Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
load_in_fp8=True quantizes a model with no FP8 repo instead of silently loading it in bf16 (#14093)
Leo Borcherding96f065bff8b4
diff --git a/tests/test_fp8_offline_quantize_gate.py b/tests/test_fp8_offline_quantize_gate.pynew file mode 100644index 00000000000..544bbe525a9--- /dev/null+++ b/tests/test_fp8_offline_quantize_gate.py@@ -0,0 +1,246 @@+# SPDX-License-Identifier: AGPL-3.0-only+"""`load_in_fp8=True` on a model with no pre-quantized FP8 repo must quantize it: get_model_name+hands back the caller's own name in that case, so the loader's `is None` check never fired and+the model loaded in bf16 with no message. Nothing is downloaded or loaded."""++from __future__ import annotations++import sys+import types++import pytest++loader = pytest.importorskip("unsloth.models.loader")+loader_utils = pytest.importorskip("unsloth.models.loader_utils")+++class _Reached(Exception):+ pass+++def _plain_checkpoint(path):+ import json+ (path / "config.json").write_text(+ json.dumps({"model_type": "llama", "architectures": ["LlamaForCausalLM"]})+ )+ return str(path)++[email protected]("entry", ["FastLanguageModel", "FastModel"])+def test_an_unmapped_model_reaches_the_fp8_quantizer(monkeypatch, tmp_path, entry):+ monkeypatch.setenv("HF_HUB_OFFLINE", "1")+ monkeypatch.setattr(loader, "get_model_name", lambda name, **_kwargs: name)++ def check(*_args, **_kwargs):+ raise _Reached()++ monkeypatch.setattr(loader, "_get_fp8_mode_and_check_settings", check)+ with pytest.raises(_Reached):+ getattr(loader, entry).from_pretrained(+ _plain_checkpoint(tmp_path),+ load_in_4bit = False,+ load_in_fp8 = True,+ fast_inference = False,+ )+++def _fake_vllm(monkeypatch, version):+ module = types.ModuleType("vllm")+ module.__version__ = version+ monkeypatch.setitem(sys.modules, "vllm", module)+ monkeypatch.setattr(+ loader_utils.importlib.util,+ "find_spec",+ lambda name, *a, **k: object() if name == "vllm" else None,+ )+++ "resolved, fast_inference, vllm, expected",+ [+ ("unsloth/x-fp8", False, None, False), # a pre-quantized repo was mapped+ ("my-org/x", False, "0.12.0", True), # vLLM installed but not the one loading+ ("my-org/x", True, "0.12.0", False), # vLLM quantizes on the fly+ ("my-org/x", True, "0.11.0", True), # too old to quantize on the fly+ ],+)+def test_who_quantizes(monkeypatch, resolved, fast_inference, vllm, expected):+ if vllm is not None:+ _fake_vllm(monkeypatch, vllm)+ assert (+ loader_utils._fp8_needs_offline_quantize(resolved, "my-org/x", fast_inference) is expected+ )++[email protected]("entry", ["FastLanguageModel", "FastModel"])+def test_an_already_quantized_checkpoint_skips_the_fp8_gate(monkeypatch, tmp_path, entry):+ # Base loaded such a checkpoint as it is; the H100 gate must not newly refuse it.+ import json++ (tmp_path / "config.json").write_text(+ json.dumps(+ {+ "model_type": "llama",+ "architectures": ["LlamaForCausalLM"],+ "quantization_config": {"quant_method": "fp8", "weight_block_size": [128, 128]},+ }+ )+ )+ monkeypatch.setenv("HF_HUB_OFFLINE", "1")+ monkeypatch.setattr(loader, "get_model_name", lambda name, **_kwargs: name)++ def check(*_args, **_kwargs):+ raise _Reached()++ monkeypatch.setattr(loader, "_get_fp8_mode_and_check_settings", check)+ monkeypatch.setattr(loader, "_offline_quantize_to_fp8", check)+ try:+ getattr(loader, entry).from_pretrained(+ str(tmp_path), load_in_4bit = False, load_in_fp8 = True, fast_inference = False+ )+ except _Reached:+ pytest.fail("an already quantized checkpoint was sent to the fp8 quantizer")+ except Exception:+ pass # No weights here: the load fails later, past the gate.++[email protected]("entry", ["FastLanguageModel", "FastModel"])+def test_an_adapter_skips_the_fp8_gate(monkeypatch, tmp_path, entry):+ # Its base is resolved later; converting the adapter dir itself fails on its missing config.json.+ import json++ (tmp_path / "adapter_config.json").write_text(+ json.dumps({"peft_type": "LORA", "base_model_name_or_path": "my-org/Base", "r": 8})+ )+ monkeypatch.setenv("HF_HUB_OFFLINE", "1")+ monkeypatch.setattr(loader, "get_model_name", lambda name, **_kwargs: name)++ def check(*_args, **_kwargs):+ raise _Reached()++ monkeypatch.setattr(loader, "_get_fp8_mode_and_check_settings", check)+ monkeypatch.setattr(loader, "_offline_quantize_to_fp8", check)+ try:+ getattr(loader, entry).from_pretrained(+ str(tmp_path), load_in_4bit = False, load_in_fp8 = True, fast_inference = False+ )+ except _Reached:+ pytest.fail("an adapter was sent to the fp8 quantizer")+ except Exception:+ pass+++def test_quantization_config_on_text_config_counts(tmp_path):+ import json+ (tmp_path / "config.json").write_text(+ json.dumps(+ {+ "model_type": "llava",+ "text_config": {+ "model_type": "llama",+ "quantization_config": {"quant_method": "fp8", "weight_block_size": [128, 128]},+ },+ "vision_config": {"model_type": "clip_vision_model"},+ }+ )+ )+ assert loader_utils._fp8_load_as_is(str(tmp_path)) is True+++def test_fp8_artifacts_of_same_named_repos_do_not_collide(monkeypatch, tmp_path):+ import transformers++ monkeypatch.setattr(+ transformers.AutoConfig,+ "from_pretrained",+ classmethod(lambda cls, *a, **k: types.SimpleNamespace(architectures = ["LlamaForCausalLM"])),+ )+ monkeypatch.setattr(loader_utils.tempfile, "gettempdir", lambda: str(tmp_path))+ # Pretend both artifacts exist so nothing is converted.+ monkeypatch.setattr(loader_utils.os.path, "isdir", lambda p: str(p).startswith(str(tmp_path)))+ a = loader_utils._offline_quantize_to_fp8("org-a/model", "row")+ b = loader_utils._offline_quantize_to_fp8("org-b/model", "row")+ assert a != b+++def test_an_adapter_with_a_config_json_counts_as_an_adapter(tmp_path):+ import json++ (tmp_path / "config.json").write_text(+ json.dumps({"model_type": "llama", "architectures": ["LlamaForCausalLM"]})+ )+ (tmp_path / "adapter_config.json").write_text(+ json.dumps({"peft_type": "LORA", "base_model_name_or_path": "my-org/Base", "r": 8})+ )+ assert loader_utils._fp8_load_as_is(str(tmp_path)) is True++[email protected]("entry", ["FastLanguageModel", "FastModel"])+def test_an_exact_model_name_still_reaches_the_fp8_quantizer(monkeypatch, tmp_path, entry):+ monkeypatch.setenv("HF_HUB_OFFLINE", "1")++ def check(*_args, **_kwargs):+ raise _Reached()++ monkeypatch.setattr(loader, "_get_fp8_mode_and_check_settings", check)+ with pytest.raises(_Reached):+ getattr(loader, entry).from_pretrained(+ _plain_checkpoint(tmp_path),+ load_in_4bit = False,+ load_in_fp8 = True,+ fast_inference = False,+ use_exact_model_name = True,+ )+++def test_float8_tensor_linear_backward_matches_dense():+ import torch++ torch.manual_seed(0)+ X = torch.randn(3, 5, 8, dtype = torch.float64, requires_grad = True)+ W = torch.randn(4, 8, dtype = torch.float64)+ b = torch.randn(4, dtype = torch.float64, requires_grad = True)+ dY = torch.randn(3, 5, 4, dtype = torch.float64)+ loader_utils._Float8TensorLinear.apply(X, W, b).backward(dY)+ got = (X.grad.clone(), b.grad.clone())+ X.grad = b.grad = None+ torch.nn.functional.linear(X, W, b).backward(dY)+ assert torch.allclose(got[0], X.grad) and torch.allclose(got[1], b.grad)++[email protected]("entry", ["FastLanguageModel", "FastModel"])+def test_an_unreadable_config_skips_the_fp8_gate(monkeypatch, tmp_path, entry):+ # A Mistral-format checkpoint has no config.json; the loader's own fallback builds the view later.+ (tmp_path / "params.json").write_text("{}")+ monkeypatch.setenv("HF_HUB_OFFLINE", "1")+ monkeypatch.setattr(loader, "get_model_name", lambda name, **_kwargs: name)++ def check(*_args, **_kwargs):+ raise _Reached()++ monkeypatch.setattr(loader, "_get_fp8_mode_and_check_settings", check)+ monkeypatch.setattr(loader, "_offline_quantize_to_fp8", check)+ try:+ getattr(loader, entry).from_pretrained(+ str(tmp_path), load_in_4bit = False, load_in_fp8 = True, fast_inference = False+ )+ except _Reached:+ pytest.fail("an unreadable config was sent to the fp8 quantizer")+ except Exception:+ pass+++def test_a_trainable_fp8_weight_keeps_the_plain_linear(monkeypatch):+ import torch++ def boom(*_args, **_kwargs):+ raise _Reached()++ monkeypatch.setattr(loader_utils._Float8TensorLinear, "apply", boom)+ layer = torch.nn.Linear(4, 3)+ X = torch.randn(2, 4, requires_grad = True)+ loader_utils._float8_tensor_linear_forward(layer, X).sum().backward()+ assert layer.weight.grad is not None+ layer.weight.requires_grad_(False)+ with pytest.raises(_Reached):+ loader_utils._float8_tensor_linear_forward(layer, X)diff --git a/unsloth/models/loader.py b/unsloth/models/loader.pyindex 5be5fe403a8..399dac45a33 100644--- a/unsloth/models/loader.py+++ b/unsloth/models/loader.py@@ -49,6 +49,8 @@ from .loader_utils import (OFFLOAD_EMBEDDING_AUTO,_exclude_rope_inv_freq_from_ddp,_get_fp8_mode_and_check_settings,+ _fp8_load_as_is,+ _fp8_needs_offline_quantize,_offline_quantize_to_fp8,_tag_model_with_fp8_torchao_config,get_model_name,@@ -1307,29 +1309,40 @@ class FastLanguageModel(FastLlamaModel):cache_dir = kwargs.get("cache_dir", None),revision = revision,)- if new_model_name is None and load_in_fp8 != False:- fp8_mode = _get_fp8_mode_and_check_settings(- load_in_fp8,- fast_inference,- full_finetuning,- load_in_4bit,- load_in_8bit,- load_in_16bit,- )- # Still the caller's repo here, so their ref is the one to quantize from.- model_name = _offline_quantize_to_fp8(- model_name,- fp8_mode,- text_only = text_only,- revision = revision,- token = token,- )- else:- assert new_model_name is not None- model_name = new_model_name- # If the mapper resolved to a pre-quantized FP8 model, disable on-the-fly quantization to avoid quantizing twice.- if load_in_fp8 != False and new_model_name != old_model_name:- load_in_fp8 = False+ else:+ new_model_name = model_name+ # Unmapped names come back unchanged, not None; conflicting flags are left to the precision check below.+ if (+ load_in_fp8 != False+ and not _precision_flags_conflict(+ load_in_4bit, load_in_8bit, load_in_16bit, load_in_fp8+ )+ and _fp8_needs_offline_quantize(new_model_name, old_model_name, fast_inference)+ and not _fp8_load_as_is(model_name, revision, token, trust_remote_code)+ ):+ fp8_mode = _get_fp8_mode_and_check_settings(+ load_in_fp8,+ fast_inference,+ full_finetuning,+ load_in_4bit,+ load_in_8bit,+ load_in_16bit,+ )+ # Still the caller's repo here, so their ref is the one to quantize from.+ model_name = _offline_quantize_to_fp8(+ model_name,+ fp8_mode,+ text_only = text_only,+ revision = revision,+ token = token,+ trust_remote_code = trust_remote_code,+ )+ elif not use_exact_model_name:+ assert new_model_name is not None+ model_name = new_model_name+ # If the mapper resolved to a pre-quantized FP8 model, disable on-the-fly quantization to avoid quantizing twice.+ if load_in_fp8 != False and new_model_name != old_model_name:+ load_in_fp8 = False# Only this block honours use_exact_model_name; the transforms below do not.mapper_moved_name = model_name != old_model_name@@ -2220,29 +2233,40 @@ class FastModel(FastBaseModel):cache_dir = kwargs.get("cache_dir", None),revision = revision,)- if new_model_name is None and load_in_fp8 != False:- fp8_mode = _get_fp8_mode_and_check_settings(- load_in_fp8,- fast_inference,- full_finetuning,- load_in_4bit,- load_in_8bit,- load_in_16bit,- )- # Still the caller's repo here, so their ref is the one to quantize from.- model_name = _offline_quantize_to_fp8(- model_name,- fp8_mode,- text_only = text_only,- revision = revision,- token = token,- )- else:- assert new_model_name is not None- model_name = new_model_name- # If the mapper resolved to a pre-quantized FP8 model, disable on-the-fly quantization to avoid quantizing twice.- if load_in_fp8 != False and new_model_name != old_model_name:- load_in_fp8 = False+ else:+ new_model_name = model_name+ # Unmapped names come back unchanged, not None; conflicting flags are left to the precision check below.+ if (+ load_in_fp8 != False+ and not _precision_flags_conflict(+ load_in_4bit, load_in_8bit, load_in_16bit, load_in_fp8+ )+ and _fp8_needs_offline_quantize(new_model_name, old_model_name, fast_inference)+ and not _fp8_load_as_is(model_name, revision, token, trust_remote_code)+ ):+ fp8_mode = _get_fp8_mode_and_check_settings(+ load_in_fp8,+ fast_inference,+ full_finetuning,+ load_in_4bit,+ load_in_8bit,+ load_in_16bit,+ )+ # Still the caller's repo here, so their ref is the one to quantize from.+ model_name = _offline_quantize_to_fp8(+ model_name,+ fp8_mode,+ text_only = text_only,+ revision = revision,+ token = token,+ trust_remote_code = trust_remote_code,+ )+ elif not use_exact_model_name:+ assert new_model_name is not None+ model_name = new_model_name+ # If the mapper resolved to a pre-quantized FP8 model, disable on-the-fly quantization to avoid quantizing twice.+ if load_in_fp8 != False and new_model_name != old_model_name:+ load_in_fp8 = False# Only this block honours use_exact_model_name; the transforms below do not.mapper_moved_name = model_name != old_model_namediff --git a/unsloth/models/loader_utils.py b/unsloth/models/loader_utils.pyindex e0947574c90..97c82b85cbb 100644--- a/unsloth/models/loader_utils.py+++ b/unsloth/models/loader_utils.py@@ -17,6 +17,8 @@ import osimport torchimport reimport tempfile+import types+import shutilimport contextlibimport threading as _threadingimport functools@@ -1711,6 +1713,7 @@ def _offline_quantize_to_fp8(text_only: bool = False,revision: str = None,token = None,+ trust_remote_code: bool = False,) -> str:"""Quantize the model to fp8 via torchao, save to a temp dir, return its path. For vllm >= 0.12.0, prefer dynamic quantization in vllm instead (via hf_overrides={"quantization_config_file": "torchao_config.json"}). The caller's revision has to reach the source loads, and the cache name has to name it too: the returned path replaces model_name, so the revision gate downstream drops the pin, and two refs of one repo would otherwise share a single artifact."""from transformers import (@@ -1722,7 +1725,9 @@ def _offline_quantize_to_fp8(AutoConfig,)- config = AutoConfig.from_pretrained(model_name, revision = revision, token = token)+ config = AutoConfig.from_pretrained(+ model_name, revision = revision, token = token, trust_remote_code = trust_remote_code+ )is_vlm = any(x.endswith(("ForConditionalGeneration", "ForVisionText2Text"))for x in (getattr(config, "architectures", None) or [])@@ -1749,6 +1754,9 @@ def _offline_quantize_to_fp8(temp_dir = tempfile.gettempdir()# Cache text-only and full-VLM artifacts separately so neither reuses the other (#5816).cache_name = model_name.split("/")[-1] + "-fp8-" + fp8_mode+ # The basename alone would let org-a/model and org-b/model share one artifact.+ source = os.path.abspath(model_name) if os.path.isdir(model_name) else model_name+ cache_name += "-" + hashlib.sha256(source.encode("utf-8")).hexdigest()[:12]if revision is not None:# Sanitizing is lossy (release/v1 and release.v1 collapse), so a digest of the raw ref rides along and two refs never share an artifact.digest = hashlib.sha256(revision.encode("utf-8")).hexdigest()[:12]@@ -1764,7 +1772,19 @@ def _offline_quantize_to_fp8(qconfig = _get_torchao_fp8_config(fp8_mode)qconfig = TorchAoConfig(qconfig)- load_kwargs = dict(torch_dtype = "auto", device_map = "auto", quantization_config = qconfig)+ # Under torchrun every rank converts: keep each on its own GPU, not all of them on "auto".+ local_rank = os.environ.get("LOCAL_RANK")+ device_map = (+ {"": int(local_rank)}+ if local_rank is not None and int(os.environ.get("WORLD_SIZE", "1")) > 1+ else "auto"+ )+ load_kwargs = dict(+ torch_dtype = "auto",+ device_map = device_map,+ quantization_config = qconfig,+ trust_remote_code = trust_remote_code,+ )if text_config is not None:_apply_text_only_key_mapping(load_kwargs, config, text_config)config = text_config@@ -1782,18 +1802,108 @@ def _offline_quantize_to_fp8(model_name,revision = revision,token = token,+ trust_remote_code = trust_remote_code,)- model.save_pretrained(new_model_name, safe_serialization = False)- del model- for _ in range(2):- clean_gpu_cache()- gc.collect()- tokenizer.save_pretrained(new_model_name)+ # Atomic publish: the isdir check above must never see another rank's half-written artifact.+ partial = tempfile.mkdtemp(prefix = cache_name + ".partial-", dir = temp_dir)+ try:+ model.save_pretrained(partial, safe_serialization = False)+ del model+ for _ in range(2):+ clean_gpu_cache()+ gc.collect()+ tokenizer.save_pretrained(partial)+ try:+ os.rename(partial, new_model_name)+ except OSError:+ if not os.path.isdir(new_model_name):+ raise+ finally:+ shutil.rmtree(partial, ignore_errors = True)return new_model_name+def _fp8_needs_offline_quantize(resolved_name, requested_name, fast_inference):+ """True when no pre-quantized FP8 repo was mapped and vLLM won't quantize it on the fly."""+ if resolved_name is not None and str(resolved_name).lower() != str(requested_name).lower():+ return False+ if not fast_inference or importlib.util.find_spec("vllm") is None:+ return True+ import vllm++ return Version(vllm.__version__) < Version("0.12.0")+++def _fp8_load_as_is(+ model_name,+ revision = None,+ token = None,+ trust_remote_code = False,+):+ """Only a readable, unquantized model config takes the offline fp8 path and its H100 gate: an already quantized checkpoint, an adapter (resolved to its base later) or an unreadable config loads as before."""+ from transformers import AutoConfig++ try:+ from peft import PeftConfig++ # The loader treats a repo holding both config.json and adapter_config.json as an adapter too.+ PeftConfig.from_pretrained(model_name, revision = revision, token = token)+ return True+ except Exception:+ pass+ try:+ config = AutoConfig.from_pretrained(+ model_name, revision = revision, token = token, trust_remote_code = trust_remote_code+ )+ except Exception:+ # Unreadable here (a Mistral-format checkpoint, say): leave it to the loader's own fallbacks.+ return True+ text_config = getattr(config, "text_config", None)+ return bool(+ getattr(config, "quantization_config", None)+ or getattr(text_config, "quantization_config", None)+ )+++class _Float8TensorLinear(torch.autograd.Function):+ """torchao's Float8Tensor linear has no backward, so a LoRA layer Unsloth's kernels do not take (a biased Qwen2 q_proj, FastModel's PEFT path) failed in backward."""++ @staticmethod+ def forward(ctx, X, weight, bias):+ ctx.weight = weight+ ctx.bias_needs_grad = bias is not None and bias.requires_grad+ return torch.nn.functional.linear(X, weight, bias)++ @staticmethod+ def backward(ctx, dY):+ dX = dY @ ctx.weight.dequantize().to(dY.dtype)+ dB = dY.reshape(-1, dY.shape[-1]).sum(0) if ctx.bias_needs_grad else None+ return dX, None, dB+++def _float8_tensor_linear_forward(self, input):+ # A trainable fp8 weight keeps torchao's own (backward-less) path, so it errors instead of silently getting no gradient.+ if torch.is_grad_enabled() and input.requires_grad and not self.weight.requires_grad:+ return _Float8TensorLinear.apply(input, self.weight, self.bias)+ return torch.nn.functional.linear(input, self.weight, self.bias)+++def _enable_float8_tensor_backward(model: torch.nn.Module):+ from ..kernels.utils import Float8Tensor+ for module in model.modules():+ weight = getattr(module, "weight", None)+ if type(module) is torch.nn.Linear and isinstance(+ getattr(weight, "data", weight), Float8Tensor+ ):+ module.forward = types.MethodType(_float8_tensor_linear_forward, module)++def _tag_model_with_fp8_torchao_config(model: torch.nn.Module, fp8_mode: str):"""Tag a model with a `TorchAOConfig` so downstream callers know how to handle it."""+ try:+ _enable_float8_tensor_backward(model)+ except Exception:+ passtry:base_config = _get_torchao_fp8_config(fp8_mode)model.torchao_config = TorchAOConfig(