Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

load_in_fp8=True quantizes a model with no FP8 repo instead of silently loading it in bf16 (#14093)

Leo Borcherding96f065bff8b4
diff --git a/tests/test_fp8_offline_quantize_gate.py b/tests/test_fp8_offline_quantize_gate.py
new file mode 100644
index 00000000000..544bbe525a9
--- /dev/null
+++ b/tests/test_fp8_offline_quantize_gate.py
@@ -0,0 +1,246 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+"""`load_in_fp8=True` on a model with no pre-quantized FP8 repo must quantize it: get_model_name
+hands back the caller's own name in that case, so the loader's `is None` check never fired and
+the model loaded in bf16 with no message. Nothing is downloaded or loaded."""
+
+from __future__ import annotations
+
+import sys
+import types
+
+import pytest
+
+loader = pytest.importorskip("unsloth.models.loader")
+loader_utils = pytest.importorskip("unsloth.models.loader_utils")
+
+
+class _Reached(Exception):
+ pass
+
+
+def _plain_checkpoint(path):
+ import json
+ (path / "config.json").write_text(
+ json.dumps({"model_type": "llama", "architectures": ["LlamaForCausalLM"]})
+ )
+ return str(path)
+
+
[email protected]("entry", ["FastLanguageModel", "FastModel"])
+def test_an_unmapped_model_reaches_the_fp8_quantizer(monkeypatch, tmp_path, entry):
+ monkeypatch.setenv("HF_HUB_OFFLINE", "1")
+ monkeypatch.setattr(loader, "get_model_name", lambda name, **_kwargs: name)
+
+ def check(*_args, **_kwargs):
+ raise _Reached()
+
+ monkeypatch.setattr(loader, "_get_fp8_mode_and_check_settings", check)
+ with pytest.raises(_Reached):
+ getattr(loader, entry).from_pretrained(
+ _plain_checkpoint(tmp_path),
+ load_in_4bit = False,
+ load_in_fp8 = True,
+ fast_inference = False,
+ )
+
+
+def _fake_vllm(monkeypatch, version):
+ module = types.ModuleType("vllm")
+ module.__version__ = version
+ monkeypatch.setitem(sys.modules, "vllm", module)
+ monkeypatch.setattr(
+ loader_utils.importlib.util,
+ "find_spec",
+ lambda name, *a, **k: object() if name == "vllm" else None,
+ )
+
+
+ "resolved, fast_inference, vllm, expected",
+ [
+ ("unsloth/x-fp8", False, None, False), # a pre-quantized repo was mapped
+ ("my-org/x", False, "0.12.0", True), # vLLM installed but not the one loading
+ ("my-org/x", True, "0.12.0", False), # vLLM quantizes on the fly
+ ("my-org/x", True, "0.11.0", True), # too old to quantize on the fly
+ ],
+)
+def test_who_quantizes(monkeypatch, resolved, fast_inference, vllm, expected):
+ if vllm is not None:
+ _fake_vllm(monkeypatch, vllm)
+ assert (
+ loader_utils._fp8_needs_offline_quantize(resolved, "my-org/x", fast_inference) is expected
+ )
+
+
[email protected]("entry", ["FastLanguageModel", "FastModel"])
+def test_an_already_quantized_checkpoint_skips_the_fp8_gate(monkeypatch, tmp_path, entry):
+ # Base loaded such a checkpoint as it is; the H100 gate must not newly refuse it.
+ import json
+
+ (tmp_path / "config.json").write_text(
+ json.dumps(
+ {
+ "model_type": "llama",
+ "architectures": ["LlamaForCausalLM"],
+ "quantization_config": {"quant_method": "fp8", "weight_block_size": [128, 128]},
+ }
+ )
+ )
+ monkeypatch.setenv("HF_HUB_OFFLINE", "1")
+ monkeypatch.setattr(loader, "get_model_name", lambda name, **_kwargs: name)
+
+ def check(*_args, **_kwargs):
+ raise _Reached()
+
+ monkeypatch.setattr(loader, "_get_fp8_mode_and_check_settings", check)
+ monkeypatch.setattr(loader, "_offline_quantize_to_fp8", check)
+ try:
+ getattr(loader, entry).from_pretrained(
+ str(tmp_path), load_in_4bit = False, load_in_fp8 = True, fast_inference = False
+ )
+ except _Reached:
+ pytest.fail("an already quantized checkpoint was sent to the fp8 quantizer")
+ except Exception:
+ pass # No weights here: the load fails later, past the gate.
+
+
[email protected]("entry", ["FastLanguageModel", "FastModel"])
+def test_an_adapter_skips_the_fp8_gate(monkeypatch, tmp_path, entry):
+ # Its base is resolved later; converting the adapter dir itself fails on its missing config.json.
+ import json
+
+ (tmp_path / "adapter_config.json").write_text(
+ json.dumps({"peft_type": "LORA", "base_model_name_or_path": "my-org/Base", "r": 8})
+ )
+ monkeypatch.setenv("HF_HUB_OFFLINE", "1")
+ monkeypatch.setattr(loader, "get_model_name", lambda name, **_kwargs: name)
+
+ def check(*_args, **_kwargs):
+ raise _Reached()
+
+ monkeypatch.setattr(loader, "_get_fp8_mode_and_check_settings", check)
+ monkeypatch.setattr(loader, "_offline_quantize_to_fp8", check)
+ try:
+ getattr(loader, entry).from_pretrained(
+ str(tmp_path), load_in_4bit = False, load_in_fp8 = True, fast_inference = False
+ )
+ except _Reached:
+ pytest.fail("an adapter was sent to the fp8 quantizer")
+ except Exception:
+ pass
+
+
+def test_quantization_config_on_text_config_counts(tmp_path):
+ import json
+ (tmp_path / "config.json").write_text(
+ json.dumps(
+ {
+ "model_type": "llava",
+ "text_config": {
+ "model_type": "llama",
+ "quantization_config": {"quant_method": "fp8", "weight_block_size": [128, 128]},
+ },
+ "vision_config": {"model_type": "clip_vision_model"},
+ }
+ )
+ )
+ assert loader_utils._fp8_load_as_is(str(tmp_path)) is True
+
+
+def test_fp8_artifacts_of_same_named_repos_do_not_collide(monkeypatch, tmp_path):
+ import transformers
+
+ monkeypatch.setattr(
+ transformers.AutoConfig,
+ "from_pretrained",
+ classmethod(lambda cls, *a, **k: types.SimpleNamespace(architectures = ["LlamaForCausalLM"])),
+ )
+ monkeypatch.setattr(loader_utils.tempfile, "gettempdir", lambda: str(tmp_path))
+ # Pretend both artifacts exist so nothing is converted.
+ monkeypatch.setattr(loader_utils.os.path, "isdir", lambda p: str(p).startswith(str(tmp_path)))
+ a = loader_utils._offline_quantize_to_fp8("org-a/model", "row")
+ b = loader_utils._offline_quantize_to_fp8("org-b/model", "row")
+ assert a != b
+
+
+def test_an_adapter_with_a_config_json_counts_as_an_adapter(tmp_path):
+ import json
+
+ (tmp_path / "config.json").write_text(
+ json.dumps({"model_type": "llama", "architectures": ["LlamaForCausalLM"]})
+ )
+ (tmp_path / "adapter_config.json").write_text(
+ json.dumps({"peft_type": "LORA", "base_model_name_or_path": "my-org/Base", "r": 8})
+ )
+ assert loader_utils._fp8_load_as_is(str(tmp_path)) is True
+
+
[email protected]("entry", ["FastLanguageModel", "FastModel"])
+def test_an_exact_model_name_still_reaches_the_fp8_quantizer(monkeypatch, tmp_path, entry):
+ monkeypatch.setenv("HF_HUB_OFFLINE", "1")
+
+ def check(*_args, **_kwargs):
+ raise _Reached()
+
+ monkeypatch.setattr(loader, "_get_fp8_mode_and_check_settings", check)
+ with pytest.raises(_Reached):
+ getattr(loader, entry).from_pretrained(
+ _plain_checkpoint(tmp_path),
+ load_in_4bit = False,
+ load_in_fp8 = True,
+ fast_inference = False,
+ use_exact_model_name = True,
+ )
+
+
+def test_float8_tensor_linear_backward_matches_dense():
+ import torch
+
+ torch.manual_seed(0)
+ X = torch.randn(3, 5, 8, dtype = torch.float64, requires_grad = True)
+ W = torch.randn(4, 8, dtype = torch.float64)
+ b = torch.randn(4, dtype = torch.float64, requires_grad = True)
+ dY = torch.randn(3, 5, 4, dtype = torch.float64)
+ loader_utils._Float8TensorLinear.apply(X, W, b).backward(dY)
+ got = (X.grad.clone(), b.grad.clone())
+ X.grad = b.grad = None
+ torch.nn.functional.linear(X, W, b).backward(dY)
+ assert torch.allclose(got[0], X.grad) and torch.allclose(got[1], b.grad)
+
+
[email protected]("entry", ["FastLanguageModel", "FastModel"])
+def test_an_unreadable_config_skips_the_fp8_gate(monkeypatch, tmp_path, entry):
+ # A Mistral-format checkpoint has no config.json; the loader's own fallback builds the view later.
+ (tmp_path / "params.json").write_text("{}")
+ monkeypatch.setenv("HF_HUB_OFFLINE", "1")
+ monkeypatch.setattr(loader, "get_model_name", lambda name, **_kwargs: name)
+
+ def check(*_args, **_kwargs):
+ raise _Reached()
+
+ monkeypatch.setattr(loader, "_get_fp8_mode_and_check_settings", check)
+ monkeypatch.setattr(loader, "_offline_quantize_to_fp8", check)
+ try:
+ getattr(loader, entry).from_pretrained(
+ str(tmp_path), load_in_4bit = False, load_in_fp8 = True, fast_inference = False
+ )
+ except _Reached:
+ pytest.fail("an unreadable config was sent to the fp8 quantizer")
+ except Exception:
+ pass
+
+
+def test_a_trainable_fp8_weight_keeps_the_plain_linear(monkeypatch):
+ import torch
+
+ def boom(*_args, **_kwargs):
+ raise _Reached()
+
+ monkeypatch.setattr(loader_utils._Float8TensorLinear, "apply", boom)
+ layer = torch.nn.Linear(4, 3)
+ X = torch.randn(2, 4, requires_grad = True)
+ loader_utils._float8_tensor_linear_forward(layer, X).sum().backward()
+ assert layer.weight.grad is not None
+ layer.weight.requires_grad_(False)
+ with pytest.raises(_Reached):
+ loader_utils._float8_tensor_linear_forward(layer, X)
diff --git a/unsloth/models/loader.py b/unsloth/models/loader.py
index 5be5fe403a8..399dac45a33 100644
--- a/unsloth/models/loader.py
+++ b/unsloth/models/loader.py
@@ -49,6 +49,8 @@ from .loader_utils import (
OFFLOAD_EMBEDDING_AUTO,
_exclude_rope_inv_freq_from_ddp,
_get_fp8_mode_and_check_settings,
+ _fp8_load_as_is,
+ _fp8_needs_offline_quantize,
_offline_quantize_to_fp8,
_tag_model_with_fp8_torchao_config,
get_model_name,
@@ -1307,29 +1309,40 @@ class FastLanguageModel(FastLlamaModel):
cache_dir = kwargs.get("cache_dir", None),
revision = revision,
)
- if new_model_name is None and load_in_fp8 != False:
- fp8_mode = _get_fp8_mode_and_check_settings(
- load_in_fp8,
- fast_inference,
- full_finetuning,
- load_in_4bit,
- load_in_8bit,
- load_in_16bit,
- )
- # Still the caller's repo here, so their ref is the one to quantize from.
- model_name = _offline_quantize_to_fp8(
- model_name,
- fp8_mode,
- text_only = text_only,
- revision = revision,
- token = token,
- )
- else:
- assert new_model_name is not None
- model_name = new_model_name
- # If the mapper resolved to a pre-quantized FP8 model, disable on-the-fly quantization to avoid quantizing twice.
- if load_in_fp8 != False and new_model_name != old_model_name:
- load_in_fp8 = False
+ else:
+ new_model_name = model_name
+ # Unmapped names come back unchanged, not None; conflicting flags are left to the precision check below.
+ if (
+ load_in_fp8 != False
+ and not _precision_flags_conflict(
+ load_in_4bit, load_in_8bit, load_in_16bit, load_in_fp8
+ )
+ and _fp8_needs_offline_quantize(new_model_name, old_model_name, fast_inference)
+ and not _fp8_load_as_is(model_name, revision, token, trust_remote_code)
+ ):
+ fp8_mode = _get_fp8_mode_and_check_settings(
+ load_in_fp8,
+ fast_inference,
+ full_finetuning,
+ load_in_4bit,
+ load_in_8bit,
+ load_in_16bit,
+ )
+ # Still the caller's repo here, so their ref is the one to quantize from.
+ model_name = _offline_quantize_to_fp8(
+ model_name,
+ fp8_mode,
+ text_only = text_only,
+ revision = revision,
+ token = token,
+ trust_remote_code = trust_remote_code,
+ )
+ elif not use_exact_model_name:
+ assert new_model_name is not None
+ model_name = new_model_name
+ # If the mapper resolved to a pre-quantized FP8 model, disable on-the-fly quantization to avoid quantizing twice.
+ if load_in_fp8 != False and new_model_name != old_model_name:
+ load_in_fp8 = False
# Only this block honours use_exact_model_name; the transforms below do not.
mapper_moved_name = model_name != old_model_name
@@ -2220,29 +2233,40 @@ class FastModel(FastBaseModel):
cache_dir = kwargs.get("cache_dir", None),
revision = revision,
)
- if new_model_name is None and load_in_fp8 != False:
- fp8_mode = _get_fp8_mode_and_check_settings(
- load_in_fp8,
- fast_inference,
- full_finetuning,
- load_in_4bit,
- load_in_8bit,
- load_in_16bit,
- )
- # Still the caller's repo here, so their ref is the one to quantize from.
- model_name = _offline_quantize_to_fp8(
- model_name,
- fp8_mode,
- text_only = text_only,
- revision = revision,
- token = token,
- )
- else:
- assert new_model_name is not None
- model_name = new_model_name
- # If the mapper resolved to a pre-quantized FP8 model, disable on-the-fly quantization to avoid quantizing twice.
- if load_in_fp8 != False and new_model_name != old_model_name:
- load_in_fp8 = False
+ else:
+ new_model_name = model_name
+ # Unmapped names come back unchanged, not None; conflicting flags are left to the precision check below.
+ if (
+ load_in_fp8 != False
+ and not _precision_flags_conflict(
+ load_in_4bit, load_in_8bit, load_in_16bit, load_in_fp8
+ )
+ and _fp8_needs_offline_quantize(new_model_name, old_model_name, fast_inference)
+ and not _fp8_load_as_is(model_name, revision, token, trust_remote_code)
+ ):
+ fp8_mode = _get_fp8_mode_and_check_settings(
+ load_in_fp8,
+ fast_inference,
+ full_finetuning,
+ load_in_4bit,
+ load_in_8bit,
+ load_in_16bit,
+ )
+ # Still the caller's repo here, so their ref is the one to quantize from.
+ model_name = _offline_quantize_to_fp8(
+ model_name,
+ fp8_mode,
+ text_only = text_only,
+ revision = revision,
+ token = token,
+ trust_remote_code = trust_remote_code,
+ )
+ elif not use_exact_model_name:
+ assert new_model_name is not None
+ model_name = new_model_name
+ # If the mapper resolved to a pre-quantized FP8 model, disable on-the-fly quantization to avoid quantizing twice.
+ if load_in_fp8 != False and new_model_name != old_model_name:
+ load_in_fp8 = False
# Only this block honours use_exact_model_name; the transforms below do not.
mapper_moved_name = model_name != old_model_name
diff --git a/unsloth/models/loader_utils.py b/unsloth/models/loader_utils.py
index e0947574c90..97c82b85cbb 100644
--- a/unsloth/models/loader_utils.py
+++ b/unsloth/models/loader_utils.py
@@ -17,6 +17,8 @@ import os
import torch
import re
import tempfile
+import types
+import shutil
import contextlib
import threading as _threading
import functools
@@ -1711,6 +1713,7 @@ def _offline_quantize_to_fp8(
text_only: bool = False,
revision: str = None,
token = None,
+ trust_remote_code: bool = False,
) -> str:
"""Quantize the model to fp8 via torchao, save to a temp dir, return its path. For vllm >= 0.12.0, prefer dynamic quantization in vllm instead (via hf_overrides={"quantization_config_file": "torchao_config.json"}). The caller's revision has to reach the source loads, and the cache name has to name it too: the returned path replaces model_name, so the revision gate downstream drops the pin, and two refs of one repo would otherwise share a single artifact."""
from transformers import (
@@ -1722,7 +1725,9 @@ def _offline_quantize_to_fp8(
AutoConfig,
)
- config = AutoConfig.from_pretrained(model_name, revision = revision, token = token)
+ config = AutoConfig.from_pretrained(
+ model_name, revision = revision, token = token, trust_remote_code = trust_remote_code
+ )
is_vlm = any(
x.endswith(("ForConditionalGeneration", "ForVisionText2Text"))
for x in (getattr(config, "architectures", None) or [])
@@ -1749,6 +1754,9 @@ def _offline_quantize_to_fp8(
temp_dir = tempfile.gettempdir()
# Cache text-only and full-VLM artifacts separately so neither reuses the other (#5816).
cache_name = model_name.split("/")[-1] + "-fp8-" + fp8_mode
+ # The basename alone would let org-a/model and org-b/model share one artifact.
+ source = os.path.abspath(model_name) if os.path.isdir(model_name) else model_name
+ cache_name += "-" + hashlib.sha256(source.encode("utf-8")).hexdigest()[:12]
if revision is not None:
# Sanitizing is lossy (release/v1 and release.v1 collapse), so a digest of the raw ref rides along and two refs never share an artifact.
digest = hashlib.sha256(revision.encode("utf-8")).hexdigest()[:12]
@@ -1764,7 +1772,19 @@ def _offline_quantize_to_fp8(
qconfig = _get_torchao_fp8_config(fp8_mode)
qconfig = TorchAoConfig(qconfig)
- load_kwargs = dict(torch_dtype = "auto", device_map = "auto", quantization_config = qconfig)
+ # Under torchrun every rank converts: keep each on its own GPU, not all of them on "auto".
+ local_rank = os.environ.get("LOCAL_RANK")
+ device_map = (
+ {"": int(local_rank)}
+ if local_rank is not None and int(os.environ.get("WORLD_SIZE", "1")) > 1
+ else "auto"
+ )
+ load_kwargs = dict(
+ torch_dtype = "auto",
+ device_map = device_map,
+ quantization_config = qconfig,
+ trust_remote_code = trust_remote_code,
+ )
if text_config is not None:
_apply_text_only_key_mapping(load_kwargs, config, text_config)
config = text_config
@@ -1782,18 +1802,108 @@ def _offline_quantize_to_fp8(
model_name,
revision = revision,
token = token,
+ trust_remote_code = trust_remote_code,
)
- model.save_pretrained(new_model_name, safe_serialization = False)
- del model
- for _ in range(2):
- clean_gpu_cache()
- gc.collect()
- tokenizer.save_pretrained(new_model_name)
+ # Atomic publish: the isdir check above must never see another rank's half-written artifact.
+ partial = tempfile.mkdtemp(prefix = cache_name + ".partial-", dir = temp_dir)
+ try:
+ model.save_pretrained(partial, safe_serialization = False)
+ del model
+ for _ in range(2):
+ clean_gpu_cache()
+ gc.collect()
+ tokenizer.save_pretrained(partial)
+ try:
+ os.rename(partial, new_model_name)
+ except OSError:
+ if not os.path.isdir(new_model_name):
+ raise
+ finally:
+ shutil.rmtree(partial, ignore_errors = True)
return new_model_name
+def _fp8_needs_offline_quantize(resolved_name, requested_name, fast_inference):
+ """True when no pre-quantized FP8 repo was mapped and vLLM won't quantize it on the fly."""
+ if resolved_name is not None and str(resolved_name).lower() != str(requested_name).lower():
+ return False
+ if not fast_inference or importlib.util.find_spec("vllm") is None:
+ return True
+ import vllm
+
+ return Version(vllm.__version__) < Version("0.12.0")
+
+
+def _fp8_load_as_is(
+ model_name,
+ revision = None,
+ token = None,
+ trust_remote_code = False,
+):
+ """Only a readable, unquantized model config takes the offline fp8 path and its H100 gate: an already quantized checkpoint, an adapter (resolved to its base later) or an unreadable config loads as before."""
+ from transformers import AutoConfig
+
+ try:
+ from peft import PeftConfig
+
+ # The loader treats a repo holding both config.json and adapter_config.json as an adapter too.
+ PeftConfig.from_pretrained(model_name, revision = revision, token = token)
+ return True
+ except Exception:
+ pass
+ try:
+ config = AutoConfig.from_pretrained(
+ model_name, revision = revision, token = token, trust_remote_code = trust_remote_code
+ )
+ except Exception:
+ # Unreadable here (a Mistral-format checkpoint, say): leave it to the loader's own fallbacks.
+ return True
+ text_config = getattr(config, "text_config", None)
+ return bool(
+ getattr(config, "quantization_config", None)
+ or getattr(text_config, "quantization_config", None)
+ )
+
+
+class _Float8TensorLinear(torch.autograd.Function):
+ """torchao's Float8Tensor linear has no backward, so a LoRA layer Unsloth's kernels do not take (a biased Qwen2 q_proj, FastModel's PEFT path) failed in backward."""
+
+ @staticmethod
+ def forward(ctx, X, weight, bias):
+ ctx.weight = weight
+ ctx.bias_needs_grad = bias is not None and bias.requires_grad
+ return torch.nn.functional.linear(X, weight, bias)
+
+ @staticmethod
+ def backward(ctx, dY):
+ dX = dY @ ctx.weight.dequantize().to(dY.dtype)
+ dB = dY.reshape(-1, dY.shape[-1]).sum(0) if ctx.bias_needs_grad else None
+ return dX, None, dB
+
+
+def _float8_tensor_linear_forward(self, input):
+ # A trainable fp8 weight keeps torchao's own (backward-less) path, so it errors instead of silently getting no gradient.
+ if torch.is_grad_enabled() and input.requires_grad and not self.weight.requires_grad:
+ return _Float8TensorLinear.apply(input, self.weight, self.bias)
+ return torch.nn.functional.linear(input, self.weight, self.bias)
+
+
+def _enable_float8_tensor_backward(model: torch.nn.Module):
+ from ..kernels.utils import Float8Tensor
+ for module in model.modules():
+ weight = getattr(module, "weight", None)
+ if type(module) is torch.nn.Linear and isinstance(
+ getattr(weight, "data", weight), Float8Tensor
+ ):
+ module.forward = types.MethodType(_float8_tensor_linear_forward, module)
+
+
def _tag_model_with_fp8_torchao_config(model: torch.nn.Module, fp8_mode: str):
"""Tag a model with a `TorchAOConfig` so downstream callers know how to handle it."""
+ try:
+ _enable_float8_tensor_backward(model)
+ except Exception:
+ pass
try:
base_config = _get_torchao_fp8_config(fp8_mode)
model.torchao_config = TorchAOConfig(