Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Pass gradients through compressed-tensors activation quantization so W8A8 checkpoints train with LoRA (#11585)

Daniel Han9d15a59883c8
diff --git a/tests/test_compressed_tensors_activation_ste.py b/tests/test_compressed_tensors_activation_ste.py
new file mode 100644
index 00000000000..23c6c836810
--- /dev/null
+++ b/tests/test_compressed_tensors_activation_ste.py
@@ -0,0 +1,122 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published
+# by the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+"""Each case runs in a subprocess: the patch changes compressed-tensors module state."""
+
+import os
+import subprocess
+import sys
+import textwrap
+
+import pytest
+
+pytest.importorskip("compressed_tensors")
+
+_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+
+_RUNNER = textwrap.dedent(
+ """
+ import importlib.util, json, sys
+ mode = sys.argv[1]
+ if mode != "base":
+ spec = importlib.util.spec_from_file_location("unsloth_import_fixes_under_test", sys.argv[2])
+ fixes = importlib.util.module_from_spec(spec)
+ spec.loader.exec_module(fixes)
+ if mode == "lazy":
+ assert "compressed_tensors.quantization.lifecycle.forward" not in sys.modules
+ fixes.fix_compressed_tensors_activation_quant_gradient()
+ import compressed_tensors # noqa: F401
+ if mode != "lazy":
+ fixes.fix_compressed_tensors_activation_quant_gradient()
+ fixes.fix_compressed_tensors_activation_quant_gradient() # idempotent
+ import torch
+ from compressed_tensors.quantization import (
+ QuantizationArgs, QuantizationConfig, QuantizationScheme, apply_quantization_config,
+ )
+ torch.manual_seed(0)
+ model = torch.nn.Sequential(torch.nn.Linear(64, 32, bias = False))
+ static = len(sys.argv) > 3 and sys.argv[3] == "static"
+ if static: # INT8 W8A8 with a calibrated per-tensor input scale that saturates
+ activations = QuantizationArgs(
+ num_bits = 8, type = "int", strategy = "tensor", dynamic = False, symmetric = True,
+ )
+ else:
+ activations = QuantizationArgs(
+ num_bits = 8, type = "float", strategy = "token", dynamic = True, symmetric = True,
+ )
+ scheme = QuantizationScheme(targets = ["Linear"], input_activations = activations)
+ apply_quantization_config(model, QuantizationConfig(config_groups = {"group_0": scheme}, quantization_status = "frozen"))
+ lin = model[0]
+ lin.weight.requires_grad_(False) # the frozen base layer under a LoRA adapter
+ if static:
+ model.to(torch.bfloat16)
+ lin.input_scale.data.fill_(0.01) # clips at 1.27, so most of randn * 4 saturates
+ x = (torch.randn(4, 64) * (4 if static else 1)).to(lin.weight.dtype).requires_grad_(True)
+ y = lin(x)
+ with torch.no_grad():
+ y_nograd = lin(x)
+ out = {"has_grad_fn": y.grad_fn is not None, "same_forward": bool(torch.equal(y.detach(), y_nograd))}
+ if y.grad_fn is not None:
+ (gx,) = torch.autograd.grad(y.sum(), x)
+ ref = torch.ones(4, 32, dtype = lin.weight.dtype) @ lin.weight
+ out["grad_err"] = float((gx - ref).abs().max().float())
+ import compressed_tensors.quantization.lifecycle.forward as f
+ out["patched"] = bool(getattr(f.forward_quantize, "_unsloth_activation_ste", False))
+ print(json.dumps(out))
+ """
+)
+
+
+def _run(mode, scheme = "dynamic"):
+ proc = subprocess.run(
+ [
+ sys.executable,
+ "-c",
+ _RUNNER,
+ mode,
+ os.path.join(_ROOT, "unsloth", "import_fixes.py"),
+ scheme,
+ ],
+ capture_output = True,
+ text = True,
+ timeout = 600,
+ env = dict(os.environ, CUDA_VISIBLE_DEVICES = ""),
+ )
+ assert proc.returncode == 0, proc.stdout[-2000:] + proc.stderr[-4000:]
+ import json
+
+ return json.loads(proc.stdout.strip().splitlines()[-1])
+
+
+def test_without_the_fix_the_input_gradient_is_lost():
+ out = _run("base")
+ assert not out["has_grad_fn"]
+
+
[email protected]("mode", ["eager", "lazy"])
+def test_activation_quantization_passes_the_gradient_straight_through(mode):
+ out = _run(mode)
+ assert out["patched"]
+ assert out["has_grad_fn"]
+ assert out["same_forward"] # the quantized forward is untouched
+ assert out["grad_err"] < 1e-5 # dX = dY @ W, as if the activation were not quantized
+
+
+def test_a_saturating_static_scale_keeps_the_forward_bit_identical():
+ out = _run("eager", "static")
+ assert out["patched"]
+ assert out["has_grad_fn"]
+ assert out["same_forward"]
diff --git a/tests/test_compressed_tensors_fp8_kernel_route.py b/tests/test_compressed_tensors_fp8_kernel_route.py
new file mode 100644
index 00000000000..23f9870f3fd
--- /dev/null
+++ b/tests/test_compressed_tensors_fp8_kernel_route.py
@@ -0,0 +1,338 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published
+# by the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+
+import types
+
+import pytest
+import torch
+
+pytestmark = pytest.mark.skipif(not torch.cuda.is_available(), reason = "needs CUDA")
+ct_quant = pytest.importorskip("compressed_tensors.quantization")
+
+
[email protected](autouse = True)
+def _zoo_with_float_only_cast(monkeypatch):
+ monkeypatch.setenv("UNSLOTH_COMPRESSED_TENSORS_FP8_KERNELS", "1")
+ from unsloth.models import loader_utils
+ monkeypatch.setattr(loader_utils, "_zoo_peft_forward_keeps_fp8_inputs", lambda: True)
+
+
+def _quantize(
+ W,
+ strategy,
+ block = None,
+):
+ o, i = W.shape
+ if strategy == "tensor":
+ s = W.abs().amax().view(1) / 448
+ full = s.expand(o, i)
+ elif strategy == "channel":
+ s = W.abs().amax(1, keepdim = True) / 448
+ full = s.expand(o, i)
+ else:
+ bo, bi = block
+ s = torch.empty(-(-o // bo), -(-i // bi), device = W.device)
+ for a in range(s.shape[0]):
+ for b in range(s.shape[1]):
+ s[a, b] = W[a * bo : (a + 1) * bo, b * bi : (b + 1) * bi].abs().amax() / 448
+ full = s.repeat_interleave(bo, 0)[:o].repeat_interleave(bi, 1)[:, :i]
+ Wq = (W / full).to(torch.float8_e4m3fn)
+ return Wq, s.to(torch.bfloat16), Wq.float() * full.to(torch.bfloat16).float()
+
+
+def _ct_model(
+ o,
+ i,
+ strategy,
+ bias = False,
+ block = None,
+ weight_type = "float",
+):
+ torch.manual_seed(0)
+ lin = torch.nn.Linear(i, o, bias = bias, device = "cuda", dtype = torch.bfloat16)
+ Wq, s, ref = _quantize(torch.randn(o, i, device = "cuda") * 0.02, strategy, block)
+ lin.weight = torch.nn.Parameter(Wq, requires_grad = False)
+ lin.weight_scale = torch.nn.Parameter(s, requires_grad = False)
+ kwargs = dict(num_bits = 8, type = weight_type, strategy = strategy, symmetric = True, dynamic = False)
+ if block is not None:
+ kwargs["block_structure"] = list(block)
+ lin.quantization_scheme = ct_quant.QuantizationScheme(
+ targets = ["Linear"],
+ weights = ct_quant.QuantizationArgs(**kwargs),
+ input_activations = ct_quant.QuantizationArgs(
+ num_bits = 8, type = "float", strategy = "token", dynamic = True, symmetric = True
+ ),
+ )
+
+ def ct_forward(self, x):
+ raise AssertionError("compressed-tensors forward must not run on a routed module")
+
+ lin.forward = types.MethodType(ct_forward, lin)
+ model = torch.nn.Module()
+ model.lin = lin
+ model.config = types.SimpleNamespace(quantization_config = {"quant_method": "compressed-tensors"})
+ return model, ref
+
+
[email protected]("shape", [(256, 256), (384, 256), (256, 384)])
+def test_weight_dequant_transposed_row_scale(shape):
+ from unsloth.kernels.fp8 import weight_dequant
+
+ Wq, s, ref = _quantize(torch.randn(*shape, device = "cuda") * 0.02, "channel")
+ torch.testing.assert_close(weight_dequant(Wq, s, torch.float32), ref)
+ torch.testing.assert_close(weight_dequant(Wq.t(), s, torch.float32), ref.t())
+
+
+ "strategy,shape,block",
+ [
+ ("tensor", (384, 256), None),
+ ("channel", (256, 256), None),
+ ("channel", (384, 256), None),
+ ("block", (256, 256), (128, 128)),
+ ("block", (320, 256), (128, 128)),
+ ],
+)
[email protected]("bias", [False, True])
+def test_fp8_modules_route_to_unsloth_kernels(strategy, shape, block, bias, monkeypatch):
+ from unsloth.models.loader_utils import _route_compressed_tensors_fp8_to_unsloth
+
+ monkeypatch.setenv("UNSLOTH_COMPRESSED_TENSORS_FP8_BLOCK_KERNELS", "1")
+ model, ref = _ct_model(*shape, strategy, bias = bias, block = block)
+ assert _route_compressed_tensors_fp8_to_unsloth(model) == 1
+ lin = model.lin
+ assert lin.weight.dtype == torch.float8_e4m3fn
+ X = torch.randn(2, 5, shape[1], device = "cuda", dtype = torch.bfloat16, requires_grad = True)
+ y = lin(X)
+ y_ref = X.float() @ ref.t() + (lin.bias.float() if bias else 0)
+ (dX,) = torch.autograd.grad(y.float().sum(), X)
+ dX_ref = torch.ones(2, 5, shape[0], device = "cuda") @ ref
+ assert float((y.float() - y_ref).norm() / y_ref.norm()) < 0.05
+ assert float((dX.float() - dX_ref).norm() / dX_ref.norm()) < 0.01
+
+
+def test_fp8_routing_is_opt_in(monkeypatch):
+ from unsloth.models.loader_utils import _route_compressed_tensors_fp8_to_unsloth
+
+ monkeypatch.delenv("UNSLOTH_COMPRESSED_TENSORS_FP8_KERNELS", raising = False)
+ model, _ = _ct_model(256, 256, "channel")
+ assert _route_compressed_tensors_fp8_to_unsloth(model) == 0
+
+
+def test_block_fp8_is_opt_in(monkeypatch):
+ from unsloth.models.loader_utils import _route_compressed_tensors_fp8_to_unsloth
+
+ monkeypatch.delenv("UNSLOTH_COMPRESSED_TENSORS_FP8_BLOCK_KERNELS", raising = False)
+ model, _ = _ct_model(256, 256, "block", block = (128, 128))
+ assert _route_compressed_tensors_fp8_to_unsloth(model) == 0
+
+
[email protected]("strategy", ["tensor", "channel"])
+def test_routed_scale_saves_in_the_checkpoint_shape(strategy):
+ from unsloth.models.loader_utils import _route_compressed_tensors_fp8_to_unsloth
+
+ model, _ = _ct_model(384, 256, strategy)
+ before = {k: v.clone() for k, v in model.state_dict().items() if k.endswith("weight_scale")}
+ if strategy == "channel":
+ model.lin.weight_scale.data = model.lin.weight_scale.data.reshape(-1)
+ before["lin.weight_scale"] = before["lin.weight_scale"].reshape(-1)
+ assert _route_compressed_tensors_fp8_to_unsloth(model) == 1
+ after = model.state_dict()
+ for key, value in before.items():
+ assert after[key].shape == value.shape
+ assert torch.equal(after[key], value)
+ runtime = model.lin.weight_scale.detach().clone()
+ model.load_state_dict(after)
+ assert torch.equal(model.lin.weight_scale, runtime)
+
+
+def test_per_tensor_scale_routes_as_per_row():
+ from unsloth.models.loader_utils import _route_compressed_tensors_fp8_to_unsloth
+
+ model, _ = _ct_model(384, 256, "tensor")
+ scalar = float(model.lin.weight_scale.reshape(-1)[0])
+ assert _route_compressed_tensors_fp8_to_unsloth(model) == 1
+ assert tuple(model.lin.weight_scale.shape) == (384, 1)
+ assert bool((model.lin.weight_scale == scalar).all())
+ assert not hasattr(model.lin.weight, "block_size")
+
+
+def test_non_fp8_or_unsupported_modules_are_left_alone(monkeypatch):
+ from unsloth.models.loader_utils import _route_compressed_tensors_fp8_to_unsloth
+
+ model, _ = _ct_model(256, 256, "channel", weight_type = "int")
+ assert _route_compressed_tensors_fp8_to_unsloth(model) == 0
+ model, _ = _ct_model(256, 256, "channel")
+ model.lin.quantization_scheme.output_activations = (
+ model.lin.quantization_scheme.input_activations
+ )
+ assert _route_compressed_tensors_fp8_to_unsloth(model) == 0
+ model, _ = _ct_model(256, 256, "channel")
+ model.lin.weight = torch.nn.Parameter(model.lin.weight.to(torch.bfloat16), requires_grad = False)
+ assert _route_compressed_tensors_fp8_to_unsloth(model) == 0
+ model, _ = _ct_model(256, 256, "channel")
+ model.lin.weight_scale = torch.nn.Parameter(
+ torch.ones(3, 1, device = "cuda"), requires_grad = False
+ )
+ assert _route_compressed_tensors_fp8_to_unsloth(model) == 0
+ model, _ = _ct_model(256, 256, "channel")
+ monkeypatch.setenv("UNSLOTH_COMPRESSED_TENSORS_FP8_KERNELS", "0")
+ assert _route_compressed_tensors_fp8_to_unsloth(model) == 0
+
+
+def test_decompress_hook_is_removed():
+ from unsloth.models.loader_utils import _route_compressed_tensors_fp8_to_unsloth
+
+ model, _ = _ct_model(256, 256, "channel")
+ model.ct_decompress_hook = model.register_forward_pre_hook(lambda module, args: None)
+ assert len(model._forward_pre_hooks) == 1
+ assert _route_compressed_tensors_fp8_to_unsloth(model) == 1
+ assert not hasattr(model, "ct_decompress_hook")
+ assert len(model._forward_pre_hooks) == 0
+
+
+def test_older_zoo_keeps_the_compressed_tensors_path(monkeypatch):
+ from unsloth.models import loader_utils
+
+ monkeypatch.setattr(loader_utils, "_zoo_peft_forward_keeps_fp8_inputs", lambda: False)
+ model, _ = _ct_model(256, 256, "channel")
+ assert loader_utils._route_compressed_tensors_fp8_to_unsloth(model) == 0
+ assert not hasattr(model.lin, "_unsloth_compressed_tensors_fp8")
+
+
[email protected]("strategy, block", [("channel", None), ("block", (128, 128))])
+def test_single_token_fast_path_adds_the_bias_once(strategy, block, monkeypatch):
+ from unsloth.kernels.utils import fast_linear_forward
+ from unsloth.models.loader_utils import _route_compressed_tensors_fp8_to_unsloth
+
+ monkeypatch.setenv("UNSLOTH_COMPRESSED_TENSORS_FP8_BLOCK_KERNELS", "1")
+ model, ref = _ct_model(256, 256, strategy, bias = True, block = block)
+ assert _route_compressed_tensors_fp8_to_unsloth(model) == 1
+ lin = model.lin
+ with torch.no_grad():
+ lin.bias.fill_(1.0)
+ X = torch.randn(1, 1, 256, device = "cuda", dtype = torch.bfloat16)
+ y = fast_linear_forward(lin, X)
+ y_ref = X.float() @ ref.t() + 1.0
+ assert float((y.float() - y_ref).norm() / y_ref.norm()) < 0.05
+
+
+def test_a_model_mixing_fp8_and_other_schemes_is_not_partly_routed():
+ from unsloth.models.loader_utils import _route_compressed_tensors_fp8_to_unsloth
+
+ model, _ = _ct_model(256, 256, "channel")
+ other, _ = _ct_model(256, 256, "channel", weight_type = "int")
+ model.other = other.lin
+ assert _route_compressed_tensors_fp8_to_unsloth(model) == 0
+ assert not hasattr(model.lin, "_unsloth_compressed_tensors_fp8")
+
+
+ "shape", [(2048, 2048), (256, 2048), (11008, 2048), (2048, 11008), (100, 300), (33, 7)]
+)
[email protected]("rows", [(1,), (1, 1), (1, 1, 1)])
+def test_decode_gemv_matches_the_dequantized_matmul(shape, rows):
+ from unsloth.kernels.fp8 import can_use_fp8_rowwise_gemv, fp8_rowwise_gemv
+
+ if _fp8_kernel_unsupported_here():
+ pytest.skip("triton cannot read float8_e4m3fn on this GPU")
+ model, ref = _ct_model(*shape, "channel")
+ W, s = model.lin.weight, model.lin.weight_scale
+ X = torch.randn(*rows, shape[1], device = "cuda", dtype = torch.bfloat16)
+ with torch.no_grad():
+ assert can_use_fp8_rowwise_gemv(X, W, s)
+ y = fp8_rowwise_gemv(X, W, s)
+ y_again = fp8_rowwise_gemv(X, W, s)
+ y_ref = X.float() @ ref.t()
+ assert y.shape == (*rows, shape[0]) and y.dtype == X.dtype
+ assert float((y.float() - y_ref).norm() / y_ref.norm()) < 0.01
+ assert torch.equal(y, y_again)
+
+
+def test_decode_gemv_refuses_what_it_cannot_compute():
+ from unsloth.kernels.fp8 import can_use_fp8_rowwise_gemv
+
+ model, _ = _ct_model(2048, 16384, "channel")
+ W, s = model.lin.weight, model.lin.weight_scale
+ X = torch.randn(1, 16384, device = "cuda", dtype = torch.bfloat16)
+ with torch.no_grad():
+ # 128x128 block grid here has 2048 == N elements.
+ assert not can_use_fp8_rowwise_gemv(X, W, torch.ones(16, 128, device = "cuda"))
+ assert not can_use_fp8_rowwise_gemv(
+ torch.randn(2, 16384, device = "cuda", dtype = torch.bfloat16), W, s
+ )
+ assert not can_use_fp8_rowwise_gemv(X.float(), W, s)
+ assert not can_use_fp8_rowwise_gemv(X[:, :2048], W.t(), s)
+ assert not can_use_fp8_rowwise_gemv(X.requires_grad_(True), W, s)
+
+
+def _fp8_kernel_unsupported_here():
+ from unsloth.kernels.fp8 import _fp8_kernel_unsupported
+ return _fp8_kernel_unsupported(torch.empty(1, device = "cuda", dtype = torch.float8_e4m3fn))
+
+
+def test_fp8_weights_off_the_gpu_keep_the_compressed_tensors_path():
+ from unsloth.models.loader_utils import _route_compressed_tensors_fp8_to_unsloth
+
+ model, _ = _ct_model(256, 256, "channel")
+ model.cpu()
+ assert _route_compressed_tensors_fp8_to_unsloth(model) == 0
+ assert not hasattr(model.lin, "_unsloth_compressed_tensors_fp8")
+
+
+def test_compiled_block_fp8_linear_passes_the_input_gradient():
+ # torch 2.11 compiled this backward to zeros once Dynamo could trace the whole autograd.Function.
+ from unsloth.kernels.fp8 import fp8_linear
+
+ torch._dynamo.reset()
+ torch.manual_seed(0)
+ W = (torch.randn(256, 256, device = "cuda") * 0.05).to(torch.float8_e4m3fn)
+ s = torch.rand(2, 2, device = "cuda") * 0.01 + 0.001
+ W.block_size = s.block_size = [128, 128]
+ ref = W.float() * s.repeat_interleave(128, 0).repeat_interleave(128, 1)
+ X = torch.randn(2, 5, 256, device = "cuda", dtype = torch.bfloat16, requires_grad = True)
+ (dX,) = torch.autograd.grad(fp8_linear(X, W, s).float().sum(), X)
+ dX_ref = torch.ones(2, 5, 256, device = "cuda") @ ref
+ assert float((dX.float() - dX_ref).norm() / dX_ref.norm()) < 0.01
+
+
[email protected]("dtype", [torch.float16, torch.bfloat16])
+def test_rowwise_dequant_fallback_follows_the_input_dtype(dtype, monkeypatch):
+ from unsloth.kernels import fp8
+
+ monkeypatch.setattr(fp8, "_has_fbgemm_rowwise", lambda: False)
+ W = (torch.randn(256, 512, device = "cuda") * 0.05).to(torch.float8_e4m3fn)
+ s = torch.rand(256, 1, device = "cuda") * 0.01 + 0.001
+ X = torch.randn(3, 512, device = "cuda", dtype = dtype, requires_grad = True)
+ y = fp8.FbgemmFp8Linear_matmul.apply(X, W, s, None)
+ (dX,) = torch.autograd.grad(y.float().sum(), X)
+ ref = W.float() * s
+ assert y.dtype == dtype and dX.dtype == dtype
+ assert float((y.float() - X.float() @ ref.t()).norm() / (X.float() @ ref.t()).norm()) < 0.01
+
+
+def test_full_finetuning_skips_the_frozen_fp8_route(monkeypatch):
+ from unsloth.models import loader_utils
+
+ model, _ = _ct_model(256, 256, "channel")
+ decompressed = []
+ monkeypatch.setattr(loader_utils, "_decompress_compressed_tensors_model", decompressed.append)
+ loader_utils._prepare_compressed_tensors_model(model, full_finetuning = True)
+ assert decompressed == [model]
+ assert not getattr(model.lin, "_unsloth_compressed_tensors_fp8", False)
+ loader_utils._prepare_compressed_tensors_model(model)
+ assert model.lin._unsloth_compressed_tensors_fp8
diff --git a/tests/test_compressed_tensors_load_decompress.py b/tests/test_compressed_tensors_load_decompress.py
new file mode 100644
index 00000000000..684a11ce717
--- /dev/null
+++ b/tests/test_compressed_tensors_load_decompress.py
@@ -0,0 +1,125 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published
+# by the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+
+"""Unsloth decompresses a compressed-tensors checkpoint once at load, outside inference mode."""
+
+import ast
+import os
+import types
+
+import pytest
+import torch
+
+
+def _load_functions(*names):
+ path = os.path.join(
+ os.path.dirname(os.path.dirname(os.path.abspath(__file__))),
+ "unsloth",
+ "models",
+ "loader_utils.py",
+ )
+ tree = ast.parse(open(path, encoding = "utf-8").read())
+ nodes = [n for n in tree.body if isinstance(n, ast.FunctionDef) and n.name in names]
+ namespace = {"torch": torch}
+ exec(compile(ast.Module(body = nodes, type_ignores = []), path, "exec"), namespace)
+ return namespace
+
+
+_decompress_compressed_tensors_model = _load_functions(
+ "_decompress_compressed_tensors_model", "_remove_same_device_compressed_tensors_offload"
+)["_decompress_compressed_tensors_model"]
+
+
+class _Status:
+ value = "compressed"
+
+
+class _Compressor:
+ def __init__(
+ self,
+ fail_first = False,
+ removes_hook = True,
+ ):
+ self.calls = []
+ self.fail_first = fail_first
+ self.removes_hook = removes_hook
+
+ def decompress_model(self, model):
+ self.calls.append(torch.is_inference_mode_enabled())
+ if self.fail_first and len(self.calls) == 1:
+ raise RuntimeError("CUDA out of memory")
+ lin = model.lin
+ lin.weight = torch.nn.Parameter(
+ torch.ones_like(lin.weight, dtype = torch.float32), requires_grad = False
+ )
+ lin.quantization_status = None
+ if self.removes_hook and hasattr(model, "ct_decompress_hook"):
+ model.ct_decompress_hook.remove()
+ delattr(model, "ct_decompress_hook")
+
+
+def _model(compressor):
+ model = torch.nn.Module()
+ model.lin = torch.nn.Linear(4, 4, bias = False)
+ model.lin.quantization_status = _Status()
+ model.config = types.SimpleNamespace(quantization_config = {"quant_method": "compressed-tensors"})
+ model.hf_quantizer = types.SimpleNamespace(compressor = compressor)
+ model.forward = lambda x: model.lin(x)
+ model.ct_decompress_hook = model.register_forward_pre_hook(
+ lambda m, a: compressor.decompress_model(m)
+ )
+ return model
+
+
+def test_decompresses_at_load_and_drops_the_lazy_hook():
+ compressor = _Compressor()
+ model = _model(compressor)
+ assert _decompress_compressed_tensors_model(model)
+ assert compressor.calls == [False]
+ assert not hasattr(model, "ct_decompress_hook") and not model._forward_pre_hooks
+
+
+def test_a_failed_load_decompression_falls_back_outside_inference_mode():
+ compressor = _Compressor(fail_first = True)
+ model = _model(compressor)
+ assert not _decompress_compressed_tensors_model(model)
+ with torch.inference_mode():
+ model(torch.ones(1, 4))
+ assert compressor.calls == [False, False]
+ assert not model.lin.weight.is_inference()
+ x = torch.ones(1, 4, requires_grad = True)
+ torch.nn.functional.linear(x, model.lin.weight).sum().backward()
+
+
[email protected]("fail_first", [False, True])
+def test_the_lazy_hook_is_dropped_when_compressed_tensors_keeps_it(fail_first):
+ compressor = _Compressor(fail_first = fail_first, removes_hook = False)
+ model = _model(compressor)
+ assert _decompress_compressed_tensors_model(model) is not fail_first
+ model(torch.ones(1, 4))
+ model(torch.ones(1, 4))
+ assert len(compressor.calls) == (2 if fail_first else 1)
+ assert not hasattr(model, "ct_decompress_hook") and not model._forward_pre_hooks
+
+
+def test_non_compressed_tensors_models_are_left_alone():
+ compressor = _Compressor()
+ model = _model(compressor)
+ model.config.quantization_config = {"quant_method": "fp8"}
+ assert not _decompress_compressed_tensors_model(model)
+ assert compressor.calls == []
+ assert hasattr(model, "ct_decompress_hook")
diff --git a/tests/test_fp8_device_context.py b/tests/test_fp8_device_context.py
index 1f72a23ec7e..7e1569a2191 100644
--- a/tests/test_fp8_device_context.py
+++ b/tests/test_fp8_device_context.py
@@ -20,6 +20,9 @@ class _FakeDeviceModule:
def device_count(self) -> int:
return self._device_count
+ def is_available(self) -> bool:
+ return self._device_count > 0
+
def device(self, device):
self.device_calls.append(device)
return ("device-context", device)
@@ -80,9 +83,14 @@ class _LaunchVisitor(ast.NodeVisitor):
def _load_device_context_helper(fake_torch: _FakeTorch):
source = FP8_SOURCE.read_text(encoding = "utf-8")
tree = ast.parse(source)
+ namespace = {"torch": fake_torch, "nullcontext": nullcontext}
for node in tree.body:
+ if isinstance(node, ast.Assign) and any(
+ getattr(t, "id", None) in ("_CUDA_MULTI_DEVICE", "_XPU_MULTI_DEVICE")
+ for t in node.targets
+ ):
+ exec(ast.get_source_segment(source, node), namespace)
if isinstance(node, ast.FunctionDef) and node.name == "_fp8_triton_device_context":
- namespace = {"torch": fake_torch, "nullcontext": nullcontext}
exec(ast.get_source_segment(source, node), namespace)
return namespace["_fp8_triton_device_context"]
raise AssertionError("_fp8_triton_device_context was not found")
diff --git a/unsloth/_gpu_init.py b/unsloth/_gpu_init.py
index 9d429386136..e2c344c43db 100644
--- a/unsloth/_gpu_init.py
+++ b/unsloth/_gpu_init.py
@@ -268,6 +268,7 @@ from .import_fixes import (
fix_vllm_aimv2_issue,
fix_vllm_lora_tokenizer_module,
fix_torchao_nf4tensor_move,
+ fix_compressed_tensors_activation_quant_gradient,
fix_torchao_safe_int_mm_repr_probe,
check_vllm_torch_sm100_compatibility,
fix_vllm_guided_decoding_params,
@@ -344,6 +345,7 @@ fix_vllm_lora_tokenizer_module()
# torchao 0.18.0 moved nf4tensor; torchtune (via xcodec2) still imports the old path. Lazy alias, so
# it costs nothing unless asked for.
fix_torchao_nf4tensor_move()
+fix_compressed_tensors_activation_quant_gradient()
fix_torchao_safe_int_mm_repr_probe()
# Check vLLM + torch < 2.9.0 + SM100 compatibility BEFORE importing vLLM
check_vllm_torch_sm100_compatibility()
@@ -397,6 +399,7 @@ del fix_flash_attn_4_namespace_shadow
del fix_vllm_aimv2_issue
del fix_vllm_lora_tokenizer_module
del fix_torchao_nf4tensor_move
+del fix_compressed_tensors_activation_quant_gradient
del fix_torchao_safe_int_mm_repr_probe
del check_vllm_torch_sm100_compatibility
del fix_vllm_guided_decoding_params
diff --git a/unsloth/import_fixes.py b/unsloth/import_fixes.py
index f5cb22135ab..9b5adcadc15 100644
--- a/unsloth/import_fixes.py
+++ b/unsloth/import_fixes.py
@@ -10561,6 +10561,137 @@ def disable_sentencepiece_on_windows():
return True
+# compressed-tensors fake-quantizes W8A8 activations under no_grad; STE so LoRA gets input gradients.
+_CT_FORWARD_MODULE = "compressed_tensors.quantization.lifecycle.forward"
+_CT_BY_NAME_MODULES = (
+ "compressed_tensors.modeling.kvcache",
+ "compressed_tensors.modeling.attention",
+)
+_CT_STE_SENTINEL = "_unsloth_activation_ste"
+_CT_FINDER_SENTINEL = "__unsloth_compressed_tensors_ste_finder__"
+
+
+def _compressed_tensors_ste_forward_quantize(original):
+ import torch
+
+ class _StraightThrough(torch.autograd.Function):
+ # Not `value + (out - value).detach()`: that rounds where a static scale saturates.
+ @staticmethod
+ def forward(ctx, value, quantized):
+ ctx.value_dtype = value.dtype
+ return quantized.to(value.dtype).view_as(quantized)
+
+ @staticmethod
+ def backward(ctx, grad_output):
+ return grad_output.to(ctx.value_dtype), None
+
+ @functools.wraps(original)
+ def forward_quantize(*args, **kwargs):
+ out = original(*args, **kwargs)
+ if not torch.is_grad_enabled():
+ return out
+ value = args[1] if len(args) > 1 else kwargs.get("value")
+ base_name = args[2] if len(args) > 2 else kwargs.get("base_name")
+ if (
+ base_name != "weight"
+ and isinstance(value, torch.Tensor)
+ and isinstance(out, torch.Tensor)
+ and value.requires_grad
+ and not out.requires_grad
+ and out.shape == value.shape
+ ):
+ return _StraightThrough.apply(value, out)
+ return out
+
+ setattr(forward_quantize, _CT_STE_SENTINEL, True)
+ return forward_quantize
+
+
+def _patch_compressed_tensors_forward_module(module):
+ original = getattr(module, "forward_quantize", None)
+ if not callable(original):
+ return False
+ if getattr(original, _CT_STE_SENTINEL, False):
+ return True
+ patched = _compressed_tensors_ste_forward_quantize(original)
+ module.forward_quantize = patched
+ # Modules that imported the function by name before this ran hold the original.
+ for name in _CT_BY_NAME_MODULES:
+ other = sys.modules.get(name)
+ if other is not None and getattr(other, "forward_quantize", None) is original:
+ other.forward_quantize = patched
+ return True
+
+
+class _CompressedTensorsSTELoader(importlib.abc.Loader):
+ __slots__ = ("_loader",)
+
+ def __init__(self, loader):
+ self._loader = loader
+
+ def create_module(self, spec):
+ create_module = getattr(self._loader, "create_module", None)
+ if create_module is None:
+ return None
+ return create_module(spec)
+
+ def exec_module(self, module):
+ self._loader.exec_module(module)
+ try:
+ _patch_compressed_tensors_forward_module(module)
+ except Exception as e:
+ logger.info(f"Unsloth: compressed-tensors activation gradient patch skipped: {e}")
+
+ def __getattr__(self, name):
+ return getattr(self._loader, name)
+
+
+class _CompressedTensorsSTEFinder(importlib.abc.MetaPathFinder):
+ __slots__ = (_CT_FINDER_SENTINEL,)
+
+ def __init__(self):
+ setattr(self, _CT_FINDER_SENTINEL, True)
+
+ def find_spec(
+ self,
+ fullname,
+ path = None,
+ target = None,
+ ):
+ if fullname != _CT_FORWARD_MODULE:
+ return None
+ spec = None
+ for finder in sys.meta_path:
+ if finder is self or getattr(finder, _CT_FINDER_SENTINEL, False):
+ continue
+ finder_find_spec = getattr(finder, "find_spec", None)
+ if finder_find_spec is None:
+ continue
+ try:
+ spec = finder_find_spec(fullname, path, target)
+ except Exception:
+ spec = None
+ if spec is not None:
+ break
+ if spec is None or spec.loader is None or not hasattr(spec.loader, "exec_module"):
+ return None
+ spec.loader = _CompressedTensorsSTELoader(spec.loader)
+ return spec
+
+
+def fix_compressed_tensors_activation_quant_gradient():
+ if importlib.util.find_spec("compressed_tensors") is None:
+ return
+ module = sys.modules.get(_CT_FORWARD_MODULE)
+ if module is not None:
+ _patch_compressed_tensors_forward_module(module)
+ return
+ for finder in sys.meta_path:
+ if getattr(finder, _CT_FINDER_SENTINEL, False):
+ return
+ sys.meta_path.insert(0, _CompressedTensorsSTEFinder())
+
+
def fix_transformers_longcat_lsa_config():
"""Answer AutoConfig's "Unrecognized model" on a LongcatCausalLM config (no model_type,
auto_map or modeling code) with ``models/longcat_lsa.py``; all other loads are untouched."""
diff --git a/unsloth/kernels/fp8.py b/unsloth/kernels/fp8.py
index db90408ca28..92973106d4a 100644
--- a/unsloth/kernels/fp8.py
+++ b/unsloth/kernels/fp8.py
@@ -24,10 +24,24 @@ from unsloth_zoo.temporary_patches.common import torch_compile
torch_matmul = torch.matmul
-def _fp8_triton_device_context(tensor: torch.Tensor):
- if tensor.device.type == "cuda" and torch.cuda.device_count() > 1:
+# Read once for the decode GEMV: device_count() in a compiled region graph-breaks ("torch.* op returned non-Tensor").
+_CUDA_MULTI_DEVICE = torch.cuda.is_available() and torch.cuda.device_count() > 1
+_XPU_MULTI_DEVICE = (
+ hasattr(torch, "xpu") and torch.xpu.is_available() and torch.xpu.device_count() > 1
+)
+
+
+def _fp8_triton_device_context(tensor: torch.Tensor, static_device_count = False):
+ # static_device_count only for forward-only kernels: torch 2.11 compiles FP8BlockQuantLinear's backward to zeros.
+ if tensor.device.type == "cuda" and (
+ _CUDA_MULTI_DEVICE if static_device_count else torch.cuda.device_count() > 1
+ ):
return torch.cuda.device(tensor.device)
- if tensor.device.type == "xpu" and hasattr(torch, "xpu") and torch.xpu.device_count() > 1:
+ if tensor.device.type == "xpu" and (
+ _XPU_MULTI_DEVICE
+ if static_device_count
+ else (hasattr(torch, "xpu") and torch.xpu.device_count() > 1)
+ ):
return torch.xpu.device(tensor.device)
return nullcontext()
@@ -112,11 +126,29 @@ def weight_dequant_block(
return y
+def _is_transposed_view(x):
+ return (
+ x.dim() == 2 and x.shape[0] > 1 and x.shape[1] > 1 and x.stride(0) == 1 and x.stride(1) != 1
+ )
+
+
+def _has_fbgemm_rowwise():
+ try:
+ return hasattr(torch.ops.fbgemm, "quantize_fp8_per_row") and hasattr(
+ torch.ops.fbgemm, "f8f8bf16_rowwise"
+ )
+ except Exception:
+ return False
+
+
def weight_dequant(
x: torch.Tensor,
s: torch.Tensor,
dtype = torch.bfloat16,
):
+ # Transposed views keep scales in storage order; square weights can't be told apart by shape.
+ if _is_transposed_view(x):
+ return weight_dequant(x.t(), s, dtype).t()
# Per-tensor scale: single value for entire weight matrix
if s.numel() == 1:
return x.to(dtype) * s.view(1, 1).to(dtype)
@@ -134,7 +166,9 @@ def weight_dequant(
else:
# Block quantized weight: scale shape is (ceil(m/block_m), ceil(n/block_n)). Go through the
# any-shape helper so fast_dequantize's callers get the pre-sm89 fallback too.
- return _blockwise_weight_dequant_any_shape(x, s, [128, 128], dtype)
+ return _blockwise_weight_dequant_any_shape(
+ x, s, getattr(s, "block_size", None) or [128, 128], dtype
+ )
# Copied from huggingface.co/deepseek-ai/DeepSeek-V3 inference/kernel.py
@@ -366,6 +400,59 @@ def _fp8_kernel_unsupported(tensor, kernel_dtype = None):
)
+def _fp8_rowwise_gemv_kernel(
+ x_ptr, w_ptr, s_ptr, y_ptr, N, K, stride_wn, BLOCK_N: tl.constexpr, BLOCK_K: tl.constexpr
+):
+ offs_n = tl.program_id(0) * BLOCK_N + tl.arange(0, BLOCK_N)
+ acc = tl.zeros((BLOCK_N, BLOCK_K), dtype = tl.float32)
+ for k0 in range(0, K, BLOCK_K):
+ offs_k = k0 + tl.arange(0, BLOCK_K)
+ x = tl.load(x_ptr + offs_k, mask = offs_k < K, other = 0.0).to(tl.float32)
+ w = tl.load(
+ w_ptr + offs_n[:, None] * stride_wn + offs_k[None, :],
+ mask = (offs_n[:, None] < N) & (offs_k[None, :] < K),
+ other = 0.0,
+ ).to(tl.float32)
+ acc += w * x[None, :]
+ s = tl.load(s_ptr + offs_n, mask = offs_n < N, other = 0.0).to(tl.float32)
+ y = tl.sum(acc, axis = 1) * s
+ tl.store(y_ptr + offs_n, y.to(y_ptr.dtype.element_ty), mask = offs_n < N)
+
+
+def can_use_fp8_rowwise_gemv(X, weight, weight_scale):
+ if not (X.is_cuda and weight.is_cuda and X.dtype in (torch.bfloat16, torch.float16)):
+ return False
+ if weight.dtype != torch.float8_e4m3fn or weight.dim() != 2 or not weight.is_contiguous():
+ return False
+ if X.numel() != X.shape[-1] or X.shape[-1] != weight.shape[1]:
+ return False
+ # Exactly one scale per output row: a block grid can have N elements by coincidence.
+ if (
+ tuple(weight_scale.shape) not in ((weight.shape[0], 1), (weight.shape[0],))
+ or weight.shape[0] == 1
+ ):
+ return False
+ if torch.is_grad_enabled() and (X.requires_grad or weight.requires_grad):
+ return False
+ return not _fp8_kernel_unsupported(weight)
+
+
+def fp8_rowwise_gemv(X, weight, weight_scale):
+ """X @ (weight * weight_scale).T for one token without a 16-bit weight copy."""
+ N, K = weight.shape
+ out = torch.empty((*X.shape[:-1], N), device = X.device, dtype = X.dtype)
+ if not X.is_contiguous():
+ X = X.contiguous()
+ scale = weight_scale if weight_scale.is_contiguous() else weight_scale.contiguous()
+ block_n = 1 if N <= 4096 else (4 if N <= 32768 else 8)
+ with _fp8_triton_device_context(X, static_device_count = True):
+ _fp8_rowwise_gemv_kernel[(triton.cdiv(N, block_n),)](
+ X, weight, scale, out, N, K, weight.stride(0), BLOCK_N = block_n, BLOCK_K = 512, num_warps = 2
+ )
+ return out
+
+
# Expanding the scale over the whole weight needs two m*n float32 temporaries, ~6x the triton
# kernel's peak; chunk by block-rows so the scratch stays bounded. Values are identical.
_DEQUANT_CHUNK_ELEMS = 8 * 1024 * 1024
@@ -509,8 +596,12 @@ class FbgemmFp8Linear_matmul(torch.autograd.Function):
weight_scale,
bias = None,
):
- if weight.shape[0] == weight_scale.shape[0] and (
- weight.shape[0] % 8 == 0 and weight.shape[1] % 8 == 0
+ if (
+ weight.shape[0] == weight_scale.shape[0]
+ and (weight.shape[0] % 8 == 0 and weight.shape[1] % 8 == 0)
+ and not _is_transposed_view(weight)
+ and _has_fbgemm_rowwise()
+ and not _fp8_kernel_unsupported(weight, torch.float8_e4m3fn)
):
# The kernel needs weight dims divisible by 8 (else "cutlass cannot implement"), and padding plus
# f8f8bf16 is slower than dequant plus bf16 matmul.
@@ -538,12 +629,9 @@ class FbgemmFp8Linear_matmul(torch.autograd.Function):
output = output.to(x.device, x.dtype)
output = output.reshape(output_shape)
del x_quantized, x_scale
- elif (
- weight.shape[0] != weight_scale.shape[0] and weight.shape[1] == weight_scale.shape[0]
- ) or (weight.shape[0] % 8 != 0 or weight.shape[1] % 8 != 0):
- # Transposed weight/scale (backward dY@W) or a non-divisible-by-8 shape (Qwen 2.5 VL 7B gate proj
- # 3420x1280): dequant is preferred.
- W_deq = weight_dequant(weight, weight_scale).T
+ elif weight_scale.shape[0] in (weight.shape[0], weight.shape[1]):
+ # Transposed, non-divisible-by-8 (Qwen 2.5 VL 7B 3420x1280), no FBGEMM, or pre-sm89: dequant.
+ W_deq = weight_dequant(weight, weight_scale, x.dtype).T
output = torch_matmul(x, W_deq)
output = output + bias if bias is not None else output
del W_deq
@@ -558,7 +646,7 @@ class FbgemmFp8Linear_matmul(torch.autograd.Function):
@staticmethod
def backward(ctx, grad_output):
- W_deq = weight_dequant(ctx.weight, ctx.weight_scale)
+ W_deq = weight_dequant(ctx.weight, ctx.weight_scale, grad_output.dtype)
grad_X = torch_matmul(grad_output, W_deq)
del W_deq
return grad_X, None, None, None, None
diff --git a/unsloth/kernels/utils.py b/unsloth/kernels/utils.py
index 12eeb3943e0..694ed690226 100644
--- a/unsloth/kernels/utils.py
+++ b/unsloth/kernels/utils.py
@@ -27,7 +27,7 @@ from ..device_type import (
ALLOW_PREQUANTIZED_MODELS,
)
from ..bnb_availability import native_kernels_ready
-from .fp8 import weight_dequant, fp8_linear
+from .fp8 import weight_dequant, fp8_linear, can_use_fp8_rowwise_gemv, fp8_rowwise_gemv
import functools
# torch.cuda.amp.custom_fwd is deprecated from 2.4.
@@ -1051,7 +1051,11 @@ def fast_linear_forward(
if W_quant is None:
out = torch_matmul(X, W.t(), out = out)
elif W.dtype == torch.float8_e4m3fn:
- out = fp8_linear(X, W, W_quant, bias)
+ # The bias is added once below; the per-channel fp8_linear path would add it a second time.
+ if can_use_fp8_rowwise_gemv(X, W, W_quant):
+ out = fp8_rowwise_gemv(X, W, W_quant)
+ else:
+ out = fp8_linear(X, W, W_quant)
elif bsz == 1 and q_len == 1:
out = fast_gemv(X, W, W_quant, out = out)
else:
diff --git a/unsloth/models/llama.py b/unsloth/models/llama.py
index 021ee7a3aba..86d956d9271 100644
--- a/unsloth/models/llama.py
+++ b/unsloth/models/llama.py
@@ -35,6 +35,7 @@ from .loader_utils import (
_exclude_rope_inv_freq_from_ddp,
_get_fp8_mode_and_check_settings,
_restore_dropped_fp8_scales,
+ _prepare_compressed_tensors_model,
planner_class_mismatch_reason,
planner_model_class,
planner_config_overrides,
@@ -2817,6 +2818,7 @@ class FastLlamaModel:
variant = kwargs.get("variant"),
dtype = dtype,
)
+ _prepare_compressed_tensors_model(model)
elif not fast_inference:
if user_config is not None or _modelopt_rewritten:
# Transformers 5.x @strict model init rejects extra kwargs next to config=, so set the override
@@ -2879,6 +2881,7 @@ class FastLlamaModel:
variant = kwargs.get("variant"),
dtype = dtype,
)
+ _prepare_compressed_tensors_model(model)
model.fast_generate = make_fast_generate_wrapper(model.generate)
model.fast_generate_batches = None
else:
diff --git a/unsloth/models/loader_utils.py b/unsloth/models/loader_utils.py
index 59212e7fb49..d3333080a53 100644
--- a/unsloth/models/loader_utils.py
+++ b/unsloth/models/loader_utils.py
@@ -1315,6 +1315,163 @@ def _match_fp8_module(module_by_name, base):
return None
+_CT_FP8_STRATEGIES = ("tensor", "channel", "block")
+
+
+def _compressed_tensors_fp8_block_size(module, weights):
+ """(out, in) block of a routable compressed-tensors FP8 weight, else None."""
+ if getattr(weights, "type", None) != "float" or getattr(weights, "num_bits", None) != 8:
+ return None
+ strategy = getattr(weights, "strategy", None)
+ strategy = str(getattr(strategy, "value", strategy))
+ # Block FP8 is opt-in: fp8_block_quant_linear is far slower than the decompressed bf16 model.
+ if (
+ strategy == "block"
+ and os.environ.get("UNSLOTH_COMPRESSED_TENSORS_FP8_BLOCK_KERNELS", "0") != "1"
+ ):
+ return None
+ if (
+ strategy not in _CT_FP8_STRATEGIES
+ or getattr(weights, "dynamic", False)
+ or getattr(weights, "actorder", None)
+ ):
+ return None
+ weight = getattr(module, "weight", None)
+ scale = getattr(module, "weight_scale", None)
+ if weight is None or scale is None or weight.dtype != torch.float8_e4m3fn or weight.dim() != 2:
+ return None
+ zero_point = getattr(module, "weight_zero_point", None)
+ if zero_point is not None and bool(torch.any(zero_point != 0)):
+ return None
+ out_features, in_features = weight.shape
+ if strategy == "tensor":
+ # Routed as per-channel (scale broadcast per row): the rowwise paths beat the block kernel.
+ return [1, in_features] if scale.numel() == 1 else None
+ if strategy == "channel":
+ return (
+ [1, in_features] if tuple(scale.shape) in ((out_features, 1), (out_features,)) else None
+ )
+ block = list(getattr(weights, "block_structure", None) or ())
+ if len(block) != 2 or scale.dim() != 2:
+ return None
+ if tuple(scale.shape) != (-(-out_features // block[0]), -(-in_features // block[1])):
+ return None
+ return block
+
+
+def _save_compressed_tensors_scale_shape(module, state_dict, prefix, local_metadata):
+ # Saved checkpoints keep the config's scale shape; a per-tensor scale was broadcast per row.
+ key = prefix + "weight_scale"
+ shape = getattr(module, "_unsloth_ct_scale_shape", None)
+ scale = state_dict.get(key)
+ if shape is None or scale is None or tuple(scale.shape) == shape:
+ return state_dict
+ numel = 1
+ for n in shape:
+ numel *= n
+ state_dict[key] = scale.reshape(-1)[:1].reshape(shape) if numel == 1 else scale.reshape(shape)
+ return state_dict
+
+
+def _load_compressed_tensors_scale_shape(module, state_dict, prefix, *args):
+ key = prefix + "weight_scale"
+ scale = state_dict.get(key)
+ current = getattr(module, "weight_scale", None)
+ if scale is None or current is None or scale.shape == current.shape:
+ return
+ if scale.numel() == 1:
+ state_dict[key] = scale.reshape(1, 1).expand(current.shape).contiguous()
+ elif scale.numel() == current.numel():
+ state_dict[key] = scale.reshape(current.shape)
+
+
+def _unsloth_compressed_tensors_fp8_forward(self, input):
+ from unsloth.kernels.fp8 import can_use_fp8_rowwise_gemv, fp8_linear, fp8_rowwise_gemv
+
+ if can_use_fp8_rowwise_gemv(input, self.weight, self.weight_scale):
+ out = fp8_rowwise_gemv(input, self.weight, self.weight_scale)
+ else:
+ out = fp8_linear(input, self.weight, self.weight_scale)
+ if self.bias is not None:
+ out = out + self.bias.to(out.dtype)
+ return out
+
+
[email protected]_cache(maxsize = 1)
+def _zoo_peft_forward_keeps_fp8_inputs():
+ """Older zoo compiled LoRA forwards cast `x` to the FP8 weight dtype; route only when it does not."""
+ try:
+ import inspect
+ from unsloth_zoo import compiler
+ source = inspect.getsource(compiler.patch_lora_forwards)
+ except Exception:
+ return False
+ return "x = x.to(self.base_layer.weight.dtype)" not in source or (
+ "self.base_layer.weight.dtype in (torch.float16, torch.bfloat16, torch.float32)" in source
+ )
+
+
+def _route_compressed_tensors_fp8_to_unsloth(model):
+ # Opt-in: saves memory, but on-the-fly dequant trains slower than the decompressed bf16 model.
+ if os.environ.get("UNSLOTH_COMPRESSED_TENSORS_FP8_KERNELS", "0") != "1":
+ return 0
+ if getattr(getattr(model, "config", None), "quantization_config", None) is None:
+ return 0
+ if not _zoo_peft_forward_keeps_fp8_inputs():
+ return 0
+ routable = []
+ for module in model.modules():
+ scheme = getattr(module, "quantization_scheme", None)
+ if scheme is None or getattr(scheme, "weights", None) is None:
+ continue
+ block = None
+ weight = getattr(module, "weight", None)
+ on_gpu = isinstance(weight, torch.Tensor) and weight.device.type in ("cuda", "xpu")
+ if (
+ on_gpu
+ and isinstance(module, torch.nn.Linear)
+ and getattr(scheme, "output_activations", None) is None
+ ):
+ block = _compressed_tensors_fp8_block_size(module, scheme.weights)
+ if block is None:
+ return 0
+ routable.append((module, block))
+ converted = 0
+ for module, block in routable:
+ scale = module.weight_scale
+ scale_shape = tuple(scale.shape)
+ if scale.numel() == 1:
+ scale.data = scale.data.reshape(1, 1).expand(module.weight.shape[0], 1).contiguous()
+ elif scale.dim() == 1:
+ scale.data = scale.data.view(-1, 1)
+ if tuple(scale.shape) != scale_shape:
+ module._unsloth_ct_scale_shape = scale_shape
+ module._register_state_dict_hook(_save_compressed_tensors_scale_shape)
+ module._register_load_state_dict_pre_hook(
+ _load_compressed_tensors_scale_shape, with_module = True
+ )
+ module.weight.requires_grad_(False)
+ scale.requires_grad_(False)
+ if block != [1, module.weight.shape[1]]:
+ module.weight.block_size = block
+ scale.block_size = block
+ module.block_size = block
+ module.forward = _unsloth_compressed_tensors_fp8_forward.__get__(module)
+ module._unsloth_compressed_tensors_fp8 = True
+ converted += 1
+ if converted:
+ for owner in (model, getattr(model, "model", None), getattr(model, "base_model", None)):
+ hook = getattr(owner, "ct_decompress_hook", None)
+ if hook is not None:
+ hook.remove()
+ try:
+ delattr(owner, "ct_decompress_hook")
+ except AttributeError:
+ pass
+ model._unsloth_compressed_tensors_fp8 = converted
+ return converted
+
+
_FP8_DEQUANT_DTYPES = (torch.bfloat16, torch.float16, torch.float32)
@@ -1556,6 +1713,79 @@ def _restore_dropped_fp8_scales(
return (0, 0)
+def _remove_same_device_compressed_tensors_offload(model):
+ """Unwrap compressed-tensors >= 0.19 non-offloading OffloadCaches, which torch.compile cannot trace."""
+ try:
+ from compressed_tensors.offload.cache import OffloadCache
+ from compressed_tensors.offload.module import remove_module_offload
+ except Exception:
+ return 0
+ removed = 0
+ for module in model.modules():
+ cache = module._parameters
+ if not isinstance(cache, OffloadCache):
+ continue
+ onload = getattr(cache, "onload_device", None)
+ offload = getattr(cache, "offload_device", None)
+ if onload is None or offload is None or torch.device(onload) != torch.device(offload):
+ continue
+ remove_module_offload(module)
+ removed += 1
+ return removed
+
+
+def _decompress_compressed_tensors_model(model):
+ """Decompress at load: PEFT skips the first-forward hook, and in `generate` it yields inference tensors."""
+ if getattr(model, "_unsloth_compressed_tensors_fp8", 0):
+ return False
+ quant_config = getattr(getattr(model, "config", None), "quantization_config", None)
+ if isinstance(quant_config, dict):
+ method = quant_config.get("quant_method", None)
+ else:
+ method = getattr(quant_config, "quant_method", None)
+ if getattr(method, "value", method) != "compressed-tensors":
+ return False
+ if not any(
+ str(getattr(getattr(module, "quantization_status", None), "value", "")) == "compressed"
+ for module in model.modules()
+ ):
+ return False
+ compressor = getattr(getattr(model, "hf_quantizer", None), "compressor", None)
+ if compressor is None or not hasattr(compressor, "decompress_model"):
+ return False
+
+ def _decompress(module, *args):
+ with torch.inference_mode(False), torch.no_grad():
+ compressor.decompress_model(module)
+ _remove_same_device_compressed_tensors_offload(module)
+ # Some compressed-tensors releases (0.19.0) leave the hook in place: drop it, also after a retry.
+ hook = getattr(model, "ct_decompress_hook", None)
+ if hook is not None:
+ hook.remove()
+ try:
+ delattr(model, "ct_decompress_hook")
+ except AttributeError:
+ pass
+
+ # Swap the hook first: decompress_model removes it by name.
+ hook = getattr(model, "ct_decompress_hook", None)
+ if hook is not None:
+ hook.remove()
+ model.ct_decompress_hook = model.register_forward_pre_hook(_decompress)
+ try:
+ _decompress(model)
+ except Exception as e:
+ print(f"Unsloth: could not decompress the compressed-tensors checkpoint after load: {e}")
+ return False
+ return True
+
+
+def _prepare_compressed_tensors_model(model, full_finetuning = False):
+ # Routed FP8 weights are frozen, so full finetuning always takes the decompressed bf16 weights.
+ if full_finetuning or not _route_compressed_tensors_fp8_to_unsloth(model):
+ _decompress_compressed_tensors_model(model)
+
+
_FP8_SCALE_SUFFIXES = (".weight_scale_inv", "_scale_inv", ".weight_scale", "_scale")
_FP8_LEFTOVER_MAX_CHUNK = 1 << 26
diff --git a/unsloth/models/vision.py b/unsloth/models/vision.py
index d778ca9aa0a..299104d410a 100644
--- a/unsloth/models/vision.py
+++ b/unsloth/models/vision.py
@@ -106,6 +106,7 @@ from .loader_utils import (
_get_fp8_mode_and_check_settings,
_dequantize_leftover_fp8_params,
_restore_dropped_fp8_scales,
+ _prepare_compressed_tensors_model,
planner_class_mismatch_reason,
planner_model_class,
planner_quantization_kwargs,
@@ -2418,6 +2419,7 @@ class FastBaseModel:
variant = kwargs.get("variant"),
dtype = torch_dtype,
)
+ _prepare_compressed_tensors_model(model, full_finetuning = full_finetuning)
if load_in_16bit and not load_in_4bit and not load_in_8bit:
_dequantize_leftover_fp8_params(
model,