Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Pass gradients through compressed-tensors activation quantization so W8A8 checkpoints train with LoRA (#11585)
Daniel Han9d15a59883c8
diff --git a/tests/test_compressed_tensors_activation_ste.py b/tests/test_compressed_tensors_activation_ste.pynew file mode 100644index 00000000000..23c6c836810--- /dev/null+++ b/tests/test_compressed_tensors_activation_ste.py@@ -0,0 +1,122 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved.+#+# This program is free software: you can redistribute it and/or modify+# it under the terms of the GNU Affero General Public License as published+# by the Free Software Foundation, either version 3 of the License, or+# (at your option) any later version.+#+# This program is distributed in the hope that it will be useful,+# but WITHOUT ANY WARRANTY; without even the implied warranty of+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+# GNU Affero General Public License for more details.+#+# You should have received a copy of the GNU Affero General Public License+# along with this program. If not, see <https://www.gnu.org/licenses/>.++"""Each case runs in a subprocess: the patch changes compressed-tensors module state."""++import os+import subprocess+import sys+import textwrap++import pytest++pytest.importorskip("compressed_tensors")++_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))++_RUNNER = textwrap.dedent(+ """+ import importlib.util, json, sys+ mode = sys.argv[1]+ if mode != "base":+ spec = importlib.util.spec_from_file_location("unsloth_import_fixes_under_test", sys.argv[2])+ fixes = importlib.util.module_from_spec(spec)+ spec.loader.exec_module(fixes)+ if mode == "lazy":+ assert "compressed_tensors.quantization.lifecycle.forward" not in sys.modules+ fixes.fix_compressed_tensors_activation_quant_gradient()+ import compressed_tensors # noqa: F401+ if mode != "lazy":+ fixes.fix_compressed_tensors_activation_quant_gradient()+ fixes.fix_compressed_tensors_activation_quant_gradient() # idempotent+ import torch+ from compressed_tensors.quantization import (+ QuantizationArgs, QuantizationConfig, QuantizationScheme, apply_quantization_config,+ )+ torch.manual_seed(0)+ model = torch.nn.Sequential(torch.nn.Linear(64, 32, bias = False))+ static = len(sys.argv) > 3 and sys.argv[3] == "static"+ if static: # INT8 W8A8 with a calibrated per-tensor input scale that saturates+ activations = QuantizationArgs(+ num_bits = 8, type = "int", strategy = "tensor", dynamic = False, symmetric = True,+ )+ else:+ activations = QuantizationArgs(+ num_bits = 8, type = "float", strategy = "token", dynamic = True, symmetric = True,+ )+ scheme = QuantizationScheme(targets = ["Linear"], input_activations = activations)+ apply_quantization_config(model, QuantizationConfig(config_groups = {"group_0": scheme}, quantization_status = "frozen"))+ lin = model[0]+ lin.weight.requires_grad_(False) # the frozen base layer under a LoRA adapter+ if static:+ model.to(torch.bfloat16)+ lin.input_scale.data.fill_(0.01) # clips at 1.27, so most of randn * 4 saturates+ x = (torch.randn(4, 64) * (4 if static else 1)).to(lin.weight.dtype).requires_grad_(True)+ y = lin(x)+ with torch.no_grad():+ y_nograd = lin(x)+ out = {"has_grad_fn": y.grad_fn is not None, "same_forward": bool(torch.equal(y.detach(), y_nograd))}+ if y.grad_fn is not None:+ (gx,) = torch.autograd.grad(y.sum(), x)+ ref = torch.ones(4, 32, dtype = lin.weight.dtype) @ lin.weight+ out["grad_err"] = float((gx - ref).abs().max().float())+ import compressed_tensors.quantization.lifecycle.forward as f+ out["patched"] = bool(getattr(f.forward_quantize, "_unsloth_activation_ste", False))+ print(json.dumps(out))+ """+)+++def _run(mode, scheme = "dynamic"):+ proc = subprocess.run(+ [+ sys.executable,+ "-c",+ _RUNNER,+ mode,+ os.path.join(_ROOT, "unsloth", "import_fixes.py"),+ scheme,+ ],+ capture_output = True,+ text = True,+ timeout = 600,+ env = dict(os.environ, CUDA_VISIBLE_DEVICES = ""),+ )+ assert proc.returncode == 0, proc.stdout[-2000:] + proc.stderr[-4000:]+ import json++ return json.loads(proc.stdout.strip().splitlines()[-1])+++def test_without_the_fix_the_input_gradient_is_lost():+ out = _run("base")+ assert not out["has_grad_fn"]++[email protected]("mode", ["eager", "lazy"])+def test_activation_quantization_passes_the_gradient_straight_through(mode):+ out = _run(mode)+ assert out["patched"]+ assert out["has_grad_fn"]+ assert out["same_forward"] # the quantized forward is untouched+ assert out["grad_err"] < 1e-5 # dX = dY @ W, as if the activation were not quantized+++def test_a_saturating_static_scale_keeps_the_forward_bit_identical():+ out = _run("eager", "static")+ assert out["patched"]+ assert out["has_grad_fn"]+ assert out["same_forward"]diff --git a/tests/test_compressed_tensors_fp8_kernel_route.py b/tests/test_compressed_tensors_fp8_kernel_route.pynew file mode 100644index 00000000000..23f9870f3fd--- /dev/null+++ b/tests/test_compressed_tensors_fp8_kernel_route.py@@ -0,0 +1,338 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved.+#+# This program is free software: you can redistribute it and/or modify+# it under the terms of the GNU Affero General Public License as published+# by the Free Software Foundation, either version 3 of the License, or+# (at your option) any later version.+#+# This program is distributed in the hope that it will be useful,+# but WITHOUT ANY WARRANTY; without even the implied warranty of+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+# GNU Affero General Public License for more details.+#+# You should have received a copy of the GNU Affero General Public License+# along with this program. If not, see <https://www.gnu.org/licenses/>.+++import types++import pytest+import torch++pytestmark = pytest.mark.skipif(not torch.cuda.is_available(), reason = "needs CUDA")+ct_quant = pytest.importorskip("compressed_tensors.quantization")++[email protected](autouse = True)+def _zoo_with_float_only_cast(monkeypatch):+ monkeypatch.setenv("UNSLOTH_COMPRESSED_TENSORS_FP8_KERNELS", "1")+ from unsloth.models import loader_utils+ monkeypatch.setattr(loader_utils, "_zoo_peft_forward_keeps_fp8_inputs", lambda: True)+++def _quantize(+ W,+ strategy,+ block = None,+):+ o, i = W.shape+ if strategy == "tensor":+ s = W.abs().amax().view(1) / 448+ full = s.expand(o, i)+ elif strategy == "channel":+ s = W.abs().amax(1, keepdim = True) / 448+ full = s.expand(o, i)+ else:+ bo, bi = block+ s = torch.empty(-(-o // bo), -(-i // bi), device = W.device)+ for a in range(s.shape[0]):+ for b in range(s.shape[1]):+ s[a, b] = W[a * bo : (a + 1) * bo, b * bi : (b + 1) * bi].abs().amax() / 448+ full = s.repeat_interleave(bo, 0)[:o].repeat_interleave(bi, 1)[:, :i]+ Wq = (W / full).to(torch.float8_e4m3fn)+ return Wq, s.to(torch.bfloat16), Wq.float() * full.to(torch.bfloat16).float()+++def _ct_model(+ o,+ i,+ strategy,+ bias = False,+ block = None,+ weight_type = "float",+):+ torch.manual_seed(0)+ lin = torch.nn.Linear(i, o, bias = bias, device = "cuda", dtype = torch.bfloat16)+ Wq, s, ref = _quantize(torch.randn(o, i, device = "cuda") * 0.02, strategy, block)+ lin.weight = torch.nn.Parameter(Wq, requires_grad = False)+ lin.weight_scale = torch.nn.Parameter(s, requires_grad = False)+ kwargs = dict(num_bits = 8, type = weight_type, strategy = strategy, symmetric = True, dynamic = False)+ if block is not None:+ kwargs["block_structure"] = list(block)+ lin.quantization_scheme = ct_quant.QuantizationScheme(+ targets = ["Linear"],+ weights = ct_quant.QuantizationArgs(**kwargs),+ input_activations = ct_quant.QuantizationArgs(+ num_bits = 8, type = "float", strategy = "token", dynamic = True, symmetric = True+ ),+ )++ def ct_forward(self, x):+ raise AssertionError("compressed-tensors forward must not run on a routed module")++ lin.forward = types.MethodType(ct_forward, lin)+ model = torch.nn.Module()+ model.lin = lin+ model.config = types.SimpleNamespace(quantization_config = {"quant_method": "compressed-tensors"})+ return model, ref++[email protected]("shape", [(256, 256), (384, 256), (256, 384)])+def test_weight_dequant_transposed_row_scale(shape):+ from unsloth.kernels.fp8 import weight_dequant++ Wq, s, ref = _quantize(torch.randn(*shape, device = "cuda") * 0.02, "channel")+ torch.testing.assert_close(weight_dequant(Wq, s, torch.float32), ref)+ torch.testing.assert_close(weight_dequant(Wq.t(), s, torch.float32), ref.t())+++ "strategy,shape,block",+ [+ ("tensor", (384, 256), None),+ ("channel", (256, 256), None),+ ("channel", (384, 256), None),+ ("block", (256, 256), (128, 128)),+ ("block", (320, 256), (128, 128)),+ ],+)[email protected]("bias", [False, True])+def test_fp8_modules_route_to_unsloth_kernels(strategy, shape, block, bias, monkeypatch):+ from unsloth.models.loader_utils import _route_compressed_tensors_fp8_to_unsloth++ monkeypatch.setenv("UNSLOTH_COMPRESSED_TENSORS_FP8_BLOCK_KERNELS", "1")+ model, ref = _ct_model(*shape, strategy, bias = bias, block = block)+ assert _route_compressed_tensors_fp8_to_unsloth(model) == 1+ lin = model.lin+ assert lin.weight.dtype == torch.float8_e4m3fn+ X = torch.randn(2, 5, shape[1], device = "cuda", dtype = torch.bfloat16, requires_grad = True)+ y = lin(X)+ y_ref = X.float() @ ref.t() + (lin.bias.float() if bias else 0)+ (dX,) = torch.autograd.grad(y.float().sum(), X)+ dX_ref = torch.ones(2, 5, shape[0], device = "cuda") @ ref+ assert float((y.float() - y_ref).norm() / y_ref.norm()) < 0.05+ assert float((dX.float() - dX_ref).norm() / dX_ref.norm()) < 0.01+++def test_fp8_routing_is_opt_in(monkeypatch):+ from unsloth.models.loader_utils import _route_compressed_tensors_fp8_to_unsloth++ monkeypatch.delenv("UNSLOTH_COMPRESSED_TENSORS_FP8_KERNELS", raising = False)+ model, _ = _ct_model(256, 256, "channel")+ assert _route_compressed_tensors_fp8_to_unsloth(model) == 0+++def test_block_fp8_is_opt_in(monkeypatch):+ from unsloth.models.loader_utils import _route_compressed_tensors_fp8_to_unsloth++ monkeypatch.delenv("UNSLOTH_COMPRESSED_TENSORS_FP8_BLOCK_KERNELS", raising = False)+ model, _ = _ct_model(256, 256, "block", block = (128, 128))+ assert _route_compressed_tensors_fp8_to_unsloth(model) == 0++[email protected]("strategy", ["tensor", "channel"])+def test_routed_scale_saves_in_the_checkpoint_shape(strategy):+ from unsloth.models.loader_utils import _route_compressed_tensors_fp8_to_unsloth++ model, _ = _ct_model(384, 256, strategy)+ before = {k: v.clone() for k, v in model.state_dict().items() if k.endswith("weight_scale")}+ if strategy == "channel":+ model.lin.weight_scale.data = model.lin.weight_scale.data.reshape(-1)+ before["lin.weight_scale"] = before["lin.weight_scale"].reshape(-1)+ assert _route_compressed_tensors_fp8_to_unsloth(model) == 1+ after = model.state_dict()+ for key, value in before.items():+ assert after[key].shape == value.shape+ assert torch.equal(after[key], value)+ runtime = model.lin.weight_scale.detach().clone()+ model.load_state_dict(after)+ assert torch.equal(model.lin.weight_scale, runtime)+++def test_per_tensor_scale_routes_as_per_row():+ from unsloth.models.loader_utils import _route_compressed_tensors_fp8_to_unsloth++ model, _ = _ct_model(384, 256, "tensor")+ scalar = float(model.lin.weight_scale.reshape(-1)[0])+ assert _route_compressed_tensors_fp8_to_unsloth(model) == 1+ assert tuple(model.lin.weight_scale.shape) == (384, 1)+ assert bool((model.lin.weight_scale == scalar).all())+ assert not hasattr(model.lin.weight, "block_size")+++def test_non_fp8_or_unsupported_modules_are_left_alone(monkeypatch):+ from unsloth.models.loader_utils import _route_compressed_tensors_fp8_to_unsloth++ model, _ = _ct_model(256, 256, "channel", weight_type = "int")+ assert _route_compressed_tensors_fp8_to_unsloth(model) == 0+ model, _ = _ct_model(256, 256, "channel")+ model.lin.quantization_scheme.output_activations = (+ model.lin.quantization_scheme.input_activations+ )+ assert _route_compressed_tensors_fp8_to_unsloth(model) == 0+ model, _ = _ct_model(256, 256, "channel")+ model.lin.weight = torch.nn.Parameter(model.lin.weight.to(torch.bfloat16), requires_grad = False)+ assert _route_compressed_tensors_fp8_to_unsloth(model) == 0+ model, _ = _ct_model(256, 256, "channel")+ model.lin.weight_scale = torch.nn.Parameter(+ torch.ones(3, 1, device = "cuda"), requires_grad = False+ )+ assert _route_compressed_tensors_fp8_to_unsloth(model) == 0+ model, _ = _ct_model(256, 256, "channel")+ monkeypatch.setenv("UNSLOTH_COMPRESSED_TENSORS_FP8_KERNELS", "0")+ assert _route_compressed_tensors_fp8_to_unsloth(model) == 0+++def test_decompress_hook_is_removed():+ from unsloth.models.loader_utils import _route_compressed_tensors_fp8_to_unsloth++ model, _ = _ct_model(256, 256, "channel")+ model.ct_decompress_hook = model.register_forward_pre_hook(lambda module, args: None)+ assert len(model._forward_pre_hooks) == 1+ assert _route_compressed_tensors_fp8_to_unsloth(model) == 1+ assert not hasattr(model, "ct_decompress_hook")+ assert len(model._forward_pre_hooks) == 0+++def test_older_zoo_keeps_the_compressed_tensors_path(monkeypatch):+ from unsloth.models import loader_utils++ monkeypatch.setattr(loader_utils, "_zoo_peft_forward_keeps_fp8_inputs", lambda: False)+ model, _ = _ct_model(256, 256, "channel")+ assert loader_utils._route_compressed_tensors_fp8_to_unsloth(model) == 0+ assert not hasattr(model.lin, "_unsloth_compressed_tensors_fp8")++[email protected]("strategy, block", [("channel", None), ("block", (128, 128))])+def test_single_token_fast_path_adds_the_bias_once(strategy, block, monkeypatch):+ from unsloth.kernels.utils import fast_linear_forward+ from unsloth.models.loader_utils import _route_compressed_tensors_fp8_to_unsloth++ monkeypatch.setenv("UNSLOTH_COMPRESSED_TENSORS_FP8_BLOCK_KERNELS", "1")+ model, ref = _ct_model(256, 256, strategy, bias = True, block = block)+ assert _route_compressed_tensors_fp8_to_unsloth(model) == 1+ lin = model.lin+ with torch.no_grad():+ lin.bias.fill_(1.0)+ X = torch.randn(1, 1, 256, device = "cuda", dtype = torch.bfloat16)+ y = fast_linear_forward(lin, X)+ y_ref = X.float() @ ref.t() + 1.0+ assert float((y.float() - y_ref).norm() / y_ref.norm()) < 0.05+++def test_a_model_mixing_fp8_and_other_schemes_is_not_partly_routed():+ from unsloth.models.loader_utils import _route_compressed_tensors_fp8_to_unsloth++ model, _ = _ct_model(256, 256, "channel")+ other, _ = _ct_model(256, 256, "channel", weight_type = "int")+ model.other = other.lin+ assert _route_compressed_tensors_fp8_to_unsloth(model) == 0+ assert not hasattr(model.lin, "_unsloth_compressed_tensors_fp8")+++ "shape", [(2048, 2048), (256, 2048), (11008, 2048), (2048, 11008), (100, 300), (33, 7)]+)[email protected]("rows", [(1,), (1, 1), (1, 1, 1)])+def test_decode_gemv_matches_the_dequantized_matmul(shape, rows):+ from unsloth.kernels.fp8 import can_use_fp8_rowwise_gemv, fp8_rowwise_gemv++ if _fp8_kernel_unsupported_here():+ pytest.skip("triton cannot read float8_e4m3fn on this GPU")+ model, ref = _ct_model(*shape, "channel")+ W, s = model.lin.weight, model.lin.weight_scale+ X = torch.randn(*rows, shape[1], device = "cuda", dtype = torch.bfloat16)+ with torch.no_grad():+ assert can_use_fp8_rowwise_gemv(X, W, s)+ y = fp8_rowwise_gemv(X, W, s)+ y_again = fp8_rowwise_gemv(X, W, s)+ y_ref = X.float() @ ref.t()+ assert y.shape == (*rows, shape[0]) and y.dtype == X.dtype+ assert float((y.float() - y_ref).norm() / y_ref.norm()) < 0.01+ assert torch.equal(y, y_again)+++def test_decode_gemv_refuses_what_it_cannot_compute():+ from unsloth.kernels.fp8 import can_use_fp8_rowwise_gemv++ model, _ = _ct_model(2048, 16384, "channel")+ W, s = model.lin.weight, model.lin.weight_scale+ X = torch.randn(1, 16384, device = "cuda", dtype = torch.bfloat16)+ with torch.no_grad():+ # 128x128 block grid here has 2048 == N elements.+ assert not can_use_fp8_rowwise_gemv(X, W, torch.ones(16, 128, device = "cuda"))+ assert not can_use_fp8_rowwise_gemv(+ torch.randn(2, 16384, device = "cuda", dtype = torch.bfloat16), W, s+ )+ assert not can_use_fp8_rowwise_gemv(X.float(), W, s)+ assert not can_use_fp8_rowwise_gemv(X[:, :2048], W.t(), s)+ assert not can_use_fp8_rowwise_gemv(X.requires_grad_(True), W, s)+++def _fp8_kernel_unsupported_here():+ from unsloth.kernels.fp8 import _fp8_kernel_unsupported+ return _fp8_kernel_unsupported(torch.empty(1, device = "cuda", dtype = torch.float8_e4m3fn))+++def test_fp8_weights_off_the_gpu_keep_the_compressed_tensors_path():+ from unsloth.models.loader_utils import _route_compressed_tensors_fp8_to_unsloth++ model, _ = _ct_model(256, 256, "channel")+ model.cpu()+ assert _route_compressed_tensors_fp8_to_unsloth(model) == 0+ assert not hasattr(model.lin, "_unsloth_compressed_tensors_fp8")+++def test_compiled_block_fp8_linear_passes_the_input_gradient():+ # torch 2.11 compiled this backward to zeros once Dynamo could trace the whole autograd.Function.+ from unsloth.kernels.fp8 import fp8_linear++ torch._dynamo.reset()+ torch.manual_seed(0)+ W = (torch.randn(256, 256, device = "cuda") * 0.05).to(torch.float8_e4m3fn)+ s = torch.rand(2, 2, device = "cuda") * 0.01 + 0.001+ W.block_size = s.block_size = [128, 128]+ ref = W.float() * s.repeat_interleave(128, 0).repeat_interleave(128, 1)+ X = torch.randn(2, 5, 256, device = "cuda", dtype = torch.bfloat16, requires_grad = True)+ (dX,) = torch.autograd.grad(fp8_linear(X, W, s).float().sum(), X)+ dX_ref = torch.ones(2, 5, 256, device = "cuda") @ ref+ assert float((dX.float() - dX_ref).norm() / dX_ref.norm()) < 0.01++[email protected]("dtype", [torch.float16, torch.bfloat16])+def test_rowwise_dequant_fallback_follows_the_input_dtype(dtype, monkeypatch):+ from unsloth.kernels import fp8++ monkeypatch.setattr(fp8, "_has_fbgemm_rowwise", lambda: False)+ W = (torch.randn(256, 512, device = "cuda") * 0.05).to(torch.float8_e4m3fn)+ s = torch.rand(256, 1, device = "cuda") * 0.01 + 0.001+ X = torch.randn(3, 512, device = "cuda", dtype = dtype, requires_grad = True)+ y = fp8.FbgemmFp8Linear_matmul.apply(X, W, s, None)+ (dX,) = torch.autograd.grad(y.float().sum(), X)+ ref = W.float() * s+ assert y.dtype == dtype and dX.dtype == dtype+ assert float((y.float() - X.float() @ ref.t()).norm() / (X.float() @ ref.t()).norm()) < 0.01+++def test_full_finetuning_skips_the_frozen_fp8_route(monkeypatch):+ from unsloth.models import loader_utils++ model, _ = _ct_model(256, 256, "channel")+ decompressed = []+ monkeypatch.setattr(loader_utils, "_decompress_compressed_tensors_model", decompressed.append)+ loader_utils._prepare_compressed_tensors_model(model, full_finetuning = True)+ assert decompressed == [model]+ assert not getattr(model.lin, "_unsloth_compressed_tensors_fp8", False)+ loader_utils._prepare_compressed_tensors_model(model)+ assert model.lin._unsloth_compressed_tensors_fp8diff --git a/tests/test_compressed_tensors_load_decompress.py b/tests/test_compressed_tensors_load_decompress.pynew file mode 100644index 00000000000..684a11ce717--- /dev/null+++ b/tests/test_compressed_tensors_load_decompress.py@@ -0,0 +1,125 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved.+#+# This program is free software: you can redistribute it and/or modify+# it under the terms of the GNU Affero General Public License as published+# by the Free Software Foundation, either version 3 of the License, or+# (at your option) any later version.+#+# This program is distributed in the hope that it will be useful,+# but WITHOUT ANY WARRANTY; without even the implied warranty of+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+# GNU Affero General Public License for more details.+#+# You should have received a copy of the GNU Affero General Public License+# along with this program. If not, see <https://www.gnu.org/licenses/>.+++"""Unsloth decompresses a compressed-tensors checkpoint once at load, outside inference mode."""++import ast+import os+import types++import pytest+import torch+++def _load_functions(*names):+ path = os.path.join(+ os.path.dirname(os.path.dirname(os.path.abspath(__file__))),+ "unsloth",+ "models",+ "loader_utils.py",+ )+ tree = ast.parse(open(path, encoding = "utf-8").read())+ nodes = [n for n in tree.body if isinstance(n, ast.FunctionDef) and n.name in names]+ namespace = {"torch": torch}+ exec(compile(ast.Module(body = nodes, type_ignores = []), path, "exec"), namespace)+ return namespace+++_decompress_compressed_tensors_model = _load_functions(+ "_decompress_compressed_tensors_model", "_remove_same_device_compressed_tensors_offload"+)["_decompress_compressed_tensors_model"]+++class _Status:+ value = "compressed"+++class _Compressor:+ def __init__(+ self,+ fail_first = False,+ removes_hook = True,+ ):+ self.calls = []+ self.fail_first = fail_first+ self.removes_hook = removes_hook++ def decompress_model(self, model):+ self.calls.append(torch.is_inference_mode_enabled())+ if self.fail_first and len(self.calls) == 1:+ raise RuntimeError("CUDA out of memory")+ lin = model.lin+ lin.weight = torch.nn.Parameter(+ torch.ones_like(lin.weight, dtype = torch.float32), requires_grad = False+ )+ lin.quantization_status = None+ if self.removes_hook and hasattr(model, "ct_decompress_hook"):+ model.ct_decompress_hook.remove()+ delattr(model, "ct_decompress_hook")+++def _model(compressor):+ model = torch.nn.Module()+ model.lin = torch.nn.Linear(4, 4, bias = False)+ model.lin.quantization_status = _Status()+ model.config = types.SimpleNamespace(quantization_config = {"quant_method": "compressed-tensors"})+ model.hf_quantizer = types.SimpleNamespace(compressor = compressor)+ model.forward = lambda x: model.lin(x)+ model.ct_decompress_hook = model.register_forward_pre_hook(+ lambda m, a: compressor.decompress_model(m)+ )+ return model+++def test_decompresses_at_load_and_drops_the_lazy_hook():+ compressor = _Compressor()+ model = _model(compressor)+ assert _decompress_compressed_tensors_model(model)+ assert compressor.calls == [False]+ assert not hasattr(model, "ct_decompress_hook") and not model._forward_pre_hooks+++def test_a_failed_load_decompression_falls_back_outside_inference_mode():+ compressor = _Compressor(fail_first = True)+ model = _model(compressor)+ assert not _decompress_compressed_tensors_model(model)+ with torch.inference_mode():+ model(torch.ones(1, 4))+ assert compressor.calls == [False, False]+ assert not model.lin.weight.is_inference()+ x = torch.ones(1, 4, requires_grad = True)+ torch.nn.functional.linear(x, model.lin.weight).sum().backward()++[email protected]("fail_first", [False, True])+def test_the_lazy_hook_is_dropped_when_compressed_tensors_keeps_it(fail_first):+ compressor = _Compressor(fail_first = fail_first, removes_hook = False)+ model = _model(compressor)+ assert _decompress_compressed_tensors_model(model) is not fail_first+ model(torch.ones(1, 4))+ model(torch.ones(1, 4))+ assert len(compressor.calls) == (2 if fail_first else 1)+ assert not hasattr(model, "ct_decompress_hook") and not model._forward_pre_hooks+++def test_non_compressed_tensors_models_are_left_alone():+ compressor = _Compressor()+ model = _model(compressor)+ model.config.quantization_config = {"quant_method": "fp8"}+ assert not _decompress_compressed_tensors_model(model)+ assert compressor.calls == []+ assert hasattr(model, "ct_decompress_hook")diff --git a/tests/test_fp8_device_context.py b/tests/test_fp8_device_context.pyindex 1f72a23ec7e..7e1569a2191 100644--- a/tests/test_fp8_device_context.py+++ b/tests/test_fp8_device_context.py@@ -20,6 +20,9 @@ class _FakeDeviceModule:def device_count(self) -> int:return self._device_count+ def is_available(self) -> bool:+ return self._device_count > 0+def device(self, device):self.device_calls.append(device)return ("device-context", device)@@ -80,9 +83,14 @@ class _LaunchVisitor(ast.NodeVisitor):def _load_device_context_helper(fake_torch: _FakeTorch):source = FP8_SOURCE.read_text(encoding = "utf-8")tree = ast.parse(source)+ namespace = {"torch": fake_torch, "nullcontext": nullcontext}for node in tree.body:+ if isinstance(node, ast.Assign) and any(+ getattr(t, "id", None) in ("_CUDA_MULTI_DEVICE", "_XPU_MULTI_DEVICE")+ for t in node.targets+ ):+ exec(ast.get_source_segment(source, node), namespace)if isinstance(node, ast.FunctionDef) and node.name == "_fp8_triton_device_context":- namespace = {"torch": fake_torch, "nullcontext": nullcontext}exec(ast.get_source_segment(source, node), namespace)return namespace["_fp8_triton_device_context"]raise AssertionError("_fp8_triton_device_context was not found")diff --git a/unsloth/_gpu_init.py b/unsloth/_gpu_init.pyindex 9d429386136..e2c344c43db 100644--- a/unsloth/_gpu_init.py+++ b/unsloth/_gpu_init.py@@ -268,6 +268,7 @@ from .import_fixes import (fix_vllm_aimv2_issue,fix_vllm_lora_tokenizer_module,fix_torchao_nf4tensor_move,+ fix_compressed_tensors_activation_quant_gradient,fix_torchao_safe_int_mm_repr_probe,check_vllm_torch_sm100_compatibility,fix_vllm_guided_decoding_params,@@ -344,6 +345,7 @@ fix_vllm_lora_tokenizer_module()# torchao 0.18.0 moved nf4tensor; torchtune (via xcodec2) still imports the old path. Lazy alias, so# it costs nothing unless asked for.fix_torchao_nf4tensor_move()+fix_compressed_tensors_activation_quant_gradient()fix_torchao_safe_int_mm_repr_probe()# Check vLLM + torch < 2.9.0 + SM100 compatibility BEFORE importing vLLMcheck_vllm_torch_sm100_compatibility()@@ -397,6 +399,7 @@ del fix_flash_attn_4_namespace_shadowdel fix_vllm_aimv2_issuedel fix_vllm_lora_tokenizer_moduledel fix_torchao_nf4tensor_move+del fix_compressed_tensors_activation_quant_gradientdel fix_torchao_safe_int_mm_repr_probedel check_vllm_torch_sm100_compatibilitydel fix_vllm_guided_decoding_paramsdiff --git a/unsloth/import_fixes.py b/unsloth/import_fixes.pyindex f5cb22135ab..9b5adcadc15 100644--- a/unsloth/import_fixes.py+++ b/unsloth/import_fixes.py@@ -10561,6 +10561,137 @@ def disable_sentencepiece_on_windows():return True+# compressed-tensors fake-quantizes W8A8 activations under no_grad; STE so LoRA gets input gradients.+_CT_FORWARD_MODULE = "compressed_tensors.quantization.lifecycle.forward"+_CT_BY_NAME_MODULES = (+ "compressed_tensors.modeling.kvcache",+ "compressed_tensors.modeling.attention",+)+_CT_STE_SENTINEL = "_unsloth_activation_ste"+_CT_FINDER_SENTINEL = "__unsloth_compressed_tensors_ste_finder__"+++def _compressed_tensors_ste_forward_quantize(original):+ import torch++ class _StraightThrough(torch.autograd.Function):+ # Not `value + (out - value).detach()`: that rounds where a static scale saturates.+ @staticmethod+ def forward(ctx, value, quantized):+ ctx.value_dtype = value.dtype+ return quantized.to(value.dtype).view_as(quantized)++ @staticmethod+ def backward(ctx, grad_output):+ return grad_output.to(ctx.value_dtype), None++ @functools.wraps(original)+ def forward_quantize(*args, **kwargs):+ out = original(*args, **kwargs)+ if not torch.is_grad_enabled():+ return out+ value = args[1] if len(args) > 1 else kwargs.get("value")+ base_name = args[2] if len(args) > 2 else kwargs.get("base_name")+ if (+ base_name != "weight"+ and isinstance(value, torch.Tensor)+ and isinstance(out, torch.Tensor)+ and value.requires_grad+ and not out.requires_grad+ and out.shape == value.shape+ ):+ return _StraightThrough.apply(value, out)+ return out++ setattr(forward_quantize, _CT_STE_SENTINEL, True)+ return forward_quantize+++def _patch_compressed_tensors_forward_module(module):+ original = getattr(module, "forward_quantize", None)+ if not callable(original):+ return False+ if getattr(original, _CT_STE_SENTINEL, False):+ return True+ patched = _compressed_tensors_ste_forward_quantize(original)+ module.forward_quantize = patched+ # Modules that imported the function by name before this ran hold the original.+ for name in _CT_BY_NAME_MODULES:+ other = sys.modules.get(name)+ if other is not None and getattr(other, "forward_quantize", None) is original:+ other.forward_quantize = patched+ return True+++class _CompressedTensorsSTELoader(importlib.abc.Loader):+ __slots__ = ("_loader",)++ def __init__(self, loader):+ self._loader = loader++ def create_module(self, spec):+ create_module = getattr(self._loader, "create_module", None)+ if create_module is None:+ return None+ return create_module(spec)++ def exec_module(self, module):+ self._loader.exec_module(module)+ try:+ _patch_compressed_tensors_forward_module(module)+ except Exception as e:+ logger.info(f"Unsloth: compressed-tensors activation gradient patch skipped: {e}")++ def __getattr__(self, name):+ return getattr(self._loader, name)+++class _CompressedTensorsSTEFinder(importlib.abc.MetaPathFinder):+ __slots__ = (_CT_FINDER_SENTINEL,)++ def __init__(self):+ setattr(self, _CT_FINDER_SENTINEL, True)++ def find_spec(+ self,+ fullname,+ path = None,+ target = None,+ ):+ if fullname != _CT_FORWARD_MODULE:+ return None+ spec = None+ for finder in sys.meta_path:+ if finder is self or getattr(finder, _CT_FINDER_SENTINEL, False):+ continue+ finder_find_spec = getattr(finder, "find_spec", None)+ if finder_find_spec is None:+ continue+ try:+ spec = finder_find_spec(fullname, path, target)+ except Exception:+ spec = None+ if spec is not None:+ break+ if spec is None or spec.loader is None or not hasattr(spec.loader, "exec_module"):+ return None+ spec.loader = _CompressedTensorsSTELoader(spec.loader)+ return spec+++def fix_compressed_tensors_activation_quant_gradient():+ if importlib.util.find_spec("compressed_tensors") is None:+ return+ module = sys.modules.get(_CT_FORWARD_MODULE)+ if module is not None:+ _patch_compressed_tensors_forward_module(module)+ return+ for finder in sys.meta_path:+ if getattr(finder, _CT_FINDER_SENTINEL, False):+ return+ sys.meta_path.insert(0, _CompressedTensorsSTEFinder())++def fix_transformers_longcat_lsa_config():"""Answer AutoConfig's "Unrecognized model" on a LongcatCausalLM config (no model_type,auto_map or modeling code) with ``models/longcat_lsa.py``; all other loads are untouched."""diff --git a/unsloth/kernels/fp8.py b/unsloth/kernels/fp8.pyindex db90408ca28..92973106d4a 100644--- a/unsloth/kernels/fp8.py+++ b/unsloth/kernels/fp8.py@@ -24,10 +24,24 @@ from unsloth_zoo.temporary_patches.common import torch_compiletorch_matmul = torch.matmul-def _fp8_triton_device_context(tensor: torch.Tensor):- if tensor.device.type == "cuda" and torch.cuda.device_count() > 1:+# Read once for the decode GEMV: device_count() in a compiled region graph-breaks ("torch.* op returned non-Tensor").+_CUDA_MULTI_DEVICE = torch.cuda.is_available() and torch.cuda.device_count() > 1+_XPU_MULTI_DEVICE = (+ hasattr(torch, "xpu") and torch.xpu.is_available() and torch.xpu.device_count() > 1+)+++def _fp8_triton_device_context(tensor: torch.Tensor, static_device_count = False):+ # static_device_count only for forward-only kernels: torch 2.11 compiles FP8BlockQuantLinear's backward to zeros.+ if tensor.device.type == "cuda" and (+ _CUDA_MULTI_DEVICE if static_device_count else torch.cuda.device_count() > 1+ ):return torch.cuda.device(tensor.device)- if tensor.device.type == "xpu" and hasattr(torch, "xpu") and torch.xpu.device_count() > 1:+ if tensor.device.type == "xpu" and (+ _XPU_MULTI_DEVICE+ if static_device_count+ else (hasattr(torch, "xpu") and torch.xpu.device_count() > 1)+ ):return torch.xpu.device(tensor.device)return nullcontext()@@ -112,11 +126,29 @@ def weight_dequant_block(return y+def _is_transposed_view(x):+ return (+ x.dim() == 2 and x.shape[0] > 1 and x.shape[1] > 1 and x.stride(0) == 1 and x.stride(1) != 1+ )+++def _has_fbgemm_rowwise():+ try:+ return hasattr(torch.ops.fbgemm, "quantize_fp8_per_row") and hasattr(+ torch.ops.fbgemm, "f8f8bf16_rowwise"+ )+ except Exception:+ return False++def weight_dequant(x: torch.Tensor,s: torch.Tensor,dtype = torch.bfloat16,):+ # Transposed views keep scales in storage order; square weights can't be told apart by shape.+ if _is_transposed_view(x):+ return weight_dequant(x.t(), s, dtype).t()# Per-tensor scale: single value for entire weight matrixif s.numel() == 1:return x.to(dtype) * s.view(1, 1).to(dtype)@@ -134,7 +166,9 @@ def weight_dequant(else:# Block quantized weight: scale shape is (ceil(m/block_m), ceil(n/block_n)). Go through the# any-shape helper so fast_dequantize's callers get the pre-sm89 fallback too.- return _blockwise_weight_dequant_any_shape(x, s, [128, 128], dtype)+ return _blockwise_weight_dequant_any_shape(+ x, s, getattr(s, "block_size", None) or [128, 128], dtype+ )# Copied from huggingface.co/deepseek-ai/DeepSeek-V3 inference/kernel.py@@ -366,6 +400,59 @@ def _fp8_kernel_unsupported(tensor, kernel_dtype = None):)+def _fp8_rowwise_gemv_kernel(+ x_ptr, w_ptr, s_ptr, y_ptr, N, K, stride_wn, BLOCK_N: tl.constexpr, BLOCK_K: tl.constexpr+):+ offs_n = tl.program_id(0) * BLOCK_N + tl.arange(0, BLOCK_N)+ acc = tl.zeros((BLOCK_N, BLOCK_K), dtype = tl.float32)+ for k0 in range(0, K, BLOCK_K):+ offs_k = k0 + tl.arange(0, BLOCK_K)+ x = tl.load(x_ptr + offs_k, mask = offs_k < K, other = 0.0).to(tl.float32)+ w = tl.load(+ w_ptr + offs_n[:, None] * stride_wn + offs_k[None, :],+ mask = (offs_n[:, None] < N) & (offs_k[None, :] < K),+ other = 0.0,+ ).to(tl.float32)+ acc += w * x[None, :]+ s = tl.load(s_ptr + offs_n, mask = offs_n < N, other = 0.0).to(tl.float32)+ y = tl.sum(acc, axis = 1) * s+ tl.store(y_ptr + offs_n, y.to(y_ptr.dtype.element_ty), mask = offs_n < N)+++def can_use_fp8_rowwise_gemv(X, weight, weight_scale):+ if not (X.is_cuda and weight.is_cuda and X.dtype in (torch.bfloat16, torch.float16)):+ return False+ if weight.dtype != torch.float8_e4m3fn or weight.dim() != 2 or not weight.is_contiguous():+ return False+ if X.numel() != X.shape[-1] or X.shape[-1] != weight.shape[1]:+ return False+ # Exactly one scale per output row: a block grid can have N elements by coincidence.+ if (+ tuple(weight_scale.shape) not in ((weight.shape[0], 1), (weight.shape[0],))+ or weight.shape[0] == 1+ ):+ return False+ if torch.is_grad_enabled() and (X.requires_grad or weight.requires_grad):+ return False+ return not _fp8_kernel_unsupported(weight)+++def fp8_rowwise_gemv(X, weight, weight_scale):+ """X @ (weight * weight_scale).T for one token without a 16-bit weight copy."""+ N, K = weight.shape+ out = torch.empty((*X.shape[:-1], N), device = X.device, dtype = X.dtype)+ if not X.is_contiguous():+ X = X.contiguous()+ scale = weight_scale if weight_scale.is_contiguous() else weight_scale.contiguous()+ block_n = 1 if N <= 4096 else (4 if N <= 32768 else 8)+ with _fp8_triton_device_context(X, static_device_count = True):+ _fp8_rowwise_gemv_kernel[(triton.cdiv(N, block_n),)](+ X, weight, scale, out, N, K, weight.stride(0), BLOCK_N = block_n, BLOCK_K = 512, num_warps = 2+ )+ return out++# Expanding the scale over the whole weight needs two m*n float32 temporaries, ~6x the triton# kernel's peak; chunk by block-rows so the scratch stays bounded. Values are identical._DEQUANT_CHUNK_ELEMS = 8 * 1024 * 1024@@ -509,8 +596,12 @@ class FbgemmFp8Linear_matmul(torch.autograd.Function):weight_scale,bias = None,):- if weight.shape[0] == weight_scale.shape[0] and (- weight.shape[0] % 8 == 0 and weight.shape[1] % 8 == 0+ if (+ weight.shape[0] == weight_scale.shape[0]+ and (weight.shape[0] % 8 == 0 and weight.shape[1] % 8 == 0)+ and not _is_transposed_view(weight)+ and _has_fbgemm_rowwise()+ and not _fp8_kernel_unsupported(weight, torch.float8_e4m3fn)):# The kernel needs weight dims divisible by 8 (else "cutlass cannot implement"), and padding plus# f8f8bf16 is slower than dequant plus bf16 matmul.@@ -538,12 +629,9 @@ class FbgemmFp8Linear_matmul(torch.autograd.Function):output = output.to(x.device, x.dtype)output = output.reshape(output_shape)del x_quantized, x_scale- elif (- weight.shape[0] != weight_scale.shape[0] and weight.shape[1] == weight_scale.shape[0]- ) or (weight.shape[0] % 8 != 0 or weight.shape[1] % 8 != 0):- # Transposed weight/scale (backward dY@W) or a non-divisible-by-8 shape (Qwen 2.5 VL 7B gate proj- # 3420x1280): dequant is preferred.- W_deq = weight_dequant(weight, weight_scale).T+ elif weight_scale.shape[0] in (weight.shape[0], weight.shape[1]):+ # Transposed, non-divisible-by-8 (Qwen 2.5 VL 7B 3420x1280), no FBGEMM, or pre-sm89: dequant.+ W_deq = weight_dequant(weight, weight_scale, x.dtype).Toutput = torch_matmul(x, W_deq)output = output + bias if bias is not None else outputdel W_deq@@ -558,7 +646,7 @@ class FbgemmFp8Linear_matmul(torch.autograd.Function):@staticmethoddef backward(ctx, grad_output):- W_deq = weight_dequant(ctx.weight, ctx.weight_scale)+ W_deq = weight_dequant(ctx.weight, ctx.weight_scale, grad_output.dtype)grad_X = torch_matmul(grad_output, W_deq)del W_deqreturn grad_X, None, None, None, Nonediff --git a/unsloth/kernels/utils.py b/unsloth/kernels/utils.pyindex 12eeb3943e0..694ed690226 100644--- a/unsloth/kernels/utils.py+++ b/unsloth/kernels/utils.py@@ -27,7 +27,7 @@ from ..device_type import (ALLOW_PREQUANTIZED_MODELS,)from ..bnb_availability import native_kernels_ready-from .fp8 import weight_dequant, fp8_linear+from .fp8 import weight_dequant, fp8_linear, can_use_fp8_rowwise_gemv, fp8_rowwise_gemvimport functools# torch.cuda.amp.custom_fwd is deprecated from 2.4.@@ -1051,7 +1051,11 @@ def fast_linear_forward(if W_quant is None:out = torch_matmul(X, W.t(), out = out)elif W.dtype == torch.float8_e4m3fn:- out = fp8_linear(X, W, W_quant, bias)+ # The bias is added once below; the per-channel fp8_linear path would add it a second time.+ if can_use_fp8_rowwise_gemv(X, W, W_quant):+ out = fp8_rowwise_gemv(X, W, W_quant)+ else:+ out = fp8_linear(X, W, W_quant)elif bsz == 1 and q_len == 1:out = fast_gemv(X, W, W_quant, out = out)else:diff --git a/unsloth/models/llama.py b/unsloth/models/llama.pyindex 021ee7a3aba..86d956d9271 100644--- a/unsloth/models/llama.py+++ b/unsloth/models/llama.py@@ -35,6 +35,7 @@ from .loader_utils import (_exclude_rope_inv_freq_from_ddp,_get_fp8_mode_and_check_settings,_restore_dropped_fp8_scales,+ _prepare_compressed_tensors_model,planner_class_mismatch_reason,planner_model_class,planner_config_overrides,@@ -2817,6 +2818,7 @@ class FastLlamaModel:variant = kwargs.get("variant"),dtype = dtype,)+ _prepare_compressed_tensors_model(model)elif not fast_inference:if user_config is not None or _modelopt_rewritten:# Transformers 5.x @strict model init rejects extra kwargs next to config=, so set the override@@ -2879,6 +2881,7 @@ class FastLlamaModel:variant = kwargs.get("variant"),dtype = dtype,)+ _prepare_compressed_tensors_model(model)model.fast_generate = make_fast_generate_wrapper(model.generate)model.fast_generate_batches = Noneelse:diff --git a/unsloth/models/loader_utils.py b/unsloth/models/loader_utils.pyindex 59212e7fb49..d3333080a53 100644--- a/unsloth/models/loader_utils.py+++ b/unsloth/models/loader_utils.py@@ -1315,6 +1315,163 @@ def _match_fp8_module(module_by_name, base):return None+_CT_FP8_STRATEGIES = ("tensor", "channel", "block")+++def _compressed_tensors_fp8_block_size(module, weights):+ """(out, in) block of a routable compressed-tensors FP8 weight, else None."""+ if getattr(weights, "type", None) != "float" or getattr(weights, "num_bits", None) != 8:+ return None+ strategy = getattr(weights, "strategy", None)+ strategy = str(getattr(strategy, "value", strategy))+ # Block FP8 is opt-in: fp8_block_quant_linear is far slower than the decompressed bf16 model.+ if (+ strategy == "block"+ and os.environ.get("UNSLOTH_COMPRESSED_TENSORS_FP8_BLOCK_KERNELS", "0") != "1"+ ):+ return None+ if (+ strategy not in _CT_FP8_STRATEGIES+ or getattr(weights, "dynamic", False)+ or getattr(weights, "actorder", None)+ ):+ return None+ weight = getattr(module, "weight", None)+ scale = getattr(module, "weight_scale", None)+ if weight is None or scale is None or weight.dtype != torch.float8_e4m3fn or weight.dim() != 2:+ return None+ zero_point = getattr(module, "weight_zero_point", None)+ if zero_point is not None and bool(torch.any(zero_point != 0)):+ return None+ out_features, in_features = weight.shape+ if strategy == "tensor":+ # Routed as per-channel (scale broadcast per row): the rowwise paths beat the block kernel.+ return [1, in_features] if scale.numel() == 1 else None+ if strategy == "channel":+ return (+ [1, in_features] if tuple(scale.shape) in ((out_features, 1), (out_features,)) else None+ )+ block = list(getattr(weights, "block_structure", None) or ())+ if len(block) != 2 or scale.dim() != 2:+ return None+ if tuple(scale.shape) != (-(-out_features // block[0]), -(-in_features // block[1])):+ return None+ return block+++def _save_compressed_tensors_scale_shape(module, state_dict, prefix, local_metadata):+ # Saved checkpoints keep the config's scale shape; a per-tensor scale was broadcast per row.+ key = prefix + "weight_scale"+ shape = getattr(module, "_unsloth_ct_scale_shape", None)+ scale = state_dict.get(key)+ if shape is None or scale is None or tuple(scale.shape) == shape:+ return state_dict+ numel = 1+ for n in shape:+ numel *= n+ state_dict[key] = scale.reshape(-1)[:1].reshape(shape) if numel == 1 else scale.reshape(shape)+ return state_dict+++def _load_compressed_tensors_scale_shape(module, state_dict, prefix, *args):+ key = prefix + "weight_scale"+ scale = state_dict.get(key)+ current = getattr(module, "weight_scale", None)+ if scale is None or current is None or scale.shape == current.shape:+ return+ if scale.numel() == 1:+ state_dict[key] = scale.reshape(1, 1).expand(current.shape).contiguous()+ elif scale.numel() == current.numel():+ state_dict[key] = scale.reshape(current.shape)+++def _unsloth_compressed_tensors_fp8_forward(self, input):+ from unsloth.kernels.fp8 import can_use_fp8_rowwise_gemv, fp8_linear, fp8_rowwise_gemv++ if can_use_fp8_rowwise_gemv(input, self.weight, self.weight_scale):+ out = fp8_rowwise_gemv(input, self.weight, self.weight_scale)+ else:+ out = fp8_linear(input, self.weight, self.weight_scale)+ if self.bias is not None:+ out = out + self.bias.to(out.dtype)+ return out++[email protected]_cache(maxsize = 1)+def _zoo_peft_forward_keeps_fp8_inputs():+ """Older zoo compiled LoRA forwards cast `x` to the FP8 weight dtype; route only when it does not."""+ try:+ import inspect+ from unsloth_zoo import compiler+ source = inspect.getsource(compiler.patch_lora_forwards)+ except Exception:+ return False+ return "x = x.to(self.base_layer.weight.dtype)" not in source or (+ "self.base_layer.weight.dtype in (torch.float16, torch.bfloat16, torch.float32)" in source+ )+++def _route_compressed_tensors_fp8_to_unsloth(model):+ # Opt-in: saves memory, but on-the-fly dequant trains slower than the decompressed bf16 model.+ if os.environ.get("UNSLOTH_COMPRESSED_TENSORS_FP8_KERNELS", "0") != "1":+ return 0+ if getattr(getattr(model, "config", None), "quantization_config", None) is None:+ return 0+ if not _zoo_peft_forward_keeps_fp8_inputs():+ return 0+ routable = []+ for module in model.modules():+ scheme = getattr(module, "quantization_scheme", None)+ if scheme is None or getattr(scheme, "weights", None) is None:+ continue+ block = None+ weight = getattr(module, "weight", None)+ on_gpu = isinstance(weight, torch.Tensor) and weight.device.type in ("cuda", "xpu")+ if (+ on_gpu+ and isinstance(module, torch.nn.Linear)+ and getattr(scheme, "output_activations", None) is None+ ):+ block = _compressed_tensors_fp8_block_size(module, scheme.weights)+ if block is None:+ return 0+ routable.append((module, block))+ converted = 0+ for module, block in routable:+ scale = module.weight_scale+ scale_shape = tuple(scale.shape)+ if scale.numel() == 1:+ scale.data = scale.data.reshape(1, 1).expand(module.weight.shape[0], 1).contiguous()+ elif scale.dim() == 1:+ scale.data = scale.data.view(-1, 1)+ if tuple(scale.shape) != scale_shape:+ module._unsloth_ct_scale_shape = scale_shape+ module._register_state_dict_hook(_save_compressed_tensors_scale_shape)+ module._register_load_state_dict_pre_hook(+ _load_compressed_tensors_scale_shape, with_module = True+ )+ module.weight.requires_grad_(False)+ scale.requires_grad_(False)+ if block != [1, module.weight.shape[1]]:+ module.weight.block_size = block+ scale.block_size = block+ module.block_size = block+ module.forward = _unsloth_compressed_tensors_fp8_forward.__get__(module)+ module._unsloth_compressed_tensors_fp8 = True+ converted += 1+ if converted:+ for owner in (model, getattr(model, "model", None), getattr(model, "base_model", None)):+ hook = getattr(owner, "ct_decompress_hook", None)+ if hook is not None:+ hook.remove()+ try:+ delattr(owner, "ct_decompress_hook")+ except AttributeError:+ pass+ model._unsloth_compressed_tensors_fp8 = converted+ return converted++_FP8_DEQUANT_DTYPES = (torch.bfloat16, torch.float16, torch.float32)@@ -1556,6 +1713,79 @@ def _restore_dropped_fp8_scales(return (0, 0)+def _remove_same_device_compressed_tensors_offload(model):+ """Unwrap compressed-tensors >= 0.19 non-offloading OffloadCaches, which torch.compile cannot trace."""+ try:+ from compressed_tensors.offload.cache import OffloadCache+ from compressed_tensors.offload.module import remove_module_offload+ except Exception:+ return 0+ removed = 0+ for module in model.modules():+ cache = module._parameters+ if not isinstance(cache, OffloadCache):+ continue+ onload = getattr(cache, "onload_device", None)+ offload = getattr(cache, "offload_device", None)+ if onload is None or offload is None or torch.device(onload) != torch.device(offload):+ continue+ remove_module_offload(module)+ removed += 1+ return removed+++def _decompress_compressed_tensors_model(model):+ """Decompress at load: PEFT skips the first-forward hook, and in `generate` it yields inference tensors."""+ if getattr(model, "_unsloth_compressed_tensors_fp8", 0):+ return False+ quant_config = getattr(getattr(model, "config", None), "quantization_config", None)+ if isinstance(quant_config, dict):+ method = quant_config.get("quant_method", None)+ else:+ method = getattr(quant_config, "quant_method", None)+ if getattr(method, "value", method) != "compressed-tensors":+ return False+ if not any(+ str(getattr(getattr(module, "quantization_status", None), "value", "")) == "compressed"+ for module in model.modules()+ ):+ return False+ compressor = getattr(getattr(model, "hf_quantizer", None), "compressor", None)+ if compressor is None or not hasattr(compressor, "decompress_model"):+ return False++ def _decompress(module, *args):+ with torch.inference_mode(False), torch.no_grad():+ compressor.decompress_model(module)+ _remove_same_device_compressed_tensors_offload(module)+ # Some compressed-tensors releases (0.19.0) leave the hook in place: drop it, also after a retry.+ hook = getattr(model, "ct_decompress_hook", None)+ if hook is not None:+ hook.remove()+ try:+ delattr(model, "ct_decompress_hook")+ except AttributeError:+ pass++ # Swap the hook first: decompress_model removes it by name.+ hook = getattr(model, "ct_decompress_hook", None)+ if hook is not None:+ hook.remove()+ model.ct_decompress_hook = model.register_forward_pre_hook(_decompress)+ try:+ _decompress(model)+ except Exception as e:+ print(f"Unsloth: could not decompress the compressed-tensors checkpoint after load: {e}")+ return False+ return True+++def _prepare_compressed_tensors_model(model, full_finetuning = False):+ # Routed FP8 weights are frozen, so full finetuning always takes the decompressed bf16 weights.+ if full_finetuning or not _route_compressed_tensors_fp8_to_unsloth(model):+ _decompress_compressed_tensors_model(model)++_FP8_SCALE_SUFFIXES = (".weight_scale_inv", "_scale_inv", ".weight_scale", "_scale")_FP8_LEFTOVER_MAX_CHUNK = 1 << 26diff --git a/unsloth/models/vision.py b/unsloth/models/vision.pyindex d778ca9aa0a..299104d410a 100644--- a/unsloth/models/vision.py+++ b/unsloth/models/vision.py@@ -106,6 +106,7 @@ from .loader_utils import (_get_fp8_mode_and_check_settings,_dequantize_leftover_fp8_params,_restore_dropped_fp8_scales,+ _prepare_compressed_tensors_model,planner_class_mismatch_reason,planner_model_class,planner_quantization_kwargs,@@ -2418,6 +2419,7 @@ class FastBaseModel:variant = kwargs.get("variant"),dtype = torch_dtype,)+ _prepare_compressed_tensors_model(model, full_finetuning = full_finetuning)if load_in_16bit and not load_in_4bit and not load_in_8bit:_dequantize_leftover_fp8_params(model,