Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Load compressed-tensors packed INT4 checkpoints for QLoRA without a 16-bit copy (exact packed INT4 by default, NF4 fallback), and train remote DeepSeek-V3 MoE code (Kimi-K2.7-Code) (#11537)

Daniel Han6ee4c0c02284
diff --git a/tests/_shared/llama_patch_isolation.py b/tests/_shared/llama_patch_isolation.py
new file mode 100644
index 00000000000..15f71184701
--- /dev/null
+++ b/tests/_shared/llama_patch_isolation.py
@@ -0,0 +1,35 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""FastLlamaModel.pre_patch rewrites the stock Llama classes process-wide; restore them after an in-process load."""
+
+import pytest
+
+
+def restore_llama_patches():
+ import peft
+ import transformers.models.llama.modeling_llama as modeling_llama
+
+ module_globals = dict(vars(modeling_llama))
+ classes = {
+ cls: dict(vars(cls))
+ for cls in module_globals.values()
+ if isinstance(cls, type) and cls.__module__ == modeling_llama.__name__
+ }
+ peft_forward = peft.PeftModelForCausalLM.__dict__["forward"]
+ try:
+ yield
+ finally:
+ for name in set(vars(modeling_llama)) - set(module_globals):
+ delattr(modeling_llama, name)
+ for name, value in module_globals.items():
+ if vars(modeling_llama).get(name) is not value:
+ setattr(modeling_llama, name, value)
+ for cls, attrs in classes.items():
+ for name in set(vars(cls)) - set(attrs):
+ delattr(cls, name)
+ for name, value in attrs.items():
+ if vars(cls).get(name) is not value:
+ setattr(cls, name, value)
+ peft.PeftModelForCausalLM.forward = peft_forward
diff --git a/tests/test_compressed_tensors_bnb.py b/tests/test_compressed_tensors_bnb.py
new file mode 100644
index 00000000000..014ce558f0c
--- /dev/null
+++ b/tests/test_compressed_tensors_bnb.py
@@ -0,0 +1,1290 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved.
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+"""Loading a compressed-tensors packed INT4 / INT8 checkpoint straight into bitsandbytes 4-bit.
+
+Offline tests cover the checkpoint classifier, the storage-dtype plan for the packed tensors,
+the config stripping (root and sub-configs) and the loader hand-off. The GPU tests build a
+tiny packed Llama with compressed-tensors' own compressor, load it through
+`FastLanguageModel.from_pretrained(load_in_4bit = True)` and compare every Linear4bit bit for
+bit against the same checkpoint decompressed to bf16 on disk first.
+"""
+
+import json
+import os
+import shutil
+import tempfile
+from types import SimpleNamespace
+
+import pytest
+from llama_patch_isolation import restore_llama_patches # noqa: F401
+from real_accelerator import has_real_cuda # tests/_shared, on sys.path via tests/conftest.py
+import torch
+
+# Import unsloth first to set UNSLOTH_IS_PRESENT env var.
+import unsloth # noqa: F401
+from unsloth.models.compressed_tensors_bnb import (
+ UNSLOTH_COMPRESSED_TENSORS_ATTR,
+ _config_and_subconfigs,
+ _generalize,
+ arm_compressed_tensors_bnb_loading,
+ compressed_tensors_bnb_plan,
+ install_compressed_tensors_bnb_quantizer,
+ packed_weight_dtype_plan,
+)
+from unsloth.models.loader_utils import check_and_disable_bitsandbytes_loading
+
+try:
+ import inspect
+ from compressed_tensors.quantization.lifecycle.forward import dequantize as _ct_dequantize
+
+ HAS_CT = True
+ # compressed-tensors 0.19 dropped GPTQ activation ordering (#840): its compressor ignores weight_g_idx.
+ CT_HONOURS_G_IDX = "g_idx" in inspect.signature(_ct_dequantize).parameters
+except Exception:
+ HAS_CT = CT_HONOURS_G_IDX = False
+
+# Needs `update_weight_conversions` (transformers 5.8+); inert on 5.0-5.7.
+from unsloth.models.compressed_tensors_bnb import _transformers_supports_weight_converters
+
+HAS_CONVERTERS = _transformers_supports_weight_converters()
+
+
+def _w4a16(**overrides):
+ weights = {
+ "num_bits": 4,
+ "type": "int",
+ "symmetric": True,
+ "strategy": "group",
+ "group_size": 32,
+ "block_structure": None,
+ "dynamic": False,
+ "actorder": None,
+ "observer": "minmax",
+ "observer_kwargs": {},
+ }
+ weights.update(overrides.pop("weights", {}))
+ quant = {
+ "quant_method": "compressed-tensors",
+ "format": "pack-quantized",
+ "quantization_status": "compressed",
+ "ignore": ["lm_head"],
+ "config_groups": {
+ "group_0": {
+ "targets": ["Linear"],
+ "weights": weights,
+ "input_activations": None,
+ "output_activations": None,
+ }
+ },
+ }
+ quant.update(overrides)
+ return quant
+
+
+class _Config(SimpleNamespace):
+ """Enough of PretrainedConfig for the classifier: attribute access plus to_dict."""
+
+ def to_dict(self):
+ return dict(self.__dict__)
+
+
+def test_plan_accepts_w4a16_pack_quantized():
+ assert compressed_tensors_bnb_plan(_Config(quantization_config = _w4a16())) is not None
+
+
+def test_plan_accepts_int8_and_asymmetric_and_actorder():
+ for weights in (
+ {"num_bits": 8},
+ {"symmetric": False},
+ {"actorder": "group"},
+ {"strategy": "channel", "group_size": None},
+ ):
+ assert (
+ compressed_tensors_bnb_plan(_Config(quantization_config = _w4a16(weights = weights)))
+ is not None
+ ), weights
+
+
+def test_plan_accepts_legacy_nested_sparseml_spelling():
+ inner = _w4a16()
+ inner.pop("quant_method")
+ inner["quant_method"] = "sparseml"
+ inner["quantization_status"] = "frozen"
+ outer = {
+ "quantization_config": inner,
+ "sparsity_config": {"format": "dense"},
+ "quant_method": "compressed-tensors",
+ }
+ plan = compressed_tensors_bnb_plan(_Config(quantization_config = outer))
+ assert plan is not None and "config_groups" in plan
+
+
+def test_plan_declines_other_formats_and_schemes():
+ cases = {
+ "no config": _Config(),
+ "bitsandbytes": _Config(
+ quantization_config = {"quant_method": "bitsandbytes", "load_in_4bit": True}
+ ),
+ "fp8": _Config(
+ quantization_config = {"quant_method": "fp8", "weight_block_size": [128, 128]}
+ ),
+ "float weights": _Config(
+ quantization_config = _w4a16(weights = {"type": "float", "num_bits": 8})
+ ),
+ "nvfp4 format": _Config(quantization_config = _w4a16(format = "nvfp4-pack-quantized")),
+ "activations quantized": _Config(quantization_config = _w4a16()),
+ "sparse": _Config(
+ quantization_config = _w4a16(sparsity_config = {"format": "sparse-24-bitmask"})
+ ),
+ "gptq": _Config(quantization_config = {"quant_method": "gptq", "bits": 4}),
+ "no groups": _Config(quantization_config = _w4a16(config_groups = {})),
+ }
+ cases["activations quantized"].quantization_config["config_groups"]["group_0"][
+ "input_activations"
+ ] = {"num_bits": 8, "type": "int"}
+ for name, config in cases.items():
+ assert compressed_tensors_bnb_plan(config) is None, name
+
+
+def test_plan_declines_mixed_groups_with_a_float_scheme():
+ quant = _w4a16()
+ quant["config_groups"]["group_1"] = {
+ "targets": ["re:.*self_attn.*"],
+ "weights": {"num_bits": 8, "type": "float", "strategy": "tensor", "symmetric": True},
+ "input_activations": None,
+ "output_activations": None,
+ }
+ assert compressed_tensors_bnb_plan(_Config(quantization_config = quant)) is None
+
+
+def test_generalize_widens_numeric_path_components_only():
+ rx = _generalize("model.layers.0.mlp.experts.12.gate_proj")
+ assert rx == r"model\.layers\.\d+\.mlp\.experts\.\d+\.gate_proj"
+ assert _generalize("model.embed_tokens") == r"model\.embed_tokens"
+ assert _generalize("layer0.block") == r"layer0\.block"
+
+
+def test_packed_dtype_plan_uses_families_and_exact_names_on_collision():
+ import re
+
+ keys = [
+ "model.layers.0.mlp.experts.12.gate_proj.weight_packed",
+ "model.layers.0.mlp.experts.12.gate_proj.weight_scale",
+ "model.layers.1.mlp.experts.3.gate_proj.weight_packed",
+ "model.layers.0.self_attn.q_proj.weight_packed",
+ "model.layers.1.self_attn.q_proj.weight", # this layer's q_proj is stored unpacked
+ "model.embed_tokens.weight",
+ "lm_head.weight",
+ ]
+ plan = packed_weight_dtype_plan(keys)
+ assert all(v is None for v in plan.values())
+ rx = re.compile("|".join(plan))
+ kept = [
+ k
+ for k in [
+ "model.layers.0.mlp.experts.12.gate_proj.weight",
+ "model.layers.7.mlp.experts.0.gate_proj.weight",
+ "model.layers.0.self_attn.q_proj.weight",
+ "model.layers.1.self_attn.q_proj.weight",
+ "model.embed_tokens.weight",
+ "lm_head.weight",
+ ]
+ if rx.search(k)
+ ]
+ assert kept == [
+ "model.layers.0.mlp.experts.12.gate_proj.weight",
+ "model.layers.7.mlp.experts.0.gate_proj.weight",
+ "model.layers.0.self_attn.q_proj.weight",
+ ]
+ assert packed_weight_dtype_plan(["model.embed_tokens.weight"]) == {}
+
+
+ not (HAS_CT and HAS_CONVERTERS), reason = "needs compressed-tensors and the transformers 5 loader"
+)
+def test_arm_strips_root_and_subconfigs_and_parks_the_plan():
+ from transformers import LlamaConfig
+
+ root = LlamaConfig(
+ hidden_size = 8,
+ num_hidden_layers = 1,
+ num_attention_heads = 2,
+ intermediate_size = 8,
+ vocab_size = 16,
+ )
+ text = LlamaConfig(
+ hidden_size = 8,
+ num_hidden_layers = 1,
+ num_attention_heads = 2,
+ intermediate_size = 8,
+ vocab_size = 16,
+ )
+ root.quantization_config = _w4a16()
+ text.quantization_config = _w4a16()
+ root.text_config = text
+ assert text in _config_and_subconfigs(root)
+ plan = arm_compressed_tensors_bnb_loading(root, verbose = False)
+ assert plan is not None
+ assert not hasattr(root, "quantization_config") and not hasattr(text, "quantization_config")
+ assert getattr(root, UNSLOTH_COMPRESSED_TENSORS_ATTR) is plan
+ assert "quantization_config" not in root.to_dict()
+
+
+def test_prepared_config_is_the_armed_config_and_nothing_else():
+ from transformers import LlamaConfig
+ from unsloth.models.loader_utils import compressed_tensors_prepared_config
+
+ plain = LlamaConfig(
+ hidden_size = 8,
+ num_hidden_layers = 1,
+ num_attention_heads = 2,
+ intermediate_size = 8,
+ vocab_size = 16,
+ )
+ assert compressed_tensors_prepared_config(plain) is None
+ assert compressed_tensors_prepared_config(None) is None
+ armed = LlamaConfig(
+ hidden_size = 8,
+ num_hidden_layers = 1,
+ num_attention_heads = 2,
+ intermediate_size = 8,
+ vocab_size = 16,
+ )
+ armed.quantization_config = _w4a16()
+ if arm_compressed_tensors_bnb_loading(armed, verbose = False) is None:
+ pytest.skip("this transformers cannot arm the re-quantization")
+ assert compressed_tensors_prepared_config(armed) is armed
+
+
+def test_planner_gets_the_prepared_config_instead_of_rebuilding_it(monkeypatch):
+ """The planner rebuilds the repo's config from the model name. For a re-quantized packed
+ checkpoint that config still carries compressed-tensors, and `merge_quantization_configs`
+ then refuses the bitsandbytes flags, so Kimi-K2.7-Code lost its plan and fell back to
+ `sequential`, which spilled to CPU and bitsandbytes refused the load. The armed config
+ object is what the planner has to size; the loaders hand it over as `planner_config`."""
+ import sys
+ import types
+ from unsloth.models import loader_utils
+
+ planner = types.ModuleType("unsloth_zoo.device_map_planner")
+ monkeypatch.setitem(sys.modules, "unsloth_zoo.device_map_planner", planner)
+
+ seen = {}
+
+ def fake_plan(
+ model_name,
+ *,
+ max_memory = None,
+ config = None,
+ **kwargs,
+ ):
+ seen.update(kwargs)
+ seen["config"] = config
+ seen["max_memory"] = max_memory
+ return None
+
+ monkeypatch.setattr(planner, "plan_device_map_for_pretrained", fake_plan, raising = False)
+ monkeypatch.setattr(loader_utils, "DEVICE_TYPE_TORCH", "cuda")
+ monkeypatch.setattr(loader_utils, "is_distributed", lambda: False)
+ monkeypatch.setattr(torch.cuda, "device_count", lambda: 2)
+ monkeypatch.setattr(torch.cuda, "mem_get_info", lambda index: (10 * 1024**3, 16 * 1024**3))
+ prepared = object()
+ device_map = loader_utils.resolve_unsloth_device_map(
+ loader_utils.UNSLOTH_DEVICE_MAP, "some/repo", planner_config = prepared, load_in_4bit = True
+ )
+ assert seen["config"] is prepared
+ assert seen["load_in_4bit"] is True
+ assert device_map == loader_utils._PLANNED_DEVICE_MAPS[loader_utils.UNSLOTH_DEVICE_MAP]
+
+ seen.clear()
+
+ def old_plan(
+ model_name,
+ *,
+ max_memory = None,
+ **kwargs,
+ ):
+ seen.update(kwargs)
+ return None
+
+ monkeypatch.setattr(planner, "plan_device_map_for_pretrained", old_plan)
+ device_map = loader_utils.resolve_unsloth_device_map(
+ loader_utils.UNSLOTH_DEVICE_MAP, "some/repo", planner_config = prepared, load_in_4bit = True
+ )
+ assert seen == {}
+ assert device_map == loader_utils._PLANNED_DEVICE_MAPS[loader_utils.UNSLOTH_DEVICE_MAP]
+
+
+ not (HAS_CT and HAS_CONVERTERS), reason = "needs compressed-tensors and the transformers 5 loader"
+)
+def test_check_and_disable_keeps_4bit_for_packed_int4():
+ from transformers import LlamaConfig
+
+ config = LlamaConfig(
+ hidden_size = 8,
+ num_hidden_layers = 1,
+ num_attention_heads = 2,
+ intermediate_size = 8,
+ vocab_size = 16,
+ )
+ config.quantization_config = _w4a16()
+ load_in_4bit, load_in_8bit, method = check_and_disable_bitsandbytes_loading(
+ config, load_in_4bit = True, load_in_8bit = False, verbose = False
+ )
+ assert (load_in_4bit, load_in_8bit, method) == (True, False, None)
+ assert not hasattr(config, "quantization_config")
+
+
+ not (HAS_CT and HAS_CONVERTERS), reason = "needs compressed-tensors and the transformers 5 loader"
+)
[email protected]("spelling", ["compressed_tensors", "sparseml"])
+def test_check_and_disable_accepts_the_method_aliases(spelling):
+ from transformers import LlamaConfig
+
+ config = LlamaConfig(
+ hidden_size = 8,
+ num_hidden_layers = 1,
+ num_attention_heads = 2,
+ intermediate_size = 8,
+ vocab_size = 16,
+ )
+ quantization_config = dict(_w4a16())
+ quantization_config["quant_method"] = spelling
+ config.quantization_config = quantization_config
+ load_in_4bit, load_in_8bit, method = check_and_disable_bitsandbytes_loading(
+ config, load_in_4bit = True, load_in_8bit = False, verbose = False
+ )
+ assert (load_in_4bit, load_in_8bit, method) == (True, False, None)
+
+
+def test_check_and_disable_still_disables_8bit_and_other_methods():
+ for quant, flags in (
+ (_w4a16(), dict(load_in_4bit = False, load_in_8bit = True)),
+ ({"quant_method": "gptq", "bits": 4}, dict(load_in_4bit = True, load_in_8bit = False)),
+ ):
+ config = _Config(quantization_config = quant)
+ load_in_4bit, load_in_8bit, method = check_and_disable_bitsandbytes_loading(
+ config, verbose = False, **flags
+ )
+ assert (load_in_4bit, load_in_8bit) == (False, False)
+ assert method in ("compressed-tensors", "gptq")
+ assert hasattr(config, "quantization_config")
+
+
+ not (HAS_CT and HAS_CONVERTERS), reason = "needs compressed-tensors and the transformers 5 loader"
+)
+def test_check_and_disable_does_not_arm_when_the_4bit_load_is_not_happening():
+ """`fast_inference` hands the packed checkpoint to vLLM and `full_finetuning` turns 4-bit off
+ right after this call: arming there stripped the checkpoint's own quantization config with
+ nothing left to consume the plan, so the packed tensors loaded as unmatched keys."""
+ from transformers import LlamaConfig
+
+ config = LlamaConfig(
+ hidden_size = 8,
+ num_hidden_layers = 1,
+ num_attention_heads = 2,
+ intermediate_size = 8,
+ vocab_size = 16,
+ )
+ config.quantization_config = _w4a16()
+ load_in_4bit, load_in_8bit, method = check_and_disable_bitsandbytes_loading(
+ config, load_in_4bit = True, load_in_8bit = False, verbose = False, requantize_packed = False
+ )
+ assert (load_in_4bit, load_in_8bit, method) == (False, False, "compressed-tensors")
+ assert hasattr(config, "quantization_config")
+ assert not hasattr(config, UNSLOTH_COMPRESSED_TENSORS_ATTR)
+
+
+ not (HAS_CT and HAS_CONVERTERS), reason = "needs compressed-tensors and the transformers 5 loader"
+)
+def test_arm_leaves_the_config_alone_when_the_quantizer_cannot_be_installed(monkeypatch):
+ """A foreign class in the bitsandbytes quantizer slot would load the packed tensors without
+ the converters; the config must then keep its own quantization config."""
+ from transformers import LlamaConfig
+ from unsloth.models import compressed_tensors_bnb
+
+ monkeypatch.setattr(
+ compressed_tensors_bnb, "install_compressed_tensors_bnb_quantizer", lambda: False
+ )
+ config = LlamaConfig(
+ hidden_size = 8,
+ num_hidden_layers = 1,
+ num_attention_heads = 2,
+ intermediate_size = 8,
+ vocab_size = 16,
+ )
+ config.quantization_config = _w4a16()
+ assert compressed_tensors_bnb.arm_compressed_tensors_bnb_loading(config, verbose = False) is None
+ assert hasattr(config, "quantization_config")
+ assert not hasattr(config, UNSLOTH_COMPRESSED_TENSORS_ATTR)
+
+
+ not (HAS_CT and HAS_CONVERTERS), reason = "needs compressed-tensors and the transformers 5 loader"
+)
+def test_arm_declines_a_plan_the_installed_compressed_tensors_cannot_parse(monkeypatch):
+ """Fields get retired between compressed-tensors releases (`actorder = "group"` left in
+ 0.19.0) and the parse used to happen inside `from_pretrained`, after the checkpoint's own
+ quantization config had already been stripped, so the load died with nothing to fall back
+ to. The plan is parsed before anything is touched."""
+ from transformers import LlamaConfig
+ from unsloth.models import compressed_tensors_bnb
+
+ def cannot_parse(plan):
+ raise ValueError("actorder='group' has been removed")
+
+ monkeypatch.setattr(compressed_tensors_bnb, "_build_quantization_config", cannot_parse)
+ config = LlamaConfig(
+ hidden_size = 8,
+ num_hidden_layers = 1,
+ num_attention_heads = 2,
+ intermediate_size = 8,
+ vocab_size = 16,
+ )
+ config.quantization_config = _w4a16(weights = {"actorder": "group"})
+ assert compressed_tensors_bnb.arm_compressed_tensors_bnb_loading(config, verbose = False) is None
+ assert config.quantization_config["config_groups"]["group_0"]["weights"]["actorder"] == "group"
+ assert not hasattr(config, UNSLOTH_COMPRESSED_TENSORS_ATTR)
+
+
+ not (HAS_CT and HAS_CONVERTERS), reason = "needs compressed-tensors and the transformers 5 loader"
+)
+def test_the_planner_pass_does_not_consume_the_plan():
+ """The device-map planner preprocesses a meta model built from the very config object the
+ load uses. The plan used to be taken off the config by the first pass, so on a multi-GPU
+ load the real quantizer found none, added no converters, and every packed expert was
+ reported missing and quantized from its random initialisation."""
+ from transformers import LlamaConfig, LlamaForCausalLM
+ from transformers.quantizers import AutoHfQuantizer
+ from transformers.utils.quantization_config import BitsAndBytesConfig
+ from accelerate import init_empty_weights
+
+ assert install_compressed_tensors_bnb_quantizer()
+ config = LlamaConfig(
+ hidden_size = 8,
+ num_hidden_layers = 1,
+ num_attention_heads = 2,
+ intermediate_size = 8,
+ vocab_size = 16,
+ )
+ config.quantization_config = _w4a16()
+ plan = arm_compressed_tensors_bnb_loading(config, verbose = False)
+ assert plan is not None
+ bnb = BitsAndBytesConfig(
+ load_in_4bit = True, bnb_4bit_compute_dtype = torch.bfloat16, bnb_4bit_quant_type = "nf4"
+ )
+
+ def preprocess():
+ quantizer = AutoHfQuantizer.from_config(bnb, pre_quantized = False)
+ with init_empty_weights():
+ model = LlamaForCausalLM(config)
+ quantizer._process_model_before_weight_loading(
+ model, dtype = torch.bfloat16, device_map = None, checkpoint_files = []
+ )
+ return quantizer, model
+
+ first, _ = preprocess() # the planner's meta pass
+ assert getattr(config, UNSLOTH_COMPRESSED_TENSORS_ATTR, None) is plan
+ second, model = preprocess() # the real load
+ assert second._unsloth_ct_config is not None
+ conversions = second.update_weight_conversions([])
+ assert any(
+ getattr(c, "source_patterns", None) and "weight_packed$" in c.source_patterns
+ for c in conversions
+ ), [getattr(c, "source_patterns", None) for c in conversions]
+ second._process_model_after_weight_loading(model)
+ assert not hasattr(config, UNSLOTH_COMPRESSED_TENSORS_ATTR)
+
+
+ not (HAS_CT and HAS_CONVERTERS), reason = "needs compressed-tensors and the transformers 5 loader"
+)
+def test_armed_conversions_still_run_a_composed_subclass_hook(monkeypatch):
+ """When another bitsandbytes 4-bit subclass is already registered, ours is layered on top of
+ it. An armed load used to build its converter list and return it directly, so the other
+ subclass's ``update_weight_conversions`` never ran."""
+ from accelerate import init_empty_weights
+ from transformers import LlamaConfig, LlamaForCausalLM
+ from transformers.quantizers import AutoHfQuantizer
+ from transformers.quantizers import auto as quantizers_auto
+
+ from unsloth.models import compressed_tensors_bnb
+ from transformers.quantizers.quantizer_bnb_4bit import Bnb4BitHfQuantizer
+ from transformers.utils.quantization_config import BitsAndBytesConfig
+
+ marker = object()
+
+ class MarkerBnb4BitHfQuantizer(Bnb4BitHfQuantizer):
+ def update_weight_conversions(self, weight_conversions):
+ return super().update_weight_conversions(weight_conversions) + [marker]
+
+ monkeypatch.setitem(
+ quantizers_auto.AUTO_QUANTIZER_MAPPING, "bitsandbytes_4bit", MarkerBnb4BitHfQuantizer
+ )
+ monkeypatch.setattr(compressed_tensors_bnb, "_installed", False)
+ assert install_compressed_tensors_bnb_quantizer()
+ composed = quantizers_auto.AUTO_QUANTIZER_MAPPING["bitsandbytes_4bit"]
+ assert issubclass(composed, MarkerBnb4BitHfQuantizer)
+
+ config = LlamaConfig(
+ hidden_size = 8,
+ num_hidden_layers = 1,
+ num_attention_heads = 2,
+ intermediate_size = 8,
+ vocab_size = 16,
+ )
+ config.quantization_config = _w4a16()
+ plan = arm_compressed_tensors_bnb_loading(config, verbose = False)
+ assert plan is not None
+ bnb = BitsAndBytesConfig(
+ load_in_4bit = True, bnb_4bit_compute_dtype = torch.bfloat16, bnb_4bit_quant_type = "nf4"
+ )
+ quantizer = AutoHfQuantizer.from_config(bnb, pre_quantized = False)
+ assert isinstance(quantizer, composed)
+ with init_empty_weights():
+ model = LlamaForCausalLM(config)
+ quantizer._process_model_before_weight_loading(
+ model, dtype = torch.bfloat16, device_map = None, checkpoint_files = []
+ )
+ assert quantizer._unsloth_ct_config is not None
+ conversions = quantizer.update_weight_conversions([])
+ assert marker in conversions
+ assert any(
+ getattr(c, "source_patterns", None) and "weight_packed$" in c.source_patterns
+ for c in conversions
+ ), [getattr(c, "source_patterns", None) for c in conversions]
+
+
[email protected](not HAS_CONVERTERS, reason = "needs the transformers 5 loader")
+def test_quantizer_registration_is_idempotent_and_a_subclass():
+ from transformers.quantizers import auto as quantizers_auto
+ from transformers.quantizers.quantizer_bnb_4bit import Bnb4BitHfQuantizer
+
+ assert install_compressed_tensors_bnb_quantizer()
+ first = quantizers_auto.AUTO_QUANTIZER_MAPPING["bitsandbytes_4bit"]
+ assert install_compressed_tensors_bnb_quantizer()
+ assert quantizers_auto.AUTO_QUANTIZER_MAPPING["bitsandbytes_4bit"] is first
+ assert issubclass(first, Bnb4BitHfQuantizer)
+
+
+def _write_tiny_packed_llama(
+ root,
+ asymmetric = False,
+ actorder = False,
+ num_bits = 4,
+ group_size = 32,
+):
+ """A 2-layer Llama with every Linear but lm_head packed by compressed-tensors' compressor.
+ Returns (packed_dir, bf16_dir) where bf16_dir holds the exact decompressed weights."""
+ from safetensors.torch import save_file
+ from transformers import LlamaConfig, LlamaForCausalLM
+ from compressed_tensors.compressors import BaseCompressor
+ from compressed_tensors.quantization import QuantizationConfig
+ from compressed_tensors.quantization.utils import calculate_qparams
+
+ torch.manual_seed(0)
+ config = LlamaConfig(
+ hidden_size = 64,
+ num_hidden_layers = 2,
+ num_attention_heads = 4,
+ num_key_value_heads = 2,
+ intermediate_size = 128,
+ vocab_size = 256,
+ max_position_embeddings = 128,
+ tie_word_embeddings = False,
+ )
+ model = LlamaForCausalLM(config).to(torch.bfloat16)
+ sd = {k: v.detach().contiguous() for k, v in model.state_dict().items()}
+ if actorder and not CT_HONOURS_G_IDX:
+ pytest.skip("this compressed-tensors cannot write or decompress activation-ordered weights")
+ quant = _w4a16(
+ weights = {
+ "num_bits": num_bits,
+ "group_size": group_size,
+ "symmetric": not asymmetric,
+ "actorder": "weight" if actorder else None,
+ }
+ )
+ ctc = QuantizationConfig.model_validate(quant)
+ scheme = list(ctc.config_groups.values())[0]
+ comp = BaseCompressor.get_value_from_registry("pack-quantized")
+ linears = {n for n, m in model.named_modules() if isinstance(m, torch.nn.Linear)}
+ packed, plain = {}, {}
+ for k, w in sd.items():
+ mod = k[: -len(".weight")] if k.endswith(".weight") else None
+ if mod is None or mod not in linears or mod == "lm_head":
+ packed[k] = w
+ plain[k] = w
+ continue
+ out_f, in_f = w.shape
+ wf = w.float()
+ g_idx = None
+ if actorder:
+ g_idx = torch.randperm(in_f) // group_size
+ g_idx = g_idx.to(torch.int32)
+ grouped = wf[:, torch.argsort(g_idx)].reshape(out_f, in_f // group_size, group_size)
+ else:
+ grouped = wf.reshape(out_f, in_f // group_size, group_size)
+ scale, zp = calculate_qparams(grouped.amin(-1), grouped.amax(-1), scheme.weights)
+ state = {"weight": wf, "weight_scale": scale.to(torch.bfloat16)}
+ if asymmetric:
+ state["weight_zero_point"] = zp.to(torch.int8)
+ if g_idx is not None:
+ state["weight_g_idx"] = g_idx
+ out = comp.compress(state, scheme)
+ for pk, pv in out.items():
+ packed[mod + "." + pk] = pv.contiguous()
+ dec = comp.decompress({k2: v2 for k2, v2 in out.items()}, scheme)["weight"].to(
+ torch.bfloat16
+ )
+ plain[k] = dec.contiguous()
+ packed_dir, bf16_dir = os.path.join(root, "packed"), os.path.join(root, "bf16")
+ os.makedirs(packed_dir)
+ os.makedirs(bf16_dir)
+ save_file(packed, os.path.join(packed_dir, "model.safetensors"), metadata = {"format": "pt"})
+ save_file(plain, os.path.join(bf16_dir, "model.safetensors"), metadata = {"format": "pt"})
+ cfg = config.to_dict()
+ json.dump(cfg, open(os.path.join(bf16_dir, "config.json"), "w"))
+ cfg["quantization_config"] = quant
+ json.dump(cfg, open(os.path.join(packed_dir, "config.json"), "w"))
+ return packed_dir, bf16_dir
+
+
+def _same_linear4bit(a, b):
+ import bitsandbytes as bnb
+
+ mods_b = dict(b.named_modules())
+ n = 0
+ for name, m in a.named_modules():
+ if not isinstance(m, bnb.nn.Linear4bit):
+ continue
+ o = mods_b[name]
+ assert isinstance(o, bnb.nn.Linear4bit), name
+ qa, qb = m.weight.quant_state, o.weight.quant_state
+ assert torch.equal(m.weight.data.view(-1), o.weight.data.view(-1)), name
+ assert (
+ torch.equal(qa.absmax, qb.absmax) and qa.shape == qb.shape and qa.dtype == qb.dtype
+ ), name
+ n += 1
+ return n
+
+
+def _tokenizer_free_load(path, root):
+ """Unsloth's loader wants a tokenizer next to the weights; a tiny one is enough."""
+ from transformers import AutoTokenizer
+
+ tok_dir = os.path.join(root, "tok")
+ if not os.path.isdir(tok_dir):
+ AutoTokenizer.from_pretrained("hf-internal-testing/llama-tokenizer").save_pretrained(
+ tok_dir
+ )
+ for f in os.listdir(tok_dir):
+ if not os.path.exists(os.path.join(path, f)):
+ shutil.copy(os.path.join(tok_dir, f), os.path.join(path, f))
+
+
[email protected](not has_real_cuda(), reason = "needs a CUDA device")
+ not (HAS_CT and HAS_CONVERTERS), reason = "needs compressed-tensors and the transformers 5 loader"
+)
[email protected]("variant", ["symmetric", "asymmetric", "actorder", "int8"])
[email protected]("restore_llama_patches")
+def test_packed_checkpoint_loads_as_linear4bit_bit_identical_to_disk_route(
+ variant, tmp_path, monkeypatch
+):
+ from unsloth import FastLanguageModel
+
+ monkeypatch.setenv("UNSLOTH_COMPRESSED_TENSORS_INT4", "nf4")
+
+ packed_dir, bf16_dir = _write_tiny_packed_llama(
+ str(tmp_path),
+ asymmetric = variant == "asymmetric",
+ actorder = variant == "actorder",
+ num_bits = 8 if variant == "int8" else 4,
+ )
+ for d in (packed_dir, bf16_dir):
+ _tokenizer_free_load(d, str(tmp_path))
+ kw = dict(max_seq_length = 64, dtype = torch.bfloat16, load_in_4bit = True)
+ model_a, _ = FastLanguageModel.from_pretrained(packed_dir, **kw)
+ model_b, _ = FastLanguageModel.from_pretrained(bf16_dir, **kw)
+ n = _same_linear4bit(model_a, model_b)
+ assert n == 2 * 7, n # q k v o gate up down per layer
+ assert not any(k.endswith("weight_packed") for k, _ in model_a.named_parameters())
+ ids = torch.randint(0, 256, (1, 16), device = "cuda:0")
+ with torch.no_grad():
+ assert torch.equal(model_a(input_ids = ids).logits, model_b(input_ids = ids).logits)
+ assert not hasattr(model_a.config, UNSLOTH_COMPRESSED_TENSORS_ATTR)
+ assert getattr(model_a.config, "quantization_config", None) is not None # the bitsandbytes one
+ assert "compressed" not in str(model_a.config.quantization_config).lower()
+
+
[email protected](not has_real_cuda(), reason = "needs a CUDA device")
+ not (HAS_CT and HAS_CONVERTERS), reason = "needs compressed-tensors and the transformers 5 loader"
+)
[email protected]("route", ["nf4", "packed"])
[email protected]("restore_llama_patches")
+def test_packed_checkpoint_trains_with_lora(tmp_path, monkeypatch, route):
+ from unsloth import FastLanguageModel
+
+ monkeypatch.setenv("UNSLOTH_COMPRESSED_TENSORS_INT4", route)
+
+ packed_dir, _ = _write_tiny_packed_llama(str(tmp_path))
+ _tokenizer_free_load(packed_dir, str(tmp_path))
+ model, _ = FastLanguageModel.from_pretrained(
+ packed_dir, max_seq_length = 64, dtype = torch.bfloat16, load_in_4bit = True
+ )
+ model = FastLanguageModel.get_peft_model(
+ model, r = 4, lora_alpha = 8, target_modules = ["q_proj", "v_proj", "down_proj"]
+ )
+ model.train()
+ ids = torch.randint(0, 256, (1, 16), device = "cuda:0")
+ loss = model(input_ids = ids, labels = ids).loss
+ loss.backward()
+ grads = [p.grad for n, p in model.named_parameters() if "lora_B" in n]
+ assert grads and all(g is not None and torch.isfinite(g).all() for g in grads)
+ assert any(g.abs().sum() > 0 for g in grads)
+
+
+ not (HAS_CT and HAS_CONVERTERS), reason = "needs compressed-tensors and the transformers 5 loader"
+)
+def test_many_to_many_expert_op_keeps_its_source_contract():
+ """transformers' ErnieFuseAndSplitTextVisionExperts iterates the converter's
+ source_patterns and requires every one of them in the collected dict. After
+ decompression the packed metadata patterns are gone, so the op must see the
+ patterns it was declared with and the decompressed buckets under them."""
+ from transformers.core_model_loading import ErnieFuseAndSplitTextVisionExperts, WeightConverter
+ from compressed_tensors.compressors import BaseCompressor
+ from compressed_tensors.quantization import QuantizationConfig
+ from compressed_tensors.quantization.utils import calculate_qparams
+ from unsloth.models.compressed_tensors_bnb import _WithOriginalSources, _DecompressPackedWeights
+
+ torch.manual_seed(0)
+ quant = _w4a16(weights = {"num_bits": 4, "group_size": 32, "symmetric": True})
+ ctc = QuantizationConfig.model_validate(quant)
+ scheme = list(ctc.config_groups.values())[0]
+ comp = BaseCompressor.get_value_from_registry("pack-quantized")
+ sources = ["mlp.experts.*.gate_proj.weight", "mlp.experts.*.up_proj.weight"]
+ targets = ["mlp.text_experts.gate_up_proj", "mlp.vision_experts.gate_up_proj"]
+
+ plain, packed = {}, {}
+ for p in sources:
+ plain[p], packed[p + "_packed$"], packed[p + "_scale$"], packed[p + "_shape$"] = (
+ [],
+ [],
+ [],
+ [],
+ )
+ for _ in range(4): # two text experts then two vision experts
+ w = torch.randn(16, 64)
+ grouped = w.reshape(16, 2, 32)
+ scale, _zp = calculate_qparams(grouped.amin(-1), grouped.amax(-1), scheme.weights)
+ out = comp.compress({"weight": w, "weight_scale": scale.to(torch.bfloat16)}, scheme)
+ plain[p].append(comp.decompress(dict(out), scheme)["weight"].to(torch.bfloat16))
+ packed[p + "_packed$"].append(out["weight_packed"])
+ packed[p + "_scale$"].append(out["weight_scale"])
+ packed[p + "_shape$"].append(out["weight_shape"])
+
+ ernie = ErnieFuseAndSplitTextVisionExperts()
+ want = ernie.convert(dict(plain), source_patterns = sources, target_patterns = targets, config = None)
+
+ rebuilt_sources = (
+ [p + "_packed$" for p in sources]
+ + [p + "_scale$" for p in sources]
+ + [p + "_shape$" for p in sources]
+ + [p + "$" for p in sources]
+ )
+ ops = [
+ _DecompressPackedWeights(ctc, torch.bfloat16, stacked = True, scheme = scheme),
+ _WithOriginalSources(ernie, sources, sources),
+ ]
+ got = dict(packed)
+ for op in ops:
+ got = op.convert(
+ got,
+ source_patterns = rebuilt_sources,
+ target_patterns = targets,
+ full_layer_name = "model.layers.0",
+ model = None,
+ config = None,
+ )
+ assert set(got) == set(targets)
+ for k in targets:
+ assert torch.equal(got[k], want[k]), k
+
+ # Without the adapter the many-to-many op fails on the consumed metadata patterns.
+ got = dict(packed)
+ got = ops[0].convert(
+ got,
+ source_patterns = rebuilt_sources,
+ target_patterns = targets,
+ full_layer_name = "model.layers.0",
+ model = None,
+ config = None,
+ )
+ with pytest.raises((ValueError, TypeError)):
+ ernie.convert(got, source_patterns = rebuilt_sources, target_patterns = targets, config = None)
+
+
+def test_classification_load_under_fast_inference_still_requantizes():
+ """fast_inference with num_labels loads through transformers (vLLM has no classification
+ head), so the packed re-quantization must stay armed there."""
+ import inspect
+ from unsloth.models import llama
+
+ assert llama._vllm_will_load_weights(True, num_labels = 2) is False
+ assert llama._vllm_will_load_weights(False, None) is False
+ source = inspect.getsource(llama.FastLlamaModel.from_pretrained)
+ assert "requantize_packed = not _vllm_will_load_weights(fast_inference, num_labels)" in source
+
+
[email protected](not HAS_CONVERTERS, reason = "needs the transformers 5 loader")
+def test_wrapped_many_to_many_op_still_passes_the_converter_type_check():
+ """transformers' WeightConverter allows a many-to-many mapping only when `operations`
+ holds an instance of its internal Ernie ops. The adapter that feeds such an op its
+ original source contract must remain an instance of the op's class, or building the
+ replacement converter raises before any weight loads."""
+ from transformers.core_model_loading import ErnieFuseAndSplitTextVisionExperts, WeightConverter
+ from unsloth.models.compressed_tensors_bnb import _WithOriginalSources, _with_original_sources
+
+ op = ErnieFuseAndSplitTextVisionExperts(stack_dim = 0, concat_dim = 1)
+ sources = ["mlp.experts.*.gate_proj.weight", "mlp.experts.*.up_proj.weight"]
+ targets = ["mlp.text_experts.gate_up_proj", "mlp.vision_experts.gate_up_proj"]
+ wrapped = _with_original_sources(op, sources, sources)
+ assert isinstance(wrapped, _WithOriginalSources) and isinstance(
+ wrapped, ErnieFuseAndSplitTextVisionExperts
+ )
+ WeightConverter(
+ source_patterns = [s + "_packed$" for s in sources] + [s + "$" for s in sources],
+ target_patterns = targets,
+ operations = [wrapped],
+ )
+
+
+def test_a_pickled_shard_is_refused_rather_than_read_in_the_model_dtype():
+ """A .bin shard is materialised in the model dtype before any converter runs, so the packed
+ words would be cast to bf16 garbage; the plan builder refuses it with an instruction."""
+ from unsloth.models.compressed_tensors_bnb import _checkpoint_keys
+
+ with pytest.raises(RuntimeError, match = "safetensors"):
+ _checkpoint_keys(["/x/pytorch_model-00001-of-00002.bin"])
+ assert _checkpoint_keys([]) == []
+
+
+def test_load_only_converters_are_dropped_before_save():
+ """transformers reverses every converter on model._weight_conversions in save_pretrained;
+ the decompression converters must not be there, or a bitsandbytes model is saved under
+ the packed names."""
+ from unsloth.models.compressed_tensors_bnb import (
+ _DecompressPackedWeights,
+ drop_load_only_conversions,
+ )
+
+ class Conv:
+ def __init__(self, ops):
+ self.operations = ops
+
+ model = SimpleNamespace(
+ _weight_conversions = [
+ Conv([object()]),
+ Conv([_DecompressPackedWeights.__new__(_DecompressPackedWeights)]),
+ ]
+ )
+ assert drop_load_only_conversions(model) == 1
+ assert len(model._weight_conversions) == 1
+ assert drop_load_only_conversions(SimpleNamespace()) == 0
+
+
+def test_partially_packed_expert_bucket_is_refused():
+ from unsloth.models.compressed_tensors_bnb import _WithOriginalSources
+
+ class Op:
+ def convert(self, d, **kw):
+ return d
+
+ adapter = _WithOriginalSources(Op(), ["experts.*.up_proj.weight"], ["experts.*.up_proj.weight"])
+ with pytest.raises(RuntimeError, match = "partially packed"):
+ adapter.convert(
+ {
+ "experts.*.up_proj.weight_packed$": [torch.zeros(2, 2)],
+ "experts.*.up_proj.weight$": [torch.zeros(2, 2)],
+ }
+ )
+
+
[email protected](not HAS_CT, reason = "needs compressed-tensors")
+def test_scheme_is_resolved_per_converter_source_and_mixed_buckets_are_refused():
+ from compressed_tensors.quantization import QuantizationConfig
+ from unsloth.models.compressed_tensors_bnb import _scheme_for_sources
+
+ quant = _w4a16()
+ quant["config_groups"] = {
+ "group_0": {
+ "targets": ["re:.*gate_proj.*"],
+ "weights": dict(quant["config_groups"]["group_0"]["weights"], num_bits = 4),
+ "input_activations": None,
+ "output_activations": None,
+ },
+ "group_1": {
+ "targets": ["re:.*up_proj.*"],
+ "weights": dict(quant["config_groups"]["group_0"]["weights"], num_bits = 8),
+ "input_activations": None,
+ "output_activations": None,
+ },
+ }
+ ctc = QuantizationConfig.model_validate(quant)
+ assert _scheme_for_sources(ctc, ["mlp.experts.*.gate_proj.weight"]).weights.num_bits == 4
+ assert _scheme_for_sources(ctc, ["mlp.experts.*.up_proj.weight"]).weights.num_bits == 8
+ with pytest.raises(RuntimeError, match = "different config groups"):
+ _scheme_for_sources(ctc, ["mlp.experts.*.gate_proj.weight", "mlp.experts.*.up_proj.weight"])
+
+
+def test_an_explicit_quantizer_other_than_bnb_4bit_keeps_the_checkpoint_config():
+ from transformers import BitsAndBytesConfig
+ from unsloth.models.loader_utils import quantization_config_selects_bnb_4bit
+
+ assert quantization_config_selects_bnb_4bit(None)
+ assert quantization_config_selects_bnb_4bit(BitsAndBytesConfig(load_in_4bit = True))
+ assert quantization_config_selects_bnb_4bit(
+ {"quant_method": "bitsandbytes", "load_in_4bit": True}
+ )
+ assert not quantization_config_selects_bnb_4bit(BitsAndBytesConfig(load_in_8bit = True))
+ assert not quantization_config_selects_bnb_4bit({"quant_method": "gptq", "bits": 4})
+ config = _Config(quantization_config = _w4a16())
+ load_in_4bit, _, method = check_and_disable_bitsandbytes_loading(
+ config,
+ load_in_4bit = True,
+ load_in_8bit = False,
+ verbose = False,
+ requantize_packed = quantization_config_selects_bnb_4bit(
+ BitsAndBytesConfig(load_in_8bit = True)
+ ),
+ )
+ assert method == "compressed-tensors" and not load_in_4bit
+ assert getattr(config, UNSLOTH_COMPRESSED_TENSORS_ATTR, None) is None
+
+
+def test_both_loaders_gate_packed_requantization_on_the_callers_quantizer():
+ import ast, inspect
+ from unsloth.models import llama, vision
+ for module in (llama, vision):
+ calls = [
+ node
+ for node in ast.walk(ast.parse(inspect.getsource(module)))
+ if isinstance(node, ast.Call)
+ and getattr(node.func, "id", None) == "check_and_disable_bitsandbytes_loading"
+ ]
+ assert calls, module.__name__
+ for call in calls:
+ flag = next(k.value for k in call.keywords if k.arg == "requantize_packed")
+ assert "quantization_config_selects_bnb_4bit" in ast.unparse(flag), module.__name__
+
+
+def test_only_an_explicit_bnb_4bit_config_keeps_the_callers_flags():
+ # Else the 4-bit branch swaps a GPTQ / AWQ / FP8 quantizer for a BitsAndBytesConfig.
+ import ast, inspect
+ from unsloth.models import llama, vision
+ for module in (llama, vision):
+ guards = [
+ ast.unparse(node.test)
+ for node in ast.walk(ast.parse(inspect.getsource(module)))
+ if isinstance(node, ast.If)
+ and any("_checked_4bit" in ast.unparse(stmt) for stmt in node.body)
+ ]
+ # ... and only while the check kept 4-bit (an unarmed pre-quantized checkpoint must not reach vLLM as bnb).
+ assert guards == ["not (_explicit_bnb_4bit and _checked_4bit)"], (module.__name__, guards)
+
+
+def test_an_explicit_bnb_4bit_config_is_not_replaced_by_the_default_nf4_one():
+ import ast, inspect
+ from unsloth.models import llama
+
+ guards = [
+ ast.unparse(node.test)
+ for node in ast.walk(ast.parse(inspect.getsource(llama)))
+ if isinstance(node, ast.If)
+ and any(
+ isinstance(stmt, ast.Assign)
+ and "quantization_config" in ast.unparse(stmt.targets[0])
+ and ast.unparse(stmt.value) == "bnb_config"
+ for stmt in node.body
+ )
+ ]
+ assert guards and all("not _explicit_bnb_4bit" in guard for guard in guards), guards
+
+
+ not (HAS_CT and HAS_CONVERTERS), reason = "needs compressed-tensors and the transformers 5 loader"
+)
+def test_both_loaders_treat_an_explicit_bnb_4bit_config_as_the_4bit_request():
+ # The loader passes load_in_4bit = False with an explicit config; a bnb 4-bit config must still arm the plan.
+ import ast, inspect
+ from transformers import BitsAndBytesConfig
+ from unsloth.models import llama, vision
+ from unsloth.models.loader_utils import quantization_config_selects_bnb_4bit
+
+ for module in (llama, vision):
+ calls = [
+ node
+ for node in ast.walk(ast.parse(inspect.getsource(module)))
+ if isinstance(node, ast.Call)
+ and getattr(node.func, "id", None) == "check_and_disable_bitsandbytes_loading"
+ ]
+ for call in calls:
+ flag = next(k.value for k in call.keywords if k.arg == "load_in_4bit")
+ assert "_explicit_bnb_4bit" in ast.unparse(flag), module.__name__
+
+ explicit = BitsAndBytesConfig(load_in_4bit = True)
+ config = _Config(quantization_config = _w4a16())
+ check_and_disable_bitsandbytes_loading(
+ config,
+ load_in_4bit = False or quantization_config_selects_bnb_4bit(explicit),
+ load_in_8bit = False,
+ verbose = False,
+ requantize_packed = quantization_config_selects_bnb_4bit(explicit),
+ )
+ assert getattr(config, UNSLOTH_COMPRESSED_TENSORS_ATTR, None) is not None
+ # Unarmed (vLLM reads the checkpoint itself), the check clears 4-bit.
+ unarmed = _Config(quantization_config = _w4a16())
+ checked_4bit, _, _ = check_and_disable_bitsandbytes_loading(
+ unarmed, load_in_4bit = True, load_in_8bit = False, verbose = False, requantize_packed = False
+ )
+ assert checked_4bit is False
+
+
+def test_the_flag_only_dict_shorthand_is_a_bnb_4bit_request():
+ # transformers' AutoQuantizationConfig.from_dict reads {"load_in_4bit": True} as bitsandbytes.
+ from unsloth.models.loader_utils import (
+ _bnb_bits_requested,
+ quantization_config_selects_bnb_4bit,
+ )
+
+ assert quantization_config_selects_bnb_4bit({"load_in_4bit": True})
+ assert _bnb_bits_requested({"load_in_4bit": True}) == 4
+ assert not quantization_config_selects_bnb_4bit({"load_in_8bit": True})
+ assert not quantization_config_selects_bnb_4bit({"quant_method": "fp8", "load_in_4bit": True})
+
+
+ "bits,route,want",
+ [(8, True, (False, True)), (4, True, (True, False)), (8, False, (True, False))],
+)
+def test_the_planner_sizes_a_packed_int8_checkpoint_at_8_bits(bits, route, want, monkeypatch):
+ # Kept packed, INT8 weights stay 8-bit: planning them as bnb 4-bit halves their budget and OOMs the load.
+ import unsloth.models.compressed_tensors_int4 as ct_int4
+ from unsloth.models.loader_utils import compressed_tensors_planner_bits
+
+ monkeypatch.setattr(ct_int4, "int4_packed_route_enabled", lambda: route)
+ config = _Config()
+ setattr(config, UNSLOTH_COMPRESSED_TENSORS_ATTR, _w4a16(weights = {"num_bits": bits}))
+ assert compressed_tensors_planner_bits(config, True, False) == want
+ assert compressed_tensors_planner_bits(_Config(), True, False) == (True, False)
+ assert compressed_tensors_planner_bits(config, False, False) == (False, False)
+ # An explicit bnb 4-bit config would override the 8-bit flags in the planner: it must be dropped too.
+ from transformers import BitsAndBytesConfig
+ from unsloth.models.loader_utils import compressed_tensors_planner_quantization
+
+ explicit = BitsAndBytesConfig(load_in_4bit = True)
+ got = compressed_tensors_planner_quantization(config, False, False, explicit)
+ if want == (False, True):
+ assert got == dict(load_in_4bit = False, load_in_8bit = True, quantization_config = None)
+ else:
+ assert got["quantization_config"] is explicit
+ import ast, inspect
+ from unsloth.models import llama, vision
+
+ for module in (llama, vision):
+ calls = [
+ node
+ for node in ast.walk(ast.parse(inspect.getsource(module)))
+ if isinstance(node, ast.Call)
+ and getattr(node.func, "id", None) == "planner_quantization_kwargs"
+ ]
+ assert calls and all(
+ "compressed_tensors_planner_quantization" in ast.unparse(call) for call in calls
+ ), module.__name__
+
+
+def test_exact_module_targets_win_over_a_class_target():
+ from types import SimpleNamespace
+ from unsloth.models.compressed_tensors_bnb import _layer_expert_scheme, _scheme_for_module
+
+ dense = SimpleNamespace(targets = ["Linear"])
+ exact = SimpleNamespace(targets = [f"model.layers.1.mlp.experts.{e}.gate_proj" for e in range(4)])
+ ct = SimpleNamespace(config_groups = {"dense": dense, "experts": exact})
+ assert _scheme_for_module(ct, "model.layers.1.mlp.experts.2.gate_proj", None) is exact
+ assert _scheme_for_module(ct, "model.layers.0.self_attn.q_proj", None) is dense
+ key = "mlp.experts.*.gate_proj.weight_packed$"
+ assert (
+ _layer_expert_scheme(ct, "model.layers.1.mlp.experts.gate_up_proj", key, 4, None) is exact
+ )
+
+
+def test_exact_and_regex_targets_win_when_the_module_is_known():
+ pytest.importorskip("compressed_tensors")
+ from types import SimpleNamespace
+ from unsloth.models.compressed_tensors_bnb import _scheme_for_module
+
+ dense = SimpleNamespace(targets = ["Linear"])
+ exact = SimpleNamespace(targets = [f"model.layers.1.mlp.experts.{e}.gate_proj" for e in range(4)])
+ ct = SimpleNamespace(config_groups = {"dense": dense, "experts": exact})
+ linear = torch.nn.Linear(4, 4)
+ assert _scheme_for_module(ct, "model.layers.1.mlp.experts.2.gate_proj", linear) is exact
+ assert _scheme_for_module(ct, "model.layers.0.self_attn.q_proj", linear) is dense
+ regex = SimpleNamespace(targets = [r"re:.*self_attn\.q_proj$"])
+ ct = SimpleNamespace(config_groups = {"dense": dense, "q": regex})
+ assert _scheme_for_module(ct, "model.layers.0.self_attn.q_proj", linear) is regex
+
+
+def test_expert_scheme_is_resolved_per_layer():
+ # One converter serves every layer: each layer must get its own scheme.
+ from types import SimpleNamespace
+ from unsloth.models.compressed_tensors_bnb import _layer_expert_scheme
+
+ early = SimpleNamespace(targets = [r"re:.*layers\.[0-3]\.mlp\.experts\..*"])
+ late = SimpleNamespace(targets = [r"re:.*layers\.([4-9]|\d\d+)\.mlp\.experts\..*"])
+ ct = SimpleNamespace(config_groups = {"group_0": early, "group_1": late})
+ key = "mlp.experts.*.gate_proj.weight_packed$"
+ assert (
+ _layer_expert_scheme(ct, "model.layers.1.mlp.experts.gate_up_proj", key, 4, None) is early
+ )
+ assert _layer_expert_scheme(ct, "model.layers.5.mlp.experts.gate_up_proj", key, 4, None) is late
+ assert (
+ _layer_expert_scheme(ct, "model.layers.12.mlp.experts.gate_up_proj", key, 4, None) is late
+ )
+
+ split = SimpleNamespace(targets = [r"re:.*experts\.[0-1]\..*"])
+ rest = SimpleNamespace(targets = [r"re:.*experts\.[2-9]\..*"])
+ mixed = SimpleNamespace(config_groups = {"a": split, "b": rest})
+ with pytest.raises(RuntimeError, match = "different config groups"):
+ _layer_expert_scheme(mixed, "model.layers.0.mlp.experts.gate_up_proj", key, 4, None)
+
+ for dotted in (".experts.*.w1.weight_packed$", "^.experts.*.w1.weight_packed"):
+ assert (
+ _layer_expert_scheme(ct, "model.layers.1.mlp.experts.gate_up_proj", dotted, 4, None)
+ is early
+ )
+ assert (
+ _layer_expert_scheme(ct, "model.layers.5.mlp.experts.gate_up_proj", dotted, 4, None)
+ is late
+ )
+
+ single = SimpleNamespace(config_groups = {"only": early})
+ assert (
+ _layer_expert_scheme(single, "model.layers.9.mlp.experts.gate_up_proj", key, 4, "d") == "d"
+ )
+
+
+ not (HAS_CT and HAS_CONVERTERS), reason = "needs compressed-tensors and the transformers 5 loader"
+)
[email protected]("stack_only_merge", [False, True])
+def test_chained_expert_ops_all_see_the_original_sources(stack_only_merge):
+ """Qwen2/Qwen3-MoE gate/up is MergeModulelist then Concatenate. Every op of the rebuilt chain
+ must run under the original source names, and a MergeModulelist that only stacks lists
+ (transformers 5.0 to 5.6) must be handed the per-expert list, not the stacked bucket."""
+ from transformers.core_model_loading import Concatenate, MergeModulelist
+ from compressed_tensors.compressors import BaseCompressor
+ from compressed_tensors.quantization import QuantizationConfig
+ from compressed_tensors.quantization.utils import calculate_qparams
+ from unsloth.models.compressed_tensors_bnb import (
+ _DecompressPackedWeights,
+ _with_original_sources,
+ )
+
+ class StackOnlyMerge(MergeModulelist):
+ def convert(self, input_dict, source_patterns, target_patterns, **kwargs):
+ size = len(input_dict)
+ return {
+ self.get_target_pattern(size, key, target_patterns): torch.stack(
+ input_dict.pop(key), dim = self.dim
+ )
+ for key in list(input_dict)
+ }
+
+ torch.manual_seed(0)
+ quant = _w4a16(weights = {"num_bits": 4, "group_size": 32, "symmetric": True})
+ ctc = QuantizationConfig.model_validate(quant)
+ scheme = list(ctc.config_groups.values())[0]
+ comp = BaseCompressor.get_value_from_registry("pack-quantized")
+ sources = ["mlp.experts.*.gate_proj.weight", "mlp.experts.*.up_proj.weight"]
+ targets = ["mlp.experts.gate_up_proj"]
+ plain, packed = {}, {}
+ for p in sources:
+ plain[p] = []
+ packed[p + "_packed$"], packed[p + "_scale$"], packed[p + "_shape$"] = [], [], []
+ for _ in range(4):
+ w = torch.randn(16, 64)
+ scale, _zp = calculate_qparams(
+ w.reshape(16, 2, 32).amin(-1), w.reshape(16, 2, 32).amax(-1), scheme.weights
+ )
+ out = comp.compress({"weight": w, "weight_scale": scale.to(torch.bfloat16)}, scheme)
+ plain[p].append(comp.decompress(dict(out), scheme)["weight"].to(torch.bfloat16))
+ packed[p + "_packed$"].append(out["weight_packed"])
+ packed[p + "_scale$"].append(out["weight_scale"])
+ packed[p + "_shape$"].append(out["weight_shape"])
+
+ merge = StackOnlyMerge(dim = 0) if stack_only_merge else MergeModulelist(dim = 0)
+ want = dict(plain)
+ for op in (MergeModulelist(dim = 0), Concatenate(dim = 1)):
+ want = op.convert(want, source_patterns = sources, target_patterns = targets)
+
+ rebuilt_sources = (
+ [p + "_packed$" for p in sources]
+ + [p + "_scale$" for p in sources]
+ + [p + "_shape$" for p in sources]
+ + [p + "$" for p in sources]
+ )
+ ops = [_DecompressPackedWeights(ctc, torch.bfloat16, stacked = True, scheme = scheme)] + [
+ _with_original_sources(op, sources, sources, receives_buckets = index == 0)
+ for index, op in enumerate([merge, Concatenate(dim = 1)])
+ ]
+ got = dict(packed)
+ for op in ops:
+ got = op.convert(
+ got,
+ source_patterns = rebuilt_sources,
+ target_patterns = targets,
+ full_layer_name = "model.layers.0",
+ model = None,
+ config = None,
+ )
+ assert set(got) == set(targets)
+ torch.testing.assert_close(got[targets[0]], want[targets[0]])
+
+
+def test_wrapped_converter_op_survives_the_loaders_deepcopy():
+ # transformers' loader deep-copies every WeightConverter per target key (native MoE merges).
+ pytest.importorskip("transformers.core_model_loading")
+ import copy
+
+ from transformers.core_model_loading import MergeModulelist, WeightConverter
+ from unsloth.models.compressed_tensors_bnb import _with_original_sources
+
+ pattern = "mlp.experts.*.gate_proj.weight"
+ op = _with_original_sources(MergeModulelist(dim = 0), [pattern], [pattern])
+ converter = WeightConverter(
+ source_patterns = [pattern],
+ target_patterns = "mlp.experts.gate_up_proj",
+ operations = [op],
+ )
+ clone = copy.deepcopy(converter).operations[0]
+ assert type(clone) is type(op) and clone.op is not op.op
+ assert clone.weight_sources == op.weight_sources and clone.dim == 0
diff --git a/tests/test_compressed_tensors_int4.py b/tests/test_compressed_tensors_int4.py
new file mode 100644
index 00000000000..2d94fa4ddb8
--- /dev/null
+++ b/tests/test_compressed_tensors_int4.py
@@ -0,0 +1,645 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved.
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+"""compressed-tensors INT4 / INT8 kept packed: exact decode kernels and the default load route.
+
+The kernels are checked against compressed-tensors' own ``decompress`` (bit for bit) and a dense
+matmul; the route loads a tiny packed Llama (built by test_compressed_tensors_bnb) and compares
+it with the same checkpoint decompressed to bf16 on disk.
+"""
+
+import copy
+import itertools
+import os
+import sys
+
+import pytest
+import torch
+from llama_patch_isolation import restore_llama_patches # noqa: F401
+from real_accelerator import has_real_cuda # tests/_shared, on sys.path via tests/conftest.py
+
+import unsloth # noqa: F401
+from unsloth.models.compressed_tensors_bnb import _transformers_supports_weight_converters
+
+try:
+ import inspect
+ from compressed_tensors.quantization.lifecycle.forward import dequantize as _ct_dequantize
+
+ HAS_CT = True
+ # compressed-tensors 0.19 dropped GPTQ activation ordering (#840): its compressor ignores weight_g_idx.
+ CT_HONOURS_G_IDX = "g_idx" in inspect.signature(_ct_dequantize).parameters
+except Exception:
+ HAS_CT = CT_HONOURS_G_IDX = False
+
+HAS_CONVERTERS = _transformers_supports_weight_converters()
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+needs_gpu = pytest.mark.skipif(not has_real_cuda(), reason = "needs a CUDA device")
+needs_ct = pytest.mark.skipif(not HAS_CT, reason = "needs compressed-tensors")
+
+
+def _packed_layer(
+ out_f,
+ in_f,
+ bits,
+ group_size,
+ symmetric,
+ actorder,
+ scale_dtype,
+ strategy = "group",
+):
+ from compressed_tensors.compressors import BaseCompressor
+ from compressed_tensors.quantization import QuantizationScheme, QuantizationArgs
+ from compressed_tensors.quantization.utils import calculate_qparams
+
+ args = QuantizationArgs(
+ num_bits = bits,
+ type = "int",
+ symmetric = symmetric,
+ strategy = strategy,
+ group_size = group_size if strategy == "group" else None,
+ actorder = "weight" if actorder else None,
+ )
+ if actorder and not CT_HONOURS_G_IDX:
+ pytest.skip("this compressed-tensors cannot write or decompress activation-ordered weights")
+ scheme = QuantizationScheme(targets = ["Linear"], weights = args)
+ w = torch.randn(out_f, in_f) * 0.05
+ gs = group_size if strategy == "group" else in_f
+ g_idx = None
+ if actorder:
+ g_idx = (torch.randperm(in_f) // gs).to(torch.int32)
+ grouped = w[:, torch.argsort(g_idx)].reshape(out_f, in_f // gs, gs)
+ else:
+ grouped = w.reshape(out_f, in_f // gs, gs)
+ scale, zp = calculate_qparams(grouped.amin(-1), grouped.amax(-1), args)
+ state = {"weight": w, "weight_scale": scale.to(scale_dtype)}
+ if not symmetric:
+ state["weight_zero_point"] = zp.to(torch.int8)
+ if g_idx is not None:
+ state["weight_g_idx"] = g_idx
+ comp = BaseCompressor.get_value_from_registry("pack-quantized")
+ packed = comp.compress(state, scheme)
+ ref = comp.decompress(dict(packed), scheme)["weight"].to(torch.bfloat16)
+ return {k: v.cuda() for k, v in packed.items()}, ref.cuda(), gs
+
+
+CASES = [
+ # bits, group, symmetric, actorder, scale dtype, strategy
+ (4, 128, True, False, torch.bfloat16, "group"),
+ (4, 64, False, False, torch.bfloat16, "group"),
+ (4, 128, False, True, torch.bfloat16, "group"),
+ (4, 32, True, False, torch.float32, "group"),
+ (8, 128, True, False, torch.bfloat16, "group"),
+ (8, 64, False, False, torch.float16, "group"),
+ (2, 64, True, False, torch.bfloat16, "group"),
+ (4, None, True, False, torch.bfloat16, "channel"),
+]
+
+
+@needs_gpu
+@needs_ct
[email protected]("bits,group,sym,actorder,scale_dtype,strategy", CASES)
+def test_kernels_decode_bit_exact_and_multiply_like_dense(
+ bits, group, sym, actorder, scale_dtype, strategy, monkeypatch
+):
+ from unsloth.kernels.int4_packed import (
+ Int4QuantState,
+ int4_dequantize,
+ int4_dequantize_weight,
+ int4_matmul,
+ int4_matmul_t,
+ )
+
+ torch.manual_seed(0)
+ out_f, in_f = 200, 512 # out_f not a multiple of the zero-point packing
+ packed, ref, gs = _packed_layer(out_f, in_f, bits, group, sym, actorder, scale_dtype, strategy)
+ qs = Int4QuantState(
+ packed["weight_scale"],
+ packed.get("weight_zero_point"),
+ packed.get("weight_g_idx"),
+ (out_f, in_f),
+ bits,
+ gs,
+ torch.bfloat16,
+ )
+ W = packed["weight_packed"]
+ assert torch.equal(int4_dequantize(W, qs), ref)
+ assert torch.equal(int4_dequantize_weight(W.t(), qs), ref.t())
+ import unsloth.kernels.int4_packed as ip
+
+ for rows, gemv in itertools.product((1, 3, 5, 64), (True, False)):
+ monkeypatch.setattr(ip, "GEMV_MAX_ROWS", 1 << 30 if gemv else 0)
+ x = torch.randn(rows, in_f, device = "cuda", dtype = torch.bfloat16)
+ y = int4_matmul(x, W, qs)
+ want = x.float() @ ref.float().t()
+ assert y.shape == (rows, out_f) and y.dtype == torch.bfloat16
+ assert ((y.float() - want).norm() / want.norm()) < 5e-3
+ dy = torch.randn(rows, out_f, device = "cuda", dtype = torch.bfloat16)
+ dx = int4_matmul_t(dy, W, qs)
+ want = dy.float() @ ref.float()
+ assert ((dx.float() - want).norm() / want.norm()) < 5e-3
+
+
+def _has_marlin():
+ if not has_real_cuda():
+ return False
+ import unsloth.kernels.int4_packed as ip
+ return bool(ip._marlin_api())
+
+
+needs_sm80 = pytest.mark.skipif(
+ not has_real_cuda() or torch.cuda.get_device_capability() < (8, 0),
+ reason = "tensor-core layouts need sm_80+",
+)
+
+
+def _qs(
+ ip,
+ packed,
+ shape,
+ bits,
+ gs,
+ dtype = torch.bfloat16,
+):
+ return ip.Int4QuantState(
+ packed["weight_scale"], packed.get("weight_zero_point"), None, shape, bits, gs, dtype
+ )
+
+
+REPACK_CASES = [
+ (128, True, 256),
+ (128, False, 512),
+ (32, False, 256),
+ (64, True, 1024),
+ (256, False, 512),
+]
+
+
+@needs_gpu
+@needs_ct
+@needs_sm80
[email protected]("layout", ["tinygemm", "marlin"])
[email protected]("group,sym,out_f", REPACK_CASES)
+def test_repacked_weights_dequantize_exactly_and_save_in_checkpoint_layout(
+ layout, group, sym, out_f, monkeypatch
+):
+ import unsloth.kernels.int4_packed as ip
+
+ if layout == "marlin" and (not _has_marlin() or group == 256):
+ pytest.skip("needs vLLM's Marlin kernels (groups <= 128)")
+ monkeypatch.setenv("UNSLOTH_INT4_LAYOUT", layout)
+ torch.manual_seed(0)
+ packed, ref, gs = _packed_layer(out_f, 1024, 4, group, sym, False, torch.bfloat16)
+ W = packed["weight_packed"]
+ checkpoint = W.clone()
+ qs = _qs(ip, packed, (out_f, 1024), 4, gs)
+ assert ip.int4_repack_(W, qs) == layout
+ # Replaced in place (no second copy), same shape; training dequantizes the exact same weights; saves unchanged.
+ assert W.shape == checkpoint.shape and not torch.equal(W, checkpoint)
+ assert torch.equal(ip.int4_dequantize(W, qs), ref)
+ assert torch.equal(ip.int4_unpack(W, qs), checkpoint)
+ fast = ip._fast_rows(layout, torch.cuda.current_device())
+ for rows in (1, 3, fast, fast + 1):
+ x = torch.randn(rows, 1024, device = "cuda", dtype = torch.bfloat16)
+ with torch.no_grad():
+ y = ip.int4_matmul(x, W, qs)
+ want = x.float() @ ref.float().t()
+ assert ((y.float() - want).norm() / want.norm()) < 5e-3
+ out = torch.empty(1, 1, out_f, device = "cuda", dtype = torch.bfloat16)
+ with torch.no_grad():
+ got = ip.int4_matmul(x[:1].view(1, 1, 1024), W, qs, out = out)
+ want = ip.int4_matmul(x[:1], W, qs)
+ assert got.data_ptr() == out.data_ptr() and torch.equal(got.view(1, -1), want)
+ # With grad enabled every row count decodes + cuBLAS: training numerics are those of the checkpoint layout.
+ assert torch.equal(ip.int4_matmul(x, W, qs), x @ ref.t())
+
+
+@needs_gpu
+@needs_ct
+ "why", ["8bit", "fp32_scale", "fp16_without_marlin", "channel", "kill_switch", "layout_changed"]
+)
+def test_layers_that_cannot_be_repacked_keep_the_checkpoint_layout(why, monkeypatch):
+ import unsloth.kernels.int4_packed as ip
+
+ monkeypatch.setattr(ip, "_MARLIN_API", False)
+ if why == "kill_switch":
+ monkeypatch.setenv("UNSLOTH_INT4_REPACK", "0")
+ if why == "layout_changed":
+ # A torch release that changes tinygemm's tile layout must fail the probe, not corrupt weights.
+ TN, TK, nfast, bits = ip._LAYOUTS["tinygemm"]
+ monkeypatch.setitem(ip._LAYOUTS, "tinygemm", (TN, TK, nfast, bits[::-1]))
+ monkeypatch.setattr(ip, "_LAYOUT_OK", {})
+ bits = 8 if why == "8bit" else 4
+ scale_dtype = {"fp32_scale": torch.float32, "fp16_without_marlin": torch.float16}.get(
+ why, torch.bfloat16
+ )
+ dtype = torch.float16 if why == "fp16_without_marlin" else torch.bfloat16
+ strategy = "channel" if why == "channel" else "group"
+ torch.manual_seed(0)
+ packed, ref, gs = _packed_layer(256, 512, bits, 128, True, False, scale_dtype, strategy)
+ W = packed["weight_packed"]
+ checkpoint = W.clone()
+ qs = _qs(ip, packed, (256, 512), bits, gs, dtype)
+ assert ip.int4_repack_(W, qs) is None and qs.layout is None and torch.equal(W, checkpoint)
+ x = torch.randn(2, 512, device = "cuda", dtype = dtype)
+ with torch.no_grad():
+ y = ip.int4_matmul(x, W, qs)
+ want = x.float() @ ref.float().t()
+ assert ((y.float() - want).norm() / want.norm()) < 5e-3
+
+
+@needs_gpu
+@needs_ct
+@needs_sm80
+def test_packed_linear_state_dict_round_trips_through_the_checkpoint_layout():
+ from unsloth.models.compressed_tensors_int4 import (
+ Int4PackedLinear,
+ finalize_int4_packed_linears,
+ )
+ import unsloth.kernels.int4_packed as ip
+
+ torch.manual_seed(0)
+ packed, ref, gs = _packed_layer(256, 1024, 4, 128, False, False, torch.bfloat16)
+ lin = Int4PackedLinear.__new__(Int4PackedLinear)
+ torch.nn.Module.__init__(lin)
+ lin.in_features, lin.out_features = 1024, 256
+ for name, tensor in packed.items():
+ lin.register_parameter(name, torch.nn.Parameter(tensor.clone(), requires_grad = False))
+ lin.register_parameter("bias", None)
+ lin._int4_bits, lin._int4_group_size = 4, gs
+ model = torch.nn.Sequential(lin)
+ finalize_int4_packed_linears(model, torch.bfloat16)
+ assert lin.quant_state.layout is not None
+ assert not torch.equal(lin.weight_packed, packed["weight_packed"])
+ state = model.state_dict()
+ assert torch.equal(state["0.weight_packed"], packed["weight_packed"])
+ assert torch.equal(lin.dequantize_weight(), ref)
+ # Training takes the exact dequantize + matmul whatever the layout; only inference uses the fused kernel.
+ x = torch.randn(4, 1024, device = "cuda", dtype = torch.bfloat16, requires_grad = True)
+ assert torch.equal(lin(x), x @ ref.t())
+ # Loading checkpoint-layout words resets the layer to that layout; unrelated loads leave it alone.
+ model.load_state_dict({}, strict = False)
+ assert lin.quant_state.layout is not None
+ model.load_state_dict(state)
+ assert lin.quant_state.layout is None and torch.equal(lin.dequantize_weight(), ref)
+
+
+def _repacked_linear():
+ from unsloth.models.compressed_tensors_int4 import (
+ Int4PackedLinear,
+ finalize_int4_packed_linears,
+ )
+
+ torch.manual_seed(0)
+ packed, ref, gs = _packed_layer(256, 1024, 4, 128, False, False, torch.bfloat16)
+ lin = Int4PackedLinear.__new__(Int4PackedLinear)
+ torch.nn.Module.__init__(lin)
+ lin.in_features, lin.out_features = 1024, 256
+ for name, tensor in packed.items():
+ lin.register_parameter(name, torch.nn.Parameter(tensor.clone(), requires_grad = False))
+ lin.register_parameter("bias", None)
+ lin._int4_bits, lin._int4_group_size = 4, gs
+ finalize_int4_packed_linears(torch.nn.Sequential(lin), torch.bfloat16)
+ assert lin.quant_state.layout is not None
+ return lin, packed, ref
+
+
+@needs_gpu
+@needs_ct
+@needs_sm80
[email protected]("cast", ["float", "half"])
+def test_repacked_linear_infers_after_a_dtype_cast(cast):
+ # The fused layout was picked for bf16; after a cast inference must take the exact dequantize path.
+ lin, _, ref = _repacked_linear()
+ getattr(lin, cast)()
+ dtype = lin.quant_state.dtype
+ x = torch.randn(2, 1024, device = "cuda", dtype = dtype)
+ with torch.no_grad():
+ y = lin(x)
+ want = x.float() @ ref.float().t()
+ assert y.dtype == dtype and ((y.float() - want).norm() / want.norm()) < 5e-3
+
+
+@needs_gpu
+@needs_ct
+@needs_sm80
+def test_adapter_saves_skip_unpacking_repacked_words(monkeypatch, tmp_path):
+ from peft import LoraConfig, get_peft_model
+ import unsloth.kernels.int4_packed as ip
+
+ lin, packed, _ = _repacked_linear()
+
+ class Tiny(torch.nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.proj = lin
+ self.head = torch.nn.Linear(256, 8, device = "cuda", dtype = torch.bfloat16)
+
+ model = get_peft_model(Tiny(), LoraConfig(r = 4, target_modules = ["head"]))
+ calls = []
+ real = ip.int4_unpack
+ monkeypatch.setattr(ip, "int4_unpack", lambda *a: calls.append(1) or real(*a))
+ model.save_pretrained(tmp_path)
+ assert calls == [] and (tmp_path / "adapter_model.safetensors").exists()
+ # A plain state_dict() still writes the checkpoint layout.
+ assert torch.equal(
+ model.state_dict()["base_model.model.proj.weight_packed"], packed["weight_packed"]
+ )
+ assert calls == [1]
+
+
+@needs_gpu
+@needs_ct
+def test_jit_launch_fallback_matches_the_compiled_launcher(monkeypatch):
+ # Triton < 3.7 launchers reject CompiledKernel[grid](*runtime_args); the JIT launch must give the same bits.
+ import unsloth.kernels.int4_packed as ip
+
+ torch.manual_seed(0)
+ packed, ref, gs = _packed_layer(200, 512, 4, 128, True, False, torch.bfloat16, "group")
+ W = packed["weight_packed"]
+
+ def run():
+ qs = ip.Int4QuantState(
+ packed["weight_scale"], None, None, (200, 512), 4, gs, torch.bfloat16
+ )
+ x = torch.randn(
+ 3,
+ 512,
+ device = "cuda",
+ dtype = torch.bfloat16,
+ generator = torch.Generator("cuda").manual_seed(1),
+ )
+ return ip.int4_dequantize(W, qs), ip.int4_matmul(x, W, qs)
+
+ fast = run()
+ monkeypatch.setattr(ip, "_FAST_LAUNCH", False)
+ jit = run()
+ assert torch.equal(fast[0], ref) and torch.equal(jit[0], ref)
+ assert torch.equal(fast[1], jit[1])
+
+
+@needs_gpu
+ not (HAS_CT and HAS_CONVERTERS), reason = "needs compressed-tensors and the transformers 5 loader"
+)
[email protected]("variant", ["symmetric", "asymmetric", "actorder", "int8"])
[email protected]("restore_llama_patches")
+def test_packed_route_keeps_the_checkpoint_weights_exactly(variant, tmp_path, monkeypatch):
+ from unsloth import FastLanguageModel
+ from unsloth.models.compressed_tensors_int4 import Int4PackedLinear
+ from test_compressed_tensors_bnb import _tokenizer_free_load, _write_tiny_packed_llama
+
+ monkeypatch.setenv("UNSLOTH_COMPRESSED_TENSORS_INT4", "packed")
+ packed_dir, bf16_dir = _write_tiny_packed_llama(
+ str(tmp_path),
+ asymmetric = variant == "asymmetric",
+ actorder = variant == "actorder",
+ num_bits = 8 if variant == "int8" else 4,
+ )
+ for d in (packed_dir, bf16_dir):
+ _tokenizer_free_load(d, str(tmp_path))
+ kw = dict(max_seq_length = 64, dtype = torch.bfloat16)
+ model_a, _ = FastLanguageModel.from_pretrained(packed_dir, load_in_4bit = True, **kw)
+ model_b, _ = FastLanguageModel.from_pretrained(
+ bf16_dir, load_in_4bit = False, load_in_16bit = True, **kw
+ )
+ dense = dict(model_b.named_modules())
+ n = 0
+ for name, m in model_a.named_modules():
+ if isinstance(m, Int4PackedLinear):
+ assert torch.equal(m.dequantize_weight(), dense[name].weight), name
+ assert m.weight.dtype == torch.int32 and m.weight.quant_state is m.quant_state
+ assert not any(p.requires_grad for p in m.parameters())
+ n += 1
+ assert n == 2 * 7, n
+ ids = torch.randint(0, 256, (1, 16), device = "cuda:0")
+ with torch.no_grad():
+ a = model_a(input_ids = ids).logits.float()
+ b = model_b(input_ids = ids).logits.float()
+ assert (a - b).abs().max() < 1e-2 * b.abs().max()
+ lin = next(m for m in model_a.modules() if isinstance(m, Int4PackedLinear))
+ scale = lin.weight_scale.clone()
+ lin.to(torch.float16)
+ assert torch.equal(lin.weight_scale, scale) and lin.quant_state.dtype == torch.float16
+
+
+@needs_gpu
+ not (HAS_CT and HAS_CONVERTERS), reason = "needs compressed-tensors and the transformers 5 loader"
+)
[email protected]("restore_llama_patches")
+def test_packed_route_lora_trains_merges_and_unmerges(tmp_path, monkeypatch):
+ from unsloth import FastLanguageModel
+ from unsloth.models.compressed_tensors_int4 import Int4PackedLinear
+ from test_compressed_tensors_bnb import _tokenizer_free_load, _write_tiny_packed_llama
+
+ monkeypatch.setenv("UNSLOTH_COMPRESSED_TENSORS_INT4", "packed")
+ packed_dir, _ = _write_tiny_packed_llama(str(tmp_path), asymmetric = True)
+ _tokenizer_free_load(packed_dir, str(tmp_path))
+ model, _ = FastLanguageModel.from_pretrained(
+ packed_dir, max_seq_length = 64, dtype = torch.bfloat16, load_in_4bit = True
+ )
+ model = FastLanguageModel.get_peft_model(
+ model,
+ r = 4,
+ lora_alpha = 8,
+ target_modules = [
+ "q_proj",
+ "k_proj",
+ "v_proj",
+ "o_proj",
+ "gate_proj",
+ "up_proj",
+ "down_proj",
+ ],
+ )
+ model.train()
+ ids = torch.randint(0, 256, (2, 16), device = "cuda:0")
+ loss = model(input_ids = ids, labels = ids).loss
+ loss.backward()
+ grads = [p.grad for n, p in model.named_parameters() if "lora_B" in n]
+ assert len(grads) == 14 and all(g is not None and torch.isfinite(g).all() for g in grads)
+ assert all(g.abs().sum() > 0 for g in grads)
+ layer = model.base_model.model.model.layers[0].self_attn.q_proj
+ base = layer.get_base_layer()
+ packed = base.weight_packed
+ before = base.dequantize_weight()
+ with torch.no_grad():
+ layer.lora_B["default"].weight.normal_()
+ layer.merge()
+ assert type(layer.get_base_layer()) is torch.nn.Linear
+ assert not torch.equal(layer.get_base_layer().weight, before)
+ layer.unmerge()
+ assert isinstance(layer.get_base_layer(), Int4PackedLinear)
+ assert layer.get_base_layer().weight_packed is packed
+
+
+@needs_ct
+def test_adopt_swaps_plain_linears_and_leaves_routers_to_the_decompress_converter(tmp_path):
+ import re
+
+ from safetensors.torch import save_file
+ from torch import nn
+ from unsloth.models.compressed_tensors_bnb import (
+ _PACKED_SUFFIXES,
+ _build_quantization_config,
+ adopt_int4_packed_linears,
+ )
+ from unsloth.models.compressed_tensors_int4 import Int4PackedLinear
+ from test_compressed_tensors_bnb import _w4a16
+
+ class Router(nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.weight = nn.Parameter(torch.empty(4, 64))
+
+ with torch.device("meta"):
+ model = nn.Module()
+ model.proj = nn.Linear(64, 16, bias = False)
+ model.gate = Router()
+ tensors = {}
+ for name, rows in (("proj", 16), ("gate", 4), ("mlp.experts.0.up", 8)):
+ tensors[f"{name}.weight_packed"] = torch.zeros(rows, 8, dtype = torch.int32)
+ tensors[f"{name}.weight_scale"] = torch.ones(rows, 2, dtype = torch.float32)
+ tensors[f"{name}.weight_shape"] = torch.tensor([rows, 64])
+ path = str(tmp_path / "model.safetensors")
+ save_file(tensors, path)
+ ct_config = _build_quantization_config(_w4a16())
+ skipped, _ = adopt_int4_packed_linears(
+ copy.deepcopy(model), ct_config, [path], torch.bfloat16, skip_modules = ["proj"]
+ )
+ assert skipped == [] # a caller-skipped module stays with the decompress converter
+ swapped, leftover = adopt_int4_packed_linears(model, ct_config, [path], torch.bfloat16)
+ assert swapped == ["proj"] and leftover == ["gate"]
+ # The stacked expert is not in the checkpoint's layout any more, so a full save must not claim it is.
+ assert model.__dict__.get("_unsloth_int4_stacked_experts") is True
+ assert isinstance(model.proj, Int4PackedLinear)
+ assert model.proj.weight_packed.dtype == torch.int32 and model.proj.weight_packed.shape == (
+ 16,
+ 8,
+ )
+ assert model.proj.weight_scale.dtype == torch.float32
+ assert type(model.gate) is Router
+ # The leftover converter pattern renames only the router's keys, and only the suffix.
+ patterns = [f"(?<={re.escape(n)}\\.){s}$" for n in leftover for s in _PACKED_SUFFIXES]
+ rx = re.compile("|".join(patterns))
+ m = rx.search("gate.weight_packed")
+ assert m and "gate.weight_packed".replace(m.group(0), "weight", 1) == "gate.weight"
+ assert rx.search("proj.weight_packed") is None
+
+
+@needs_gpu
+ not (HAS_CT and HAS_CONVERTERS), reason = "needs compressed-tensors and the transformers 5 loader"
+)
[email protected]("restore_llama_patches")
+def test_a_full_save_of_the_packed_route_reloads(tmp_path, monkeypatch):
+ # The saved tensors stay packed, so the saved config must be the checkpoint's, not the runtime bnb one.
+ import json
+ from transformers import AutoModelForCausalLM
+ from unsloth import FastLanguageModel
+ from test_compressed_tensors_bnb import _tokenizer_free_load, _write_tiny_packed_llama
+
+ monkeypatch.setenv("UNSLOTH_COMPRESSED_TENSORS_INT4", "packed")
+ packed_dir, _ = _write_tiny_packed_llama(str(tmp_path))
+ _tokenizer_free_load(packed_dir, str(tmp_path))
+ kw = dict(max_seq_length = 64, dtype = torch.bfloat16)
+ model, tokenizer = FastLanguageModel.from_pretrained(packed_dir, load_in_4bit = True, **kw)
+ runtime = model.config.quantization_config
+ out = str(tmp_path / "saved")
+ model.save_pretrained(out)
+ tokenizer.save_pretrained(out)
+ assert model.config.quantization_config is runtime
+ saved = json.load(open(f"{out}/config.json"))["quantization_config"]
+ assert saved["quant_method"] == "compressed-tensors"
+ ids = torch.randint(0, 256, (1, 16), device = "cuda:0")
+ with torch.no_grad():
+ want = model(input_ids = ids).logits.float()
+ again, _ = FastLanguageModel.from_pretrained(out, load_in_4bit = True, **kw)
+ assert torch.equal(again(input_ids = ids).logits.float(), want)
+ plain = AutoModelForCausalLM.from_pretrained(out, dtype = torch.bfloat16, device_map = {"": 0})
+ got = plain(input_ids = ids).logits.float()
+ assert (got - want).abs().max() < 1e-2 * want.abs().max()
+ del plain, again
+ merged = FastLanguageModel.get_peft_model(
+ model, r = 4, target_modules = ["q_proj"]
+ ).merge_and_unload()
+ merged.save_pretrained(str(tmp_path / "merged"))
+ tokenizer.save_pretrained(str(tmp_path / "merged"))
+ # A permanent merge must not keep the packed weights stashed for an unmerge that can never come.
+ assert not any("_unsloth_int4_packed_state" in m.__dict__ for m in merged.modules())
+ ignore = json.load(open(f"{tmp_path}/merged/config.json"))["quantization_config"]["ignore"]
+ assert (
+ "model.layers.0.self_attn.q_proj" in ignore
+ and "model.layers.0.self_attn.k_proj" not in ignore
+ )
+ with torch.no_grad():
+ want = merged(input_ids = ids).logits.float()
+ plain = AutoModelForCausalLM.from_pretrained(
+ str(tmp_path / "merged"), dtype = torch.bfloat16, device_map = {"": 0}
+ )
+ got = plain(input_ids = ids).logits.float()
+ assert (got - want).abs().max() < 1e-2 * want.abs().max()
+
+
+@needs_gpu
+@needs_ct
[email protected]("grad", [True, False])
[email protected]("lead", [(0,), (2, 0)])
+def test_an_empty_batch_returns_an_empty_output(grad, lead):
+ from unsloth.kernels.int4_packed import Int4QuantState, int4_matmul
+
+ packed, _, gs = _packed_layer(200, 512, 4, 128, True, False, torch.bfloat16)
+ qs = Int4QuantState(packed["weight_scale"], None, None, (200, 512), 4, gs, torch.bfloat16)
+ x = torch.randn(*lead, 512, device = "cuda", dtype = torch.bfloat16)
+ with torch.set_grad_enabled(grad):
+ assert int4_matmul(x, packed["weight_packed"], qs).shape == (*lead, 200)
+
+
+def test_a_mixed_packed_layout_refuses_a_full_save():
+ # Packed layers beside converted experts: no single saved config could reload them.
+ from torch import nn
+ from unsloth.models.compressed_tensors_int4 import (
+ Int4PackedLinear,
+ refuse_mixed_packed_full_save,
+ )
+
+ class Model(nn.Module):
+ def save_pretrained(self, *args, **kwargs):
+ return "saved"
+
+ model = Model()
+ layer = Int4PackedLinear.__new__(Int4PackedLinear)
+ nn.Module.__init__(layer)
+ model.layer = layer
+ refuse_mixed_packed_full_save(model)
+ with pytest.raises(RuntimeError, match = "full save is refused"):
+ model.save_pretrained("out")
+ layer.__class__ = nn.Linear # what a LoRA merge's densify leaves behind
+ assert model.save_pretrained("out") == "saved"
+
+
+@needs_gpu
+@needs_ct
+def test_training_single_row_forward_uses_the_backward_weights():
+ # fast = False is the autograd forward: it must multiply the same rounded weights int4_matmul_t uses.
+ from unsloth.kernels.int4_packed import Int4QuantState, int4_dequantize, int4_matmul
+
+ packed, _, gs = _packed_layer(200, 512, 4, 128, True, True, torch.bfloat16)
+ qs = Int4QuantState(
+ packed["weight_scale"], None, packed.get("weight_g_idx"), (200, 512), 4, gs, torch.bfloat16
+ )
+ W = packed["weight_packed"]
+ x = torch.randn(1, 512, device = "cuda", dtype = torch.bfloat16)
+ want = x @ int4_dequantize(W, qs, torch.bfloat16).t()
+ assert torch.equal(int4_matmul(x, W, qs, fast = False), want)
diff --git a/tests/test_remote_gradient_checkpointing.py b/tests/test_remote_gradient_checkpointing.py
new file mode 100644
index 00000000000..81ce59d9578
--- /dev/null
+++ b/tests/test_remote_gradient_checkpointing.py
@@ -0,0 +1,224 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved.
+
+"""Remote decoder loops that never call a checkpoint function (Kimi-K2.7's DeepSeek-V3 port)
+must still checkpoint once gradient checkpointing is enabled.
+
+The backbone below mirrors that file: it declares `gradient_checkpointing` and support for it,
+but its loop calls every layer directly, so enabling the flag used to change nothing.
+"""
+
+import sys
+
+import pytest
+import torch
+import torch.nn as nn
+
+transformers = pytest.importorskip("transformers")
+from transformers import PretrainedConfig, PreTrainedModel
+
+from unsloth.models.loader_utils import install_remote_gradient_checkpointing
+
+REMOTE = "transformers_modules.some_repo.modeling_x"
+
+
+class RemoteLayer(nn.Module):
+ calls = 0
+
+ def __init__(self, hidden):
+ super().__init__()
+ self.up = nn.Linear(hidden, 4 * hidden)
+ self.down = nn.Linear(4 * hidden, hidden)
+
+ def forward(
+ self,
+ hidden_states,
+ attention_mask = None,
+ past_key_value = None,
+ use_cache = False,
+ ):
+ type(self).calls += 1
+ return (hidden_states + self.down(torch.relu(self.up(hidden_states))),)
+
+
+class RemoteBackbone(PreTrainedModel):
+ config_class = PretrainedConfig
+ supports_gradient_checkpointing = True
+
+ def __init__(
+ self,
+ config,
+ layer_cls = RemoteLayer,
+ ):
+ super().__init__(config)
+ self.layers = nn.ModuleList(layer_cls(16) for _ in range(3))
+ self.gradient_checkpointing = False
+
+ def forward(
+ self,
+ hidden_states,
+ past_key_values = None,
+ ):
+ for layer in self.layers:
+ hidden_states = layer(
+ hidden_states, attention_mask = None, past_key_value = past_key_values
+ )[0]
+ return hidden_states
+
+
+for _cls in (RemoteLayer, RemoteBackbone):
+ _cls.__module__ = REMOTE
+# transformers reads the defining module's source for some class checks.
+sys.modules.setdefault(REMOTE, sys.modules[__name__])
+
+
+def _build(layer_cls = RemoteLayer):
+ torch.manual_seed(0)
+ return RemoteBackbone(PretrainedConfig(), layer_cls = layer_cls)
+
+
+def _step(model, **kwargs):
+ model.train()
+ x = torch.randn(2, 8, 16, requires_grad = True)
+ before = type(model.layers[0]).calls
+ model(x, **kwargs).square().sum().backward()
+ grads = [p.grad.clone() for p in model.parameters()]
+ model.zero_grad()
+ return type(model.layers[0]).calls - before, grads
+
+
+def test_the_flag_really_checkpoints_a_remote_loop():
+ reference = _build()
+ install_remote_gradient_checkpointing(reference, verbose = False)
+ calls_off, grads_off = _step(reference)
+ assert calls_off == 3
+
+ model = _build()
+ install_remote_gradient_checkpointing(model, verbose = False)
+ assert RemoteLayer.__dict__["forward"]._unsloth_manual_checkpoint
+ model.gradient_checkpointing_enable(gradient_checkpointing_kwargs = {"use_reentrant": True})
+ calls_on, grads_on = _step(model)
+ assert calls_on == 6 # every layer recomputed in the backward
+ assert all(torch.allclose(a, b) for a, b in zip(grads_off, grads_on))
+
+
+def test_the_installed_checkpoint_function_is_used():
+ model = _build()
+ install_remote_gradient_checkpointing(model, verbose = False)
+ model.gradient_checkpointing_enable()
+ seen = []
+ original = model._gradient_checkpointing_func
+
+ def spy(function, *args):
+ seen.append(len(args))
+ return original(function, *args)
+
+ for module in model.modules():
+ if hasattr(module, "gradient_checkpointing"):
+ module._gradient_checkpointing_func = spy
+ _step(model)
+ assert seen == [1, 1, 1]
+
+
+def test_a_cache_is_never_recomputed():
+ model = _build()
+ install_remote_gradient_checkpointing(model, verbose = False)
+ model.gradient_checkpointing_enable()
+ calls, _ = _step(model, past_key_values = object())
+ assert calls == 3
+
+
+def test_a_loop_that_already_checkpoints_is_left_alone():
+ class CheckpointingBackbone(RemoteBackbone):
+ def forward(
+ self,
+ hidden_states,
+ past_key_values = None,
+ ):
+ for layer in self.layers:
+ hidden_states = self._gradient_checkpointing_func(layer.__call__, hidden_states)[0]
+ return hidden_states
+
+ CheckpointingBackbone.__module__ = REMOTE
+ torch.manual_seed(0)
+ model = CheckpointingBackbone(PretrainedConfig())
+ assert install_remote_gradient_checkpointing(model, verbose = False) == []
+
+
+def test_native_models_are_never_wrapped():
+ class NativeLayer(nn.Module):
+ def __init__(self, hidden):
+ super().__init__()
+ self.up = nn.Linear(hidden, hidden)
+
+ def forward(self, hidden_states, **kwargs):
+ return (self.up(hidden_states),)
+
+ class NativeBackbone(RemoteBackbone):
+ pass
+
+ native = "transformers.models.x.modeling_x"
+ NativeLayer.__module__ = NativeBackbone.__module__ = native
+ sys.modules.setdefault(native, sys.modules[__name__])
+ model = NativeBackbone(PretrainedConfig(), layer_cls = NativeLayer)
+ assert install_remote_gradient_checkpointing(model, verbose = False) == []
+ assert not hasattr(NativeLayer.forward, "_unsloth_manual_checkpoint")
+
+
+class ResidualLayer(RemoteLayer):
+ """Kimi-K3's attention-residual layers: a second tensor that needs a gradient is carried
+ from layer to layer as a keyword."""
+
+ def forward(
+ self,
+ hidden_states,
+ attention_mask = None,
+ past_key_value = None,
+ block_residual = None,
+ ):
+ type(self).calls += 1
+ out = hidden_states + self.down(torch.relu(self.up(hidden_states + block_residual)))
+ return out, block_residual + out
+
+
+class ResidualBackbone(RemoteBackbone):
+ def forward(
+ self,
+ hidden_states,
+ past_key_values = None,
+ ):
+ block_residual = hidden_states * 0.5
+ for layer in self.layers:
+ hidden_states, block_residual = layer(
+ hidden_states,
+ attention_mask = None,
+ past_key_value = past_key_values,
+ block_residual = block_residual,
+ )
+ return hidden_states + block_residual
+
+
+for _cls in (ResidualLayer, ResidualBackbone):
+ _cls.__module__ = REMOTE
+
+
[email protected]("use_reentrant", [True, False])
+def test_a_keyword_tensor_carried_across_layers_is_recomputed_once(use_reentrant):
+ def build():
+ torch.manual_seed(0)
+ return ResidualBackbone(PretrainedConfig(), layer_cls = ResidualLayer)
+
+ reference = build()
+ install_remote_gradient_checkpointing(reference, verbose = False)
+ calls_off, grads_off = _step(reference)
+ assert calls_off == 3
+
+ model = build()
+ install_remote_gradient_checkpointing(model, verbose = False)
+ model.gradient_checkpointing_enable(
+ gradient_checkpointing_kwargs = {"use_reentrant": use_reentrant}
+ )
+ calls_on, grads_on = _step(model)
+ # A closure-held block_residual tied each recompute to earlier layers' graphs.
+ assert calls_on == 6
+ assert all(torch.allclose(a, b, atol = 1e-6) for a, b in zip(grads_off, grads_on))
diff --git a/tests/test_remote_moe_shims.py b/tests/test_remote_moe_shims.py
new file mode 100644
index 00000000000..2fad9243639
--- /dev/null
+++ b/tests/test_remote_moe_shims.py
@@ -0,0 +1,448 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Training forwards for remote DeepSeek-V3-style MoE code (Kimi-K2.7) and the gradient
+checkpointing flag on a remote composition. Runs offline on CPU with a synthetic copy of the
+port's `MoEGate` / `DeepseekV3MoE` idioms defined in a `transformers_modules` namespace."""
+
+import math
+import os
+import sys
+import tempfile
+import types
+
+import pytest
+import torch
+from torch import nn
+from torch.nn import functional as F
+
+# Import unsloth first to set UNSLOTH_IS_PRESENT env var.
+import unsloth # noqa: F401
+from unsloth.models.remote_moe_shims import (
+ is_remote_deepseek_gate,
+ is_remote_deepseek_moe,
+ prepare_remote_moe_for_training,
+)
+from unsloth.models.loader_utils import enable_composite_gradient_checkpointing
+
+
+def _remote_module(name = "transformers_modules.tiny_kimi.modeling_deepseek", edits = ()):
+ """The port's classes, verbatim in the parts that matter, under a remote-code module name.
+ ``edits`` are (old, new) source replacements for ports derived from it."""
+ src = """
+import math, torch
+import torch.nn.functional as F
+from torch import nn
+
+class Cfg:
+ def __init__(self):
+ self.hidden_size = 16; self.n_routed_experts = 8; self.num_experts_per_tok = 2
+ self.moe_intermediate_size = 8; self.n_shared_experts = 1; self.routed_scaling_factor = 2.5
+ self.scoring_func = "sigmoid"; self.seq_aux = True; self.topk_method = "noaux_tc"
+ self.n_group = 1; self.topk_group = 1; self.norm_topk_prob = True; self.hidden_act = "silu"
+
+class MLP(nn.Module):
+ def __init__(self, config, intermediate_size):
+ super().__init__()
+ self.gate_proj = nn.Linear(config.hidden_size, intermediate_size, bias=False)
+ self.up_proj = nn.Linear(config.hidden_size, intermediate_size, bias=False)
+ self.down_proj = nn.Linear(intermediate_size, config.hidden_size, bias=False)
+ def forward(self, x):
+ return self.down_proj(F.silu(self.gate_proj(x)) * self.up_proj(x))
+
+class MoEGate(nn.Module):
+ def __init__(self, config):
+ super().__init__()
+ self.config = config; self.top_k = config.num_experts_per_tok; self.n_routed_experts = config.n_routed_experts
+ self.routed_scaling_factor = config.routed_scaling_factor; self.scoring_func = config.scoring_func
+ self.topk_method = config.topk_method; self.n_group = config.n_group; self.topk_group = config.topk_group
+ self.norm_topk_prob = config.norm_topk_prob; self.gating_dim = config.hidden_size
+ self.weight = nn.Parameter(torch.empty((self.n_routed_experts, self.gating_dim)))
+ self.e_score_correction_bias = nn.Parameter(torch.zeros((self.n_routed_experts)))
+ nn.init.kaiming_uniform_(self.weight, a=math.sqrt(5))
+ def forward(self, hidden_states):
+ bsz, seq_len, h = hidden_states.shape
+ hidden_states = hidden_states.view(-1, h)
+ logits = F.linear(hidden_states.type(torch.float32), self.weight.type(torch.float32), None)
+ scores = logits.sigmoid()
+ assert not self.training
+ scores_for_choice = scores.view(bsz * seq_len, -1) + self.e_score_correction_bias.unsqueeze(0)
+ group_scores = scores_for_choice.view(bsz * seq_len, self.n_group, -1).topk(2, dim=-1)[0].sum(dim=-1)
+ group_idx = torch.topk(group_scores, k=self.topk_group, dim=-1, sorted=False)[1]
+ group_mask = torch.zeros_like(group_scores); group_mask.scatter_(1, group_idx, 1)
+ score_mask = group_mask.unsqueeze(-1).expand(bsz * seq_len, self.n_group, self.n_routed_experts // self.n_group).reshape(bsz * seq_len, -1)
+ tmp_scores = scores_for_choice.masked_fill(~score_mask.bool(), 0.0)
+ _, topk_idx = torch.topk(tmp_scores, k=self.top_k, dim=-1, sorted=False)
+ topk_weight = scores.gather(1, topk_idx)
+ if self.top_k > 1 and self.norm_topk_prob:
+ topk_weight = topk_weight / (topk_weight.sum(dim=-1, keepdim=True) + 1e-20)
+ return topk_idx, topk_weight * self.routed_scaling_factor
+
+class DeepseekV3MoE(nn.Module):
+ def __init__(self, config):
+ super().__init__()
+ self.config = config; self.num_experts_per_tok = config.num_experts_per_tok
+ self.ep_size = 1; self.experts_per_rank = config.n_routed_experts; self.ep_rank = 0
+ self.experts = nn.ModuleList([MLP(config, config.moe_intermediate_size) for _ in range(config.n_routed_experts)])
+ self.gate = MoEGate(config)
+ self.shared_experts = MLP(config, config.moe_intermediate_size * config.n_shared_experts)
+ def forward(self, hidden_states):
+ identity = hidden_states; orig_shape = hidden_states.shape
+ topk_idx, topk_weight = self.gate(hidden_states)
+ hidden_states = hidden_states.view(-1, hidden_states.shape[-1])
+ if not self.training:
+ y = self.moe_infer(hidden_states, topk_idx, topk_weight).view(*orig_shape)
+ if self.config.n_shared_experts is not None:
+ y = y + self.shared_experts(identity)
+ return y
+ @torch.no_grad()
+ def moe_infer(self, x, topk_ids, topk_weight):
+ cnts = topk_ids.new_zeros((topk_ids.shape[0], len(self.experts))); cnts.scatter_(1, topk_ids, 1)
+ tokens_per_expert = cnts.sum(dim=0); idxs = topk_ids.view(-1).argsort()
+ sorted_tokens = x[idxs // topk_ids.shape[1]]
+ outputs = []; start_idx = 0
+ for i, num_tokens in enumerate(tokens_per_expert.cpu().numpy()):
+ end_idx = start_idx + num_tokens
+ if num_tokens == 0: continue
+ outputs.append(self.experts[i](sorted_tokens[start_idx:end_idx])); start_idx = end_idx
+ outs = torch.cat(outputs, dim=0) if len(outputs) else sorted_tokens.new_empty(0)
+ new_x = torch.empty_like(outs); new_x[idxs] = outs
+ return (new_x.view(*topk_ids.shape, -1).type(topk_weight.dtype).mul_(topk_weight.unsqueeze(dim=-1)).sum(dim=1).type(new_x.dtype))
+"""
+ for before, after in edits:
+ assert before in src, before
+ src = src.replace(before, after)
+ mod = types.ModuleType(name)
+ # The shim predicate reads forward's source: register the exec'd fixture in linecache.
+ import linecache
+
+ filename = f"<{name}>"
+ linecache.cache[filename] = (len(src), None, src.splitlines(True), filename)
+ exec(compile(src, filename, "exec"), mod.__dict__)
+ sys.modules[name] = mod
+ for cls in (mod.MoEGate, getattr(mod, "DeepseekV3MoE", None) or mod.SarvamMLAMoE, mod.MLP):
+ cls.__module__ = name
+ return mod
+
+
+# sarvamai/sarvam-105b: renamed block, `else:` still calls `moe_infer`, `num_shared_experts` key.
+_SARVAM_EDITS = (
+ ("class DeepseekV3MoE(nn.Module):", "class SarvamMLAMoE(nn.Module):"),
+ (
+ " if not self.training:\n y = self.moe_infer(hidden_states, topk_idx, topk_weight).view(*orig_shape)\n",
+ " if not self.training:\n y = self.moe_infer(hidden_states, topk_idx, topk_weight).view(*orig_shape)\n"
+ " else:\n y = self.moe_infer(hidden_states, topk_idx, topk_weight).view(*orig_shape)\n",
+ ),
+ (
+ " if self.config.n_shared_experts is not None:",
+ " if self.shared_experts is not None:",
+ ),
+)
+
+
+def test_detection_is_structural_and_ignores_native_classes():
+ mod = _remote_module()
+ block = mod.DeepseekV3MoE(mod.Cfg())
+ assert is_remote_deepseek_gate(block.gate)
+ assert is_remote_deepseek_moe(block)
+
+ class MoEGate(nn.Module): # same name, not remote code
+ pass
+
+ assert not is_remote_deepseek_gate(MoEGate())
+ assert not is_remote_deepseek_moe(nn.Linear(2, 2))
+
+
+def test_training_forward_matches_eval_and_backpropagates():
+ torch.manual_seed(0)
+ mod = _remote_module("transformers_modules.tiny_kimi_b.modeling_deepseek")
+ block = mod.DeepseekV3MoE(mod.Cfg())
+ x = torch.randn(2, 5, 16)
+ block.train()
+ with pytest.raises(AssertionError):
+ block(x)
+ patched = prepare_remote_moe_for_training(block, verbose = False)
+ assert sorted(patched) == ["DeepseekV3MoE", "MoEGate"]
+ assert prepare_remote_moe_for_training(block, verbose = False) == [] # idempotent
+ block.eval()
+ with torch.no_grad():
+ reference = block(x)
+ block.train()
+ out = block(x)
+ assert torch.allclose(out, reference, atol = 1e-5, rtol = 1e-5)
+ out.float().pow(2).sum().backward()
+ assert block.gate.weight.grad is not None and torch.isfinite(block.gate.weight.grad).all()
+ expert_grads = [
+ e.down_proj.weight.grad for e in block.experts if e.down_proj.weight.grad is not None
+ ]
+ assert expert_grads and any(g.abs().sum() > 0 for g in expert_grads)
+ assert block.shared_experts.down_proj.weight.grad is not None
+ assert block.gate.training and block.training # the gate flag is restored after the call
+
+
+def test_shims_reach_a_module_behind_an_accelerate_hook():
+ """A device_map load wraps forward with an accelerate hook that keeps the original as
+ `_old_forward`; the class-level shim alone is never called from `module(...)`. Kimi-K2.7-Code
+ on four GPUs still hit the gate's training assert this way."""
+ from accelerate.hooks import ModelHook, add_hook_to_module
+
+ torch.manual_seed(0)
+ mod = _remote_module("transformers_modules.tiny_kimi_c.modeling_deepseek")
+ block = mod.DeepseekV3MoE(mod.Cfg())
+ for sub in (block, block.gate):
+ add_hook_to_module(sub, ModelHook())
+ assert "_old_forward" in vars(block.gate)
+ block.train()
+ x = torch.randn(2, 5, 16)
+ with pytest.raises(AssertionError):
+ block(x)
+ prepare_remote_moe_for_training(block, verbose = False)
+ out = block(x) # goes through the hook, which must now reach the shim
+ block.eval()
+ with torch.no_grad():
+ reference = block(x)
+ assert torch.allclose(out, reference, atol = 1e-5, rtol = 1e-5)
+ # A hook attached after the classes were shimmed (a second model of the same remote code) is rebound too.
+ block2 = mod.DeepseekV3MoE(mod.Cfg())
+ add_hook_to_module(block2.gate, ModelHook())
+ block2.train()
+ assert prepare_remote_moe_for_training(block2, verbose = False) == []
+ block2(x)
+
+
+def test_composite_gradient_checkpointing_flag():
+ from transformers import LlamaConfig, LlamaForCausalLM, PreTrainedModel, PretrainedConfig
+
+ class OuterConfig(PretrainedConfig):
+ model_type = "outer_test"
+
+ class Outer(PreTrainedModel):
+ config_class = OuterConfig
+
+ def __init__(self, config):
+ super().__init__(config)
+ self.language_model = LlamaForCausalLM(
+ LlamaConfig(
+ hidden_size = 8,
+ num_hidden_layers = 1,
+ num_attention_heads = 2,
+ intermediate_size = 8,
+ vocab_size = 16,
+ )
+ )
+
+ Outer.supports_gradient_checkpointing = False
+ outer = Outer(OuterConfig())
+ with pytest.raises(ValueError):
+ outer.gradient_checkpointing_enable()
+ assert enable_composite_gradient_checkpointing(outer, verbose = False)
+ assert not enable_composite_gradient_checkpointing(outer, verbose = False)
+ outer.gradient_checkpointing_enable()
+ assert outer.language_model.model.gradient_checkpointing
+ assert not enable_composite_gradient_checkpointing(nn.Linear(2, 2), verbose = False)
+
+
+def test_a_training_capable_remote_moe_is_left_alone():
+ """The standard DeepSeek-V2/V3 implementation branches on self.training and its gate
+ returns an auxiliary loss; shimming it would replace a working training path."""
+ import sys, types
+ from unsloth.models.remote_moe_shims import is_remote_deepseek_moe
+
+ module = types.ModuleType("transformers_modules.capable.modeling_deepseek")
+
+ class DeepseekV3MoE(nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.experts = nn.ModuleList([nn.Linear(2, 2)])
+ self.gate = nn.Linear(2, 1)
+
+ def forward(self, hidden_states):
+ if self.training:
+ return hidden_states * 2
+ else:
+ return self.moe_infer(hidden_states)
+
+ def moe_infer(self, x):
+ return x
+
+ DeepseekV3MoE.__module__ = module.__name__
+ sys.modules[module.__name__] = module
+ try:
+ assert not is_remote_deepseek_moe(DeepseekV3MoE())
+ finally:
+ sys.modules.pop(module.__name__, None)
+
+
+def test_the_gate_of_a_training_capable_block_is_not_patched():
+ """A training-capable block's gate computes its auxiliary loss in train mode; the gate is
+ patched only when its block is the inference-only port."""
+ import linecache, sys, types
+ from unsloth.models.remote_moe_shims import prepare_remote_moe_for_training
+
+ name = "transformers_modules.capable2.modeling_deepseek"
+ src = """
+import torch
+from torch import nn
+class MoEGate(nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.topk_method = "greedy"; self.n_group = 1; self.topk_group = 1
+ self.routed_scaling_factor = 1.0; self.norm_topk_prob = False; self.top_k = 1
+ def forward(self, x):
+ return x
+class DeepseekV3MoE(nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.gate = MoEGate(); self.experts = nn.ModuleList([nn.Linear(2, 2)])
+ def forward(self, x):
+ if self.training:
+ return x * 2
+ else:
+ return self.moe_infer(x)
+ def moe_infer(self, x):
+ return x
+"""
+ filename = f"<{name}>"
+ linecache.cache[filename] = (len(src), None, src.splitlines(True), filename)
+ mod = types.ModuleType(name)
+ exec(compile(src, filename, "exec"), mod.__dict__)
+ for cls in (mod.MoEGate, mod.DeepseekV3MoE):
+ cls.__module__ = name
+ sys.modules[name] = mod
+ try:
+ block = mod.DeepseekV3MoE()
+ assert prepare_remote_moe_for_training(block, verbose = False) == []
+ assert not getattr(mod.MoEGate.forward, "_unsloth_remote_moe_shim", False)
+ finally:
+ sys.modules.pop(name, None)
+
+
[email protected]("shared", [True, False])
+def test_a_renamed_port_with_a_no_grad_else_branch_is_shimmed(shared):
+ """sarvam-105b trains through the same shims: its block is not called DeepseekV3MoE, its
+ `else:` branch computes the output under `@torch.no_grad()` (so LoRA on the experts would
+ get no gradient), and its shared expert may be None."""
+ torch.manual_seed(0)
+ mod = _remote_module(
+ f"transformers_modules.tiny_sarvam_{int(shared)}.modeling_sarvam_moe", _SARVAM_EDITS
+ )
+ block = mod.SarvamMLAMoE(mod.Cfg())
+ if not shared:
+ block.shared_experts = None
+ assert is_remote_deepseek_moe(block)
+ x = torch.randn(2, 5, 16)
+ block.train()
+ with pytest.raises(AssertionError):
+ block(x)
+ patched = prepare_remote_moe_for_training(block, verbose = False)
+ assert sorted(patched) == ["MoEGate", "SarvamMLAMoE"]
+ block.eval()
+ with torch.no_grad():
+ reference = block(x)
+ block.train()
+ out = block(x)
+ assert torch.allclose(out, reference, atol = 1e-5, rtol = 1e-5)
+ out.float().pow(2).sum().backward()
+ assert any(
+ e.down_proj.weight.grad is not None and e.down_proj.weight.grad.abs().sum() > 0
+ for e in block.experts
+ )
+ if shared:
+ assert block.shared_experts.down_proj.weight.grad is not None
+
+
+def test_an_eval_first_training_capable_remote_moe_is_left_alone():
+ """`if not self.training: moe_infer(...) else: <training dispatch>` trains on its own."""
+ import sys, types
+ from unsloth.models.remote_moe_shims import is_remote_deepseek_moe
+
+ module = types.ModuleType("transformers_modules.eval_first.modeling_deepseek")
+
+ class EvalFirstMoE(nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.experts = nn.ModuleList([nn.Linear(2, 2)])
+ self.gate = nn.Linear(2, 1)
+
+ def forward(self, hidden_states):
+ if not self.training:
+ y = self.moe_infer(hidden_states)
+ else:
+ y = self.experts[0](hidden_states)
+ return y
+
+ def moe_infer(self, x):
+ return x
+
+ EvalFirstMoE.__module__ = module.__name__
+ sys.modules[module.__name__] = module
+ try:
+ assert not is_remote_deepseek_moe(EvalFirstMoE())
+ finally:
+ sys.modules.pop(module.__name__, None)
+
+
+ "predicate, trains_in_body",
+ [
+ ("self.training and self.ep_size == 1", True),
+ ("self.ep_size == 1 and self.training", True),
+ ("not self.training and self.ep_size == 1", False),
+ ("self.training or self.ep_size == 2", True),
+ ("(self.training and self.ep_size == 1) or self.use_fused", True),
+ ("not (self.training and self.ep_size == 1)", False),
+ ("not (not self.training or self.ep_size != 1)", True),
+ ],
+)
+def test_a_compound_training_predicate_is_recognised(predicate, trains_in_body):
+ """`if self.training and self.ep_size == 1:` dispatches its own experts in training."""
+ import sys, textwrap, types
+ from unsloth.models.remote_moe_shims import is_remote_deepseek_moe
+
+ module = types.ModuleType("transformers_modules.compound.modeling_deepseek")
+ train, infer = "y = self.experts[0](hidden_states)", "y = self.moe_infer(hidden_states)"
+ body, other = (train, infer) if trains_in_body else (infer, train)
+ source = textwrap.dedent(f"""
+ class CompoundMoE(nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.experts = nn.ModuleList([nn.Linear(2, 2)])
+ self.gate = nn.Linear(2, 1)
+ self.ep_size = 1
+ self.use_fused = False
+
+ def forward(self, hidden_states):
+ if {predicate}:
+ {body}
+ else:
+ {other}
+ return y
+
+ def moe_infer(self, x):
+ return x
+ """)
+ path = os.path.join(tempfile.mkdtemp(dir = os.environ.get("TMPDIR")), "modeling_compound.py")
+ with open(path, "w") as f:
+ f.write("import torch.nn as nn\n" + source)
+ module.__file__ = path
+ namespace = {"nn": nn, "__name__": module.__name__}
+ exec(compile("import torch.nn as nn\n" + source, path, "exec"), namespace)
+ CompoundMoE = namespace["CompoundMoE"]
+ CompoundMoE.__module__ = module.__name__
+ sys.modules[module.__name__] = module
+ try:
+ assert not is_remote_deepseek_moe(CompoundMoE())
+ finally:
+ sys.modules.pop(module.__name__, None)
diff --git a/unsloth/kernels/int4_packed.py b/unsloth/kernels/int4_packed.py
new file mode 100644
index 00000000000..8363080164d
--- /dev/null
+++ b/unsloth/kernels/int4_packed.py
@@ -0,0 +1,804 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved.
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+"""Triton kernels for compressed-tensors ``pack-quantized`` integer weights kept packed.
+
+Layout (compressed_tensors/compressors/pack_quantized/helpers.py): ``weight_packed`` is int32
+``[N, ceil(K / pf)]`` with ``pf = 32 // bits`` values per word, element ``j`` in bits
+``[j * bits, (j + 1) * bits)``, stored as ``q + 2 ** (bits - 1)``. ``weight_scale`` is ``[N, G]``;
+an asymmetric ``weight_zero_point`` is packed the same way along the OUTPUT dim ``[ceil(N / pf), G]``;
+act-order checkpoints carry ``weight_g_idx`` ``[K]`` (column -> group). The decode
+``(q - zp) * scale`` is exact in fp32, then rounded like compressed-tensors (scale dtype, then output).
+"""
+
+import functools
+import inspect
+import os
+
+import torch
+import triton
+import triton.language as tl
+
+__all__ = [
+ "Int4QuantState",
+ "int4_dequantize",
+ "int4_dequantize_weight",
+ "int4_matmul",
+ "int4_matmul_t",
+ "int4_repack_",
+ "int4_unpack",
+]
+
+
+class Int4QuantState:
+ """Carried as ``weight.quant_state`` so Unsloth's LoRA kernels dispatch like bitsandbytes."""
+
+ __slots__ = (
+ "scale",
+ "zero_point",
+ "g_idx",
+ "shape",
+ "bits",
+ "group_size",
+ "dtype",
+ "_launch",
+ "_launchers",
+ "layout",
+ "_fast",
+ )
+
+ def __init__(self, scale, zero_point, g_idx, shape, bits, group_size, dtype):
+ self.scale = scale
+ self.zero_point = zero_point
+ self.g_idx = g_idx
+ self.shape = torch.Size(shape)
+ self.bits = int(bits)
+ self.group_size = int(group_size)
+ self.dtype = dtype
+ self._launch = None
+ self._launchers = None
+ self.layout = None
+ self._fast = None
+
+
+def _as_float(v):
+ # Exact int -> fp32 for |v| < 2**22 without the quarter-rate cvt: place v in the mantissa of 2**23.
+ return ((v + 0x4B400000).to(tl.float32, bitcast = True)) - 12582912.0
+
+
+def _unpack_tile3(
+ P,
+ S,
+ Z,
+ G,
+ rn,
+ rkp,
+ N,
+ K,
+ KP,
+ stride_pn,
+ stride_sn,
+ stride_zn,
+ BITS: tl.constexpr,
+ PF: tl.constexpr,
+ GROUP: tl.constexpr,
+ HAS_Z: tl.constexpr,
+ HAS_G: tl.constexpr,
+):
+ # fp32 decode of W[rn, words rkp] as [BLOCK_N, BLOCK_KP, PF] (column = word * PF + j).
+ MASK: tl.constexpr = (1 << BITS) - 1
+ n_ok = rn < N
+ kp_ok = rkp < KP
+ words = tl.load(
+ P + rn[:, None] * stride_pn + rkp[None, :],
+ mask = n_ok[:, None] & kp_ok[None, :],
+ other = 0,
+ )
+ j = tl.arange(0, PF)
+ q = (words[:, :, None] >> (j * BITS)[None, None, :]) & MASK
+ if HAS_G:
+ # act-order: every column has its own group.
+ k3 = rkp[:, None] * PF + j[None, :]
+ k3_ok = k3 < K
+ g3 = tl.load(G + k3, mask = k3_ok, other = 0)
+ m3 = n_ok[:, None, None] & k3_ok[None, :, :]
+ s = tl.load(S + rn[:, None, None] * stride_sn + g3[None, :, :], mask = m3, other = 0.0)
+ if HAS_Z:
+ zw = tl.load(
+ Z + (rn // PF)[:, None, None] * stride_zn + g3[None, :, :], mask = m3, other = 0
+ )
+ z = (zw >> ((rn % PF) * BITS)[:, None, None]) & MASK
+ else:
+ z = 1 << (BITS - 1)
+ w = _as_float(q - z) * s.to(tl.float32)
+ else:
+ # GROUP % PF == 0 (checked on the host): one group per packed word.
+ g = (rkp * PF) // GROUP
+ m2 = n_ok[:, None] & kp_ok[None, :]
+ s = tl.load(S + rn[:, None] * stride_sn + g[None, :], mask = m2, other = 0.0)
+ if HAS_Z:
+ zw = tl.load(Z + (rn // PF)[:, None] * stride_zn + g[None, :], mask = m2, other = 0)
+ z = (zw >> ((rn % PF) * BITS)[:, None]) & MASK
+ zf = _as_float(z)
+ w = (_as_float(q) - zf[:, :, None]) * s.to(tl.float32)[:, :, None]
+ else:
+ w = (_as_float(q) - (1 << (BITS - 1))) * s.to(tl.float32)[:, :, None]
+ return w
+
+
+def _unpack_tile(
+ P,
+ S,
+ Z,
+ G,
+ rn,
+ rkp,
+ rk,
+ N,
+ K,
+ KP,
+ stride_pn,
+ stride_sn,
+ stride_zn,
+ BITS: tl.constexpr,
+ PF: tl.constexpr,
+ GROUP: tl.constexpr,
+ HAS_Z: tl.constexpr,
+ HAS_G: tl.constexpr,
+ BLOCK_N: tl.constexpr,
+ BLOCK_KP: tl.constexpr,
+):
+ w = _unpack_tile3(
+ P, S, Z, G, rn, rkp, N, K, KP, stride_pn, stride_sn, stride_zn,
+ BITS, PF, GROUP, HAS_Z, HAS_G,
+ ) # fmt: skip
+ return tl.reshape(w, (BLOCK_N, BLOCK_KP * PF))
+
+
+def _dequant_kernel(
+ P,
+ S,
+ Z,
+ G,
+ Out,
+ N,
+ K,
+ KP,
+ stride_pn,
+ stride_sn,
+ stride_zn,
+ stride_on,
+ BITS: tl.constexpr,
+ PF: tl.constexpr,
+ GROUP: tl.constexpr,
+ HAS_Z: tl.constexpr,
+ HAS_G: tl.constexpr,
+ BLOCK_N: tl.constexpr,
+ BLOCK_KP: tl.constexpr,
+):
+ pid_n = tl.program_id(0)
+ pid_k = tl.program_id(1)
+ rn = pid_n * BLOCK_N + tl.arange(0, BLOCK_N)
+ rkp = pid_k * BLOCK_KP + tl.arange(0, BLOCK_KP)
+ rk = pid_k * (BLOCK_KP * PF) + tl.arange(0, BLOCK_KP * PF)
+ w = _unpack_tile(
+ P, S, Z, G, rn, rkp, rk, N, K, KP, stride_pn, stride_sn, stride_zn,
+ BITS, PF, GROUP, HAS_Z, HAS_G, BLOCK_N, BLOCK_KP,
+ ) # fmt: skip
+ # compressed-tensors multiplies in the scale dtype, then casts: round the same way twice.
+ tl.store(
+ Out + rn[:, None] * stride_on + rk[None, :],
+ w.to(S.dtype.element_ty).to(Out.dtype.element_ty),
+ mask = (rn < N)[:, None] & (rk < K)[None, :],
+ )
+
+
+def _gemv_kernel(
+ X,
+ P,
+ S,
+ Z,
+ G,
+ Y,
+ M,
+ N,
+ K,
+ KP,
+ stride_xm,
+ stride_pn,
+ stride_sn,
+ stride_zn,
+ stride_ym,
+ BITS: tl.constexpr,
+ PF: tl.constexpr,
+ GROUP: tl.constexpr,
+ HAS_Z: tl.constexpr,
+ HAS_G: tl.constexpr,
+ BLOCK_N: tl.constexpr,
+ BLOCK_KP: tl.constexpr,
+ SPLIT_K: tl.constexpr,
+):
+ # Decode: one activation row per program axis 2 (M tiny), CUDA-core reduction.
+ pid_n = tl.program_id(0)
+ pid_k = tl.program_id(1)
+ m = tl.program_id(2)
+ rn = pid_n * BLOCK_N + tl.arange(0, BLOCK_N)
+ acc = tl.zeros((BLOCK_N,), dtype = tl.float32)
+ n_k = tl.cdiv(KP, BLOCK_KP)
+ j = tl.arange(0, PF)
+ for kb in range(pid_k, n_k, SPLIT_K):
+ rkp = kb * BLOCK_KP + tl.arange(0, BLOCK_KP)
+ k2 = rkp[:, None] * PF + j[None, :]
+ x = tl.load(X + m * stride_xm + k2, mask = k2 < K, other = 0.0).to(tl.float32)
+ w = _unpack_tile3(
+ P, S, Z, G, rn, rkp, N, K, KP, stride_pn, stride_sn, stride_zn,
+ BITS, PF, GROUP, HAS_Z, HAS_G,
+ ) # fmt: skip
+ acc += tl.sum(tl.sum(w * x[None, :, :], axis = 2), axis = 1)
+ ptrs = Y + m * stride_ym + rn
+ if SPLIT_K == 1:
+ tl.store(ptrs, acc.to(Y.dtype.element_ty), mask = rn < N)
+ else:
+ tl.atomic_add(ptrs, acc, mask = rn < N, sem = "relaxed")
+
+
+def _launch_args(packed, qs):
+ """``((P, S, Z, G), (N, K, KP), strides, constexprs)``, cached on the quant state (decode is launch bound)."""
+ cached = qs._launch
+ if cached is not None and cached[0] == packed.data_ptr():
+ return cached[1]
+ N, K = qs.shape
+ bits = qs.bits
+ pf = 32 // bits
+ group = qs.group_size if qs.group_size > 0 else K
+ if qs.g_idx is None and group % pf != 0:
+ # A word would straddle two groups: address the groups per column instead.
+ qs.g_idx = torch.arange(K, device = packed.device, dtype = torch.int32) // group
+ zp, g_idx, scale = qs.zero_point, qs.g_idx, qs.scale
+ Z = zp if zp is not None else packed
+ G = g_idx if g_idx is not None else packed
+ args = (
+ (packed, scale, Z, G),
+ (N, K, packed.shape[1]),
+ (packed.stride(0), scale.stride(0), Z.stride(0) if zp is not None else 0),
+ dict(BITS = bits, PF = pf, GROUP = group, HAS_Z = zp is not None, HAS_G = g_idx is not None),
+ )
+ qs._launch = (packed.data_ptr(), args)
+ qs._launchers = None
+ return args
+
+
+class _on_device:
+ """``torch.cuda.device`` only when ``device`` is not already current (the context costs a few us)."""
+
+ __slots__ = ("ctx",)
+
+ def __init__(self, device):
+ self.ctx = None
+ index = device.index
+ if index is not None and index != torch.cuda.current_device():
+ self.ctx = torch.cuda.device(index)
+
+ def __enter__(self):
+ if self.ctx is not None:
+ self.ctx.__enter__()
+
+ def __exit__(self, *exc):
+ if self.ctx is not None:
+ self.ctx.__exit__(*exc)
+
+
+# Triton < 3.7 CompiledKernel launchers also want constexprs (TypeError): fall back to the JIT launch.
+_FAST_LAUNCH = None
+
+
+def _launch(kernel, launchers, key, grid, args, **constexprs):
+ global _FAST_LAUNCH
+ if _FAST_LAUNCH is not False:
+ compiled = launchers.get(key)
+ if compiled is None:
+ compiled = launchers[key] = kernel.warmup(*args, grid = grid, **constexprs)
+ try:
+ compiled[grid](*args)
+ _FAST_LAUNCH = True
+ return
+ except TypeError:
+ if _FAST_LAUNCH:
+ raise
+ _FAST_LAUNCH = False
+ kernel[grid](*args, **constexprs)
+
+
+def int4_dequantize(
+ packed,
+ qs,
+ dtype = None,
+ out = None,
+):
+ dtype = dtype or qs.dtype or torch.bfloat16
+ if qs.layout is not None:
+ return _dequant_repacked(packed, qs, dtype, out)
+ (P, S, Z, G), (N, K, KP), (sp, ss, sz), meta = _launch_args(packed, qs)
+ if out is None:
+ out = torch.empty((N, K), dtype = dtype, device = packed.device)
+ BLOCK_N, BLOCK_KP = 16, max(1, 512 // meta["PF"])
+ grid = (triton.cdiv(N, BLOCK_N), triton.cdiv(KP, BLOCK_KP), 1)
+ args = (P, S, Z, G, out, N, K, KP, sp, ss, sz, out.stride(0))
+ key = (packed.device.index, "dq", out.dtype, out.data_ptr() % 16 == 0, out.stride(0) % 16 == 0)
+ launchers = qs._launchers
+ if launchers is None:
+ launchers = qs._launchers = {}
+ with _on_device(packed.device):
+ _launch(
+ _dequant_kernel,
+ launchers,
+ key,
+ grid,
+ args,
+ BLOCK_N = BLOCK_N,
+ BLOCK_KP = BLOCK_KP,
+ num_warps = 4,
+ **meta,
+ )
+ return out
+
+
+def int4_dequantize_weight(
+ W,
+ qs,
+ dtype = None,
+):
+ """``fast_dequantize`` contract: ``W`` is the packed weight or its ``.t()``; returns the matching dense view."""
+ if W.stride(-1) != 1 and W.dim() == 2 and W.shape[1] == qs.shape[0]:
+ return int4_dequantize(W.t(), qs, dtype).t()
+ return int4_dequantize(W, qs, dtype)
+
+
[email protected]_cache(maxsize = None)
+def _sm_count(index):
+ return torch.cuda.get_device_properties(index).multi_processor_count
+
+
+# The fused GEMV decodes the weight once per row: only one row beats decode + cuBLAS.
+GEMV_MAX_ROWS = 1
+
+
+# Weights are repacked once at load (in place, no second copy) into vLLM Marlin's layout when vLLM is installed, else
+# tinygemm's (bf16 only; UNSLOTH_INT4_LAYOUT=tinygemm forces it). Rows where the fused kernel beats dequantize + cuBLAS,
+# by CC major (training always dequantizes, so its numerics never depend on the layout):
+MARLIN_MAX_ROWS = {8: 512, 10: 32, 11: 32, 12: 1024}
+TINYGEMM_MAX_ROWS = {8: 16, 10: 4, 11: 4, 12: 16}
+# tile rows, tile columns, tiles advance along N first, destination bit of each source nibble-index bit
+_LAYOUTS = {
+ "tinygemm": (8, 128, False, (3, 4, 0, 5, 6, 1, 2, 7, 8, 9)),
+ "marlin": (64, 16, True, (3, 7, 0, 8, 9, 1, 2, 4, 5, 6)),
+}
+_LAYOUT_OK = {}
+_FAST_CHECKED = {}
+
+
[email protected]_cache(maxsize = None)
+def _fast_rows(layout, index):
+ table = MARLIN_MAX_ROWS if layout == "marlin" else TINYGEMM_MAX_ROWS
+ return table.get(torch.cuda.get_device_capability(index)[0], 32 if layout == "marlin" else 4)
+
+
+_MARLIN_API = None
+
+
+def _marlin_api():
+ global _MARLIN_API
+ if _MARLIN_API is None:
+ _MARLIN_API = False
+ if os.environ.get("UNSLOTH_INT4_MARLIN", "1") != "0" and torch.version.hip is None:
+ try:
+ from vllm import _custom_ops as ops
+ from vllm.scalar_type import scalar_types
+
+ new = hasattr(torch.ops._C, "marlin_gemm")
+ if not new and not hasattr(torch.ops._C, "gptq_marlin_gemm"):
+ return _MARLIN_API
+ repack_perm = "perm" in inspect.signature(ops.gptq_marlin_repack).parameters
+ types = {False: scalar_types.uint4b8, True: scalar_types.uint4}
+ _MARLIN_API = (ops, new, repack_perm, types, {})
+ except Exception:
+ pass
+ return _MARLIN_API
+
+
+def _marlin_scale_perm():
+ return [i + 8 * j for i in range(8) for j in range(8)]
+
+
+def _repack_words(layout, packed, N, K):
+ """The checkpoint's ``[N, K / 8]`` words in ``layout``, viewed as ``[N, K / 8]`` again."""
+ if layout == "marlin":
+ ops, _, repack_perm, _, _ = _marlin_api()
+ gptq = packed.t().contiguous() # GPTQ packs along K too, transposed
+ if repack_perm:
+ words = ops.gptq_marlin_repack(
+ gptq, torch.empty(0, dtype = torch.int32, device = packed.device), K, N, 4
+ )
+ else:
+ words = ops.gptq_marlin_repack(gptq, K, N, 4)
+ else:
+ # Nibble i of word w is column 8w + i; tinygemm wants bytes (q[2j] << 4) | q[2j + 1].
+ b = packed.unsqueeze(-1) >> torch.arange(0, 32, 8, device = packed.device, dtype = torch.int32)
+ pairs = (((b & 15) << 4) | ((b >> 4) & 15)).to(torch.uint8).reshape(N, K // 2)
+ words = torch._convert_weight_to_int4pack(pairs, 8)
+ return words.reshape(N, K // 8)
+
+
+def _layout_verified(layout, device):
+ """Probe the repack with base-16 digits of each position; it must be the tile bit permutation hard-coded above."""
+ key = (layout, device)
+ ok = _LAYOUT_OK.get(key)
+ if ok is None:
+ ok = False
+ try:
+ TN, TK, nfast, bits = _LAYOUTS[layout]
+ Np, Kp = (128, 256) if layout == "marlin" else (16, 256)
+ pos = torch.arange(Np * Kp, device = device, dtype = torch.int32).view(Np, Kp)
+ idx = torch.zeros(Np * Kp, dtype = torch.int32, device = device)
+ nib = torch.arange(0, 32, 4, device = device, dtype = torch.int32)
+ for d in range(4):
+ q = ((pos >> (4 * d)) & 15).view(Np, -1, 8)
+ probe = torch.zeros((Np, Kp // 8), dtype = torch.int32, device = device)
+ for i in range(8):
+ probe |= q[:, :, i] << (4 * i)
+ words = _repack_words(layout, probe, Np, Kp).reshape(-1)
+ idx |= ((words.unsqueeze(-1) >> nib) & 15).reshape(-1) << (4 * d)
+ e = torch.arange(Np * Kp, device = device, dtype = torch.int32)
+ tile, local = e // 1024, e % 1024
+ NT_N, NT_K = Np // TN, Kp // TK
+ tn, tk = (tile % NT_N, tile // NT_N) if nfast else (tile // NT_K, tile % NT_K)
+ dest = torch.zeros_like(local)
+ for b, d in enumerate(bits):
+ dest |= ((local >> b) & 1) << d
+ want = (tn * TN + dest // TK) * Kp + tk * TK + dest % TK
+ ok = bool(torch.equal(idx, want))
+ except Exception:
+ ok = False
+ _LAYOUT_OK[key] = ok
+ return ok
+
+
+def _pick_layout(packed, qs):
+ if (
+ os.environ.get("UNSLOTH_INT4_REPACK", "1") == "0"
+ or not packed.is_cuda
+ or torch.version.hip is not None
+ or qs.bits != 4
+ or qs.g_idx is not None
+ or packed.dtype != torch.int32
+ or torch.cuda.get_device_capability(packed.device) < (8, 0)
+ ):
+ return None
+ N, K = qs.shape
+ group, zp, dtype = qs.group_size, qs.zero_point, qs.dtype
+ if (
+ packed.shape != (N, K // 8)
+ or group <= 0
+ or K % max(group, 128)
+ or qs.scale.dtype != dtype
+ or qs.scale.shape != (N, K // max(group, 1))
+ or (zp is not None and zp.shape != ((N + 7) // 8, K // max(group, 1)))
+ ):
+ return None
+ tinygemm = (
+ dtype == torch.bfloat16
+ and group in (32, 64, 128, 256)
+ and N % 8 == 0
+ and hasattr(torch, "_weight_int4pack_mm")
+ and os.environ.get("UNSLOTH_INT4_TINYGEMM", "1") != "0"
+ )
+ marlin = (
+ dtype in (torch.float16, torch.bfloat16)
+ and group in (32, 64, 128)
+ and N % 64 == 0
+ and bool(_marlin_api())
+ )
+ if marlin and (not tinygemm or os.environ.get("UNSLOTH_INT4_LAYOUT") != "tinygemm"):
+ return "marlin"
+ return "tinygemm" if tinygemm else None
+
+
+def int4_repack_(packed, qs):
+ """Repack ``packed`` in place into a fused-kernel layout; returns it, or None (left untouched)."""
+ layout = _pick_layout(packed, qs)
+ if layout is None or not _layout_verified(layout, packed.device):
+ return None
+ N, K = qs.shape
+ with torch.no_grad(), _on_device(packed.device):
+ words = _repack_words(layout, packed, N, K)
+ before = int4_dequantize(packed, qs, qs.dtype)
+ qs.layout = layout
+ same = torch.equal(_dequant_repacked(words, qs, qs.dtype), before)
+ del before
+ if not same:
+ qs.layout = None
+ return None
+ packed.copy_(words)
+ qs._launch = qs._launchers = qs._fast = None
+ return layout
+
+
+def _dequant_repacked_kernel(
+ P,
+ S,
+ Z,
+ Out,
+ NT_N,
+ NT_K,
+ stride_sn,
+ stride_zn,
+ stride_on,
+ MARLIN: tl.constexpr,
+ TN: tl.constexpr,
+ TK: tl.constexpr,
+ R: tl.constexpr,
+ GROUP: tl.constexpr,
+ HAS_Z: tl.constexpr,
+ RAW: tl.constexpr,
+):
+ # One program: tile row tn, R tiles along K, written as [TN, R * TK] contiguous rows.
+ pid = tl.program_id(0)
+ tn = pid // (NT_K // R)
+ tk0 = (pid % (NT_K // R)) * R
+ r = tl.arange(0, R)
+ if MARLIN:
+ tile = (tk0 + r) * NT_N + tn
+ else:
+ tile = tn * NT_K + tk0 + r
+ words = tl.load(P + tile[:, None] * 128 + tl.arange(0, 128)[None, :])
+ q = (words[:, :, None] >> (tl.arange(0, 8) * 4)[None, None, :]) & 15
+ # The tile's nibble-index bits, grouped by where they land (_LAYOUTS), permuted into row-major [TN, R * TK].
+ if MARLIN: # E(9:7) D(6:5) C(4:3) B(2) A2(1) A1(0) -> rows C A2 E, columns A1 D B
+ q = tl.permute(tl.reshape(q, (R, 8, 4, 4, 2, 2, 2)), (3, 5, 1, 0, 6, 2, 4))
+ else: # E(9:7) D(6:5) C(4:3) B(2) A(1:0) -> rows E, columns C A D B
+ q = tl.permute(tl.reshape(q, (R, 8, 4, 4, 2, 4)), (1, 0, 3, 5, 2, 4))
+ q = tl.reshape(q, (TN, R * TK))
+ n = tn * TN + tl.arange(0, TN)
+ offs = n[:, None] * stride_on + tk0 * TK + tl.arange(0, R * TK)[None, :]
+ if RAW:
+ tl.store(Out + offs, q.to(tl.int8))
+ else:
+ NG: tl.constexpr = (R * TK) // GROUP
+ g = (tk0 * TK) // GROUP + tl.arange(0, NG)
+ s = tl.load(S + n[:, None] * stride_sn + g[None, :]).to(tl.float32)
+ if HAS_Z:
+ zw = tl.load(Z + (n // 8)[:, None] * stride_zn + g[None, :])
+ z = _as_float((zw >> ((n % 8) * 4)[:, None]) & 15)
+ else:
+ z = tl.full((TN, NG), 8.0, tl.float32)
+ s = tl.reshape(tl.broadcast_to(s[:, :, None], (TN, NG, GROUP)), (TN, R * TK))
+ z = tl.reshape(tl.broadcast_to(z[:, :, None], (TN, NG, GROUP)), (TN, R * TK))
+ # Same fp32 math and double rounding as _dequant_kernel: bit-identical weights.
+ w = (_as_float(q) - z) * s
+ tl.store(Out + offs, w.to(S.dtype.element_ty).to(Out.dtype.element_ty))
+
+
+def _dequant_repacked(
+ words,
+ qs,
+ dtype,
+ out = None,
+ raw = False,
+):
+ N, K = qs.shape
+ TN, TK, _, _ = _LAYOUTS[qs.layout]
+ NT_N, NT_K = N // TN, K // TK
+ group = qs.group_size
+ R = max(8, group // TK)
+ while NT_K % R:
+ R //= 2
+ if out is None:
+ out = torch.empty((N, K), dtype = torch.int8 if raw else dtype, device = words.device)
+ zp = qs.zero_point
+ with _on_device(words.device):
+ _dequant_repacked_kernel[(NT_N * (NT_K // R),)](
+ words, qs.scale, qs.scale if zp is None else zp, out, NT_N, NT_K, qs.scale.stride(0),
+ 0 if zp is None else zp.stride(0), out.stride(0), MARLIN = qs.layout == "marlin", TN = TN, TK = TK,
+ R = R, GROUP = group, HAS_Z = zp is not None, RAW = raw, num_warps = 4,
+ ) # fmt: skip
+ return out
+
+
+def int4_unpack(packed, qs):
+ """The checkpoint-layout ``[N, K / 8]`` words of a repacked weight (what gets saved)."""
+ if qs.layout is None:
+ return packed
+ N, K = qs.shape
+ q = _dequant_repacked(packed, qs, None, raw = True).to(torch.int32).view(N, K // 8, 8)
+ out = torch.zeros((N, K // 8), dtype = torch.int32, device = packed.device)
+ for i in range(8):
+ out |= q[:, :, i] << (4 * i)
+ return out
+
+
+def _fast_args(packed, qs):
+ """Cached fused-kernel call for a repacked weight, self-checked once per (layout, device, zero points, dtype)."""
+ fast = qs._fast
+ if fast is not None and fast[0] == packed.data_ptr():
+ return fast[1]
+ N, K = qs.shape
+ group, zp, scale = qs.group_size, qs.zero_point, qs.scale
+ if qs.layout == "tinygemm":
+ s = scale.reshape(N, -1).t()
+ if zp is None:
+ zero = torch.zeros_like(s)
+ else:
+ # (q - z) * s == (q - 8) * s + (8 - z) * s, tinygemm's float zero.
+ z = (
+ zp.t().unsqueeze(-1) >> torch.arange(0, 32, 4, device = zp.device, dtype = torch.int32)
+ ) & 15
+ zero = ((8 - z.reshape(zp.shape[1], -1)[:, :N]).float() * s.float()).to(s.dtype)
+ call = (
+ torch.ops.aten._weight_int4pack_mm.default,
+ packed.view(N // 8, K // 128, 32, 4),
+ group,
+ torch.stack((s, zero), -1).contiguous(),
+ )
+ else:
+ _, new, _, types, workspaces = _marlin_api()
+ perm = _marlin_scale_perm()
+ ms = scale.t().contiguous().reshape(-1, 64)[:, perm].reshape(-1, N).contiguous()
+ empty = torch.empty(0, dtype = torch.int32, device = packed.device)
+ mz = empty
+ if zp is not None:
+ G = zp.shape[1]
+ z = (
+ (
+ zp.t().unsqueeze(-1)
+ >> torch.arange(0, 32, 4, device = zp.device, dtype = torch.int32)
+ )
+ & 15
+ ).reshape(G, N)
+ z = (
+ z.reshape(-1, 64)[:, perm]
+ .reshape(-1, 8)[:, [0, 2, 4, 6, 1, 3, 5, 7]]
+ .reshape(G, N // 8, 8)
+ )
+ mz = torch.zeros((G, N // 8), dtype = torch.int32, device = zp.device)
+ for i in range(8):
+ mz |= z[:, :, i] << (4 * i)
+ ws = workspaces.get(packed.device)
+ if ws is None:
+ ws = workspaces[packed.device] = torch.zeros(
+ _sm_count(packed.device.index or 0), dtype = torch.int32, device = packed.device
+ )
+ mq, sid = packed.view(K // 16, 2 * N), types[zp is not None].id
+ # vLLM's Python wrapper costs ~2 us per call (decode is launch bound): call the op overload.
+ if new:
+ call = (
+ torch.ops._C.marlin_gemm.default,
+ (None, mq, None, ms, None, None, mz, ws, sid),
+ (N, K, False, True, False),
+ )
+ else:
+ call = (
+ torch.ops._C.gptq_marlin_gemm.default,
+ (None, mq, None, ms, None, mz, empty, empty, ws, sid),
+ (N, K, True, False, True, False),
+ )
+ key = (qs.layout, packed.device, zp is not None, qs.dtype)
+ if key not in _FAST_CHECKED:
+ g = torch.Generator(device = packed.device).manual_seed(0)
+ probe = torch.randn(4, K, device = packed.device, dtype = qs.dtype, generator = g)
+ ref = probe.float() @ int4_dequantize(packed, qs, qs.dtype).float().t()
+ got = _fast_call(call, probe).float()
+ _FAST_CHECKED[key] = bool(((got - ref).abs().max() <= 1e-2 * ref.abs().max() + 1e-3).item())
+ call = call if _FAST_CHECKED[key] else None
+ qs._fast = (packed.data_ptr(), call)
+ return call
+
+
+def _fast_call(call, x2):
+ if len(call) == 4:
+ return call[0](x2, call[1], call[2], call[3])
+ op, pre, post = call
+ return op(x2, *pre, x2.shape[0], *post)
+
+
+def int4_matmul(
+ x,
+ packed,
+ qs,
+ out = None,
+ fast = True,
+):
+ """``x @ W.T`` for packed ``W``; ``x`` is ``[..., K]``. ``fast = False`` (training) keeps the exact dequantize + matmul."""
+ shape = x.shape
+ x2 = x.reshape(-1, shape[-1])
+ M = x2.shape[0]
+ N = qs.shape[0]
+ if (
+ M == 0
+ ): # Nothing to launch; the kernels reject a zero grid and the split below divides by M.
+ return x.new_empty((*shape[:-1], N)) if out is None else out
+ if qs.layout is not None:
+ call = None
+ if (
+ fast
+ and not torch.is_grad_enabled()
+ # The layout was picked for this compute dtype; a later .float() / .half() leaves the scales behind.
+ and qs.dtype == qs.scale.dtype
+ and M <= _fast_rows(qs.layout, x2.device.index or 0)
+ ):
+ call = _fast_args(packed, qs)
+ if call is not None:
+ y = _fast_call(call, x2 if x2.is_contiguous() else x2.contiguous())
+ if out is not None:
+ out.view(M, N).copy_(y)
+ y = out
+ return y.view(*shape[:-1], N)
+ W = int4_dequantize(packed, qs, x.dtype)
+ y = torch.matmul(x2, W.t(), out = None if out is None else out.view(M, N))
+ return y.view(*shape[:-1], N)
+ if M > GEMV_MAX_ROWS or not fast:
+ W = int4_dequantize(packed, qs, x.dtype)
+ y = torch.matmul(x2, W.t(), out = None if out is None else out.view(M, N))
+ return y.view(*shape[:-1], N)
+ if x2.stride(-1) != 1:
+ x2 = x2.contiguous()
+ (P, S, Z, G), (N, K, KP), (sp, ss, sz), meta = _launch_args(packed, qs)
+ BLOCK_N, BLOCK_KP = 8, max(1, 512 // meta["PF"])
+ n_blocks = triton.cdiv(N, BLOCK_N)
+ # Split K only when the rows alone cannot fill the GPU (the fp32 atomics cost two extra launches).
+ sms = _sm_count(x.device.index or 0)
+ split = 1 if n_blocks * M >= sms else min(4, triton.cdiv(sms, n_blocks * M))
+ if split > 1:
+ y = torch.zeros((M, N), dtype = torch.float32, device = x.device)
+ elif out is not None and out.is_contiguous() and out.dtype == x.dtype:
+ y = out.view(M, N)
+ else:
+ y = torch.empty((M, N), dtype = x.dtype, device = x.device)
+ grid = (n_blocks, split, M)
+ args = (x2, P, S, Z, G, y, M, N, K, KP, x2.stride(0), sp, ss, sz, y.stride(0))
+ # Reuse the compiled kernel (half the JIT launch cost); Triton specializes on ==1 and 16-divisibility.
+ key = (
+ x.device.index,
+ M,
+ split,
+ x2.dtype,
+ x2.data_ptr() % 16 == 0,
+ y.data_ptr() % 16 == 0,
+ x2.stride(0) % 16 == 0,
+ y.stride(0) % 16 == 0,
+ )
+ launchers = qs._launchers
+ if launchers is None:
+ launchers = qs._launchers = {}
+ with _on_device(x.device):
+ _launch(
+ _gemv_kernel, launchers, key, grid, args, BLOCK_N = BLOCK_N, BLOCK_KP = BLOCK_KP, SPLIT_K = split,
+ num_warps = 4, **meta,
+ ) # fmt: skip
+ if y.dtype != x.dtype:
+ y = y.to(x.dtype)
+ if out is not None and y.data_ptr() != out.data_ptr():
+ out.view(M, N).copy_(y)
+ y = out
+ return y.view(*shape[:-1], N)
+
+
+def int4_matmul_t(dy, packed, qs):
+ """``dy @ W`` (the dX of ``x @ W.T``) for packed ``W``; ``dy`` is ``[..., N]``."""
+ W = int4_dequantize(packed, qs, dy.dtype)
+ return torch.matmul(dy, W)
diff --git a/unsloth/kernels/utils.py b/unsloth/kernels/utils.py
index 694ed690226..c0452830366 100644
--- a/unsloth/kernels/utils.py
+++ b/unsloth/kernels/utils.py
@@ -1037,6 +1037,38 @@ else:
pass
+from .int4_packed import (
+ Int4QuantState,
+ int4_dequantize_weight as _int4_dequantize_weight,
+ int4_matmul as _int4_matmul,
+)
+
+_fast_dequantize_bnb = fast_dequantize
+_fast_gemv_bnb = fast_gemv
+
+
+def fast_dequantize(
+ W,
+ quant_state = None,
+ out = None,
+ use_global_buffer = False,
+):
+ if type(quant_state) is Int4QuantState:
+ return _int4_dequantize_weight(W, quant_state)
+ return _fast_dequantize_bnb(W, quant_state, out = out, use_global_buffer = use_global_buffer)
+
+
+def fast_gemv(
+ X,
+ W,
+ quant_state,
+ out = None,
+):
+ if type(quant_state) is Int4QuantState:
+ return _int4_matmul(X, W, quant_state, out = out)
+ return _fast_gemv_bnb(X, W, quant_state, out = out)
+
+
def fast_linear_forward(
proj,
X,
@@ -1056,6 +1088,8 @@ def fast_linear_forward(
out = fp8_rowwise_gemv(X, W, W_quant)
else:
out = fp8_linear(X, W, W_quant)
+ elif type(W_quant) is Int4QuantState:
+ out = _int4_matmul(X, W, W_quant, out = out)
elif bsz == 1 and q_len == 1:
out = fast_gemv(X, W, W_quant, out = out)
else:
@@ -1114,6 +1148,9 @@ def matmul_lora(
out = torch_matmul(X, W.t(), out = out)
elif W.dtype == torch.float8_e4m3fn:
out = fp8_linear(X, W, W_quant)
+ elif type(W_quant) is Int4QuantState and W.stride(-1) == 1:
+ # Forward orientation: few rows use the fused kernel, the rest decode once + cuBLAS.
+ out = _int4_matmul(X, W, W_quant, out = out)
else:
W = fast_dequantize(W, W_quant, use_global_buffer = True)
out = torch_matmul(X, W.t(), out = out)
diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.py
index c035a6b7fea..142b2f37914 100644
--- a/unsloth/models/_utils.py
+++ b/unsloth/models/_utils.py
@@ -5620,11 +5620,14 @@ def fast_inference_setup(model_name, model_config):
def save_lora_adapter(model, save_directory, *args, **kwargs):
"""`save_pretrained` over the adapter, cast to the embedding dtype. PEFT's own selection decides what an adapter contains, so it is handed the whole state dict and only the adapter tensors are cast: filtering down to `.lora_A.`/`.lora_B.` first, as the Zoo helper does, makes PEFT raise `KeyError` looking up `modules_to_save.<adapter>.weight`, and a DoRA run loses its `lora_magnitude_vector` the same way. Both are reachable without vLLM, since `get_peft_model` adds `embed_tokens` and `lm_head` to `modules_to_save` once new tokens are trained. Non-adapter entries pass through by reference."""
+ from .compressed_tensors_int4 import adapter_only_state_dict
+
dtype = model.get_input_embeddings().weight.dtype
- kwargs["state_dict"] = {
- key: (value.to(dtype) if "lora_" in key else value)
- for key, value in model.state_dict().items()
- }
+ with adapter_only_state_dict():
+ kwargs["state_dict"] = {
+ key: (value.to(dtype) if "lora_" in key else value)
+ for key, value in model.state_dict().items()
+ }
return model.save_pretrained(save_directory, *args, **kwargs)
diff --git a/unsloth/models/compressed_tensors_bnb.py b/unsloth/models/compressed_tensors_bnb.py
new file mode 100644
index 00000000000..2f32a50a75f
--- /dev/null
+++ b/unsloth/models/compressed_tensors_bnb.py
@@ -0,0 +1,946 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved.
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+"""Re-quantize a compressed-tensors ``pack-quantized`` W4A16/W8A16 checkpoint into bitsandbytes
+4-bit per tensor during ``from_pretrained`` (no 16-bit copy on disk). Native MoE stacks keep
+transformers' ``DecompressExperts``; needs the transformers 5 weight-conversion loader."""
+
+import inspect
+import os
+import re
+from typing import Any, Optional
+
+import torch
+
+__all__ = [
+ "compressed_tensors_bnb_plan",
+ "arm_compressed_tensors_bnb_loading",
+ "install_compressed_tensors_bnb_quantizer",
+ "UNSLOTH_COMPRESSED_TENSORS_ATTR",
+]
+
+# Popped by the quantizer after loading so it never reaches a saved config.
+UNSLOTH_COMPRESSED_TENSORS_ATTR = "_unsloth_compressed_tensors_bnb"
+
+_SUPPORTED_FORMATS = ("pack-quantized",)
+
+
+def _quant_dict(config) -> Optional[dict]:
+ quant = getattr(config, "quantization_config", None)
+ if quant is None:
+ return None
+ if hasattr(quant, "to_dict"):
+ try:
+ quant = quant.to_dict()
+ except Exception:
+ return None
+ if not isinstance(quant, dict):
+ return None
+ # Legacy llm-compressor checkpoints nest the real config one level down.
+ inner = quant.get("quantization_config")
+ if isinstance(inner, dict) and "config_groups" in inner:
+ merged = dict(inner)
+ merged.setdefault("quant_method", quant.get("quant_method"))
+ if "sparsity_config" in quant:
+ merged["sparsity_config"] = quant["sparsity_config"]
+ quant = merged
+ return quant
+
+
+def _config_and_subconfigs(config) -> list:
+ seen, out, stack = set(), [], [config]
+ while stack:
+ cfg = stack.pop()
+ if cfg is None or id(cfg) in seen:
+ continue
+ seen.add(id(cfg))
+ out.append(cfg)
+ for name, value in list(getattr(cfg, "__dict__", {}).items()):
+ if name.startswith("_"):
+ continue
+ if (
+ hasattr(value, "to_dict")
+ and hasattr(value, "__dict__")
+ and not isinstance(value, (str, bytes, dict, list, tuple))
+ ):
+ stack.append(value)
+ return out
+
+
+def _transformers_supports_weight_converters() -> bool:
+ try:
+ from transformers.core_model_loading import WeightConverter # noqa: F401
+ from transformers.quantizers.base import HfQuantizer
+ return hasattr(HfQuantizer, "update_weight_conversions")
+ except Exception:
+ return False
+
+
+def compressed_tensors_bnb_plan(config) -> Optional[dict]:
+ """Quant dict for integer weight-only pack-quantized, non-sparse checkpoints; else ``None``."""
+ quant = _quant_dict(config)
+ if quant is None:
+ return None
+ method = str(quant.get("quant_method") or "").lower()
+ if method not in ("compressed-tensors", "compressed_tensors", "sparseml"):
+ return None
+ fmt = quant.get("format")
+ if fmt not in _SUPPORTED_FORMATS:
+ return None
+ groups = quant.get("config_groups") or {}
+ if not isinstance(groups, dict) or len(groups) == 0:
+ return None
+ for group in groups.values():
+ if not isinstance(group, dict):
+ return None
+ weights = group.get("weights")
+ if not isinstance(weights, dict):
+ return None
+ if str(weights.get("type", "int")).lower() != "int":
+ return None
+ if int(weights.get("num_bits", 0)) not in (2, 3, 4, 5, 6, 7, 8):
+ return None
+ if group.get("input_activations") is not None:
+ return None
+ if group.get("output_activations") is not None:
+ return None
+ group_format = group.get("format")
+ if group_format is not None and group_format not in _SUPPORTED_FORMATS:
+ return None
+ sparsity = quant.get("sparsity_config")
+ if isinstance(sparsity, dict):
+ if str(sparsity.get("format", "dense")) != "dense":
+ return None
+ return quant
+
+
+def arm_compressed_tensors_bnb_loading(config, verbose: bool = True) -> Optional[dict]:
+ """Swap the checkpoint quant config for the plan + quantizer; ``None`` (config untouched) if unsupported."""
+ plan = compressed_tensors_bnb_plan(config)
+ if plan is None:
+ return None
+ if not _transformers_supports_weight_converters():
+ if verbose:
+ print(
+ "Unsloth: This checkpoint is compressed-tensors packed INT4/INT8. Re-quantizing it "
+ "to bitsandbytes 4-bit on the fly needs transformers 5.8 or later; loading it as published instead."
+ )
+ return None
+ try:
+ import compressed_tensors # noqa: F401
+ except Exception:
+ if verbose:
+ print(
+ "Unsloth: This checkpoint is compressed-tensors packed INT4/INT8 but `compressed_tensors` "
+ "is not installed; loading it as published. `pip install compressed-tensors` to train it in 4-bit."
+ )
+ return None
+ # Parse now: compressed-tensors retires fields (`actorder = "group"`, 0.19.0) and a late failure is costly.
+ try:
+ _build_quantization_config(plan)
+ except Exception as error:
+ if verbose:
+ print(
+ "Unsloth: This checkpoint is compressed-tensors packed INT4/INT8 but the installed "
+ f"compressed-tensors cannot read its quantization config ({type(error).__name__}); "
+ "loading it as published. Upgrading or downgrading `compressed-tensors` may help."
+ )
+ return None
+ if not install_compressed_tensors_bnb_quantizer():
+ # No converter hook (transformers < 5.8) or foreign bnb quantizer: keep the checkpoint config.
+ if verbose:
+ if not _transformers_supports_weight_converters():
+ reason = "this transformers has no quantizer weight-conversion hook (5.8 or later has it)"
+ else:
+ reason = "another library owns the bitsandbytes quantizer"
+ print(
+ "Unsloth: This checkpoint is compressed-tensors packed INT4/INT8 but "
+ f"{reason}; loading it as published."
+ )
+ return None
+ # Composite configs (Kimi-K2.7) copy the quant config onto sub-configs transformers also reads.
+ for sub in _config_and_subconfigs(config):
+ try:
+ delattr(sub, "quantization_config")
+ except AttributeError:
+ pass
+ if "quantization_config" in getattr(sub, "__dict__", {}):
+ sub.__dict__.pop("quantization_config", None)
+ setattr(config, UNSLOTH_COMPRESSED_TENSORS_ATTR, plan)
+ if verbose:
+ bits = sorted({int(g["weights"]["num_bits"]) for g in plan["config_groups"].values()})
+ from .compressed_tensors_int4 import int4_packed_route_enabled, int4_packed_supported_plan
+
+ if int4_packed_route_enabled() and int4_packed_supported_plan(plan):
+ how = (
+ "Keeping its Linear weights packed and decoding them exactly per use "
+ f"(set {'UNSLOTH_COMPRESSED_TENSORS_INT4'}=nf4 to re-quantize to bitsandbytes 4-bit instead)."
+ )
+ else:
+ how = (
+ "Decompressing each weight on the fly and re-quantizing to bitsandbytes 4-bit "
+ "(no 16-bit copy on disk)."
+ )
+ print(
+ f"Unsloth: Checkpoint is compressed-tensors packed INT{'/'.join(map(str, bits))}. {how}"
+ )
+ return plan
+
+
+# dtype_plan None keeps int32 words uncast; keyed by renamed name (``<module>.weight`` or merged stack), so both spellings.
+
+
+def _checkpoint_keys(checkpoint_files) -> list:
+ keys = []
+ try:
+ from safetensors import safe_open
+ except Exception:
+ return keys
+ others = [str(p) for p in (checkpoint_files or []) if not str(p).endswith(".safetensors")]
+ if others:
+ # Pickled shards are cast to the model dtype before converters run.
+ raise RuntimeError(
+ "Unsloth: re-quantizing a compressed-tensors packed checkpoint on the fly needs "
+ f"safetensors shards; {os.path.basename(others[0])} is not one. Convert the "
+ "checkpoint to safetensors, or load it as published without `load_in_4bit = True`."
+ )
+ for path in checkpoint_files or []:
+ if not str(path).endswith(".safetensors"):
+ continue
+ try:
+ with safe_open(str(path), framework = "pt") as f:
+ keys.extend(f.keys())
+ except Exception:
+ continue
+ return keys
+
+
+_SAFETENSORS_DTYPES = {
+ "I8": torch.int8,
+ "I16": torch.int16,
+ "I32": torch.int32,
+ "I64": torch.int64,
+ "U8": torch.uint8,
+ "F16": torch.float16,
+ "BF16": torch.bfloat16,
+ "F32": torch.float32,
+ "F64": torch.float64,
+}
+
+
+def _packed_tensor_meta(checkpoint_files) -> dict:
+ """``{module: {suffix: (shape, dtype)}}`` for every packed module, from the shard headers only."""
+ from safetensors import safe_open
+
+ meta: dict = {}
+ for path in checkpoint_files or []:
+ if not str(path).endswith(".safetensors"):
+ continue
+ with safe_open(str(path), framework = "pt") as f:
+ for key in f.keys():
+ module, _, suffix = key.rpartition(".")
+ if suffix not in _PACKED_SUFFIXES:
+ continue
+ piece = f.get_slice(key)
+ dtype = _SAFETENSORS_DTYPES.get(piece.get_dtype())
+ if dtype is None:
+ return {}
+ meta.setdefault(module, {})[suffix] = (tuple(piece.get_shape()), dtype)
+ return {m: v for m, v in meta.items() if "weight_packed" in v}
+
+
+_STACKED_PACKED_EXPERTS = "_unsloth_int4_stacked_experts"
+
+
+def adopt_int4_packed_linears(
+ model,
+ ct_config,
+ checkpoint_files,
+ dtype,
+ skip_modules = (),
+) -> tuple:
+ """Swap every packed plain ``nn.Linear`` for an ``Int4PackedLinear`` shell.
+
+ Returns ``(swapped, leftover)``: ``leftover`` packed modules (routers, renamed prefixes) keep the
+ per-module decompress converter; stacked MoE experts keep their own converter."""
+ from torch import nn
+ from .compressed_tensors_int4 import Int4PackedLinear, make_int4_packed_linear
+
+ meta = _packed_tensor_meta(checkpoint_files)
+ swaps, leftover = [], []
+ for name, shapes in meta.items():
+ try:
+ module = model.get_submodule(name)
+ except AttributeError:
+ module = None
+ if isinstance(module, Int4PackedLinear):
+ swaps.append((name, None, None, None))
+ continue
+ # The caller's bnb skip list keeps a module dense: its converter decompresses it instead.
+ if any(f".{key}." in f".{name}." for key in skip_modules or ()):
+ leftover.append(name)
+ continue
+ if module is None and ".experts." in name:
+ # Stacked by their own converter, so no longer in the checkpoint's per-expert layout.
+ model.__dict__[_STACKED_PACKED_EXPERTS] = True
+ continue
+ if type(module) is not nn.Linear:
+ leftover.append(name)
+ continue
+ scheme = _scheme_for_module(ct_config, name, module)
+ if (
+ int(scheme.weights.num_bits) not in (2, 4, 8)
+ or shapes["weight_packed"][0][0] != module.out_features
+ or "weight_scale" not in shapes
+ ):
+ leftover.append(name)
+ continue
+ swaps.append((name, module, scheme, shapes))
+ for name, module, scheme, shapes in swaps:
+ if module is None:
+ continue
+ parent_name, _, child = name.rpartition(".")
+ parent = model.get_submodule(parent_name) if parent_name else model
+ device = module.weight.device if module.weight is not None else "meta"
+ setattr(parent, child, make_int4_packed_linear(module, scheme, shapes, dtype, device))
+ return [name for name, *_ in swaps], leftover
+
+
+def _generalize(name: str) -> str:
+ escaped = re.escape(name)
+ return re.sub(r"(?<=\\.)\d+(?=\\.|$)", r"\\d+", escaped)
+
+
+def packed_weight_dtype_plan(keys) -> dict:
+ """Regex -> ``None`` for ``dtype_plan``; exact names where widening would catch unpacked modules."""
+ packed = [k[: -len(".weight_packed")] for k in keys if k.endswith(".weight_packed")]
+ if not packed:
+ return {}
+ plain = {k[: -len(".weight")] for k in keys if k.endswith(".weight")}
+ packed_set = set(packed)
+ families: dict = {}
+ for name in packed:
+ families.setdefault(_generalize(name), []).append(name)
+ plan: dict = {}
+ for pattern, members in families.items():
+ rx = re.compile(pattern + r"$")
+ collides = any(rx.match(p) for p in plain if p not in packed_set)
+ if collides:
+ for name in members:
+ plan[re.escape(name) + r"\.weight$"] = None
+ else:
+ plan[pattern + r"\.weight$"] = None
+ return plan
+
+
+_PACKED_SUFFIXES = (
+ "weight_packed",
+ "weight_scale",
+ "weight_shape",
+ "weight_zero_point",
+ "weight_g_idx",
+)
+
+
+def _build_quantization_config(plan: dict):
+ from compressed_tensors.quantization import QuantizationConfig
+
+ clean = {
+ k: v for k, v in plan.items() if k not in ("sparsity_config", "global_compression_ratio")
+ }
+ clean.setdefault("quant_method", "compressed-tensors")
+ return QuantizationConfig.model_validate(clean)
+
+
+def _experts_scheme(ct_config):
+ try:
+ from transformers.integrations.compressed_tensors import get_experts_scheme
+ return get_experts_scheme(ct_config)
+ except Exception:
+ groups = list(ct_config.config_groups.values())
+ for group in groups:
+ if any(t.startswith("re:") and "experts" in t for t in group.targets):
+ return group
+ return groups[0]
+
+
+def _scheme_for_sources(ct_config, weight_sources):
+ groups = list(ct_config.config_groups.values())
+ if len(groups) == 1:
+ return groups[0]
+ chosen = []
+ for pattern in weight_sources:
+ name = pattern[: -len(".weight")] if pattern.endswith(".weight") else pattern
+ name = name.replace("*", "0")
+ scheme = _scheme_for_module(ct_config, name, None)
+ if all(scheme is not s for s in chosen):
+ chosen.append(scheme)
+ if len(chosen) > 1:
+ raise RuntimeError(
+ "Unsloth: the compressed-tensors checkpoint quantizes the projections of one expert "
+ f"bucket ({', '.join(weight_sources)}) under different config groups; re-quantizing it "
+ "on the fly is not supported. Load it as published without `load_in_4bit = True`."
+ )
+ return chosen[0] if chosen else _experts_scheme(ct_config)
+
+
+def _layer_expert_scheme(ct_config, full_layer_name, packed_key, n_experts, default):
+ if len(ct_config.config_groups) < 2:
+ return default
+ parent = full_layer_name.rsplit(".", 1)[0]
+ pattern = packed_key.rstrip("$")
+ for suffix in (".weight_packed", "_packed"):
+ if pattern.endswith(suffix):
+ pattern = pattern[: -len(suffix)]
+ break
+ if pattern.endswith(".weight"):
+ pattern = pattern[: -len(".weight")]
+ pattern = pattern.lstrip("^").lstrip(".")
+ parts = parent.split(".")
+ tail = None
+ for start in range(len(parts)):
+ head = ".".join(parts[start:])
+ if pattern.startswith(head + "."):
+ tail = pattern[len(head) :]
+ break
+ if tail is None or "*" not in tail:
+ return default
+ schemes = []
+ for i in range(n_experts):
+ scheme = _scheme_for_module(ct_config, parent + tail.replace("*", str(i), 1), None)
+ if all(scheme is not s for s in schemes):
+ schemes.append(scheme)
+ if len(schemes) > 1:
+ raise RuntimeError(
+ f"Unsloth: the compressed-tensors checkpoint quantizes the experts of {parent} under "
+ "different config groups; re-quantizing it on the fly is not supported. Load it as "
+ "published without `load_in_4bit = True`."
+ )
+ return schemes[0] if schemes else default
+
+
+def drop_load_only_conversions(model) -> int:
+ conversions = getattr(model, "_weight_conversions", None)
+ if not conversions:
+ return 0
+ kept, dropped = [], 0
+ for conv in conversions:
+ ops = getattr(conv, "operations", None) or []
+ if any(isinstance(op, (_DecompressPackedWeights, _WithOriginalSources)) for op in ops):
+ dropped += 1
+ continue
+ kept.append(conv)
+ if dropped:
+ try:
+ model._weight_conversions = kept
+ except Exception:
+ return 0
+ return dropped
+
+
+def _scheme_for_module(ct_config, name: str, module: Optional[torch.nn.Module]):
+ groups = list(ct_config.config_groups.values())
+ if len(groups) == 1:
+ return groups[0]
+ # compressed-tensors precedence: an exact module path, then a regex, then a class name.
+ for group in groups:
+ if name in group.targets:
+ return group
+ for group in groups:
+ for target in group.targets:
+ if target.startswith("re:") and re.match(target[3:], name):
+ return group
+ try:
+ from compressed_tensors.utils.match import is_match
+ if module is not None:
+ for group in groups:
+ if is_match(name, module, group.targets):
+ return group
+ except Exception:
+ pass
+ class_name = type(module).__name__ if module is not None else "Linear"
+ for group in groups:
+ for target in group.targets:
+ if target == class_name or target == "Linear":
+ return group
+ return groups[0]
+
+
+def _decompress_one_triton(scheme, packed, scale, shape, zero_point, g_idx, dtype):
+ """Exact Triton decode (bit-identical to compressed-tensors) for CUDA tensors; ``None`` otherwise."""
+ if packed.device.type != "cuda" or dtype not in (torch.bfloat16, torch.float16, torch.float32):
+ return None
+ weights = scheme.weights
+ bits = int(weights.num_bits)
+ strategy = str(getattr(weights.strategy, "value", weights.strategy)).lower()
+ if bits not in (2, 4, 8) or strategy not in ("group", "channel") or scale.dim() != 2:
+ return None
+ if getattr(torch.version, "hip", None):
+ return None
+ try:
+ from ..kernels.int4_packed import Int4QuantState, int4_dequantize
+ except Exception:
+ return None
+ rows = packed.shape[0]
+ cols = (
+ int(shape[1])
+ if shape is not None and shape.numel() == 2
+ else packed.shape[1] * (32 // bits)
+ )
+ group = int(weights.group_size) if strategy == "group" and weights.group_size else cols
+ if scale.shape[0] != rows or cols > packed.shape[1] * (32 // bits):
+ return None
+ qs = Int4QuantState(
+ scale,
+ None if weights.symmetric else zero_point,
+ g_idx if g_idx is not None and g_idx.device.type == "cuda" and -1 not in g_idx else None,
+ (rows, cols),
+ bits,
+ group,
+ dtype,
+ )
+ if not weights.symmetric and zero_point is None:
+ return None
+ if g_idx is not None and qs.g_idx is None:
+ return None
+ return int4_dequantize(packed, qs, dtype)
+
+
+def _decompress_one(compressor, scheme, packed, scale, shape, zero_point, g_idx, dtype):
+ fast = _decompress_one_triton(scheme, packed, scale, shape, zero_point, g_idx, dtype)
+ if fast is not None:
+ return fast
+ state = {"weight_packed": packed, "weight_scale": scale}
+ if shape is not None and shape.numel():
+ state["weight_shape"] = shape
+ else:
+ pack_factor = 32 // int(scheme.weights.num_bits)
+ state["weight_shape"] = torch.tensor([packed.shape[0], packed.shape[1] * pack_factor])
+ if zero_point is not None:
+ state["weight_zero_point"] = zero_point
+ if g_idx is not None:
+ state["weight_g_idx"] = g_idx
+ out = compressor.decompress(state, scheme)
+ weight = out["weight"]
+ if dtype is not None and weight.dtype != dtype:
+ weight = weight.to(dtype)
+ return weight.contiguous()
+
+
+class _DecompressPackedWeights:
+ def __init__(
+ self,
+ ct_config,
+ dtype,
+ stacked: bool,
+ scheme = None,
+ ):
+ self.ct_config = ct_config
+ self.dtype = dtype
+ self.stacked = stacked
+ self.scheme = scheme
+
+ def _compressor(self, scheme):
+ from compressed_tensors.compressors import BaseCompressor
+ fmt = scheme.format or "pack-quantized"
+ return BaseCompressor.get_value_from_registry(str(fmt))
+
+ def convert(
+ self,
+ input_dict,
+ source_patterns = None,
+ target_patterns = None,
+ full_layer_name = None,
+ model = None,
+ **kwargs,
+ ):
+ packed_keys = [k for k in input_dict if "weight_packed" in k]
+ if not packed_keys:
+ return input_dict
+ out = {}
+ as_list = lambda v: v if isinstance(v, list) else [v] # noqa: E731
+ for packed_key in packed_keys:
+ packed = input_dict[packed_key]
+ get = lambda suffix: input_dict.get(packed_key.replace("weight_packed", suffix)) # noqa: E731
+ scale, shape, zero_point, g_idx = (
+ get(s)
+ for s in ("weight_scale", "weight_shape", "weight_zero_point", "weight_g_idx")
+ )
+ scheme = self.scheme
+ if self.stacked and full_layer_name:
+ scheme = _layer_expert_scheme(
+ self.ct_config, full_layer_name, packed_key, len(as_list(packed)), scheme
+ )
+ if scheme is None:
+ module = None
+ if model is not None and full_layer_name is not None and not self.stacked:
+ try:
+ from transformers.quantizers.quantizers_utils import get_module_from_name
+ module, _ = get_module_from_name(model, full_layer_name)
+ except Exception:
+ module = None
+ scheme = _scheme_for_module(
+ self.ct_config, (full_layer_name or "").rsplit(".", 1)[0], module
+ )
+ compressor = self._compressor(scheme)
+ packed_list = as_list(packed)
+ n = len(packed_list)
+ scales, shapes, zps, gidxs = (
+ (as_list(v) if v is not None else [None] * n)
+ for v in (scale, shape, zero_point, g_idx)
+ )
+ if self.stacked:
+ stacked_out = None
+ for i, p in enumerate(packed_list):
+ w = _decompress_one(
+ compressor, scheme, p, scales[i], shapes[i], zps[i], gidxs[i], self.dtype
+ )
+ if stacked_out is None:
+ stacked_out = torch.empty((n, *w.shape), dtype = w.dtype, device = w.device)
+ stacked_out[i].copy_(w)
+ del w
+ value = stacked_out
+ else:
+ value = _decompress_one(
+ compressor,
+ scheme,
+ packed_list[0],
+ scales[0],
+ shapes[0],
+ zps[0],
+ gidxs[0],
+ self.dtype,
+ )
+ if self.stacked:
+ out[packed_key] = value
+ else:
+ out[(target_patterns or ["weight"])[0]] = value
+ for key, value in input_dict.items():
+ if not any(s in key for s in _PACKED_SUFFIXES) and key not in out:
+ out[key] = value
+ return out
+
+ @property
+ def reverse_op(self):
+ try:
+ from transformers.core_model_loading import _IdentityOp
+ return _IdentityOp()
+ except Exception:
+ return None
+
+
+class _WithOriginalSources:
+ """Run a converter's own op under the source contract it was declared with.
+
+ The decompression step leaves each expert bucket under its `<weight>_packed$`
+ key and the converter's `source_patterns` name the packed metadata. An op
+ that iterates `source_patterns` and requires every one of them present
+ (transformers' ErnieFuseAndSplitTextVisionExperts) then fails on the consumed
+ scale and shape patterns. Here every decompressed bucket is renamed to the
+ plain weight pattern and the op sees the original `source_patterns`, which
+ is exactly what it sees on an unpacked checkpoint.
+ """
+
+ def __init__(
+ self,
+ op,
+ original_sources,
+ weight_sources,
+ receives_buckets = True,
+ ):
+ self.op = op
+ self.original_sources = list(original_sources)
+ self.weight_sources = list(weight_sources)
+ # Later ops in a chain (Concatenate after MergeModulelist) get the previous op's output.
+ self.receives_buckets = receives_buckets
+
+ def convert(
+ self,
+ input_dict,
+ source_patterns = None,
+ target_patterns = None,
+ **kwargs,
+ ):
+ if not self.receives_buckets:
+ return self.op.convert(
+ input_dict,
+ source_patterns = self.original_sources,
+ target_patterns = target_patterns,
+ **kwargs,
+ )
+ keeps_stack = _merge_takes_a_stacked_tensor(self.op)
+ for pattern in self.weight_sources:
+ if (pattern + "_packed$") in input_dict and (pattern + "$") in input_dict:
+ # The loader drops indices of mixed subsets, so a merge would reorder experts.
+ raise RuntimeError(
+ "Unsloth: the compressed-tensors checkpoint packs only some of the experts of "
+ f"`{pattern}`; a partially packed expert bucket cannot be re-quantized on the fly. "
+ "Load it as published without `load_in_4bit = True`."
+ )
+ renamed = {}
+ for key, value in input_dict.items():
+ new_key = key
+ for pattern in self.weight_sources:
+ if key in (pattern + "_packed$", pattern + "$"):
+ new_key = pattern
+ if isinstance(value, torch.Tensor) and not keeps_stack:
+ value = list(value.unbind(0))
+ break
+ renamed[new_key] = value
+ return self.op.convert(
+ renamed,
+ source_patterns = self.original_sources,
+ target_patterns = target_patterns,
+ **kwargs,
+ )
+
+ @property
+ def reverse_op(self):
+ return getattr(self.op, "reverse_op", None)
+
+ def __getattr__(self, name):
+ # copy.deepcopy (the loader copies each converter) probes attributes before `op` exists.
+ try:
+ op = self.__dict__["op"]
+ except KeyError:
+ raise AttributeError(name) from None
+ return getattr(op, name)
+
+
+_with_sources_classes = {}
+
+
+def _merge_takes_a_stacked_tensor(op):
+ try:
+ from transformers.core_model_loading import MergeModulelist
+ except Exception: # pragma: no cover
+ return False
+ if not isinstance(op, MergeModulelist):
+ return False
+ try:
+ source = inspect.getsource(type(op).convert)
+ except (OSError, TypeError):
+ return False
+ return "isinstance(tensors, torch.Tensor)" in source
+
+
+def _with_original_sources(
+ op,
+ original_sources,
+ weight_sources,
+ receives_buckets = True,
+):
+ """Subclass of ``op``'s own class: WeightConverter only allows many-to-many for its Ernie ops."""
+ cls = _with_sources_classes.get(type(op))
+ if cls is None:
+ cls = _with_sources_classes[type(op)] = type(
+ "WithOriginalSources" + type(op).__name__, (_WithOriginalSources, type(op)), {}
+ )
+ return cls(op, original_sources, weight_sources, receives_buckets)
+
+
+_installed = False
+
+
+def install_compressed_tensors_bnb_quantizer() -> bool:
+ """Idempotent; ``False`` on transformers without the weight-conversion loader."""
+ global _installed
+ if _installed:
+ return True
+ if not _transformers_supports_weight_converters():
+ return False
+ from transformers.quantizers import auto as quantizers_auto
+ from transformers.quantizers.quantizer_bnb_4bit import Bnb4BitHfQuantizer
+ from transformers.core_model_loading import ConversionOps, WeightConverter
+
+ op_cls = type("DecompressPackedWeights", (_DecompressPackedWeights, ConversionOps), {})
+ with_sources_cls = _with_original_sources
+
+ class UnslothBnb4BitHfQuantizer(Bnb4BitHfQuantizer):
+ _unsloth_ct_config = None
+ _unsloth_ct_dtype = None
+ _unsloth_dtype_plan = None
+ _unsloth_plan_dicts = ()
+ _unsloth_int4_packed = ()
+ _unsloth_int4_leftover = ()
+
+ def _process_model_before_weight_loading(self, model, **kwargs):
+ config = getattr(model, "config", None)
+ plan = (
+ getattr(config, UNSLOTH_COMPRESSED_TENSORS_ATTR, None)
+ if config is not None
+ else None
+ )
+ if plan is not None:
+ # Read, not popped: device-map planning runs first on meta; popping left random experts (Kimi, 4 GPUs).
+ self._unsloth_ct_config = _build_quantization_config(plan)
+ dtype = kwargs.get("dtype", None)
+ if not isinstance(dtype, torch.dtype):
+ dtype = getattr(model, "dtype", None) or torch.bfloat16
+ self._unsloth_ct_dtype = dtype
+ self._unsloth_dtype_plan = packed_weight_dtype_plan(
+ _checkpoint_keys(kwargs.get("checkpoint_files"))
+ )
+ self._unsloth_plan_dicts = []
+ original_get_dtype_plan = model._get_dtype_plan
+ quantizer = self
+
+ def _get_dtype_plan(dtype, _original = original_get_dtype_plan):
+ merged = dict(_original(dtype))
+ merged.update(quantizer._unsloth_dtype_plan)
+ quantizer._unsloth_plan_dicts.append(merged)
+ return merged
+
+ model._get_dtype_plan = _get_dtype_plan
+ self._unsloth_int4_packed = []
+ from .compressed_tensors_int4 import (
+ int4_packed_route_enabled,
+ int4_packed_supported_plan,
+ )
+
+ self._unsloth_int4_leftover = []
+ if int4_packed_route_enabled() and int4_packed_supported_plan(plan):
+ swapped, leftover = adopt_int4_packed_linears(
+ model,
+ self._unsloth_ct_config,
+ kwargs.get("checkpoint_files"),
+ dtype,
+ getattr(self.quantization_config, "llm_int8_skip_modules", None),
+ )
+ self._unsloth_int4_packed = swapped
+ self._unsloth_int4_leftover = leftover if swapped else []
+ if self._unsloth_int4_packed:
+ # Load as stored: a cast would change int32 words and round a fp32 scale.
+ self._unsloth_dtype_plan[
+ r"\.weight_(packed|scale|zero_point|g_idx|shape)$"
+ ] = None
+ return super()._process_model_before_weight_loading(model, **kwargs)
+
+ def _process_model_after_weight_loading(self, model, **kwargs):
+ config = getattr(model, "config", None)
+ plan = getattr(config, UNSLOTH_COMPRESSED_TENSORS_ATTR, None)
+ if plan is not None:
+ try:
+ delattr(config, UNSLOTH_COMPRESSED_TENSORS_ATTR)
+ except AttributeError:
+ config.__dict__.pop(UNSLOTH_COMPRESSED_TENSORS_ATTR, None)
+ # Load-only: save_pretrained would reverse them into packed names without metadata.
+ drop_load_only_conversions(model)
+ if getattr(self, "_unsloth_int4_packed", None):
+ from .compressed_tensors_int4 import (
+ finalize_int4_packed_linears,
+ refuse_mixed_packed_full_save,
+ save_packed_with_checkpoint_config,
+ )
+ finalize_int4_packed_linears(model, self._unsloth_ct_dtype)
+ # Leftover layers and stacked experts were converted, which the checkpoint config cannot describe.
+ if (
+ plan is not None
+ and not self._unsloth_int4_leftover
+ and not model.__dict__.get(_STACKED_PACKED_EXPERTS)
+ ):
+ save_packed_with_checkpoint_config(model, plan)
+ else:
+ refuse_mixed_packed_full_save(model)
+ return super()._process_model_after_weight_loading(model, **kwargs)
+
+ def _unsloth_keep_storage_dtype(self, target_patterns):
+ for target in target_patterns:
+ key = re.escape(target) + r"$"
+ self._unsloth_dtype_plan[key] = None
+ for plan in self._unsloth_plan_dicts:
+ plan[key] = None
+
+ def update_weight_conversions(self, weight_conversions):
+ ct_config = self._unsloth_ct_config
+ if ct_config is None:
+ return super().update_weight_conversions(weight_conversions)
+ dtype = self._unsloth_ct_dtype
+ updated = []
+ for conv in weight_conversions:
+ if isinstance(conv, WeightConverter) and any(
+ "experts" in p for p in conv.source_patterns
+ ):
+ weight_sources = [p for p in conv.source_patterns if p.endswith(".weight")]
+ if weight_sources:
+ scheme = _scheme_for_sources(ct_config, weight_sources)
+ other = [p for p in conv.source_patterns if not p.endswith(".weight")]
+ # Plain `.weight` kept so `ignore`d unpacked experts still merge; anchored.
+ new_sources = (
+ [p + "_packed$" for p in weight_sources]
+ + [p + "_scale$" for p in weight_sources]
+ + [p + "_shape$" for p in weight_sources]
+ + [p + "_zero_point$" for p in weight_sources]
+ + [p + "_g_idx$" for p in weight_sources]
+ + [p + "$" for p in weight_sources]
+ + other
+ )
+ original_sources = list(
+ getattr(conv, "_original_source_patterns", conv.source_patterns)
+ )
+ conv = WeightConverter(
+ source_patterns = new_sources,
+ target_patterns = conv._original_target_patterns,
+ operations = [op_cls(ct_config, dtype, stacked = True, scheme = scheme)]
+ + [
+ with_sources_cls(
+ op,
+ original_sources,
+ weight_sources,
+ receives_buckets = index == 0,
+ )
+ for index, op in enumerate(conv.operations)
+ ],
+ )
+ self._unsloth_keep_storage_dtype(conv._original_target_patterns)
+ updated.append(conv)
+ if not getattr(self, "_unsloth_int4_packed", None):
+ sources = [s + "$" for s in _PACKED_SUFFIXES]
+ else:
+ # Only modules left out of the packed route; the lookbehind keeps the rename to `weight`.
+ sources = [
+ f"(?<={re.escape(name)}\\.){s}$"
+ for name in getattr(self, "_unsloth_int4_leftover", None) or ()
+ for s in _PACKED_SUFFIXES
+ ]
+ if sources:
+ updated.append(
+ WeightConverter(
+ source_patterns = sources,
+ target_patterns = "weight",
+ operations = [op_cls(ct_config, dtype, stacked = False)],
+ )
+ )
+ parent = getattr(super(), "update_weight_conversions", None)
+ if parent is None:
+ return updated + list(self.get_weight_conversions())
+ return parent(updated)
+
+ mapping = getattr(quantizers_auto, "AUTO_QUANTIZER_MAPPING", None)
+ if mapping is None:
+ return False
+ for key in ("bitsandbytes_4bit",):
+ current = mapping.get(key)
+ if current is Bnb4BitHfQuantizer:
+ mapping[key] = UnslothBnb4BitHfQuantizer
+ elif current is not None and not issubclass(current, Bnb4BitHfQuantizer):
+ return False
+ elif (
+ current is not None
+ and current is not UnslothBnb4BitHfQuantizer
+ and issubclass(current, Bnb4BitHfQuantizer)
+ ):
+ mapping[key] = type(
+ "UnslothBnb4BitHfQuantizer", (UnslothBnb4BitHfQuantizer, current), {}
+ )
+ _installed = True
+ return True
diff --git a/unsloth/models/compressed_tensors_int4.py b/unsloth/models/compressed_tensors_int4.py
new file mode 100644
index 00000000000..d0924625a12
--- /dev/null
+++ b/unsloth/models/compressed_tensors_int4.py
@@ -0,0 +1,438 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved.
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+"""Keep compressed-tensors ``pack-quantized`` INT4/INT8 Linears packed for LoRA training.
+
+The checkpoint's own ``weight_packed`` / ``weight_scale`` / ``weight_zero_point`` / ``weight_g_idx``
+load straight into ``Int4PackedLinear`` (no decompress, no re-quantization). ``.weight`` is the packed
+int32 tensor carrying an ``Int4QuantState``, so Unsloth's LoRA kernels treat it like a bitsandbytes
+weight and decode it exactly per use (kernels/int4_packed.py). ``UNSLOTH_COMPRESSED_TENSORS_INT4=nf4``
+selects the older NF4 re-quantization instead.
+"""
+
+import contextlib
+import functools
+import os
+import types
+
+import torch
+from torch import nn
+
+__all__ = [
+ "Int4PackedLinear",
+ "int4_packed_route_enabled",
+ "int4_packed_supported_plan",
+ "make_int4_packed_linear",
+ "finalize_int4_packed_linears",
+ "INT4_ROUTE_ENV",
+]
+
+INT4_ROUTE_ENV = "UNSLOTH_COMPRESSED_TENSORS_INT4"
+_PACKED_BITS = (2, 4, 8) # 32 // bits must be a power of two for the kernels.
+
+
+def int4_packed_route_enabled() -> bool:
+ route = os.environ.get(INT4_ROUTE_ENV, "packed").strip().lower()
+ if route in ("nf4", "bnb", "bitsandbytes", "0", "off"):
+ return False
+ if not torch.cuda.is_available() or getattr(torch.version, "hip", None):
+ return False
+ try:
+ import triton # noqa: F401
+ except Exception:
+ return False
+ return True
+
+
+def int4_packed_supported_plan(plan) -> bool:
+ """Every config group is a group / channel INT scheme the kernels decode exactly."""
+ groups = (plan or {}).get("config_groups") or {}
+ if not groups:
+ return False
+ for group in groups.values():
+ weights = group.get("weights") or {}
+ if int(weights.get("num_bits", 0)) not in _PACKED_BITS:
+ return False
+ strategy = str(weights.get("strategy", "group")).lower()
+ if strategy not in ("group", "channel"):
+ return False
+ if strategy == "group" and not int(weights.get("group_size") or 0) > 0:
+ return False
+ return True
+
+
+class _Int4LinearFunction(torch.autograd.Function):
+ @staticmethod
+ def forward(ctx, x, packed, module, bias):
+ from ..kernels.int4_packed import int4_matmul
+
+ qs = module.quant_state
+ out = int4_matmul(x.to(qs.dtype), packed, qs, fast = False)
+ if bias is not None:
+ out = out + bias.to(out.dtype)
+ ctx.module = module
+ ctx.x_dtype = x.dtype
+ ctx.save_for_backward(packed)
+ return out
+
+ @staticmethod
+ def backward(ctx, grad_output):
+ from ..kernels.int4_packed import int4_matmul_t
+
+ (packed,) = ctx.saved_tensors
+ grad_x = grad_bias = None
+ if ctx.needs_input_grad[0]:
+ grad_x = int4_matmul_t(grad_output, packed, ctx.module.quant_state).to(ctx.x_dtype)
+ if ctx.needs_input_grad[3]:
+ grad_bias = grad_output.reshape(-1, grad_output.shape[-1]).sum(0)
+ return grad_x, None, None, grad_bias
+
+
+class Int4PackedLinear(nn.Linear):
+ """``nn.Linear`` over a compressed-tensors packed integer weight (frozen)."""
+
+ _unsloth_int4_packed_linear = True
+
+ @property
+ def quant_state(self):
+ from ..kernels.int4_packed import Int4QuantState
+
+ packed = self._parameters["weight_packed"]
+ qs = self.__dict__.get("_int4_quant_state")
+ scale = self._parameters["weight_scale"]
+ zero = self._parameters.get("weight_zero_point")
+ g_idx = self._parameters.get("weight_g_idx")
+ if (
+ qs is None
+ or qs.scale is not scale
+ or qs.zero_point is not zero
+ or (g_idx is not None and qs.g_idx is not g_idx)
+ ):
+ qs = Int4QuantState(
+ scale,
+ zero,
+ g_idx,
+ (self.out_features, self.in_features),
+ self._int4_bits,
+ self._int4_group_size,
+ self.__dict__.get("_int4_dtype") or torch.bfloat16,
+ )
+ qs.layout = self.__dict__.get("_int4_layout")
+ self.__dict__["_int4_quant_state"] = qs
+ packed.quant_state = qs
+ elif getattr(packed, "quant_state", None) is not qs:
+ packed.quant_state = qs
+ return qs
+
+ @property
+ def weight(self):
+ # The packed int32 words with `.quant_state`: the bitsandbytes contract Unsloth's kernels read.
+ packed = self._parameters["weight_packed"]
+ if packed.__dict__.get("quant_state") is None:
+ self.quant_state
+ return packed
+
+ def forward(self, x):
+ bias = self.bias
+ if torch.is_grad_enabled() and (
+ x.requires_grad or (bias is not None and bias.requires_grad)
+ ):
+ return _Int4LinearFunction.apply(x, self._parameters["weight_packed"], self, bias)
+ # Nothing to differentiate: autograd.Function.apply alone costs ~20 us per call (inference is launch bound).
+ from ..kernels.int4_packed import int4_matmul
+
+ qs = self.quant_state
+ out = int4_matmul(x.to(qs.dtype), self._parameters["weight_packed"], qs)
+ return out if bias is None else out + bias.to(out.dtype)
+
+ def dequantize_weight(self, dtype = None):
+ from ..kernels.int4_packed import int4_dequantize
+ qs = self.quant_state
+ return int4_dequantize(self._parameters["weight_packed"], qs, dtype or qs.dtype)
+
+ def reset_parameters(self):
+ return
+
+ def _save_to_state_dict(self, destination, prefix, keep_vars):
+ super()._save_to_state_dict(destination, prefix, keep_vars)
+ # A repacked weight is saved in the checkpoint's own layout; adapter saves drop base weights, so skip the copy.
+ if (
+ self.__dict__.get("_int4_layout")
+ and not _ADAPTER_ONLY_SAVES[0]
+ and prefix + "weight_packed" in destination
+ ):
+ from ..kernels.int4_packed import int4_unpack
+ destination[prefix + "weight_packed"] = int4_unpack(
+ self._parameters["weight_packed"], self.quant_state
+ )
+
+ def _load_from_state_dict(self, state_dict, prefix, *args, **kwargs):
+ super()._load_from_state_dict(state_dict, prefix, *args, **kwargs)
+ # Incoming words are in the checkpoint layout.
+ if prefix + "weight_packed" in state_dict:
+ self.__dict__.pop("_int4_layout", None)
+ self.__dict__.pop("_int4_quant_state", None)
+ self._parameters["weight_packed"].__dict__.pop("quant_state", None)
+
+ def _apply(self, fn, *args, **kwargs):
+ # Casts skip packed words, scales and zero points (fp32 scale stays exact); moves still apply.
+ try:
+ probe = fn(torch.empty(0, dtype = torch.bfloat16))
+ dtype, device = probe.dtype, probe.device
+ except Exception:
+ dtype = device = None
+ frozen = {
+ n: self._parameters.pop(n) for n in list(self._parameters) if n.startswith("weight_")
+ }
+ try:
+ result = super()._apply(fn, *args, **kwargs)
+ finally:
+ for name, param in frozen.items():
+ if param is not None and param.device.type != "meta":
+ target = fn(torch.empty(0, device = param.device)).device
+ if target != param.device:
+ param.data = param.data.to(target)
+ self._parameters[name] = param
+ if dtype is not None and dtype in (torch.float16, torch.bfloat16, torch.float32):
+ self.__dict__["_int4_dtype"] = dtype
+ self.__dict__.pop("_int4_quant_state", None)
+ packed = self._parameters.get("weight_packed")
+ if packed is not None:
+ packed.__dict__.pop("quant_state", None)
+ return result
+
+ def extra_repr(self):
+ return (
+ f"in_features={self.in_features}, out_features={self.out_features}, "
+ f"bias={self.bias is not None}, bits={self._int4_bits}, group_size={self._int4_group_size}"
+ )
+
+
+def make_int4_packed_linear(
+ module,
+ scheme,
+ shapes,
+ dtype,
+ device = "meta",
+):
+ """``Int4PackedLinear`` shell for ``module`` whose parameters take the checkpoint tensors as-is.
+
+ ``shapes`` maps suffix (``weight_packed`` ...) to ``(shape, torch dtype)`` read from the header."""
+ new = Int4PackedLinear.__new__(Int4PackedLinear)
+ nn.Module.__init__(new)
+ new.in_features = module.in_features
+ new.out_features = module.out_features
+ weights = scheme.weights
+ new._int4_bits = int(weights.num_bits)
+ group_size = weights.group_size
+ new._int4_group_size = int(group_size) if group_size else module.in_features
+ new.__dict__["_int4_dtype"] = dtype
+ for suffix in ("weight_packed", "weight_scale", "weight_zero_point", "weight_g_idx"):
+ if suffix not in shapes:
+ continue
+ shape, tdtype = shapes[suffix]
+ new.register_parameter(
+ suffix,
+ nn.Parameter(torch.empty(shape, dtype = tdtype, device = device), requires_grad = False),
+ )
+ if "weight_shape" in shapes:
+ shape, tdtype = shapes["weight_shape"]
+ new.register_buffer("weight_shape", torch.empty(shape, dtype = tdtype, device = device))
+ bias = getattr(module, "bias", None)
+ if isinstance(bias, nn.Parameter):
+ new.bias = bias
+ else:
+ new.register_parameter("bias", None)
+ new.quantization_scheme = scheme
+ return new
+
+
+def finalize_int4_packed_linears(model, dtype = None) -> int:
+ from ..kernels.int4_packed import int4_repack_
+
+ count = 0
+ for module in model.modules():
+ if isinstance(module, Int4PackedLinear):
+ for name, param in module._parameters.items():
+ if name.startswith("weight_") and param is not None:
+ param.requires_grad_(False)
+ if isinstance(dtype, torch.dtype) and dtype.is_floating_point:
+ module.__dict__["_int4_dtype"] = dtype
+ module.__dict__.pop("_int4_quant_state", None)
+ module._parameters["weight_packed"].__dict__.pop("quant_state", None)
+ qs = module.quant_state
+ if qs.layout is None and int4_repack_(module._parameters["weight_packed"], qs):
+ module.__dict__["_int4_layout"] = qs.layout
+ count += 1
+ return count
+
+
+def save_packed_with_checkpoint_config(model, quantization_config) -> None:
+ """A full save writes the packed tensors as loaded, so it must carry the checkpoint's compressed-tensors config
+ (not the runtime bitsandbytes one) for the directory to reload through this route."""
+ original = model.save_pretrained
+ packed = [(name, m) for name, m in model.named_modules() if isinstance(m, Int4PackedLinear)]
+
+ @functools.wraps(original)
+ def save_pretrained(*args, **kwargs):
+ # Layers a LoRA merge densified are written as plain weights: compressed-tensors skips `ignore` entries.
+ dense = [name for name, m in packed if not isinstance(m, Int4PackedLinear)]
+ saved = dict(quantization_config)
+ if dense:
+ saved["ignore"] = list(saved.get("ignore") or []) + dense
+ config = model.config
+ previous = config.__dict__.get("quantization_config", _MISSING)
+ config.quantization_config = saved
+ try:
+ return original(*args, **kwargs)
+ finally:
+ if previous is _MISSING:
+ config.__dict__.pop("quantization_config", None)
+ else:
+ config.quantization_config = previous
+
+ model.save_pretrained = save_pretrained
+
+
+def refuse_mixed_packed_full_save(model) -> None:
+ """Packed layers beside converted ones (stacked experts, leftovers): no one config reloads that mix."""
+ original = model.save_pretrained
+ packed = [m for m in model.modules() if isinstance(m, Int4PackedLinear)]
+
+ @functools.wraps(original)
+ def save_pretrained(*args, **kwargs):
+ if any(isinstance(m, Int4PackedLinear) for m in packed):
+ raise RuntimeError(
+ "Unsloth: This model keeps compressed-tensors packed layers next to converted ones, and no "
+ "single config can reload that mix, so a full save is refused. Save the LoRA adapter with the "
+ "PEFT model's save_pretrained, or reload with UNSLOTH_COMPRESSED_TENSORS_INT4=nf4 for a full "
+ "bitsandbytes save."
+ )
+ return original(*args, **kwargs)
+
+ model.save_pretrained = save_pretrained
+
+
+_MISSING = object()
+_ADAPTER_ONLY_SAVES = [0]
+
+
+def adapter_only_state_dict():
+ """``state_dict()`` calls inside keep repacked words by reference instead of unpacking a copy per layer."""
+ _ADAPTER_ONLY_SAVES[0] += 1
+ try:
+ yield
+ finally:
+ _ADAPTER_ONLY_SAVES[0] -= 1
+
+
+# PEFT merge on a packed base would write into a throwaway view: densify first, restore after.
+_PACKED_STATE = "_unsloth_int4_packed_state"
+
+
+def _swap_class(module, cls):
+ old = module.__dict__.get("_old_forward")
+ hooked = old is not None and getattr(old, "__func__", None) is type(module).forward
+ module.__class__ = cls
+ if hooked:
+ module._old_forward = types.MethodType(cls.forward, module)
+
+
+def _densify(module):
+ weight = module.dequantize_weight()
+ saved = {
+ name: module._parameters.pop(name)
+ for name in list(module._parameters)
+ if name.startswith("weight_")
+ }
+ module.__dict__[_PACKED_STATE] = saved
+ _swap_class(module, nn.Linear)
+ module.weight = nn.Parameter(weight, requires_grad = False)
+
+
+def _restore_packed(module):
+ saved = module.__dict__.pop(_PACKED_STATE)
+ del module._parameters["weight"]
+ _swap_class(module, Int4PackedLinear)
+ for name, param in saved.items():
+ module.register_parameter(name, param)
+ module.__dict__.pop("_int4_quant_state", None)
+
+
+def patch_peft_merge_for_int4_packed_linears() -> bool:
+ try:
+ from peft.tuners.lora.layer import Linear as LoraLinear
+ except Exception:
+ return False
+ if getattr(LoraLinear.merge, "_unsloth_int4_patched", False):
+ return True
+ original_merge = LoraLinear.merge
+ original_unmerge = LoraLinear.unmerge
+
+ @functools.wraps(original_merge)
+ def merge(self, *args, **kwargs):
+ base = self.get_base_layer()
+ if not isinstance(base, Int4PackedLinear):
+ return original_merge(self, *args, **kwargs)
+ _densify(base)
+ try:
+ return original_merge(self, *args, **kwargs)
+ finally:
+ if not self.merged_adapters:
+ _restore_packed(base)
+
+ @functools.wraps(original_unmerge)
+ def unmerge(self, *args, **kwargs):
+ result = original_unmerge(self, *args, **kwargs)
+ base = self.get_base_layer()
+ if _PACKED_STATE in base.__dict__ and not self.merged_adapters:
+ _restore_packed(base)
+ return result
+
+ merge._unsloth_int4_patched = True
+ unmerge._unsloth_int4_patched = True
+ LoraLinear.merge = merge
+ LoraLinear.unmerge = unmerge
+ try:
+ from peft.tuners.tuners_utils import BaseTuner
+ except Exception:
+ return True
+ original_unload = BaseTuner._unload_and_optionally_merge
+
+ @functools.wraps(original_unload)
+ def _unload_and_optionally_merge(self, *args, **kwargs):
+ model = original_unload(self, *args, **kwargs)
+ # Unloaded, a merge is permanent: the stash kept only for unmerge would pin the packed weights.
+ for module in model.modules():
+ module.__dict__.pop(_PACKED_STATE, None)
+ return model
+
+ BaseTuner._unload_and_optionally_merge = _unload_and_optionally_merge
+ try:
+ from peft import PeftModel
+ except Exception:
+ return True
+ original_save = PeftModel.save_pretrained
+
+ @functools.wraps(original_save)
+ def save_pretrained(self, *args, **kwargs):
+ # PEFT builds the adapter from model.state_dict() and keeps only adapter tensors.
+ with adapter_only_state_dict():
+ return original_save(self, *args, **kwargs)
+
+ PeftModel.save_pretrained = save_pretrained
+ return True
+
+
+patch_peft_merge_for_int4_packed_linears()
diff --git a/unsloth/models/llama.py b/unsloth/models/llama.py
index 9b373ec9430..431081f91d4 100644
--- a/unsloth/models/llama.py
+++ b/unsloth/models/llama.py
@@ -40,6 +40,8 @@ from .loader_utils import (
planner_class_mismatch_reason,
planner_model_class,
planner_config_overrides,
+ compressed_tensors_planner_quantization,
+ compressed_tensors_prepared_config,
planner_hub_kwargs,
planner_kwargs_with_max_memory,
planner_quantization_kwargs,
@@ -2610,6 +2612,7 @@ class FastLlamaModel:
from .loader_utils import (
check_and_disable_bitsandbytes_loading,
+ quantization_config_selects_bnb_4bit,
sync_unsloth_model_name_bnb_flags,
)
from unsloth_zoo.utils import get_quant_type
@@ -2617,10 +2620,19 @@ class FastLlamaModel:
load_in_8bit = kwargs.get("load_in_8bit", False)
# Disable bitsandbytes loading if the model has non-bitsandbytes quantization.
- load_in_4bit, load_in_8bit, _ckpt_quant_method = check_and_disable_bitsandbytes_loading(
+ # The loader passes load_in_4bit=False with an explicit config, so a bnb 4-bit config is the request.
+ _user_quantization_config = kwargs.get("quantization_config", None)
+ _explicit_bnb_4bit = _user_quantization_config is not None and (
+ quantization_config_selects_bnb_4bit(_user_quantization_config)
+ )
+ _checked_4bit, _checked_8bit, _ckpt_quant_method = check_and_disable_bitsandbytes_loading(
model_config,
- load_in_4bit = load_in_4bit,
+ load_in_4bit = load_in_4bit or _explicit_bnb_4bit,
load_in_8bit = load_in_8bit,
+ # vLLM reads packed checkpoints itself; a num_labels load stays in-process even with fast_inference.
+ requantize_packed = not _vllm_will_load_weights(fast_inference, num_labels)
+ # A caller's own quantizer must stay authoritative: only a bitsandbytes 4-bit one consumes the plan.
+ and quantization_config_selects_bnb_4bit(_user_quantization_config),
rewrite_modelopt = not _vllm_will_load_weights(fast_inference, num_labels),
token = token,
model_name = model_name,
@@ -2633,6 +2645,10 @@ class FastLlamaModel:
"local_files_only": kwargs.get("local_files_only", False),
},
)
+ # Only an explicit bnb 4-bit config the check kept keeps the caller's flags: a pre-quantized
+ # checkpoint left unarmed (vLLM reads it itself) must not reach vLLM as a bitsandbytes load.
+ if not (_explicit_bnb_4bit and _checked_4bit):
+ load_in_4bit, load_in_8bit = _checked_4bit, _checked_8bit
from .modelopt_fp8 import (
keep_fp8_scale_names_on_save,
move_config_overrides_onto_config,
@@ -2708,6 +2724,9 @@ class FastLlamaModel:
fast_inference = fast_inference,
planner_kwargs = planner_kwargs_with_max_memory(device_map_planner_kwargs, kwargs),
skip_reason = _planner_skip_reason,
+ # Re-quantized packed checkpoint: config.json would size it as compressed-tensors and refuse bnb flags.
+ planner_config = compressed_tensors_prepared_config(model_config),
+ planner_config_reason = "this unsloth_zoo cannot plan from the prepared config of a re-quantized checkpoint",
**planner_config_overrides(kwargs),
token = token,
trust_remote_code = trust_remote_code,
@@ -2719,9 +2738,12 @@ class FastLlamaModel:
# The caller's own config, still untouched in kwargs here, overrides the flags: loader.py clears
# them whenever it forwards one.
**planner_quantization_kwargs(
- load_in_4bit = load_in_4bit,
- load_in_8bit = load_in_8bit,
- quantization_config = kwargs.get("quantization_config", None),
+ **compressed_tensors_planner_quantization(
+ model_config,
+ load_in_4bit,
+ load_in_8bit,
+ kwargs.get("quantization_config", None),
+ ),
rewritten_quantization_config = modelopt_planner_quantization_config(model_config)
if _modelopt_rewritten
else fp8_to_nf4_planner_quantization_config(
@@ -2773,7 +2795,8 @@ class FastLlamaModel:
kwargs.pop("attn_implementation", None) # No need since we auto call it
# Cannot be None, since HF now checks for the config.
- if load_in_4bit:
+ # A caller's own BitsAndBytesConfig stays authoritative (fast_inference forwards load_in_4bit=True).
+ if load_in_4bit and not _explicit_bnb_4bit:
kwargs["quantization_config"] = bnb_config
kwargs = add_dtype_kwargs(dtype, kwargs)
@@ -2852,7 +2875,13 @@ class FastLlamaModel:
)
_prepare_compressed_tensors_model(model)
elif not fast_inference:
- if user_config is not None or _modelopt_rewritten or _fp8_to_nf4:
+ # Re-quantized packed checkpoint: use model_config (quant config dropped), not config.json.
+ from .compressed_tensors_bnb import UNSLOTH_COMPRESSED_TENSORS_ATTR
+
+ _ct_requant = (
+ getattr(model_config, UNSLOTH_COMPRESSED_TENSORS_ATTR, None) is not None
+ )
+ if user_config is not None or _modelopt_rewritten or _ct_requant or _fp8_to_nf4:
# Transformers 5.x @strict model init rejects extra kwargs next to config=, so set the override
# on the config and pass the single config object through.
if max_position_embeddings is not None:
diff --git a/unsloth/models/loader_utils.py b/unsloth/models/loader_utils.py
index 2478ab68f9f..44f550b83f5 100644
--- a/unsloth/models/loader_utils.py
+++ b/unsloth/models/loader_utils.py
@@ -257,6 +257,58 @@ def planner_quantization_kwargs(
return kwargs
+def compressed_tensors_planner_bits(model_config, load_in_4bit, load_in_8bit):
+ """(load_in_4bit, load_in_8bit) to size an armed compressed-tensors load at: the packed route keeps INT8 at 8 bits."""
+ if not load_in_4bit or compressed_tensors_prepared_config(model_config) is None:
+ return load_in_4bit, load_in_8bit
+ try:
+ from .compressed_tensors_bnb import UNSLOTH_COMPRESSED_TENSORS_ATTR
+ from .compressed_tensors_int4 import int4_packed_route_enabled, int4_packed_supported_plan
+
+ plan = getattr(model_config, UNSLOTH_COMPRESSED_TENSORS_ATTR)
+ if int4_packed_route_enabled() and int4_packed_supported_plan(plan):
+ if max(int(g["weights"]["num_bits"]) for g in plan["config_groups"].values()) > 4:
+ return False, True
+ except Exception:
+ pass
+ return load_in_4bit, load_in_8bit
+
+
+def compressed_tensors_planner_quantization(
+ model_config, load_in_4bit, load_in_8bit, quantization_config
+):
+ """Planner flags + config: a packed INT8 load is sized at 8 bits even under an explicit bnb 4-bit config."""
+ wants_4bit = bool(load_in_4bit) or (
+ quantization_config is not None
+ and quantization_config_selects_bnb_4bit(quantization_config)
+ )
+ if (
+ compressed_tensors_planner_bits(model_config, wants_4bit, load_in_8bit) == (False, True)
+ and wants_4bit
+ ):
+ return dict(load_in_4bit = False, load_in_8bit = True, quantization_config = None)
+ return dict(
+ load_in_4bit = load_in_4bit,
+ load_in_8bit = load_in_8bit,
+ quantization_config = quantization_config,
+ )
+
+
+def compressed_tensors_prepared_config(model_config):
+ """`model_config` if armed for packed compressed-tensors re-quantization (plan must size from it), else None."""
+ if model_config is None:
+ return None
+ try:
+ from .compressed_tensors_bnb import UNSLOTH_COMPRESSED_TENSORS_ATTR
+ except Exception:
+ return None
+ return (
+ model_config
+ if getattr(model_config, UNSLOTH_COMPRESSED_TENSORS_ATTR, None) is not None
+ else None
+ )
+
+
def planner_model_class(config, trust_remote_code = False):
"""The model class the planner's own rules pick for `config`, or None if unknown. The planner never sees the auto class the load chose: `config` is whatever the caller passed, while the planner rebuilds the repo's from `model_name`, and the two can disagree."""
try:
@@ -1713,6 +1765,127 @@ def _restore_dropped_fp8_scales(
return (0, 0)
+def _forward_calls_checkpointing(cls):
+ """True when a backbone's own `forward` hands its layers to a checkpoint function."""
+ import inspect
+
+ try:
+ source = inspect.getsource(cls.forward)
+ except Exception:
+ return True # cannot read it: assume it does and change nothing
+ return "_gradient_checkpointing_func" in source or "checkpoint(" in source
+
+
+def _checkpointed_layer_forward(original):
+ @functools.wraps(original)
+ def forward(self, *args, **kwargs):
+ holder = self.__dict__.get("_unsloth_gradient_checkpointing_holder")
+ holder = holder() if holder is not None else None
+ if (
+ not self.training
+ or holder is None
+ or not getattr(holder, "gradient_checkpointing", False)
+ or not torch.is_grad_enabled()
+ # A cache would be written a second time by the recompute.
+ or kwargs.get("past_key_value") is not None
+ or kwargs.get("past_key_values") is not None
+ ):
+ return original(self, *args, **kwargs)
+ if not args and "hidden_states" in kwargs:
+ # A reentrant checkpoint only tracks gradients through positional tensors.
+ args = (kwargs.pop("hidden_states"),)
+ # Grad-requiring kwargs (Kimi-K3 block_residual) go positional: a closure ties recompute to earlier layers.
+ grad_keys = [k for k, v in kwargs.items() if torch.is_tensor(v) and v.requires_grad]
+ n_args = len(args)
+ grad_values = tuple(kwargs.pop(k) for k in grad_keys)
+
+ def run(*inputs):
+ return original(
+ self, *inputs[:n_args], **kwargs, **dict(zip(grad_keys, inputs[n_args:]))
+ )
+
+ # gradient_checkpointing_enable()'s function: keeps offloading and the caller's use_reentrant.
+ checkpoint = getattr(holder, "_gradient_checkpointing_func", None)
+ if checkpoint is None:
+ return torch.utils.checkpoint.checkpoint(run, *args, *grad_values, use_reentrant = False)
+ return checkpoint(run, *args, *grad_values)
+
+ forward._unsloth_manual_checkpoint = True
+ return forward
+
+
+def install_remote_gradient_checkpointing(model, verbose = True):
+ """Wrap decoder layers of remote-code backbones whose loop ignores `self.gradient_checkpointing`."""
+ import types
+ import weakref
+
+ try:
+ from transformers.modeling_layers import GradientCheckpointingLayer
+ except Exception:
+ GradientCheckpointingLayer = ()
+ wrapped = []
+ for _, holder in model.named_modules():
+ holder_cls = type(holder)
+ if "transformers_modules" not in (getattr(holder_cls, "__module__", "") or ""):
+ continue
+ if not hasattr(holder, "gradient_checkpointing"):
+ continue
+ layers = getattr(holder, "layers", None)
+ if not isinstance(layers, torch.nn.ModuleList) or len(layers) == 0:
+ continue
+ if _forward_calls_checkpointing(holder_cls):
+ continue
+ for layer in layers:
+ cls = type(layer)
+ if GradientCheckpointingLayer and isinstance(layer, GradientCheckpointingLayer):
+ continue
+ layer.__dict__["_unsloth_gradient_checkpointing_holder"] = weakref.ref(holder)
+ if not getattr(cls.__dict__.get("forward"), "_unsloth_manual_checkpoint", False):
+ cls.forward = _checkpointed_layer_forward(cls.forward)
+ wrapped.append(cls.__name__)
+ # device_map hooks keep the bound original in `_old_forward`.
+ if "_old_forward" in vars(layer):
+ layer._old_forward = types.MethodType(cls.forward, layer)
+ wrapped = sorted(set(wrapped))
+ if wrapped and verbose:
+ print(
+ "Unsloth: the remote modeling code never calls a checkpoint function; wrapping "
+ + ", ".join(wrapped)
+ + " so gradient checkpointing actually saves activations."
+ )
+ return wrapped
+
+
+def enable_composite_gradient_checkpointing(model, verbose = True):
+ """Mark a remote-code outer class checkpointable when an inner PreTrainedModel is (Kimi-K2.7)."""
+ try:
+ from transformers import PreTrainedModel
+ except Exception:
+ return False
+ if not isinstance(model, PreTrainedModel):
+ return False
+ # Always install: the flag is what transformers checks, the wrapper makes it act.
+ install_remote_gradient_checkpointing(model, verbose = verbose)
+ if getattr(type(model), "supports_gradient_checkpointing", False):
+ return False
+ inner = [
+ type(m).__name__
+ for name, m in model.named_modules()
+ if name
+ and isinstance(m, PreTrainedModel)
+ and getattr(type(m), "supports_gradient_checkpointing", False)
+ ]
+ if not inner:
+ return False
+ type(model).supports_gradient_checkpointing = True
+ if verbose:
+ print(
+ f"Unsloth: {type(model).__name__} did not declare gradient checkpointing support but its "
+ f"{inner[0]} does; enabling it on the outer model."
+ )
+ return True
+
+
def _remove_same_device_compressed_tensors_offload(model):
"""Unwrap compressed-tensors >= 0.19 non-offloading OffloadCaches, which torch.compile cannot trace."""
try:
@@ -2163,6 +2336,9 @@ def warn_if_bitsandbytes_quantized_nothing(
for module in model.modules():
if type(module).__name__ in _BNB_QUANTIZED_TYPES:
return False
+ # compressed-tensors INT4 kept packed (compressed_tensors_int4.py) is 4-bit too.
+ if getattr(module, "_unsloth_int4_packed_linear", False):
+ return False
for param in module.parameters(recurse = False):
if type(param).__name__ in _BNB_QUANTIZED_TYPES:
return False
@@ -2180,6 +2356,7 @@ def check_and_disable_bitsandbytes_loading(
load_in_4bit = True,
load_in_8bit = False,
verbose = True,
+ requantize_packed = True,
rewrite_modelopt = True,
token = None,
model_name = None,
@@ -2187,7 +2364,9 @@ def check_and_disable_bitsandbytes_loading(
hub_kwargs = None,
allow_fp8_to_nf4 = True,
):
- """Disable bitsandbytes loading (load_in_4bit/load_in_8bit) when the model already carries a non-bitsandbytes quantization config. Returns ``(load_in_4bit, load_in_8bit, quant_method)``, with both flags False if they were disabled and quant_method the detected method or None. ``rewrite_modelopt`` converts ModelOpt FP8 to fp8; pass False when vLLM loads it natively."""
+ """Disable bnb flags for non-bnb quantized checkpoints; returns ``(load_in_4bit, load_in_8bit, quant_method)``.
+
+ ``requantize_packed = False`` under fast_inference / full_finetuning: nothing would consume the plan."""
quant_method = get_quant_type(model_config)
if quant_method is None:
# Also under vLLM: it reads the file itself, but the bitsandbytes flags must still drop.
@@ -2204,6 +2383,17 @@ def check_and_disable_bitsandbytes_loading(
if quant_method is None or quant_method == "bitsandbytes":
return load_in_4bit, load_in_8bit, quant_method
+ # Packed compressed-tensors: drop its quant config here and keep load_in_4bit for on-the-fly bnb re-quantization.
+ if (
+ requantize_packed
+ and load_in_4bit
+ and not load_in_8bit
+ and str(quant_method).lower() in ("compressed-tensors", "compressed_tensors", "sparseml")
+ ):
+ from .compressed_tensors_bnb import arm_compressed_tensors_bnb_loading
+ if arm_compressed_tensors_bnb_loading(model_config, verbose = verbose) is not None:
+ return True, False, None
+
if str(quant_method).lower() == "modelopt":
# Whoever loads the weights (vLLM included), a merged_16bit save must dequantize them.
from .modelopt_fp8 import enable_modelopt_merged_save
@@ -2247,6 +2437,25 @@ def check_and_disable_bitsandbytes_loading(
return load_in_4bit, load_in_8bit, quant_method
+def quantization_config_selects_bnb_4bit(quantization_config):
+ """True when ``quantization_config`` keeps the load on bnb 4-bit; other quantizers must keep the checkpoint config."""
+ if quantization_config is None:
+ return True
+ if isinstance(quantization_config, dict):
+ get = quantization_config.get
+ else:
+ get = lambda key, default = None: getattr(quantization_config, key, default)
+ method = get("quant_method", "") or ""
+ # BitsAndBytesConfig stores a QuantizationMethod enum, whose str() is the member name on some Pythons.
+ method = str(getattr(method, "value", method)).lower()
+ # The dict shorthand {"load_in_4bit": True} has no quant_method; transformers reads it as bitsandbytes.
+ if not method and isinstance(quantization_config, dict):
+ method = "bitsandbytes"
+ if "bitsandbytes" not in method:
+ return False
+ return bool(get("load_in_4bit", False)) and not bool(get("load_in_8bit", False))
+
+
def sync_unsloth_model_name_bnb_flags(load_in_4bit, load_in_8bit):
"""Make UNSLOTH_MODEL_NAME's `_load_in_4bit_`/`_load_in_8bit_` tokens match the EFFECTIVE bnb state (after get_model_name remap + check_and_disable). The per-load env is built from the pre-remap config (None for adapter-only PEFT repos), so its tokens can be wrong once the base resolves. Only the gpt-oss patch reads them, so this is gated to gpt-oss."""
name = os.environ.get("UNSLOTH_MODEL_NAME", "")
diff --git a/unsloth/models/remote_moe_shims.py b/unsloth/models/remote_moe_shims.py
new file mode 100644
index 00000000000..6598d1a8cf1
--- /dev/null
+++ b/unsloth/models/remote_moe_shims.py
@@ -0,0 +1,232 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved.
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+"""Train-mode forwards for DeepSeek-V3-derived remote MoE ports (Kimi-K2.7, sarvam) whose gate
+asserts `not self.training` and whose block only computes under no-grad `moe_infer`. Keyed on
+structure, never repo name; no-op for transformers' own models."""
+
+import functools
+
+import torch
+
+__all__ = ["prepare_remote_moe_for_training", "is_remote_deepseek_gate", "is_remote_deepseek_moe"]
+
+
+def _is_remote_code(cls) -> bool:
+ return "transformers_modules" in (getattr(cls, "__module__", "") or "")
+
+
+def is_remote_deepseek_gate(module) -> bool:
+ cls = type(module)
+ return (
+ cls.__name__ == "MoEGate"
+ and _is_remote_code(cls)
+ and all(
+ hasattr(module, a)
+ for a in (
+ "topk_method",
+ "n_group",
+ "topk_group",
+ "routed_scaling_factor",
+ "norm_topk_prob",
+ "top_k",
+ )
+ )
+ )
+
+
+def _calls_moe_infer(nodes) -> bool:
+ import ast
+ return any(
+ isinstance(sub, ast.Attribute) and sub.attr == "moe_infer"
+ for node in nodes
+ for sub in ast.walk(node)
+ )
+
+
+def _has_own_training_dispatch(body) -> bool:
+ import ast
+
+ def value_in_training(node):
+ if isinstance(node, ast.Attribute) and node.attr == "training":
+ return True
+ if isinstance(node, ast.UnaryOp) and isinstance(node.op, ast.Not):
+ inner = value_in_training(node.operand)
+ return None if inner is None else not inner
+ if isinstance(node, ast.BoolOp):
+ values = [value_in_training(value) for value in node.values]
+ decisive = isinstance(node.op, ast.Or)
+ if decisive in values:
+ return decisive
+ if all(value is (not decisive) for value in values):
+ return not decisive
+ return None
+
+ def training_branches(branch):
+ test = branch.test
+ if not any(isinstance(n, ast.Attribute) and n.attr == "training" for n in ast.walk(test)):
+ return []
+ value = value_in_training(test)
+ if value is None:
+ return [branch.body, branch.orelse]
+ return [branch.body] if value else [branch.orelse]
+
+ for node in body:
+ for sub in ast.walk(node):
+ if not isinstance(sub, ast.If):
+ continue
+ for branch in training_branches(sub):
+ if branch and not _calls_moe_infer(branch):
+ return True
+ return False
+
+
+def _forward_has_no_training_branch(cls) -> bool:
+ """True for inference-only ports (training reaches `moe_infer` or nothing); unreadable source -> False."""
+ import ast
+ import inspect
+ import textwrap
+
+ try:
+ tree = ast.parse(textwrap.dedent(inspect.getsource(cls.forward)))
+ except (OSError, TypeError, SyntaxError):
+ return False
+ function = tree.body[0] if tree.body else None
+ if not isinstance(function, (ast.FunctionDef, ast.AsyncFunctionDef)):
+ return False
+ return _calls_moe_infer(function.body) and not _has_own_training_dispatch(function.body)
+
+
+def is_remote_deepseek_moe(module) -> bool:
+ # Structural match: derived code renames the block (sarvam's `SarvamMLAMoE`).
+ cls = type(module)
+ return (
+ _is_remote_code(cls)
+ and isinstance(getattr(module, "experts", None), torch.nn.ModuleList)
+ and hasattr(module, "moe_infer")
+ and hasattr(module, "gate")
+ and _forward_has_no_training_branch(cls)
+ )
+
+
+def _gate_forward_without_training_assert(original):
+ @functools.wraps(original)
+ def forward(self, hidden_states):
+ if not self.training:
+ return original(self, hidden_states)
+ # Gate has no children: flipping `training` only bypasses the assert.
+ self.training = False
+ try:
+ return original(self, hidden_states)
+ finally:
+ self.training = True
+
+ forward._unsloth_remote_moe_shim = True
+ return forward
+
+
+def _moe_train_dispatch(block, x, topk_idx, topk_weight):
+ num_tokens, top_k = topk_idx.shape
+ experts = block.experts
+ flat_idx = topk_idx.reshape(-1)
+ order = torch.argsort(flat_idx, stable = True)
+ sorted_tokens = x[order // top_k]
+ counts = torch.bincount(flat_idx, minlength = len(experts)).tolist()
+ outputs = []
+ start = 0
+ for expert_index, count in enumerate(counts):
+ if count == 0:
+ continue
+ expert = experts[expert_index]
+ outputs.append(expert(sorted_tokens[start : start + count]))
+ start += count
+ if outputs:
+ outs = torch.cat(outputs, dim = 0)
+ else:
+ outs = sorted_tokens.new_zeros((0, x.shape[-1]))
+ unsorted = torch.empty_like(outs)
+ unsorted[order] = outs
+ weighted = unsorted.view(num_tokens, top_k, -1).to(torch.float32) * topk_weight.to(
+ torch.float32
+ ).unsqueeze(-1)
+ return weighted.sum(dim = 1).to(x.dtype)
+
+
+def _moe_forward_with_training_path(original):
+ @functools.wraps(original)
+ def forward(self, hidden_states):
+ if not self.training or getattr(self, "ep_size", 1) > 1:
+ return original(self, hidden_states)
+ identity = hidden_states
+ orig_shape = hidden_states.shape
+ topk_idx, topk_weight = self.gate(hidden_states)
+ flat = hidden_states.view(-1, hidden_states.shape[-1])
+ y = _moe_train_dispatch(self, flat, topk_idx, topk_weight).view(*orig_shape)
+ # Config key varies (sarvam: `num_shared_experts`); the module decides.
+ shared = getattr(self, "shared_experts", None)
+ if shared is not None:
+ y = y + shared(identity)
+ return y
+
+ forward._unsloth_remote_moe_shim = True
+ return forward
+
+
+def _rebind_accelerate_hook(module):
+ # device_map hooks keep the bound original in `_old_forward`, bypassing class patches (multi-GPU Kimi hit the assert).
+ if "_old_forward" not in vars(module):
+ return False
+ import types
+
+ module._old_forward = types.MethodType(type(module).forward, module)
+ return True
+
+
+def prepare_remote_moe_for_training(model, verbose = True):
+ patched = []
+ seen = set()
+ shimmed_classes = set()
+ # Only gates of inference-only blocks: eval mode would drop a trainable gate's aux loss.
+ shimmable_gates = set()
+ for module in model.modules():
+ gate = getattr(module, "gate", None)
+ if gate is not None and is_remote_deepseek_moe(module):
+ shimmable_gates.add(type(gate))
+ for module in model.modules():
+ cls = type(module)
+ if cls in seen:
+ if cls in shimmed_classes:
+ _rebind_accelerate_hook(module)
+ continue
+ seen.add(cls)
+ current = cls.__dict__.get("forward")
+ if getattr(current, "_unsloth_remote_moe_shim", False):
+ shimmed_classes.add(cls)
+ _rebind_accelerate_hook(module)
+ continue
+ if is_remote_deepseek_gate(module) and cls in shimmable_gates:
+ cls.forward = _gate_forward_without_training_assert(cls.forward)
+ patched.append(cls.__name__)
+ shimmed_classes.add(cls)
+ _rebind_accelerate_hook(module)
+ elif is_remote_deepseek_moe(module):
+ cls.forward = _moe_forward_with_training_path(cls.forward)
+ patched.append(cls.__name__)
+ shimmed_classes.add(cls)
+ _rebind_accelerate_hook(module)
+ if patched and verbose:
+ print(
+ "Unsloth: The remote MoE code only had an inference path; added a training forward to "
+ + ", ".join(patched)
+ + "."
+ )
+ return patched
diff --git a/unsloth/models/vision.py b/unsloth/models/vision.py
index 79ac7d3f08b..5853291aa47 100644
--- a/unsloth/models/vision.py
+++ b/unsloth/models/vision.py
@@ -102,6 +102,8 @@ from .loader_utils import (
DEFAULT_DEVICE_MAP,
OFFLOAD_EMBEDDING_AUTO,
planner_config_overrides,
+ compressed_tensors_planner_quantization,
+ compressed_tensors_prepared_config,
planner_hub_kwargs,
planner_kwargs_with_max_memory,
_exclude_rope_inv_freq_from_ddp,
@@ -2220,14 +2222,23 @@ class FastBaseModel:
from .loader_utils import (
check_and_disable_bitsandbytes_loading,
+ quantization_config_selects_bnb_4bit,
sync_unsloth_model_name_bnb_flags,
)
from .fp8_to_nf4 import fp8_to_nf4_planner_quantization_config
- load_in_4bit, load_in_8bit, _ = check_and_disable_bitsandbytes_loading(
+ # The loader passes load_in_4bit=False with an explicit config, so a bnb 4-bit config is the request.
+ _explicit_bnb_4bit = user_quantization_config is not None and (
+ quantization_config_selects_bnb_4bit(user_quantization_config)
+ )
+ _checked_4bit, _checked_8bit, _ = check_and_disable_bitsandbytes_loading(
auto_config,
- load_in_4bit = load_in_4bit,
+ load_in_4bit = load_in_4bit or _explicit_bnb_4bit,
load_in_8bit = load_in_8bit,
+ # Re-quantize needs the transformers 4-bit load: not vLLM, not full finetuning; explicit config must be bnb 4-bit.
+ requantize_packed = not fast_inference
+ and not full_finetuning
+ and quantization_config_selects_bnb_4bit(user_quantization_config),
rewrite_modelopt = not (fast_inference and is_vLLM_available()),
token = token,
model_name = model_name,
@@ -2240,6 +2251,10 @@ class FastBaseModel:
"local_files_only": local_files_only,
},
)
+ # Only an explicit bnb 4-bit config the check kept keeps the caller's flags: a pre-quantized
+ # checkpoint left unarmed (vLLM reads it itself) must not reach vLLM as a bitsandbytes load.
+ if not (_explicit_bnb_4bit and _checked_4bit):
+ load_in_4bit, load_in_8bit = _checked_4bit, _checked_8bit
from .modelopt_fp8 import (
keep_fp8_scale_names_on_save,
keep_task_heads_unquantized,
@@ -2273,6 +2288,13 @@ class FastBaseModel:
# text_only builds the bare decoder; from model_name the planner would plan the whole VLM.
_planner_skip_reason = None
_planner_config = auto_config if text_only_decoder else None
+ _planner_config_reason = "text_only loads a decoder the repo config does not describe"
+ # Re-quantized packed checkpoint: config.json would size it as compressed-tensors and refuse bnb flags.
+ if _planner_config is None and compressed_tensors_prepared_config(auto_config) is not None:
+ _planner_config = auto_config
+ _planner_config_reason = (
+ "this unsloth_zoo cannot plan from the prepared config of a re-quantized checkpoint"
+ )
# Same failure from the other direction: num_labels (or an explicit auto_model) loads a task head whose `score` replaces the planned lm_head, and dispatch refuses a map with no score.weight.
if _planner_skip_reason is None:
_planner_skip_reason = planner_class_mismatch_reason(
@@ -2304,7 +2326,7 @@ class FastBaseModel:
planner_kwargs = planner_kwargs_with_max_memory(device_map_planner_kwargs, kwargs),
skip_reason = _planner_skip_reason,
planner_config = _planner_config,
- planner_config_reason = "text_only loads a decoder the repo config does not describe",
+ planner_config_reason = _planner_config_reason,
**planner_config_overrides(kwargs),
token = token,
trust_remote_code = trust_remote_code,
@@ -2315,9 +2337,9 @@ class FastBaseModel:
**add_dtype_kwargs(torch_dtype),
# A caller-supplied config overrides the flags: loader.py clears them when it forwards one, so the flags alone would size a 4bit load at full precision.
**planner_quantization_kwargs(
- load_in_4bit = load_in_4bit,
- load_in_8bit = load_in_8bit,
- quantization_config = user_quantization_config,
+ **compressed_tensors_planner_quantization(
+ auto_config, load_in_4bit, load_in_8bit, user_quantization_config
+ ),
rewritten_quantization_config = modelopt_planner_quantization_config(
auto_config, dequantize = load_in_16bit
)
@@ -3351,6 +3373,12 @@ class FastBaseModel:
lora_config = LoraConfig(
**{k: v for k, v in local_variables.items() if k in allowed_parameters},
)
+ from .loader_utils import enable_composite_gradient_checkpointing
+ from .remote_moe_shims import prepare_remote_moe_for_training
+
+ enable_composite_gradient_checkpointing(model)
+ prepare_remote_moe_for_training(model)
+ # Block-diagonal grouped linears (DeepSeek-V4's o_a_proj) need a LoRA forward that is grouped too.
_grouped_classes = register_grouped_linear_lora(lora_config, model)
if _grouped_classes:
print(
@@ -3488,6 +3516,11 @@ class FastBaseModel:
torch_checkpoint.checkpoint = _nonre_checkpoint
hf_modeling_utils.checkpoint = _nonre_checkpoint
+ from .loader_utils import enable_composite_gradient_checkpointing
+ from .remote_moe_shims import prepare_remote_moe_for_training
+
+ enable_composite_gradient_checkpointing(model)
+ prepare_remote_moe_for_training(model)
model = prepare_model_for_training(
model,
use_gradient_checkpointing = use_gradient_checkpointing,