Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

fix: kimi 2.6 dequantization is not correct (#2260)

Xin He1ea1d3cddbf8
diff --git a/auto_round/compressors/model_free.py b/auto_round/compressors/model_free.py
index f024b79dd..fd4dbafbf 100644
--- a/auto_round/compressors/model_free.py
+++ b/auto_round/compressors/model_free.py
@@ -117,6 +117,7 @@ from auto_round.logger import logger
from auto_round.schemes import QuantizationScheme, preset_name_to_scheme
from auto_round.utils.common import AUDIO_MM_KEYS, VISION_MM_KEYS, compress_layer_names
from auto_round.utils.device import clear_memory, memory_monitor
+from auto_round.utils.device_manager import default_enable_torch_compile
from auto_round.utils.model_free_utils import (
_LM_HEAD_PATTERNS,
_apply_scheme_overrides,
@@ -187,6 +188,48 @@ _SUPPORTED_MXFP_BITS: tuple[int, ...] = (4, 8)
_NVFP4_E5M3_DATA_TYPE = "nvfp4_v2"
+# Standalone quantization metadata used by different source formats. Only the
+# newly generated quantization_config.json may remain in model-free output.
+_QUANTIZATION_CONFIG_FILENAMES: tuple[str, ...] = (
+ "quantization_config.json",
+ "quantize_config.json",
+ "quant_config.json",
+)
+
+
+def _find_quantization_config(config: Any) -> dict:
+ """Return the first quantization_config found in a nested model config."""
+ if isinstance(config, dict):
+ quantization_config = config.get("quantization_config")
+ if isinstance(quantization_config, dict):
+ return quantization_config
+ for value in config.values():
+ nested = _find_quantization_config(value)
+ if nested:
+ return nested
+ elif isinstance(config, list):
+ for value in config:
+ nested = _find_quantization_config(value)
+ if nested:
+ return nested
+ return {}
+
+
+def _remove_quantization_configs(config: Any) -> None:
+ """Remove every quantization_config entry from a nested model config."""
+ if isinstance(config, dict):
+ config.pop("quantization_config", None)
+ for value in config.values():
+ _remove_quantization_configs(value)
+ elif isinstance(config, list):
+ for value in config:
+ _remove_quantization_configs(value)
+
+
+def _is_stale_quantization_metadata(filename: str) -> bool:
+ return filename in _QUANTIZATION_CONFIG_FILENAMES or filename.lower() == "readme.md"
+
+
# Multimodal keywords kept in full precision by default.
_NONTEXT_KEYWORDS: tuple[str, ...] = VISION_MM_KEYS + AUDIO_MM_KEYS
@@ -493,12 +536,14 @@ class _ModelFreeCompressorCore:
self.device = device
self.quant_lm_head = quant_lm_head
self.quant_nontext_module = quant_nontext_module
- # Model-free is always zero-shot RTN/opt-RTN (each layer is quantized in a
- # single pass), so torch.compile only adds one-off compilation overhead.
- # Keep it off by default, but honor an explicit user request.
+ # Keep default torch.compile policy consistent with standard compressor
+ # behavior, while still honoring explicit user overrides.
if enable_torch_compile is None:
- self.enable_torch_compile = False
- logger.info("`torch.compile` is disabled, as RTN/OPT-RTN quantizes each layer in a single pass")
+ self.enable_torch_compile = default_enable_torch_compile(self.device)
+ logger.info(
+ "`torch.compile` is %s (default policy)",
+ "enabled" if self.enable_torch_compile else "disabled",
+ )
else:
self.enable_torch_compile = enable_torch_compile
logger.info("`torch.compile` is %s", "enabled" if enable_torch_compile else "disabled")
@@ -510,6 +555,7 @@ class _ModelFreeCompressorCore:
self.layer_config: dict = {}
self.ignore_patterns: list[str] = []
self.config: dict = {}
+ self.source_quantization_config: dict = {}
self.fp8_block_size: list | None = None
self.model_type: str = ""
self.is_streaming: bool = False
@@ -690,6 +736,8 @@ class _ModelFreeCompressorCore:
)
self.config = _load_config(self.source_dir)
+ self.source_quantization_config = copy.deepcopy(_find_quantization_config(self.config))
+
def _check_conv1d_and_embedding(self) -> None:
"""Detect Conv1d and embedding layers and automatically add them to the ignore list."""
local_dir = self.work_dir if self.is_streaming else self.source_dir
@@ -733,7 +781,7 @@ class _ModelFreeCompressorCore:
self.ignore_patterns.extend(predefined)
def _detect_fp8_source(self) -> None:
- quant_config = self.config.get("quantization_config", {})
+ quant_config = self.source_quantization_config
is_fp8 = (
quant_config.get("quant_method") == "fp8"
or quant_config.get("quantization_type") == "fp8"
@@ -960,7 +1008,7 @@ class _ModelFreeCompressorCore:
ignore_patterns=self.ignore_patterns,
fp8_block_size=self.fp8_block_size,
model_type=self.model_type,
- source_quantization_config=self.config.get("quantization_config", {}),
+ source_quantization_config=self.source_quantization_config,
enable_torch_compile=self.enable_torch_compile,
disable_opt_rtn=self.disable_opt_rtn,
quant_output_dir=self._quant_output_dir,
@@ -1152,7 +1200,7 @@ class _ModelFreeCompressorCore:
ignore_patterns=self.ignore_patterns,
fp8_block_size=self.fp8_block_size,
model_type=self.model_type,
- source_quantization_config=self.config.get("quantization_config", {}),
+ source_quantization_config=self.source_quantization_config,
quant_output_dir=self._quant_output_dir,
total_shards=total_shards,
enable_torch_compile=self.enable_torch_compile,
@@ -1201,6 +1249,17 @@ class _ModelFreeCompressorCore:
def _write_index(self) -> None:
_write_index_file(self._quant_output_dir, self.output_weight_map)
+ def _remove_stale_quantization_config_files(self) -> None:
+ """Remove source/output quantization metadata before writing the new config."""
+ for directory in {self.output_dir, self._quant_output_dir}:
+ if not os.path.isdir(directory):
+ continue
+ for filename in os.listdir(directory):
+ if _is_stale_quantization_metadata(filename):
+ path = os.path.join(directory, filename)
+ if os.path.isfile(path):
+ os.remove(path)
+
def _write_config_files(self) -> None:
block_prefixes = []
for layer_name in self.all_quantized_layers:
@@ -1222,6 +1281,8 @@ class _ModelFreeCompressorCore:
format=self.format,
)
+ self._remove_stale_quantization_config_files()
+ _remove_quantization_configs(self.config)
self.config["quantization_config"] = quantization_config
with open(os.path.join(self._quant_output_dir, "config.json"), "w") as f:
json.dump(self.config, f, indent=2)
@@ -1243,6 +1304,8 @@ class _ModelFreeCompressorCore:
# copytree's ``not os.path.exists(dst)`` guard prevents
# overwriting it.
for fname in os.listdir(self.diffusion_root_dir):
+ if _is_stale_quantization_metadata(fname):
+ continue
src = os.path.join(self.diffusion_root_dir, fname)
dst = os.path.join(self.output_dir, fname)
if os.path.isdir(src):
@@ -1253,7 +1316,7 @@ class _ModelFreeCompressorCore:
return
for fname in os.listdir(self.source_dir):
- if _is_weight_shard(fname):
+ if _is_weight_shard(fname) or _is_stale_quantization_metadata(fname):
continue
src = os.path.join(self.source_dir, fname)
dst = os.path.join(self.output_dir, fname)
diff --git a/auto_round/utils/model_free_utils.py b/auto_round/utils/model_free_utils.py
index 0aa968938..deb495118 100644
--- a/auto_round/utils/model_free_utils.py
+++ b/auto_round/utils/model_free_utils.py
@@ -2521,7 +2521,7 @@ def _dequant_kimi_k25_int4_tensors(
if not entries:
return raw_tensors
- default_group_size, default_sym = _resolve_kimi_k25_int4_params(source_quant_config)
+ default_group_size, _ = _resolve_kimi_k25_int4_params(source_quant_config)
dequant_device = str(device or "cpu")
shard_prefix = f"[{shard_name}] " if shard_name else ""
logger.info(
@@ -2583,10 +2583,9 @@ def _dequant_kimi_k25_int4_tensors(
zp = zp[:, : q.shape[1]].to(torch.int16)
q = q - zp
else:
- if default_sym:
- q = torch.where(q >= 8, q - 16, q)
- else:
- q = q - 8
+ # Kimi RAWINT4 stores signed values with a fixed bias of 8:
+ # logical_q = nibble - 8, not 4-bit two's-complement.
+ q = q - 8
sc = sc.repeat_interleave(group_size, dim=1)[:, : q.shape[1]]
return (q.to(torch.float32) * sc).to(torch.bfloat16)
diff --git a/test/unit/common/core/test_init.py b/test/unit/common/core/test_init.py
index fdc16cea5..ef50a5ef7 100644
--- a/test/unit/common/core/test_init.py
+++ b/test/unit/common/core/test_init.py
@@ -124,33 +124,6 @@ def _assert_compile(ar, expected: bool):
assert compress_context.enable_torch_compile is expected
-def test_torch_compile_disabled_for_rtn_and_short_signround(tiny_opt_model_path, monkeypatch):
- """RTN / opt-RTN and `iters` < 10 never amortize the torch.compile cost."""
- # Pin the platform default to "enabled" so the algorithm heuristic is observable
- # regardless of the host device / OS.
- monkeypatch.setattr("auto_round.compressors.base.default_enable_torch_compile", lambda *a, **k: True)
-
- # Plain RTN (routes to the model-free compressor) and opt-RTN both stay off.
- for disable_opt_rtn in (True, False):
- ar = AutoRound(
- model=tiny_opt_model_path,
- scheme="W4A16",
- iters=0,
- nsamples=1,
- disable_opt_rtn=disable_opt_rtn,
- )
- _assert_compile(ar, False)
-
- # SignRound with too few iterations.
- for iters in (1, MIN_ITERS_FOR_TORCH_COMPILE - 1):
- ar = AutoRound(model=tiny_opt_model_path, scheme="W4A16", iters=iters, nsamples=1)
- _assert_compile(ar, False)
-
- # Enough iterations to pay back the compilation cost.
- ar = AutoRound(model=tiny_opt_model_path, scheme="W4A16", iters=MIN_ITERS_FOR_TORCH_COMPILE, nsamples=1)
- _assert_compile(ar, True)
-
-
def test_explicit_torch_compile_overrides_algorithm_heuristic(tiny_opt_model_path, monkeypatch):
"""An explicit `enable_torch_compile` is always honored, even for RTN / tiny iters."""
monkeypatch.setattr("auto_round.compressors.base.default_enable_torch_compile", lambda *a, **k: False)
diff --git a/test/unit/common/quantization/test_model_free.py b/test/unit/common/quantization/test_model_free.py
index 5d70475af..172959623 100644
--- a/test/unit/common/quantization/test_model_free.py
+++ b/test/unit/common/quantization/test_model_free.py
@@ -119,6 +119,26 @@ def test_model_free_entry_preserves_enabled_opt_rtn(tiny_opt_model_path):
assert compressor.disable_opt_rtn is False
[email protected]("default_enabled", [True, False])
+def test_model_free_core_uses_default_torch_compile_policy_when_unset(tmp_path, monkeypatch, default_enabled):
+ model_dir = _make_model_dir(tmp_path, _SIMPLE_CONFIG, _SIMPLE_TENSORS)
+ output_dir = str(tmp_path / "output")
+ os.makedirs(output_dir, exist_ok=True)
+
+ monkeypatch.setattr(
+ "auto_round.compressors.model_free.default_enable_torch_compile", lambda _device: default_enabled
+ )
+
+ core = _ModelFreeCompressorCore(
+ model_name_or_path=model_dir,
+ output_dir=output_dir,
+ scheme="W4A16",
+ enable_torch_compile=None,
+ )
+
+ assert core.enable_torch_compile is default_enabled
+
+
from ...envs import require_compressed_tensors
# ---------------------------------------------------------------------------
@@ -1140,6 +1160,50 @@ class TestKimiK25Int4Source:
assert os.path.isdir(os.path.join(output_dir, "tokenizer", "nested"))
assert os.path.isfile(os.path.join(output_dir, "tokenizer", "nested", "vocab.txt"))
+ def test_replaces_source_quantization_metadata(self, tmp_path):
+ config = dict(_SIMPLE_CONFIG)
+ config["text_config"] = {
+ "quantization_config": {
+ "quant_method": "gptq",
+ "bits": 4,
+ "stale_source_field": True,
+ }
+ }
+ model_dir = _make_model_dir(tmp_path, config, _SIMPLE_TENSORS)
+ for filename in ("quantization_config.json", "quantize_config.json", "quant_config.json"):
+ with open(os.path.join(model_dir, filename), "w") as f:
+ json.dump({"quant_method": "gptq", "stale_source_field": True}, f)
+ with open(os.path.join(model_dir, "README.md"), "w") as f:
+ f.write("stale source quantization documentation")
+
+ output_dir = str(tmp_path / "output")
+ os.makedirs(output_dir)
+ with open(os.path.join(output_dir, "quantize_config.json"), "w") as f:
+ json.dump({"stale_output_field": True}, f)
+ with open(os.path.join(output_dir, "README.md"), "w") as f:
+ f.write("stale output quantization documentation")
+
+ core = _ModelFreeCompressorCore(
+ model_name_or_path=model_dir,
+ output_dir=output_dir,
+ scheme="W4A16",
+ )
+ core.run()
+
+ with open(os.path.join(output_dir, "config.json")) as f:
+ output_config = json.load(f)
+ with open(os.path.join(output_dir, "quantization_config.json")) as f:
+ standalone_config = json.load(f)
+
+ assert output_config["quantization_config"] == standalone_config
+ assert standalone_config["quant_method"] == "auto-round"
+ assert "stale_source_field" not in standalone_config
+ assert core.source_quantization_config["stale_source_field"] is True
+ assert "quantization_config" not in output_config["text_config"]
+ assert not os.path.exists(os.path.join(output_dir, "quantize_config.json"))
+ assert not os.path.exists(os.path.join(output_dir, "quant_config.json"))
+ assert not os.path.exists(os.path.join(output_dir, "README.md"))
+
def test_diffusion_copies_subfolders(self, tmp_path):
"""Diffusion model: non-transformer subdirectories should be copied."""
root_dir = _make_diffusion_model_dir(tmp_path, _TRANSFORMER_CONFIG, _TRANSFORMER_TENSORS)
diff --git a/test/unit/test_cpu/utils/test_model_free_utils.py b/test/unit/test_cpu/utils/test_model_free_utils.py
index 4c7568a31..5cbb2bfc9 100644
--- a/test/unit/test_cpu/utils/test_model_free_utils.py
+++ b/test/unit/test_cpu/utils/test_model_free_utils.py
@@ -1394,6 +1394,25 @@ class TestFP8Source:
class TestKimiK25Int4Source:
+ def test_kimi_k25_int4_dequant_uses_biased_signed_values(self):
+ nibbles = torch.arange(16, dtype=torch.int32).reshape(2, 8)
+ shifts = torch.arange(8, dtype=torch.int32) * 4
+ packed = (nibbles << shifts).sum(dim=1, keepdim=True).to(torch.int32)
+ raw = {
+ "layer.weight_packed": packed,
+ "layer.weight_scale": torch.full((2, 1), 0.5, dtype=torch.float16),
+ }
+
+ out = _handle_model_type_low_precision_source_tensors(
+ raw,
+ model_type="kimi_k25",
+ source_quant_config=None,
+ device="cpu",
+ )
+
+ expected = ((nibbles - 8).float() * 0.5).to(torch.bfloat16)
+ assert torch.equal(out["layer.weight"], expected)
+
def test_kimi_k25_int4_dequant_helper(self):
raw = {
"layer.weight_packed": torch.randint(0, 255, (128, 64), dtype=torch.uint8),
@@ -1413,12 +1432,14 @@ class TestKimiK25Int4Source:
@require_compressed_tensors
def test_kimi_k25_int4_to_mxfp4_via_model_free(self, tmp_path):
+ config = {key: value for key, value in _KIMI_K25_CFG.items() if key != "quantization_config"}
+ config["text_config"] = {"quantization_config": _KIMI_K25_CFG["quantization_config"]}
tensors = {
"model.layers.0.mlp.fc1.weight_packed": torch.randint(0, 255, (128, 64), dtype=torch.uint8),
"model.layers.0.mlp.fc1.weight_scale": torch.ones(128, 16, dtype=torch.float16),
"lm_head.weight": torch.randn(1000, 128),
}
- model_dir = _make_model_dir(tmp_path, _KIMI_K25_CFG, tensors)
+ model_dir = _make_model_dir(tmp_path, config, tensors)
output_dir = str(tmp_path / "output")
_ModelFreeCompressorCore(
@@ -1431,6 +1452,9 @@ class TestKimiK25Int4Source:
qc = _read_qconfig(output_dir)
assert qc["format"] == "mxfp4-pack-quantized"
assert qc["quant_method"] == "compressed-tensors"
+ with open(os.path.join(output_dir, "config.json")) as f:
+ output_config = json.load(f)
+ assert "quantization_config" not in output_config["text_config"]
found_scale_dtype = None
found_packed_shape = None