PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
fix: kimi 2.6 dequantization is not correct (#2260)
Xin He1ea1d3cddbf8
diff --git a/auto_round/compressors/model_free.py b/auto_round/compressors/model_free.pyindex f024b79dd..fd4dbafbf 100644--- a/auto_round/compressors/model_free.py+++ b/auto_round/compressors/model_free.py@@ -117,6 +117,7 @@ from auto_round.logger import loggerfrom auto_round.schemes import QuantizationScheme, preset_name_to_schemefrom auto_round.utils.common import AUDIO_MM_KEYS, VISION_MM_KEYS, compress_layer_namesfrom auto_round.utils.device import clear_memory, memory_monitor+from auto_round.utils.device_manager import default_enable_torch_compilefrom auto_round.utils.model_free_utils import (_LM_HEAD_PATTERNS,_apply_scheme_overrides,@@ -187,6 +188,48 @@ _SUPPORTED_MXFP_BITS: tuple[int, ...] = (4, 8)_NVFP4_E5M3_DATA_TYPE = "nvfp4_v2"+# Standalone quantization metadata used by different source formats. Only the+# newly generated quantization_config.json may remain in model-free output.+_QUANTIZATION_CONFIG_FILENAMES: tuple[str, ...] = (+ "quantization_config.json",+ "quantize_config.json",+ "quant_config.json",+)+++def _find_quantization_config(config: Any) -> dict:+ """Return the first quantization_config found in a nested model config."""+ if isinstance(config, dict):+ quantization_config = config.get("quantization_config")+ if isinstance(quantization_config, dict):+ return quantization_config+ for value in config.values():+ nested = _find_quantization_config(value)+ if nested:+ return nested+ elif isinstance(config, list):+ for value in config:+ nested = _find_quantization_config(value)+ if nested:+ return nested+ return {}+++def _remove_quantization_configs(config: Any) -> None:+ """Remove every quantization_config entry from a nested model config."""+ if isinstance(config, dict):+ config.pop("quantization_config", None)+ for value in config.values():+ _remove_quantization_configs(value)+ elif isinstance(config, list):+ for value in config:+ _remove_quantization_configs(value)+++def _is_stale_quantization_metadata(filename: str) -> bool:+ return filename in _QUANTIZATION_CONFIG_FILENAMES or filename.lower() == "readme.md"++# Multimodal keywords kept in full precision by default._NONTEXT_KEYWORDS: tuple[str, ...] = VISION_MM_KEYS + AUDIO_MM_KEYS@@ -493,12 +536,14 @@ class _ModelFreeCompressorCore:self.device = deviceself.quant_lm_head = quant_lm_headself.quant_nontext_module = quant_nontext_module- # Model-free is always zero-shot RTN/opt-RTN (each layer is quantized in a- # single pass), so torch.compile only adds one-off compilation overhead.- # Keep it off by default, but honor an explicit user request.+ # Keep default torch.compile policy consistent with standard compressor+ # behavior, while still honoring explicit user overrides.if enable_torch_compile is None:- self.enable_torch_compile = False- logger.info("`torch.compile` is disabled, as RTN/OPT-RTN quantizes each layer in a single pass")+ self.enable_torch_compile = default_enable_torch_compile(self.device)+ logger.info(+ "`torch.compile` is %s (default policy)",+ "enabled" if self.enable_torch_compile else "disabled",+ )else:self.enable_torch_compile = enable_torch_compilelogger.info("`torch.compile` is %s", "enabled" if enable_torch_compile else "disabled")@@ -510,6 +555,7 @@ class _ModelFreeCompressorCore:self.layer_config: dict = {}self.ignore_patterns: list[str] = []self.config: dict = {}+ self.source_quantization_config: dict = {}self.fp8_block_size: list | None = Noneself.model_type: str = ""self.is_streaming: bool = False@@ -690,6 +736,8 @@ class _ModelFreeCompressorCore:)self.config = _load_config(self.source_dir)+ self.source_quantization_config = copy.deepcopy(_find_quantization_config(self.config))+def _check_conv1d_and_embedding(self) -> None:"""Detect Conv1d and embedding layers and automatically add them to the ignore list."""local_dir = self.work_dir if self.is_streaming else self.source_dir@@ -733,7 +781,7 @@ class _ModelFreeCompressorCore:self.ignore_patterns.extend(predefined)def _detect_fp8_source(self) -> None:- quant_config = self.config.get("quantization_config", {})+ quant_config = self.source_quantization_configis_fp8 = (quant_config.get("quant_method") == "fp8"or quant_config.get("quantization_type") == "fp8"@@ -960,7 +1008,7 @@ class _ModelFreeCompressorCore:ignore_patterns=self.ignore_patterns,fp8_block_size=self.fp8_block_size,model_type=self.model_type,- source_quantization_config=self.config.get("quantization_config", {}),+ source_quantization_config=self.source_quantization_config,enable_torch_compile=self.enable_torch_compile,disable_opt_rtn=self.disable_opt_rtn,quant_output_dir=self._quant_output_dir,@@ -1152,7 +1200,7 @@ class _ModelFreeCompressorCore:ignore_patterns=self.ignore_patterns,fp8_block_size=self.fp8_block_size,model_type=self.model_type,- source_quantization_config=self.config.get("quantization_config", {}),+ source_quantization_config=self.source_quantization_config,quant_output_dir=self._quant_output_dir,total_shards=total_shards,enable_torch_compile=self.enable_torch_compile,@@ -1201,6 +1249,17 @@ class _ModelFreeCompressorCore:def _write_index(self) -> None:_write_index_file(self._quant_output_dir, self.output_weight_map)+ def _remove_stale_quantization_config_files(self) -> None:+ """Remove source/output quantization metadata before writing the new config."""+ for directory in {self.output_dir, self._quant_output_dir}:+ if not os.path.isdir(directory):+ continue+ for filename in os.listdir(directory):+ if _is_stale_quantization_metadata(filename):+ path = os.path.join(directory, filename)+ if os.path.isfile(path):+ os.remove(path)+def _write_config_files(self) -> None:block_prefixes = []for layer_name in self.all_quantized_layers:@@ -1222,6 +1281,8 @@ class _ModelFreeCompressorCore:format=self.format,)+ self._remove_stale_quantization_config_files()+ _remove_quantization_configs(self.config)self.config["quantization_config"] = quantization_configwith open(os.path.join(self._quant_output_dir, "config.json"), "w") as f:json.dump(self.config, f, indent=2)@@ -1243,6 +1304,8 @@ class _ModelFreeCompressorCore:# copytree's ``not os.path.exists(dst)`` guard prevents# overwriting it.for fname in os.listdir(self.diffusion_root_dir):+ if _is_stale_quantization_metadata(fname):+ continuesrc = os.path.join(self.diffusion_root_dir, fname)dst = os.path.join(self.output_dir, fname)if os.path.isdir(src):@@ -1253,7 +1316,7 @@ class _ModelFreeCompressorCore:returnfor fname in os.listdir(self.source_dir):- if _is_weight_shard(fname):+ if _is_weight_shard(fname) or _is_stale_quantization_metadata(fname):continuesrc = os.path.join(self.source_dir, fname)dst = os.path.join(self.output_dir, fname)diff --git a/auto_round/utils/model_free_utils.py b/auto_round/utils/model_free_utils.pyindex 0aa968938..deb495118 100644--- a/auto_round/utils/model_free_utils.py+++ b/auto_round/utils/model_free_utils.py@@ -2521,7 +2521,7 @@ def _dequant_kimi_k25_int4_tensors(if not entries:return raw_tensors- default_group_size, default_sym = _resolve_kimi_k25_int4_params(source_quant_config)+ default_group_size, _ = _resolve_kimi_k25_int4_params(source_quant_config)dequant_device = str(device or "cpu")shard_prefix = f"[{shard_name}] " if shard_name else ""logger.info(@@ -2583,10 +2583,9 @@ def _dequant_kimi_k25_int4_tensors(zp = zp[:, : q.shape[1]].to(torch.int16)q = q - zpelse:- if default_sym:- q = torch.where(q >= 8, q - 16, q)- else:- q = q - 8+ # Kimi RAWINT4 stores signed values with a fixed bias of 8:+ # logical_q = nibble - 8, not 4-bit two's-complement.+ q = q - 8sc = sc.repeat_interleave(group_size, dim=1)[:, : q.shape[1]]return (q.to(torch.float32) * sc).to(torch.bfloat16)diff --git a/test/unit/common/core/test_init.py b/test/unit/common/core/test_init.pyindex fdc16cea5..ef50a5ef7 100644--- a/test/unit/common/core/test_init.py+++ b/test/unit/common/core/test_init.py@@ -124,33 +124,6 @@ def _assert_compile(ar, expected: bool):assert compress_context.enable_torch_compile is expected-def test_torch_compile_disabled_for_rtn_and_short_signround(tiny_opt_model_path, monkeypatch):- """RTN / opt-RTN and `iters` < 10 never amortize the torch.compile cost."""- # Pin the platform default to "enabled" so the algorithm heuristic is observable- # regardless of the host device / OS.- monkeypatch.setattr("auto_round.compressors.base.default_enable_torch_compile", lambda *a, **k: True)-- # Plain RTN (routes to the model-free compressor) and opt-RTN both stay off.- for disable_opt_rtn in (True, False):- ar = AutoRound(- model=tiny_opt_model_path,- scheme="W4A16",- iters=0,- nsamples=1,- disable_opt_rtn=disable_opt_rtn,- )- _assert_compile(ar, False)-- # SignRound with too few iterations.- for iters in (1, MIN_ITERS_FOR_TORCH_COMPILE - 1):- ar = AutoRound(model=tiny_opt_model_path, scheme="W4A16", iters=iters, nsamples=1)- _assert_compile(ar, False)-- # Enough iterations to pay back the compilation cost.- ar = AutoRound(model=tiny_opt_model_path, scheme="W4A16", iters=MIN_ITERS_FOR_TORCH_COMPILE, nsamples=1)- _assert_compile(ar, True)--def test_explicit_torch_compile_overrides_algorithm_heuristic(tiny_opt_model_path, monkeypatch):"""An explicit `enable_torch_compile` is always honored, even for RTN / tiny iters."""monkeypatch.setattr("auto_round.compressors.base.default_enable_torch_compile", lambda *a, **k: False)diff --git a/test/unit/common/quantization/test_model_free.py b/test/unit/common/quantization/test_model_free.pyindex 5d70475af..172959623 100644--- a/test/unit/common/quantization/test_model_free.py+++ b/test/unit/common/quantization/test_model_free.py@@ -119,6 +119,26 @@ def test_model_free_entry_preserves_enabled_opt_rtn(tiny_opt_model_path):assert compressor.disable_opt_rtn is False[email protected]("default_enabled", [True, False])+def test_model_free_core_uses_default_torch_compile_policy_when_unset(tmp_path, monkeypatch, default_enabled):+ model_dir = _make_model_dir(tmp_path, _SIMPLE_CONFIG, _SIMPLE_TENSORS)+ output_dir = str(tmp_path / "output")+ os.makedirs(output_dir, exist_ok=True)++ monkeypatch.setattr(+ "auto_round.compressors.model_free.default_enable_torch_compile", lambda _device: default_enabled+ )++ core = _ModelFreeCompressorCore(+ model_name_or_path=model_dir,+ output_dir=output_dir,+ scheme="W4A16",+ enable_torch_compile=None,+ )++ assert core.enable_torch_compile is default_enabled++from ...envs import require_compressed_tensors# ---------------------------------------------------------------------------@@ -1140,6 +1160,50 @@ class TestKimiK25Int4Source:assert os.path.isdir(os.path.join(output_dir, "tokenizer", "nested"))assert os.path.isfile(os.path.join(output_dir, "tokenizer", "nested", "vocab.txt"))+ def test_replaces_source_quantization_metadata(self, tmp_path):+ config = dict(_SIMPLE_CONFIG)+ config["text_config"] = {+ "quantization_config": {+ "quant_method": "gptq",+ "bits": 4,+ "stale_source_field": True,+ }+ }+ model_dir = _make_model_dir(tmp_path, config, _SIMPLE_TENSORS)+ for filename in ("quantization_config.json", "quantize_config.json", "quant_config.json"):+ with open(os.path.join(model_dir, filename), "w") as f:+ json.dump({"quant_method": "gptq", "stale_source_field": True}, f)+ with open(os.path.join(model_dir, "README.md"), "w") as f:+ f.write("stale source quantization documentation")++ output_dir = str(tmp_path / "output")+ os.makedirs(output_dir)+ with open(os.path.join(output_dir, "quantize_config.json"), "w") as f:+ json.dump({"stale_output_field": True}, f)+ with open(os.path.join(output_dir, "README.md"), "w") as f:+ f.write("stale output quantization documentation")++ core = _ModelFreeCompressorCore(+ model_name_or_path=model_dir,+ output_dir=output_dir,+ scheme="W4A16",+ )+ core.run()++ with open(os.path.join(output_dir, "config.json")) as f:+ output_config = json.load(f)+ with open(os.path.join(output_dir, "quantization_config.json")) as f:+ standalone_config = json.load(f)++ assert output_config["quantization_config"] == standalone_config+ assert standalone_config["quant_method"] == "auto-round"+ assert "stale_source_field" not in standalone_config+ assert core.source_quantization_config["stale_source_field"] is True+ assert "quantization_config" not in output_config["text_config"]+ assert not os.path.exists(os.path.join(output_dir, "quantize_config.json"))+ assert not os.path.exists(os.path.join(output_dir, "quant_config.json"))+ assert not os.path.exists(os.path.join(output_dir, "README.md"))+def test_diffusion_copies_subfolders(self, tmp_path):"""Diffusion model: non-transformer subdirectories should be copied."""root_dir = _make_diffusion_model_dir(tmp_path, _TRANSFORMER_CONFIG, _TRANSFORMER_TENSORS)diff --git a/test/unit/test_cpu/utils/test_model_free_utils.py b/test/unit/test_cpu/utils/test_model_free_utils.pyindex 4c7568a31..5cbb2bfc9 100644--- a/test/unit/test_cpu/utils/test_model_free_utils.py+++ b/test/unit/test_cpu/utils/test_model_free_utils.py@@ -1394,6 +1394,25 @@ class TestFP8Source:class TestKimiK25Int4Source:+ def test_kimi_k25_int4_dequant_uses_biased_signed_values(self):+ nibbles = torch.arange(16, dtype=torch.int32).reshape(2, 8)+ shifts = torch.arange(8, dtype=torch.int32) * 4+ packed = (nibbles << shifts).sum(dim=1, keepdim=True).to(torch.int32)+ raw = {+ "layer.weight_packed": packed,+ "layer.weight_scale": torch.full((2, 1), 0.5, dtype=torch.float16),+ }++ out = _handle_model_type_low_precision_source_tensors(+ raw,+ model_type="kimi_k25",+ source_quant_config=None,+ device="cpu",+ )++ expected = ((nibbles - 8).float() * 0.5).to(torch.bfloat16)+ assert torch.equal(out["layer.weight"], expected)+def test_kimi_k25_int4_dequant_helper(self):raw = {"layer.weight_packed": torch.randint(0, 255, (128, 64), dtype=torch.uint8),@@ -1413,12 +1432,14 @@ class TestKimiK25Int4Source:@require_compressed_tensorsdef test_kimi_k25_int4_to_mxfp4_via_model_free(self, tmp_path):+ config = {key: value for key, value in _KIMI_K25_CFG.items() if key != "quantization_config"}+ config["text_config"] = {"quantization_config": _KIMI_K25_CFG["quantization_config"]}tensors = {"model.layers.0.mlp.fc1.weight_packed": torch.randint(0, 255, (128, 64), dtype=torch.uint8),"model.layers.0.mlp.fc1.weight_scale": torch.ones(128, 16, dtype=torch.float16),"lm_head.weight": torch.randn(1000, 128),}- model_dir = _make_model_dir(tmp_path, _KIMI_K25_CFG, tensors)+ model_dir = _make_model_dir(tmp_path, config, tensors)output_dir = str(tmp_path / "output")_ModelFreeCompressorCore(@@ -1431,6 +1452,9 @@ class TestKimiK25Int4Source:qc = _read_qconfig(output_dir)assert qc["format"] == "mxfp4-pack-quantized"assert qc["quant_method"] == "compressed-tensors"+ with open(os.path.join(output_dir, "config.json")) as f:+ output_config = json.load(f)+ assert "quantization_config" not in output_config["text_config"]found_scale_dtype = Nonefound_packed_shape = None