PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
Add support for kimi_k25 INT4 packed source models and related tests (#2156)
Xin He674377982561
diff --git a/auto_round/compressors/model_free.py b/auto_round/compressors/model_free.pyindex f71575aa8..f44833419 100755--- a/auto_round/compressors/model_free.py+++ b/auto_round/compressors/model_free.py@@ -663,16 +663,281 @@ def _collect_mxfp_source_entries(raw_tensors: dict[str, torch.Tensor]) -> list[tif name.endswith(".weight") and tensor.dtype == torch.float8_e4m3fn:layer_name = name[: -len(".weight")]scale_key = f"{layer_name}.weight_scale"- if scale_key in raw_tensors:+ # MXFP scales are raw E8M0 bytes stored as uint8.+ if scale_key in raw_tensors and raw_tensors[scale_key].dtype == torch.uint8:entries.append((layer_name, name, scale_key, 8))elif name.endswith(".weight_packed") and tensor.dtype in (torch.int8, torch.uint8):layer_name = name[: -len(".weight_packed")]scale_key = f"{layer_name}.weight_scale"- if scale_key in raw_tensors:+ # Guard against incorrectly detecting non-MXFP INT4 sources that also use+ # ".weight_packed" + ".weight_scale" but with floating-point scales.+ if scale_key in raw_tensors and raw_tensors[scale_key].dtype == torch.uint8:entries.append((layer_name, name, scale_key, 4))return entries+def _read_int_like(value: Any) -> int | None:+ """Best-effort conversion for int-like config values."""+ if isinstance(value, bool) or value is None:+ return None+ if isinstance(value, int):+ return value+ if isinstance(value, float) and value.is_integer():+ return int(value)+ if isinstance(value, str):+ v = value.strip()+ if v.isdigit() or (v.startswith("-") and v[1:].isdigit()):+ return int(v)+ return None+++def _read_bool_like(value: Any) -> bool | None:+ """Best-effort conversion for bool-like config values."""+ if isinstance(value, bool):+ return value+ if isinstance(value, str):+ v = value.strip().lower()+ if v in {"true", "1", "yes"}:+ return True+ if v in {"false", "0", "no"}:+ return False+ return None+++def _resolve_kimi_k25_int4_params(source_quant_config: dict | None) -> tuple[int | None, bool]:+ """Resolve default (group_size, sym) for kimi_k25 INT4 source dequant.++ The source tensor shapes are still treated as the source of truth, while+ config values are used as defaults or for logging consistency checks.+ """+ group_size: int | None = None+ sym = True++ if not isinstance(source_quant_config, dict):+ return group_size, sym++ candidates = [source_quant_config]+ weights_cfg = source_quant_config.get("weights")+ if isinstance(weights_cfg, dict):+ candidates.append(weights_cfg)++ for cfg in candidates:+ if group_size is None:+ for key in ("group_size", "weight_group_size"):+ maybe = _read_int_like(cfg.get(key))+ if maybe is not None and maybe > 0:+ group_size = maybe+ break++ sym_val = _read_bool_like(cfg.get("sym"))+ if sym_val is not None:+ sym = sym_val++ # Some configs store asymmetry as zero_point=True.+ zp_val = _read_bool_like(cfg.get("zero_point"))+ if zp_val is not None:+ sym = not zp_val++ return group_size, sym+++def _collect_kimi_k25_int4_source_entries(+ raw_tensors: dict[str, torch.Tensor],+) -> list[tuple[str, str, str, str | None]]:+ """Collect kimi_k25 INT4 source tensors.++ Expected source layout per layer:+ * ``<layer>.weight_packed``: uint8/int8 packed 2x int4 values per byte,+ **or** int32 packed 8x int4 values per element (Kimi-K2.6 format)+ * ``<layer>.weight_scale``: floating-point per-group scales+ * optional ``<layer>.weight_zero_point`` or ``<layer>.zero_point``+ * optional ``<layer>.weight_shape``: int32 tensor storing original shape+ """+ entries: list[tuple[str, str, str, str | None]] = []+ for name, packed in raw_tensors.items():+ if not name.endswith(".weight_packed"):+ continue+ if packed.dtype not in (torch.uint8, torch.int8, torch.int32):+ continue++ layer_name = name[: -len(".weight_packed")]+ scale_key = f"{layer_name}.weight_scale"+ if scale_key not in raw_tensors:+ continue++ scale = raw_tensors[scale_key]+ if scale.dtype not in (torch.float16, torch.bfloat16, torch.float32, torch.float64):+ continue+ if packed.dim() != 2 or scale.dim() != 2:+ continue++ zp_key = None+ for candidate in (f"{layer_name}.weight_zero_point", f"{layer_name}.zero_point"):+ if candidate in raw_tensors:+ zp_key = candidate+ break++ entries.append((layer_name, name, scale_key, zp_key))++ return entries+++def _unpack_int4_weight_packed(packed: torch.Tensor) -> torch.Tensor:+ """Unpack packed int4 bytes/ints into int16 ``[out, in]`` nibbles.++ Supports two packing formats:+ * ``uint8`` / ``int8``: 2 nibbles per byte — ``[out, in/2]`` → ``[out, in]``+ * ``int32``: 8 nibbles per element (Kimi-K2.6) — ``[out, in/8]`` → ``[out, in]``+ Nibbles are stored LSB-first: bits 0-3 → nibble 0, bits 4-7 → nibble 1, …+ """+ if packed.dtype == torch.int32:+ # Each int32 packs 8 nibbles (4-bit values), LSB-first.+ packed_i32 = packed.view(torch.int32)+ rows, cols = packed_i32.shape+ # Extract all 8 nibbles per int32 element.+ nibble_list = [(packed_i32 >> (4 * i)) & 0xF for i in range(8)]+ # Stack into [rows, cols, 8] then reshape to [rows, cols*8].+ out = torch.stack(nibble_list, dim=-1).reshape(rows, cols * 8).to(torch.int16)+ return out++ packed_u8 = packed.view(torch.uint8)+ lo = packed_u8 & 0x0F+ hi = (packed_u8 >> 4) & 0x0F++ out = torch.empty(+ (packed_u8.shape[0], packed_u8.shape[1] * 2),+ dtype=torch.int16,+ device=packed_u8.device,+ )+ out[:, 0::2] = lo.to(torch.int16)+ out[:, 1::2] = hi.to(torch.int16)+ return out+++def _dequant_kimi_k25_int4_tensors(+ raw_tensors: dict[str, torch.Tensor],+ source_quant_config: dict | None = None,+ device: str = "cpu",+ shard_name: str | None = None,+) -> dict[str, torch.Tensor]:+ """Dequantize kimi_k25-style INT4 packed weights to bfloat16 ``.weight``.++ This normalizes source tensors so the downstream model-free path can+ requantize them into the requested target format (for example MXFP4).+ """+ entries = _collect_kimi_k25_int4_source_entries(raw_tensors)+ if not entries:+ return raw_tensors++ default_group_size, default_sym = _resolve_kimi_k25_int4_params(source_quant_config)+ dequant_device = str(device or "cpu")+ shard_prefix = f"[{shard_name}] " if shard_name else ""+ logger.info(+ f"{shard_prefix}Dequantizing kimi_k25 INT4 tensor(s) to bfloat16 on {dequant_device}: " f"total={len(entries)}."+ )++ for layer_name, packed_key, scale_key, zp_key in entries:+ packed = raw_tensors.pop(packed_key)+ scale = raw_tensors.pop(scale_key)+ zero = raw_tensors.pop(zp_key) if zp_key is not None else None+ # Remove auxiliary weight_shape tensor if present (Kimi-K2.6 format).+ raw_tensors.pop(f"{layer_name}.weight_shape", None)++ # int32 packs 8 nibbles per element; uint8/int8 packs 2 nibbles per byte.+ nibbles_per_element = 8 if packed.dtype == torch.int32 else 2+ in_features = packed.shape[1] * nibbles_per_element+ groups = scale.shape[1]+ inferred_group_size = (in_features // groups) if groups > 0 and in_features % groups == 0 else None+ group_size = inferred_group_size or default_group_size+ if group_size is None or group_size <= 0:+ raise ValueError(+ f"Cannot resolve group_size for INT4 tensor '{layer_name}': "+ f"packed_shape={tuple(packed.shape)}, scale_shape={tuple(scale.shape)}"+ )+ if in_features % group_size != 0:+ raise ValueError(+ f"Invalid group_size={group_size} for INT4 tensor '{layer_name}': "+ f"in_features={in_features} is not divisible by group_size."+ )++ if (+ inferred_group_size is not None+ and default_group_size is not None+ and inferred_group_size != default_group_size+ ):+ logger.warning(+ f"{shard_prefix}k25 INT4 group_size mismatch for {layer_name}: "+ f"config={default_group_size}, inferred={inferred_group_size}. "+ "Using inferred value from tensor shapes."+ )++ n_groups = in_features // group_size+ if scale.shape[1] != n_groups:+ raise ValueError(+ f"Scale shape mismatch for INT4 tensor '{layer_name}': "+ f"scale.shape={tuple(scale.shape)}, expected second dim {n_groups}."+ )++ def _dequant_impl(target_device: str) -> torch.Tensor:+ q = _unpack_int4_weight_packed(packed.to(target_device, non_blocking=True))+ sc = scale.to(target_device, non_blocking=True).to(torch.float32)++ if zero is not None:+ zp = zero.to(target_device, non_blocking=True)+ if zp.dim() == 2 and zp.shape == packed.shape and zp.dtype in (torch.int8, torch.uint8):+ zp = _unpack_int4_weight_packed(zp)+ elif zp.dim() == 2 and zp.shape[1] != n_groups:+ raise ValueError(f"Unsupported zero-point shape for INT4 tensor '{layer_name}': {tuple(zp.shape)}")+ if zp.dim() == 2 and zp.shape[1] == n_groups:+ zp = zp.to(torch.int16).repeat_interleave(group_size, dim=1)+ zp = zp[:, : q.shape[1]].to(torch.int16)+ q = q - zp+ else:+ if default_sym:+ q = torch.where(q >= 8, q - 16, q)+ else:+ # Conservative fallback for asymmetric INT4 without explicit+ # zero-point tensor: recenter to [-8, 7].+ q = q - 8++ sc = sc.repeat_interleave(group_size, dim=1)[:, : q.shape[1]]+ return (q.to(torch.float32) * sc).to(torch.bfloat16)++ dq_weight = _dequantize_with_device_fallback(+ dequant_device=dequant_device,+ shard_prefix=shard_prefix,+ op_name="kimi_k25 INT4 dequant",+ tensor_label=layer_name,+ on_device=lambda: _dequant_impl(dequant_device).to("cpu"),+ on_cpu=lambda: _dequant_impl("cpu"),+ )+ raw_tensors[f"{layer_name}.weight"] = dq_weight++ return raw_tensors+++def _handle_model_type_low_precision_source_tensors(+ raw_tensors: dict[str, torch.Tensor],+ model_type: str | None,+ source_quant_config: dict | None = None,+ device: str = "cpu",+ shard_name: str | None = None,+) -> dict[str, torch.Tensor]:+ """Handle model-type-specific low-precision source tensors.++ Currently supports:+ * ``model_type='kimi_k25'`` INT4 packed tensors.+ """+ if (model_type or "").lower() == "kimi_k25":+ return _dequant_kimi_k25_int4_tensors(+ raw_tensors,+ source_quant_config=source_quant_config,+ device=device,+ shard_name=shard_name,+ )+ return raw_tensors++def _is_out_of_memory_error(exc: Exception) -> bool:if isinstance(exc, torch.OutOfMemoryError):return True@@ -929,6 +1194,7 @@ def _process_shard(matcher: "_PatternMatcher | None" = None,fp8_block_size: list | None = None,model_type: str | None = None,+ source_quant_config: dict | None = None,enable_torch_compile: bool = False,disable_opt_rtn: bool = False,) -> tuple[dict[str, torch.Tensor], list[str], list[str]]:@@ -1006,6 +1272,15 @@ def _process_shard(# 1) model-type-specific preprocessing (format conversion only)raw_tensors, source_state = _preprocess_model_type_source_tensors(raw_tensors, model_type=model_type)+ # 1.5) model-type-specific low-precision dequantization (e.g. kimi_k25 INT4)+ raw_tensors = _handle_model_type_low_precision_source_tensors(+ raw_tensors,+ model_type=model_type,+ source_quant_config=source_quant_config,+ device=device,+ shard_name=shard_name,+ )+# 2) generic MXFP handling for both preprocessed and normal source modelsraw_tensors, passthrough_tensors, passthrough_layers = _handle_mxfp_source_tensors(raw_tensors,@@ -1560,6 +1835,7 @@ def _process_single_shard_task(model_type: str | None,quant_output_dir: str,total_shards: int,+ source_quant_config: dict | None = None,enable_torch_compile: bool = False,disable_opt_rtn: bool = False,) -> tuple[int, str, str | None, str | None, list[str] | None, list[str] | None, list[str] | None]:@@ -1587,6 +1863,7 @@ def _process_single_shard_task(device=device,fp8_block_size=fp8_block_size,model_type=model_type,+ source_quant_config=source_quant_config,enable_torch_compile=enable_torch_compile,disable_opt_rtn=disable_opt_rtn,)@@ -2297,6 +2574,7 @@ class _ModelFreeCompressorCore:ignore_patterns=self.ignore_patterns,fp8_block_size=self.fp8_block_size,model_type=self.model_type,+ source_quant_config=self.config.get("quantization_config"),enable_torch_compile=self.enable_torch_compile,disable_opt_rtn=self.disable_opt_rtn,quant_output_dir=self._quant_output_dir,diff --git a/test/unit/test_cpu/quantization/test_model_free.py b/test/unit/test_cpu/quantization/test_model_free.pyindex 662bd7f20..5e39a42cf 100644--- a/test/unit/test_cpu/quantization/test_model_free.py+++ b/test/unit/test_cpu/quantization/test_model_free.py@@ -59,6 +59,7 @@ from auto_round.compressors.model_free import (_dequant_fp8_tensors,_dequant_mxfp_tensors,_expand_e8m0_block_scale,+ _handle_model_type_low_precision_source_tensors,_handle_mxfp_source_tensors,_looks_like_auto_scheme,_ModelFreeCompressorCore,@@ -933,6 +934,19 @@ class TestLLMCompressorMXFPSource:raw_out, passthrough, layers = _handle_mxfp_source_tensors(raw, matcher)assert raw_out is raw and passthrough == {} and layers == []+ def test_reject_non_mxfp_packed_float_scale(self):+ """weight_packed + float scale must NOT be treated as MXFP source."""+ raw = {+ "layer.weight_packed": torch.randint(0, 255, (64, 64), dtype=torch.uint8),+ "layer.weight_scale": torch.ones(64, 8, dtype=torch.float16),+ }+ matcher = _matcher(default={"bits": 4, "group_size": 32, "sym": True, "data_type": "mx_fp"})+ raw_out, passthrough, layers = _handle_mxfp_source_tensors(raw, matcher)++ assert raw_out is raw+ assert passthrough == {}+ assert layers == []+@require_compressed_tensorsdef test_e2e_mxfp8_passthrough(self, tmp_path):"""End-to-end: MXFP8 source + MXFP8 target → passthrough, weight bytes unchanged."""@@ -1141,6 +1155,75 @@ class TestSchemeValidation:assert is_model_free_supported_scheme("DOES_NOT_EXIST") is False+# ===========================================================================+# kimi_k25 INT4 packed source models+# ===========================================================================+++_KIMI_K25_CFG = {+ "architectures": ["KimiK25ForConditionalGeneration"],+ "model_type": "kimi_k25",+ "quantization_config": {+ "quant_method": "compressed-tensors",+ "weights": {"num_bits": 4, "type": "int", "group_size": 8, "symmetric": True},+ },+}+++class TestKimiK25Int4Source:+ def test_kimi_k25_int4_dequant_helper(self):+ raw = {+ "layer.weight_packed": torch.randint(0, 255, (128, 64), dtype=torch.uint8),+ "layer.weight_scale": torch.ones(128, 16, dtype=torch.float16),+ }+ out = _handle_model_type_low_precision_source_tensors(+ raw,+ model_type="kimi_k25",+ source_quant_config=_KIMI_K25_CFG["quantization_config"],+ device="cpu",+ )+ assert "layer.weight" in out+ assert out["layer.weight"].dtype == torch.bfloat16+ assert out["layer.weight"].shape == (128, 128)+ assert "layer.weight_packed" not in out+ assert "layer.weight_scale" not in out++ @require_compressed_tensors+ def test_kimi_k25_int4_to_mxfp4_via_model_free(self, tmp_path):+ tensors = {+ "model.layers.0.mlp.fc1.weight_packed": torch.randint(0, 255, (128, 64), dtype=torch.uint8),+ "model.layers.0.mlp.fc1.weight_scale": torch.ones(128, 16, dtype=torch.float16),+ "lm_head.weight": torch.randn(1000, 128),+ }+ model_dir = _make_model_dir(tmp_path, _KIMI_K25_CFG, tensors)+ output_dir = str(tmp_path / "output")++ _ModelFreeCompressorCore(+ model_name_or_path=model_dir,+ output_dir=output_dir,+ scheme="MXFP4",+ format="llm_compressor",+ ).run()++ qc = _read_qconfig(output_dir)+ assert qc["format"] == "mxfp4-pack-quantized"+ assert qc["quant_method"] == "compressed-tensors"++ found_scale_dtype = None+ found_packed_shape = None+ for fname in os.listdir(output_dir):+ if not fname.endswith(".safetensors"):+ continue+ with safe_open(os.path.join(output_dir, fname), framework="pt") as sf:+ if "model.layers.0.mlp.fc1.weight_scale" in sf.keys():+ found_scale_dtype = sf.get_tensor("model.layers.0.mlp.fc1.weight_scale").dtype+ found_packed_shape = sf.get_tensor("model.layers.0.mlp.fc1.weight_packed").shape++ # Re-quantized MXFP4 scales are uint8 E8M0 (source INT4 scales were fp16).+ assert found_scale_dtype == torch.uint8+ assert found_packed_shape == (128, 64)++# ===========================================================================# CLI auto-routing# ===========================================================================