PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
# Copyright (c) 2025 Intel Corporation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# Background
# ----------
# When ``transformers`` loads a model, it only instantiates the sub-modules
# declared in the model class. Some architectures include auxiliary
# parameters — most notably **MTP (Multi-Token Prediction)** layers — that are
# stored in the original checkpoint but are *not* part of the default module
# graph. As a result, ``model.save_pretrained`` silently omits those tensors
# from the quantized output.
#
# This module provides :func:`copy_missing_tensors_from_source`, which detects
# and copies such tensors from the original checkpoint into the saved output
# directory. Detection works by comparing the **parent layer names**
# (``tensor_name.rsplit(".", 1)[0]``) present in the source checkpoint against
# those present in the saved output: any source tensor whose parent layer has
# no tensors at all in the saved output (and is itself absent) is considered
# "missing". This requires no model/config loading.
#
# Before writing, the function optionally:
# - **dequantizes FP8 weights** to BF16 (when a matching ``weight_scale_inv``
# tensor is present in the source checkpoint), and
# - **re-quantizes with RTN** into the packed WOQ format used by the rest of
# the saved model (when ``quant_method == "auto-round"`` and
# ``packing_format == "auto_round:auto_gptq"`` are detected in
# ``config.json``).
import json
import os
import tempfile
import torch
from auto_round.logger import logger
from auto_round.utils.common import compress_layer_names
from auto_round.utils.model_free_utils import (
_normalize_tensor_name_for_warning,
quantize_weight_rtn,
split_fused_expert_tensors,
)
from auto_round.utils.path_safety import resolve_within_directory, validate_weight_map
from auto_round.utils.weight_handler import _dequant_fp8_linear_weight
def _restore_special_fp32_tensors(
source_tensor_to_file: dict[str, str],
saved_tensor_to_file: dict[str, str],
) -> None:
"""Restore source FP32 tensors that were saved as FP16 or BF16."""
from safetensors import safe_open
from safetensors.torch import save_file
source_to_saved: dict[str, str] = {}
for source_name in source_tensor_to_file:
if source_name in saved_tensor_to_file:
source_to_saved[source_name] = source_name
source_names_by_shard: dict[str, list[str]] = {}
for tensor_name, source_shard in source_tensor_to_file.items():
source_names_by_shard.setdefault(source_shard, []).append(tensor_name)
candidates: list[str] = []
has_low_precision_float = False
for source_shard, tensor_names in source_names_by_shard.items():
with safe_open(source_shard, framework="pt", device="cpu") as source_file:
for tensor_name in tensor_names:
dtype = source_file.get_slice(tensor_name).get_dtype()
if dtype in {"F16", "BF16"} or dtype.startswith("F8"):
has_low_precision_float = True
elif dtype == "F32" and tensor_name in source_to_saved:
candidates.append(tensor_name)
# An all-FP32 source has no "special" FP32 tensors; the downcast to the export dtype is intended.
if not has_low_precision_float:
return
target_names_by_shard: dict[str, list[str]] = {}
for tensor_name in candidates:
target_name = source_to_saved[tensor_name]
target_names_by_shard.setdefault(saved_tensor_to_file[target_name], []).append(tensor_name)
tensors_to_restore: dict[str, torch.Tensor] = {}
for target_shard, tensor_names in target_names_by_shard.items():
with safe_open(target_shard, framework="pt", device="cpu") as target_file:
for tensor_name in tensor_names:
target_name = source_to_saved[tensor_name]
if target_file.get_slice(target_name).get_dtype() in {"F16", "BF16"}:
source_shard = source_tensor_to_file[tensor_name]
with safe_open(source_shard, framework="pt", device="cpu") as source_file:
source_tensor = source_file.get_tensor(tensor_name)
target_tensor = target_file.get_tensor(target_name)
# Skip tensors changed beyond a plain downcast (e.g. fake-quantized weights).
if source_tensor.shape == target_tensor.shape and torch.equal(
source_tensor.to(target_tensor.dtype), target_tensor
):
tensors_to_restore[tensor_name] = source_tensor
restore_names_by_shard: dict[str, list[str]] = {}
for tensor_name in tensors_to_restore:
restore_names_by_shard.setdefault(saved_tensor_to_file[source_to_saved[tensor_name]], []).append(tensor_name)
for target_shard, tensor_names in restore_names_by_shard.items():
with safe_open(target_shard, framework="pt", device="cpu") as target_file:
metadata = target_file.metadata()
shard_tensors = {name: target_file.get_tensor(name) for name in target_file.keys()}
for tensor_name in tensor_names:
shard_tensors[source_to_saved[tensor_name]] = tensors_to_restore[tensor_name]
# Preserve the original shard permission bits so the temporary file's
# 0600 mode is not inherited by the final replacement target.
original_mode = os.stat(target_shard).st_mode
temporary_path = None
try:
with tempfile.NamedTemporaryFile(
dir=os.path.dirname(target_shard), prefix=".restore_fp32_", suffix=".safetensors", delete=False
) as temporary_file:
temporary_path = temporary_file.name
save_file({name: tensor.contiguous() for name, tensor in shard_tensors.items()}, temporary_path, metadata)
os.replace(temporary_path, target_shard)
os.chmod(target_shard, original_mode)
finally:
if temporary_path and os.path.exists(temporary_path):
os.remove(temporary_path)
if tensors_to_restore:
tensor_summary = compress_layer_names(list(tensors_to_restore))
logger.info(
f"Restored {len(tensors_to_restore)} tensor(s) from FP16/BF16 to their original FP32 values: "
f"{tensor_summary}."
)
def keep_special_fp32_params(model: torch.nn.Module, source_dir: str) -> list[str]:
"""Preserve checkpoint FP32 tensors after loading or building a meta skeleton.
Only applies to mixed-precision checkpoints (e.g. a BF16 model that stores
``linear_attn.norm.weight`` in FP32); for an all-FP32 checkpoint the load dtype is
intentional. Restored names are appended to ``model._keep_in_fp32_modules`` so that
later ``cast_model_dtype`` calls and export keep them in FP32.
Meta tensors are promoted without reading their payload; materialization later
loads the original FP32 values. Returns the restored or marked tensor names.
"""
source_tensor_to_file = _resolve_source_tensor_map(source_dir)
if not source_tensor_to_file:
return []
from safetensors import safe_open
model_tensors = dict(model.named_parameters(remove_duplicate=False))
model_tensors.update(model.named_buffers(remove_duplicate=False))
names_by_shard: dict[str, list[str]] = {}
for tensor_name, shard in source_tensor_to_file.items():
names_by_shard.setdefault(shard, []).append(tensor_name)
has_low_precision_float = False
candidates_by_shard: dict[str, list[str]] = {}
for shard, tensor_names in names_by_shard.items():
with safe_open(shard, framework="pt", device="cpu") as f:
for tensor_name in tensor_names:
dtype = f.get_slice(tensor_name).get_dtype()
if dtype in {"F16", "BF16"} or dtype.startswith("F8"):
has_low_precision_float = True
elif dtype == "F32":
tensor = model_tensors.get(tensor_name)
if tensor is not None and tensor.dtype in (torch.float16, torch.bfloat16, torch.float32):
candidates_by_shard.setdefault(shard, []).append(tensor_name)
if not has_low_precision_float or not candidates_by_shard:
return []
restored: list[str] = []
for shard, tensor_names in candidates_by_shard.items():
with safe_open(shard, framework="pt", device="cpu") as f:
for tensor_name in tensor_names:
tensor = model_tensors[tensor_name]
if tensor.device.type == "meta":
if tuple(f.get_slice(tensor_name).get_shape()) != tuple(tensor.shape):
continue
tensor.data = tensor.to(dtype=torch.float32)
restored.append(tensor_name)
continue
value = f.get_tensor(tensor_name)
if value.shape != tensor.shape:
continue
tensor.data = value.to(tensor.device)
restored.append(tensor_name)
if restored:
keep = getattr(model, "_keep_in_fp32_modules", None) or []
keep = [keep] if isinstance(keep, str) else list(keep)
model._keep_in_fp32_modules = keep + [name for name in restored if name not in keep]
logger.info(
f"Kept {len(restored)} checkpoint FP32 tensor(s) in FP32 after loading: "
f"{compress_layer_names(restored)}."
)
return restored
def _resolve_source_tensor_map(source_dir: str) -> dict[str, str] | None:
"""Return ``tensor_name -> shard_file`` for the safetensors checkpoint in *source_dir*."""
try:
from safetensors import safe_open
except ImportError:
logger.warning("safetensors not available, skipping tensor comparison against source checkpoint")
return None
if not os.path.isdir(source_dir):
try:
from auto_round.utils.model import download_hf_model
source_dir = download_hf_model(source_dir)
except Exception as e:
logger.debug(f"Could not resolve source model path to check for missing tensors: {e}")
return None
if not source_dir or not os.path.isdir(source_dir):
return None
source_index_file = os.path.join(source_dir, "model.safetensors.index.json")
source_single_file = os.path.join(source_dir, "model.safetensors")
source_tensor_to_file: dict = {}
if os.path.exists(source_index_file):
with open(source_index_file) as f:
src_index = json.load(f)
# Shard names come from the source checkpoint's own index: validate them
# against source_dir, then resolve at the point of use below.
for tensor_name, shard_file in validate_weight_map(
src_index["weight_map"], source_dir, index_path=source_index_file
).items():
source_tensor_to_file[tensor_name] = str(resolve_within_directory(source_dir, shard_file))
elif os.path.exists(source_single_file):
with safe_open(source_single_file, framework="pt", device="cpu") as f:
for key in f.keys():
source_tensor_to_file[key] = source_single_file
else:
return None
return source_tensor_to_file
def _resolve_tensor_shard_maps(
source_dir: str,
target_dir: str,
) -> tuple[dict[str, str], dict[str, str]] | None:
"""Resolve ``tensor_name -> shard_file`` maps for *source_dir* and *target_dir*.
Returns ``None`` when either checkpoint cannot be resolved (missing config,
unresolvable source repo, or no safetensors present), in which case callers
should skip whatever tensor-level processing they were about to do.
"""
config_path = os.path.join(target_dir, "config.json")
if not os.path.exists(config_path):
return None
source_tensor_to_file = _resolve_source_tensor_map(source_dir)
if source_tensor_to_file is None:
return None
from safetensors import safe_open
saved_tensor_to_file: dict[str, str] = {}
saved_index_file = os.path.join(target_dir, "model.safetensors.index.json")
saved_single_file = os.path.join(target_dir, "model.safetensors")
if os.path.exists(saved_index_file):
with open(saved_index_file) as f:
saved_idx = json.load(f)
saved_tensor_to_file = {
tensor_name: os.path.join(target_dir, shard_file)
for tensor_name, shard_file in saved_idx["weight_map"].items()
}
elif os.path.exists(saved_single_file):
with safe_open(saved_single_file, framework="pt", device="cpu") as f:
saved_tensor_to_file = {tensor_name: saved_single_file for tensor_name in f.keys()}
else:
return None
return source_tensor_to_file, saved_tensor_to_file
def restore_fp32_tensors_from_source(
source_dir: str,
target_dir: str,
) -> None:
"""Restore FP32 source tensors that were downcast to FP16/BF16 in the saved output.
This is independent of MTP/missing-tensor handling: it must run by default
whenever a source checkpoint can be resolved, regardless of whether
``AR_DISABLE_COPY_MTP_WEIGHTS`` disables the (separate) missing-tensor copy
performed by :func:`copy_missing_tensors_from_source`.
"""
maps = _resolve_tensor_shard_maps(source_dir, target_dir)
if maps is None:
return
source_tensor_to_file, saved_tensor_to_file = maps
_restore_special_fp32_tensors(source_tensor_to_file, saved_tensor_to_file)
def copy_missing_tensors_from_source(
source_dir: str,
target_dir: str,
) -> None:
"""Copy tensors from the source checkpoint that are absent from the saved output.
Some parameters (e.g., MTP layers) are not loaded by ``transformers`` and
therefore not written by ``model.save_pretrained``. This function copies
those tensors from the original checkpoint into the quantized output dir.
Detection works by comparing **parent layer names**
(``tensor_name.rsplit(".", 1)[0]``) between the source checkpoint and the
saved output. A source tensor is considered "missing" when:
1. its name is **not** already present in the saved output, **and**
2. its parent layer name is **not** among the parent layer names of any
saved tensor (i.e. the layer was entirely skipped by
``save_pretrained``).
No model or config loading is required.
FP8 handling:
Missing weight tensors in FP8 dtype that have a corresponding
``weight_scale_inv`` tensor are dequantized to BF16 before saving.
WOQ quantization:
If the saved ``config.json`` contains a ``quantization_config`` with
``quant_method == "auto-round"`` and
``packing_format == "auto_round:auto_gptq"``, missing 2-D weight tensors
will be quantized with RTN and packed into qweight/qzeros/scales format.
Args:
source_dir: Path to the original (pre-quantization) checkpoint directory
or a HuggingFace repo id.
target_dir: Directory to which the quantized model was saved. Must
contain a ``config.json``.
"""
try:
from safetensors import safe_open
from safetensors.torch import save_file
except ImportError:
logger.warning("safetensors not available, skipping copy of missing tensors from source checkpoint")
return
maps = _resolve_tensor_shard_maps(source_dir, target_dir)
if maps is None:
return
source_tensor_to_file, saved_tensor_to_file = maps
# Also restore FP32 tensors here so direct callers of this function keep
# getting the restore; ``restore_fp32_tensors_from_source`` is the
# preferred entry point and is safe to call again (no-op once restored).
_restore_special_fp32_tensors(source_tensor_to_file, saved_tensor_to_file)
saved_tensor_names = set(saved_tensor_to_file)
saved_index_file = os.path.join(target_dir, "model.safetensors.index.json")
saved_single_file = os.path.join(target_dir, "model.safetensors")
# ------------------------------------------------------------------ #
# Identify missing tensors via block-prefix statistics #
# ------------------------------------------------------------------ #
# A source tensor is considered "missing" when:
# 1. its name is absent from the saved output, AND
# 2. its parent layer name is NOT among the parent layers of any saved
# tensor (catches layers whose suffix changed, e.g. .weight →
# .qweight after quantization), AND
# 3. its "block prefix" — the path up to and including the first
# numeric segment, e.g. ``model.layers.0`` — is NOT present in the
# set of block prefixes collected from the saved output.
saved_parent_layers: set = {name.rsplit(".", 1)[0] for name in saved_tensor_names}
# To match model.language_model.layers.0.fc1 with language_model.layers.0.fc1
shortcut_parent_layers: set = {name.split(".", 1)[1] for name in saved_parent_layers if "." in name}
saved_parent_layers.update(shortcut_parent_layers)
def _first_numeric_prefix(tensor_name: str) -> str | None:
"""Return the path up to and including the first numeric segment, or None.
e.g. ``model.layers.0.mlp.gate.qweight`` → ``model.layers.0``
``model.embed_tokens.weight`` → ``None``
"""
parts = tensor_name.split(".")
for i, p in enumerate(parts):
if p.isdigit():
return ".".join(parts[: i + 1])
return None
# Collect block prefixes from the saved output: path up to the first
# numeric segment in each saved tensor name (e.g. "model.layers.0",
# "mlp.experts.0").
saved_block_prefix: set = set()
for tensor_name in saved_tensor_names:
bp = _first_numeric_prefix(tensor_name)
if bp is not None:
saved_block_prefix.add(bp)
# To match model.language_model.layers.0 with language_model.layers.0
shortcut_block_prefix: set = {name.split(".", 1)[1] for name in saved_block_prefix if "." in name}
saved_block_prefix.update(shortcut_block_prefix)
def _is_missing_by_name(name: str) -> bool:
if name in saved_tensor_names:
return False
parent = name.rsplit(".", 1)[0]
if parent in saved_parent_layers:
return False
# For split experts, name is changed but block name is the same.
block = _first_numeric_prefix(name)
if block is not None:
return block not in saved_block_prefix
return True
nested_trunk_source, nested_trunk_saved = "language_model.backbone.", "language_model.model."
def _is_truly_missing(name: str) -> bool:
# Special case: Qwen/Qwen3-0.6B-FP8
# lm_head is tied but still in source_dir → not missing
if name == "lm_head.weight":
return False
if not _is_missing_by_name(name):
return False
# Nemotron-H Omni checkpoints store the language-model trunk as
# ``language_model.backbone.*`` while the quantized model saves it as
# ``language_model.model.*``; without this alias every quantized LM block
# would be re-copied from the source in BF16.
if name.startswith(nested_trunk_source):
return _is_missing_by_name(nested_trunk_saved + name[len(nested_trunk_source) :])
return True
missing_tensor_names: list = [name for name in source_tensor_to_file if _is_truly_missing(name)]
if not missing_tensor_names:
return
try:
from tqdm import tqdm as _tqdm
except ImportError:
_tqdm = None
# Compressed parent-layer summary for display
parent_summary = compress_layer_names(list({name.rsplit(".", 1)[0] for name in missing_tensor_names}))
logger.info(
f"Found {len(missing_tensor_names)} tensor(s) in the source checkpoint that are "
f"absent from the saved output (e.g., MTP parameters): {parent_summary}. Copying them now...\n"
)
# ------------------------------------------------------------------ #
# Load missing tensors from source shards #
# ------------------------------------------------------------------ #
shard_to_missing: dict = {}
for tensor_name in missing_tensor_names:
shard_file = source_tensor_to_file[tensor_name]
shard_to_missing.setdefault(shard_file, []).append(tensor_name)
missing_tensors_dict: dict = {}
_iter_shards = (
_tqdm(
shard_to_missing.items(),
desc="Loading missing tensors",
unit="shard",
total=len(shard_to_missing),
)
if _tqdm
else shard_to_missing.items()
)
for shard_file, tensor_names in _iter_shards:
with safe_open(shard_file, framework="pt", device="cpu") as f:
_iter_tensors = (
_tqdm(
tensor_names,
desc=f" {os.path.basename(shard_file)}",
unit="tensor",
leave=False,
)
if _tqdm
else tensor_names
)
for tensor_name in _iter_tensors:
missing_tensors_dict[tensor_name] = f.get_tensor(tensor_name)
# ------------------------------------------------------------------ #
# Split fused expert tensors (3-D) into per-expert 2-D tensors #
# ------------------------------------------------------------------ #
missing_tensors_dict = split_fused_expert_tensors(missing_tensors_dict)
# ------------------------------------------------------------------ #
# FP8 dequantization: if a weight is FP8 and its scale_inv is present #
# ------------------------------------------------------------------ #
fp8_dtypes = {"torch.float8_e4m3fn", "torch.float8_e5m2", "torch.float8_e4m3fnuz", "torch.float8_e5m2fnuz"}
# Find FP8 weight candidates upfront to show accurate progress bar
fp8_candidates = [
(name, t)
for name, t in list(missing_tensors_dict.items())
if str(t.dtype) in fp8_dtypes and name.endswith(".weight")
]
dequantized_keys: list = []
keys_to_remove: list = []
_fp8_iter = (
_tqdm(fp8_candidates, desc="Dequantizing FP8 weights", unit="weight")
if _tqdm and fp8_candidates
else fp8_candidates
)
for tensor_name, tensor in _fp8_iter:
# Look for the corresponding scale_inv tensor
base = tensor_name[: -len(".weight")]
scale_inv_name = base + ".weight_scale_inv"
scale_name = base + ".weight_scale"
weight_scale = missing_tensors_dict.get(scale_inv_name)
if weight_scale is None:
weight_scale = missing_tensors_dict.get(scale_name)
if weight_scale is None:
continue
# Try to determine block_size from config or fall back to per-tensor
block_size = None
if weight_scale.dim() >= 2 and weight_scale.shape != tensor.shape:
block_m = tensor.shape[-2] // weight_scale.shape[-2] if weight_scale.shape[-2] > 0 else 1
block_n = tensor.shape[-1] // weight_scale.shape[-1] if weight_scale.shape[-1] > 0 else 1
if block_m > 1 or block_n > 1:
block_size = [block_m, block_n]
dq_weight = _dequant_fp8_linear_weight(tensor, weight_scale, block_size)
missing_tensors_dict[tensor_name] = dq_weight.to(torch.bfloat16)
dequantized_keys.append(tensor_name)
for s_name in [scale_inv_name, scale_name]:
if s_name in missing_tensors_dict:
keys_to_remove.append(s_name)
for k in keys_to_remove:
missing_tensors_dict.pop(k, None)
if dequantized_keys:
dq_summary = compress_layer_names([k.rsplit(".", 1)[0] for k in dequantized_keys])
logger.info(f"Dequantized {len(dequantized_keys)} FP8 weight(s) to BF16 before saving: " f"{dq_summary}")
if not missing_tensors_dict:
return
# ------------------------------------------------------------------ #
# WOQ quantization: quantize missing Linear weights if model is WOQ #
# ------------------------------------------------------------------ #
missing_tensors_dict = _woq_quantize_missing_tensors(target_dir, missing_tensors_dict)
if not missing_tensors_dict:
return
# ------------------------------------------------------------------ #
# Write missing tensors to a new shard and update the index #
# ------------------------------------------------------------------ #
is_sharded = os.path.exists(saved_index_file)
new_shard_name = "model_extra_tensors.safetensors"
new_shard_path = os.path.join(target_dir, new_shard_name)
missing_tensors_dict = {k: v.contiguous() for k, v in missing_tensors_dict.items()}
save_file(missing_tensors_dict, new_shard_path)
if is_sharded:
with open(saved_index_file) as f:
saved_index = json.load(f)
for tensor_name in missing_tensors_dict:
saved_index["weight_map"][tensor_name] = new_shard_name
with open(saved_index_file, "w") as f:
json.dump(saved_index, f, indent=2)
elif os.path.exists(saved_single_file):
weight_map: dict = {}
with safe_open(saved_single_file, framework="pt", device="cpu") as f:
for key in f.keys():
weight_map[key] = "model.safetensors"
for tensor_name in missing_tensors_dict:
weight_map[tensor_name] = new_shard_name
new_index = {"metadata": {"total_size": 0}, "weight_map": weight_map}
with open(saved_index_file, "w") as f:
json.dump(new_index, f, indent=2)
logger.info(
f"Successfully wrote {len(missing_tensors_dict)} missing tensor(s) to " f"'{new_shard_name}' in {target_dir}."
)
def _woq_quantize_missing_tensors(target_dir: str, missing_tensors_dict: dict) -> dict:
"""Apply WOQ (Weight-Only Quantization) to missing Linear weight tensors.
Reads ``config.json`` from *target_dir* to obtain ``quantization_config``.
Only activates when ``quant_method == "auto-round"`` and
``packing_format == "auto_round:auto_gptq"``.
Uses :func:`quantize_weight_rtn` for RTN quantisation + packing so that
there is **no dependency on the model object** or on QuantLinear classes.
Non-weight tensors (bias, norms, embeddings, etc.) are kept as-is.
Per-layer resolution:
For each weight, ``extra_config`` is checked first (exact layer name,
then regex pattern match), and the result is merged with global
defaults. This means entries like ``".*mtp.*": {"bits": 8}`` or
``"mtp.fc": {"bits": 16, "data_type": "fp"}`` are honoured, while
layers absent from ``extra_config`` fall back to global ``bits`` /
``group_size`` / ``sym``.
Args:
target_dir: Output directory that contains ``config.json``.
missing_tensors_dict: Dict mapping tensor names to tensor values.
Returns:
Updated dict with quantized+packed tensors replacing original weight tensors.
"""
import re as _re
BLOCK_NAME_TO_IGNORE = [".shared_expert_gate.", ".gate.", ".g_proj.", "mtp.fc."]
qconfig = _get_woq_config_from_dir(target_dir)
if qconfig is None:
return missing_tensors_dict
global_bits = qconfig["bits"]
global_group_size = qconfig["group_size"]
global_sym = qconfig["sym"]
declared_packing = qconfig.get("packing_format", "auto_round:auto_gptq")
block_name_to_quantize = qconfig.get("block_name_to_quantize", None)
extra_config: dict = qconfig.get("extra_config", {}) or {}
# Pre-compile all valid regex patterns once to avoid repeated re.compile() calls
# for every tensor lookup (O(N×M) → O(M) compile + O(N×M) match).
_compiled_patterns: list = []
for pattern, pattern_cfg in extra_config.items():
try:
_compiled_patterns.append((_re.compile(pattern), pattern, pattern_cfg))
except _re.error as exc:
logger.warning(
"Invalid regex key in extra_config ignored during pre-compilation: %r (%s)",
pattern,
exc,
)
# Cache resolved layer configs to avoid repeated regex scans for the same name.
_layer_cfg_cache: dict = {}
def _resolve_layer_cfg(layer_name: str) -> dict:
"""Return effective {bits, group_size, sym, data_type} for *layer_name*.
Lookup order:
1. Exact key match in extra_config.
2. Among all pre-compiled regex keys that match layer_name, pick the longest
pattern (longer pattern == more specific).
3. Global defaults.
"""
if layer_name in _layer_cfg_cache:
return _layer_cfg_cache[layer_name]
override: dict = {}
# 1. exact match
if layer_name in extra_config:
override = extra_config[layer_name]
else:
# 2. use pre-compiled patterns, keep the most specific (longest)
best_pattern: str | None = None
for compiled, pattern, cfg in _compiled_patterns:
if pattern == layer_name:
continue # already handled above
if compiled.search(layer_name):
if best_pattern is None or len(pattern) > len(best_pattern):
best_pattern = pattern
override = cfg
result = {
"bits": override.get("bits", global_bits),
"group_size": override.get("group_size", global_group_size),
"sym": override.get("sym", global_sym),
"data_type": override.get("data_type", "int"),
}
_layer_cfg_cache[layer_name] = result
return result
def _is_fp_layer(layer_cfg: dict) -> bool:
"""Return True when the resolved config indicates full-precision (no quantization)."""
dt = layer_cfg.get("data_type", "int")
return layer_cfg["bits"] >= 16 or dt in ("fp", "float", "float16", "bfloat16", "float32")
def _is_covered_by_extra_config(layer_name: str) -> bool:
"""Return True if layer_name is matched by any entry in extra_config (exact or regex)."""
if layer_name in extra_config:
return True
for compiled, pattern, _ in _compiled_patterns:
if pattern != layer_name and compiled.search(layer_name):
return True
return False
if block_name_to_quantize:
existing_blocks = (
block_name_to_quantize
if isinstance(block_name_to_quantize, list)
else [b.strip() for b in block_name_to_quantize.split(",") if b.strip()]
)
existing_set = set(existing_blocks)
new_prefixes = {
".".join(k.split(".")[:i])
for k in missing_tensors_dict
if k.endswith(".weight") and missing_tensors_dict[k].dim() == 2
for i, p in enumerate(k.split("."))
if p.isdigit() and ".".join(k.split(".")[:i])
}
added = sorted(new_prefixes - existing_set)
if added:
block_name_to_quantize = existing_blocks + added
# Identify weight tensors eligible for quantization (2D Linear weights)
def _is_eligible(k: str) -> bool:
if not k.endswith(".weight"):
return False
if missing_tensors_dict[k].dim() != 2:
return False
if k.startswith("talker."):
return False
# Only quantize weights that fall under the quantized blocks.
if block_name_to_quantize:
blocks = (
block_name_to_quantize
if isinstance(block_name_to_quantize, list)
else [b.strip() for b in block_name_to_quantize.split(",") if b.strip()]
)
if not any(k.startswith((b + ".", b + "[")) for b in blocks):
return False
layer_name = k[: -len(".weight")]
layer_cfg = _resolve_layer_cfg(layer_name)
# If extra_config explicitly covers this layer, trust its decision
if _is_covered_by_extra_config(layer_name):
return not _is_fp_layer(layer_cfg)
# Fall back to BLOCK_NAME_TO_IGNORE heuristic
if any(block in k for block in BLOCK_NAME_TO_IGNORE):
return False
return not _is_fp_layer(layer_cfg)
weight_keys = [k for k in missing_tensors_dict if _is_eligible(k)]
# Collect 2-D weight tensors that will NOT be quantized (for extra_config bookkeeping).
ignored_weight_keys = [
k
for k in missing_tensors_dict
if k.endswith(".weight") and missing_tensors_dict[k].dim() == 2 and k not in weight_keys
]
# Update quantization config: extra_config for ignored layers + block_name_to_quantize for new blocks
if ignored_weight_keys or weight_keys:
def _update_qcfg(qcfg, show_log=True):
if ignored_weight_keys:
ec = qcfg.get("extra_config", {})
for k in ignored_weight_keys:
layer_name = k[: -len(".weight")]
ec.setdefault(layer_name, {"bits": 16, "data_type": "fp"})
qcfg["extra_config"] = ec
if show_log:
# 7 is the len('.weight')
ignored_layers = [k[:-7] for k in ignored_weight_keys]
compressed_ignored_layers = compress_layer_names(ignored_layers)
logger.info(
f"Updated extra_config for {len(ignored_weight_keys)} ignored layer(s): "
f"{compressed_ignored_layers}"
)
if weight_keys:
existing = qcfg.get("block_name_to_quantize") or []
# In cases where this attribute is lacking
if not existing:
return qcfg
if isinstance(existing, str):
existing = [b.strip() for b in existing.split(",") if b.strip()]
existing_set = set(existing)
new_prefixes = {
".".join(k.split(".")[:i])
for k in weight_keys
for i, p in enumerate(k.split("."))
if p.isdigit() and ".".join(k.split(".")[:i])
}
added = sorted(new_prefixes - existing_set)
if added:
merged = existing + added
merged = [
b for b in merged if not any(b != other and b.startswith(other + ".") for other in merged)
]
qcfg["block_name_to_quantize"] = merged
if show_log:
logger.info(f"Updated block_name_to_quantize: {merged}")
return qcfg
for cfg_file in ["config.json", "quantization_config.json"]:
cfg_path = os.path.join(target_dir, cfg_file)
if not os.path.exists(cfg_path):
continue
logger.info(f"Processing {cfg_file} to update quantization_config for missing tensors...")
try:
with open(cfg_path) as f:
cfg_data = json.load(f)
if cfg_file == "config.json":
cfg_data["quantization_config"] = _update_qcfg(
cfg_data.get("quantization_config", {}), show_log=True
)
else:
cfg_data = _update_qcfg(cfg_data, show_log=False)
with open(cfg_path, "w") as f:
json.dump(cfg_data, f, indent=2)
except Exception as e:
logger.warning(f"Failed to update {cfg_file}: {e}")
if not weight_keys:
return missing_tensors_dict
try:
from tqdm import tqdm as _tqdm_woq
except ImportError:
_tqdm_woq = None
logger.info(
f"Applying WOQ[RTN] to "
f"{len(weight_keys)} missing Linear weight(s) (per-layer overrides from extra_config applied)..."
)
new_tensors: dict = {}
packed_weight_keys: set = set()
_woq_iter = (
_tqdm_woq(weight_keys, desc="WOQ[RTN] quantizing missing weights", unit="weight") if _tqdm_woq else weight_keys
)
for weight_key in _woq_iter:
weight = missing_tensors_dict[weight_key]
out_features, in_features = weight.shape
layer_name = weight_key[: -len(".weight")]
layer_cfg = _resolve_layer_cfg(layer_name)
bits = layer_cfg["bits"]
if bits not in [4, 8]:
if bits < 4:
bits = 4
elif 4 < bits < 8:
bits = 8
logger.warning(
f"Unsupported bits={layer_cfg['bits']} for layer '{layer_name}' in extra_config, "
f"falling back to bits={bits} for quantization."
)
group_size = layer_cfg["group_size"]
sym = layer_cfg["sym"]
effective_gs = group_size if group_size != -1 else in_features
if effective_gs <= 0:
continue
logger.debug(
f"WOQ[RTN] [{layer_name}]: bits={bits}, group_size={effective_gs}, sym={sym}, "
f"shape={list(weight.shape)}"
)
base_name = layer_name
try:
qweight, qzeros, scales = quantize_weight_rtn(
weight, bits=bits, group_size=effective_gs, sym=sym, packing=declared_packing
)
except Exception as e:
logger.warning(f"Failed to quantize {weight_key}: {e}, keeping original weight")
continue
new_tensors[base_name + ".qweight"] = qweight
new_tensors[base_name + ".qzeros"] = qzeros
new_tensors[base_name + ".scales"] = scales
packed_weight_keys.add(weight_key)
if not packed_weight_keys:
return missing_tensors_dict
# Replace original weights with packed versions
result = {}
for k, v in missing_tensors_dict.items():
if k not in packed_weight_keys:
result[k] = v
result.update(new_tensors)
logger.info(
f"Successfully packed {len(packed_weight_keys)} weight(s) into WOQ format "
f"({len(new_tensors)} packed tensor(s) created)."
)
return result
def _is_valid_regex(pattern: str) -> bool:
"""Return True if *pattern* is a valid regular expression."""
import re as _re
try:
_re.compile(pattern)
return True
except _re.error:
return False
def _get_woq_config_from_dir(target_dir: str) -> dict | None:
"""Read ``quantization_config`` from ``config.json`` in *target_dir*.
Only supports ``quant_method == "auto-round"`` with
``packing_format == "auto_round:auto_gptq"``. Returns ``None`` (skip
quantization) for any other combination or if any required parameter
(``bits``, ``group_size``, ``sym``) is missing.
Returns a dict with keys ``bits``, ``group_size``, ``sym``
or ``None``.
"""
config_path = os.path.join(target_dir, "config.json")
if not os.path.exists(config_path):
return None
with open(config_path) as f:
config_data = json.load(f)
qcfg = config_data.get("quantization_config", None)
if not isinstance(qcfg, dict):
return None
quant_method = qcfg.get("quant_method", "")
packing_format = qcfg.get("packing_format", "")
bits = qcfg.get("bits", None)
group_size = qcfg.get("group_size", None)
sym = qcfg.get("sym", None)
if quant_method != "auto-round":
logger.debug(
f"Skipping WOQ quantization of missing tensors: "
f"quant_method='{quant_method}' (only 'auto-round' is supported)"
)
return None
if packing_format != "auto_round:auto_gptq":
logger.debug(
f"Skipping WOQ quantization of missing tensors: "
f"packing_format='{packing_format}' (only 'auto_round:auto_gptq' is supported)"
)
return None
if bits is None or group_size is None or sym is None:
logger.debug(
f"Skipping WOQ quantization of missing tensors: "
f"incomplete quantization_config (bits={bits}, group_size={group_size}, sym={sym})"
)
return None
if bits > 8:
return None
block_name_to_quantize = qcfg.get("block_name_to_quantize", None)
extra_config = qcfg.get("extra_config", {})
return {
"bits": bits,
"group_size": group_size,
"sym": sym,
"block_name_to_quantize": block_name_to_quantize,
"extra_config": extra_config,
}