Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

[AutoScheme] fix lm_head no_grad issue and support MOE model with shared mix_score (#1971)

Xin Hee6b35d13200a
diff --git a/auto_round/auto_scheme/delta_loss.py b/auto_round/auto_scheme/delta_loss.py
index 1a6a94a90..7861acb52 100644
--- a/auto_round/auto_scheme/delta_loss.py
+++ b/auto_round/auto_scheme/delta_loss.py
@@ -14,6 +14,7 @@
import copy
import gc
+import math
from dataclasses import asdict
from functools import wraps
from typing import Iterable, Optional, Union
@@ -25,8 +26,16 @@ from tqdm import tqdm
from auto_round.auto_scheme.gen_auto_scheme import AutoScheme
from auto_round.auto_scheme.register import register_scheme_methods
from auto_round.auto_scheme.utils import (
+ _describe_layer_config,
+ _fill_inactive_expert_scores,
+ _log_batch_avg_loss,
+ _log_scheme_loss_matrix,
+ _log_score_summary_by_block_and_nonblock,
+ _scheme_short_name,
apply_quant_scheme,
+ build_expert_groups,
compute_layer_bits,
+ merge_lists_unionfind,
parse_shared_layers,
remove_quant_scheme,
)
@@ -62,6 +71,7 @@ from auto_round.utils import (
)
from auto_round.utils.device import MemoryMonitor, memory_monitor
from auto_round.utils.device_manager import get_current_device_manager
+from auto_round.utils.model import is_moe_model as _is_moe_model
from auto_round.utils.offload import OffloadManager
from auto_round.wrapper import WrapperLinear
@@ -81,6 +91,9 @@ class AutoSchemeWrapperLinear(WrapperLinear):
enable_torch_compile=False,
**kwargs,
):
+ """Wrap ``orig_layer`` to accumulate a ``mix_score`` (weight + activation loss) during
+ forward/backward, used by Delta Loss to rank candidate quantization schemes.
+ """
super().__init__(
orig_layer,
enable_minmax_tuning,
@@ -90,7 +103,6 @@ class AutoSchemeWrapperLinear(WrapperLinear):
enable_torch_compile=enable_torch_compile,
**kwargs,
)
- self.total_act_score = 0.0
self.act_score = 0.0
self.avg_act_score = 0.0
self.act_cnt = 0.0
@@ -105,6 +117,9 @@ class AutoSchemeWrapperLinear(WrapperLinear):
self.orig_layer.weight.requires_grad = True
def _qdq_act(self, x, act_min_scale=1.0, act_max_scale=1.0, act_max=None):
+ """Quant-dequant the activation and, in ``grad_mode``, register a backward hook that
+ accumulates ``act_score`` from ``|grad * (x - qdq_x)|``.
+ """
if hasattr(self.orig_layer, "act_bits") and self.orig_layer.act_bits > 8:
return x, 1.0, None
@@ -118,6 +133,7 @@ class AutoSchemeWrapperLinear(WrapperLinear):
self.x_diff = x_diff.to("cpu")
def save_grad(grad):
+ """Backward hook: accumulate activation score from grad * (x - qdq_x)."""
if self.max_act_value == 0:
if torch.abs(grad).max() != 0:
raise ValueError
@@ -131,8 +147,7 @@ class AutoSchemeWrapperLinear(WrapperLinear):
self.act_cnt -= 1
return None
- self.total_act_score += torch.abs((grad * self.x_diff.to(grad.device))).sum().item()
- self.act_score = 0.0 if self.act_cnt <= 0 else self.total_act_score / self.act_cnt
+ self.act_score += torch.abs((grad * self.x_diff.to(grad.device))).sum().item()
self.mix_score = self.weight_score + self.act_score
self.x_diff = None
return None
@@ -141,6 +156,11 @@ class AutoSchemeWrapperLinear(WrapperLinear):
return qdq_x, scale, zp
def _qdq_weight(self, value, min_scale, max_scale):
+ """Quant-dequant the weight and, in ``grad_mode``, register a backward hook that
+ accumulates ``weight_score`` from ``|grad * (weight - qdq_w)|``. Weight quantization
+ for the hook is only recomputed lazily inside the hook itself, so layers whose
+ forward never runs (e.g. unrouted MoE experts) never pay this cost.
+ """
device = self.device
if self.orig_layer.bits > 8 or not self.need_weight_grad:
qdq_w, scale, zp = super()._qdq_weight(
@@ -156,13 +176,13 @@ class AutoSchemeWrapperLinear(WrapperLinear):
if self.grad_mode:
def save_grad(grad):
+ """Backward hook: accumulate weight score from grad * (weight - qdq_w)."""
qdq_w, scale, zp = self.super_qdq_func(
torch.tensor(0, device=device), torch.tensor(1.0, device=device), torch.tensor(1.0, device=device)
)
w_diff = self.orig_layer.weight - qdq_w.to(self.orig_layer.weight.device)
self.weight_score += torch.abs((grad.to(w_diff.device) * w_diff)).sum().item()
- act_score = 0.0 if self.act_cnt <= 0 else self.total_act_score / self.act_cnt
- self.mix_score = self.weight_score + act_score
+ self.mix_score = self.weight_score + self.act_score
return None
qdq_w.register_hook(save_grad)
@@ -170,6 +190,7 @@ class AutoSchemeWrapperLinear(WrapperLinear):
class AutoSchemeWrapperLinearIMatrix(WrapperLinear):
+ """GGUF-K wrapper that scores a layer using an imatrix-aware quant search (RTN, iters=0)."""
def __init__(
self,
@@ -182,6 +203,7 @@ class AutoSchemeWrapperLinearIMatrix(WrapperLinear):
enable_torch_compile=False,
**kwargs,
):
+ """Wrap ``orig_layer`` and eagerly run the imatrix-aware quant search to build ``qdq_w``."""
super().__init__(
orig_layer,
enable_minmax_tuning,
@@ -191,7 +213,6 @@ class AutoSchemeWrapperLinearIMatrix(WrapperLinear):
enable_torch_compile=enable_torch_compile,
**kwargs,
)
- self.total_act_score = 0.0
self.act_score = 0.0
self.avg_act_score = 0.0
self.act_cnt = 0.0
@@ -216,8 +237,9 @@ class AutoSchemeWrapperLinearIMatrix(WrapperLinear):
@torch.no_grad()
def post_init_qdqw(self, device):
- # Could not place in qdq_w, otherwise vram is much higher
-
+ """Run the imatrix-aware quant search once and cache the result as buffer ``qdq_w``,
+ registering a backward hook on it to accumulate ``weight_score``.
+ """
qdq_w, _, _ = self.weight_search_quant_func(
self.orig_layer.weight.to(device),
bits=self.orig_layer.bits,
@@ -235,10 +257,10 @@ class AutoSchemeWrapperLinearIMatrix(WrapperLinear):
self.register_buffer("qdq_w", qdq_w.detach().clone().to(self.orig_layer.weight.device))
def save_grad(grad):
+ """Backward hook: accumulate weight score from grad * (weight - qdq_w)."""
w_diff = self.orig_layer.weight - self.qdq_w.to(self.orig_layer.weight.device)
self.weight_score += torch.abs((grad.to(torch.float32) * w_diff.to(grad.device))).sum().item()
- act_score = 0.0 if self.act_cnt <= 0 else self.total_act_score / self.act_cnt
- self.mix_score = self.weight_score + act_score
+ self.mix_score = self.weight_score + self.act_score
return None
self.qdq_w.requires_grad_(True)
@@ -247,6 +269,9 @@ class AutoSchemeWrapperLinearIMatrix(WrapperLinear):
self.qdq_w.register_hook(save_grad)
def _qdq_act(self, x, act_min_scale=1.0, act_max_scale=1.0, act_max=None):
+ """Quant-dequant the activation and, in ``grad_mode``, register a backward hook that
+ accumulates ``act_score`` from ``|grad * (x - qdq_x)|``.
+ """
if hasattr(self.orig_layer, "act_bits") and self.orig_layer.act_bits > 8:
return x, 1.0, None
@@ -260,6 +285,7 @@ class AutoSchemeWrapperLinearIMatrix(WrapperLinear):
self.x_diff = x_diff.to("cpu")
def save_grad(grad):
+ """Backward hook: accumulate activation score from grad * (x - qdq_x)."""
if self.max_act_value == 0:
if torch.abs(grad).max() != 0:
raise ValueError
@@ -273,8 +299,7 @@ class AutoSchemeWrapperLinearIMatrix(WrapperLinear):
self.act_cnt -= 1
return None
- self.total_act_score += torch.abs((grad * self.x_diff.to(grad.device))).sum().item()
- self.act_score = 0.0 if self.act_cnt <= 0 else self.total_act_score / self.act_cnt
+ self.act_score += torch.abs((grad * self.x_diff.to(grad.device))).sum().item()
self.mix_score = self.weight_score + self.act_score
self.x_diff = None
return None
@@ -283,10 +308,12 @@ class AutoSchemeWrapperLinearIMatrix(WrapperLinear):
return qdq_x, scale, zp
def _qdq_weight(self, value, min_scale, max_scale):
+ """Return the cached ``qdq_w`` computed eagerly in ``__init__`` (via ``post_init_qdqw``)."""
return self.qdq_w, 1.0, None
class AutoSchemeWrapperLinearForGGUFK(AutoSchemeWrapperLinear):
+ """GGUF-K wrapper (no imatrix): scores a layer using the plain GGUF K-quant search."""
def __init__(
self,
@@ -298,6 +325,7 @@ class AutoSchemeWrapperLinearForGGUFK(AutoSchemeWrapperLinear):
need_weight_grad=False,
**kwargs,
):
+ """Wrap ``orig_layer`` and eagerly run the GGUF K-quant search to build ``qdq_w``."""
super().__init__(
orig_layer,
enable_minmax_tuning,
@@ -307,18 +335,24 @@ class AutoSchemeWrapperLinearForGGUFK(AutoSchemeWrapperLinear):
need_weight_grad,
**kwargs,
)
- with torch.no_grad():
- qdq_w, scale, zp = self.super_qdq_func(
- torch.tensor(0).to(device), torch.tensor(1.0).to(device), torch.tensor(1.0).to(device)
- )
+ self.post_init_qdqw(device)
+
+ @torch.no_grad()
+ def post_init_qdqw(self, device):
+ """Run the GGUF K-quant search once and cache the result as buffer ``qdq_w``,
+ registering a backward hook on it to accumulate ``weight_score``.
+ """
+ qdq_w, scale, zp = self.super_qdq_func(
+ torch.tensor(0).to(device), torch.tensor(1.0).to(device), torch.tensor(1.0).to(device)
+ )
self.register_buffer("qdq_w", qdq_w.detach().clone().to(self.orig_layer.weight.device))
def save_grad(grad):
+ """Backward hook: accumulate weight score from grad * (weight - qdq_w)."""
w_diff = self.orig_layer.weight - self.qdq_w.to(self.orig_layer.weight.device)
# TODO strange, grad could be in CPU
self.weight_score += torch.abs((grad.to(w_diff.device).to(torch.float32) * w_diff)).sum().item()
- act_score = 0.0 if self.act_cnt <= 0 else self.total_act_score / self.act_cnt
- self.mix_score = self.weight_score + act_score
+ self.mix_score = self.weight_score + self.act_score
return None
self.qdq_w.requires_grad_(True)
@@ -326,10 +360,14 @@ class AutoSchemeWrapperLinearForGGUFK(AutoSchemeWrapperLinear):
self.qdq_w.register_hook(save_grad)
def _qdq_weight(self, value, min_scale, max_scale):
+ """Return the cached ``qdq_w`` computed eagerly in ``__init__`` (via ``post_init_qdqw``)."""
return self.qdq_w, 1.0, None
class AutoSchemeWrapperLinearForGGUFKImatrix(AutoSchemeWrapperLinear):
+ """GGUF-K wrapper (with imatrix): scores a layer using the imatrix-weighted GGUF K-quant
+ search (``_init_scale``).
+ """
def __init__(
self,
@@ -342,6 +380,9 @@ class AutoSchemeWrapperLinearForGGUFKImatrix(AutoSchemeWrapperLinear):
enable_torch_compile=False,
**kwargs,
):
+ """Wrap ``orig_layer`` and eagerly run the imatrix-weighted GGUF K-quant search to
+ build ``qdq_w``.
+ """
super().__init__(
orig_layer,
enable_minmax_tuning,
@@ -356,14 +397,17 @@ class AutoSchemeWrapperLinearForGGUFKImatrix(AutoSchemeWrapperLinear):
@torch.no_grad()
def post_init_qdqw(self, device): # Could not place in qdq_w, otherwise vram is much higher
+ """Run the imatrix-weighted GGUF K-quant search once and cache the result as buffer
+ ``qdq_w``, registering a backward hook on it to accumulate ``weight_score``.
+ """
qdq_w = self._init_scale(device).detach()
self.register_buffer("qdq_w", qdq_w.detach().clone().to(self.orig_layer.weight.device))
def save_grad(grad):
+ """Backward hook: accumulate weight score from grad * (weight - qdq_w)."""
w_diff = self.orig_layer.weight - self.qdq_w.to(self.orig_layer.weight.device)
self.weight_score += torch.abs((grad.to(torch.float32) * w_diff.to(grad.device))).sum().item()
- act_score = 0.0 if self.act_cnt <= 0 else self.total_act_score / self.act_cnt
- self.mix_score = self.weight_score + act_score
+ self.mix_score = self.weight_score + self.act_score
return None
self.qdq_w.requires_grad_(True)
@@ -373,6 +417,9 @@ class AutoSchemeWrapperLinearForGGUFKImatrix(AutoSchemeWrapperLinear):
@torch.no_grad()
def _init_scale(self, device):
+ """Compute the imatrix-weighted GGUF K-quant quant-dequant weight for ``bits`` in
+ [2,3,4,5,6], returned in the original weight dtype.
+ """
tensor = self.orig_layer.weight.data.to(device)
bits = self.orig_layer.bits
scale_dtype = self.orig_layer.scale_dtype
@@ -407,6 +454,7 @@ class AutoSchemeWrapperLinearForGGUFKImatrix(AutoSchemeWrapperLinear):
return qdq_w.to(orig_dtype)
def _qdq_weight(self, value, min_scale, max_scale):
+ """Return the cached ``qdq_w`` computed eagerly in ``__init__`` (via ``post_init_qdqw``)."""
return self.qdq_w, 1.0, None
@@ -414,6 +462,7 @@ def register_imatrix_hook(model):
"""Registers hooks to accumulate activation squared norms into `imatrix`."""
def get_imatrix_hook(module, input, output):
+ """Forward hook: accumulate the per-channel squared-activation sum into ``module.imatrix``."""
input = input[0] if isinstance(input, (tuple, list)) else input
flattened = input.reshape(-1, input.shape[-1]).to(torch.float32)
squared = torch.sum(torch.pow(flattened, 2), dim=0).to(torch.float32)
@@ -433,7 +482,10 @@ def register_imatrix_hook(model):
@torch.no_grad()
def cal_imatrix(model, dataloader, major_device, low_gpu_mem_usage):
-
+ """Accumulate an activation-based imatrix on every supported layer by running the
+ calibration ``dataloader`` through ``model`` once (dispatches to the low-GPU-memory or
+ full-forward variant based on ``low_gpu_mem_usage``).
+ """
if low_gpu_mem_usage:
cal_imatrix_low_gpu(model, dataloader, major_device)
else:
@@ -446,18 +498,24 @@ def cal_imatrix(model, dataloader, major_device, low_gpu_mem_usage):
def cal_imatrix_low_gpu(model, dataloader, major_device):
+ """Low-GPU-memory variant of ``cal_imatrix``: moves each block to ``major_device`` only
+ for the duration of its own forward pass (via pre/post forward hooks), then back to CPU.
+ """
imatrix_hooks = register_imatrix_hook(model)
block_names = get_block_names(model, quant_vision=True)
block_names = flatten_list(block_names)
def move_to_gpu_hook(module, inputs):
+ """Pre-forward hook: move this block (and its inputs) to ``major_device``."""
module.to(major_device)
to_device(inputs, major_device)
def move_to_cpu(module, inputs, outputs):
+ """Forward hook: move this block back to CPU once its forward pass is done."""
module.to("cpu")
def move_to_cpu_clear_memory(module, inputs, outputs):
+ """Forward hook: move this block back to CPU and free the device memory it used."""
module.to("cpu")
clear_memory(device_list=major_device)
@@ -484,8 +542,12 @@ def cal_imatrix_low_gpu(model, dataloader, major_device):
class MyCustomError(Exception):
+ """Raised from ``backward_pre_hook`` to deliberately interrupt ``loss.backward()`` at the
+ last block, so gradients can be replayed manually block-by-block in ``model_forward_low_gpu``.
+ """
def __init__(self, message):
+ """Create the interrupt signal with the given ``message``."""
super().__init__(message)
@@ -493,6 +555,14 @@ last_grad_input = None
def prepare_model_low_gpu(model, block_inputs: dict = None, pbar=None, major_device="cpu"):
+ """Wrap every block's forward so that, for one calibration batch, it (1) moves itself to
+ ``major_device`` on demand, (2) records its own inputs into ``block_inputs`` (on CPU) so
+ they can be replayed later, and (3) moves itself back to CPU once done.
+
+ Called once per calibration batch before ``model_forward_low_gpu`` runs the actual
+ forward+backward -- the recorded ``block_inputs`` are what let the backward pass be
+ replayed manually, one block at a time, without keeping every block resident on GPU.
+ """
block_inputs.clear()
for n, m in model.named_modules():
if hasattr(m, "grad_mode"):
@@ -501,10 +571,16 @@ def prepare_model_low_gpu(model, block_inputs: dict = None, pbar=None, major_dev
block_names = get_block_names(model)[0]
def wrap_forward(module, module_name):
+ """Build a replacement ``forward`` for ``module`` that captures its inputs/outputs
+ (see ``prepare_model_low_gpu`` docstring) while moving it to/from ``major_device``.
+ """
original_forward = module.forward
@wraps(original_forward)
def new_forward(*args, **kwargs):
+ """Move the block to device, run its original forward, cache its (CPU) inputs
+ for later replay, then move the block back to CPU.
+ """
move_module_to_tuning_device(module, major_device=major_device)
# for n,m in module.named_modules():
# if hasattr(m, "post_init_qdqw"):
@@ -595,8 +671,17 @@ def model_forward(model, data, **forward_kwargs):
return model(**prepared, **forward_kwargs), prepared
-def model_forward_low_gpu(model, dataloader, major_device="cuda", pbar=None):
+def model_forward_low_gpu(model, dataloader, major_device="cuda", pbar=None, scheme_tag=None):
+ """Run one full scoring pass (all calibration batches) in low-GPU-memory mode.
+
+ For each batch: capture per-block inputs via ``prepare_model_low_gpu``, run a forward
+ pass whose backward is deliberately interrupted at the last block (``backward_pre_hook``
+ raising ``MyCustomError``), then manually replay the backward pass block-by-block
+ (moving each block to ``major_device`` only for its own recompute + backward, then back
+ to CPU) so only one block's weights need to be resident on GPU at a time.
+ """
block_inputs = {}
+ total_batches = len(dataloader) if hasattr(dataloader, "__len__") else None
block_names = get_block_names(model)[0]
for name in block_names:
@@ -610,9 +695,25 @@ def model_forward_low_gpu(model, dataloader, major_device="cuda", pbar=None):
get_current_device_manager().synchronize()
raise MyCustomError("Interrupt backward pass")
- for data in dataloader:
+ for batch_idx, data in enumerate(dataloader, start=1):
prepare_model_low_gpu(model, block_inputs, major_device=major_device, pbar=pbar)
+ # lm_head sits outside every decoder block, so it never gets `grad_mode=True`
+ # in the manual block-by-block backward below. Scope the fix narrowly to
+ # just lm_head (rather than every non-block module) to avoid enabling grad
+ # tracking / scoring hooks on unrelated out-of-block layers, which would
+ # add extra autograd-graph memory for no benefit. The backward flow is:
+ # loss → lm_head (hook fires here) → norm → last_block (hook raises error)
+ head_name = get_lm_head_name(model)
+ if head_name is not None:
+ # Once lm_head has been wrapped for scoring, `get_lm_head_name` resolves
+ # to the inner original Linear (e.g. "lm_head.orig_layer") rather than
+ # the wrapper itself ("lm_head") -- strip the suffix to reach the wrapper.
+ head_name = head_name.removesuffix(".orig_layer")
+ head_module = get_module(model, head_name)
+ if hasattr(head_module, "grad_mode"):
+ head_module.grad_mode = True
+
# Register backward hook on the last block
last_block = get_module(model, block_names[-1])
last_block_backward_hook = last_block.register_full_backward_pre_hook(backward_pre_hook)
@@ -701,6 +802,15 @@ def model_forward_low_gpu(model, dataloader, major_device="cuda", pbar=None):
pbar.update(1)
+ _log_batch_avg_loss(
+ model,
+ batch_idx,
+ pbar=pbar,
+ block_names=block_names,
+ total_batches=total_batches,
+ scheme_tag=scheme_tag,
+ )
+
def get_score_for_scheme(
model,
@@ -723,8 +833,19 @@ def get_score_for_scheme(
is_vlm: bool = False,
force_mllm: bool = False,
model_name: Optional[str] = None,
+ scheme_tag: Optional[str] = None,
):
+ """Wrap every quantizable layer in ``quant_layer_names`` with a scoring wrapper, run
+ forward(+backward, unless RTN-only) calibration over ``nsamples`` examples from
+ ``dataset``/``dataloader``, then unwrap and return each layer's ``[bits, loss]``.
+ """
scores_dict = {} # Key=name,Val=[quant_total_bits, loss]
+ # Include the visual block(s) when scoring VLMs with ``--quant_nontext_module``
+ # (``force_mllm=True``) so vision-tower layer losses match a block below instead
+ # of silently falling through to "non_block" in the logging/inactive-expert-fill
+ # helpers. Mirrors the same ``quant_vision=force_mllm`` pattern used in
+ # ``_gen_layer_config``.
+ block_names = get_block_names(model, quant_vision=force_mllm)[0]
for n, m in model.named_modules():
if type(m) in SUPPORTED_LAYER_TYPES:
m.weight.requires_grad = False
@@ -931,11 +1052,11 @@ def get_score_for_scheme(
"AutoScheme(force_mllm): cannot build mllm dataloader. "
"Provide a `processor` and a multimodal `dataset`."
)
- model_forward_low_gpu(model, mllm_loader, major_device=major_device, pbar=pbar)
+ model_forward_low_gpu(model, mllm_loader, major_device=major_device, pbar=pbar, scheme_tag=scheme_tag)
else:
try:
dataloader = _build_calib_dataloader()
- model_forward_low_gpu(model, dataloader, major_device=major_device, pbar=pbar)
+ model_forward_low_gpu(model, dataloader, major_device=major_device, pbar=pbar, scheme_tag=scheme_tag)
except Exception as exc: # noqa: BLE001
if not is_vlm:
raise
@@ -947,7 +1068,7 @@ def get_score_for_scheme(
batch_size = 1
if mllm_loader is None:
raise
- model_forward_low_gpu(model, mllm_loader, major_device=major_device, pbar=pbar)
+ model_forward_low_gpu(model, mllm_loader, major_device=major_device, pbar=pbar, scheme_tag=scheme_tag)
else:
for n, m in model.named_modules():
if hasattr(m, "grad_mode"):
@@ -956,6 +1077,11 @@ def get_score_for_scheme(
# m.post_init_qdqw()
def _run_forward_loop(loader):
+ """Run the full (non-low-GPU) forward+backward calibration loop over ``loader``,
+ accumulating ``mix_score`` on every wrapped layer and periodically logging progress
+ via ``_log_batch_avg_loss``.
+ """
+ total_batches = len(loader) if hasattr(loader, "__len__") else None
_checked_pixel = False
_pixel_keys = (
"pixel_values",
@@ -965,7 +1091,7 @@ def get_score_for_scheme(
"images",
"image",
)
- for data in loader:
+ for batch_idx, data in enumerate(loader, start=1):
# Pull labels out of the batch (VLM datasets often carry them;
# LLM ones don't) before mllm_model_forward casts dtypes.
_src = data if isinstance(data, dict) else None
@@ -1004,6 +1130,14 @@ def get_score_for_scheme(
m.grad = None
if pbar is not None:
pbar.update(1)
+ _log_batch_avg_loss(
+ model,
+ batch_idx,
+ pbar=pbar,
+ block_names=block_names,
+ total_batches=total_batches,
+ scheme_tag=scheme_tag,
+ )
if force_mllm:
mllm_loader = _build_mllm_calib_dataloader()
@@ -1031,7 +1165,6 @@ def get_score_for_scheme(
for n, m in model.named_parameters():
m.grad = None
- scores_dict = {}
for n, m in model.named_modules():
if hasattr(m, "mix_score"):
if m.orig_layer.act_bits <= 8:
@@ -1041,6 +1174,15 @@ def get_score_for_scheme(
)
layer_bits, _ = compute_layer_bits(m.orig_layer, ignore_scale_zp_bits=ignore_scale_zp_bits)
scores_dict[n] = [layer_bits, m.mix_score]
+ _fill_inactive_expert_scores(scores_dict, block_names)
+ _log_score_summary_by_block_and_nonblock(
+ scores_dict,
+ block_names,
+ model=model,
+ scheme_tag=scheme_tag,
+ summary_stage="final",
+ )
+
for n, m in model.named_modules():
if hasattr(m, "orig_layer"):
# Explicitly break reference cycles to ensure GC can free the wrapper.
@@ -1085,10 +1227,11 @@ def choose_bits_per_layer_with_path(layers: dict, P: int, max_states: int = None
solution exists.
"""
# dp: total_params -> (accumulated_loss, chosen_path)
- # The path explicitly stores the selected options.
- dp: dict[int, tuple[float, list]] = {0: (0.0, [])}
+ # The path is stored as a tuple to avoid the high overhead of repeatedly
+ # copying Python lists for every DP transition.
+ dp: dict[int, tuple[float, tuple]] = {0: (0.0, ())}
for layer_name, opts in layers.items():
- new_dp: dict[int, tuple[float, list]] = {}
+ new_dp: dict[int, tuple[float, tuple]] = {}
for cur_params, (cur_loss, cur_path) in dp.items():
for opt in opts:
scheme, bits_cost, loss_cost, layer_names = opt
@@ -1097,7 +1240,7 @@ def choose_bits_per_layer_with_path(layers: dict, P: int, max_states: int = None
continue
new_loss = cur_loss + loss_cost
- new_path = cur_path + [(layer_names, scheme)]
+ new_path = cur_path + ((layer_names, scheme),)
# Keep the path with smaller loss for the same parameter budget
if np_total not in new_dp or new_loss < new_dp[np_total][0]:
@@ -1107,7 +1250,7 @@ def choose_bits_per_layer_with_path(layers: dict, P: int, max_states: int = None
return None, None
# Pareto pruning: remove dominated (params, loss) states
items = sorted(new_dp.items(), key=lambda x: x[0]) # (params, (loss, path))
- pruned: dict[int, tuple[float, list]] = {}
+ pruned: dict[int, tuple[float, tuple]] = {}
best_loss_so_far = float("inf")
for params_val, (loss_val, path_val) in items:
if loss_val < best_loss_so_far:
@@ -1115,7 +1258,7 @@ def choose_bits_per_layer_with_path(layers: dict, P: int, max_states: int = None
best_loss_so_far = loss_val
# Beam width limit: if too many states survive Pareto pruning,
- # uniformly subsample to bound memory usage. For models with many
+ # uniformly subsample to bound memory usage. For models with many
# layers whose sizes are incommensurate, the number of distinct
# cumulative-bit sums can grow to millions, each storing a full
# path copy — easily exceeding 70 GB of RAM.
@@ -1128,9 +1271,11 @@ def choose_bits_per_layer_with_path(layers: dict, P: int, max_states: int = None
n = len(sorted_keys)
# Uniformly pick max_states indices (always include first and last)
step = (n - 1) / (max_states - 1)
- selected: dict[int, tuple[float, list]] = {}
+ selected: dict[int, tuple[float, tuple]] = {}
for i in range(max_states):
idx = int(round(i * step))
+ if idx >= n:
+ idx = n - 1
k = sorted_keys[idx]
selected[k] = pruned[k]
pruned = selected
@@ -1140,15 +1285,24 @@ def choose_bits_per_layer_with_path(layers: dict, P: int, max_states: int = None
# Select the solution with the minimum loss
best_params = min(dp.keys(), key=lambda k: dp[k][0])
best_loss, best_path = dp[best_params]
- return best_loss, best_path
+ return best_loss, list(best_path)
def move_module_to_tuning_device(module, major_device="cpu"):
+ """Move every submodule of ``module`` to its own tuning device: wrapper submodules go to
+ ``orig_layer.tuning_device``/``tuning_device`` (set per-layer earlier), leaf modules with
+ no such attribute fall back to ``major_device``, and any directly-held parameters/buffers
+ (not just the standard ``.to()`` targets) are relocated along with their ``.grad``.
+ """
def _normalize(dev):
+ """Coerce ``dev`` (str or ``torch.device``) into a ``torch.device``."""
return dev if isinstance(dev, torch.device) else torch.device(dev)
def _move_own_tensors(m, device):
+ """Move ``m``'s directly-owned (non-recursive) parameters/buffers (and their
+ ``.grad``) to ``device``.
+ """
# Cover non-leaf modules that directly hold nn.Parameter / buffers
# (e.g. Mamba/GDN linear_attn with A_log & dt_bias). Also relocate
# p.grad together with p.data — otherwise the next backward's grad
@@ -1294,6 +1448,14 @@ def _gen_layer_config(
processor=None,
is_vlm: bool = False,
):
+ """Score every candidate scheme in ``auto_scheme.options`` against ``quant_layer_names``
+ and return per-layer per-scheme losses used by the caller to pick a final bit-width
+ assignment (via the DP knapsack in ``choose_bits_per_layer_with_path``).
+
+ For each scheme: wraps every quantizable layer with a scoring wrapper, runs
+ forward+backward calibration to accumulate ``mix_score`` (weight + activation loss), then
+ unwraps and records the result before moving to the next scheme.
+ """
# Initialize memory tracking for AutoScheme
memory_monitor = MemoryMonitor()
# memory_monitor.reset()
@@ -1338,6 +1500,8 @@ def _gen_layer_config(
embedding_layers_names.append(name)
quant_layer_names = list(set(quant_layer_names) - set(embedding_layers_names))
+ is_moe_model = _is_moe_model(model)
+
# Decide whether AutoScheme has to score vision-tower layers (typically
# because the user passed ``--quant_nontext_module``). Used below to
# clamp batch_size to 1 (image sizes vary) and to pick the multimodal
@@ -1346,7 +1510,11 @@ def _gen_layer_config(
vision_markers = ("vision", "visual", "image", "img")
force_mllm = is_vlm and any(any(marker in n.lower() for marker in vision_markers) for n in quant_layer_names)
- block_name = get_block_names(model)[0] # TODO need change to support vlm
+ # When scoring vision-tower layers (``force_mllm``, typically because the
+ # caller passed ``--quant_nontext_module``), include the visual block(s)
+ # so they get ``in_block=True``, count towards ``block_num``/progress, and
+ # participate in offload hooks below just like language blocks.
+ block_name = get_block_names(model, quant_vision=force_mllm)[0]
for name in block_name:
module = get_module(model, name)
module.in_block = True
@@ -1364,6 +1532,9 @@ def _gen_layer_config(
schemes = auto_scheme.options
def check_bf16_scheme(scheme):
+ """Return True if ``scheme`` is effectively BF16/no-op (bits >= 16 and act_bits >= 16),
+ in which case scoring can skip the expensive wrap/forward/backward cycle entirely.
+ """
if isinstance(scheme, str) and scheme.upper() == "BF16":
return True
if isinstance(scheme, QuantizationScheme):
@@ -1372,32 +1543,34 @@ def _gen_layer_config(
return True
return False
- if auto_scheme.nsamples is not None:
+ from auto_round import envs as _envs
+
+ _env_nsamples = _envs.AR_AUTO_SCHEME_NSAMPLES
+ # Priority for nsamples: env > API > default
+ if _env_nsamples is not None:
+ nsamples = _env_nsamples
+ elif auto_scheme.nsamples is not None:
nsamples = auto_scheme.nsamples
else:
- is_moe_model = False
- if hasattr(model, "config"):
- for key in model.config.to_dict().keys():
- if "moe" in key or "expert" in key:
- is_moe_model = True
- break
- if is_moe_model:
- logger.info(
- "The model appears to be an MoE model. "
- "Using more samples to help generate a better auto-scheme recipe."
- )
- nsamples = 64
- else:
- nsamples = 16
- seqlen = auto_scheme.seqlen if auto_scheme.seqlen is not None else 256
+ nsamples = 16
+
+ # seqlen: API explicit setting takes priority; otherwise use MoE-aware default
+ if auto_scheme.seqlen is not None:
+ seqlen = auto_scheme.seqlen
+ else:
+ seqlen = 128 if is_moe_model else 256
if auto_scheme.batch_size is not None:
batch_size = auto_scheme.batch_size
else:
- if auto_scheme.low_gpu_mem_usage:
- batch_size = 8
+ _env_batch_size = _envs.AR_AUTO_SCHEME_BATCH_SIZE
+ if _env_batch_size is not None:
+ batch_size = _env_batch_size
else:
- batch_size = 1
+ if auto_scheme.low_gpu_mem_usage:
+ batch_size = 8
+ else:
+ batch_size = 1
# ------------------------------------------------------------------ #
# Multimodal calibration: ``batch_size`` must be 1 because image #
@@ -1412,9 +1585,12 @@ def _gen_layer_config(
pbar_cnt = 0
need_weight_grad = False
need_imatrix = False # only trigger it for gguf q-k quant
+ effective_scheme_num = 0
+ block_num = len(block_name)
for index, scheme in enumerate(schemes):
if check_bf16_scheme(scheme):
continue
+ effective_scheme_num += 1
if isinstance(scheme, str):
scheme = asdict(preset_name_to_scheme(scheme))
elif isinstance(scheme, QuantizationScheme):
@@ -1429,8 +1605,70 @@ def _gen_layer_config(
pbar_cnt += nsamples
if auto_scheme.low_gpu_mem_usage:
pbar_cnt += len(block_name) * 2 * ((nsamples + batch_size - 1) // batch_size) # forward backward
+
+ # Formula-style step log for paper/debug readability.
+ # In low_gpu mode, one calibration mini-batch uses block-wise forward+backward replay.
+ # so base_step_per_scheme = block_num * 2.
+ base_total_steps = effective_scheme_num * block_num * 2
+ logger.info(f"AutoScheme steps(total)={base_total_steps}")
+ logger.info(
+ "AutoScheme steps variables: "
+ f"scheme_num={effective_scheme_num}, block_num={block_num}, "
+ f"nsamples={nsamples}, batch_size={batch_size}"
+ )
+ logger.info(
+ "AutoScheme: nsamples/batch_size can be overridden via env vars "
+ "AR_AUTO_SCHEME_NSAMPLES / AR_AUTO_SCHEME_BATCH_SIZE "
+ "(e.g. `export AR_AUTO_SCHEME_NSAMPLES=1` for a quick run); "
+ "see docs/environments.md for details."
+ )
+ if auto_scheme.low_gpu_mem_usage:
+ n_batches = (nsamples + batch_size - 1) // batch_size
+ logger.info(
+ "AutoScheme steps expanded(low_gpu): "
+ "total_steps = scheme_num * block_num * 2(forward+backward) * n_batches = "
+ f"{effective_scheme_num} * {block_num} * 2 * {n_batches} = {pbar_cnt}"
+ )
+ else:
+ logger.info(
+ "AutoScheme steps expanded(full_backward): "
+ f"total_steps = scheme_num * nsamples = {effective_scheme_num} * {nsamples} = {pbar_cnt}"
+ )
shared_layers = parse_shared_layers(model, auto_scheme.shared_layers)
+ # Auto-group MoE expert layers so DP treats all experts in one block as a unit.
+ if is_moe_model:
+ expert_groups = build_expert_groups(model, quant_layer_names, fixed_layer_scheme)
+ if expert_groups:
+ shared_layers = merge_lists_unionfind(shared_layers + expert_groups)
+
+ # Pre-compute per-key weight numel (for loss/elem display). Mirrors the
+ # shared_layers grouping used in the scoring loop so keys match total_scores.
+ _dp_names = set(quant_layer_names) - set(fixed_layer_scheme.keys())
+ _shared_seen: set[str] = set()
+ layer_numel: dict[str, int] = {}
+ for _share_layer in shared_layers:
+ _nl = [n for n in _share_layer if n in _dp_names]
+ if not _nl:
+ continue
+ _total = 0
+ for _n in _nl:
+ _m = get_module(model, _n)
+ _np = _m.weight.numel() if hasattr(_m, "weight") and _m.weight is not None else 0
+ if _np == 0 and hasattr(_m, "_cached_weight_numel"):
+ _np = _m._cached_weight_numel
+ _total += _np
+ _shared_seen.add(_n)
+ layer_numel[_nl[0]] = _total
+ for _n in _dp_names:
+ if _n in _shared_seen:
+ continue
+ _m = get_module(model, _n)
+ _np = _m.weight.numel() if hasattr(_m, "weight") and _m.weight is not None else 0
+ if _np == 0 and hasattr(_m, "_cached_weight_numel"):
+ _np = _m._cached_weight_numel
+ layer_numel[_n] = _np
+
options_scores = []
if need_imatrix:
@@ -1455,6 +1693,8 @@ def _gen_layer_config(
pbar = tqdm(total=pbar_cnt, desc="Generating AutoScheme")
for index, scheme in enumerate(schemes):
+ scheme_tag = f"{index + 1}/{len(schemes)} {_scheme_short_name(scheme)}"
+ logger.info(f"AutoScheme transition: switch to scheme {index + 1}/{len(schemes)} ({scheme})")
apply_quant_scheme(
model, quant_layer_names=quant_layer_names, fixed_layer_scheme=fixed_layer_scheme, scheme=scheme
)
@@ -1487,6 +1727,7 @@ def _gen_layer_config(
is_vlm=is_vlm,
force_mllm=force_mllm,
model_name=model_name,
+ scheme_tag=scheme_tag,
)
# Track peak RAM after each scheme scoring
memory_monitor.update()
@@ -1514,6 +1755,10 @@ def _gen_layer_config(
else:
total_scores[key] = [item]
options_scores.append(options_total_loss)
+ logger.info(
+ f"AutoScheme transition: scheme {index + 1}/{len(schemes)} "
+ f"scoring finished (total_loss={options_total_loss:.6f})"
+ )
clear_memory(device_list=device_list)
# Remove hooks and restore original weights from disk for final bit-budget computations
@@ -1589,7 +1834,6 @@ def _gen_layer_config(
In all cases, the selected scheme must not exceed the total bit budget
(i.e., embedding bits + fixed bits + min DP bits <= target_params_cnt).
"""
- import math
if has_tied_lm_head:
if target_bits > 6:
@@ -1673,6 +1917,8 @@ def _gen_layer_config(
layer_scheme = options[item[1]]
for layer_name in layer_names:
layer_config[layer_name] = asdict(layer_scheme)
+ _log_scheme_loss_matrix(total_scores, options, block_name, model=model, layer_numel=layer_numel)
+ _describe_layer_config(layer_config, total_scores, options, block_name, model=model)
if model_name is not None:
model = None
del model
@@ -1722,6 +1968,11 @@ def gen_layer_config(
processor=None,
**kwargs,
):
+ """Public AutoScheme entry.
+
+ This wrapper performs model loading/dispatch and environment preparation,
+ then delegates to `_gen_layer_config` for staged scoring + DP selection.
+ """
model_name = None
is_vlm = False
if isinstance(model, str):
@@ -1747,6 +1998,9 @@ def gen_layer_config(
except Exception: # noqa: BLE001
is_vlm = False
+ from auto_round.modeling.fused_moe.replace_modules import materialize_model_
+
+ materialize_model_(model)
# ---- Vision-tower scoring requires a full backward ---- #
# ``model_forward_low_gpu`` only walks the language tower (it uses
# ``get_block_names(model)[0]``, which excludes vision blocks by default)
@@ -1807,6 +2061,9 @@ def gen_layer_config(
# ``use_reentrant=False`` (saved-tensor-hooks impl) does not have this
# restriction.
def _enable_gc(mod):
+ """Enable gradient checkpointing on ``mod`` with ``use_reentrant=False`` if supported
+ (see rationale above); no-op if the module doesn't support checkpointing.
+ """
if not getattr(mod, "supports_gradient_checkpointing", False):
return
try:
diff --git a/auto_round/auto_scheme/utils.py b/auto_round/auto_scheme/utils.py
index 8a3ae951d..83e74fe57 100644
--- a/auto_round/auto_scheme/utils.py
+++ b/auto_round/auto_scheme/utils.py
@@ -11,27 +11,34 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
+import logging
+import math
import re
from dataclasses import asdict, fields
-from typing import Iterable, Union
+from typing import Iterable, Optional, Union
import torch
from accelerate import dispatch_model, infer_auto_device_map
from accelerate.utils import get_balanced_memory
-from auto_round.schemes import QuantizationScheme, preset_name_to_scheme
+from auto_round.logger import logger
+from auto_round.schemes import QuantizationScheme, preset_name_to_scheme, scheme_to_preset_name
from auto_round.utils import (
DEVICE_ENVIRON_VARIABLE_MAPPING,
SUPPORTED_LAYER_TYPES,
check_to_quantized,
get_block_names,
get_layer_features,
+ get_lm_head_name,
get_module,
is_hpex_available,
normalize_no_split_modules,
parse_available_devices,
)
+_EXPERT_ID_PATTERN = re.compile(r"^(.*?\.experts\.\d+)(?:\.|$)")
+_ZERO_EPS = 1e-12
+
def apply_quant_scheme(
model: torch.nn.Module,
@@ -223,15 +230,22 @@ def compute_layer_bits(
def merge_lists_unionfind(list_of_lists):
+ """Merge lists that share at least one common element using union-find, returning the
+ resulting disjoint groups (each input list's elements are unioned together).
+ """
parent = {}
def find(x):
+ """Return the representative (root) element of ``x``'s union-find set, with path
+ compression.
+ """
while parent[x] != x:
parent[x] = parent[parent[x]]
x = parent[x]
return x
def union(x, y):
+ """Merge the sets containing ``x`` and ``y``."""
root_x, root_y = find(x), find(y)
if root_x != root_y:
parent[root_y] = root_x
@@ -325,3 +339,558 @@ def parse_shared_layers(model: torch.nn.Module, shared_patterns: Iterable[Iterab
matched_groups.append(sorted(matched_layers))
matched_groups = merge_lists_unionfind(matched_groups)
return matched_groups
+
+
+def _expert_key_from_layer_name(layer_name: str) -> Optional[str]:
+ """Map one MoE-related linear layer to a unique expert key.
+
+ Gate/up/down projections belonging to the same expert should map to one key.
+ """
+ match = _EXPERT_ID_PATTERN.search(layer_name)
+ if match:
+ return match.group(1)
+
+ if ".moe." in layer_name:
+ # Fallback for models that expose MoE expert layers without
+ # `.experts.<id>` naming. Best-effort collapse by dropping the last
+ # projection suffix (gate/up/down/etc.).
+ parts = layer_name.split(".")
+ if len(parts) > 1:
+ return ".".join(parts[:-1])
+ return layer_name
+
+ return None
+
+
+def _short_summary_name(layer_name: str) -> str:
+ """Shorten ``layer_name`` to its last two dotted segments when the final segment is a
+ numeric index (e.g. expert id), otherwise return it unchanged.
+ """
+ parts = layer_name.rsplit(".", 2)
+ if len(parts) >= 2 and parts[-1].isdigit():
+ return ".".join(parts[-2:])
+ return layer_name
+
+
+def _scheme_short_name(scheme) -> str:
+ """Concise human-readable label for a quantization scheme, e.g. 'MXFP4' or 'W4A16'."""
+ if isinstance(scheme, str):
+ return scheme
+ if isinstance(scheme, QuantizationScheme):
+ preset_name = scheme_to_preset_name(scheme)
+ if preset_name:
+ return preset_name
+ scheme = asdict(scheme)
+ if isinstance(scheme, dict):
+ try:
+ preset_name = scheme_to_preset_name(QuantizationScheme.from_dict(scheme))
+ if preset_name:
+ return preset_name
+ except Exception: # noqa: BLE001
+ pass
+ data_type = str(scheme.get("data_type", "")).lower()
+ bits = scheme.get("bits", 16)
+ act_bits = scheme.get("act_bits", 16)
+ if data_type.startswith("mx_fp"):
+ return f"MXFP{bits}"
+ if data_type.startswith("nv_fp"):
+ return f"NVFP{bits}"
+ if data_type.startswith("gguf"):
+ return data_type.upper()
+ if act_bits >= 16:
+ return f"W{bits}"
+ return f"W{bits}A{act_bits}"
+ return str(scheme)
+
+
+def build_expert_groups(
+ model: torch.nn.Module,
+ quant_layer_names: list[str],
+ fixed_layer_scheme: dict[str, dict],
+) -> list[list[str]]:
+ """Auto-detect MoE expert layers and group all experts in the same block together.
+
+ All expert projection layers (gate/up/down etc.) across all experts within a single
+ transformer block are grouped into one list, so that DP treats them as a single entity
+ with summed loss and summed numel.
+
+ Returns a list of groups, each group is a list of layer names.
+ """
+ block_names = get_block_names(model, quant_vision=False)
+ block_names = [item for sublist in block_names for item in sublist]
+ block_prefixes = [(name, name + ".") for name in sorted(block_names, key=len, reverse=True)]
+
+ dp_names = set(quant_layer_names) - set(fixed_layer_scheme.keys())
+ # Collect expert layers per block
+ block_expert_layers: dict[str, list[str]] = {}
+ for layer_name in dp_names:
+ if _expert_key_from_layer_name(layer_name) is None:
+ continue
+ for block_name, block_prefix in block_prefixes:
+ if layer_name.startswith(block_prefix):
+ block_expert_layers.setdefault(block_name, []).append(layer_name)
+ break
+
+ groups = []
+ for block_name in block_names:
+ layers = block_expert_layers.get(block_name)
+ if layers and len(layers) > 1:
+ groups.append(sorted(layers))
+ return groups
+
+
+def _fill_inactive_expert_scores(scores_dict: dict[str, list[float]], block_names: list[str]):
+ """Fill inactive experts with the average loss of active experts in each block.
+
+ Inactive expert means all its tracked projection losses are zero.
+ """
+ block_prefixes = [(name, name + ".") for name in sorted(block_names, key=len, reverse=True)]
+ block_expert_stats: dict[str, dict[str, dict[str, object]]] = {}
+
+ for layer_name, values in scores_dict.items():
+ layer_loss = float(values[1])
+ if not math.isfinite(layer_loss):
+ continue
+
+ matched_block = None
+ for block_name, block_prefix in block_prefixes:
+ if layer_name == block_name or layer_name.startswith(block_prefix):
+ matched_block = block_name
+ break
+ if matched_block is None:
+ continue
+
+ expert_key = _expert_key_from_layer_name(layer_name)
+ if expert_key is None:
+ continue
+
+ expert_stats = block_expert_stats.setdefault(matched_block, {}).setdefault(
+ expert_key,
+ {"layers": [], "has_active": False},
+ )
+ expert_stats["layers"].append(layer_name)
+
+ if abs(layer_loss) >= _ZERO_EPS:
+ expert_stats["has_active"] = True
+
+ for block_name, expert_stats_map in block_expert_stats.items():
+ active_expert_avg_losses = []
+ for expert_stats in expert_stats_map.values():
+ if not expert_stats["has_active"]:
+ continue
+ active_losses = [
+ float(scores_dict[layer_name][1])
+ for layer_name in expert_stats["layers"]
+ if abs(float(scores_dict[layer_name][1])) >= _ZERO_EPS
+ ]
+ if not active_losses:
+ continue
+ active_expert_avg_losses.append(sum(active_losses) / len(active_losses))
+ if not active_expert_avg_losses:
+ continue
+ fill_value = sum(active_expert_avg_losses) / len(active_expert_avg_losses)
+ for _, expert_stats in expert_stats_map.items():
+ if expert_stats["has_active"]:
+ continue
+ for layer_name in expert_stats["layers"]:
+ scores_dict[layer_name][1] = fill_value
+
+
+def _log_score_summary_by_block_and_nonblock(
+ scores_dict: dict[str, list[float]],
+ block_names: list[str],
+ model=None,
+ scheme_tag: Optional[str] = None,
+ summary_stage: Optional[str] = None,
+):
+ """Log a per-block (and non-block) breakdown of ``scores_dict`` losses at debug level."""
+ if not scores_dict:
+ logger.info("AutoScheme score summary: empty.")
+ return
+
+ head_name = get_lm_head_name(model) if model is not None else None
+ if head_name is not None and head_name.endswith(".orig_layer"):
+ head_name = head_name[: -len(".orig_layer")]
+ if head_name is None and "lm_head" in scores_dict:
+ head_name = "lm_head"
+
+ block_prefixes = [(name, name + ".") for name in sorted(block_names, key=len, reverse=True)]
+ block_stats: dict[str, list[float]] = {name: [0.0, 0.0] for name in block_names}
+ non_block_items: list[tuple[str, float]] = []
+
+ for layer_name, values in scores_dict.items():
+ layer_loss = float(values[1])
+ if not math.isfinite(layer_loss):
+ continue
+ matched_block = None
+ for block_name, block_prefix in block_prefixes:
+ if layer_name == block_name or layer_name.startswith(block_prefix):
+ matched_block = block_name
+ break
+ if matched_block is None:
+ if layer_name != head_name:
+ non_block_items.append((layer_name, layer_loss))
+ else:
+ block_stats[matched_block][0] += layer_loss
+ block_stats[matched_block][1] += 1
+
+ tag = f"[{scheme_tag}] " if scheme_tag else ""
+ stage_str = f" ({summary_stage})" if summary_stage else ""
+ logger.debug("AutoScheme %sblock loss summary%s:", tag, stage_str)
+ logger.debug("AutoScheme | block | avg_loss |")
+
+ for block_name in block_names:
+ total_loss, cnt = block_stats.get(block_name, [0.0, 0.0])
+ avg_loss = 0.0 if cnt <= 0 else total_loss / cnt
+ logger.debug("AutoScheme | %s | %.6f |", _short_summary_name(block_name), avg_loss)
+
+ if head_name is not None:
+ head_loss = None
+ if head_name in scores_dict:
+ head_loss = float(scores_dict[head_name][1])
+ head_avg = "N/A" if head_loss is None or not math.isfinite(head_loss) else f"{head_loss:.6f}"
+ logger.debug("AutoScheme | %s | %s |", head_name, head_avg)
+
+ if non_block_items:
+ non_block_items.sort(key=lambda x: x[0])
+ for layer_name, layer_loss in non_block_items:
+ logger.info("AutoScheme non_block=%s loss=%.6f", layer_name, layer_loss)
+ else:
+ logger.info("AutoScheme non_block loss: none")
+
+
+def _collect_current_scores(model):
+ """Snapshot the current ``mix_score`` of every wrapped module into a
+ ``{name: [0.0, loss]}`` dict.
+ """
+ scores_dict = {}
+ for name, module in model.named_modules():
+ if not hasattr(module, "mix_score"):
+ continue
+ loss = float(module.mix_score)
+ if not math.isfinite(loss):
+ continue
+ scores_dict[name] = [0.0, loss]
+ return scores_dict
+
+
+_BATCH_SUMMARY_LOG_INTERVAL = 10
+
+
+def _log_batch_avg_loss(model, batch_idx: int, pbar=None, block_names=None, total_batches=None, scheme_tag=None):
+ """Log the running average ``mix_score`` after processing one calibration batch."""
+ is_last_batch = total_batches is not None and batch_idx == total_batches
+ should_log = logger.isEnabledFor(logging.DEBUG) and (is_last_batch or batch_idx % _BATCH_SUMMARY_LOG_INTERVAL == 0)
+ if not should_log:
+ return
+
+ total_loss = 0.0
+ layer_cnt = 0
+ for _, module in model.named_modules():
+ if not hasattr(module, "mix_score"):
+ continue
+ loss = float(module.mix_score)
+ if not math.isfinite(loss):
+ continue
+ total_loss += loss
+ layer_cnt += 1
+
+ avg_loss = 0.0 if layer_cnt == 0 else total_loss / layer_cnt
+ tag = f"[{scheme_tag}] " if scheme_tag else ""
+ batch_str = f"{batch_idx}/{total_batches}" if total_batches is not None else str(batch_idx)
+ msg = f"AutoScheme {tag}cumulative batch {batch_str} avg_loss={avg_loss:.6f} layers={layer_cnt}"
+ if pbar is not None:
+ pbar.write(msg)
+ logger.debug(msg)
+
+ if block_names:
+ scores_dict = _collect_current_scores(model)
+ if scores_dict:
+ tag = f"[{scheme_tag}] " if scheme_tag else ""
+ batch_str = f"{batch_idx}/{total_batches}" if total_batches is not None else str(batch_idx)
+ if is_last_batch:
+ logger.debug("AutoScheme %scumulative batch %s block summary skipped (same as final)", tag, batch_str)
+ else:
+ logger.debug("AutoScheme %scumulative batch %s block summary:", tag, batch_str)
+ _log_score_summary_by_block_and_nonblock(
+ scores_dict,
+ block_names,
+ model=model,
+ scheme_tag=scheme_tag,
+ summary_stage="cumulative",
+ )
+
+
+def _build_layer_config_header_rows(columns: list[str]) -> list[list[str]]:
+ """Build a compact two-row header for the layer-config matrix.
+
+ The first row keeps a shared prefix for each grouped set of columns (for
+ example ``mlp`` or ``self_attn``), while the second row keeps the leaf
+ suffix (for example ``down_proj``).
+ """
+ if not columns:
+ return [["block"], [""]]
+
+ leaves = []
+ first_row = []
+ prev_prefix = None
+ for column in columns:
+ parts = column.split(".")
+ if len(parts) > 1:
+ prefix = ".".join(parts[:-1])
+ leaves.append(parts[-1])
+ if prefix == prev_prefix:
+ first_row.append("")
+ else:
+ first_row.append(prefix)
+ prev_prefix = prefix
+ else:
+ leaves.append(column)
+ first_row.append("")
+ prev_prefix = None
+
+ header_rows = [["block"] + first_row]
+ header_rows.append([""] + leaves)
+ return header_rows
+
+
+def _log_scheme_loss_matrix(total_scores, options, block_names, model=None, layer_numel=None):
+ """For every scheme in *options* log a block×layer matrix of per-element losses.
+
+ Cells show ``loss / weight.numel()`` in scientific notation with three
+ decimal places (e.g. ``1.234E-05``). Expert layers are aggregated into an
+ ``experts`` column that shows the average per-element loss across all active
+ expert layers in each block.
+ """
+ if not total_scores or not options:
+ return
+
+ head_name = get_lm_head_name(model) if model is not None else None
+ if head_name is not None and head_name.endswith(".orig_layer"):
+ head_name = head_name[: -len(".orig_layer")]
+ block_prefixes = [(name, name + ".") for name in sorted(block_names, key=len, reverse=True)]
+
+ # Classify every key in total_scores into a (block, short_name) pair or expert bucket.
+ # lm_head is treated as a regular non-block row (not skipped).
+ block_col_keys: dict[str, dict[str, str]] = {name: {} for name in block_names}
+ block_expert_keys: dict[str, list[str]] = {name: [] for name in block_names}
+ other_keys: list[str] = [] # keys not belonging to any block (e.g. lm_head)
+ for key in sorted(total_scores):
+ matched_block = None
+ for block_name, block_prefix in block_prefixes:
+ if key == block_name or key.startswith(block_prefix):
+ matched_block = block_name
+ break
+ if matched_block is None:
+ other_keys.append(key)
+ continue
+ if _expert_key_from_layer_name(key) is not None:
+ block_expert_keys[matched_block].append(key)
+ else:
+ short_name = key[len(matched_block) + 1 :] if key.startswith(matched_block + ".") else key
+ block_col_keys[matched_block][short_name] = key
+
+ columns = sorted({col for row in block_col_keys.values() for col in row.keys()})
+ if not columns:
+ columns = ["layer"]
+ has_expert_layers = any(block_expert_keys.get(b) for b in block_names)
+
+ block_display_names = [_short_summary_name(b) for b in block_names]
+ header_rows = _build_layer_config_header_rows(columns)
+
+ # Fixed cell width for scientific-notation values: "1.234E-05" = 9 chars.
+ _CELL_W = 9
+ widths: dict[str, int] = {"block": max(8, max((len(b) for b in block_display_names), default=5))}
+ for col in columns:
+ leaf = col.split(".")[-1]
+ widths[col] = max(len(leaf), _CELL_W)
+ if has_expert_layers:
+ widths["experts"] = max(len("experts"), _CELL_W)
+ # Ensure the first column of each prefix group is wide enough for the prefix text in header row 1.
+ for col, prefix in zip(columns, header_rows[0][1:]):
+ if prefix:
+ widths[col] = max(widths[col], len(prefix))
+
+ header_keys = ["block"] + columns + (["experts"] if has_expert_layers else [])
+ header_vals = [header_rows[0][0]] + header_rows[0][1:]
+ header_vals_row2 = ([""]) + (header_rows[1][1:] if len(header_rows) > 1 else [""] * len(columns))
+ if has_expert_layers:
+ header_vals.append("experts")
+ header_vals_row2.append("")
+
+ def _fmt_row(values: list[str], keys: list[str]) -> str:
+ return "|".join(v.ljust(widths[k]) for v, k in zip(values, keys))
+
+ sep = "|".join("-" * widths[k] for k in header_keys)
+
+ for scheme_idx, scheme in enumerate(options):
+ scheme_name = _scheme_short_name(scheme)
+ block_loss_cells: dict[str, dict[str, str]] = {name: {} for name in block_names}
+ block_expert_avg: dict[str, str] = {}
+
+ _numel = layer_numel or {}
+ for block_name in block_names:
+ for short_name, key in block_col_keys[block_name].items():
+ loss = next((item[2] for item in total_scores.get(key, []) if item[0] == scheme_idx), None)
+ if loss is not None:
+ n = _numel.get(key, 0)
+ block_loss_cells[block_name][short_name] = f"{loss / n:.3E}" if n > 0 else f"{loss:.3E}"
+ else:
+ block_loss_cells[block_name][short_name] = "-"
+
+ expert_losses_per_elem: list[float] = []
+ for ek in block_expert_keys.get(block_name, []):
+ el = next((item[2] for item in total_scores.get(ek, []) if item[0] == scheme_idx), None)
+ if el is not None:
+ n = _numel.get(ek, 0)
+ expert_losses_per_elem.append(el / n if n > 0 else el)
+ if expert_losses_per_elem:
+ block_expert_avg[block_name] = f"{sum(expert_losses_per_elem) / len(expert_losses_per_elem):.3E}"
+ else:
+ block_expert_avg[block_name] = "-"
+
+ logger.debug("AutoScheme [%s] per-op loss/elem matrix:", scheme_name)
+ logger.debug(" %s", _fmt_row(header_vals, header_keys))
+ logger.debug(" %s", _fmt_row(header_vals_row2, header_keys))
+ logger.debug(" %s", sep)
+ for block_name, block_display_name in zip(block_names, block_display_names):
+ row_vals = [block_display_name]
+ for col in columns:
+ row_vals.append(block_loss_cells.get(block_name, {}).get(col, "-"))
+ if has_expert_layers:
+ row_vals.append(block_expert_avg.get(block_name, "-"))
+ logger.debug(" %s", _fmt_row(row_vals, header_keys))
+ if other_keys:
+ for key in other_keys:
+ loss = next((item[2] for item in total_scores.get(key, []) if item[0] == scheme_idx), None)
+ if loss is not None:
+ n = _numel.get(key, 0)
+ loss_str = f"{loss / n:.3E}" if n > 0 else f"{loss:.3E}"
+ else:
+ loss_str = "-"
+ logger.debug(" %s -> %s", _short_summary_name(key), loss_str)
+
+
+def _describe_layer_config(layer_config, total_scores, options, block_names, model=None):
+ """Log final ``layer_config`` as a block-row / layer-column matrix.
+
+ Cells show the selected scheme name only; the detailed loss delta is no
+ longer printed to keep the table compact.
+ """
+
+ if not layer_config:
+ logger.info("AutoScheme final layer_config: empty.")
+ return
+
+ head_name = get_lm_head_name(model) if model is not None else None
+ if head_name is not None and head_name.endswith(".orig_layer"):
+ head_name = head_name[: -len(".orig_layer")]
+ block_prefixes = [(name, name + ".") for name in sorted(block_names, key=len, reverse=True)]
+
+ block_cells: dict[str, dict[str, str]] = {name: {} for name in block_names}
+ block_expert_scheme_counts: dict[str, dict[str, int]] = {name: {} for name in block_names}
+ block_expert_layers: dict[str, dict[str, list[str]]] = {name: {} for name in block_names}
+ other_rows: list[tuple[str, str]] = []
+
+ for layer_name in sorted(layer_config):
+ cell = _scheme_short_name(layer_config[layer_name])
+
+ if head_name is not None and layer_name == head_name:
+ continue
+
+ matched_block = None
+ for block_name, block_prefix in block_prefixes:
+ if layer_name == block_name or layer_name.startswith(block_prefix):
+ matched_block = block_name
+ break
+
+ expert_key = _expert_key_from_layer_name(layer_name)
+ if matched_block is None:
+ other_rows.append((layer_name, cell))
+ continue
+
+ if expert_key is not None:
+ block_expert_layers[matched_block].setdefault(expert_key, []).append(layer_name)
+ continue
+
+ short_name = layer_name[len(matched_block) + 1 :] if layer_name.startswith(matched_block + ".") else layer_name
+ block_cells[matched_block][short_name] = cell
+
+ for block_name, expert_map in block_expert_layers.items():
+ if not expert_map:
+ continue
+ counts: dict[str, int] = {}
+ for layer_names in expert_map.values():
+ scheme_set = {_scheme_short_name(layer_config[layer_name]) for layer_name in layer_names}
+ rep_scheme = (
+ next(iter(sorted(scheme_set))) if len(scheme_set) == 1 else f"MIXED({', '.join(sorted(scheme_set))})"
+ )
+ counts[rep_scheme] = counts.get(rep_scheme, 0) + 1
+ block_expert_scheme_counts[block_name] = counts
+
+ columns = sorted({col for row in block_cells.values() for col in row.keys()})
+ if not columns:
+ columns = ["layer"]
+
+ expert_text_by_block: dict[str, str] = {}
+ for block_name in block_names:
+ expert_counts = block_expert_scheme_counts.get(block_name, {})
+ if not expert_counts:
+ expert_text_by_block[block_name] = "-"
+ continue
+ expert_total = len(block_expert_layers.get(block_name, {}))
+ expert_text_by_block[block_name] = ", ".join(
+ f"{count}/{expert_total} {scheme_name}"
+ for scheme_name, count in sorted(expert_counts.items(), key=lambda item: (-item[1], item[0]))
+ )
+
+ has_expert_layers = any(block_expert_layers.get(block_name) for block_name in block_names)
+
+ block_display_names = [_short_summary_name(block_name) for block_name in block_names]
+ header_rows = _build_layer_config_header_rows(columns)
+
+ # Width is driven by the leaf name and cell content — NOT the full dotted column name,
+ # so columns stay tight (e.g. "down_proj" width, not "mlp.down_proj" width).
+ widths: dict[str, int] = {"block": max(8, max((len(b) for b in block_display_names), default=5))}
+ for col in columns:
+ leaf = col.split(".")[-1]
+ max_cell_len = max((len(block_cells.get(b, {}).get(col, "-")) for b in block_names), default=1)
+ widths[col] = max(len(leaf), max_cell_len)
+ if has_expert_layers:
+ widths["experts"] = max(len("experts"), max((len(v) for v in expert_text_by_block.values()), default=1))
+ # Ensure the first column of each prefix group is wide enough for the prefix text in header row 1.
+ for col, prefix in zip(columns, header_rows[0][1:]):
+ if prefix:
+ widths[col] = max(widths[col], len(prefix))
+
+ def _fmt_row(values: list[str], keys: list[str]) -> str:
+ return "|".join(v.ljust(widths[k]) for v, k in zip(values, keys))
+
+ header_keys = ["block"] + columns + (["experts"] if has_expert_layers else [])
+ header_vals = [header_rows[0][0]] + header_rows[0][1:]
+ if len(header_rows) > 1:
+ header_vals_row2 = [header_rows[1][0]] + header_rows[1][1:]
+ else:
+ header_vals_row2 = [""] + [""] * len(columns)
+
+ logger.info("AutoScheme final layer_config matrix:")
+ logger.info("AutoScheme note: cell=`scheme`.")
+ logger.info(" %s", _fmt_row(header_vals, header_keys))
+ logger.info(" %s", _fmt_row(header_vals_row2, header_keys))
+ logger.info(" %s", "|".join("-" * widths[k] for k in header_keys))
+
+ for block_name, block_display_name in zip(block_names, block_display_names):
+ row_vals = [block_display_name]
+ for col in columns:
+ row_vals.append(block_cells.get(block_name, {}).get(col, "-"))
+ if has_expert_layers:
+ row_vals.append(expert_text_by_block.get(block_name, "-"))
+ logger.info(" %s", _fmt_row(row_vals, header_keys))
+
+ if head_name is not None and head_name in layer_config:
+ logger.info(" %s is using %s", head_name, _scheme_short_name(layer_config[head_name]))
+
+ if other_rows:
+ logger.info(" other:")
+ for layer_name, cell in other_rows:
+ logger.info(" %s -> %s", layer_name, cell)
diff --git a/auto_round/envs.py b/auto_round/envs.py
index a6df79435..714eaacc6 100644
--- a/auto_round/envs.py
+++ b/auto_round/envs.py
@@ -24,6 +24,8 @@ if TYPE_CHECKING:
AUTO_ROUND_CACHE: Optional[str] = None
AUTO_ROUND_GGUF_AUTO_UPDATE: bool = False
LLAMA_CPP_ROOT: Optional[str] = None
+ AR_AUTO_SCHEME_NSAMPLES: Optional[int] = None
+ AR_AUTO_SCHEME_BATCH_SIZE: Optional[int] = None
def _get_optional_positive_int_env(name: str) -> Optional[int]:
@@ -75,6 +77,14 @@ environment_variables: dict[str, Callable[[], Any]] = {
"AUTO_ROUND_GGUF_AUTO_UPDATE": lambda: os.getenv("AUTO_ROUND_GGUF_AUTO_UPDATE", "0").lower()
in ("1", "true", "yes", "on"),
"LLAMA_CPP_ROOT": lambda: os.getenv("LLAMA_CPP_ROOT", None),
+ # Controls the default number of calibration samples used by AutoScheme scoring
+ # when ``AutoScheme.nsamples`` is not explicitly set.
+ # When unset, AutoScheme uses 16.
+ "AR_AUTO_SCHEME_NSAMPLES": lambda: _get_optional_positive_int_env("AR_AUTO_SCHEME_NSAMPLES"),
+ # Controls the default batch size used by AutoScheme scoring
+ # when ``AutoScheme.batch_size`` is not explicitly set.
+ # When unset, AutoScheme uses its built-in heuristic (8 for low GPU memory mode, 1 for normal mode).
+ "AR_AUTO_SCHEME_BATCH_SIZE": lambda: _get_optional_positive_int_env("AR_AUTO_SCHEME_BATCH_SIZE"),
}
diff --git a/auto_round/utils/model.py b/auto_round/utils/model.py
index 1cd28df7a..120055f99 100644
--- a/auto_round/utils/model.py
+++ b/auto_round/utils/model.py
@@ -1652,10 +1652,13 @@ def safe_device_move_with_meta_handling(
def is_moe_model(model: torch.nn.Module) -> bool:
+ """Heuristically detect whether ``model`` is a MoE architecture by scanning its config
+ for "moe"/"expert" markers or by checking module names for "expert".
+ """
if hasattr(model, "config") and hasattr(model.config, "to_dict"):
- for key in model.config.to_dict().keys():
- if "moe" in key or "expert" in key:
- return True
+ config_str = str(model.config.to_dict()).lower()
+ if "moe" in config_str or "expert" in config_str:
+ return True
for n, m in model.named_modules():
if "expert" in n:
return True
diff --git a/docs/auto_scheme_acc.md b/docs/auto_scheme_acc.md
index ad404c313..8b6981831 100644
--- a/docs/auto_scheme_acc.md
+++ b/docs/auto_scheme_acc.md
@@ -3,42 +3,31 @@
We tested with a fake model because the main branch currently has layer name mismatches between Transformers and GGUF.
~~~bash
-python3 -m auto_round Qwen/Qwen3-8B --options "gguf:q2_k_s,gguf:q4_k_s" --target_bits 3.5 --ignore_scale_zp_bits --iters 0 --format fake --output_dir "./test_gguf" --tasks mmlu,lambada_openai
+python3 -m auto_round Qwen/Qwen3-8B --options "gguf:q2_k_s,gguf:q4_k_s" --target_bits 3.5 --ignore_scale_zp_bits --iters 0 --format fake --output_dir "./test_gguf"
~~~
+
eval
~~~
- vllm serve ./test_gguf/Qwen3-8B-w2g16/ --port 8000 --max-model-len 8192 --host 127.0.0.1 --served-model-name qwen
-evalscope eval --model qwen --api-url http://127.0.0.1:8000/v1 --api-key EMPTY --datasets math_500 gpqa_diamoid mmlu_pro --eval-batch-size 32
-~~~
-
-| evalscope options q2ks,q4ks avgbits 3.5, ignore_scale_zp | math_500 | gpqa_diamond | mmlu_pro |
-|----------------------------------------------------------|------------|---------------|----------|
-| qwen3-8b bf16 | 0.79 | 0.4242 | 0.6934 |
-| qwen3-8b (AR<=0.13.1) iters 0 | 0.76/0.768 | 0.3434/0.3687 | 0.6419 |
-| qwen3-8b (AR>0.13.1) iters 0 | 0.782 | 0.4293 | 0.6763 |
-
-| evalscope, options q2ks,q4ks avgbits 3.5, ignore_scale_zp | math_500 | gpqa_diamond | mmlu_pro |
-|-----------------------------------------------------------|----------|--------------|----------|
-| qwen3.5-4b bf16 | 0.48 | 0.4192 | 0.5891 |
-| qwen3.5-4b (AR<=0.13.1) iters 0 | 0.228 | 0.298 | 0.4583 |
-| pr branch (AR>0.13.1) iters 0 | 0.432 | 0.4444 | 0.5922 |
-
-[//]: # (| options W2A16G64,W4A16 avgbits 3.5 ignore_scale_zp | math_500 | gpqa_diamond | mmlu_pro |)
-
-[//]: # (| -------------------------------------------------- | -------- | ------------ | -------- |)
+# Start vLLM serve
+vllm serve ./test_gguf/Qwen3-8B-w2g16/ --port 8000 --max-model-len 8192 --host 127.0.0.1 --served-model-name qwen3-8b
-[//]: # (| qwen3-8b bf16 | 0.79 | 0.4242 | 0.6934 |)
-
-[//]: # (| qwen3-8b main branch | 0.738 | 0.2778 | 0.576 |)
-
-[//]: # (| main branch iters 200 enable_alg_ext | 0.766 | 0.3434 | |)
-
-[//]: # (| pr branch | 0.72 | 0.2677 | - |)
-
-[//]: # (| pr branch + iters 200 enable_alg_ext | 0.742 | 0.3232 | - |)
+# Perform five repeated evaluations on math_500 and gpqa_diamond.
+evalscope eval --model qwen3-8b --api-url http://127.0.0.1:8000/v1 --api-key EMPTY --datasets math_500 gpqa_diamond --eval-batch-size 64 --generation-config "{\"n\":5}"
+# Evaluate mmlu_pro
+evalscope eval --model qwen3-8b --api-url http://127.0.0.1:8001/v1 --api-key EMPTY --datasets mmlu_pro --eval-batch-size 128
+~~~
+| evalscope, options q2ks,q4ks avgbits 3.5, ignore_scale_zp | math_500 (repeat=5) | gpqa_diamond (repeat=5) | mmlu_pro |
+|-----------------------------------------------------------|---------------------|-------------------------|----------|
+| qwen3-8b: BF16 | 0.8083 | 0.4586 | 0.6934 |
+| qwen3-8b: Fake quantized model | 0.7924 | 0.4313 | 0.6751 |
+
+| evalscope, options q2ks,q4ks avgbits 3.5, ignore_scale_zp | math_500 (repeat=5) | gpqa_diamond (repeat=5) | mmlu_pro |
+|-----------------------------------------------------------|---------------------|-------------------------|----------|
+| qwen3.5-4b: BF16 | 0.5365 | 0.3263 | 0.5891 |
+| qwen3.5-4b: Fake quantized model | 0.505 | 0.3172 | 0.5948 |
### Other results
diff --git a/docs/environments.md b/docs/environments.md
index 89d9bf100..296bf6c28 100644
--- a/docs/environments.md
+++ b/docs/environments.md
@@ -121,6 +121,26 @@ export AR_DYNAMO_CACHE_SIZE_LIMIT=32
export AR_MODEL_FREE_SHARD_PARALLELISM=4
```
+### AR_AUTO_SCHEME_NSAMPLES
+- **Description**: Controls the default number of calibration samples used by AutoScheme scoring when `AutoScheme.nsamples` is not explicitly set.
+- **Default**: unset → 16
+- **Valid Values**: any positive integer, e.g. `8`, `16`, `32`
+- **Usage**: Set this to override the automatic sample-count selection for AutoScheme
+
+```bash
+export AR_AUTO_SCHEME_NSAMPLES=1 # set 1 for quick execution
+```
+
+### AR_AUTO_SCHEME_BATCH_SIZE
+- **Description**: Controls the default batch size used by AutoScheme scoring when `AutoScheme.batch_size` is not explicitly set.
+- **Default**: unset → built-in heuristic applies (8 for low GPU memory mode, 1 for normal mode)
+- **Valid Values**: any positive integer, e.g. `1`, `2`, `4`
+- **Usage**: Set this to override the default batch size for AutoScheme
+
+```bash
+export AR_AUTO_SCHEME_BATCH_SIZE=1
+```
+
## Usage Examples
### Setting Environment Variables
diff --git a/docs/environments_CN.md b/docs/environments_CN.md
index 606f6b1f3..c04c45568 100644
--- a/docs/environments_CN.md
+++ b/docs/environments_CN.md
@@ -121,6 +121,26 @@ export AR_DYNAMO_CACHE_SIZE_LIMIT=32
export AR_MODEL_FREE_SHARD_PARALLELISM=4
```
+### AR_AUTO_SCHEME_NSAMPLES
+- **描述**:控制 AutoScheme 评分时使用的校准样本数默认值,仅在 `AutoScheme.nsamples` 未显式设置时生效。
+- **默认值**:未设置 → 16
+- **有效值**:任意正整数,如 `8`、`16`、`32`
+- **用途**:覆盖 AutoScheme 的自动样本数选择
+
+```bash
+export AR_AUTO_SCHEME_NSAMPLES=1
+```
+
+### AR_AUTO_SCHEME_BATCH_SIZE
+- **描述**:控制 AutoScheme 评分时使用的批大小默认值,仅在 `AutoScheme.batch_size` 未显式设置时生效。
+- **默认值**:未设置 → 走内置启发式规则(低GPU内存模式为 8,普通模式为 1)
+- **有效值**:任意正整数,如 `1`、`2`、`4`
+- **用途**:覆盖 AutoScheme 的默认批大小
+
+```bash
+export AR_AUTO_SCHEME_BATCH_SIZE=1
+```
+
## 使用示例
### 设置环境变量
diff --git a/docs/step_by_step.md b/docs/step_by_step.md
index d1eb3d013..0091b4af9 100644
--- a/docs/step_by_step.md
+++ b/docs/step_by_step.md
@@ -422,6 +422,8 @@ ar.quantize_and_save()
In some serving frameworks, certain layers (e.g., QKV or MoE) are fused to accelerate inference. These fused layers may require the same data type and bit configuration. The shared_layers option simplifies this setup by supporting both regex and full-name matching. **Note that regex matching is applied in a block-wise manner.**
+**MoE expert layers are automatically grouped per block** — all expert projections (gate/up/down across all experts) within the same transformer block are treated as a single entity during DP optimization. This means they share the same quantization scheme and their losses/numel are summed together. No manual `shared_layers` configuration is needed for expert layers.
+
```python
from auto_round import AutoRound, AutoScheme
diff --git a/docs/step_by_step_CN.md b/docs/step_by_step_CN.md
index 057f68563..131324697 100644
--- a/docs/step_by_step_CN.md
+++ b/docs/step_by_step_CN.md
@@ -421,6 +421,8 @@ ar.quantize_and_save()
为加速推理,在部分框架中,会对特定层(如QKV、MoE)进行融合。这些融合层必须是相同的数据类型和量化配置。`shared_layers` 参数可以简化该配置,**同时支持正则表达式匹配和完整层名匹配**。注意**正则匹配按块匹配规则生效**。
+**MoE 的 expert 层会自动按块分组** — 同一个 transformer block 内所有 expert 的投影层(gate/up/down,跨所有 experts)会被视为一个整体进行 DP 优化。它们共享相同的量化方案,loss 和 numel 直接求和。无需手动配置 `shared_layers` 来处理 expert 层。
+
示例代码如下:
```python
from auto_round import AutoRound, AutoScheme
diff --git a/test/test_cpu/schemes/test_auto_scheme_internal.py b/test/test_cpu/schemes/test_auto_scheme_internal.py
index 35a6e69c5..d5080fa86 100644
--- a/test/test_cpu/schemes/test_auto_scheme_internal.py
+++ b/test/test_cpu/schemes/test_auto_scheme_internal.py
@@ -1,55 +1,116 @@
-from auto_round.auto_scheme.delta_loss import _apply_head_trick
+import os
+
from auto_round.auto_scheme.gen_auto_scheme import AutoScheme
+from auto_round.auto_scheme.utils import _build_layer_config_header_rows, _short_summary_name
+
+
+def test_env_ar_auto_scheme_nsamples_overrides_default(monkeypatch):
+ """AR_AUTO_SCHEME_NSAMPLES env var should override the built-in nsamples heuristic."""
+ import auto_round.envs as envs
+
+ monkeypatch.setenv("AR_AUTO_SCHEME_NSAMPLES", "7")
+ assert envs.AR_AUTO_SCHEME_NSAMPLES == 7
+
+
+def test_env_ar_auto_scheme_batch_size_overrides_default(monkeypatch):
+ """AR_AUTO_SCHEME_BATCH_SIZE env var should override the built-in batch_size default."""
+ import auto_round.envs as envs
+
+ monkeypatch.setenv("AR_AUTO_SCHEME_BATCH_SIZE", "4")
+ assert envs.AR_AUTO_SCHEME_BATCH_SIZE == 4
-def test_apply_head_trick_keeps_lowest_loss_candidate_when_budget_allows():
- schemes = AutoScheme(avg_bits=3, options=("GGUF:Q2_K_S", "GGUF:Q4_K_M")).options
- total_scores = {
- "lm_head": [
- [0, 1244659712, 41.0, ["lm_head"]],
- [1, 2489319424, 11.9, ["lm_head"]],
- ],
- "model.layers.0.mlp.down_proj": [
- [0, 100, 10.0, ["model.layers.0.mlp.down_proj"]],
- [1, 200, 5.0, ["model.layers.0.mlp.down_proj"]],
- ],
- }
-
- _apply_head_trick(
- head_name="lm_head",
- schemes=schemes,
- sorted_indices=[1, 0],
- target_bits=3,
- target_params_cnt=4891670016,
- total_scores=total_scores,
- )
-
- assert total_scores["lm_head"] == [[1, 2489319424, 11.9, ["lm_head"]]]
-
-
-def test_apply_head_trick_relaxes_lowest_loss_candidate_when_budget_is_tight():
- schemes = AutoScheme(avg_bits=3, options=("GGUF:Q2_K_S", "GGUF:Q4_K_M")).options
- total_scores = {
- "lm_head": [
- [0, 1244659712, 41.0, ["lm_head"]],
- [1, 2489319424, 11.9, ["lm_head"]],
- ],
- "model.layers.0.mlp.down_proj": [
- [0, 100, 10.0, ["model.layers.0.mlp.down_proj"]],
- [1, 200, 5.0, ["model.layers.0.mlp.down_proj"]],
- ],
- }
-
- _apply_head_trick(
- head_name="lm_head",
- schemes=schemes,
- sorted_indices=[1, 0],
- target_bits=3,
- target_params_cnt=1244659812,
- total_scores=total_scores,
- )
-
- assert total_scores["lm_head"] == [
- [0, 1244659712, 41.0, ["lm_head"]],
- [1, 2489319424, 11.9, ["lm_head"]],
+def test_env_ar_auto_scheme_batch_size_zero_raises(monkeypatch):
+ """Zero value for AR_AUTO_SCHEME_BATCH_SIZE should raise ValueError."""
+ import pytest
+
+ import auto_round.envs as envs
+
+ monkeypatch.setenv("AR_AUTO_SCHEME_BATCH_SIZE", "0")
+ with pytest.raises(ValueError):
+ _ = envs.AR_AUTO_SCHEME_BATCH_SIZE
+
+
+def test_build_layer_config_header_rows_merges_adjacent_prefixes():
+ """Adjacent columns with the same prefix should be merged into one compact header cell."""
+ columns = ["mlp.down_proj", "mlp.gate_proj", "self_attn.q_proj", "self_attn.v_proj"]
+ assert _build_layer_config_header_rows(columns) == [
+ ["block", "mlp", "", "self_attn", ""],
+ ["", "down_proj", "gate_proj", "q_proj", "v_proj"],
]
+
+
+def test_short_summary_name_keeps_one_field_before_numeric_suffix():
+ """Numeric block suffixes should be shortened to keep the preceding field."""
+ assert _short_summary_name("model.layers.0") == "layers.0"
+
+
+def test_build_expert_groups_groups_experts_per_block():
+ """Expert layers in the same block should be grouped together."""
+ import torch
+ from torch import nn
+
+ from auto_round.auto_scheme.utils import build_expert_groups
+
+ # Build a minimal MoE-like model with 2 blocks, each with 2 experts having 2 projections
+ class FakeModel(nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.model = nn.Module()
+ self.model.layers = nn.ModuleList()
+ for i in range(2):
+ block = nn.Module()
+ block.mlp = nn.Module()
+ block.mlp.experts = nn.ModuleList()
+ for j in range(2):
+ expert = nn.Module()
+ expert.gate_proj = nn.Linear(8, 8, bias=False)
+ expert.up_proj = nn.Linear(8, 8, bias=False)
+ expert.down_proj = nn.Linear(8, 8, bias=False)
+ block.mlp.experts.append(expert)
+ block.self_attn = nn.Module()
+ block.self_attn.q_proj = nn.Linear(8, 8, bias=False)
+ self.model.layers.append(block)
+
+ model = FakeModel()
+ quant_layer_names = [n for n, m in model.named_modules() if isinstance(m, nn.Linear)]
+ fixed_layer_scheme = {}
+
+ groups = build_expert_groups(model, quant_layer_names, fixed_layer_scheme)
+ # Should have 2 groups (one per block), each containing all 6 expert projections
+ assert len(groups) == 2
+ for group in groups:
+ expert_layers = [n for n in group if "experts" in n]
+ assert len(expert_layers) == 6 # 2 experts * 3 projections
+ # Non-expert layers (q_proj) should NOT be in the group
+ assert all("self_attn" not in n for n in group)
+
+
+def test_build_expert_groups_skips_fixed_layers():
+ """Expert layers already in fixed_layer_scheme should not be grouped."""
+ import torch
+ from torch import nn
+
+ from auto_round.auto_scheme.utils import build_expert_groups
+
+ class FakeModel(nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.model = nn.Module()
+ self.model.layers = nn.ModuleList()
+ block = nn.Module()
+ block.mlp = nn.Module()
+ block.mlp.experts = nn.ModuleList()
+ for j in range(2):
+ expert = nn.Module()
+ expert.gate_proj = nn.Linear(8, 8, bias=False)
+ block.mlp.experts.append(expert)
+ self.model.layers.append(block)
+
+ model = FakeModel()
+ quant_layer_names = [n for n, m in model.named_modules() if isinstance(m, nn.Linear)]
+ # Fix all expert layers
+ fixed_layer_scheme = {n: {} for n in quant_layer_names if "experts" in n}
+
+ groups = build_expert_groups(model, quant_layer_names, fixed_layer_scheme)
+ assert len(groups) == 0
diff --git a/test/test_cuda/algorithms/test_auto_scheme.py b/test/test_cuda/algorithms/test_auto_scheme.py
index 6ff82bba3..c721b6b20 100644
--- a/test/test_cuda/algorithms/test_auto_scheme.py
+++ b/test/test_cuda/algorithms/test_auto_scheme.py
@@ -260,6 +260,41 @@ class TestAutoScheme:
# cause using tiny model
assert 4.8 < avg_bits <= target_bits + 1e-3
+ def test_lm_head_mix_score_nonzero(self, tiny_untied_qwen_model_path):
+ """Verify that lm_head gets a non-zero mix_score in the low_gpu_mem_usage path."""
+ import auto_round.auto_scheme.delta_loss as dl
+
+ target_bits = 5
+ scheme = AutoScheme(avg_bits=target_bits, options=("MXFP4", "MXFP8"))
+
+ scores_captured = {}
+ orig_head_trick = dl._apply_head_trick
+
+ def capture_head_scores(head_name, schemes, sorted_indices, target_bits, target_params_cnt, total_scores):
+ if head_name in total_scores:
+ scores_captured["head_scores"] = [list(opt) for opt in total_scores[head_name]]
+ return orig_head_trick(head_name, schemes, sorted_indices, target_bits, target_params_cnt, total_scores)
+
+ dl._apply_head_trick = capture_head_scores
+ try:
+ ar = AutoRound(
+ tiny_untied_qwen_model_path,
+ scheme=scheme,
+ iters=0,
+ disable_opt_rtn=True,
+ nsamples=1,
+ quant_lm_head=True,
+ )
+ ar.quantize()
+ finally:
+ dl._apply_head_trick = orig_head_trick
+
+ assert "head_scores" in scores_captured, "lm_head was not found in total_scores"
+ head_scores = scores_captured["head_scores"]
+ assert len(head_scores) >= 2, f"Expected at least 2 scheme options, got {len(head_scores)}"
+ has_nonzero = any(opt[2] > 0 for opt in head_scores)
+ assert has_nonzero, f"lm_head mix_score is 0 for all schemes: {head_scores}"
+
@pytest.mark.skip_ci(reason="The evaluation is time-consuming")
def test_auto_scheme_export(self):
model_name = get_model_path("facebook/opt-125m")