PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
[AutoScheme] fix lm_head no_grad issue and support MOE model with shared mix_score (#1971)
Xin Hee6b35d13200a
diff --git a/auto_round/auto_scheme/delta_loss.py b/auto_round/auto_scheme/delta_loss.pyindex 1a6a94a90..7861acb52 100644--- a/auto_round/auto_scheme/delta_loss.py+++ b/auto_round/auto_scheme/delta_loss.py@@ -14,6 +14,7 @@import copyimport gc+import mathfrom dataclasses import asdictfrom functools import wrapsfrom typing import Iterable, Optional, Union@@ -25,8 +26,16 @@ from tqdm import tqdmfrom auto_round.auto_scheme.gen_auto_scheme import AutoSchemefrom auto_round.auto_scheme.register import register_scheme_methodsfrom auto_round.auto_scheme.utils import (+ _describe_layer_config,+ _fill_inactive_expert_scores,+ _log_batch_avg_loss,+ _log_scheme_loss_matrix,+ _log_score_summary_by_block_and_nonblock,+ _scheme_short_name,apply_quant_scheme,+ build_expert_groups,compute_layer_bits,+ merge_lists_unionfind,parse_shared_layers,remove_quant_scheme,)@@ -62,6 +71,7 @@ from auto_round.utils import ()from auto_round.utils.device import MemoryMonitor, memory_monitorfrom auto_round.utils.device_manager import get_current_device_manager+from auto_round.utils.model import is_moe_model as _is_moe_modelfrom auto_round.utils.offload import OffloadManagerfrom auto_round.wrapper import WrapperLinear@@ -81,6 +91,9 @@ class AutoSchemeWrapperLinear(WrapperLinear):enable_torch_compile=False,**kwargs,):+ """Wrap ``orig_layer`` to accumulate a ``mix_score`` (weight + activation loss) during+ forward/backward, used by Delta Loss to rank candidate quantization schemes.+ """super().__init__(orig_layer,enable_minmax_tuning,@@ -90,7 +103,6 @@ class AutoSchemeWrapperLinear(WrapperLinear):enable_torch_compile=enable_torch_compile,**kwargs,)- self.total_act_score = 0.0self.act_score = 0.0self.avg_act_score = 0.0self.act_cnt = 0.0@@ -105,6 +117,9 @@ class AutoSchemeWrapperLinear(WrapperLinear):self.orig_layer.weight.requires_grad = Truedef _qdq_act(self, x, act_min_scale=1.0, act_max_scale=1.0, act_max=None):+ """Quant-dequant the activation and, in ``grad_mode``, register a backward hook that+ accumulates ``act_score`` from ``|grad * (x - qdq_x)|``.+ """if hasattr(self.orig_layer, "act_bits") and self.orig_layer.act_bits > 8:return x, 1.0, None@@ -118,6 +133,7 @@ class AutoSchemeWrapperLinear(WrapperLinear):self.x_diff = x_diff.to("cpu")def save_grad(grad):+ """Backward hook: accumulate activation score from grad * (x - qdq_x)."""if self.max_act_value == 0:if torch.abs(grad).max() != 0:raise ValueError@@ -131,8 +147,7 @@ class AutoSchemeWrapperLinear(WrapperLinear):self.act_cnt -= 1return None- self.total_act_score += torch.abs((grad * self.x_diff.to(grad.device))).sum().item()- self.act_score = 0.0 if self.act_cnt <= 0 else self.total_act_score / self.act_cnt+ self.act_score += torch.abs((grad * self.x_diff.to(grad.device))).sum().item()self.mix_score = self.weight_score + self.act_scoreself.x_diff = Nonereturn None@@ -141,6 +156,11 @@ class AutoSchemeWrapperLinear(WrapperLinear):return qdq_x, scale, zpdef _qdq_weight(self, value, min_scale, max_scale):+ """Quant-dequant the weight and, in ``grad_mode``, register a backward hook that+ accumulates ``weight_score`` from ``|grad * (weight - qdq_w)|``. Weight quantization+ for the hook is only recomputed lazily inside the hook itself, so layers whose+ forward never runs (e.g. unrouted MoE experts) never pay this cost.+ """device = self.deviceif self.orig_layer.bits > 8 or not self.need_weight_grad:qdq_w, scale, zp = super()._qdq_weight(@@ -156,13 +176,13 @@ class AutoSchemeWrapperLinear(WrapperLinear):if self.grad_mode:def save_grad(grad):+ """Backward hook: accumulate weight score from grad * (weight - qdq_w)."""qdq_w, scale, zp = self.super_qdq_func(torch.tensor(0, device=device), torch.tensor(1.0, device=device), torch.tensor(1.0, device=device))w_diff = self.orig_layer.weight - qdq_w.to(self.orig_layer.weight.device)self.weight_score += torch.abs((grad.to(w_diff.device) * w_diff)).sum().item()- act_score = 0.0 if self.act_cnt <= 0 else self.total_act_score / self.act_cnt- self.mix_score = self.weight_score + act_score+ self.mix_score = self.weight_score + self.act_scorereturn Noneqdq_w.register_hook(save_grad)@@ -170,6 +190,7 @@ class AutoSchemeWrapperLinear(WrapperLinear):class AutoSchemeWrapperLinearIMatrix(WrapperLinear):+ """GGUF-K wrapper that scores a layer using an imatrix-aware quant search (RTN, iters=0)."""def __init__(self,@@ -182,6 +203,7 @@ class AutoSchemeWrapperLinearIMatrix(WrapperLinear):enable_torch_compile=False,**kwargs,):+ """Wrap ``orig_layer`` and eagerly run the imatrix-aware quant search to build ``qdq_w``."""super().__init__(orig_layer,enable_minmax_tuning,@@ -191,7 +213,6 @@ class AutoSchemeWrapperLinearIMatrix(WrapperLinear):enable_torch_compile=enable_torch_compile,**kwargs,)- self.total_act_score = 0.0self.act_score = 0.0self.avg_act_score = 0.0self.act_cnt = 0.0@@ -216,8 +237,9 @@ class AutoSchemeWrapperLinearIMatrix(WrapperLinear):@torch.no_grad()def post_init_qdqw(self, device):- # Could not place in qdq_w, otherwise vram is much higher-+ """Run the imatrix-aware quant search once and cache the result as buffer ``qdq_w``,+ registering a backward hook on it to accumulate ``weight_score``.+ """qdq_w, _, _ = self.weight_search_quant_func(self.orig_layer.weight.to(device),bits=self.orig_layer.bits,@@ -235,10 +257,10 @@ class AutoSchemeWrapperLinearIMatrix(WrapperLinear):self.register_buffer("qdq_w", qdq_w.detach().clone().to(self.orig_layer.weight.device))def save_grad(grad):+ """Backward hook: accumulate weight score from grad * (weight - qdq_w)."""w_diff = self.orig_layer.weight - self.qdq_w.to(self.orig_layer.weight.device)self.weight_score += torch.abs((grad.to(torch.float32) * w_diff.to(grad.device))).sum().item()- act_score = 0.0 if self.act_cnt <= 0 else self.total_act_score / self.act_cnt- self.mix_score = self.weight_score + act_score+ self.mix_score = self.weight_score + self.act_scorereturn Noneself.qdq_w.requires_grad_(True)@@ -247,6 +269,9 @@ class AutoSchemeWrapperLinearIMatrix(WrapperLinear):self.qdq_w.register_hook(save_grad)def _qdq_act(self, x, act_min_scale=1.0, act_max_scale=1.0, act_max=None):+ """Quant-dequant the activation and, in ``grad_mode``, register a backward hook that+ accumulates ``act_score`` from ``|grad * (x - qdq_x)|``.+ """if hasattr(self.orig_layer, "act_bits") and self.orig_layer.act_bits > 8:return x, 1.0, None@@ -260,6 +285,7 @@ class AutoSchemeWrapperLinearIMatrix(WrapperLinear):self.x_diff = x_diff.to("cpu")def save_grad(grad):+ """Backward hook: accumulate activation score from grad * (x - qdq_x)."""if self.max_act_value == 0:if torch.abs(grad).max() != 0:raise ValueError@@ -273,8 +299,7 @@ class AutoSchemeWrapperLinearIMatrix(WrapperLinear):self.act_cnt -= 1return None- self.total_act_score += torch.abs((grad * self.x_diff.to(grad.device))).sum().item()- self.act_score = 0.0 if self.act_cnt <= 0 else self.total_act_score / self.act_cnt+ self.act_score += torch.abs((grad * self.x_diff.to(grad.device))).sum().item()self.mix_score = self.weight_score + self.act_scoreself.x_diff = Nonereturn None@@ -283,10 +308,12 @@ class AutoSchemeWrapperLinearIMatrix(WrapperLinear):return qdq_x, scale, zpdef _qdq_weight(self, value, min_scale, max_scale):+ """Return the cached ``qdq_w`` computed eagerly in ``__init__`` (via ``post_init_qdqw``)."""return self.qdq_w, 1.0, Noneclass AutoSchemeWrapperLinearForGGUFK(AutoSchemeWrapperLinear):+ """GGUF-K wrapper (no imatrix): scores a layer using the plain GGUF K-quant search."""def __init__(self,@@ -298,6 +325,7 @@ class AutoSchemeWrapperLinearForGGUFK(AutoSchemeWrapperLinear):need_weight_grad=False,**kwargs,):+ """Wrap ``orig_layer`` and eagerly run the GGUF K-quant search to build ``qdq_w``."""super().__init__(orig_layer,enable_minmax_tuning,@@ -307,18 +335,24 @@ class AutoSchemeWrapperLinearForGGUFK(AutoSchemeWrapperLinear):need_weight_grad,**kwargs,)- with torch.no_grad():- qdq_w, scale, zp = self.super_qdq_func(- torch.tensor(0).to(device), torch.tensor(1.0).to(device), torch.tensor(1.0).to(device)- )+ self.post_init_qdqw(device)++ @torch.no_grad()+ def post_init_qdqw(self, device):+ """Run the GGUF K-quant search once and cache the result as buffer ``qdq_w``,+ registering a backward hook on it to accumulate ``weight_score``.+ """+ qdq_w, scale, zp = self.super_qdq_func(+ torch.tensor(0).to(device), torch.tensor(1.0).to(device), torch.tensor(1.0).to(device)+ )self.register_buffer("qdq_w", qdq_w.detach().clone().to(self.orig_layer.weight.device))def save_grad(grad):+ """Backward hook: accumulate weight score from grad * (weight - qdq_w)."""w_diff = self.orig_layer.weight - self.qdq_w.to(self.orig_layer.weight.device)# TODO strange, grad could be in CPUself.weight_score += torch.abs((grad.to(w_diff.device).to(torch.float32) * w_diff)).sum().item()- act_score = 0.0 if self.act_cnt <= 0 else self.total_act_score / self.act_cnt- self.mix_score = self.weight_score + act_score+ self.mix_score = self.weight_score + self.act_scorereturn Noneself.qdq_w.requires_grad_(True)@@ -326,10 +360,14 @@ class AutoSchemeWrapperLinearForGGUFK(AutoSchemeWrapperLinear):self.qdq_w.register_hook(save_grad)def _qdq_weight(self, value, min_scale, max_scale):+ """Return the cached ``qdq_w`` computed eagerly in ``__init__`` (via ``post_init_qdqw``)."""return self.qdq_w, 1.0, Noneclass AutoSchemeWrapperLinearForGGUFKImatrix(AutoSchemeWrapperLinear):+ """GGUF-K wrapper (with imatrix): scores a layer using the imatrix-weighted GGUF K-quant+ search (``_init_scale``).+ """def __init__(self,@@ -342,6 +380,9 @@ class AutoSchemeWrapperLinearForGGUFKImatrix(AutoSchemeWrapperLinear):enable_torch_compile=False,**kwargs,):+ """Wrap ``orig_layer`` and eagerly run the imatrix-weighted GGUF K-quant search to+ build ``qdq_w``.+ """super().__init__(orig_layer,enable_minmax_tuning,@@ -356,14 +397,17 @@ class AutoSchemeWrapperLinearForGGUFKImatrix(AutoSchemeWrapperLinear):@torch.no_grad()def post_init_qdqw(self, device): # Could not place in qdq_w, otherwise vram is much higher+ """Run the imatrix-weighted GGUF K-quant search once and cache the result as buffer+ ``qdq_w``, registering a backward hook on it to accumulate ``weight_score``.+ """qdq_w = self._init_scale(device).detach()self.register_buffer("qdq_w", qdq_w.detach().clone().to(self.orig_layer.weight.device))def save_grad(grad):+ """Backward hook: accumulate weight score from grad * (weight - qdq_w)."""w_diff = self.orig_layer.weight - self.qdq_w.to(self.orig_layer.weight.device)self.weight_score += torch.abs((grad.to(torch.float32) * w_diff.to(grad.device))).sum().item()- act_score = 0.0 if self.act_cnt <= 0 else self.total_act_score / self.act_cnt- self.mix_score = self.weight_score + act_score+ self.mix_score = self.weight_score + self.act_scorereturn Noneself.qdq_w.requires_grad_(True)@@ -373,6 +417,9 @@ class AutoSchemeWrapperLinearForGGUFKImatrix(AutoSchemeWrapperLinear):@torch.no_grad()def _init_scale(self, device):+ """Compute the imatrix-weighted GGUF K-quant quant-dequant weight for ``bits`` in+ [2,3,4,5,6], returned in the original weight dtype.+ """tensor = self.orig_layer.weight.data.to(device)bits = self.orig_layer.bitsscale_dtype = self.orig_layer.scale_dtype@@ -407,6 +454,7 @@ class AutoSchemeWrapperLinearForGGUFKImatrix(AutoSchemeWrapperLinear):return qdq_w.to(orig_dtype)def _qdq_weight(self, value, min_scale, max_scale):+ """Return the cached ``qdq_w`` computed eagerly in ``__init__`` (via ``post_init_qdqw``)."""return self.qdq_w, 1.0, None@@ -414,6 +462,7 @@ def register_imatrix_hook(model):"""Registers hooks to accumulate activation squared norms into `imatrix`."""def get_imatrix_hook(module, input, output):+ """Forward hook: accumulate the per-channel squared-activation sum into ``module.imatrix``."""input = input[0] if isinstance(input, (tuple, list)) else inputflattened = input.reshape(-1, input.shape[-1]).to(torch.float32)squared = torch.sum(torch.pow(flattened, 2), dim=0).to(torch.float32)@@ -433,7 +482,10 @@ def register_imatrix_hook(model):@torch.no_grad()def cal_imatrix(model, dataloader, major_device, low_gpu_mem_usage):-+ """Accumulate an activation-based imatrix on every supported layer by running the+ calibration ``dataloader`` through ``model`` once (dispatches to the low-GPU-memory or+ full-forward variant based on ``low_gpu_mem_usage``).+ """if low_gpu_mem_usage:cal_imatrix_low_gpu(model, dataloader, major_device)else:@@ -446,18 +498,24 @@ def cal_imatrix(model, dataloader, major_device, low_gpu_mem_usage):def cal_imatrix_low_gpu(model, dataloader, major_device):+ """Low-GPU-memory variant of ``cal_imatrix``: moves each block to ``major_device`` only+ for the duration of its own forward pass (via pre/post forward hooks), then back to CPU.+ """imatrix_hooks = register_imatrix_hook(model)block_names = get_block_names(model, quant_vision=True)block_names = flatten_list(block_names)def move_to_gpu_hook(module, inputs):+ """Pre-forward hook: move this block (and its inputs) to ``major_device``."""module.to(major_device)to_device(inputs, major_device)def move_to_cpu(module, inputs, outputs):+ """Forward hook: move this block back to CPU once its forward pass is done."""module.to("cpu")def move_to_cpu_clear_memory(module, inputs, outputs):+ """Forward hook: move this block back to CPU and free the device memory it used."""module.to("cpu")clear_memory(device_list=major_device)@@ -484,8 +542,12 @@ def cal_imatrix_low_gpu(model, dataloader, major_device):class MyCustomError(Exception):+ """Raised from ``backward_pre_hook`` to deliberately interrupt ``loss.backward()`` at the+ last block, so gradients can be replayed manually block-by-block in ``model_forward_low_gpu``.+ """def __init__(self, message):+ """Create the interrupt signal with the given ``message``."""super().__init__(message)@@ -493,6 +555,14 @@ last_grad_input = Nonedef prepare_model_low_gpu(model, block_inputs: dict = None, pbar=None, major_device="cpu"):+ """Wrap every block's forward so that, for one calibration batch, it (1) moves itself to+ ``major_device`` on demand, (2) records its own inputs into ``block_inputs`` (on CPU) so+ they can be replayed later, and (3) moves itself back to CPU once done.++ Called once per calibration batch before ``model_forward_low_gpu`` runs the actual+ forward+backward -- the recorded ``block_inputs`` are what let the backward pass be+ replayed manually, one block at a time, without keeping every block resident on GPU.+ """block_inputs.clear()for n, m in model.named_modules():if hasattr(m, "grad_mode"):@@ -501,10 +571,16 @@ def prepare_model_low_gpu(model, block_inputs: dict = None, pbar=None, major_devblock_names = get_block_names(model)[0]def wrap_forward(module, module_name):+ """Build a replacement ``forward`` for ``module`` that captures its inputs/outputs+ (see ``prepare_model_low_gpu`` docstring) while moving it to/from ``major_device``.+ """original_forward = module.forward@wraps(original_forward)def new_forward(*args, **kwargs):+ """Move the block to device, run its original forward, cache its (CPU) inputs+ for later replay, then move the block back to CPU.+ """move_module_to_tuning_device(module, major_device=major_device)# for n,m in module.named_modules():# if hasattr(m, "post_init_qdqw"):@@ -595,8 +671,17 @@ def model_forward(model, data, **forward_kwargs):return model(**prepared, **forward_kwargs), prepared-def model_forward_low_gpu(model, dataloader, major_device="cuda", pbar=None):+def model_forward_low_gpu(model, dataloader, major_device="cuda", pbar=None, scheme_tag=None):+ """Run one full scoring pass (all calibration batches) in low-GPU-memory mode.++ For each batch: capture per-block inputs via ``prepare_model_low_gpu``, run a forward+ pass whose backward is deliberately interrupted at the last block (``backward_pre_hook``+ raising ``MyCustomError``), then manually replay the backward pass block-by-block+ (moving each block to ``major_device`` only for its own recompute + backward, then back+ to CPU) so only one block's weights need to be resident on GPU at a time.+ """block_inputs = {}+ total_batches = len(dataloader) if hasattr(dataloader, "__len__") else Noneblock_names = get_block_names(model)[0]for name in block_names:@@ -610,9 +695,25 @@ def model_forward_low_gpu(model, dataloader, major_device="cuda", pbar=None):get_current_device_manager().synchronize()raise MyCustomError("Interrupt backward pass")- for data in dataloader:+ for batch_idx, data in enumerate(dataloader, start=1):prepare_model_low_gpu(model, block_inputs, major_device=major_device, pbar=pbar)+ # lm_head sits outside every decoder block, so it never gets `grad_mode=True`+ # in the manual block-by-block backward below. Scope the fix narrowly to+ # just lm_head (rather than every non-block module) to avoid enabling grad+ # tracking / scoring hooks on unrelated out-of-block layers, which would+ # add extra autograd-graph memory for no benefit. The backward flow is:+ # loss → lm_head (hook fires here) → norm → last_block (hook raises error)+ head_name = get_lm_head_name(model)+ if head_name is not None:+ # Once lm_head has been wrapped for scoring, `get_lm_head_name` resolves+ # to the inner original Linear (e.g. "lm_head.orig_layer") rather than+ # the wrapper itself ("lm_head") -- strip the suffix to reach the wrapper.+ head_name = head_name.removesuffix(".orig_layer")+ head_module = get_module(model, head_name)+ if hasattr(head_module, "grad_mode"):+ head_module.grad_mode = True+# Register backward hook on the last blocklast_block = get_module(model, block_names[-1])last_block_backward_hook = last_block.register_full_backward_pre_hook(backward_pre_hook)@@ -701,6 +802,15 @@ def model_forward_low_gpu(model, dataloader, major_device="cuda", pbar=None):pbar.update(1)+ _log_batch_avg_loss(+ model,+ batch_idx,+ pbar=pbar,+ block_names=block_names,+ total_batches=total_batches,+ scheme_tag=scheme_tag,+ )+def get_score_for_scheme(model,@@ -723,8 +833,19 @@ def get_score_for_scheme(is_vlm: bool = False,force_mllm: bool = False,model_name: Optional[str] = None,+ scheme_tag: Optional[str] = None,):+ """Wrap every quantizable layer in ``quant_layer_names`` with a scoring wrapper, run+ forward(+backward, unless RTN-only) calibration over ``nsamples`` examples from+ ``dataset``/``dataloader``, then unwrap and return each layer's ``[bits, loss]``.+ """scores_dict = {} # Key=name,Val=[quant_total_bits, loss]+ # Include the visual block(s) when scoring VLMs with ``--quant_nontext_module``+ # (``force_mllm=True``) so vision-tower layer losses match a block below instead+ # of silently falling through to "non_block" in the logging/inactive-expert-fill+ # helpers. Mirrors the same ``quant_vision=force_mllm`` pattern used in+ # ``_gen_layer_config``.+ block_names = get_block_names(model, quant_vision=force_mllm)[0]for n, m in model.named_modules():if type(m) in SUPPORTED_LAYER_TYPES:m.weight.requires_grad = False@@ -931,11 +1052,11 @@ def get_score_for_scheme("AutoScheme(force_mllm): cannot build mllm dataloader. ""Provide a `processor` and a multimodal `dataset`.")- model_forward_low_gpu(model, mllm_loader, major_device=major_device, pbar=pbar)+ model_forward_low_gpu(model, mllm_loader, major_device=major_device, pbar=pbar, scheme_tag=scheme_tag)else:try:dataloader = _build_calib_dataloader()- model_forward_low_gpu(model, dataloader, major_device=major_device, pbar=pbar)+ model_forward_low_gpu(model, dataloader, major_device=major_device, pbar=pbar, scheme_tag=scheme_tag)except Exception as exc: # noqa: BLE001if not is_vlm:raise@@ -947,7 +1068,7 @@ def get_score_for_scheme(batch_size = 1if mllm_loader is None:raise- model_forward_low_gpu(model, mllm_loader, major_device=major_device, pbar=pbar)+ model_forward_low_gpu(model, mllm_loader, major_device=major_device, pbar=pbar, scheme_tag=scheme_tag)else:for n, m in model.named_modules():if hasattr(m, "grad_mode"):@@ -956,6 +1077,11 @@ def get_score_for_scheme(# m.post_init_qdqw()def _run_forward_loop(loader):+ """Run the full (non-low-GPU) forward+backward calibration loop over ``loader``,+ accumulating ``mix_score`` on every wrapped layer and periodically logging progress+ via ``_log_batch_avg_loss``.+ """+ total_batches = len(loader) if hasattr(loader, "__len__") else None_checked_pixel = False_pixel_keys = ("pixel_values",@@ -965,7 +1091,7 @@ def get_score_for_scheme("images","image",)- for data in loader:+ for batch_idx, data in enumerate(loader, start=1):# Pull labels out of the batch (VLM datasets often carry them;# LLM ones don't) before mllm_model_forward casts dtypes._src = data if isinstance(data, dict) else None@@ -1004,6 +1130,14 @@ def get_score_for_scheme(m.grad = Noneif pbar is not None:pbar.update(1)+ _log_batch_avg_loss(+ model,+ batch_idx,+ pbar=pbar,+ block_names=block_names,+ total_batches=total_batches,+ scheme_tag=scheme_tag,+ )if force_mllm:mllm_loader = _build_mllm_calib_dataloader()@@ -1031,7 +1165,6 @@ def get_score_for_scheme(for n, m in model.named_parameters():m.grad = None- scores_dict = {}for n, m in model.named_modules():if hasattr(m, "mix_score"):if m.orig_layer.act_bits <= 8:@@ -1041,6 +1174,15 @@ def get_score_for_scheme()layer_bits, _ = compute_layer_bits(m.orig_layer, ignore_scale_zp_bits=ignore_scale_zp_bits)scores_dict[n] = [layer_bits, m.mix_score]+ _fill_inactive_expert_scores(scores_dict, block_names)+ _log_score_summary_by_block_and_nonblock(+ scores_dict,+ block_names,+ model=model,+ scheme_tag=scheme_tag,+ summary_stage="final",+ )+for n, m in model.named_modules():if hasattr(m, "orig_layer"):# Explicitly break reference cycles to ensure GC can free the wrapper.@@ -1085,10 +1227,11 @@ def choose_bits_per_layer_with_path(layers: dict, P: int, max_states: int = Nonesolution exists."""# dp: total_params -> (accumulated_loss, chosen_path)- # The path explicitly stores the selected options.- dp: dict[int, tuple[float, list]] = {0: (0.0, [])}+ # The path is stored as a tuple to avoid the high overhead of repeatedly+ # copying Python lists for every DP transition.+ dp: dict[int, tuple[float, tuple]] = {0: (0.0, ())}for layer_name, opts in layers.items():- new_dp: dict[int, tuple[float, list]] = {}+ new_dp: dict[int, tuple[float, tuple]] = {}for cur_params, (cur_loss, cur_path) in dp.items():for opt in opts:scheme, bits_cost, loss_cost, layer_names = opt@@ -1097,7 +1240,7 @@ def choose_bits_per_layer_with_path(layers: dict, P: int, max_states: int = Nonecontinuenew_loss = cur_loss + loss_cost- new_path = cur_path + [(layer_names, scheme)]+ new_path = cur_path + ((layer_names, scheme),)# Keep the path with smaller loss for the same parameter budgetif np_total not in new_dp or new_loss < new_dp[np_total][0]:@@ -1107,7 +1250,7 @@ def choose_bits_per_layer_with_path(layers: dict, P: int, max_states: int = Nonereturn None, None# Pareto pruning: remove dominated (params, loss) statesitems = sorted(new_dp.items(), key=lambda x: x[0]) # (params, (loss, path))- pruned: dict[int, tuple[float, list]] = {}+ pruned: dict[int, tuple[float, tuple]] = {}best_loss_so_far = float("inf")for params_val, (loss_val, path_val) in items:if loss_val < best_loss_so_far:@@ -1115,7 +1258,7 @@ def choose_bits_per_layer_with_path(layers: dict, P: int, max_states: int = Nonebest_loss_so_far = loss_val# Beam width limit: if too many states survive Pareto pruning,- # uniformly subsample to bound memory usage. For models with many+ # uniformly subsample to bound memory usage. For models with many# layers whose sizes are incommensurate, the number of distinct# cumulative-bit sums can grow to millions, each storing a full# path copy — easily exceeding 70 GB of RAM.@@ -1128,9 +1271,11 @@ def choose_bits_per_layer_with_path(layers: dict, P: int, max_states: int = Nonen = len(sorted_keys)# Uniformly pick max_states indices (always include first and last)step = (n - 1) / (max_states - 1)- selected: dict[int, tuple[float, list]] = {}+ selected: dict[int, tuple[float, tuple]] = {}for i in range(max_states):idx = int(round(i * step))+ if idx >= n:+ idx = n - 1k = sorted_keys[idx]selected[k] = pruned[k]pruned = selected@@ -1140,15 +1285,24 @@ def choose_bits_per_layer_with_path(layers: dict, P: int, max_states: int = None# Select the solution with the minimum lossbest_params = min(dp.keys(), key=lambda k: dp[k][0])best_loss, best_path = dp[best_params]- return best_loss, best_path+ return best_loss, list(best_path)def move_module_to_tuning_device(module, major_device="cpu"):+ """Move every submodule of ``module`` to its own tuning device: wrapper submodules go to+ ``orig_layer.tuning_device``/``tuning_device`` (set per-layer earlier), leaf modules with+ no such attribute fall back to ``major_device``, and any directly-held parameters/buffers+ (not just the standard ``.to()`` targets) are relocated along with their ``.grad``.+ """def _normalize(dev):+ """Coerce ``dev`` (str or ``torch.device``) into a ``torch.device``."""return dev if isinstance(dev, torch.device) else torch.device(dev)def _move_own_tensors(m, device):+ """Move ``m``'s directly-owned (non-recursive) parameters/buffers (and their+ ``.grad``) to ``device``.+ """# Cover non-leaf modules that directly hold nn.Parameter / buffers# (e.g. Mamba/GDN linear_attn with A_log & dt_bias). Also relocate# p.grad together with p.data — otherwise the next backward's grad@@ -1294,6 +1448,14 @@ def _gen_layer_config(processor=None,is_vlm: bool = False,):+ """Score every candidate scheme in ``auto_scheme.options`` against ``quant_layer_names``+ and return per-layer per-scheme losses used by the caller to pick a final bit-width+ assignment (via the DP knapsack in ``choose_bits_per_layer_with_path``).++ For each scheme: wraps every quantizable layer with a scoring wrapper, runs+ forward+backward calibration to accumulate ``mix_score`` (weight + activation loss), then+ unwraps and records the result before moving to the next scheme.+ """# Initialize memory tracking for AutoSchemememory_monitor = MemoryMonitor()# memory_monitor.reset()@@ -1338,6 +1500,8 @@ def _gen_layer_config(embedding_layers_names.append(name)quant_layer_names = list(set(quant_layer_names) - set(embedding_layers_names))+ is_moe_model = _is_moe_model(model)+# Decide whether AutoScheme has to score vision-tower layers (typically# because the user passed ``--quant_nontext_module``). Used below to# clamp batch_size to 1 (image sizes vary) and to pick the multimodal@@ -1346,7 +1510,11 @@ def _gen_layer_config(vision_markers = ("vision", "visual", "image", "img")force_mllm = is_vlm and any(any(marker in n.lower() for marker in vision_markers) for n in quant_layer_names)- block_name = get_block_names(model)[0] # TODO need change to support vlm+ # When scoring vision-tower layers (``force_mllm``, typically because the+ # caller passed ``--quant_nontext_module``), include the visual block(s)+ # so they get ``in_block=True``, count towards ``block_num``/progress, and+ # participate in offload hooks below just like language blocks.+ block_name = get_block_names(model, quant_vision=force_mllm)[0]for name in block_name:module = get_module(model, name)module.in_block = True@@ -1364,6 +1532,9 @@ def _gen_layer_config(schemes = auto_scheme.optionsdef check_bf16_scheme(scheme):+ """Return True if ``scheme`` is effectively BF16/no-op (bits >= 16 and act_bits >= 16),+ in which case scoring can skip the expensive wrap/forward/backward cycle entirely.+ """if isinstance(scheme, str) and scheme.upper() == "BF16":return Trueif isinstance(scheme, QuantizationScheme):@@ -1372,32 +1543,34 @@ def _gen_layer_config(return Truereturn False- if auto_scheme.nsamples is not None:+ from auto_round import envs as _envs++ _env_nsamples = _envs.AR_AUTO_SCHEME_NSAMPLES+ # Priority for nsamples: env > API > default+ if _env_nsamples is not None:+ nsamples = _env_nsamples+ elif auto_scheme.nsamples is not None:nsamples = auto_scheme.nsampleselse:- is_moe_model = False- if hasattr(model, "config"):- for key in model.config.to_dict().keys():- if "moe" in key or "expert" in key:- is_moe_model = True- break- if is_moe_model:- logger.info(- "The model appears to be an MoE model. "- "Using more samples to help generate a better auto-scheme recipe."- )- nsamples = 64- else:- nsamples = 16- seqlen = auto_scheme.seqlen if auto_scheme.seqlen is not None else 256+ nsamples = 16++ # seqlen: API explicit setting takes priority; otherwise use MoE-aware default+ if auto_scheme.seqlen is not None:+ seqlen = auto_scheme.seqlen+ else:+ seqlen = 128 if is_moe_model else 256if auto_scheme.batch_size is not None:batch_size = auto_scheme.batch_sizeelse:- if auto_scheme.low_gpu_mem_usage:- batch_size = 8+ _env_batch_size = _envs.AR_AUTO_SCHEME_BATCH_SIZE+ if _env_batch_size is not None:+ batch_size = _env_batch_sizeelse:- batch_size = 1+ if auto_scheme.low_gpu_mem_usage:+ batch_size = 8+ else:+ batch_size = 1# ------------------------------------------------------------------ ## Multimodal calibration: ``batch_size`` must be 1 because image #@@ -1412,9 +1585,12 @@ def _gen_layer_config(pbar_cnt = 0need_weight_grad = Falseneed_imatrix = False # only trigger it for gguf q-k quant+ effective_scheme_num = 0+ block_num = len(block_name)for index, scheme in enumerate(schemes):if check_bf16_scheme(scheme):continue+ effective_scheme_num += 1if isinstance(scheme, str):scheme = asdict(preset_name_to_scheme(scheme))elif isinstance(scheme, QuantizationScheme):@@ -1429,8 +1605,70 @@ def _gen_layer_config(pbar_cnt += nsamplesif auto_scheme.low_gpu_mem_usage:pbar_cnt += len(block_name) * 2 * ((nsamples + batch_size - 1) // batch_size) # forward backward++ # Formula-style step log for paper/debug readability.+ # In low_gpu mode, one calibration mini-batch uses block-wise forward+backward replay.+ # so base_step_per_scheme = block_num * 2.+ base_total_steps = effective_scheme_num * block_num * 2+ logger.info(f"AutoScheme steps(total)={base_total_steps}")+ logger.info(+ "AutoScheme steps variables: "+ f"scheme_num={effective_scheme_num}, block_num={block_num}, "+ f"nsamples={nsamples}, batch_size={batch_size}"+ )+ logger.info(+ "AutoScheme: nsamples/batch_size can be overridden via env vars "+ "AR_AUTO_SCHEME_NSAMPLES / AR_AUTO_SCHEME_BATCH_SIZE "+ "(e.g. `export AR_AUTO_SCHEME_NSAMPLES=1` for a quick run); "+ "see docs/environments.md for details."+ )+ if auto_scheme.low_gpu_mem_usage:+ n_batches = (nsamples + batch_size - 1) // batch_size+ logger.info(+ "AutoScheme steps expanded(low_gpu): "+ "total_steps = scheme_num * block_num * 2(forward+backward) * n_batches = "+ f"{effective_scheme_num} * {block_num} * 2 * {n_batches} = {pbar_cnt}"+ )+ else:+ logger.info(+ "AutoScheme steps expanded(full_backward): "+ f"total_steps = scheme_num * nsamples = {effective_scheme_num} * {nsamples} = {pbar_cnt}"+ )shared_layers = parse_shared_layers(model, auto_scheme.shared_layers)+ # Auto-group MoE expert layers so DP treats all experts in one block as a unit.+ if is_moe_model:+ expert_groups = build_expert_groups(model, quant_layer_names, fixed_layer_scheme)+ if expert_groups:+ shared_layers = merge_lists_unionfind(shared_layers + expert_groups)++ # Pre-compute per-key weight numel (for loss/elem display). Mirrors the+ # shared_layers grouping used in the scoring loop so keys match total_scores.+ _dp_names = set(quant_layer_names) - set(fixed_layer_scheme.keys())+ _shared_seen: set[str] = set()+ layer_numel: dict[str, int] = {}+ for _share_layer in shared_layers:+ _nl = [n for n in _share_layer if n in _dp_names]+ if not _nl:+ continue+ _total = 0+ for _n in _nl:+ _m = get_module(model, _n)+ _np = _m.weight.numel() if hasattr(_m, "weight") and _m.weight is not None else 0+ if _np == 0 and hasattr(_m, "_cached_weight_numel"):+ _np = _m._cached_weight_numel+ _total += _np+ _shared_seen.add(_n)+ layer_numel[_nl[0]] = _total+ for _n in _dp_names:+ if _n in _shared_seen:+ continue+ _m = get_module(model, _n)+ _np = _m.weight.numel() if hasattr(_m, "weight") and _m.weight is not None else 0+ if _np == 0 and hasattr(_m, "_cached_weight_numel"):+ _np = _m._cached_weight_numel+ layer_numel[_n] = _np+options_scores = []if need_imatrix:@@ -1455,6 +1693,8 @@ def _gen_layer_config(pbar = tqdm(total=pbar_cnt, desc="Generating AutoScheme")for index, scheme in enumerate(schemes):+ scheme_tag = f"{index + 1}/{len(schemes)} {_scheme_short_name(scheme)}"+ logger.info(f"AutoScheme transition: switch to scheme {index + 1}/{len(schemes)} ({scheme})")apply_quant_scheme(model, quant_layer_names=quant_layer_names, fixed_layer_scheme=fixed_layer_scheme, scheme=scheme)@@ -1487,6 +1727,7 @@ def _gen_layer_config(is_vlm=is_vlm,force_mllm=force_mllm,model_name=model_name,+ scheme_tag=scheme_tag,)# Track peak RAM after each scheme scoringmemory_monitor.update()@@ -1514,6 +1755,10 @@ def _gen_layer_config(else:total_scores[key] = [item]options_scores.append(options_total_loss)+ logger.info(+ f"AutoScheme transition: scheme {index + 1}/{len(schemes)} "+ f"scoring finished (total_loss={options_total_loss:.6f})"+ )clear_memory(device_list=device_list)# Remove hooks and restore original weights from disk for final bit-budget computations@@ -1589,7 +1834,6 @@ def _gen_layer_config(In all cases, the selected scheme must not exceed the total bit budget(i.e., embedding bits + fixed bits + min DP bits <= target_params_cnt)."""- import mathif has_tied_lm_head:if target_bits > 6:@@ -1673,6 +1917,8 @@ def _gen_layer_config(layer_scheme = options[item[1]]for layer_name in layer_names:layer_config[layer_name] = asdict(layer_scheme)+ _log_scheme_loss_matrix(total_scores, options, block_name, model=model, layer_numel=layer_numel)+ _describe_layer_config(layer_config, total_scores, options, block_name, model=model)if model_name is not None:model = Nonedel model@@ -1722,6 +1968,11 @@ def gen_layer_config(processor=None,**kwargs,):+ """Public AutoScheme entry.++ This wrapper performs model loading/dispatch and environment preparation,+ then delegates to `_gen_layer_config` for staged scoring + DP selection.+ """model_name = Noneis_vlm = Falseif isinstance(model, str):@@ -1747,6 +1998,9 @@ def gen_layer_config(except Exception: # noqa: BLE001is_vlm = False+ from auto_round.modeling.fused_moe.replace_modules import materialize_model_++ materialize_model_(model)# ---- Vision-tower scoring requires a full backward ---- ## ``model_forward_low_gpu`` only walks the language tower (it uses# ``get_block_names(model)[0]``, which excludes vision blocks by default)@@ -1807,6 +2061,9 @@ def gen_layer_config(# ``use_reentrant=False`` (saved-tensor-hooks impl) does not have this# restriction.def _enable_gc(mod):+ """Enable gradient checkpointing on ``mod`` with ``use_reentrant=False`` if supported+ (see rationale above); no-op if the module doesn't support checkpointing.+ """if not getattr(mod, "supports_gradient_checkpointing", False):returntry:diff --git a/auto_round/auto_scheme/utils.py b/auto_round/auto_scheme/utils.pyindex 8a3ae951d..83e74fe57 100644--- a/auto_round/auto_scheme/utils.py+++ b/auto_round/auto_scheme/utils.py@@ -11,27 +11,34 @@# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.# See the License for the specific language governing permissions and# limitations under the License.+import logging+import mathimport refrom dataclasses import asdict, fields-from typing import Iterable, Union+from typing import Iterable, Optional, Unionimport torchfrom accelerate import dispatch_model, infer_auto_device_mapfrom accelerate.utils import get_balanced_memory-from auto_round.schemes import QuantizationScheme, preset_name_to_scheme+from auto_round.logger import logger+from auto_round.schemes import QuantizationScheme, preset_name_to_scheme, scheme_to_preset_namefrom auto_round.utils import (DEVICE_ENVIRON_VARIABLE_MAPPING,SUPPORTED_LAYER_TYPES,check_to_quantized,get_block_names,get_layer_features,+ get_lm_head_name,get_module,is_hpex_available,normalize_no_split_modules,parse_available_devices,)+_EXPERT_ID_PATTERN = re.compile(r"^(.*?\.experts\.\d+)(?:\.|$)")+_ZERO_EPS = 1e-12+def apply_quant_scheme(model: torch.nn.Module,@@ -223,15 +230,22 @@ def compute_layer_bits(def merge_lists_unionfind(list_of_lists):+ """Merge lists that share at least one common element using union-find, returning the+ resulting disjoint groups (each input list's elements are unioned together).+ """parent = {}def find(x):+ """Return the representative (root) element of ``x``'s union-find set, with path+ compression.+ """while parent[x] != x:parent[x] = parent[parent[x]]x = parent[x]return xdef union(x, y):+ """Merge the sets containing ``x`` and ``y``."""root_x, root_y = find(x), find(y)if root_x != root_y:parent[root_y] = root_x@@ -325,3 +339,558 @@ def parse_shared_layers(model: torch.nn.Module, shared_patterns: Iterable[Iterabmatched_groups.append(sorted(matched_layers))matched_groups = merge_lists_unionfind(matched_groups)return matched_groups+++def _expert_key_from_layer_name(layer_name: str) -> Optional[str]:+ """Map one MoE-related linear layer to a unique expert key.++ Gate/up/down projections belonging to the same expert should map to one key.+ """+ match = _EXPERT_ID_PATTERN.search(layer_name)+ if match:+ return match.group(1)++ if ".moe." in layer_name:+ # Fallback for models that expose MoE expert layers without+ # `.experts.<id>` naming. Best-effort collapse by dropping the last+ # projection suffix (gate/up/down/etc.).+ parts = layer_name.split(".")+ if len(parts) > 1:+ return ".".join(parts[:-1])+ return layer_name++ return None+++def _short_summary_name(layer_name: str) -> str:+ """Shorten ``layer_name`` to its last two dotted segments when the final segment is a+ numeric index (e.g. expert id), otherwise return it unchanged.+ """+ parts = layer_name.rsplit(".", 2)+ if len(parts) >= 2 and parts[-1].isdigit():+ return ".".join(parts[-2:])+ return layer_name+++def _scheme_short_name(scheme) -> str:+ """Concise human-readable label for a quantization scheme, e.g. 'MXFP4' or 'W4A16'."""+ if isinstance(scheme, str):+ return scheme+ if isinstance(scheme, QuantizationScheme):+ preset_name = scheme_to_preset_name(scheme)+ if preset_name:+ return preset_name+ scheme = asdict(scheme)+ if isinstance(scheme, dict):+ try:+ preset_name = scheme_to_preset_name(QuantizationScheme.from_dict(scheme))+ if preset_name:+ return preset_name+ except Exception: # noqa: BLE001+ pass+ data_type = str(scheme.get("data_type", "")).lower()+ bits = scheme.get("bits", 16)+ act_bits = scheme.get("act_bits", 16)+ if data_type.startswith("mx_fp"):+ return f"MXFP{bits}"+ if data_type.startswith("nv_fp"):+ return f"NVFP{bits}"+ if data_type.startswith("gguf"):+ return data_type.upper()+ if act_bits >= 16:+ return f"W{bits}"+ return f"W{bits}A{act_bits}"+ return str(scheme)+++def build_expert_groups(+ model: torch.nn.Module,+ quant_layer_names: list[str],+ fixed_layer_scheme: dict[str, dict],+) -> list[list[str]]:+ """Auto-detect MoE expert layers and group all experts in the same block together.++ All expert projection layers (gate/up/down etc.) across all experts within a single+ transformer block are grouped into one list, so that DP treats them as a single entity+ with summed loss and summed numel.++ Returns a list of groups, each group is a list of layer names.+ """+ block_names = get_block_names(model, quant_vision=False)+ block_names = [item for sublist in block_names for item in sublist]+ block_prefixes = [(name, name + ".") for name in sorted(block_names, key=len, reverse=True)]++ dp_names = set(quant_layer_names) - set(fixed_layer_scheme.keys())+ # Collect expert layers per block+ block_expert_layers: dict[str, list[str]] = {}+ for layer_name in dp_names:+ if _expert_key_from_layer_name(layer_name) is None:+ continue+ for block_name, block_prefix in block_prefixes:+ if layer_name.startswith(block_prefix):+ block_expert_layers.setdefault(block_name, []).append(layer_name)+ break++ groups = []+ for block_name in block_names:+ layers = block_expert_layers.get(block_name)+ if layers and len(layers) > 1:+ groups.append(sorted(layers))+ return groups+++def _fill_inactive_expert_scores(scores_dict: dict[str, list[float]], block_names: list[str]):+ """Fill inactive experts with the average loss of active experts in each block.++ Inactive expert means all its tracked projection losses are zero.+ """+ block_prefixes = [(name, name + ".") for name in sorted(block_names, key=len, reverse=True)]+ block_expert_stats: dict[str, dict[str, dict[str, object]]] = {}++ for layer_name, values in scores_dict.items():+ layer_loss = float(values[1])+ if not math.isfinite(layer_loss):+ continue++ matched_block = None+ for block_name, block_prefix in block_prefixes:+ if layer_name == block_name or layer_name.startswith(block_prefix):+ matched_block = block_name+ break+ if matched_block is None:+ continue++ expert_key = _expert_key_from_layer_name(layer_name)+ if expert_key is None:+ continue++ expert_stats = block_expert_stats.setdefault(matched_block, {}).setdefault(+ expert_key,+ {"layers": [], "has_active": False},+ )+ expert_stats["layers"].append(layer_name)++ if abs(layer_loss) >= _ZERO_EPS:+ expert_stats["has_active"] = True++ for block_name, expert_stats_map in block_expert_stats.items():+ active_expert_avg_losses = []+ for expert_stats in expert_stats_map.values():+ if not expert_stats["has_active"]:+ continue+ active_losses = [+ float(scores_dict[layer_name][1])+ for layer_name in expert_stats["layers"]+ if abs(float(scores_dict[layer_name][1])) >= _ZERO_EPS+ ]+ if not active_losses:+ continue+ active_expert_avg_losses.append(sum(active_losses) / len(active_losses))+ if not active_expert_avg_losses:+ continue+ fill_value = sum(active_expert_avg_losses) / len(active_expert_avg_losses)+ for _, expert_stats in expert_stats_map.items():+ if expert_stats["has_active"]:+ continue+ for layer_name in expert_stats["layers"]:+ scores_dict[layer_name][1] = fill_value+++def _log_score_summary_by_block_and_nonblock(+ scores_dict: dict[str, list[float]],+ block_names: list[str],+ model=None,+ scheme_tag: Optional[str] = None,+ summary_stage: Optional[str] = None,+):+ """Log a per-block (and non-block) breakdown of ``scores_dict`` losses at debug level."""+ if not scores_dict:+ logger.info("AutoScheme score summary: empty.")+ return++ head_name = get_lm_head_name(model) if model is not None else None+ if head_name is not None and head_name.endswith(".orig_layer"):+ head_name = head_name[: -len(".orig_layer")]+ if head_name is None and "lm_head" in scores_dict:+ head_name = "lm_head"++ block_prefixes = [(name, name + ".") for name in sorted(block_names, key=len, reverse=True)]+ block_stats: dict[str, list[float]] = {name: [0.0, 0.0] for name in block_names}+ non_block_items: list[tuple[str, float]] = []++ for layer_name, values in scores_dict.items():+ layer_loss = float(values[1])+ if not math.isfinite(layer_loss):+ continue+ matched_block = None+ for block_name, block_prefix in block_prefixes:+ if layer_name == block_name or layer_name.startswith(block_prefix):+ matched_block = block_name+ break+ if matched_block is None:+ if layer_name != head_name:+ non_block_items.append((layer_name, layer_loss))+ else:+ block_stats[matched_block][0] += layer_loss+ block_stats[matched_block][1] += 1++ tag = f"[{scheme_tag}] " if scheme_tag else ""+ stage_str = f" ({summary_stage})" if summary_stage else ""+ logger.debug("AutoScheme %sblock loss summary%s:", tag, stage_str)+ logger.debug("AutoScheme | block | avg_loss |")++ for block_name in block_names:+ total_loss, cnt = block_stats.get(block_name, [0.0, 0.0])+ avg_loss = 0.0 if cnt <= 0 else total_loss / cnt+ logger.debug("AutoScheme | %s | %.6f |", _short_summary_name(block_name), avg_loss)++ if head_name is not None:+ head_loss = None+ if head_name in scores_dict:+ head_loss = float(scores_dict[head_name][1])+ head_avg = "N/A" if head_loss is None or not math.isfinite(head_loss) else f"{head_loss:.6f}"+ logger.debug("AutoScheme | %s | %s |", head_name, head_avg)++ if non_block_items:+ non_block_items.sort(key=lambda x: x[0])+ for layer_name, layer_loss in non_block_items:+ logger.info("AutoScheme non_block=%s loss=%.6f", layer_name, layer_loss)+ else:+ logger.info("AutoScheme non_block loss: none")+++def _collect_current_scores(model):+ """Snapshot the current ``mix_score`` of every wrapped module into a+ ``{name: [0.0, loss]}`` dict.+ """+ scores_dict = {}+ for name, module in model.named_modules():+ if not hasattr(module, "mix_score"):+ continue+ loss = float(module.mix_score)+ if not math.isfinite(loss):+ continue+ scores_dict[name] = [0.0, loss]+ return scores_dict+++_BATCH_SUMMARY_LOG_INTERVAL = 10+++def _log_batch_avg_loss(model, batch_idx: int, pbar=None, block_names=None, total_batches=None, scheme_tag=None):+ """Log the running average ``mix_score`` after processing one calibration batch."""+ is_last_batch = total_batches is not None and batch_idx == total_batches+ should_log = logger.isEnabledFor(logging.DEBUG) and (is_last_batch or batch_idx % _BATCH_SUMMARY_LOG_INTERVAL == 0)+ if not should_log:+ return++ total_loss = 0.0+ layer_cnt = 0+ for _, module in model.named_modules():+ if not hasattr(module, "mix_score"):+ continue+ loss = float(module.mix_score)+ if not math.isfinite(loss):+ continue+ total_loss += loss+ layer_cnt += 1++ avg_loss = 0.0 if layer_cnt == 0 else total_loss / layer_cnt+ tag = f"[{scheme_tag}] " if scheme_tag else ""+ batch_str = f"{batch_idx}/{total_batches}" if total_batches is not None else str(batch_idx)+ msg = f"AutoScheme {tag}cumulative batch {batch_str} avg_loss={avg_loss:.6f} layers={layer_cnt}"+ if pbar is not None:+ pbar.write(msg)+ logger.debug(msg)++ if block_names:+ scores_dict = _collect_current_scores(model)+ if scores_dict:+ tag = f"[{scheme_tag}] " if scheme_tag else ""+ batch_str = f"{batch_idx}/{total_batches}" if total_batches is not None else str(batch_idx)+ if is_last_batch:+ logger.debug("AutoScheme %scumulative batch %s block summary skipped (same as final)", tag, batch_str)+ else:+ logger.debug("AutoScheme %scumulative batch %s block summary:", tag, batch_str)+ _log_score_summary_by_block_and_nonblock(+ scores_dict,+ block_names,+ model=model,+ scheme_tag=scheme_tag,+ summary_stage="cumulative",+ )+++def _build_layer_config_header_rows(columns: list[str]) -> list[list[str]]:+ """Build a compact two-row header for the layer-config matrix.++ The first row keeps a shared prefix for each grouped set of columns (for+ example ``mlp`` or ``self_attn``), while the second row keeps the leaf+ suffix (for example ``down_proj``).+ """+ if not columns:+ return [["block"], [""]]++ leaves = []+ first_row = []+ prev_prefix = None+ for column in columns:+ parts = column.split(".")+ if len(parts) > 1:+ prefix = ".".join(parts[:-1])+ leaves.append(parts[-1])+ if prefix == prev_prefix:+ first_row.append("")+ else:+ first_row.append(prefix)+ prev_prefix = prefix+ else:+ leaves.append(column)+ first_row.append("")+ prev_prefix = None++ header_rows = [["block"] + first_row]+ header_rows.append([""] + leaves)+ return header_rows+++def _log_scheme_loss_matrix(total_scores, options, block_names, model=None, layer_numel=None):+ """For every scheme in *options* log a block×layer matrix of per-element losses.++ Cells show ``loss / weight.numel()`` in scientific notation with three+ decimal places (e.g. ``1.234E-05``). Expert layers are aggregated into an+ ``experts`` column that shows the average per-element loss across all active+ expert layers in each block.+ """+ if not total_scores or not options:+ return++ head_name = get_lm_head_name(model) if model is not None else None+ if head_name is not None and head_name.endswith(".orig_layer"):+ head_name = head_name[: -len(".orig_layer")]+ block_prefixes = [(name, name + ".") for name in sorted(block_names, key=len, reverse=True)]++ # Classify every key in total_scores into a (block, short_name) pair or expert bucket.+ # lm_head is treated as a regular non-block row (not skipped).+ block_col_keys: dict[str, dict[str, str]] = {name: {} for name in block_names}+ block_expert_keys: dict[str, list[str]] = {name: [] for name in block_names}+ other_keys: list[str] = [] # keys not belonging to any block (e.g. lm_head)+ for key in sorted(total_scores):+ matched_block = None+ for block_name, block_prefix in block_prefixes:+ if key == block_name or key.startswith(block_prefix):+ matched_block = block_name+ break+ if matched_block is None:+ other_keys.append(key)+ continue+ if _expert_key_from_layer_name(key) is not None:+ block_expert_keys[matched_block].append(key)+ else:+ short_name = key[len(matched_block) + 1 :] if key.startswith(matched_block + ".") else key+ block_col_keys[matched_block][short_name] = key++ columns = sorted({col for row in block_col_keys.values() for col in row.keys()})+ if not columns:+ columns = ["layer"]+ has_expert_layers = any(block_expert_keys.get(b) for b in block_names)++ block_display_names = [_short_summary_name(b) for b in block_names]+ header_rows = _build_layer_config_header_rows(columns)++ # Fixed cell width for scientific-notation values: "1.234E-05" = 9 chars.+ _CELL_W = 9+ widths: dict[str, int] = {"block": max(8, max((len(b) for b in block_display_names), default=5))}+ for col in columns:+ leaf = col.split(".")[-1]+ widths[col] = max(len(leaf), _CELL_W)+ if has_expert_layers:+ widths["experts"] = max(len("experts"), _CELL_W)+ # Ensure the first column of each prefix group is wide enough for the prefix text in header row 1.+ for col, prefix in zip(columns, header_rows[0][1:]):+ if prefix:+ widths[col] = max(widths[col], len(prefix))++ header_keys = ["block"] + columns + (["experts"] if has_expert_layers else [])+ header_vals = [header_rows[0][0]] + header_rows[0][1:]+ header_vals_row2 = ([""]) + (header_rows[1][1:] if len(header_rows) > 1 else [""] * len(columns))+ if has_expert_layers:+ header_vals.append("experts")+ header_vals_row2.append("")++ def _fmt_row(values: list[str], keys: list[str]) -> str:+ return "|".join(v.ljust(widths[k]) for v, k in zip(values, keys))++ sep = "|".join("-" * widths[k] for k in header_keys)++ for scheme_idx, scheme in enumerate(options):+ scheme_name = _scheme_short_name(scheme)+ block_loss_cells: dict[str, dict[str, str]] = {name: {} for name in block_names}+ block_expert_avg: dict[str, str] = {}++ _numel = layer_numel or {}+ for block_name in block_names:+ for short_name, key in block_col_keys[block_name].items():+ loss = next((item[2] for item in total_scores.get(key, []) if item[0] == scheme_idx), None)+ if loss is not None:+ n = _numel.get(key, 0)+ block_loss_cells[block_name][short_name] = f"{loss / n:.3E}" if n > 0 else f"{loss:.3E}"+ else:+ block_loss_cells[block_name][short_name] = "-"++ expert_losses_per_elem: list[float] = []+ for ek in block_expert_keys.get(block_name, []):+ el = next((item[2] for item in total_scores.get(ek, []) if item[0] == scheme_idx), None)+ if el is not None:+ n = _numel.get(ek, 0)+ expert_losses_per_elem.append(el / n if n > 0 else el)+ if expert_losses_per_elem:+ block_expert_avg[block_name] = f"{sum(expert_losses_per_elem) / len(expert_losses_per_elem):.3E}"+ else:+ block_expert_avg[block_name] = "-"++ logger.debug("AutoScheme [%s] per-op loss/elem matrix:", scheme_name)+ logger.debug(" %s", _fmt_row(header_vals, header_keys))+ logger.debug(" %s", _fmt_row(header_vals_row2, header_keys))+ logger.debug(" %s", sep)+ for block_name, block_display_name in zip(block_names, block_display_names):+ row_vals = [block_display_name]+ for col in columns:+ row_vals.append(block_loss_cells.get(block_name, {}).get(col, "-"))+ if has_expert_layers:+ row_vals.append(block_expert_avg.get(block_name, "-"))+ logger.debug(" %s", _fmt_row(row_vals, header_keys))+ if other_keys:+ for key in other_keys:+ loss = next((item[2] for item in total_scores.get(key, []) if item[0] == scheme_idx), None)+ if loss is not None:+ n = _numel.get(key, 0)+ loss_str = f"{loss / n:.3E}" if n > 0 else f"{loss:.3E}"+ else:+ loss_str = "-"+ logger.debug(" %s -> %s", _short_summary_name(key), loss_str)+++def _describe_layer_config(layer_config, total_scores, options, block_names, model=None):+ """Log final ``layer_config`` as a block-row / layer-column matrix.++ Cells show the selected scheme name only; the detailed loss delta is no+ longer printed to keep the table compact.+ """++ if not layer_config:+ logger.info("AutoScheme final layer_config: empty.")+ return++ head_name = get_lm_head_name(model) if model is not None else None+ if head_name is not None and head_name.endswith(".orig_layer"):+ head_name = head_name[: -len(".orig_layer")]+ block_prefixes = [(name, name + ".") for name in sorted(block_names, key=len, reverse=True)]++ block_cells: dict[str, dict[str, str]] = {name: {} for name in block_names}+ block_expert_scheme_counts: dict[str, dict[str, int]] = {name: {} for name in block_names}+ block_expert_layers: dict[str, dict[str, list[str]]] = {name: {} for name in block_names}+ other_rows: list[tuple[str, str]] = []++ for layer_name in sorted(layer_config):+ cell = _scheme_short_name(layer_config[layer_name])++ if head_name is not None and layer_name == head_name:+ continue++ matched_block = None+ for block_name, block_prefix in block_prefixes:+ if layer_name == block_name or layer_name.startswith(block_prefix):+ matched_block = block_name+ break++ expert_key = _expert_key_from_layer_name(layer_name)+ if matched_block is None:+ other_rows.append((layer_name, cell))+ continue++ if expert_key is not None:+ block_expert_layers[matched_block].setdefault(expert_key, []).append(layer_name)+ continue++ short_name = layer_name[len(matched_block) + 1 :] if layer_name.startswith(matched_block + ".") else layer_name+ block_cells[matched_block][short_name] = cell++ for block_name, expert_map in block_expert_layers.items():+ if not expert_map:+ continue+ counts: dict[str, int] = {}+ for layer_names in expert_map.values():+ scheme_set = {_scheme_short_name(layer_config[layer_name]) for layer_name in layer_names}+ rep_scheme = (+ next(iter(sorted(scheme_set))) if len(scheme_set) == 1 else f"MIXED({', '.join(sorted(scheme_set))})"+ )+ counts[rep_scheme] = counts.get(rep_scheme, 0) + 1+ block_expert_scheme_counts[block_name] = counts++ columns = sorted({col for row in block_cells.values() for col in row.keys()})+ if not columns:+ columns = ["layer"]++ expert_text_by_block: dict[str, str] = {}+ for block_name in block_names:+ expert_counts = block_expert_scheme_counts.get(block_name, {})+ if not expert_counts:+ expert_text_by_block[block_name] = "-"+ continue+ expert_total = len(block_expert_layers.get(block_name, {}))+ expert_text_by_block[block_name] = ", ".join(+ f"{count}/{expert_total} {scheme_name}"+ for scheme_name, count in sorted(expert_counts.items(), key=lambda item: (-item[1], item[0]))+ )++ has_expert_layers = any(block_expert_layers.get(block_name) for block_name in block_names)++ block_display_names = [_short_summary_name(block_name) for block_name in block_names]+ header_rows = _build_layer_config_header_rows(columns)++ # Width is driven by the leaf name and cell content — NOT the full dotted column name,+ # so columns stay tight (e.g. "down_proj" width, not "mlp.down_proj" width).+ widths: dict[str, int] = {"block": max(8, max((len(b) for b in block_display_names), default=5))}+ for col in columns:+ leaf = col.split(".")[-1]+ max_cell_len = max((len(block_cells.get(b, {}).get(col, "-")) for b in block_names), default=1)+ widths[col] = max(len(leaf), max_cell_len)+ if has_expert_layers:+ widths["experts"] = max(len("experts"), max((len(v) for v in expert_text_by_block.values()), default=1))+ # Ensure the first column of each prefix group is wide enough for the prefix text in header row 1.+ for col, prefix in zip(columns, header_rows[0][1:]):+ if prefix:+ widths[col] = max(widths[col], len(prefix))++ def _fmt_row(values: list[str], keys: list[str]) -> str:+ return "|".join(v.ljust(widths[k]) for v, k in zip(values, keys))++ header_keys = ["block"] + columns + (["experts"] if has_expert_layers else [])+ header_vals = [header_rows[0][0]] + header_rows[0][1:]+ if len(header_rows) > 1:+ header_vals_row2 = [header_rows[1][0]] + header_rows[1][1:]+ else:+ header_vals_row2 = [""] + [""] * len(columns)++ logger.info("AutoScheme final layer_config matrix:")+ logger.info("AutoScheme note: cell=`scheme`.")+ logger.info(" %s", _fmt_row(header_vals, header_keys))+ logger.info(" %s", _fmt_row(header_vals_row2, header_keys))+ logger.info(" %s", "|".join("-" * widths[k] for k in header_keys))++ for block_name, block_display_name in zip(block_names, block_display_names):+ row_vals = [block_display_name]+ for col in columns:+ row_vals.append(block_cells.get(block_name, {}).get(col, "-"))+ if has_expert_layers:+ row_vals.append(expert_text_by_block.get(block_name, "-"))+ logger.info(" %s", _fmt_row(row_vals, header_keys))++ if head_name is not None and head_name in layer_config:+ logger.info(" %s is using %s", head_name, _scheme_short_name(layer_config[head_name]))++ if other_rows:+ logger.info(" other:")+ for layer_name, cell in other_rows:+ logger.info(" %s -> %s", layer_name, cell)diff --git a/auto_round/envs.py b/auto_round/envs.pyindex a6df79435..714eaacc6 100644--- a/auto_round/envs.py+++ b/auto_round/envs.py@@ -24,6 +24,8 @@ if TYPE_CHECKING:AUTO_ROUND_CACHE: Optional[str] = NoneAUTO_ROUND_GGUF_AUTO_UPDATE: bool = FalseLLAMA_CPP_ROOT: Optional[str] = None+ AR_AUTO_SCHEME_NSAMPLES: Optional[int] = None+ AR_AUTO_SCHEME_BATCH_SIZE: Optional[int] = Nonedef _get_optional_positive_int_env(name: str) -> Optional[int]:@@ -75,6 +77,14 @@ environment_variables: dict[str, Callable[[], Any]] = {"AUTO_ROUND_GGUF_AUTO_UPDATE": lambda: os.getenv("AUTO_ROUND_GGUF_AUTO_UPDATE", "0").lower()in ("1", "true", "yes", "on"),"LLAMA_CPP_ROOT": lambda: os.getenv("LLAMA_CPP_ROOT", None),+ # Controls the default number of calibration samples used by AutoScheme scoring+ # when ``AutoScheme.nsamples`` is not explicitly set.+ # When unset, AutoScheme uses 16.+ "AR_AUTO_SCHEME_NSAMPLES": lambda: _get_optional_positive_int_env("AR_AUTO_SCHEME_NSAMPLES"),+ # Controls the default batch size used by AutoScheme scoring+ # when ``AutoScheme.batch_size`` is not explicitly set.+ # When unset, AutoScheme uses its built-in heuristic (8 for low GPU memory mode, 1 for normal mode).+ "AR_AUTO_SCHEME_BATCH_SIZE": lambda: _get_optional_positive_int_env("AR_AUTO_SCHEME_BATCH_SIZE"),}diff --git a/auto_round/utils/model.py b/auto_round/utils/model.pyindex 1cd28df7a..120055f99 100644--- a/auto_round/utils/model.py+++ b/auto_round/utils/model.py@@ -1652,10 +1652,13 @@ def safe_device_move_with_meta_handling(def is_moe_model(model: torch.nn.Module) -> bool:+ """Heuristically detect whether ``model`` is a MoE architecture by scanning its config+ for "moe"/"expert" markers or by checking module names for "expert".+ """if hasattr(model, "config") and hasattr(model.config, "to_dict"):- for key in model.config.to_dict().keys():- if "moe" in key or "expert" in key:- return True+ config_str = str(model.config.to_dict()).lower()+ if "moe" in config_str or "expert" in config_str:+ return Truefor n, m in model.named_modules():if "expert" in n:return Truediff --git a/docs/auto_scheme_acc.md b/docs/auto_scheme_acc.mdindex ad404c313..8b6981831 100644--- a/docs/auto_scheme_acc.md+++ b/docs/auto_scheme_acc.md@@ -3,42 +3,31 @@We tested with a fake model because the main branch currently has layer name mismatches between Transformers and GGUF.~~~bash-python3 -m auto_round Qwen/Qwen3-8B --options "gguf:q2_k_s,gguf:q4_k_s" --target_bits 3.5 --ignore_scale_zp_bits --iters 0 --format fake --output_dir "./test_gguf" --tasks mmlu,lambada_openai+python3 -m auto_round Qwen/Qwen3-8B --options "gguf:q2_k_s,gguf:q4_k_s" --target_bits 3.5 --ignore_scale_zp_bits --iters 0 --format fake --output_dir "./test_gguf"~~~+eval~~~- vllm serve ./test_gguf/Qwen3-8B-w2g16/ --port 8000 --max-model-len 8192 --host 127.0.0.1 --served-model-name qwen-evalscope eval --model qwen --api-url http://127.0.0.1:8000/v1 --api-key EMPTY --datasets math_500 gpqa_diamoid mmlu_pro --eval-batch-size 32-~~~--| evalscope options q2ks,q4ks avgbits 3.5, ignore_scale_zp | math_500 | gpqa_diamond | mmlu_pro |-|----------------------------------------------------------|------------|---------------|----------|-| qwen3-8b bf16 | 0.79 | 0.4242 | 0.6934 |-| qwen3-8b (AR<=0.13.1) iters 0 | 0.76/0.768 | 0.3434/0.3687 | 0.6419 |-| qwen3-8b (AR>0.13.1) iters 0 | 0.782 | 0.4293 | 0.6763 |--| evalscope, options q2ks,q4ks avgbits 3.5, ignore_scale_zp | math_500 | gpqa_diamond | mmlu_pro |-|-----------------------------------------------------------|----------|--------------|----------|-| qwen3.5-4b bf16 | 0.48 | 0.4192 | 0.5891 |-| qwen3.5-4b (AR<=0.13.1) iters 0 | 0.228 | 0.298 | 0.4583 |-| pr branch (AR>0.13.1) iters 0 | 0.432 | 0.4444 | 0.5922 |--[//]: # (| options W2A16G64,W4A16 avgbits 3.5 ignore_scale_zp | math_500 | gpqa_diamond | mmlu_pro |)--[//]: # (| -------------------------------------------------- | -------- | ------------ | -------- |)+# Start vLLM serve+vllm serve ./test_gguf/Qwen3-8B-w2g16/ --port 8000 --max-model-len 8192 --host 127.0.0.1 --served-model-name qwen3-8b-[//]: # (| qwen3-8b bf16 | 0.79 | 0.4242 | 0.6934 |)--[//]: # (| qwen3-8b main branch | 0.738 | 0.2778 | 0.576 |)--[//]: # (| main branch iters 200 enable_alg_ext | 0.766 | 0.3434 | |)--[//]: # (| pr branch | 0.72 | 0.2677 | - |)--[//]: # (| pr branch + iters 200 enable_alg_ext | 0.742 | 0.3232 | - |)+# Perform five repeated evaluations on math_500 and gpqa_diamond.+evalscope eval --model qwen3-8b --api-url http://127.0.0.1:8000/v1 --api-key EMPTY --datasets math_500 gpqa_diamond --eval-batch-size 64 --generation-config "{\"n\":5}"+# Evaluate mmlu_pro+evalscope eval --model qwen3-8b --api-url http://127.0.0.1:8001/v1 --api-key EMPTY --datasets mmlu_pro --eval-batch-size 128+~~~+| evalscope, options q2ks,q4ks avgbits 3.5, ignore_scale_zp | math_500 (repeat=5) | gpqa_diamond (repeat=5) | mmlu_pro |+|-----------------------------------------------------------|---------------------|-------------------------|----------|+| qwen3-8b: BF16 | 0.8083 | 0.4586 | 0.6934 |+| qwen3-8b: Fake quantized model | 0.7924 | 0.4313 | 0.6751 |++| evalscope, options q2ks,q4ks avgbits 3.5, ignore_scale_zp | math_500 (repeat=5) | gpqa_diamond (repeat=5) | mmlu_pro |+|-----------------------------------------------------------|---------------------|-------------------------|----------|+| qwen3.5-4b: BF16 | 0.5365 | 0.3263 | 0.5891 |+| qwen3.5-4b: Fake quantized model | 0.505 | 0.3172 | 0.5948 |### Other resultsdiff --git a/docs/environments.md b/docs/environments.mdindex 89d9bf100..296bf6c28 100644--- a/docs/environments.md+++ b/docs/environments.md@@ -121,6 +121,26 @@ export AR_DYNAMO_CACHE_SIZE_LIMIT=32export AR_MODEL_FREE_SHARD_PARALLELISM=4```+### AR_AUTO_SCHEME_NSAMPLES+- **Description**: Controls the default number of calibration samples used by AutoScheme scoring when `AutoScheme.nsamples` is not explicitly set.+- **Default**: unset → 16+- **Valid Values**: any positive integer, e.g. `8`, `16`, `32`+- **Usage**: Set this to override the automatic sample-count selection for AutoScheme++```bash+export AR_AUTO_SCHEME_NSAMPLES=1 # set 1 for quick execution+```++### AR_AUTO_SCHEME_BATCH_SIZE+- **Description**: Controls the default batch size used by AutoScheme scoring when `AutoScheme.batch_size` is not explicitly set.+- **Default**: unset → built-in heuristic applies (8 for low GPU memory mode, 1 for normal mode)+- **Valid Values**: any positive integer, e.g. `1`, `2`, `4`+- **Usage**: Set this to override the default batch size for AutoScheme++```bash+export AR_AUTO_SCHEME_BATCH_SIZE=1+```+## Usage Examples### Setting Environment Variablesdiff --git a/docs/environments_CN.md b/docs/environments_CN.mdindex 606f6b1f3..c04c45568 100644--- a/docs/environments_CN.md+++ b/docs/environments_CN.md@@ -121,6 +121,26 @@ export AR_DYNAMO_CACHE_SIZE_LIMIT=32export AR_MODEL_FREE_SHARD_PARALLELISM=4```+### AR_AUTO_SCHEME_NSAMPLES+- **描述**:控制 AutoScheme 评分时使用的校准样本数默认值,仅在 `AutoScheme.nsamples` 未显式设置时生效。+- **默认值**:未设置 → 16+- **有效值**:任意正整数,如 `8`、`16`、`32`+- **用途**:覆盖 AutoScheme 的自动样本数选择++```bash+export AR_AUTO_SCHEME_NSAMPLES=1+```++### AR_AUTO_SCHEME_BATCH_SIZE+- **描述**:控制 AutoScheme 评分时使用的批大小默认值,仅在 `AutoScheme.batch_size` 未显式设置时生效。+- **默认值**:未设置 → 走内置启发式规则(低GPU内存模式为 8,普通模式为 1)+- **有效值**:任意正整数,如 `1`、`2`、`4`+- **用途**:覆盖 AutoScheme 的默认批大小++```bash+export AR_AUTO_SCHEME_BATCH_SIZE=1+```+## 使用示例### 设置环境变量diff --git a/docs/step_by_step.md b/docs/step_by_step.mdindex d1eb3d013..0091b4af9 100644--- a/docs/step_by_step.md+++ b/docs/step_by_step.md@@ -422,6 +422,8 @@ ar.quantize_and_save()In some serving frameworks, certain layers (e.g., QKV or MoE) are fused to accelerate inference. These fused layers may require the same data type and bit configuration. The shared_layers option simplifies this setup by supporting both regex and full-name matching. **Note that regex matching is applied in a block-wise manner.**+**MoE expert layers are automatically grouped per block** — all expert projections (gate/up/down across all experts) within the same transformer block are treated as a single entity during DP optimization. This means they share the same quantization scheme and their losses/numel are summed together. No manual `shared_layers` configuration is needed for expert layers.+```pythonfrom auto_round import AutoRound, AutoSchemediff --git a/docs/step_by_step_CN.md b/docs/step_by_step_CN.mdindex 057f68563..131324697 100644--- a/docs/step_by_step_CN.md+++ b/docs/step_by_step_CN.md@@ -421,6 +421,8 @@ ar.quantize_and_save()为加速推理,在部分框架中,会对特定层(如QKV、MoE)进行融合。这些融合层必须是相同的数据类型和量化配置。`shared_layers` 参数可以简化该配置,**同时支持正则表达式匹配和完整层名匹配**。注意**正则匹配按块匹配规则生效**。+**MoE 的 expert 层会自动按块分组** — 同一个 transformer block 内所有 expert 的投影层(gate/up/down,跨所有 experts)会被视为一个整体进行 DP 优化。它们共享相同的量化方案,loss 和 numel 直接求和。无需手动配置 `shared_layers` 来处理 expert 层。+示例代码如下:```pythonfrom auto_round import AutoRound, AutoSchemediff --git a/test/test_cpu/schemes/test_auto_scheme_internal.py b/test/test_cpu/schemes/test_auto_scheme_internal.pyindex 35a6e69c5..d5080fa86 100644--- a/test/test_cpu/schemes/test_auto_scheme_internal.py+++ b/test/test_cpu/schemes/test_auto_scheme_internal.py@@ -1,55 +1,116 @@-from auto_round.auto_scheme.delta_loss import _apply_head_trick+import os+from auto_round.auto_scheme.gen_auto_scheme import AutoScheme+from auto_round.auto_scheme.utils import _build_layer_config_header_rows, _short_summary_name+++def test_env_ar_auto_scheme_nsamples_overrides_default(monkeypatch):+ """AR_AUTO_SCHEME_NSAMPLES env var should override the built-in nsamples heuristic."""+ import auto_round.envs as envs++ monkeypatch.setenv("AR_AUTO_SCHEME_NSAMPLES", "7")+ assert envs.AR_AUTO_SCHEME_NSAMPLES == 7+++def test_env_ar_auto_scheme_batch_size_overrides_default(monkeypatch):+ """AR_AUTO_SCHEME_BATCH_SIZE env var should override the built-in batch_size default."""+ import auto_round.envs as envs++ monkeypatch.setenv("AR_AUTO_SCHEME_BATCH_SIZE", "4")+ assert envs.AR_AUTO_SCHEME_BATCH_SIZE == 4-def test_apply_head_trick_keeps_lowest_loss_candidate_when_budget_allows():- schemes = AutoScheme(avg_bits=3, options=("GGUF:Q2_K_S", "GGUF:Q4_K_M")).options- total_scores = {- "lm_head": [- [0, 1244659712, 41.0, ["lm_head"]],- [1, 2489319424, 11.9, ["lm_head"]],- ],- "model.layers.0.mlp.down_proj": [- [0, 100, 10.0, ["model.layers.0.mlp.down_proj"]],- [1, 200, 5.0, ["model.layers.0.mlp.down_proj"]],- ],- }-- _apply_head_trick(- head_name="lm_head",- schemes=schemes,- sorted_indices=[1, 0],- target_bits=3,- target_params_cnt=4891670016,- total_scores=total_scores,- )-- assert total_scores["lm_head"] == [[1, 2489319424, 11.9, ["lm_head"]]]---def test_apply_head_trick_relaxes_lowest_loss_candidate_when_budget_is_tight():- schemes = AutoScheme(avg_bits=3, options=("GGUF:Q2_K_S", "GGUF:Q4_K_M")).options- total_scores = {- "lm_head": [- [0, 1244659712, 41.0, ["lm_head"]],- [1, 2489319424, 11.9, ["lm_head"]],- ],- "model.layers.0.mlp.down_proj": [- [0, 100, 10.0, ["model.layers.0.mlp.down_proj"]],- [1, 200, 5.0, ["model.layers.0.mlp.down_proj"]],- ],- }-- _apply_head_trick(- head_name="lm_head",- schemes=schemes,- sorted_indices=[1, 0],- target_bits=3,- target_params_cnt=1244659812,- total_scores=total_scores,- )-- assert total_scores["lm_head"] == [- [0, 1244659712, 41.0, ["lm_head"]],- [1, 2489319424, 11.9, ["lm_head"]],+def test_env_ar_auto_scheme_batch_size_zero_raises(monkeypatch):+ """Zero value for AR_AUTO_SCHEME_BATCH_SIZE should raise ValueError."""+ import pytest++ import auto_round.envs as envs++ monkeypatch.setenv("AR_AUTO_SCHEME_BATCH_SIZE", "0")+ with pytest.raises(ValueError):+ _ = envs.AR_AUTO_SCHEME_BATCH_SIZE+++def test_build_layer_config_header_rows_merges_adjacent_prefixes():+ """Adjacent columns with the same prefix should be merged into one compact header cell."""+ columns = ["mlp.down_proj", "mlp.gate_proj", "self_attn.q_proj", "self_attn.v_proj"]+ assert _build_layer_config_header_rows(columns) == [+ ["block", "mlp", "", "self_attn", ""],+ ["", "down_proj", "gate_proj", "q_proj", "v_proj"],]+++def test_short_summary_name_keeps_one_field_before_numeric_suffix():+ """Numeric block suffixes should be shortened to keep the preceding field."""+ assert _short_summary_name("model.layers.0") == "layers.0"+++def test_build_expert_groups_groups_experts_per_block():+ """Expert layers in the same block should be grouped together."""+ import torch+ from torch import nn++ from auto_round.auto_scheme.utils import build_expert_groups++ # Build a minimal MoE-like model with 2 blocks, each with 2 experts having 2 projections+ class FakeModel(nn.Module):+ def __init__(self):+ super().__init__()+ self.model = nn.Module()+ self.model.layers = nn.ModuleList()+ for i in range(2):+ block = nn.Module()+ block.mlp = nn.Module()+ block.mlp.experts = nn.ModuleList()+ for j in range(2):+ expert = nn.Module()+ expert.gate_proj = nn.Linear(8, 8, bias=False)+ expert.up_proj = nn.Linear(8, 8, bias=False)+ expert.down_proj = nn.Linear(8, 8, bias=False)+ block.mlp.experts.append(expert)+ block.self_attn = nn.Module()+ block.self_attn.q_proj = nn.Linear(8, 8, bias=False)+ self.model.layers.append(block)++ model = FakeModel()+ quant_layer_names = [n for n, m in model.named_modules() if isinstance(m, nn.Linear)]+ fixed_layer_scheme = {}++ groups = build_expert_groups(model, quant_layer_names, fixed_layer_scheme)+ # Should have 2 groups (one per block), each containing all 6 expert projections+ assert len(groups) == 2+ for group in groups:+ expert_layers = [n for n in group if "experts" in n]+ assert len(expert_layers) == 6 # 2 experts * 3 projections+ # Non-expert layers (q_proj) should NOT be in the group+ assert all("self_attn" not in n for n in group)+++def test_build_expert_groups_skips_fixed_layers():+ """Expert layers already in fixed_layer_scheme should not be grouped."""+ import torch+ from torch import nn++ from auto_round.auto_scheme.utils import build_expert_groups++ class FakeModel(nn.Module):+ def __init__(self):+ super().__init__()+ self.model = nn.Module()+ self.model.layers = nn.ModuleList()+ block = nn.Module()+ block.mlp = nn.Module()+ block.mlp.experts = nn.ModuleList()+ for j in range(2):+ expert = nn.Module()+ expert.gate_proj = nn.Linear(8, 8, bias=False)+ block.mlp.experts.append(expert)+ self.model.layers.append(block)++ model = FakeModel()+ quant_layer_names = [n for n, m in model.named_modules() if isinstance(m, nn.Linear)]+ # Fix all expert layers+ fixed_layer_scheme = {n: {} for n in quant_layer_names if "experts" in n}++ groups = build_expert_groups(model, quant_layer_names, fixed_layer_scheme)+ assert len(groups) == 0diff --git a/test/test_cuda/algorithms/test_auto_scheme.py b/test/test_cuda/algorithms/test_auto_scheme.pyindex 6ff82bba3..c721b6b20 100644--- a/test/test_cuda/algorithms/test_auto_scheme.py+++ b/test/test_cuda/algorithms/test_auto_scheme.py@@ -260,6 +260,41 @@ class TestAutoScheme:# cause using tiny modelassert 4.8 < avg_bits <= target_bits + 1e-3+ def test_lm_head_mix_score_nonzero(self, tiny_untied_qwen_model_path):+ """Verify that lm_head gets a non-zero mix_score in the low_gpu_mem_usage path."""+ import auto_round.auto_scheme.delta_loss as dl++ target_bits = 5+ scheme = AutoScheme(avg_bits=target_bits, options=("MXFP4", "MXFP8"))++ scores_captured = {}+ orig_head_trick = dl._apply_head_trick++ def capture_head_scores(head_name, schemes, sorted_indices, target_bits, target_params_cnt, total_scores):+ if head_name in total_scores:+ scores_captured["head_scores"] = [list(opt) for opt in total_scores[head_name]]+ return orig_head_trick(head_name, schemes, sorted_indices, target_bits, target_params_cnt, total_scores)++ dl._apply_head_trick = capture_head_scores+ try:+ ar = AutoRound(+ tiny_untied_qwen_model_path,+ scheme=scheme,+ iters=0,+ disable_opt_rtn=True,+ nsamples=1,+ quant_lm_head=True,+ )+ ar.quantize()+ finally:+ dl._apply_head_trick = orig_head_trick++ assert "head_scores" in scores_captured, "lm_head was not found in total_scores"+ head_scores = scores_captured["head_scores"]+ assert len(head_scores) >= 2, f"Expected at least 2 scheme options, got {len(head_scores)}"+ has_nonzero = any(opt[2] > 0 for opt in head_scores)+ assert has_nonzero, f"lm_head mix_score is 0 for all schemes: {head_scores}"+@pytest.mark.skip_ci(reason="The evaluation is time-consuming")def test_auto_scheme_export(self):model_name = get_model_path("facebook/opt-125m")