PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
Refine AutoScheme logic for gguf, reduce memory consumption and improve speed (#1916)
Wenhua Cheng0c9c5b1dadfa
diff --git a/README.md b/README.mdindex 7a54f8eba..e41599c9d 100644--- a/README.md+++ b/README.md@@ -36,9 +36,12 @@ See our papers [SignRoundV1](https://arxiv.org/pdf/2309.05516) and [SignRoundV2]## 🆕 What's New++* [2026/06] AutoScheme has been refined to improve accuracy for gguf format. See [AutoScheme Accuracy](./docs/auto_scheme_acc.md) for details. This enhancement incurs additional tuning cost.+* [2026/06] AutoRound is now fully integrated into vLLM-Omni: [*vLLM blog*](https://vllm.ai/blog/2026-06-02-vllm-omni-autoround)-* [2026/05] We provide **free** devices for calibration-free quantization via pure RTN mode; please visit [Intel Low Bit Open LLM Leaderboard](https://huggingface.co/spaces/Intel/low_bit_open_llm_leaderboard) for more details.+* [2026/05] We provide **free** devices for calibration-free quantization; please visit [Intel Low Bit Open LLM Leaderboard](https://huggingface.co/spaces/Intel/low_bit_open_llm_leaderboard) for more details.* [2026/05] **Model free** quantization is available, `auto-round-rtn` will now default to using the model-free approach: [Doc](https://github.com/intel/auto-round/blob/main/docs/step_by_step.md#model-free-mode).diff --git a/README_CN.md b/README_CN.mdindex 9bbcf9358..23a43c374 100644--- a/README_CN.md+++ b/README_CN.md@@ -35,9 +35,12 @@ AutoRound 是专为大语言模型(LLMs)和视觉-语言模型(VLMs)设## 🆕 最新进展++* [2026/06] AutoScheme对gguf格式优化,精度有所提升。具体结果请参考[文档](docs/auto_scheme_acc.md)。不过这些精度提升是以更高的调优时间和计算开销为代价的。+* [2026/06] **vLLM-Omni** 现已完全集成 AutoRound: [*vLLM 博客*](https://vllm.ai/blog/2026-06-02-vllm-omni-autoround)-* [2026/05] 我们提供**免费**的硬件资源,用于通过纯 RTN 方法进行模型量化。更多详情请访问 [Intel Low Bit Open LLM Leaderboard](https://huggingface.co/spaces/Intel/low_bit_open_llm_leaderboard)。+* [2026/05] 我们提供**免费**的硬件资源进行模型量化。更多详情请访问 [Intel Low Bit Open LLM Leaderboard](https://huggingface.co/spaces/Intel/low_bit_open_llm_leaderboard)。* [2026/05] 现已支持**免模型架构量化**模式, `auto-round-rtn` 现在会默认启用免模型方式: [文档](https://github.com/intel/auto-round/blob/main/docs/step_by_step_CN.md#%E5%85%8D%E6%A8%A1%E5%9E%8B%E6%9E%B6%E6%9E%84%E9%87%8F%E5%8C%96%E6%A8%A1%E5%BC%8F).diff --git a/auto_round/auto_scheme/delta_loss.py b/auto_round/auto_scheme/delta_loss.pyindex 7184d4300..b0fc7f60f 100644--- a/auto_round/auto_scheme/delta_loss.py+++ b/auto_round/auto_scheme/delta_loss.py@@ -13,6 +13,7 @@# limitations under the License.import copy+import gcfrom dataclasses import asdictfrom functools import wrapsfrom typing import Iterable, Optional, Union@@ -25,7 +26,6 @@ from auto_round.auto_scheme.gen_auto_scheme import AutoSchemefrom auto_round.auto_scheme.register import register_scheme_methodsfrom auto_round.auto_scheme.utils import (apply_quant_scheme,- compute_avg_bits_for_scheme,compute_layer_bits,parse_shared_layers,remove_quant_scheme,@@ -37,7 +37,7 @@ from auto_round.data_type.gguf import (search_gguf_scale_min_asym,search_gguf_scale_min_sym,)-from auto_round.data_type.utils import reshape_pad_tensor_by_group_size, revert_tensor_by_pad+from auto_round.data_type.utils import get_quant_func, reshape_pad_tensor_by_group_size, revert_tensor_by_padfrom auto_round.logger import loggerfrom auto_round.schemes import QuantizationScheme, preset_name_to_schemefrom auto_round.utils import (@@ -45,7 +45,9 @@ from auto_round.utils import (check_to_quantized,clear_memory,dispatch_model_by_all_available_devices,+ flatten_list,get_block_names,+ get_lm_head_name,get_major_device,get_module,is_mllm_model,@@ -58,7 +60,7 @@ from auto_round.utils import (to_device,to_dtype,)-from auto_round.utils.device import MemoryMonitor+from auto_round.utils.device import MemoryMonitor, memory_monitorfrom auto_round.utils.device_manager import get_current_device_managerfrom auto_round.utils.offload import OffloadManagerfrom auto_round.wrapper import WrapperLinear@@ -96,7 +98,6 @@ class AutoSchemeWrapperLinear(WrapperLinear):self.mix_score = 0.0self.super_qdq_func = super()._qdq_weightself.act_qdq_func = super()._qdq_act- # self.device = deviceself.max_act_value = 0self.need_weight_grad = need_weight_gradself.grad_mode = False@@ -168,6 +169,123 @@ class AutoSchemeWrapperLinear(WrapperLinear):return qdq_w, 1.0, None+class AutoSchemeWrapperLinearIMatrix(WrapperLinear):++ def __init__(+ self,+ orig_layer,+ enable_minmax_tuning=True,+ enable_norm_bias_tuning=False,+ device="cpu",+ enable_round_tuning=True,+ need_weight_grad=False,+ enable_torch_compile=False,+ **kwargs,+ ):+ super().__init__(+ orig_layer,+ enable_minmax_tuning,+ enable_norm_bias_tuning,+ device,+ enable_round_tuning,+ enable_torch_compile=enable_torch_compile,+ **kwargs,+ )+ self.total_act_score = 0.0+ self.act_score = 0.0+ self.avg_act_score = 0.0+ self.act_cnt = 0.0+ self.weight_score = 0.0+ self.mix_score = 0.0+ self.super_qdq_func = super()._qdq_weight+ self.act_qdq_func = super()._qdq_act+ self.max_act_value = 0+ self.need_weight_grad = need_weight_grad+ self.grad_mode = False+ if self.need_weight_grad:+ self.orig_layer.weight.requires_grad = True+ self.weight_search_quant_func, _ = get_quant_func(+ orig_layer.data_type,+ orig_layer.bits,+ orig_layer.sym,+ disable_opt_rtn=False,+ group_size=orig_layer.group_size,+ iters=0,+ )+ self.post_init_qdqw(device)++ @torch.no_grad()+ def post_init_qdqw(self, device):+ # Could not place in qdq_w, otherwise vram is much higher++ qdq_w, _, _ = self.weight_search_quant_func(+ self.orig_layer.weight.to(device),+ bits=self.orig_layer.bits,+ group_size=self.orig_layer.group_size,+ v=torch.tensor(0, device=device),+ min_scale=torch.tensor(1.0, device=device),+ max_scale=torch.tensor(1.0, device=device),+ scale_dtype=self.orig_layer.scale_dtype,+ data_type=self.data_type,+ q_scale_thresh=self.q_scale_thresh,+ imatrix=self.orig_layer.imatrix.to(device) if hasattr(self.orig_layer, "imatrix") else None,+ global_scale=getattr(self, "weight_global_scale", None),+ )++ self.register_buffer("qdq_w", qdq_w.detach().clone().to(self.orig_layer.weight.device))++ def save_grad(grad):+ w_diff = self.orig_layer.weight - self.qdq_w.to(self.orig_layer.weight.device)+ self.weight_score += torch.abs((grad.to(torch.float32) * w_diff.to(grad.device))).sum().item()+ act_score = 0.0 if self.act_cnt <= 0 else self.total_act_score / self.act_cnt+ self.mix_score = self.weight_score + act_score+ return None++ self.qdq_w.requires_grad_(True)+ self.orig_layer.weight.requires_grad_(False)++ self.qdq_w.register_hook(save_grad)++ def _qdq_act(self, x, act_min_scale=1.0, act_max_scale=1.0, act_max=None):+ if hasattr(self.orig_layer, "act_bits") and self.orig_layer.act_bits > 8:+ return x, 1.0, None++ qdq_x, scale, zp = self.act_qdq_func(x, act_min_scale, act_max_scale, act_max)+ if self.grad_mode:+ with torch.no_grad():+ self.max_act_value = torch.abs(x).max()+ if torch.abs(x).max() != 0:+ self.act_cnt += 1+ x_diff = x - qdq_x+ self.x_diff = x_diff.to("cpu")++ def save_grad(grad):+ if self.max_act_value == 0:+ if torch.abs(grad).max() != 0:+ raise ValueError+ """+ this ut will cause NAN issue sometimes, need to investigate+ @multi_card+ def test_multi_card(self):+ model_name = "/models/Qwen3-8B"+ """+ if torch.isnan(grad).any() or torch.isnan(self.x_diff).any():+ self.act_cnt -= 1+ return None++ self.total_act_score += torch.abs((grad * self.x_diff.to(grad.device))).sum().item()+ self.act_score = 0.0 if self.act_cnt <= 0 else self.total_act_score / self.act_cnt+ self.mix_score = self.weight_score + self.act_score+ self.x_diff = None+ return None++ qdq_x.register_hook(save_grad)+ return qdq_x, scale, zp++ def _qdq_weight(self, value, min_scale, max_scale):+ return self.qdq_w, 1.0, None++class AutoSchemeWrapperLinearForGGUFK(AutoSchemeWrapperLinear):def __init__(@@ -195,20 +313,17 @@ class AutoSchemeWrapperLinearForGGUFK(AutoSchemeWrapperLinear):)self.register_buffer("qdq_w", qdq_w.detach().clone().to(self.orig_layer.weight.device))- if self.need_weight_grad:-- def save_grad(grad):- w_diff = self.orig_layer.weight - self.qdq_w.to(self.orig_layer.weight.device)- # TODO strange, grad could be in CPU- self.weight_score += torch.abs((grad.to(w_diff.device) * w_diff)).sum().item() # TODO add 2nd order- act_score = 0.0 if self.act_cnt <= 0 else self.total_act_score / self.act_cnt- self.mix_score = self.weight_score + act_score- return None-- self.qdq_w.requires_grad_(True)- self.orig_layer.weight.requires_grad_(False)+ def save_grad(grad):+ w_diff = self.orig_layer.weight - self.qdq_w.to(self.orig_layer.weight.device)+ # TODO strange, grad could be in CPU+ self.weight_score += torch.abs((grad.to(w_diff.device).to(torch.float32) * w_diff)).sum().item()+ act_score = 0.0 if self.act_cnt <= 0 else self.total_act_score / self.act_cnt+ self.mix_score = self.weight_score + act_score+ return None- self.qdq_w.register_hook(save_grad)+ self.qdq_w.requires_grad_(True)+ self.orig_layer.weight.requires_grad_(False)+ self.qdq_w.register_hook(save_grad)def _qdq_weight(self, value, min_scale, max_scale):return self.qdq_w, 1.0, None@@ -237,29 +352,31 @@ class AutoSchemeWrapperLinearForGGUFKImatrix(AutoSchemeWrapperLinear):enable_torch_compile=enable_torch_compile,**kwargs,)- with torch.no_grad():- qdq_w = self._init_scale().detach()- self.register_buffer("qdq_w", qdq_w.detach().clone().to(self.orig_layer.weight.device))- if self.need_weight_grad:+ self.post_init_qdqw(device)- def save_grad(grad):- w_diff = self.orig_layer.weight - self.qdq_w.to(self.orig_layer.weight.device)- self.weight_score += torch.abs((grad * w_diff.to(grad.device))).sum().item()- act_score = 0.0 if self.act_cnt <= 0 else self.total_act_score / self.act_cnt- self.mix_score = self.weight_score + act_score- return None+ @torch.no_grad()+ def post_init_qdqw(self, device): # Could not place in qdq_w, otherwise vram is much higher+ qdq_w = self._init_scale(device).detach()+ self.register_buffer("qdq_w", qdq_w.detach().clone().to(self.orig_layer.weight.device))- self.qdq_w.requires_grad_(True)- self.orig_layer.weight.requires_grad_(False)+ def save_grad(grad):+ w_diff = self.orig_layer.weight - self.qdq_w.to(self.orig_layer.weight.device)+ self.weight_score += torch.abs((grad.to(torch.float32) * w_diff.to(grad.device))).sum().item()+ act_score = 0.0 if self.act_cnt <= 0 else self.total_act_score / self.act_cnt+ self.mix_score = self.weight_score + act_score+ return None++ self.qdq_w.requires_grad_(True)+ self.orig_layer.weight.requires_grad_(False)- self.qdq_w.register_hook(save_grad)+ self.qdq_w.register_hook(save_grad)@torch.no_grad()- def _init_scale(self):- tensor = self.orig_layer.weight.data+ def _init_scale(self, device):+ tensor = self.orig_layer.weight.data.to(device)bits = self.orig_layer.bitsscale_dtype = self.orig_layer.scale_dtype- imatrix = self.orig_layer.imatrix+ imatrix = self.orig_layer.imatrix.to(tensor.device)orig_dtype = tensor.dtypeif self.orig_layer.bits in [2, 4, 5]:group_size = 16 if bits == 2 else 32@@ -293,34 +410,77 @@ class AutoSchemeWrapperLinearForGGUFKImatrix(AutoSchemeWrapperLinear):return self.qdq_w, 1.0, None+def register_imatrix_hook(model):+ """Registers hooks to accumulate activation squared norms into `imatrix`."""++ def get_imatrix_hook(module, input, output):+ input = input[0] if isinstance(input, (tuple, list)) else input+ flattened = input.reshape(-1, input.shape[-1]).to(torch.float32)+ squared = torch.sum(torch.pow(flattened, 2), dim=0).to(torch.float32)++ if not hasattr(module, "imatrix"):+ module.imatrix = squared.to("cpu")+ else:+ module.imatrix += squared.to(module.imatrix.device).to("cpu")++ hook_handles = []+ for name, module in model.named_modules():+ if isinstance(module, SUPPORTED_LAYER_TYPES):+ hook = module.register_forward_hook(get_imatrix_hook)+ hook_handles.append(hook)+ return hook_handles++@torch.no_grad()-def cal_imatrix(model, dataloader):+def cal_imatrix(model, dataloader, major_device, low_gpu_mem_usage):- def register_act_hook(model):- """Registers hooks to accumulate activation squared norms into `imatrix`."""+ if low_gpu_mem_usage:+ cal_imatrix_low_gpu(model, dataloader, major_device)+ else:+ hooks = register_imatrix_hook(model)+ model = model.to(model.device)+ for data in dataloader:+ model.forward(**to_device(data, model.device))+ for hook in hooks:+ hook.remove()- def get_imatrix_hook(module, input, output):- input = input[0] if isinstance(input, (tuple, list)) else input- flattened = input.reshape(-1, input.shape[-1]).to(torch.float32)- squared = torch.sum(torch.pow(flattened, 2), dim=0).to(torch.float32)- if not hasattr(module, "imatrix"):- module.imatrix = squared.to("cpu")- else:- module.imatrix += squared.to(module.imatrix.device).to("cpu")+def cal_imatrix_low_gpu(model, dataloader, major_device):+ imatrix_hooks = register_imatrix_hook(model)+ block_names = get_block_names(model, quant_vision=True)+ block_names = flatten_list(block_names)- hook_handles = []- for name, module in model.named_modules():- if isinstance(module, SUPPORTED_LAYER_TYPES):- hook = module.register_forward_hook(get_imatrix_hook)- hook_handles.append(hook)- return hook_handles+ def move_to_gpu_hook(module, inputs):+ module.to(major_device)+ to_device(inputs, major_device)++ def move_to_cpu(module, inputs, outputs):+ module.to("cpu")++ def move_to_cpu_clear_memory(module, inputs, outputs):+ module.to("cpu")+ clear_memory(device_list=major_device)++ all_move_device_hooks = []+ i = 0+ for block_name in block_names:+ i += 1+ block_module = get_module(model, block_name)+ hook_move_gpu = block_module.register_forward_pre_hook(move_to_gpu_hook)++ hook_move_cpu = block_module.register_forward_hook(move_to_cpu)++ all_move_device_hooks.append(hook_move_gpu)+ all_move_device_hooks.append(hook_move_cpu)- hooks = register_act_hook(model)for data in dataloader:model.forward(**to_device(data, model.device))- for hook in hooks:++ for hook in imatrix_hooks:+ hook.remove()+ for hook in all_move_device_hooks:hook.remove()+ clear_memory(device_list=major_device)class MyCustomError(Exception):@@ -346,6 +506,9 @@ def prepare_model_low_gpu(model, block_inputs: dict = None, pbar=None, major_dev@wraps(original_forward)def new_forward(*args, **kwargs):move_module_to_tuning_device(module, major_device=major_device)+ # for n,m in module.named_modules():+ # if hasattr(m, "post_init_qdqw"):+ # m.post_init_qdqw()# Call the original forwardwith torch.no_grad():@@ -361,7 +524,9 @@ def prepare_model_low_gpu(model, block_inputs: dict = None, pbar=None, major_devblock_inputs[module_name] = input_infomodule.to("cpu")- clear_memory()+ memory_monitor.update(device_list=major_device)+ # clear_memory(device_list=major_device) #slow+ # memory_monitor.log_summary()# Enable gradients for the output of the last blockif module.tmp_name == block_names[-1]:@@ -417,7 +582,7 @@ def _prepare_mllm_inputs(data, model):return new, "dict"-def mllm_model_forward(model, data, **forward_kwargs):+def model_forward(model, data, **forward_kwargs):"""Single entry point for "run a (possibly multimodal) batch through themodel". Used by both AutoScheme paths so that ``pixel_values`` / ``images``are cast to ``model.dtype`` (otherwise VLMs silently skip the vision tower@@ -463,17 +628,18 @@ def model_forward_low_gpu(model, dataloader, major_device="cuda", pbar=None):# Route through the unified mllm forward so ``pixel_values`` /# ``images`` get cast to ``model.dtype`` (otherwise the vision tower# is silently bypassed on dtype mismatch and vision grad stays 0).- output, _prepared = mllm_model_forward(model, data_for_forward, labels=labels, use_cache=False)+ output, _prepared = model_forward(model, data_for_forward, labels=labels, use_cache=False)+ clear_memory(device_list=major_device)+ memory_monitor.log_summary()try:# Backward pass (will be interrupted by the hook)- output.loss.backward()+ output.loss.to(torch.float32).backward()except MyCustomError:passcurrent_grad = last_grad_inputdel output, data- clear_memory()# Manually compute gradients block by blocklast_block_backward_hook.remove()@@ -481,9 +647,9 @@ def model_forward_low_gpu(model, dataloader, major_device="cuda", pbar=None):for name in block_names:module = get_module(model, name)module.forward = module.orig_forward-+ index = 0for block_name in reversed(block_names):-+ index += 1# Retrieve stored inputs for the blockblock_input_info = block_inputs.get(block_name, {})@@ -524,12 +690,15 @@ def model_forward_low_gpu(model, dataloader, major_device="cuda", pbar=None):if block_input_args and isinstance(block_input_args[0], torch.Tensor):current_grad = block_input_args[0].grad.detach().clone()else:- print(f"Warning: No suitable input gradient found for {block_name}")+ logger.warning(f"No suitable input gradient found for {block_name}")breakdel block_output, main_output, block_input_args, block_input_kwargsblock_module.to("cpu")- clear_memory()++ # clear_memory(device_list=major_device) # this one is very slow and seems does not affect max ram usage+ memory_monitor.update()+pbar.update(1)@@ -549,7 +718,6 @@ def get_score_for_scheme(low_gpu_mem_usage=True,major_device="cpu",batch_size=1,- disable_opt_rtn=True,offload_context: Optional[OffloadManager] = None,processor=None,is_vlm: bool = False,@@ -590,6 +758,8 @@ def get_score_for_scheme(m.scale_dtype = torch.bfloat16WrapperLayer = AutoSchemeWrapperLinear+ # if has_imatrix: # no better result+ # WrapperLayer = AutoSchemeWrapperLinearIMatrixif hasattr(m, "super_group_size") and m.super_group_size is not None:if has_imatrix:WrapperLayer = AutoSchemeWrapperLinearForGGUFKImatrix@@ -609,12 +779,11 @@ def get_score_for_scheme(new_m = WrapperLayer(m,device=device,- enable_minmax_tuning=False, # TODO this should be change+ enable_minmax_tuning=False,enable_norm_bias_tuning=False,enable_round_tuning=False,need_weight_grad=need_weight_grad,enable_torch_compile=enable_torch_compile,- disable_opt_rtn=disable_opt_rtn,)set_module(model, name, new_m)if offload_context is not None:@@ -639,13 +808,13 @@ def get_score_for_scheme(if id(_p) not in wrapper_weight_ids and _p.requires_grad:_p.requires_grad_(False)_trimmed += 1- if _trimmed:- logger.info(- "AutoScheme: disabled requires_grad on %d non-wrapper parameters "- "(only wrapper.orig_layer.weight needs grad for scoring; saves "- "~one model-worth of grad buffer during backward).",- _trimmed,- )+ # if _trimmed:+ # logger.info(+ # "AutoScheme: disabled requires_grad on %d non-wrapper parameters "+ # "(only wrapper.orig_layer.weight needs grad for scoring; saves "+ # "~one model-worth of grad buffer during backward).",+ # _trimmed,+ # )# When scoring vision-tower layers, keep the autograd chain alive end-to-end:# (1) every wrapper's orig weight must require grad — otherwise the STE@@ -775,6 +944,7 @@ def get_score_for_scheme(f"falling back to multimodal calibration dataloader.")mllm_loader = _build_mllm_calib_dataloader()+ batch_size = 1if mllm_loader is None:raisemodel_forward_low_gpu(model, mllm_loader, major_device=major_device, pbar=pbar)@@ -782,6 +952,8 @@ def get_score_for_scheme(for n, m in model.named_modules():if hasattr(m, "grad_mode"):m.grad_mode = True+ # if hasattr(m, "post_init_qdqw"):+ # m.post_init_qdqw()def _run_forward_loop(loader):_checked_pixel = False@@ -810,7 +982,7 @@ def get_score_for_scheme(# Unified mllm-aware forward (casts pixel_values/images to# model.dtype, handles dict-with-text/str/tuple paths the same# way AutoRoundMLLM.calib does).- output, _prepared = mllm_model_forward(model, data_for_forward, labels=labels, use_cache=False)+ output, _prepared = model_forward(model, data_for_forward, labels=labels, use_cache=False)output.loss.backward()# One-shot sanity check: when scoring vision layers, the batch@@ -871,16 +1043,41 @@ def get_score_for_scheme(scores_dict[n] = [layer_bits, m.mix_score]for n, m in model.named_modules():if hasattr(m, "orig_layer"):+ # Explicitly break reference cycles to ensure GC can free the wrapper.+ # Hook closures capture `self` (wrapper), creating cycles:+ # wrapper → qdq_w → _backward_hooks → closure → wrapper+ # PyTorch's C-level tensor storage prevents Python's cyclic GC from+ # collecting these without explicitly breaking the cycle first.+ if hasattr(m, "qdq_w") and m.qdq_w is not None:+ if hasattr(m.qdq_w, "_backward_hooks") and m.qdq_w._backward_hooks:+ m.qdq_w._backward_hooks.clear()+ # Use detach_() rather than requires_grad_(False) because+ # block_module.to("cpu") may have turned qdq_w into a non-leaf+ # (ToCopyBackward grad_fn from .to()), and requires_grad_ only+ # works on leaf tensors.+ m.qdq_w.detach_()+ m.qdq_w = None+ if hasattr(m, "x_diff"):+ m.x_diff = None+ if hasattr(m, "super_qdq_func"):+ m.super_qdq_func = None+ if hasattr(m, "act_qdq_func"):+ m.act_qdq_func = Noneset_module(model, n, m.orig_layer)++ gc.collect()return scores_dict-def choose_bits_per_layer_with_path(layers: dict, P: int):+def choose_bits_per_layer_with_path(layers: dict, P: int, max_states: int = None):"""Args:layers: A dict mapping each layer name to a list of candidate options.Each option is a tuple of (scheme, bits_cost, loss_cost, layer_names).P: Upper bound on the total parameter (bit) budget.+ max_states: Maximum number of DP states to retain after each layer+ (beam width). Limits memory usage for models with many+ layers and incommensurate layer sizes.Returns:(min_loss, best_path), where best_path is a list of@@ -890,7 +1087,6 @@ def choose_bits_per_layer_with_path(layers: dict, P: int):# dp: total_params -> (accumulated_loss, chosen_path)# The path explicitly stores the selected options.dp: dict[int, tuple[float, list]] = {0: (0.0, [])}-for layer_name, opts in layers.items():new_dp: dict[int, tuple[float, list]] = {}for cur_params, (cur_loss, cur_path) in dp.items():@@ -909,7 +1105,6 @@ def choose_bits_per_layer_with_path(layers: dict, P: int):if not new_dp:return None, None-# Pareto pruning: remove dominated (params, loss) statesitems = sorted(new_dp.items(), key=lambda x: x[0]) # (params, (loss, path))pruned: dict[int, tuple[float, list]] = {}@@ -919,6 +1114,27 @@ def choose_bits_per_layer_with_path(layers: dict, P: int):pruned[params_val] = (loss_val, path_val)best_loss_so_far = loss_val+ # Beam width limit: if too many states survive Pareto pruning,+ # uniformly subsample to bound memory usage. For models with many+ # layers whose sizes are incommensurate, the number of distinct+ # cumulative-bit sums can grow to millions, each storing a full+ # path copy — easily exceeding 70 GB of RAM.+ if max_states is not None and len(pruned) > max_states:+ if max_states <= 1:+ best_k = min(pruned.keys(), key=lambda k: pruned[k][0])+ pruned = {best_k: pruned[best_k]}+ else:+ sorted_keys = sorted(pruned.keys())+ n = len(sorted_keys)+ # Uniformly pick max_states indices (always include first and last)+ step = (n - 1) / (max_states - 1)+ selected: dict[int, tuple[float, list]] = {}+ for i in range(max_states):+ idx = int(round(i * step))+ k = sorted_keys[idx]+ selected[k] = pruned[k]+ pruned = selected+dp = pruned# Select the solution with the minimum loss@@ -965,6 +1181,102 @@ def move_module_to_tuning_device(module, major_device="cpu"):_move_own_tensors(m, major_device)+def _get_scheme_bits(scheme):+ """Extract the weight bits from a scheme (str or dict)."""+ if isinstance(scheme, str):+ scheme = asdict(preset_name_to_scheme(scheme))+ elif isinstance(scheme, QuantizationScheme):+ scheme = asdict(scheme)+ return scheme.get("bits", 16)+++# Delta loss does not handle lm-head well, it is prone to assign low bit to lm-head which is not optimal+def _apply_head_trick(head_name, schemes, sorted_indices, target_bits, target_params_cnt, total_scores):++ # ------------------------------------------------------------------ #+ # lm_head option restriction for DP #+ # lm_head is critical — its quantization error goes directly into #+ # logits with no subsequent LayerNorm dampening. Instead of removing #+ # it from DP, we restrict its candidate options so the DP can only #+ # assign it >= 6 bit schemes, then let DP globally optimize. #+ # #+ # Rules (only if user hasn't already fixed it): #+ # 1. No option has bits >= 6 → keep all options (DP picks best)#+ # 2. Exactly one option bits >= 6 → restrict to only that option #+ # 3. Multiple options bits >= 6: #+ # - target_bits > 6 → restrict to only the highest-bit option #+ # - target_bits <= 6 → keep all >=6 options, let DP decide #+ # ------------------------------------------------------------------ #++ high_bit_indices = [i for i in range(len(schemes)) if _get_scheme_bits(schemes[i]) >= 6]++ if len(high_bit_indices) == 0:+ # Rule 1: no option >= 6 bit → restrict to the lowest-loss scheme+ allowed_indices = {sorted_indices[0]} if sorted_indices else None+ elif len(high_bit_indices) == 1:+ # Rule 2: exactly one >= 6 bit option → restrict to it+ allowed_indices = set(high_bit_indices)+ else:+ # Rule 3: multiple >= 6 bit options+ if target_bits > 6:+ # Restrict to only the highest-bit option+ highest_idx = max(high_bit_indices, key=lambda i: _get_scheme_bits(schemes[i]))+ allowed_indices = {highest_idx}+ else:+ # Keep all >= 6 bit options, let DP decide among them+ allowed_indices = set(high_bit_indices)++ # Feasibility check: ensure the restricted lm_head options + min bits+ # for all other layers don't exceed the budget. If infeasible, relax+ # by adding options from sorted_indices (lowest loss first) until+ # a feasible combination exists.+ if allowed_indices is not None:+ # Compute budget remaining after fixed layers++ _remaining_budget = target_params_cnt++ # Compute min bits for non-lm_head DP layers+ _min_other_bits = 0+ for key, opts in total_scores.items():+ if key != head_name:+ _min_other_bits += min(opt[1] for opt in opts)++ # Compute min bits for lm_head under allowed_indices+ _min_head_bits = 0++ if head_name in total_scores:+ head_opts = [opt for opt in total_scores[head_name] if opt[0] in allowed_indices]+ if head_opts:+ _min_head_bits += min(opt[1] for opt in head_opts)+ else:+ _min_head_bits += min(opt[1] for opt in total_scores[head_name])++ # If infeasible, relax by adding cheaper options from sorted_indices+ if _min_head_bits + _min_other_bits > _remaining_budget:+ for fallback_idx in sorted_indices:+ if fallback_idx in allowed_indices:+ continue+ allowed_indices.add(fallback_idx)+ # Recompute min head bits with expanded options+ _min_head_bits = 0++ if head_name in total_scores:+ head_opts = [opt for opt in total_scores[head_name] if opt[0] in allowed_indices]+ if head_opts:+ _min_head_bits += min(opt[1] for opt in head_opts)+ else:+ _min_head_bits += min(opt[1] for opt in total_scores[head_name])+ if _min_head_bits + _min_other_bits <= _remaining_budget:+ break++ # Filter lm_head's entries in total_scores to only allowed options+ if allowed_indices is not None:+ if head_name in total_scores:+ filtered = [opt for opt in total_scores[head_name] if opt[0] in allowed_indices]+ if filtered:+ total_scores[head_name] = filtered++def _gen_layer_config(auto_scheme: AutoScheme,model: Union[str, torch.nn.Module],@@ -975,7 +1287,6 @@ def _gen_layer_config(tokenizer=None,device_map=None,enable_torch_compile=False,- disable_opt_rtn=True,model_name=None,major_device="cpu",device_list=None,@@ -984,7 +1295,7 @@ def _gen_layer_config():# Initialize memory tracking for AutoSchemememory_monitor = MemoryMonitor()- memory_monitor.reset()+ # memory_monitor.reset()memory_monitor.update_cpu()# Create offload context for CPU RAM optimization@@ -1099,6 +1410,7 @@ def _gen_layer_config(pbar_cnt = 0need_weight_grad = False+ need_imatrix = False # only trigger it for gguf q-k quantfor index, scheme in enumerate(schemes):if check_bf16_scheme(scheme):continue@@ -1108,6 +1420,8 @@ def _gen_layer_config(scheme = asdict(scheme)bits = scheme.get("bits", 16)act_bits = scheme.get("act_bits", 16)+ if scheme.get("super_group_size"):+ need_imatrix = Trueif bits <= 8 < act_bits:need_weight_grad = Trueif not auto_scheme.low_gpu_mem_usage:@@ -1117,25 +1431,21 @@ def _gen_layer_config(shared_layers = parse_shared_layers(model, auto_scheme.shared_layers)options_scores = []- if auto_scheme.low_gpu_mem_usage and not disable_opt_rtn:- logger.warning("low_gpu_mem_usage is enabled, disable_opt_rtn will be set to True automatically")- disable_opt_rtn = True- if not disable_opt_rtn:- need_imatrix = False- for scheme in schemes:- if isinstance(scheme, str):- scheme = asdict(preset_name_to_scheme(scheme))- elif isinstance(scheme, QuantizationScheme):- scheme = asdict(scheme)-- need_imatrix = scheme["super_bits"] is not None- if need_imatrix:- break- if need_imatrix: # TODO change to block way in low_gpu_mem_usage- dataloader = get_dataloader(tokenizer, seqlen=256, dataset_name=dataset, seed=42, bs=8, nsamples=nsamples)- logger.info("start to compute imatrix for GGUF-K quantization in AutoScheme")- cal_imatrix(model, dataloader)- logger.info("finish calculating imatrix")++ if need_imatrix:+ dataloader = get_dataloader(+ tokenizer,+ seqlen=max(seqlen * 2, 2048),+ dataset_name=dataset,+ seed=42,+ bs=batch_size,+ nsamples=min(nsamples, 128),+ )+ logger.info("start to compute imatrix in AutoScheme")+ cal_imatrix(model, dataloader, major_device, low_gpu_mem_usage=auto_scheme.low_gpu_mem_usage)+ memory_monitor.update()+ memory_monitor.log_summary()+ logger.info("finish calculating imatrix")# Register hooks and clear all block weights before the scheme loop.# Hooks will transparently reload weights on demand during forward passes.@@ -1171,7 +1481,6 @@ def _gen_layer_config(low_gpu_mem_usage=auto_scheme.low_gpu_mem_usage,major_device=major_device,batch_size=batch_size,- disable_opt_rtn=auto_scheme.disable_opt_rtn,offload_context=offload_context,processor=processor,is_vlm=is_vlm,@@ -1179,7 +1488,8 @@ def _gen_layer_config(model_name=model_name,)# Track peak RAM after each scheme scoring- memory_monitor.update_cpu()+ memory_monitor.update()+ memory_monitor.log_summary()new_scores = {}for share_layer in shared_layers:@@ -1220,47 +1530,135 @@ def _gen_layer_config(target_params_cnt = int(total_params * target_bits)sorted_indices = sorted(range(len(options_scores)), key=lambda i: options_scores[i])# Layers that are not fixed in fixed_layer_scheme- not_fixed_embedding_layers_names = [name for name in embedding_layers_names if name not in fixed_layer_scheme]- for sorted_index in sorted_indices:- tmp_scheme = schemes[sorted_index]- if isinstance(tmp_scheme, str):- tmp_scheme = asdict(preset_name_to_scheme(tmp_scheme))+ not_fixed_embedding_layers_names = [+ name for name in embedding_layers_names if (name not in fixed_layer_scheme and name in quant_layer_names)+ ]- for embedding_layer_name in not_fixed_embedding_layers_names:-- fixed_layer_scheme[embedding_layer_name] = tmp_scheme- embedding_layer = get_module(model, embedding_layer_name)- for key, item in tmp_scheme.items():- setattr(embedding_layer, key, item)- current_avg_bits, _ = compute_avg_bits_for_scheme(- model,- quant_layer_names + embedding_layers_names,- fixed_layer_scheme,- min_avg_bit_scheme,- ignore_scale_zp_bits=auto_scheme.ignore_scale_zp_bits,- clean_scheme=False,- )+ # Determine if model has shared lm_head (tie_word_embeddings)+ has_tied_lm_head = getattr(getattr(model, "config", None), "tie_word_embeddings", False)- if current_avg_bits <= target_bits: # compute_avg_bit remove setting- for embedding_layer_name in not_fixed_embedding_layers_names:+ def _to_scheme_dict(scheme):+ """Normalize a scheme (str/QuantizationScheme/dict) to a plain dict."""+ if isinstance(scheme, str):+ return asdict(preset_name_to_scheme(scheme))+ elif isinstance(scheme, QuantizationScheme):+ return asdict(scheme)+ return scheme++ def _compute_embedding_bits(scheme_dict):+ """Compute total bits consumed by non-fixed embedding layers under scheme_dict."""+ total = 0+ for emb_name in not_fixed_embedding_layers_names:+ emb_layer = get_module(model, emb_name)+ n_param = emb_layer.weight.numel()+ if n_param == 0 and hasattr(emb_layer, "_cached_weight_numel"):+ n_param = emb_layer._cached_weight_numel+ # With ignore_scale_zp_bits, bits_cost = n_param * bits+ emb_bits = scheme_dict.get("bits", 16)+ total += n_param * emb_bits+ return total++ # Compute bits already consumed by user-fixed layers (excluding embeddings we'll set)+ already_fixed_bits = 0+ for name in fixed_layer_scheme.keys():+ m = get_module(model, name)+ layer_bits, _ = compute_layer_bits(m, auto_scheme.ignore_scale_zp_bits)+ already_fixed_bits += layer_bits++ # Compute minimum bits needed for DP layers (non-fixed, non-embedding)+ min_dp_bits = 0+ for layer_name, opts in total_scores.items():+ min_dp_bits += min(opt[1] for opt in opts)++ def _fits_budget(scheme_dict):+ """Check if applying scheme_dict to embeddings leaves enough budget for DP layers."""+ emb_bits = _compute_embedding_bits(scheme_dict)+ remaining = target_params_cnt - already_fixed_bits - emb_bits+ return remaining >= min_dp_bits++ def _select_embedding_scheme_index():+ """Select the best scheme index for embedding layers based on model type and target_bits.++ For models with shared lm_head (tie_word_embeddings=True):+ - target_bits > 6: use the lowest-loss option (same as before)+ - target_bits <= 6: use the lowest-loss option among those with bits <= 6+ For models without shared lm_head:+ - use the lowest-loss option among those with bits >= ceil(target_bits)++ In all cases, the selected scheme must not exceed the total bit budget+ (i.e., embedding bits + fixed bits + min DP bits <= target_params_cnt).+ """+ import math- fixed_layer_scheme[embedding_layer_name] = tmp_scheme- embedding_layer = get_module(model, embedding_layer_name)- for key, item in tmp_scheme.items():- setattr(embedding_layer, key, item)- break+ if has_tied_lm_head:+ if target_bits > 6:+ candidates = list(sorted_indices)+ else:+ # Prefer options with bits <= 6, sorted by loss+ candidates = [idx for idx in sorted_indices if _get_scheme_bits(schemes[idx]) <= 6]+ if not candidates:+ candidates = list(sorted_indices)+ else:+ # Not shared lm_head: prefer options with bits < floor(target_bits)+ floor_bits = math.floor(target_bits)+ candidates = [idx for idx in sorted_indices if _get_scheme_bits(schemes[idx]) == floor_bits]+ if not candidates:+ # find the first bits that greater than floor bits+ embedding_bits = [bits for idx in sorted_indices if _get_scheme_bits(schemes[idx]) > floor_bits]+ if len(embedding_bits) > 0:+ sorted(embedding_bits)+ embedding_bits = embedding_bits[0]+ candidates = [idx for idx in sorted_indices if _get_scheme_bits(schemes[idx]) == embedding_bits]+ candidates.extend(sorted_indices) # to make sure if the above candidate exceed the budget++ # Among candidates (ordered by loss), pick the first that fits the budget+ for idx in candidates:+ scheme_dict = _to_scheme_dict(schemes[idx])+ if _fits_budget(scheme_dict):+ return idx++ # Fallback: try ALL options sorted by bits ascending (cheapest first)+ all_by_bits = sorted(range(len(schemes)), key=lambda i: _get_scheme_bits(schemes[i]))+ for idx in all_by_bits:+ scheme_dict = _to_scheme_dict(schemes[idx])+ if _fits_budget(scheme_dict):+ return idx++ # Last resort: use the cheapest option regardless+ return all_by_bits[0] if all_by_bits else 0# Minus fixed_layerfor name in fixed_layer_scheme.keys(): # The Scheme should have been appliedm = get_module(model, name)layer_bits, _ = compute_layer_bits(m, auto_scheme.ignore_scale_zp_bits)target_params_cnt -= layer_bits++ head_name = get_lm_head_name(model)+ if head_name is not None and (head_name not in fixed_layer_scheme and head_name in quant_layer_names):+ _apply_head_trick(head_name, schemes, sorted_indices, target_bits, target_params_cnt, total_scores)+ # As only a small amount of calibration data is used and embedding layers are inherently sparse,+ # we cannot obtain a reliable score.+ if not_fixed_embedding_layers_names:+ selected_index = _select_embedding_scheme_index()+ tmp_scheme = _to_scheme_dict(schemes[selected_index])++ for embedding_layer_name in not_fixed_embedding_layers_names:+ fixed_layer_scheme[embedding_layer_name] = tmp_scheme+ embedding_layer = get_module(model, embedding_layer_name)+ for key, item in tmp_scheme.items():+ setattr(embedding_layer, key, item)+ layer_bits, _ = compute_layer_bits(embedding_layer, auto_scheme.ignore_scale_zp_bits)+ target_params_cnt -= layer_bits+if target_params_cnt <= 0:raise ValueError("Avg bits is too small")remove_quant_scheme(model) # Must place after minus fixed_layer+ memory_monitor.update()+ memory_monitor.log_summary()best_loss, best_path = choose_bits_per_layer_with_path(total_scores, target_params_cnt)+# print(best_loss, best_path) # TODO better loglayer_config = copy.deepcopy(fixed_layer_scheme)options = list(copy.deepcopy(auto_scheme.options))@@ -1273,7 +1671,6 @@ def _gen_layer_config(layer_scheme = options[item[1]]for layer_name in layer_names:layer_config[layer_name] = asdict(layer_scheme)-if model_name is not None:model = Nonedel model@@ -1298,8 +1695,8 @@ def _gen_layer_config(last_grad_input = Noneclear_memory(device_list=device_list)- # Log AutoScheme memory usage- memory_monitor.update_cpu()+ # # Log AutoScheme memory usage+ # memory_monitor.update_cpu()low_cpu_str = "enabled" if auto_scheme.low_cpu_mem_usage else "disabled"memory_monitor.log_summary(f"AutoScheme complete (low_cpu_mem_usage={low_cpu_str})")@@ -1318,7 +1715,6 @@ def gen_layer_config(tokenizer=None,device_map=None,enable_torch_compile=False,- disable_opt_rtn=True,low_gpu_mem_usage=True,min_avg_bit_scheme=None,processor=None,@@ -1433,7 +1829,6 @@ def gen_layer_config(tokenizer=tokenizer,model_name=model_name,enable_torch_compile=enable_torch_compile,- disable_opt_rtn=disable_opt_rtn,device_map=device_map,major_device=major_device,device_list=device_list,@@ -1465,7 +1860,6 @@ def gen_layer_config(tokenizer=tokenizer,model_name=model_name,enable_torch_compile=enable_torch_compile,- disable_opt_rtn=disable_opt_rtn,device_map=device_map,major_device=major_device,device_list=device_list,diff --git a/auto_round/auto_scheme/gen_auto_scheme.py b/auto_round/auto_scheme/gen_auto_scheme.pyindex 737d7d43e..7068b7fbc 100644--- a/auto_round/auto_scheme/gen_auto_scheme.py+++ b/auto_round/auto_scheme/gen_auto_scheme.py@@ -38,7 +38,6 @@ class AutoScheme:dataset: Optional[str] = None # Import Notice no comma for each itemdevice_map: Optional[Union[str, torch.device, int, dict]] = Noneenable_torch_compile: Optional[bool] = None- disable_opt_rtn: bool = Truelow_gpu_mem_usage: bool = Truelow_cpu_mem_usage: bool = True@@ -46,6 +45,43 @@ class AutoScheme:if isinstance(self.options, str):options = self.options.upper().replace(" ", "")self.options = options.split(",")+ self.options = self._deduplicate_options(self.options)++ @staticmethod+ def _deduplicate_options(+ options: list[Union["QuantizationScheme", str]],+ ) -> list[Union["QuantizationScheme", str]]:+ """Remove options that resolve to the same quantization scheme.++ For example, GGUF:Q4_K_S and GGUF:Q4_K_M share identical quantization+ parameters (bits, group_size, sym, data_type, super_bits, super_group_size)+ so only the first one is kept.+ """+ from auto_round.schemes import QuantizationScheme, preset_name_to_scheme++ seen_schemes: list[QuantizationScheme] = []+ unique_options = []++ for opt in options:+ if isinstance(opt, str):+ try:+ scheme = preset_name_to_scheme(opt.upper())+ except KeyError:+ unique_options.append(opt)+ continue+ elif isinstance(opt, QuantizationScheme):+ scheme = opt+ else:+ unique_options.append(opt)+ continue++ if any(scheme == s for s in seen_schemes):+ logger.info("AutoScheme: removed duplicate option '%s' (same scheme already present).", opt)+ continue+ seen_schemes.append(scheme)+ unique_options.append(opt)++ return unique_optionsclass GenScheme:@@ -64,6 +100,9 @@ class GenScheme:processor=None,):self.auto_scheme = auto_scheme+ if self.auto_scheme.low_cpu_mem_usage:+ logger.info("force not using `low_cpu_mem_usage` in AutoScheme")+ self.auto_scheme.low_cpu_mem_usage = Falseself.model = modelself.tokenizer = tokenizerself.processor = processor@@ -76,7 +115,6 @@ class GenScheme:if self.auto_scheme.enable_torch_compile is Noneelse self.auto_scheme.enable_torch_compile)- self.disable_opt_rtn = self.auto_scheme.disable_opt_rtnself.min_avg_bit, self.max_avg_bit, self.min_avg_bit_scheme = self.compute_avg_bit_range()self._check_configs()@@ -129,7 +167,6 @@ class GenScheme:self.tokenizer,device_map=self.device_map,enable_torch_compile=self.enable_torch_compile,- disable_opt_rtn=self.disable_opt_rtn,low_gpu_mem_usage=self.auto_scheme.low_gpu_mem_usage,min_avg_bit_scheme=self.min_avg_bit_scheme,processor=self.processor,diff --git a/auto_round/compressors/entry.py b/auto_round/compressors/entry.pyindex ca6407420..23d11518e 100644--- a/auto_round/compressors/entry.py+++ b/auto_round/compressors/entry.py@@ -339,6 +339,9 @@ class AutoRound(object):) -> "BaseCompressor":from auto_round.utils.model import is_model_free_route+ if alg_configs is None:+ alg_configs = "auto_round"+device_map = normalize_default_device_map(device_map)split_kwargs = _split_entry_kwargs(kwargs)route_kwargs = dict(split_kwargs["route"])@@ -347,9 +350,6 @@ class AutoRound(object):mllm_kwargs = dict(split_kwargs["mllm"])diffusion_kwargs = dict(split_kwargs["diffusion"])- if alg_configs is None:- alg_configs = "auto_round"-# Resolve string alias(es) to config instance(s) before routing.alg_configs = cls._resolve_config(alg_configs)if isinstance(alg_configs, list):diff --git a/auto_round/utils/device_manager.py b/auto_round/utils/device_manager.pyindex 8ed74643c..14fd97d1b 100644--- a/auto_round/utils/device_manager.py+++ b/auto_round/utils/device_manager.py@@ -1070,6 +1070,7 @@ def _clear_memory_for_cpu_and_cuda(tensor[i] = Nonetensor = Nonegc.collect()+# Lazy import: malloc-trim helpers live in utils/device.py.from auto_round.utils.device import _maybe_trim_mallocdiff --git a/docs/auto_scheme_acc.md b/docs/auto_scheme_acc.mdindex 522a5f607..50d64f90b 100644--- a/docs/auto_scheme_acc.md+++ b/docs/auto_scheme_acc.md@@ -1,3 +1,48 @@+### The GGUF results++We tested with a fake model because the main branch currently has layer name mismatches between Transformers and GGUF.++~~~bash+python3 -m auto_round Qwen/Qwen3-8B --options "gguf:q2_k_s,gguf:q4_k_s" --target_bits 3.5 --ignore_scale_zp_bits --iters 0 --format fake --output_dir "./test_gguf" --tasks mmlu,lambada_openai+~~~++eval++~~~+ llm serve ./test_gguf/Qwen3-8B-w2g16/ --port 8001 --max-model-len 8192 --host 127.0.0.1 --served-model-name qwen+evalscope eval --model qwen --api-url http://127.0.0.1:8000/v1 --api-key EMPTY --datasets math_500 gpqa_diamoid mmlu_pro --eval-batch-size 32+~~~++| evalscope options q2ks,q4ks avgbits 3.5, ignore_scale_zp | math_500 | gpqa_diamond | mmlu_pro |+|----------------------------------------------------------|------------|---------------|----------|+| qwen3-8b bf16 | 0.79 | 0.4242 | 0.6934 |+| qwen3-8b (AR<=0.13.1) iters 0 | 0.76/0.768 | 0.3434/0.3687 | 0.6419 |+| qwen3-8b (AR>0.13.1) iters 0 | 0.782 | 0.4293 | 0.6763 |++| evalscope, options q2ks,q4ks avgbits 3.5, ignore_scale_zp | math_500 | gpqa_diamond | mmlu_pro |+|-----------------------------------------------------------|----------|--------------|----------|+| qwen3.5-4b bf16 | 0.48 | 0.4192 | 0.5891 |+| qwen3.5-4b (AR<=0.13.1) iters 0 | 0.228 | 0.298 | 0.4583 |+| pr branch (AR>0.13.1) iters 0 | 0.432 | 0.4444 | 0.5922 |++[//]: # (| options W2A16G64,W4A16 avgbits 3.5 ignore_scale_zp | math_500 | gpqa_diamond | mmlu_pro |)++[//]: # (| -------------------------------------------------- | -------- | ------------ | -------- |)++[//]: # (| qwen3-8b bf16 | 0.79 | 0.4242 | 0.6934 |)++[//]: # (| qwen3-8b main branch | 0.738 | 0.2778 | 0.576 |)++[//]: # (| main branch iters 200 enable_alg_ext | 0.766 | 0.3434 | |)++[//]: # (| pr branch | 0.72 | 0.2677 | - |)++[//]: # (| pr branch + iters 200 enable_alg_ext | 0.742 | 0.3232 | - |)++++### Other results+We use **lm-eval** for evaluation. For LLaMA, we enabled `add_bos_token` and`removed @use_kernel_forward_from_hub("RMSNorm")`in [modeling_llama.py](https://github.com/huggingface/transformers/blob/main/src/transformers/models/llama/modeling_llama.py#L52C1-L52C40)@@ -6,14 +51,15 @@ to stabilize accuracy during evaluation. All other settings follow the default cWe ignore the scale and zp bits in the tables below. The accuracy may change a little as we modified a little of theimplementation. We will rerun all the experiments.-For mxfp experiment, we use fake model while for weight only model we use real model. **No tuning is applied unless explicit stated.**+For mxfp experiment, we use fake model while for weight only model we use real model. **No tuning is applied unless+explicitly stated.***Average accuracy across `lambada_openai`, `hellaswag`, `piqa`, `winogrande`, and `mmlu`.*### Table 1 MXFP4/8 mixed accuracy.-| Average bits | Llama3.1-8B-I | Qwen2.5-7B-I | Qwen3-8B | Qwen3-32B |-|:------------------|:----------------:|:----------------:|:----------------:|:----------------:|+| Average bits | Llama3.1-8B-I | Qwen2.5-7B-I | Qwen3-8B | Qwen3-32B |+|:-----------------|:--------------:|:--------------:|:--------------:|:--------------:|| **BF16** | 0.7076 (100%) | 0.7075 (100%) | 0.6764 (100%) | 0.7321 (100%) || **Pure 4-bit** | 0.6626 (93.6%) | 0.6550 (92.6%) | 0.6316 (93.4%) | 0.6901 (94.3%) || **Ours 4.5-bit** | 0.6808 (96.2%) | 0.6776 (95.8%) | 0.6550 (96.8%) | 0.7176 (98.0%) |@@ -26,25 +72,24 @@ performance advantages.### Table 2 Comparison with other recipes at an average of 5 bits of mxfp datatype-| Avg. bits = 5 | Llama3.1-8B-I | Qwen2.5-7B-I | Qwen3-8B |-|:------------------|:----------------:|:----------------:|:----------------:|-| **Tail layers 8-bit** | 0.6671 (94.3%) | 0.6616 (93.5%) | 0.6410 (94.8%) |-| **Head layers 8-bit** | 0.6657 (94.1%) | 0.6686 (94.5%) | 0.6356 (94.0%) |+| Avg. bits = 5 | Llama3.1-8B-I | Qwen2.5-7B-I | Qwen3-8B |+|:----------------------|:------------------:|:------------------:|:------------------:|+| **Tail layers 8-bit** | 0.6671 (94.3%) | 0.6616 (93.5%) | 0.6410 (94.8%) |+| **Head layers 8-bit** | 0.6657 (94.1%) | 0.6686 (94.5%) | 0.6356 (94.0%) || **Ours** | **0.6857 (96.9%)** | **0.6823 (96.4%)** | **0.6594 (97.5%)** |### Table 3 Comparison with other recipes at an average of 4.5 bits of mxfp datatype-| Avg. bits = 4.5 | Llama3.1-8B-I | Qwen2.5-7B-I | Qwen3-8B |-|:------------------|:----------------:|:----------------:|:----------------:|-| **Tail layers 8-bit** | 0.6614 (93.5%) | 0.6535 (92.4%) | 0.6373 (94.2%) |-| **Head layers 8-bit** | 0.6568 (92.8%) | 0.6642 (93.9%) | 0.6305 (93.2%) |+| Avg. bits = 4.5 | Llama3.1-8B-I | Qwen2.5-7B-I | Qwen3-8B |+|:----------------------|:------------------:|:------------------:|:------------------:|+| **Tail layers 8-bit** | 0.6614 (93.5%) | 0.6535 (92.4%) | 0.6373 (94.2%) |+| **Head layers 8-bit** | 0.6568 (92.8%) | 0.6642 (93.9%) | 0.6305 (93.2%) || **Ours** | **0.6808 (96.2%)** | **0.6776 (95.5%)** | **0.6550 (95.8%)** |-### Table4 Comparison with other recipes at an average of 3 bits of W2G128 and W4G128| Avg. bits = 4.5 | Llama3.1-8B-I | Qwen2.5-7B-I | Qwen3-8B |-|:------------------|:----------------:|:----------------:|:----------------:|-| **Tail layers 4-bit** | 0.6058 | 0.3798 | 0.4536 |-| **Head layers 4-bit** | 0.3198 | 0.3270 | 0.3196 |-| **Ours** | 0.6148 | 0.4058 | 0.4862 |+|:----------------------|:-------------:|:------------:|:--------:|+| **Tail layers 4-bit** | 0.6058 | 0.3798 | 0.4536 |+| **Head layers 4-bit** | 0.3198 | 0.3270 | 0.3196 |+| **Ours** | 0.6148 | 0.4058 | 0.4862 |