PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
New architecture for auto_round (#1542)
Heng Guo91585c7d5fbc
diff --git a/auto_round/__main__.py b/auto_round/__main__.pyindex 8cb79004d..0fe576c75 100644--- a/auto_round/__main__.py+++ b/auto_round/__main__.py@@ -38,6 +38,7 @@ RECIPES = {class BasicArgumentParser(argparse.ArgumentParser):+def __init__(self, *args, **kwargs):super().__init__(*args, **kwargs)self.add_argument(@@ -750,45 +751,10 @@ def tune(args):rotation_config=rot_config,)- model_name = args.model.rstrip("/")-- if model_name.split("/")[-1].strip(".") == "" and "gguf" not in args.format:- if autoround.group_size <= 0:- if "fp" in autoround.act_data_type:- suffix = f"afp{autoround.act_bits}"- else:- suffix = f"a{autoround.act_bits}"- else:- suffix = f"g{autoround.group_size}"- export_dir = os.path.join(args.output_dir, f"w{autoround.bits}{suffix}")- elif model_name.split("/")[-1].strip(".") == "" and "gguf" in args.format:- export_dir = args.output_dir- elif model_name.split("./")[-1].strip("./") != "" and "gguf" in args.format:- export_dir = os.path.join(args.output_dir, model_name.split("/")[-1] + "-gguf")- else:- if isinstance(autoround.group_size, tuple):- assert len(autoround.group_size) == 2, f"Only support 2D group_size, but get {autoround.group_size}"- suffix = f"g{autoround.group_size[0]}x{autoround.group_size[1]}"- else:- if autoround.group_size <= 0:- if "fp" in autoround.act_data_type:- suffix = f"afp{autoround.act_bits}"- else:- suffix = f"a{autoround.act_bits}"- else:- suffix = f"g{autoround.group_size}"- prefix = (- autoround.data_type.lower().replace("_", "")- if "int" not in autoround.data_type or "mx" in autoround.data_type- else ""- )- export_dir = os.path.join(- args.output_dir,- model_name.split("/")[-1] + (f"-{prefix}" if prefix else "") + f"-w{autoround.bits}{suffix}",- )-# ======================= Quantize and save model =======================- model, folders = autoround.quantize_and_save(export_dir, format=args.format) # pylint: disable=E1101+ # Export directory is now derived automatically inside quantize_and_save via+ # BaseCompressor._get_export_dir(), so we only need to pass the base output_dir.+ model, folders = autoround.quantize_and_save(args.output_dir, format=args.format) # pylint: disable=E1101tokenizer = autoround.tokenizer # pylint: disable=E1101model.eval()diff --git a/auto_round/algorithms/__init__.py b/auto_round/algorithms/__init__.pynew file mode 100644index 000000000..14a492441--- /dev/null+++ b/auto_round/algorithms/__init__.py@@ -0,0 +1,13 @@+# Copyright (c) 2026 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.diff --git a/auto_round/algorithms/alg_config.py b/auto_round/algorithms/alg_config.pynew file mode 100644index 000000000..d9d5f0c75--- /dev/null+++ b/auto_round/algorithms/alg_config.py@@ -0,0 +1,18 @@+# Copyright (c) 2026 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+++class AlgConfig:+ def __init__(self):+ passdiff --git a/auto_round/algorithms/base.py b/auto_round/algorithms/base.pynew file mode 100644index 000000000..4590536c5--- /dev/null+++ b/auto_round/algorithms/base.py@@ -0,0 +1,17 @@+# Copyright (c) 2026 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+++class BaseAlgorithm:+ passdiff --git a/auto_round/algorithms/quantization/__init__.py b/auto_round/algorithms/quantization/__init__.pynew file mode 100644index 000000000..6a727f31b--- /dev/null+++ b/auto_round/algorithms/quantization/__init__.py@@ -0,0 +1,21 @@+# Copyright (c) 2026 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.++from auto_round.algorithms.quantization.base import BaseQuantizers+from auto_round.algorithms.quantization.config import QuantizationConfig+from auto_round.algorithms.quantization.sign_round.config import SignRoundConfig+from auto_round.algorithms.quantization.sign_round.quantizer import SignRoundQuantizer+from auto_round.algorithms.quantization.adam_round.adam import AdamRoundQuantizer+from auto_round.algorithms.quantization.rtn.config import RTNConfig+from auto_round.algorithms.quantization.rtn.quantizer import RTNQuantizer, OptimizedRTNQuantizerdiff --git a/auto_round/algorithms/quantization/adam_round/__init__.py b/auto_round/algorithms/quantization/adam_round/__init__.pynew file mode 100644index 000000000..14a492441--- /dev/null+++ b/auto_round/algorithms/quantization/adam_round/__init__.py@@ -0,0 +1,13 @@+# Copyright (c) 2026 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.diff --git a/auto_round/algorithms/quantization/adam_round/adam.py b/auto_round/algorithms/quantization/adam_round/adam.pynew file mode 100644index 000000000..96835b533--- /dev/null+++ b/auto_round/algorithms/quantization/adam_round/adam.py@@ -0,0 +1,66 @@+# Copyright (c) 2026 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+from typing import Union++import torch++from auto_round.algorithms.quantization.sign_round.quantizer import SignRoundQuantizer+from auto_round.schemes import QuantizationScheme+from auto_round.utils import check_is_cpu, htcore, is_hpex_available+++class AdamRoundQuantizer(SignRoundQuantizer):++ def __init__(self, config):+ super().__init__(config)+ self.momentum = None # AdamW handles momentum internally++ def _get_optimizer(self, optimizer):+ if optimizer is None:+ optimizer = torch.optim.AdamW+ elif isinstance(optimizer, str):+ optimizer = getattr(torch.optim, optimizer)+ else:+ optimizer = optimizer+ return optimizer++ def _get_scaler(self):+ scaler = None+ if self.model_context.amp and not check_is_cpu(self.compress_context.device):+ from torch.cuda.amp import GradScaler++ scaler = GradScaler(init_scale=1024, growth_interval=100000)+ return scaler++ def _scale_loss_and_backward(self, scaler, loss):+ if scaler is not None:+ loss = scaler.scale(loss)++ loss.backward()+ if is_hpex_available():+ htcore.mark_step()+ return loss++ def _step(self, scaler, optimizer, lr_schedule):+ if scaler is not None:+ scaler.step(optimizer)+ optimizer.zero_grad()+ lr_schedule.step()+ scaler.update()+ else:+ optimizer.step()+ optimizer.zero_grad()+ lr_schedule.step()+ if is_hpex_available():+ htcore.mark_step()diff --git a/auto_round/algorithms/quantization/base.py b/auto_round/algorithms/quantization/base.pynew file mode 100644index 000000000..12d972554--- /dev/null+++ b/auto_round/algorithms/quantization/base.py@@ -0,0 +1,562 @@+# Copyright (c) 2026 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+import importlib+import traceback+from collections import defaultdict+from typing import Union++import torch++from auto_round.algorithms.quantization.config import QuantizationConfig+from auto_round.compressors_new.utils import (+ block_forward,+ check_need_act_calibration,+)+from auto_round.data_type import QUANT_FUNC_WITH_DTYPE+from auto_round.data_type.utils import reshape_pad_tensor_by_group_size+from auto_round.logger import logger+from auto_round.utils import (+ INNER_SUPPORTED_LAYER_TYPES,+ SUPPORTED_LAYER_TYPES,+ check_to_quantized,+ clear_memory,+ compile_func,+)+++class BaseQuantizers:+ # Class-level attribute declarations for convenient access in quantization methods.+ # Scheme-related attrs (layer_config, scale_dtype, has_qlayer_outside_block, etc.)+ # are resolved by SchemeMixin in BaseCompressor and synced here after post_init().+ model_context = None+ compress_context = None+ dataset = None+ supported_types = SUPPORTED_LAYER_TYPES+ inner_supported_types = INNER_SUPPORTED_LAYER_TYPES+ enable_alg_ext = False+ # Subclasses that support diffusion models should override this with the+ # appropriate output key mapping, e.g.:+ # DIFFUSION_OUTPUT_CONFIGS = {"FluxTransformerBlock": ["encoder_hidden_states", "hidden_states"]}+ DIFFUSION_OUTPUT_CONFIGS: dict = {+ "FluxTransformerBlock": ["encoder_hidden_states", "hidden_states"],+ "FluxSingleTransformerBlock": ["encoder_hidden_states", "hidden_states"],+ "OvisImageTransformerBlock": ["encoder_hidden_states", "hidden_states"],+ "OvisImageSingleTransformerBlock": ["encoder_hidden_states", "hidden_states"],+ }++ def __init__(self, config: QuantizationConfig):+ self.config = config+ self.layer_config = None+ self.bits = config.bits+ self.group_size = config.group_size+ self.sym = config.sym+ self.data_type = config.data_type+ self.act_bits = config.act_bits+ self.act_group_size = config.act_group_size+ self.act_sym = config.act_sym+ self.act_data_type = config.act_data_type+ self.act_dynamic = config.act_dynamic+ self.super_bits = config.super_bits+ self.super_group_size = config.super_group_size+ self.scale_dtype = config.scale_dtype+ self.ignore_layers = config.ignore_layers+ self.quant_lm_head = config.quant_lm_head+ self.to_quant_block_names = config.to_quant_block_names+ # Calibration / sampling attrs – synced from compressor in post_init.+ self.seqlen = 2048+ self.nsamples = 128+ self.batch_size = getattr(config, "batch_size", 8)+ self.batch_dim = getattr(config, "batch_dim", None)+ self.infer_bs_coeff = getattr(config, "infer_bs_coeff", 1)+ # Whether to feed quantized-block outputs as inputs to the next block.+ # Subclasses that support cascaded quantized-input (e.g. SignRoundQuantizer)+ # override this from their config. Defaults to False for zero-shot algorithms+ # (RTN) where activations are not used during weight optimization.+ self.enable_quanted_input = getattr(config, "enable_quanted_input", False)++ @classmethod+ def from_config(cls, config: QuantizationConfig):+ if cls.__name__ == config._alg_cls:+ return cls(config)+ else:+ module = importlib.import_module("auto_round.algorithms.quantization")+ alg_cls = getattr(module, config._alg_cls)+ return alg_cls(config)++ @property+ def formats(self):+ return getattr(self.compress_context, "formats", None)++ @property+ def amp(self):+ return getattr(self.model_context, "amp", False)++ @property+ def amp_dtype(self):+ import torch++ return getattr(self.model_context, "amp_dtype", torch.float32)++ def _register_act_max_hook(self, model):++ def get_act_max_hook(module, input, output):+ if isinstance(input, (tuple, list)):+ input = input[0]+ if input.numel() == 0:+ return # as no needs for act_max update+ input, _, _ = reshape_pad_tensor_by_group_size(input, self.act_group_size)+ act_max = torch.max(torch.abs(input), dim=-1).values+ if not hasattr(module, "act_max") or module.act_max.numel() == 0:+ module.act_max = act_max+ if self.config.is_act_nv_fp: ## for nvfp per-tensor input_global_scale calculation usage+ max_val = act_max.max()+ module.act_max = max_val.unsqueeze(0) if max_val.dim() == 0 else max_val+ else:+ act_max = act_max.to(module.act_max.device)+ if self.config.is_act_nv_fp: ## for nvfp per-tensor input_global_scale calculation usage+ max_val = torch.max(act_max.max(), module.act_max.max())+ module.act_max = max_val.unsqueeze(0) if max_val.dim() == 0 else max_val+ else:+ module.act_max = torch.max(act_max, module.act_max)++ hook_handles = []+ # for single layers out of blocks, like lm_head+ if isinstance(model, SUPPORTED_LAYER_TYPES):+ m = model+ if (+ hasattr(m, "act_dynamic")+ and check_need_act_calibration(m.act_dynamic, m.act_data_type, m.act_bits)+ and check_to_quantized(m)+ ):+ hook = m.register_forward_hook(get_act_max_hook)+ hook_handles.append(hook)+ return hook_handles++ for n, m in model.named_modules():+ if (+ hasattr(m, "act_dynamic")+ and check_need_act_calibration(m.act_dynamic, m.act_data_type, m.act_bits)+ and check_to_quantized(m)+ ):+ hook = m.register_forward_hook(get_act_max_hook)+ hook_handles.append(hook)+ continue++ # for whole model, RTN+ if n in self.layer_config:+ config = self.layer_config[n]+ act_dynamic = config.get("act_dynamic", True)+ act_data_type = config.get("act_data_type", None)+ act_bits = config.get("act_bits", 16)+ if (+ config["bits"] <= 8+ and check_need_act_calibration(act_dynamic, act_data_type, act_bits)+ and check_to_quantized(config)+ ):+ hook = m.register_forward_hook(get_act_max_hook)+ hook_handles.append(hook)+ continue+ return hook_handles++ @torch.inference_mode()+ def _quantize_embedding_layer(self):+ """Quantizes embedding layers in the model according to the configuration.++ This method iterates through all modules in the model, identifies embedding+ layers specified in `self.quantizer.layer_config`, and applies the appropriate quantization+ function based on bit precision, grouping strategy, and dtype.++ Returns:+ bool: True if the quantization process completes without critical errors.+ """+ is_quantized = False+ for name, module in self.model_context.model.named_modules():+ # Skip non-Embedding modules or layers not in config+ if not isinstance(module, torch.nn.Embedding) or name not in self.layer_config:+ continue++ config = self.layer_config[name]++ # Skip layers that are not marked for quantization+ if not check_to_quantized(config):+ continue+ is_quantized = True+ config["scale_dtype"] = self.scale_dtype+ dtype = config["data_type"]++ # Determine quantization function key with symmetry/asymmetry+ if dtype not in QUANT_FUNC_WITH_DTYPE:+ dtype = f"{dtype}_{'sym' if config['sym'] else 'asym'}"++ quant_func = QUANT_FUNC_WITH_DTYPE[dtype]+ dtype = module.weight.dtype+ # As typically float32 are used in RTN to search scale zp,+ # to avoid cache a bf16 copy we'd better use float32+ if config.get("super_group_size", None) is not None:+ dtype = torch.float32++ # Attempt quantization on GPU, fall back to CPU if OOM+ try:+ weight, scale, zp = quant_func(+ module.weight.to(dtype=dtype, device=self.compress_context.device),+ **{+ k: config.get(k, None)+ for k in ["bits", "group_size", "super_bits", "super_group_size", "scale_dtype"]+ },+ )+ except torch.OutOfMemoryError:+ cuda_error_msg = traceback.format_exc()+ try:+ logger.error(cuda_error_msg)+ logger.warning("falling back to CPU")+ weight, scale, zp = quant_func(+ module.weight.to("cpu"),+ **{+ k: config.get(k, None)+ for k in ["bits", "group_size", "super_bits", "super_group_size", "scale_dtype"]+ },+ )+ except Exception as e:+ raise++ # Overwrite the module's weights with the quantized version+ module.weight.data.copy_(weight.cpu())++ # Attach scale and zero point (zp) to the module+ for param_name, value in zip(["scale", "zp"], [scale, zp]):+ if isinstance(value, dict):+ for k, v in value.items():+ setattr(module, k if k == "scale" else f"w_{k}", v.cpu())+ elif isinstance(value, torch.Tensor):+ setattr(module, param_name, value.cpu())+ else:+ setattr(module, param_name, value)++ # Update config+ self.layer_config.setdefault(name, {}).update(config)+ del weight+ del scale+ del zp+ clear_memory(device_list=self.compress_context.device_list)++ return is_quantized++ def quantize_block(+ self, block: torch.nn.Module, input_ids=None, input_others=None, reference_output=None, **kwargs+ ) -> dict:+ """Apply the quantization algorithm to a prepared block.++ This is the **pure-algorithm** entry point called by the Compressor after+ all infrastructure work (device placement, data collection, act-max hook+ registration, DDP setup) has been completed.++ Implementations should:+ - Perform the algorithm-specific weight/activation quantization on ``block``.+ - Return a dict of best parameters (may be empty for zero-shot algorithms).++ Args:+ block: Module already placed on the correct device(s).+ input_ids: Calibration inputs on cache_device (None for zero-shot RTN).+ input_others: Additional inputs (None for zero-shot RTN).+ reference_output: FP reference outputs collected by Compressor+ (None for algorithms that don't need a reconstruction loss).+ **kwargs: Algorithm-specific keyword arguments (e.g. ``loss_device``,+ ``card_0_in_high_risk`` for SignRoundQuantizer).++ Returns:+ dict: Best quantization parameters found, or ``{}`` if not applicable.+ """+ raise NotImplementedError("quantize_block must be implemented in subclasses of BaseQuantizers")++ def quantize_layer(self, layer_name: str, **kwargs):+ """Quantizes a single layer of the model.++ Args:+ layer_name (str): The name of the layer to quantize. The layer module is+ retrieved internally via get_module(model, layer_name).+ """+ raise NotImplementedError("quantize_layer must be implemented in subclasses of BaseQuantizers")++ def quantize_layer_outside_block(self, layer_name: str, **kwargs):+ """Quantizes a single layer of the model outside of a block.++ Args:+ layer_name (str): The name of the layer to quantize. The layer module is+ retrieved internally via get_module(model, layer_name).+ """+ raise NotImplementedError("quantize_layer_outside_block must be implemented in subclasses of BaseQuantizers")++ @torch.no_grad()+ def _get_block_outputs(+ self,+ block: torch.nn.Module,+ input_ids,+ input_others,+ bs: int,+ save_output: bool = True,+ ):+ """Compute the output of a block for calibration inputs.++ Shared by SignRoundQuantizer and OptimizedRTNQuantizer. Algorithm-specific+ block-forward selection (compile vs. plain) is handled here based on+ ``enable_alg_ext`` and act-quantization flags.+ """+ diffusion_fn = getattr(self, "_get_diffusion_block_outputs", None)+ if getattr(self.model_context, "is_diffusion", False):+ return self._get_diffusion_block_outputs(+ block,+ input_ids,+ input_others,+ bs,+ self.compress_context.device,+ self.compress_context.cache_device,+ )++ _bf = self._resolve_block_forward()++ output = []+ nsamples = len(input_ids)+ for i in range(0, nsamples, bs):+ end_index = min(nsamples, i + bs)+ indices = torch.arange(i, end_index).to(torch.long)+ tmp_input_ids, tmp_input_others = self._sampling_inputs(+ input_ids,+ input_others,+ indices,+ self.seqlen,+ self.batch_dim,+ share_cache_keys=self.model_context.shared_cache_keys,+ )+ tmp_output = _bf(+ block,+ tmp_input_ids,+ tmp_input_others,+ self.model_context.amp,+ self.model_context.amp_dtype,+ self.compress_context.device,+ ).to(self.compress_context.cache_device)+ if save_output:+ if self.batch_size == 1:+ output.append(tmp_output)+ else:+ output.extend(list(torch.split(tmp_output, 1, dim=self.batch_dim)))+ self.compress_context.clear_memory()++ return output++ def _resolve_block_forward(self):+ """Resolve and cache the block forward function once.++ This avoids repeated attribute checks in the hot training loop+ (called thousands of times per block).++ For activation-quantization schemes (e.g. FP8_STATIC) or when+ algorithm extensions are enabled, forward hooks are attached to layers+ inside the block. ``torch.compile`` is incompatible with these hooks,+ so we must fall back to the plain ``block_forward``. This mirrors the+ old-arch behaviour where ``self.block_forward`` was set in ``__init__``+ to the uncompiled function for these cases.+ """+ cached = self.__dict__.get("_resolved_block_forward")+ if cached is not None:+ return cached+ # Act-quantization hooks / alg-extension hooks are incompatible with+ # torch.compile → always use the plain (uncompiled) block_forward.+ if (+ self.config.is_act_quantize and (not self.config.act_dynamic or self.config.is_act_nv_fp)+ ) or self.enable_alg_ext:+ self._resolved_block_forward = block_forward+ elif self.compress_context.enable_torch_compile:+ compiled = self.__dict__.get("_compiled_block_forward")+ if compiled is None:+ compiled = compile_func(block_forward, self.compress_context.device)+ self._compiled_block_forward = compiled+ self._resolved_block_forward = compiled+ else:+ self._resolved_block_forward = block_forward+ return self._resolved_block_forward++ def _invalidate_block_forward_cache(self):+ """Clear the cached block forward function (call when block changes)."""+ self.__dict__.pop("_resolved_block_forward", None)+ self.__dict__.pop("_compiled_block_forward", None)++ def _get_current_q_output(+ self,+ block: torch.nn.Module,+ input_ids,+ input_others: dict,+ indices,+ device,+ cache_device: str = "cpu",+ ) -> torch.Tensor:+ """Compute block output for a mini-batch selected by *indices* (used during training).++ Handles both LLM and diffusion model block formats. Uses the compiled+ block_forward when enable_torch_compile is True (same as _get_block_outputs),+ matching old-arch behaviour where self.block_forward was compiled at init.+ """+ current_input_ids, current_input_others = self._sampling_inputs(+ input_ids,+ input_others,+ indices,+ seqlen=self.seqlen,+ batch_dim=self.batch_dim,+ share_cache_keys=self.model_context.shared_cache_keys,+ )+ _bf = self._resolve_block_forward()++ if getattr(self.model_context, "is_diffusion", False):+ output_config = self.DIFFUSION_OUTPUT_CONFIGS.get(block.__class__.__name__, ["hidden_states"])+ idx = None if "hidden_states" not in output_config else output_config.index("hidden_states")+ if isinstance(current_input_ids, dict):+ hidden_states = current_input_ids.pop("hidden_states")+ current_input_others.update(current_input_ids)+ current_input_ids = hidden_states+ output_q = _bf(+ block,+ current_input_ids,+ current_input_others,+ self.model_context.amp,+ self.model_context.amp_dtype,+ device,+ idx,+ )+ else:+ output_q = _bf(+ block,+ current_input_ids,+ current_input_others,+ self.model_context.amp,+ self.model_context.amp_dtype,+ device,+ )+ return output_q.to(cache_device)++ @classmethod+ @torch.no_grad()+ def _sampling_inputs(+ cls,+ input_ids: Union[list[torch.Tensor], dict],+ input_others: dict,+ indices,+ seqlen: int,+ batch_dim: int = 0,+ share_cache_keys: tuple = (),+ ):+ """Sample a mini-batch of calibration inputs by indices.++ Shared by SignRoundQuantizer and OptimizedRTNQuantizer.+ """+ if isinstance(input_ids, list):+ current_input_ids = [input_ids[i] for i in indices]+ current_input_ids = torch.cat(current_input_ids, dim=batch_dim)+ elif isinstance(input_ids, dict):+ current_input_ids = defaultdict(list)+ for k in input_ids.keys():+ current_input_ids[k].extend([input_ids[k][i] for i in indices])+ current_input_ids[k] = torch.cat(current_input_ids[k], dim=batch_dim)++ current_input_others = {"positional_inputs": input_others["positional_inputs"]}+ for key in input_others.keys():+ if "positional_inputs" in key:+ continue+ if key in share_cache_keys:+ # Shared keys are stored once (not per-sample), often wrapped in a+ # 1-element list by the caching hook. Unwrap so the model receives+ # the raw value (e.g. (cos, sin) tuple, not [(cos, sin)]).+ val = input_others[key]+ if isinstance(val, list) and len(val) == 1:+ current_input_others[key] = val[0]+ else:+ current_input_others[key] = val+ elif not isinstance(input_others[key], (str, bool, type(None))):+ current_input_others[key] = None+ if input_others[key] is not None:+ current_input_others[key] = [input_others[key][i] for i in indices]+ if len(indices) == 1:+ current_input_others[key] = current_input_others[key][0]+ else:+ try:+ current_input_others[key] = torch.cat(current_input_others[key], dim=0)+ except TypeError as err:+ logger.warning_once("Please check the model cache inputs or try setting batch_size to 1.")+ else:+ current_input_others[key] = input_others[key]++ return current_input_ids, current_input_others++ @torch.no_grad()+ def _get_diffusion_block_outputs(+ self,+ block: torch.nn.Module,+ input_ids: Union[torch.Tensor, dict],+ input_others,+ bs: int,+ device: Union[str, torch.device],+ cache_device: Union[str, torch.device],+ save_output: bool = True,+ ):+ """Compute block outputs for diffusion models.++ Uses ``self.DIFFUSION_OUTPUT_CONFIGS`` to map block class names to their+ output keys. Subclasses override ``DIFFUSION_OUTPUT_CONFIGS`` to add+ support for new diffusion architectures.+ """+ output = defaultdict(list)+ output_config = self.DIFFUSION_OUTPUT_CONFIGS.get(block.__class__.__name__, ["hidden_states"])+ if isinstance(input_ids, dict):+ nsamples = len(input_ids["hidden_states"])+ else:+ nsamples = len(input_ids)++ for i in range(0, nsamples, bs):+ end_index = min(nsamples, i + bs)+ indices = torch.arange(i, end_index).to(torch.long)+ tmp_input_ids, tmp_input_others = self._sampling_inputs(+ input_ids,+ input_others,+ indices,+ self.seqlen,+ self.batch_dim,+ share_cache_keys=self.model_context.shared_cache_keys,+ )+ if isinstance(tmp_input_ids, dict):+ hidden_states = tmp_input_ids.pop("hidden_states")+ tmp_input_others.update(tmp_input_ids)+ tmp_input_ids = hidden_states++ tmp_output = block_forward(+ block,+ tmp_input_ids,+ tmp_input_others,+ self.model_context.amp,+ self.model_context.amp_dtype,+ device,+ None,+ )+ if isinstance(tmp_output, torch.Tensor):+ tmp_output = [tmp_output]+ assert len(output_config) == len(tmp_output)+ tmp_output = dict(zip(output_config, tmp_output))++ if save_output:+ for name, out in tmp_output.items():+ if self.batch_size == 1:+ output[name].append(out.to(cache_device))+ else:+ output[name].extend(list(torch.split(out.to(cache_device), 1, dim=self.batch_dim)))+ self.compress_context.clear_memory()++ return outputdiff --git a/auto_round/algorithms/quantization/config.py b/auto_round/algorithms/quantization/config.pynew file mode 100644index 000000000..d99219e51--- /dev/null+++ b/auto_round/algorithms/quantization/config.py@@ -0,0 +1,227 @@+# Copyright (c) 2026 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+from dataclasses import dataclass+from enum import Enum+from typing import ClassVar, Union++from auto_round.algorithms.alg_config import AlgConfig+from auto_round.export.export_to_gguf.config import GGUF_INNER_CONFIG+from auto_round.logger import logger+from auto_round.schemes import QuantizationScheme+++class BackendDataType(str, Enum):+ STANDARD_FP = "fp"+ MX_FP = "mx_fp"+ NV_FP = "nv_fp"+ FP8_STATIC = "fp8_static"+ FP8 = "fp8"+++@dataclass(kw_only=True)+class QuantizationConfig:+ _alg_cls: ClassVar[str] = None++ # quantization args+ bits: int = None+ group_size: int = None+ sym: bool = None+ data_type: str = None+ act_bits: int = None+ act_group_size: int = None+ act_sym: bool = None+ act_data_type: str = None+ act_dynamic: bool = None+ super_bits: int = None+ super_group_size: int = None+ scale_dtype: str = None+ ignore_layers: str = ""+ quant_lm_head: bool = False+ to_quant_block_names: Union[str, list, None] = None++ def __post_init__(self):+ # Run block-wise validation early (at construction time, before model loading).+ # Scheme resolution is deferred to BaseCompressor.post_init() via SchemeMixin.+ # Guard with None checks in case the user hasn't explicitly set data_type/bits+ # (they will be resolved from scheme by the compressor before use).+ if self.group_size is not None and isinstance(self.group_size, (tuple, list)):+ if not (+ self.data_type is not None+ and self.bits is not None+ and self.data_type.startswith("fp")+ and self.bits == 8+ ):+ raise ValueError(+ "Block-wise quantization (tuple group_size) only supports fp8 weight quantization, "+ f"but got data_type='{self.data_type}', bits={self.bits}."+ )+ if (+ self.act_dynamic is not None+ and self.act_data_type is not None+ and self.act_bits is not None+ and not (self.act_dynamic and self.act_data_type.startswith("fp") and self.act_bits == 8)+ ):+ raise NotImplementedError(+ "Block-wise fp8 weight quantization only supports dynamic fp8 activation quantization. "+ f"Got act_dynamic={self.act_dynamic}, act_data_type='{self.act_data_type}', "+ f"act_bits={self.act_bits}."+ )+ if self.act_group_size is not None and isinstance(self.act_group_size, (tuple, list)):+ raise ValueError(+ "`act_group_size` must be -1 (per channel), 0 (per-tensor), or a positive integer, not a tuple."+ )++ @staticmethod+ def _is_valid_group_size(gs) -> bool:+ """Return True if gs is a valid group_size value.++ Accepts -1 (per-channel), 0 (per-tensor), a positive integer,+ or a tuple/list of such values (e.g. (128, 128) for block-wise FP8).+ """+ if isinstance(gs, (tuple, list)):+ return all(QuantizationConfig._is_valid_group_size(g) for g in gs)+ return gs == -1 or gs >= 0++ def check_config(self) -> None:+ """Checks if the configurations are valid.++ Raises:+ ValueError, TypeError: If any of the configurations are invalid.+ """+ if self.bits <= 0:+ raise ValueError("`bits` must be positive")+ if self.act_bits <= 0:+ raise ValueError("`act_bits` must be positive")+ if not self._is_valid_group_size(self.group_size):+ raise ValueError(+ "`group_size` must be -1 (per channel), 0 (per-tensor), a positive integer, "+ "or a tuple thereof (e.g. (128, 128) for block-wise quantization)"+ )+ if isinstance(self.act_group_size, (tuple, list)):+ raise ValueError(+ "`act_group_size` must be -1 (per channel), 0 (per-tensor), or a positive integer, not a tuple."+ )+ if not self._is_valid_group_size(self.act_group_size):+ raise ValueError(+ "`act_group_size` must be -1 (per channel), 0 (per-tensor), a positive integer, " "or a tuple thereof"+ )+ # Block-wise (tuple group_size) is only valid for fp8 weight quantization+ if isinstance(self.group_size, (tuple, list)):+ if not (self.data_type.startswith("fp") and self.bits == 8):+ raise ValueError(+ "Block-wise quantization (tuple group_size) only supports fp8 weight quantization, "+ f"but got data_type='{self.data_type}', bits={self.bits}."+ )+ if not (self.act_dynamic and self.act_data_type.startswith("fp") and self.act_bits == 8):+ raise NotImplementedError(+ "Block-wise fp8 weight quantization only supports dynamic fp8 activation quantization. "+ f"Got act_dynamic={self.act_dynamic}, act_data_type='{self.act_data_type}', "+ f"act_bits={self.act_bits}."+ )+ # Reset the default value of super_bits and super_group_size+ if self.data_type.endswith("_dq"):+ gguf_config = GGUF_INNER_CONFIG[f"gguf:q{self.bits}_k"]+ self.super_bits = gguf_config.get("super_bits", None) if self.super_bits is None else self.super_bits+ self.super_group_size = (+ gguf_config.get("super_group_size", None) if self.super_group_size is None else self.super_group_size+ )++ if (+ self.is_act_quantize+ and (not self.is_act_nv_fp or "static_gs" not in self.act_data_type)+ and not self.is_act_mx_fp+ and not self.is_dynamic_wint8aint8+ and not self.is_static_afp8+ ):+ logger.warning(+ "activation quantization is an experimental feature with limited support and a complex API. "+ "And please save the quantized model to fake format as real deployment is not supported currently"+ )+ # For block-wise group_size (tuple), skip the scalar-only warnings+ scalar_gs = self.group_size if not isinstance(self.group_size, (tuple, list)) else None+ if self.is_mx_fp and scalar_gs != 32:+ logger.warning("dtype mx_fp should only support group_size of 32 in real deployment")+ if self.is_nv_fp and scalar_gs != 16:+ logger.warning("dtype nv_fp should only support group_size of 16 in real deployment")++ @property+ def is_act_quantize(self):+ return self.act_bits is not None and self.act_bits <= 8++ @property+ def is_nv_fp(self):+ return self.data_type is not None and BackendDataType.NV_FP in self.data_type++ @property+ def is_act_nv_fp(self):+ return self.act_data_type is not None and BackendDataType.NV_FP in self.act_data_type++ @property+ def is_mx_fp(self):+ return self.data_type is not None and BackendDataType.MX_FP in self.data_type++ @property+ def is_act_mx_fp(self):+ return self.act_data_type is not None and BackendDataType.MX_FP in self.act_data_type++ @property+ def is_dynamic_wint8aint8(self):+ if self.act_dynamic:+ return True+ if self.act_data_type is not None and self.data_type is not None:+ if ("int8" in self.act_data_type or ("int" in self.act_data_type and self.act_bits == 8)) and (+ "int8" in self.data_type or ("int" in self.data_type and self.bits == 8)+ ):+ return True+ return False++ @property+ def is_standard_fp(self):+ return (+ self.data_type is not None+ and BackendDataType.STANDARD_FP in self.data_type+ and not self.is_mx_fp+ and not self.is_nv_fp+ )++ @property+ def is_act_standard_fp(self):+ return (+ self.act_data_type is not None+ and BackendDataType.STANDARD_FP in self.act_data_type+ and not self.is_act_mx_fp+ and not self.is_act_nv_fp+ )++ @property+ def is_static_afp8(self):+ return self.act_data_type is not None and BackendDataType.FP8_STATIC in self.act_data_type++ @property+ def is_static_wfp8afp8(self):+ return self.data_type is not None and BackendDataType.FP8_STATIC in self.data_type and self.is_static_afp8++ @property+ def is_wfp8afp8(self):+ if self.act_data_type is None or self.data_type is None:+ return False+ if (+ ("fp8" in self.act_data_type or ("fp" in self.act_data_type and self.act_bits == 8))+ and ("fp8" in self.data_type or ("fp" in self.data_type and self.bits == 8))+ and self.is_act_standard_fp+ and self.is_standard_fp+ ):+ return True+ else:+ return Falsediff --git a/auto_round/algorithms/quantization/rtn/__init__.py b/auto_round/algorithms/quantization/rtn/__init__.pynew file mode 100644index 000000000..14a492441--- /dev/null+++ b/auto_round/algorithms/quantization/rtn/__init__.py@@ -0,0 +1,13 @@+# Copyright (c) 2026 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.diff --git a/auto_round/algorithms/quantization/rtn/config.py b/auto_round/algorithms/quantization/rtn/config.pynew file mode 100644index 000000000..6afc41b0f--- /dev/null+++ b/auto_round/algorithms/quantization/rtn/config.py@@ -0,0 +1,56 @@+# Copyright (c) 2026 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.++from auto_round.algorithms.quantization.config import QuantizationConfig+from auto_round.data_type import QUANT_FUNC_WITH_DTYPE+from auto_round.logger import logger+++class RTNConfig(QuantizationConfig):+ _alg_cls = "RTNQuantizer"++ def __init__(+ self,+ *,+ disable_opt_rtn: bool = None,+ # for opt-rtn+ batch_size: int = 8,+ **kwargs,+ ):+ # pop before super().__init__ so it doesn't leak into QuantizationConfig as an unknown kwarg+ enable_opt_rtn = kwargs.pop("enable_opt_rtn", None)+ super().__init__(**kwargs)++ self.batch_size = batch_size++ # Some helpers+ self.infer_bs_coeff = 1+ self.batch_dim = None++ if enable_opt_rtn:+ disable_opt_rtn = False+ self.orig_disable_opt_rtn = disable_opt_rtn++ if disable_opt_rtn is None:+ if self.bits and self.bits >= 8 and self.act_bits and self.act_bits >= 8 and self.data_type == "int":+ logger.warning("`disable_opt_rtn` is turned on for W8A16/W8A8 quantization to improve efficiency.")+ disable_opt_rtn = True+ if disable_opt_rtn is None:+ logger.info(+ "`enable_opt_rtn` is turned on, set `--disable_opt_rtn` for higher speed at the cost of accuracy."+ )+ disable_opt_rtn = False+ self.disable_opt_rtn = disable_opt_rtn+ if not self.disable_opt_rtn:+ self._alg_cls = "OptimizedRTNQuantizer"diff --git a/auto_round/algorithms/quantization/rtn/quantizer.py b/auto_round/algorithms/quantization/rtn/quantizer.pynew file mode 100644index 000000000..864ca40fe--- /dev/null+++ b/auto_round/algorithms/quantization/rtn/quantizer.py@@ -0,0 +1,253 @@+# Copyright (c) 2026 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+import traceback+from collections import defaultdict+from typing import Any, Callable, Optional, Union++import accelerate+import torch++from auto_round.algorithms.quantization.base import BaseQuantizers+from auto_round.algorithms.quantization.rtn.config import RTNConfig+from auto_round.algorithms.quantization.sign_round.quantizer import SignRoundQuantizer+from auto_round.compressors_new.shard_writer import ShardWriter+from auto_round.compressors_new.utils import (+ IndexSampler,+ block_forward,+ check_need_act_calibration,+ check_skippable_keywords,+ collect_best_params,+ get_shared_keys,+ immediate_pack,+ infer_bits_by_data_type,+ init_cache,+ reset_params,+ set_layer_config,+)+from auto_round.data_type.utils import update_block_global_scale_if_needed+from auto_round.logger import logger+from auto_round.utils import (+ check_to_quantized,+ convert_module_to_hp_if_necessary,+ get_lm_head_name,+ get_module,+ htcore,+ is_auto_device_mapping,+ is_hpex_available,+ memory_monitor,+ set_amax_for_all_moe_layers,+ set_module,+)+from auto_round.utils.device import (+ clear_memory_if_reached_threshold,+ get_major_device,+ parse_available_devices,+ set_auto_device_map_for_block_with_tuning,+ set_non_auto_device_map,+)+from auto_round.wrapper import WrapperLinear, WrapperMultiblock, unwrapper_block, unwrapper_layer, wrapper_block+++class RTNQuantizer(BaseQuantizers):++ def __init__(self, config: RTNConfig):+ BaseQuantizers.__init__(self, config)++ def quantize_block(+ self, block: torch.nn.Module, input_ids=None, input_others=None, reference_output=None, **kwargs+ ) -> dict:+ """Apply zero-shot RTN quantization to a block.++ Pure-algorithm entry point. Infrastructure (materialize, shard writing,+ device cleanup) is handled by the Compressor before/after this call.++ Args:+ block: Module already materialized and placed on the correct device.+ input_ids: Unused for zero-shot RTN (accepted for interface consistency).+ input_others: Unused for zero-shot RTN.+ reference_output: Unused for zero-shot RTN.++ Returns:+ dict: Empty dict (zero-shot RTN has no tunable parameters to return).+ """+ if (+ self.config.is_act_nv_fp+ or self.config.is_static_afp8+ or (self.config.is_wfp8afp8 and not self.config.act_dynamic)+ ):+ # For FP8 static / NVFP paths, expert input scales are derived during+ # layer quantization from the current act_max. Unify MoE input-proj+ # act_max values before quantizing each expert so exported input_scale+ # stays aligned across experts.+ set_amax_for_all_moe_layers(block, attr_name="act_max")++ for _name, m in block.named_modules():+ if hasattr(m, "global_name") and check_to_quantized(m):+ self.quantize_layer(m.global_name)+ return {}++ def quantize_layer(self, name: str, dtype: torch.dtype = None) -> None:+ """Quantizes a layer using RTN (Round-To-Nearest) if available.++ This function attempts to quantize a layer by switching its data type to a+ `rtn_*` version if supported, then wraps and unwraps the module to apply+ quantization. If GPU memory is insufficient, it falls back to CPU.++ If packing is enabled (`immediate_packing`), the function will also export+ the quantized layer to the appropriate backend format.++ Args:+ name (str): Name of the layer to quantize.++ Raises:+ RuntimeError: If quantization fails for reasons unrelated to memory.+ """++ m = get_module(self.model, name)+ if dtype is not None:+ m = m.to(dtype)++ m = convert_module_to_hp_if_necessary(m, self.model_context.amp_dtype, self.compress_context.device)+ set_module(self.model, name, m)+ tuning_device = m.tuning_device if hasattr(m, "tuning_device") else self.compress_context.device+ # Step 1: let gguf merge layers or rename module first and we will handle the RTN is gguf specific logic+ if (+ self.compress_context.is_immediate_packing+ and self.compress_context.formats[0].is_gguf()+ and not getattr(self.config, "disable_opt_rtn", False)+ ):+ m = m.to(tuning_device)+ m.scale = None+ m.zp = None+ else:+ try:+ disable_opt_rtn = bool(getattr(self.config, "disable_opt_rtn", False))+ if (+ not disable_opt_rtn+ and self.config.orig_disable_opt_rtn is None+ and self.model_context.is_moe_model+ and "expert" in m.global_name+ and "shared_expert" not in m.global_name+ and self.config.super_bits is None # GGUF still uses the optimized RTN for MoE layers+ ):+ disable_opt_rtn = True+ logger.warning_once(+ "MoE layer detected: optimized RTN is disabled for efficiency. "+ "Use `--enable_opt_rtn` to force-enable it for MoE layers."+ )+ m = m.to(tuning_device)+ m = WrapperLinear(+ m,+ device=tuning_device,+ enable_minmax_tuning=False,+ enable_norm_bias_tuning=False,+ enable_round_tuning=False,+ enable_torch_compile=self.compress_context.enable_torch_compile,+ disable_opt_rtn=disable_opt_rtn,+ enable_rtn=True,+ )+ m = m.unwrapper({})+ except torch.OutOfMemoryError:+ cuda_error_msg = traceback.format_exc()+ m = m.orig_layer if hasattr(m, "orig_layer") else m+ try:+ logger.error(cuda_error_msg)+ logger.warning("falling back to CPU.")+ m.to("cpu")+ m = WrapperLinear(+ m,+ enable_minmax_tuning=False,+ enable_norm_bias_tuning=False,+ enable_round_tuning=False,+ enable_torch_compile=self.compress_context.enable_torch_compile,+ enable_rtn=True,+ )+ m = m.unwrapper({})+ except Exception as e:+ raise++ set_module(self.model, name, m)+ self._immediate_pack_and_save_module(name)++ def _immediate_pack_and_save_module(self, module_name):+ shard_writer = ShardWriter.get_shard_writer()+ to_cpu = self.compress_context.low_gpu_mem_usage+ module = get_module(self.model, module_name)+ if self.compress_context.is_immediate_packing: # For gguf, packing conducts on block level+ immediate_pack(module_name, self.layer_config)+ if to_cpu:+ module = module.to("cpu")+ packed_module = get_module(self.model, module_name)+ set_module(self.model, module_name, packed_module.to("cpu"))+ else:+ if to_cpu:+ module = module.to("cpu")+ set_module(self.model, module_name, module)+ if self.compress_context.is_immediate_saving:+ module = get_module(self.model, module_name)+ module.to("cpu")+ shard_writer.write(module, module_name, False)+ # Free RAM immediately: the data is now in the shard-writer buffer+ # (and will be flushed to disk). Keeping it also in the model tree+ # causes linear RAM growth for large models.+ module.to("meta")+++class OptimizedRTNQuantizer(RTNQuantizer):++ def __init__(self, config: RTNConfig):+ BaseQuantizers.__init__(self, config)+ self.batch_size = config.batch_size+ self.batch_dim = config.batch_dim+ self.data_type = config.data_type+ self.group_size = config.group_size+ self.infer_bs_coeff = config.infer_bs_coeff++ self.enable_alg_ext = True++ def quantize_layer_outside_block(self, *args, **kwargs):+ return self.quantize_layer(*args, **kwargs)++ def quantize_block(+ self, block: torch.nn.Module, input_ids=None, input_others=None, reference_output=None, **kwargs+ ):+ """Apply imatrix-informed RTN quantization to a block.++ Pure-algorithm entry point. All infrastructure (device placement,+ act-max hook registration, imatrix collection, cleanup) is handled+ by the Compressor before calling this method.++ Args:+ block: Module already placed on the correct device(s) with act_max+ attributes populated by the Compressor's hook pass.+ input_ids: Unused for optimized RTN; accepted for interface consistency.+ input_others: Unused for optimized RTN.+ reference_output: Unused for optimized RTN.+ """+ update_block_global_scale_if_needed(block, self.data_type, self.group_size)+ if (+ self.config.is_act_nv_fp+ or self.config.is_static_afp8+ or (self.config.is_wfp8afp8 and not self.config.act_dynamic)+ ):+ # enable moe experts act_max automatic generation for Linear+ set_amax_for_all_moe_layers(block, attr_name="act_max")+ # Normalize imatrix and quantize layers+ for name, m in block.named_modules():+ if hasattr(m, "imatrix"):+ m.imatrix /= m.imatrix_cnt+ if hasattr(m, "global_name") and check_to_quantized(m):+ self.quantize_layer_outside_block(m.global_name)++ # _get_block_outputs and _sampling_inputs are defined in BaseQuantizers and inherited.diff --git a/auto_round/algorithms/quantization/sign_round/__init__.py b/auto_round/algorithms/quantization/sign_round/__init__.pynew file mode 100644index 000000000..14a492441--- /dev/null+++ b/auto_round/algorithms/quantization/sign_round/__init__.py@@ -0,0 +1,13 @@+# Copyright (c) 2026 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.diff --git a/auto_round/algorithms/quantization/sign_round/config.py b/auto_round/algorithms/quantization/sign_round/config.pynew file mode 100644index 000000000..90b7d86b1--- /dev/null+++ b/auto_round/algorithms/quantization/sign_round/config.py@@ -0,0 +1,111 @@+# Copyright (c) 2026 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+from typing import Union++from auto_round.algorithms.quantization.config import QuantizationConfig+from auto_round.logger import logger+++class SignRoundConfig(QuantizationConfig):+ """++ Args:+ iters (int): Number of iterations (default is 200).+ lr (float): The learning rate (default is 0.005).+ minmax_lr (float): The learning rate for min-max tuning (default is None).+ lr_scheduler: The learning rate scheduler to be used.+ batch_size (int): Batch size for training (default is 8).+ enable_minmax_tuning (bool): Whether to enable min-max tuning (default is True).+ enable_norm_bias_tuning (bool): Whether to enable fast norm/layer_bias tuning+ """++ _alg_cls = "SignRoundQuantizer"++ def __init__(+ self,+ *,+ iters: int = 200,+ lr: float = None,+ minmax_lr: float = None,+ lr_scheduler=None,+ momentum: float = 0.0,+ batch_size: int = 8,+ nblocks: int = 1,+ enable_minmax_tuning: bool = True,+ enable_norm_bias_tuning: bool = False,+ gradient_accumulate_steps: int = 1,+ enable_alg_ext: bool = False,+ not_use_best_mse: bool = False,+ dynamic_max_gap: int = -1,+ enable_quanted_input: bool = True,+ optimizer: str = None,+ enable_adam: bool = False,+ **kwargs,+ ):+ super().__init__(**kwargs)+ self.iters = iters+ if self.iters < 0:+ logger.warning("`iters` must be non-negative, reset it to 200")+ self.iters = 200++ if not lr:+ # TODO need to check 4 bits lr setting for auto-round-best, 3bits only validate on small models+ if self.iters >= 1000 and self.bits is not None and self.bits <= 3:+ self.lr = 2.0 / self.iters+ logger.info("set the lr to 2.0/iters for better accuracy")+ else:+ self.lr = 1.0 / self.iters+ else:+ self.lr = lr+ self.minmax_lr = minmax_lr or self.lr+ self.lr_scheduler = lr_scheduler++ self.batch_size, self.gradient_accumulate_steps = batch_size, gradient_accumulate_steps+ self.nblocks = nblocks+ self.momentum = momentum+ self.enable_alg_ext = enable_alg_ext++ # Some helpers+ self.infer_bs_coeff = 1+ self.batch_dim = None++ self.enable_minmax_tuning = enable_minmax_tuning+ self.enable_norm_bias_tuning = enable_norm_bias_tuning+ if self.enable_norm_bias_tuning:+ logger.warning("the `enable_norm_bias_tuning` feature is experimental and currently has limited support.")+ self.not_use_best_mse = not_use_best_mse+ self.dynamic_max_gap = dynamic_max_gap+ self.enable_quanted_input = enable_quanted_input+ self.optimizer = optimizer+ self.enable_adam = enable_adam++ if self.enable_adam:+ self._alg_cls = "AdamRoundQuantizer"++ def check_configs(self) -> None:+ """Checks if the configurations are valid.++ Raises:+ ValueError, TypeError: If any of the configurations are invalid.+ """+ super().check_config()++ if self.batch_size <= 0:+ raise ValueError("`batch_size` must be positive")+ if self.iters < 0:+ raise ValueError("`iters` must be non-negative")+ if self.nblocks <= 0:+ raise ValueError("`nblocks` must be positive")+ if self.gradient_accumulate_steps <= 0:+ raise ValueError("`gradient_accumulate_steps` must be positive")diff --git a/auto_round/algorithms/quantization/sign_round/quantizer.py b/auto_round/algorithms/quantization/sign_round/quantizer.pynew file mode 100644index 000000000..1f7b5b539--- /dev/null+++ b/auto_round/algorithms/quantization/sign_round/quantizer.py@@ -0,0 +1,568 @@+# Copyright (c) 2026 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+import copy+from collections import defaultdict+from contextlib import nullcontext+from functools import partial+from typing import Any, Callable, Optional, Union++import accelerate+import torch+from torch import autocast++from auto_round.algorithms.quantization.base import BaseQuantizers+from auto_round.algorithms.quantization.sign_round.config import SignRoundConfig+from auto_round.algorithms.quantization.sign_round.sign_sgd import SignSGD+from auto_round.compressors_new.utils import (+ IndexSampler,+ block_forward,+ check_need_act_calibration,+ collect_best_params,+ immediate_pack,+)+from auto_round.data_type.utils import reshape_pad_tensor_by_group_size, update_fused_layer_global_scales+from auto_round.logger import logger+from auto_round.utils import (+ check_to_quantized,+ compile_func,+ convert_module_to_hp_if_necessary,+ get_module,+ htcore,+ is_auto_device_mapping,+ is_hpex_available,+ memory_monitor,+ mv_module_from_gpu,+ set_amax_for_all_moe_layers,+ to_device,+)+from auto_round.utils.device import (+ clear_memory_if_reached_threshold,+ set_auto_device_map_for_block_with_tuning,+)+from auto_round.utils.distributed import setup_ddp_if_needed_+from auto_round.wrapper import WrapperLinear, unwrapper_block, unwrapper_layer, wrapper_block+++class SignRoundQuantizer(BaseQuantizers):++ def __init__(self, config: SignRoundConfig):+ super().__init__(config)+ self.attention_mask = []++ self.iters = config.iters+ self.lr = config.lr+ self.minmax_lr = config.minmax_lr+ self.lr_scheduler = config.lr_scheduler+ self.batch_size = config.batch_size+ self.batch_dim = config.batch_dim+ self.momentum = config.momentum+ self.infer_bs_coeff = config.infer_bs_coeff+ self.enable_minmax_tuning = config.enable_minmax_tuning+ self.enable_norm_bias_tuning = config.enable_norm_bias_tuning+ self.gradient_accumulate_steps = config.gradient_accumulate_steps+ self.enable_alg_ext = config.enable_alg_ext+ self.not_use_best_mse = config.not_use_best_mse+ self.enable_quanted_input = config.enable_quanted_input+ self.dynamic_max_gap = config.dynamic_max_gap++ self.optimizer = self._get_optimizer(optimizer=config.optimizer)+ self.wrapper_block = wrapper_block++ if self.enable_alg_ext:+ try:+ logger.info("using algorithm extension for quantization.")+ from auto_round.alg_ext import wrapper_autoround++ wrapper_autoround(self)+ except (ImportError, ModuleNotFoundError):+ logger.error("algorithm extension import error, fallback to default mode")++ def _get_current_output(self, output: list[torch.Tensor], indices: list[int]) -> torch.Tensor:+ if self.model_context.is_diffusion:+ assert "hidden_states" in output+ current_output = [output["hidden_states"][x] for x in indices]+ current_output = torch.cat(current_output, dim=self.batch_dim)+ return current_output+ current_output = [output[x] for x in indices]+ current_output = torch.cat(current_output, dim=self.batch_dim)+ return current_output++ def _get_current_num_elm(+ self,+ input_ids: list[torch.Tensor],+ indices: list[int],+ ) -> int:+ if self.model_context.is_diffusion:+ current_input_ids = [input_ids["hidden_states"][i] for i in indices]+ return sum(id.numel() for id in current_input_ids)++ current_input_ids = [input_ids[i] for i in indices]+ return sum(id.numel() for id in current_input_ids)++ def _get_non_zero_cnt(self, tensor: list[torch.Tensor], indices: list[int]) -> int:+ current_tensors = [tensor[i] for i in indices]+ non_zero_cnt = 0+ for t in current_tensors:+ non_zero_cnt += torch.count_nonzero(t).item()+ return non_zero_cnt++ def _get_loss(+ self,+ output_q: torch.Tensor,+ current_output: torch.Tensor,+ indices: torch.Tensor,+ mse_loss: Callable,+ device: Union[str, torch.device] = "cpu",+ ):+ autocast_ctx = (+ nullcontext()+ if self.model_context.amp+ else autocast(device_type=str(device).split(":")[0], dtype=self.model_context.amp_dtype)+ )+ if self.attention_mask:+ tmp_attention_mask = [self.attention_mask[i] for i in indices]+ tmp_attention_mask = torch.cat(tmp_attention_mask, dim=0).to(device)+ tmp_attention_mask.unsqueeze_(-1)++ with autocast_ctx:+ loss = mse_loss( # pylint: disable=not-callable+ (output_q * tmp_attention_mask).to(torch.float32),+ (current_output * tmp_attention_mask).to(torch.float32),+ )+ else:+ with autocast_ctx:+ loss = mse_loss( # pylint: disable=not-callable+ output_q.to(torch.float32), current_output.to(torch.float32)+ )++ return loss++ def quantize_block(+ self,+ block: torch.nn.Module,+ input_ids: Union[list[torch.Tensor], dict],+ input_others: dict,+ reference_output,+ *,+ loss_device: Union[str, torch.device],+ mid_iter_mem_check: bool = False,+ **kwargs,+ ) -> dict:+ """Apply the AutoRound optimization algorithm to a block.++ This is the pure-algorithm entry point. All infrastructure concerns+ (device placement, act-max hook collection, reference-output caching,+ DDP setup, memory cleanup, logging) are handled by the Compressor+ before and after this call.++ Args:+ block: Module already placed on the correct device(s).+ input_ids: Calibration inputs (already on cache_device).+ input_others: Additional inputs for the block's forward pass.+ reference_output: FP reference outputs collected by the Compressor.+ loss_device: Device on which to compute the MSE loss.+ mid_iter_mem_check: Pre-evaluated by the Compressor as+ ``low_gpu_mem_usage and card_0_in_high_risk``. When True,+ triggers mid-iteration memory threshold checks to reduce+ fragmentation on the primary GPU.++ Returns:+ best_params: Best quantization parameters found during optimization.+ Empty dict if no trainable parameters were found.+ """+ device = self.compress_context.device++ quantized_layer_names, unquantized_layer_names = self.wrapper_block(+ block,+ self.enable_minmax_tuning,+ self.enable_norm_bias_tuning,+ enable_torch_compile=self.compress_context.enable_torch_compile,+ device=device,+ )+ if self.config.is_nv_fp:+ for module in block.modules():+ update_fused_layer_global_scales(module)+ round_params = []+ minmax_params = []+ for n, m in block.named_modules():+ if hasattr(m, "orig_layer"):+ for key in m.params.keys():+ if "min" in key or "max" in key:+ minmax_params.append(m.params[key])+ else:+ round_params.append(m.params[key])++ lr = torch.tensor(self.lr)+ minmax_lr = torch.tensor(self.minmax_lr)++ extra_kwargs = {} if self.momentum is None else {"momentum": self.momentum}++ if self.enable_minmax_tuning:+ params = [+ {"params": round_params},+ {"params": minmax_params, "lr": minmax_lr},+ ]+ else:+ params = round_params++ optimizer = self.optimizer(+ params,+ lr=lr,+ weight_decay=0,+ **extra_kwargs,+ )++ if len(round_params) + len(minmax_params) <= 0:+ dump_info = (+ f"quantized {len(quantized_layer_names)}/{(len(quantized_layer_names) + len(unquantized_layer_names))} "+ f"layers in the block"+ )+ logger.info(dump_info)+ unwrapper_block(block, {})+ return {}++ if self.lr_scheduler is None:+ lr_schedule = torch.optim.lr_scheduler.LinearLR(+ optimizer, start_factor=1.0, end_factor=0.0, total_iters=self.iters+ )+ else:+ lr_schedule = copy.deepcopy(self.lr_scheduler)++ if isinstance(input_ids, dict): # input_ids of Flux is dict+ nsamples = len(input_ids["hidden_states"])+ else:+ nsamples = len(input_ids)+ last_best_iter = 0+ best_loss = torch.finfo(torch.float).max+ num_elm = 1+ mse_reduction = "mean"+ if self.gradient_accumulate_steps != 1:+ mse_reduction = "sum"+ mse_loss = torch.nn.MSELoss(reduction=mse_reduction).to(device)+ scaler = self._get_scaler() # pylint: disable=assignment-from-none+ init_loss = None+ best_params = {}+ total_loss = 0+ global_batch_size = self.batch_size * self.gradient_accumulate_steps+ global_batch_size = min(nsamples, global_batch_size)+ # We assume the block input and output shape is same+ if self.gradient_accumulate_steps != 1 and not self.attention_mask:+ whole_indices = torch.arange(global_batch_size)+ num_elm = self._get_current_num_elm(input_ids, whole_indices)+ setup_ddp_if_needed_(self, block, self.compress_context.device_list)+ index_sampler = IndexSampler(nsamples, global_batch_size)+ batch_size = self.batch_size+ for i in range(self.iters):+ if self.enable_alg_ext and self.data_type.endswith("dq"):+ for n, m in block.named_modules():+ m.cur_iter = i+ total_loss = 0+ global_indices = index_sampler.next_batch()+ if self.attention_mask:+ num_elm = self._get_non_zero_cnt(self.attention_mask, global_indices)++ for batch_start in range(0, len(global_indices), batch_size):+ indices = global_indices[batch_start : batch_start + batch_size]+ current_output = self._get_current_output(reference_output, indices)+ current_output = to_device(current_output, loss_device)+ output_q = self._get_current_q_output(block, input_ids, input_others, indices, device, loss_device)+ loss = self._get_loss(output_q, current_output, indices, mse_loss, device)+ num_elm = 1 if num_elm <= 0 else num_elm+ total_loss += loss.item() / num_elm++ if mid_iter_mem_check:+ # clear memory to avoid OOM due to memory fragmentation+ clear_memory_if_reached_threshold(threshold=0.5, device_list=self.compress_context.device_list)++ self._scale_loss_and_backward(scaler, loss)++ if mid_iter_mem_check:+ # clear memory to avoid OOM due to memory fragmentation+ clear_memory_if_reached_threshold(threshold=0.8, device_list=self.compress_context.device_list)++ if i == 0:+ init_loss = total_loss++ if total_loss < best_loss:+ best_loss = total_loss+ if not self.not_use_best_mse:+ best_params = collect_best_params(block, self.compress_context.cache_device)+ last_best_iter = i+ if self.not_use_best_mse and i == self.iters - 1:+ best_params = collect_best_params(block, self.compress_context.cache_device)++ if not self.not_use_best_mse:+ if 0 < self.dynamic_max_gap <= i - last_best_iter:+ break+ self._step(scaler, optimizer, lr_schedule)++ last_loss = total_loss+ best_iter = self.iters+ if not self.not_use_best_mse:+ last_loss = best_loss+ best_iter = last_best_iter+ if self.iters > 0:+ dump_info = (+ f"quantized {len(quantized_layer_names)}/{(len(quantized_layer_names) + len(unquantized_layer_names))} "+ f"layers in the block, loss iter 0: {init_loss:.6f} -> iter {best_iter}: {last_loss:.6f}"+ )+ else:+ dump_info = (+ f"quantized {len(quantized_layer_names)}/{(len(quantized_layer_names) + len(unquantized_layer_names))} "+ "layers in the block"+ )++ self.compress_context.clear_memory() # clear cached memory during training+ if len(unquantized_layer_names) != 0:+ logger.info(f"Unquantized layers: {unquantized_layer_names}")+ with torch.no_grad():+ unwrapper_block(block, best_params)++ if self.config.is_act_nv_fp:+ # enable moe experts act_max automatic generation for WrapperWALayer+ set_amax_for_all_moe_layers(block, attr_name="orig_layer.act_max")++ logger.infoclean(dump_info)+ return best_params++ def quantize_layer_outside_block(+ self, layer_name: str, input_ids: torch.Tensor, q_inputs: torch.Tensor = None, device: str = "cpu", **kwargs+ ):+ """Quantize a specific layer of the model using the provided inputs.++ Args:+ layer_name (str): The name of the layer to quantize.+ inputs (torch.Tensor): Input data for quantization.+ q_inputs (torch.Tensor, optional): Quantized input data. Defaults to None.+ device (torch.device, optional): The device to use for quantization. Defaults to torch.device("cpu").++ Returns:+ None+ """+ logger.info(f"quantizing layer {layer_name}")+ layer = get_module(self.model, layer_name)+ if hasattr(layer, "tuning_device"):+ device = layer.tuning_device++ layer = layer.to(device)+ for i in range(len(input_ids)):+ input_ids[i] = input_ids[i].to(layer.weight.dtype)+ if q_inputs is not None:+ q_inputs[i] = q_inputs[i].to(layer.weight.dtype)++ static_kv_dtype = self.compress_context.static_kv_dtype+ static_attention_dtype = self.compress_context.static_attention_dtype+ if self.config.is_act_quantize and check_need_act_calibration(+ self.config.act_dynamic,+ self.config.act_data_type,+ self.config.act_bits,+ static_kv_dtype,+ static_attention_dtype,+ ):+ tmp_inputs = q_inputs if q_inputs is not None else input_ids+ hook_handles = self._register_act_max_hook(layer)+ with torch.no_grad():+ for input in tmp_inputs:+ layer(input)+ for handle in hook_handles:+ handle.remove()++ wrapper_linear = WrapperLinear(+ layer,+ enable_minmax_tuning=self.enable_minmax_tuning,+ enable_torch_compile=self.compress_context.enable_torch_compile,+ device=device,+ ).to(device)+ round_params = []+ minmax_params = []+ for key in wrapper_linear.params.keys():+ if "min" in key or "max" in key:+ minmax_params.append(wrapper_linear.params[key])+ else:+ round_params.append(wrapper_linear.value)+ if len(round_params) + len(minmax_params) <= 0:+ dump_info = f"quantized {layer_name}"+ logger.info(dump_info)+ with torch.no_grad():+ unwrapper_layer(self.model, wrapper_linear, layer_name, {})+ mv_module_from_gpu(layer)++ lr = torch.tensor(self.lr)+ minmax_lr = torch.tensor(self.minmax_lr)+ if self.enable_minmax_tuning:+ optimizer = self.optimizer(+ [{"params": round_params}, {"params": minmax_params, "lr": minmax_lr}], lr=lr, weight_decay=0+ )+ else:+ optimizer = self.optimizer(round_params, lr=lr, weight_decay=0)++ if self.lr_scheduler is None:+ lr_schedule = torch.optim.lr_scheduler.LinearLR(+ optimizer, start_factor=1.0, end_factor=0.0, total_iters=self.iters+ )+ else:+ lr_schedule = copy.deepcopy(self.lr_scheduler)+ nsamples = len(input_ids)+ last_best_iter = 0+ best_loss = torch.finfo(torch.float).max+ best_params = None+ scaler = self._get_scaler() # pylint: disable=assignment-from-none+ init_loss = None+ gradient_accumulate_steps = self.batch_size # Force to low gpu++ total_loss = 0+ num_elm = 1+ mse_reduction = "mean"+ if gradient_accumulate_steps != 1:+ mse_reduction = "sum"+ mse_loss = torch.nn.MSELoss(reduction=mse_reduction).to(device)+ batch_size = 1 # Force to low gpu+ global_batch_size = self.batch_size * gradient_accumulate_steps+ global_batch_size = min(nsamples, global_batch_size)+ if gradient_accumulate_steps != 1 and not self.attention_mask:+ whole_indices = torch.arange(global_batch_size)+ if q_inputs is not None:+ num_elm = self._get_current_num_elm(q_inputs, whole_indices)+ else:+ num_elm = self._get_current_num_elm(input_ids, whole_indices)++ index_sampler = IndexSampler(nsamples, global_batch_size)++ for i in range(self.iters):+ total_loss = 0+ global_indices = index_sampler.next_batch()+ if self.attention_mask:+ num_elm = self._get_non_zero_cnt(self.attention_mask, global_indices)++ for batch_start in range(0, len(global_indices), batch_size):+ indices = global_indices[batch_start : batch_start + batch_size]+ if q_inputs is not None:+ current_input = [q_inputs[i] for i in indices]+ current_input = torch.cat(current_input, dim=0).to(device)+ org_input = [input_ids[i] for i in indices]+ org_input = torch.cat(org_input, dim=0).to(device)+ else:+ current_input = [input_ids[i] for i in indices]+ current_input = torch.cat(current_input, dim=0).to(device)+ org_input = current_input+ with torch.no_grad():+ current_output = layer(org_input)+ autocast_ctx = (+ nullcontext()+ if not self.model_context.amp+ else autocast(device_type=str(device).split(":")[0], dtype=self.model_context.amp_dtype)+ )+ if self.attention_mask:+ tmp_attention_mask = [self.attention_mask[i] for i in indices]+ tmp_attention_mask = torch.cat(tmp_attention_mask, dim=0).to(device)+ tmp_attention_mask.unsqueeze_(-1)++ with autocast_ctx:+ output_q = wrapper_linear(current_input) # pylint: disable=not-callable+ loss = mse_loss( # pylint: disable=not-callable+ (output_q * tmp_attention_mask).to(torch.float32),+ (current_output * tmp_attention_mask).to(torch.float32),+ )++ else:+ with autocast_ctx:+ output_q = wrapper_linear(current_input) # pylint: disable=not-callable+ loss = mse_loss( # pylint: disable=not-callable+ output_q.to(torch.float32),+ current_output.to(torch.float32), # mul 1.0 will copy the output+ )++ num_elm = 1 if num_elm <= 0 else num_elm+ total_loss += loss.item() / num_elm++ self._scale_loss_and_backward(scaler, loss)+ if i == 0:+ init_loss = total_loss++ if total_loss < best_loss:+ best_loss = total_loss+ if not self.not_use_best_mse:+ best_params = collect_best_params(wrapper_linear, self.compress_context.cache_device)+ last_best_iter = i+ if self.not_use_best_mse and i == self.iters - 1:+ best_params = collect_best_params(wrapper_linear, self.compress_context.cache_device)++ if not self.not_use_best_mse:+ if 0 < self.dynamic_max_gap <= i - last_best_iter:+ break+ self._step(scaler, optimizer, lr_schedule)++ last_loss = total_loss+ best_iter = self.iters+ if not self.not_use_best_mse:+ last_loss = best_loss+ best_iter = last_best_iter+ with torch.no_grad():+ unwrapper_layer(self.model, wrapper_linear, layer_name, best_params)+ mv_module_from_gpu(layer)+ dump_info = f"quantized {layer_name}, loss iter 0: {init_loss:.6f} -> iter {best_iter}: {last_loss:.6f}"+ logger.info(dump_info)++ def _get_optimizer(self, optimizer: Any):+ """Returns the specified optimizer. In SignRound, we fix the optimizer.++ Args:+ optimizer: The optimizer to be used.++ Returns:+ The specified optimizer.+ """+ if optimizer is not None:+ logger.warning_once(+ "The optimizer setting in config will be ignored in AutoRound, using SignSGD as default."+ )+ return SignSGD++ def _get_scaler(self):+ """Returns scaler, in SignRound, no need to use scaler."""+ return None++ def _scale_loss_and_backward(self, scaler: Any, loss: torch.Tensor) -> torch.Tensor:+ """Scales the loss and performs backward pass.++ Args:+ scaler: The scaler to be used.+ loss: The loss to be scaled.++ Returns:+ The scaled loss.+ """+ scale_loss = loss * 1000+ scale_loss.backward()+ if is_hpex_available():+ htcore.mark_step()+ return scale_loss++ def _step(self, scaler: Any, optimizer: Any, lr_schedule: Any):+ """Performs a step in the optimization process.++ Args:+ scaler: The scaler to be used.+ optimizer: The optimizer for the step.+ lr_schedule: The learning rate schedule.++ Returns:+ None+ """+ optimizer.step()+ # for hpu+ if is_hpex_available():+ htcore.mark_step()+ optimizer.zero_grad()+ lr_schedule.step()diff --git a/auto_round/sign_sgd.py b/auto_round/algorithms/quantization/sign_round/sign_sgd.pysimilarity index 100%rename from auto_round/sign_sgd.pyrename to auto_round/algorithms/quantization/sign_round/sign_sgd.pydiff --git a/auto_round/algorithms/transforms/__init__.py b/auto_round/algorithms/transforms/__init__.pynew file mode 100644index 000000000..6648cf6d6--- /dev/null+++ b/auto_round/algorithms/transforms/__init__.py@@ -0,0 +1,146 @@+# Copyright (c) 2026 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+"""Weight/activation rotation algorithm package.++This package houses all *pre-quantisation rotation/transform* algorithms –+mathematical operations applied to model weights or activations before the+quantisation step to improve numerical properties.++Current algorithms+------------------+* **hadamard** – Block-diagonal Hadamard rotations (QuaRot / SpinQuant style).+ See :mod:`auto_round.algorithms.transforms.rotation`.++Adding a new algorithm+-----------------------+1. Create ``algorithms/transforms/<name>/`` with ``config.py`` and ``apply.py``.+2. Subclass :class:`BaseRotationConfig` and :class:`BaseRotation`; register+ with ``@BaseRotation.register("<name>")``.+3. Re-export from this ``__init__.py``.++Typical usage+-------------+>>> from auto_round.algorithms.transforms import apply_rotation+>>> model = apply_rotation(model, config={"hadamard_type": "random_hadamard"})+"""++from __future__ import annotations++from typing import Any++import torch++from auto_round.algorithms.transforms.base import (+ BaseRotation,+ BaseRotationConfig,+ ROTATION_SUPPORTED_SCHEMES,+ check_supported_schemes,+)+from auto_round.algorithms.transforms.rotation import (+ HadamardRotation,+ apply_rotation_transform,+ normalize_rotation_config as _normalize_hadamard_config,+ RotationConfig,+)++__all__ = [+ # Base interfaces+ "BaseRotation",+ "BaseRotationConfig",+ "ROTATION_SUPPORTED_SCHEMES",+ "check_supported_schemes",+ # Config+ "RotationConfig",+ "HadamardRotation",+ "apply_rotation_transform",+ # Unified entry+ "apply_rotation",+ "normalize_rotation_config",+]+++def normalize_rotation_config(+ config: Any,+) -> BaseRotationConfig | None:+ """Normalise any supported config form to the canonical :class:`BaseRotationConfig` subclass.++ Dispatches by inspecting the ``algorithm`` field (or missing field for+ legacy dicts that only carry Hadamard keys).++ Args:+ config: One of: ``None``, :class:`RotationConfig`, a ``dict`` with+ an ``"algorithm"`` key, or a plain Hadamard shorthand string.++ Returns:+ The appropriate :class:`BaseRotationConfig` subclass, or ``None``+ when *config* is ``None`` / empty.+ """+ if config is None:+ return None++ if isinstance(config, BaseRotationConfig):+ return config++ if isinstance(config, dict):+ alg = config.get("algorithm", "hadamard")+ if alg == "hadamard":+ return RotationConfig.model_validate(config)+ raise ValueError(+ f"Unknown rotation algorithm: {alg!r}. " f"Registered algorithms: {sorted(BaseRotation._REGISTRY)}"+ )++ if isinstance(config, str):+ # String shorthand → treat as Hadamard config.+ return RotationConfig.model_validate(_normalize_hadamard_config(config))++ raise TypeError(+ f"Unsupported rotation config type: {type(config).__name__}. "+ "Expected None, dict, str, or a BaseRotationConfig subclass."+ )+++def apply_rotation(+ model: torch.nn.Module,+ config: Any,+ data_type: str = "mx_fp",+ **kwargs: Any,+) -> torch.nn.Module:+ """Apply a rotation/transform algorithm to *model*.++ This is the single, algorithm-agnostic entry point. The correct+ :class:`BaseRotation` subclass is selected automatically from *config*.++ Args:+ model: Model to transform (modified in-place).+ config: Rotation configuration. Accepts:++ * ``None`` – no-op, returns *model* unmodified.+ * :class:`RotationConfig` or compatible ``dict``/``str``.+ * Any :class:`BaseRotationConfig` subclass.++ data_type: Quantization data type (e.g. ``"mx_fp"``).+ **kwargs: Forwarded to :meth:`BaseRotation.apply_to_model`.++ Returns:+ The transformed model.+ """+ if config is None:+ return model++ normalised = normalize_rotation_config(config)+ if normalised is None:+ return model++ rotation = BaseRotation.from_config(normalised)+ return rotation.apply_to_model(model, data_type=data_type, **kwargs)diff --git a/auto_round/algorithms/transforms/base.py b/auto_round/algorithms/transforms/base.pynew file mode 100644index 000000000..236e64347--- /dev/null+++ b/auto_round/algorithms/transforms/base.py@@ -0,0 +1,166 @@+# Copyright (c) 2026 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+"""Base classes and utilities for weight/activation rotation algorithms.++All rotation algorithms (Hadamard, SpinQuant, QuaRot, …) must subclass+``BaseRotation`` and declare a corresponding ``BaseRotationConfig``.++"""++from __future__ import annotations++from abc import ABC, abstractmethod+from dataclasses import dataclass+from typing import Any++import torch++# ---------------------------------------------------------------------------+# Config base+# ---------------------------------------------------------------------------+++@dataclass+class BaseRotationConfig:+ """Minimal base for all rotation algorithm configs.++ Every concrete config subclass should be a ``dataclass`` so it is+ trivially serialisable / comparable.+ """++ #: Human-readable algorithm name, must be unique across all subclasses.+ algorithm: str = "base"+++# ---------------------------------------------------------------------------+# Algorithm base+# ---------------------------------------------------------------------------+++class BaseRotation(ABC):+ """Unified interface for all weight/activation rotation transforms.++ Concrete subclasses implement :meth:`apply_to_model` for their specific+ mathematical transform (Hadamard rotation, random rotation, …).++ Example+ -------+ >>> from auto_round.algorithms.transforms import apply_rotation+ >>> model = apply_rotation(model, config={"algorithm": "hadamard", ...})+ """++ # Registry populated by subclasses via ``BaseRotation.register``.+ _REGISTRY: dict[str, type["BaseRotation"]] = {}++ def __init__(self, config: BaseRotationConfig) -> None:+ self.config = config++ # ------------------------------------------------------------------+ # Abstract interface+ # ------------------------------------------------------------------++ @abstractmethod+ def apply_to_model(+ self,+ model: torch.nn.Module,+ data_type: str = "mx_fp",+ **kwargs: Any,+ ) -> torch.nn.Module:+ """Apply this rotation to *model* and return the (possibly mutated) model.++ Args:+ model: The model to transform.+ data_type: Quantization data type (e.g. ``"mx_fp"``).+ **kwargs: Algorithm-specific extra arguments.++ Returns:+ The transformed model.+ """++ # ------------------------------------------------------------------+ # Factory+ # ------------------------------------------------------------------++ @classmethod+ def register(cls, algorithm_name: str):+ """Class decorator to register a ``BaseRotation`` subclass.++ Usage::++ @BaseRotation.register("hadamard")+ class HadamardRotation(BaseRotation):+ ...+ """++ def _decorator(subclass: type[BaseRotation]) -> type[BaseRotation]:+ cls._REGISTRY[algorithm_name] = subclass+ return subclass++ return _decorator++ @classmethod+ def from_config(cls, config: BaseRotationConfig) -> "BaseRotation":+ """Instantiate the correct ``BaseRotation`` subclass for *config*.++ The algorithm is looked up by ``config.algorithm`` in the registry.+ Sub-packages are imported lazily on first access so that optional+ dependencies (e.g. ``pydantic``) are not required unless actually used.+ """+ # Lazy-load all sub-packages to populate the registry.+ _ensure_registry_populated()++ name = getattr(config, "algorithm", None)+ if name not in cls._REGISTRY:+ raise ValueError(f"No rotation algorithm registered under {name!r}. " f"Available: {sorted(cls._REGISTRY)}")+ return cls._REGISTRY[name](config)+++# ---------------------------------------------------------------------------+# Scheme compatibility check+# ---------------------------------------------------------------------------++#: Quantization schemes that support (and require) rotation transforms.+ROTATION_SUPPORTED_SCHEMES: list[str] = ["MXFP8", "MXFP4", "NVFP4"]+++def check_supported_schemes(scheme: str) -> None:+ """Raise ``ValueError`` if *scheme* does not support rotation transforms."""+ if scheme not in ROTATION_SUPPORTED_SCHEMES:+ raise ValueError(+ f"Rotation transforms are not supported for scheme {scheme!r}. "+ f"Currently supported schemes: {ROTATION_SUPPORTED_SCHEMES}"+ )+++# ---------------------------------------------------------------------------+# Lazy registry population+# ---------------------------------------------------------------------------++_registry_populated = False+++def _ensure_registry_populated() -> None:+ """Import all known sub-packages so their ``@BaseRotation.register`` calls run."""+ global _registry_populated+ if _registry_populated:+ return+ # Import each sub-package here. Add new entries as more algorithms land.+ import importlib++ for sub in ("hadamard",):+ try:+ importlib.import_module(f"auto_round.algorithms.transforms.{sub}")+ except ImportError:+ pass+ _registry_populated = Truediff --git a/auto_round/algorithms/transforms/rotation/__init__.py b/auto_round/algorithms/transforms/rotation/__init__.pynew file mode 100644index 000000000..640cf06ab--- /dev/null+++ b/auto_round/algorithms/transforms/rotation/__init__.py@@ -0,0 +1,44 @@+# Copyright (c) 2026 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+"""Hadamard rotation sub-package for ``algorithms/transforms``."""++from auto_round.algorithms.transforms.rotation.apply import (+ HadamardRotation,+ apply_rotation_transform,+)+from auto_round.algorithms.transforms.rotation.config import (+ RotationConfig,+ normalize_rotation_config,+)+from auto_round.algorithms.transforms.rotation.transforms import (+ HADAMARDS,+ HadamardTransform,+ RandomHadamardTransform,+ build_hadamard_transform,+)++__all__ = [+ # Algorithm class+ "HadamardRotation",+ # Config+ "RotationConfig",+ "normalize_rotation_config",+ # Transform modules+ "HadamardTransform",+ "RandomHadamardTransform",+ "HADAMARDS",+ "build_hadamard_transform",+ # One-shot convenience+ "apply_rotation_transform",+]diff --git a/auto_round/algorithms/transforms/rotation/apply.py b/auto_round/algorithms/transforms/rotation/apply.pynew file mode 100644index 000000000..62ceb02b5--- /dev/null+++ b/auto_round/algorithms/transforms/rotation/apply.py@@ -0,0 +1,309 @@+# Copyright (c) 2026 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+"""Hadamard rotation – concrete ``BaseRotation`` implementation.++Public entry points+-------------------+* :class:`HadamardRotation` – the stateful algorithm object.+* :func:`apply_rotation_transform` – convenience one-shot function.+"""++from __future__ import annotations++from typing import Any++import torch+import tqdm++from auto_round.algorithms.transforms.base import BaseRotation+from auto_round.algorithms.transforms.rotation.config import RotationConfig, normalize_rotation_config+from auto_round.algorithms.transforms.rotation.transforms import build_hadamard_transform+from auto_round.compressors.utils import is_nv_fp+from auto_round.experimental.qmodules.base import QModuleBase++__all__ = ["HadamardRotation", "apply_rotation_transform"]+++def _triton_available(data_type: str = "mx_fp") -> bool:+ """Best-effort check for whether Triton kernel path can be used."""+ if is_nv_fp(data_type):+ return False+ try:+ import triton # noqa: F401 # pylint: disable=E0401++ if not torch.cuda.is_available():+ return False+ from auto_round.algorithms.transforms.rotation.utils.triton.mxfp4 import ( # noqa: F401+ mxfp4_forward_kernel_wrapper,+ )++ return True+ except Exception:+ return False++[email protected]("hadamard")+class HadamardRotation(BaseRotation):+ """Hadamard rotation algorithm.++ Registered under ``"hadamard"`` in the+ :class:`~auto_round.algorithms.transforms.base.BaseRotation` registry.++ Typical usage (via the top-level helper)::++ from auto_round.algorithms.transforms import apply_rotation+ model = apply_rotation(model, config={"hadamard_type": "random_hadamard"})++ Or directly::++ from auto_round.algorithms.transforms.rotation import apply_rotation_transform+ model = apply_rotation_transform(model, config=RotationConfig(), need_calibration=True)+ """++ def __init__(self, config: RotationConfig) -> None:+ super().__init__(config)++ @classmethod+ def from_config(cls, config: dict | RotationConfig) -> "HadamardRotation":+ """Build a :class:`HadamardRotation` from a raw dict or :class:`RotationConfig`."""+ if isinstance(config, dict):+ config = RotationConfig.model_validate(config)+ return cls(config)++ def apply_to_model(+ self,+ model: torch.nn.Module,+ location: str = "weight",+ use_tqdm: bool = True,+ desc: str | None = None,+ data_type: str = "mx_fp",+ **kwargs: Any,+ ) -> torch.nn.Module:+ """Apply the Hadamard rotation to *model*.++ Args:+ model: Target model; modified in-place.+ location: ``"weight"`` (eager, fused into weights) or+ ``"input"`` (activation-side, via forward hook).+ use_tqdm: Show a progress bar while iterating modules.+ desc: Custom progress-bar description.+ data_type: Quantization data type (e.g. ``"mx_fp"``).+ **kwargs: Reserved for future use.++ Returns:+ The mutated *model* with ``model.rotation_config`` set to the+ normalised :class:`RotationConfig` dict.+ """+ cfg = self.config++ # Dispatch by backend. The transform backend (triton-fused per-Linear)+ # is implemented below; the inplace (QuaRot) backend is delegated to+ # :mod:`auto_round.algorithms.transforms.rotation.inplace`.+ from auto_round.algorithms.transforms.rotation.dispatcher import resolve_hadamard_backend++ backend = resolve_hadamard_backend(cfg, data_type)+ if backend == "inplace":+ import auto_round.envs as envs+ from auto_round.algorithms.transforms.rotation.inplace import apply_rotation_transform as _inplace_apply++ # Resolve fuse flag: explicit > env var > default(False).+ fuse_online_to_weight = cfg.fuse_online_to_weight+ if cfg.fuse_online_to_weight is not None:+ fuse_online_to_weight = bool(cfg.fuse_online_to_weight)+ elif envs.AR_FUSE_ONLINE_ROTATION:+ fuse_online_to_weight = bool(envs.AR_FUSE_ONLINE_ROTATION)++ bs = cfg.block_size+ group_size = bs if (bs is not None and bs > 0) else None++ compute_device = kwargs.get("compute_device")+ model, _hooks = _inplace_apply(+ model,+ group_size=group_size,+ allow_online_rotation=cfg.allow_online_rotation,+ rotation_matrix=cfg.hadamard_type,+ fuse_online_to_weight=fuse_online_to_weight,+ compute_device=compute_device,+ )+ setattr(model, "rotation_config", cfg.model_dump())+ return model++ # backend == "transform": original per-Linear triton-fused path.+ # Collect target modules.+ target_types = (torch.nn.Linear, QModuleBase)++ modules = [(name, module) for name, module in model.named_modules() if isinstance(module, target_types)]++ _desc = desc or f"Applying {cfg.hadamard_type} transforms"+ for name, module in tqdm.tqdm(modules, desc=_desc, disable=not use_tqdm):+ if "lm_head" in name:+ continue+ _apply_to_module(model, module, cfg, location, data_type)++ # Store config on model for serialisation / downstream inspection.+ setattr(model, "rotation_config", cfg.model_dump())+ return model+++# ---------------------------------------------------------------------------+# Module-level application helper+# ---------------------------------------------------------------------------+++def _apply_to_module(+ model: torch.nn.Module,+ module: torch.nn.Module,+ config: RotationConfig,+ location: str,+ data_type: str = "mx_fp",+) -> None:+ """Apply the configured Hadamard transform to a single *module*."""+ if location == "input":+ _apply_input_transform(module, config, data_type)++ elif location == "weight":+ _apply_weight_transform(module, config)++ else:+ raise NotImplementedError(f"Unsupported transform location: {location!r}")+++def _apply_input_transform(module: torch.nn.Module, config: RotationConfig, data_type: str = "mx_fp") -> None:+ """Register a forward pre-hook that applies the Hadamard to the input activation."""+ from auto_round.algorithms.transforms.rotation.utils.matrix import multihead_matmul++ inp_transform = build_hadamard_transform(+ **config.model_dump(),+ location="input",+ inverse=True,+ device="cpu",+ precision=module.dtype if hasattr(module, "dtype") else None,+ )++ if config.hadamard_type != "random_hadamard":+ hadamard_weight = inp_transform.weight+ else:+ hadamard_weight = None++ if _triton_available(data_type):+ from auto_round.algorithms.transforms.rotation.utils.triton.mxfp4 import mxfp4_forward_kernel_wrapper++ def _input_hook(self, args):+ x = args[0]+ orig_shape = x.shape+ orig_dtype = x.dtype+ x_flat = x.contiguous().flatten(end_dim=-2)+ w = hadamard_weight.to(orig_dtype) if hadamard_weight is not None else self.hadamard_matrix.T.to(orig_dtype)+ qdq_input, _ = mxfp4_forward_kernel_wrapper(x_flat, w)+ return qdq_input.reshape(orig_shape).to(orig_dtype)++ module.pre_dequantized_input = True+ module.register_forward_pre_hook(_input_hook, prepend=True)+ else:++ def _input_hook(self, args):+ x = args[0]+ ori_shape = x.shape+ orig_dtype = x.dtype+ if hadamard_weight is not None:+ x = x.view(-1, hadamard_weight.shape[0])+ return multihead_matmul(x, hadamard_weight.to(x.device).to(orig_dtype)).view(ori_shape).to(orig_dtype)+ else:+ x = x.view(-1, self.hadamard_matrix.shape[0])+ return multihead_matmul(x, self.hadamard_matrix.T.to(orig_dtype)).view(ori_shape).to(orig_dtype)++ module.pre_dequantized_input = False+ module.register_forward_pre_hook(_input_hook, prepend=True)+++def _apply_weight_transform(+ module: torch.nn.Module,+ config: RotationConfig,+) -> None:+ """Fuse or patch the Hadamard rotation into the weight of *module*."""+ from auto_round.algorithms.transforms.rotation.patch import (+ patch_quantlinear,+ patch_wrapperlinear_to_apply_transform,+ patch_wrapperwalayer_forward_to_apply_transform,+ )++ assert hasattr(module, "weight"), "Weight transform requires module to have a 'weight' attribute"++ w_transform = build_hadamard_transform(+ **config.model_dump(),+ location="weight",+ device=module.weight.device,+ )++ # For random Hadamard, save the matrix as a submodule for serialisation.+ if config.hadamard_type == "random_hadamard":+ from auto_round.algorithms.transforms.rotation.patch import patch_quantlinear as _patch_ql++ _patch_ql(w_transform)++ # Patch WrapperLinear and WrapperWALayer so the transform is applied+ # during calibration tuning.+ inp_transform = build_hadamard_transform(+ **config.model_dump(),+ location="input",+ inverse=True,+ device=module.weight.device,+ precision=module.weight.dtype,+ )++ patch_wrapperlinear_to_apply_transform(w_transform, inp_transform)+ patch_wrapperwalayer_forward_to_apply_transform(inp_transform)+++# ---------------------------------------------------------------------------+# Convenience one-shot function+# ---------------------------------------------------------------------------+++def apply_rotation_transform(+ model: torch.nn.Module,+ config: str | dict | RotationConfig | None,+ location: str = "weight",+ use_tqdm: bool = True,+ desc: str | None = None,+ data_type: str = "mx_fp",+) -> torch.nn.Module:+ """Apply a Hadamard rotation to *model*.++ This is the main public entry point when you only want Hadamard (rather+ than the polymorphic :func:`~auto_round.algorithms.transforms.apply_rotation`).++ Args:+ model: Target model.+ config: One of: :class:`RotationConfig`, ``dict``, ``str``+ shorthand, or ``None`` (no-op).+ location: ``"weight"`` or ``"input"``.+ use_tqdm: Show progress bar.+ desc: Custom progress-bar label.+ data_type: Quantization data type (e.g. ``"mx_fp"``).++ Returns:+ The transformed model.+ """+ normalised = normalize_rotation_config(config)+ if not normalised:+ return model+ rotation = HadamardRotation.from_config(normalised)+ return rotation.apply_to_model(+ model,+ location=location,+ use_tqdm=use_tqdm,+ desc=desc,+ data_type=data_type,+ )diff --git a/auto_round/algorithms/transforms/rotation/config.py b/auto_round/algorithms/transforms/rotation/config.pynew file mode 100644index 000000000..2a8c50697--- /dev/null+++ b/auto_round/algorithms/transforms/rotation/config.py@@ -0,0 +1,188 @@+# Copyright (c) 2026 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+"""Rotation/transform configuration (canonical, unified).++This module is the **single source of truth** for the ``RotationConfig``+schema. The legacy location+``auto_round.experimental.transform.rotation_config`` re-exports from here.++Two implementation backends share this one schema (method B):++* ``backend="inplace"`` – QuaRot-style residual-stream rotation, implemented+ under :mod:`auto_round.experimental.rotation_inplace`. Works for any+ weight/activation dtype and can optionally fuse the online Hadamard into+ weights (``fuse_online_to_weight=True``).++* ``backend="transform"`` – Per-Linear weight + activation Hadamard with a+ fused triton kernel, implemented under+ :mod:`auto_round.algorithms.transforms.rotation.apply`. Supports only+ MXFP4 / NVFP4 and cannot fuse online to weight.++* ``backend="auto"`` – dispatcher picks inplace when a fused online rotation+ is requested, transform when the data_type is MX/NV-FP, inplace otherwise.+"""++from __future__ import annotations++from typing import Any, Optional++from pydantic import BaseModel, Field, field_validator++from auto_round.algorithms.transforms.base import BaseRotationConfig+from auto_round.compressors.utils import is_mx_fp, is_nv_fp+from auto_round.utils import logger++__all__ = [+ "RotationConfig",+ "normalize_rotation_config",+ "to_dict_rotation_config",+ "dump_group_size_to_rotation_config",+]+++# Supported Hadamard transform types (also used by HadamardTransform registry).+HADAMARD_TYPES: frozenset[str] = frozenset({"hadamard", "random_hadamard", "quarot_hadamard"})+_SUPPORTED_BACKENDS: frozenset[str] = frozenset({"auto", "inplace", "transform"})+++class RotationConfig(BaseModel, BaseRotationConfig):+ """Unified configuration for Hadamard rotation/transform applied to a model.++ See the module docstring for a description of the three backends.++ Notes:+ * ``block_size`` is the group/block size for grouped Hadamard.+ For ``backend="inplace"`` it is forwarded as ``group_size``+ (``None`` / ``-1`` means full-dimension Hadamard).+ """++ # Registry key consumed by BaseRotation.from_config (kept for API parity+ # with other BaseRotationConfig subclasses).+ algorithm: str = Field(default="hadamard", frozen=True)++ # ---- shared ----+ backend: str = Field(default="auto")+ block_size: Optional[int] = Field(default=None)+ hadamard_type: str = Field(default="hadamard")++ # ---- inplace-only ----+ fuse_online_to_weight: Optional[bool] = Field(default=None)+ allow_online_rotation: bool = Field(default=True)++ # for random hadamard (transform path)+ random_seed: bool = Field(default=False, exclude=True)++ model_config = {"arbitrary_types_allowed": True}++ @field_validator("backend")+ @classmethod+ def _validate_backend(cls, v: str) -> str:+ if v not in _SUPPORTED_BACKENDS:+ raise ValueError(f"Unsupported backend: {v}. Supported values: {sorted(_SUPPORTED_BACKENDS)}")+ return v++ @field_validator("hadamard_type")+ @classmethod+ def _validate_hadamard_type(cls, v: str) -> str:+ if v not in HADAMARD_TYPES:+ raise ValueError(f"Unsupported hadamard_type: {v!r}. Supported values: {sorted(HADAMARD_TYPES)}")+ return v+++# ---------------------------------------------------------------------------+# Helpers (free functions – match the old experimental/utils.py API)+# ---------------------------------------------------------------------------+++def to_dict_rotation_config(rotation_config: str | dict | RotationConfig | None) -> dict[str, Any]:+ """Convert any supported config form to a plain ``dict`` (no data-type logic).++ Accepts:+ * ``None`` → ``{}``+ * :class:`RotationConfig` → ``model_dump()``+ * ``dict`` → shallow-copied+ * ``str`` → ``{"hadamard_type": key}`` (``"default"`` ⇒ plain default)+ """+ if rotation_config is None:+ return {}+ if isinstance(rotation_config, str):+ key = rotation_config.strip()+ if not key:+ return {}+ if key == "default":+ return {"hadamard_type": "hadamard"}+ return {"hadamard_type": key}+ if isinstance(rotation_config, RotationConfig):+ return rotation_config.model_dump()+ return dict(rotation_config)+++def dump_group_size_to_rotation_config(rotation_config: str | dict | RotationConfig, group_size: int) -> dict[str, Any]:+ """Return *rotation_config* as a dict with ``block_size`` populated from *group_size* (if unset)."""+ rotation_dict = to_dict_rotation_config(rotation_config)+ if rotation_dict.get("block_size", None) is None:+ rotation_dict["block_size"] = group_size+ return rotation_dict+++def normalize_rotation_config(+ rotation_config: str | dict | RotationConfig | None,+ data_type: str = "mx_fp",+) -> dict[str, Any]:+ """Normalise *rotation_config* to a validated ``dict`` ready for ``RotationConfig(**)``.++ Behaviour:+ * ``None`` → ``{}``+ * If ``block_size`` is not set:+ - ``mx_fp`` → default 32+ - ``nv_fp`` → default 16+ - other data types → emit a warning (no default)+ * If ``block_size`` mismatches the data-type recommendation, emit a warning.++ Raises:+ ValueError: If the resulting config is invalid.+ """++ def _apply_data_type_block_size(cfg_dict: dict[str, Any], block_size_explicitly_set: bool) -> dict[str, Any]:+ block_size = cfg_dict.get("block_size")++ if not block_size_explicitly_set or block_size is None:+ if is_mx_fp(data_type):+ cfg_dict["block_size"] = 32+ elif is_nv_fp(data_type):+ cfg_dict["block_size"] = 16+ logger.warning("block_size is not set for data_type 'nv_fp'; defaulting to 16.")+ else:+ logger.warning(+ f"block_size is not set and cannot be inferred for data_type {data_type!r}; "+ "please set block_size explicitly in rotation_config if needed."+ )+ else:+ if is_mx_fp(data_type) and block_size != 32:+ logger.warning(f"data_type is 'mx_fp' but block_size={block_size}; recommended value is 32.")+ elif is_nv_fp(data_type) and block_size != 16:+ logger.warning(f"data_type is 'nv_fp' but block_size={block_size}; recommended value is 16.")++ return cfg_dict++ if rotation_config is None:+ return {}++ rotation_dict = to_dict_rotation_config(rotation_config)+ block_size_explicitly_set = "block_size" in rotation_dict+ cfg_dict = _apply_data_type_block_size(rotation_dict, block_size_explicitly_set)+ try:+ return RotationConfig.model_validate(cfg_dict).model_dump()+ except Exception as exc:+ raise ValueError(f"Invalid RotationConfig: {exc}") from excdiff --git a/auto_round/algorithms/transforms/rotation/dispatcher.py b/auto_round/algorithms/transforms/rotation/dispatcher.pynew file mode 100644index 000000000..ecae6652f--- /dev/null+++ b/auto_round/algorithms/transforms/rotation/dispatcher.py@@ -0,0 +1,153 @@+# Copyright (c) 2026 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0++"""Unified entry point for Hadamard rotation/transform.++Two backend implementations exist:++* ``inplace`` – :mod:`auto_round.algorithms.transforms.rotation.inplace`+ QuaRot-style residual-stream rotation. Works for any weight/activation+ dtype. Optionally fuses the online Hadamard into weights+ (``fuse_online_to_weight=True``).+* ``transform`` – :mod:`auto_round.experimental.transform`+ Per-Linear weight + activation Hadamard with a fused triton kernel.+ Only supports MXFP4 / NVFP4 and **cannot** fuse online to weight.++Routing is controlled by :class:`RotationConfig.backend`:++ "inplace" -> always inplace+ "transform" -> always transform (validates dtype + no-fuse)+ "auto" -> if user asked to fuse -> inplace+ elif data_type is mx_fp / nv_fp -> transform+ else -> inplace+"""++from __future__ import annotations++from typing import Any, Union++import torch++import auto_round.envs as envs+from auto_round.algorithms.transforms.rotation.config import RotationConfig, normalize_rotation_config+from auto_round.compressors.utils import is_mx_fp, is_nv_fp+from auto_round.utils import logger++__all__ = ["apply_hadamard_rotation", "resolve_hadamard_backend"]+++def _to_config(+ rotation_config: Union[str, dict, RotationConfig, None],+ data_type: str,+) -> RotationConfig:+ """Normalise *rotation_config* and return a :class:`RotationConfig` instance."""+ cfg_dict = normalize_rotation_config(rotation_config, data_type)+ if isinstance(cfg_dict, RotationConfig):+ return cfg_dict+ return RotationConfig.model_validate(cfg_dict or {})+++def resolve_hadamard_backend(config: RotationConfig, data_type: str) -> str:+ """Resolve the actual backend (``"inplace"`` / ``"transform"``) from config."""+ requested = config.backend+ fuse_requested = bool(config.fuse_online_to_weight)+ allow_online_rotation: bool = config.allow_online_rotation++ if requested == "inplace":+ return "inplace"++ transform_backend_name = "transform"+ if requested == "transform":+ if fuse_requested:+ raise ValueError(+ f"backend='{transform_backend_name}' does not support fuse_online_to_weight=True. "+ "Use backend='inplace' (or backend='auto' with fuse_online_to_weight=True) instead."+ )+ if not (is_mx_fp(data_type) or is_nv_fp(data_type)):+ raise ValueError(+ f"backend='{transform_backend_name}' only supports MXFP4 / NVFP4 (got data_type={data_type!r}). "+ "Use backend='inplace' or backend='auto' for other dtypes."+ )+ if not allow_online_rotation:+ raise ValueError(f"backend='{transform_backend_name}' only supports `allow_online_rotation`=True")++ return "transform"++ # backend == "auto"+ if fuse_requested:+ return "inplace"+ if is_mx_fp(data_type) or is_nv_fp(data_type):+ return "transform"+ return "inplace"+++def apply_hadamard_rotation(+ model: torch.nn.Module,+ rotation_config: Union[str, dict, RotationConfig, None],+ data_type: str,+ compute_device: torch.device | str = None,+) -> (torch.nn.Module, Any):+ """Apply Hadamard rotation/transform to *model*, dispatching by backend.++ Args:+ model: Target model.+ rotation_config: ``str`` / ``dict`` / :class:`RotationConfig` / ``None``.+ See :class:`RotationConfig` for fields.+ data_type: Quantization data type (e.g. ``"mx_fp"``, ``"nv_fp"``,+ ``"int"``, ``"fp"``).+ compute_device: Device for inplace-backend computation. Ignored by+ the transform backend.++ Returns:+ The same model (for chaining); also stored on ``model.rotation_config``.+ """+ config = _to_config(rotation_config, data_type)+ backend = resolve_hadamard_backend(config, data_type)++ # Resolve fuse flag: explicit > env var > default(True)+ fuse_online_to_weight = config.fuse_online_to_weight+ if config.fuse_online_to_weight is not None:+ fuse_online_to_weight = bool(config.fuse_online_to_weight)+ elif envs.AR_FUSE_ONLINE_ROTATION:+ fuse_online_to_weight = bool(envs.AR_FUSE_ONLINE_ROTATION)++ logger.info(+ f"Applying Hadamard (backend={backend}, "+ f"data_type={data_type}, fuse_online_to_weight={fuse_online_to_weight if backend == 'inplace' else False})."+ )++ if backend == "inplace":+ logger.warning("this backend does not support real exporting, please export the model to fake format")+ from auto_round.algorithms.transforms.rotation.inplace import apply_rotation_transform++ # block_size -> group_size (None / -1 / 0 means full-dimension)+ bs = config.block_size+ group_size = bs if (bs is not None and bs > 0) else None++ model, hooks = apply_rotation_transform(+ model,+ group_size=group_size,+ allow_online_rotation=config.allow_online_rotation,+ rotation_matrix=config.hadamard_type,+ fuse_online_to_weight=fuse_online_to_weight,+ compute_device=compute_device,+ )+ # Stash for downstream (export / serialization). Plain dict so JSON+ # serialization (HF save_pretrained -> config.json) round-trips.+ setattr(model, "rotation_config", config.model_dump() if hasattr(config, "model_dump") else config)+ return model, hooks++ elif backend == "transform":+ supported_hadamard_types = ("hadamard", "random_hadamard")+ if config.hadamard_type not in supported_hadamard_types:+ raise ValueError("this backend only supports hadamard or random_hadamard")+ from auto_round.algorithms.transforms.rotation.apply import apply_rotation_transform++ return apply_rotation_transform(model, config, data_type=data_type)+ else:+ raise ValueError(f"Unsupported Hadamard backend {backend!r}")diff --git a/auto_round/algorithms/transforms/rotation/inplace/__init__.py b/auto_round/algorithms/transforms/rotation/inplace/__init__.pynew file mode 100644index 000000000..9bef07da9--- /dev/null+++ b/auto_round/algorithms/transforms/rotation/inplace/__init__.py@@ -0,0 +1,12 @@+# # Copyright (C) 2026 Intel Corporation+# # SPDX-License-Identifier: Apache-2.0+"""Inplace (QuaRot-style) Hadamard rotation backend.++Canonical home of the residual-stream Hadamard rotation implementation+(formerly under :mod:`auto_round.experimental.rotation_inplace`).+"""++from auto_round.algorithms.transforms.rotation.inplace.apply import apply_rotation_transform # noqa: F401+from auto_round.algorithms.transforms.rotation.inplace.hooks import clear_random_hadamard_cache # noqa: F401++__all__ = ["apply_rotation_transform", "clear_random_hadamard_cache"]diff --git a/auto_round/algorithms/transforms/rotation/inplace/apply.py b/auto_round/algorithms/transforms/rotation/inplace/apply.pynew file mode 100644index 000000000..3177d6fa9--- /dev/null+++ b/auto_round/algorithms/transforms/rotation/inplace/apply.py@@ -0,0 +1,882 @@+# # Copyright (C) 2026 Intel Corporation+# # SPDX-License-Identifier: Apache-2.0++"""Hadamard inplace rotation — public API and rotation primitives.++Supports LLaMA-2, LLaMA-3, Qwen-3 (and any model with the same layout).+The entry point is :func:`apply_hadamard_rotation`.+"""++import gc+import typing+from typing import Dict, Union++import torch+import tqdm++from auto_round.algorithms.transforms.rotation.inplace.hooks import (+ CrossHeadOnlineHadamardHook,+ FullOnlineHadamardHook,+ GroupOnlineHadamardHook,+ _get_custom_had,+ _normalize_rotation_matrix,+ _resolve_compute_device,+ _rotate_embedding_grouped,+ _rotate_linear_grouped,+ apply_cross_head_had_to_linear,+ apply_exact_had_to_linear,+ deterministic_hadamard_matrix,+ get_hadK,+ get_or_create_random_hadamard,+)+from auto_round.algorithms.transforms.rotation.inplace.model_config import (+ MAPPING_REGISTRY,+ RotationMapping,+ _resolve,+ infer_mapping_from_model,+)++# ---------------------------------------------------------------------------+# Low-level primitives (model-agnostic via RotationMapping)+# ---------------------------------------------------------------------------+++def _fuse_ln_linear(+ layernorm: torch.nn.Module,+ linear_layers: typing.Iterable[torch.nn.Linear],+) -> None:+ """Fuse the linear operations in LayerNorm into adjacent linear blocks."""+ for linear in linear_layers:+ linear_dtype = linear.weight.dtype+ dev = linear.weight.device++ W_ = linear.weight.data.double()+ ln_weight = layernorm.weight.double().to(dev)+ linear.weight.data = (W_ * ln_weight).to(linear_dtype)++ if hasattr(layernorm, "bias") and layernorm.bias is not None:+ if linear.bias is None:+ linear.bias = torch.nn.Parameter(torch.zeros(linear.out_features, dtype=torch.float64, device=dev))+ ln_bias = layernorm.bias.double().to(dev)+ linear.bias.data = linear.bias.data.double() + torch.matmul(W_, ln_bias)+ linear.bias.data = linear.bias.data.to(linear_dtype)+++def _reset_ln_params(layernorm: torch.nn.Module) -> None:+ """Reset LayerNorm to identity: weight=1, bias=0."""+ layernorm.weight.data.fill_(1.0)+ if hasattr(layernorm, "bias") and layernorm.bias is not None:+ layernorm.bias.data.fill_(0.0)+++def _rotate_linear_by_Q(module: torch.nn.Linear, Q: torch.Tensor, side: str, compute_device=None) -> None:+ """Apply rotation *Q* to a Linear layer's weight (and bias if present).++ Args:+ side: ``'input'`` → W = W @ Q (rotate input side)+ ``'output'`` → W = Q^T @ W (rotate output side)+ compute_device: Device to run computation on. If None, auto-detects GPU.+ """+ dtype = module.weight.data.dtype+ dev = module.weight.data.device+ cdev = _resolve_compute_device(compute_device)+ W_ = module.weight.data.to(device=cdev, dtype=torch.float64)+ Q_ = Q.to(device=cdev)+ if side == "input":+ new_W = torch.matmul(W_, Q_).to(device=dev, dtype=dtype)+ else:+ new_W = torch.matmul(Q_.T, W_).to(device=dev, dtype=dtype)+ # Release fp64 copy before assigning back so peak memory ≈ 1× weight + 1× rotated.+ del W_+ module.weight.data = new_W+ if side == "output" and module.bias is not None:+ b = module.bias.data.to(device=cdev, dtype=torch.float64)+ new_b = torch.matmul(Q_.T, b).to(device=dev, dtype=dtype)+ del b+ module.bias.data = new_b+ del Q_+++def _untie_word_embeddings(model, mapping: RotationMapping) -> None:+ """Break tied weights between lm_head and embedding if they share the same tensor."""+ embedding = _resolve(model, mapping.embedding)+ lm_head = _resolve(model, mapping.lm_head)++ if lm_head.weight.data_ptr() != embedding.weight.data_ptr():+ return++ lm_head.weight = torch.nn.Parameter(lm_head.weight.data.clone())+ if hasattr(model.config, "tie_word_embeddings"):+ model.config.tie_word_embeddings = False+++def _uses_layernorm_with_mean(model, mapping: RotationMapping) -> bool:+ """Check whether the model uses standard LayerNorm (which subtracts mean)."""+ layers = _resolve(model, mapping.layers_attr)+ first_ln = _resolve(layers[0], mapping.attn_input_ln)+ return isinstance(first_ln, torch.nn.LayerNorm)+++def _bake_mean_into_linear(linear: torch.nn.Linear) -> None:+ """Subtract column-wise mean from a Linear layer's weight (and mean from bias)."""+ linear_dtype = linear.weight.dtype+ W_ = linear.weight.data.double()+ linear.weight.data = (W_ - W_.mean(dim=-2, keepdim=True)).to(linear_dtype)+ if linear.bias is not None:+ b_ = linear.bias.data.double()+ linear.bias.data = (b_ - b_.mean()).to(linear_dtype)+++def _subtract_embedding_mean(model, mapping: RotationMapping) -> None:+ """Subtract per-row mean from the embedding weight matrix."""+ W = _resolve(model, mapping.embedding)+ dtype = W.weight.data.dtype+ W_ = W.weight.data.to(dtype=torch.float64)+ W.weight.data = (W_ - W_.mean(dim=-1, keepdim=True)).to(dtype=dtype)++ if mapping.positional_embedding is not None:+ P = _resolve(model, mapping.positional_embedding)+ p_dtype = P.weight.data.dtype+ P_ = P.weight.data.to(dtype=torch.float64)+ P.weight.data = (P_ - P_.mean(dim=-1, keepdim=True)).to(dtype=p_dtype)+++class _RMSNorm(torch.nn.Module):+ """RMS Normalization (no mean subtraction)."""++ def __init__(self, dim: int, eps: float = 1e-5):+ super().__init__()+ self.eps = eps+ self.register_buffer("weight", torch.ones(dim))++ def forward(self, x: torch.Tensor) -> torch.Tensor:+ rms = torch.sqrt(x.pow(2).mean(dim=-1, keepdim=True) + self.eps)+ return x / rms * self.weight+++def _replace_layernorms_with_rmsnorm(model) -> None:+ """Replace all ``nn.LayerNorm`` modules with ``_RMSNorm``."""+ replacements = []+ for name, module in model.named_modules():+ if isinstance(module, torch.nn.LayerNorm):+ replacements.append((name, module))++ for name, module in replacements:+ parts = name.rsplit(".", 1)+ if len(parts) == 2:+ parent = _resolve(model, parts[0])+ attr = parts[1]+ else:+ parent = model+ attr = parts[0]+ rms = _RMSNorm(module.normalized_shape[0], eps=module.eps)+ rms = rms.to(device=module.weight.device, dtype=module.weight.dtype)+ setattr(parent, attr, rms)+++# ---------------------------------------------------------------------------+# High-level steps driven by RotationMapping+# ---------------------------------------------------------------------------+++def _fuse_layer_norms(model, mapping: RotationMapping) -> None:+ """Fuse all LayerNorm parameters into adjacent Linear layers."""+ layers = _resolve(model, mapping.layers_attr)++ for layer in layers:+ mlp_ln = _resolve(layer, mapping.mlp_input_ln)+ mlp_linears = [_resolve(layer, p) for p in mapping.mlp_in]+ _fuse_ln_linear(mlp_ln, mlp_linears)+ _reset_ln_params(mlp_ln)++ attn_ln = _resolve(layer, mapping.attn_input_ln)+ attn_linears = [+ _resolve(layer, mapping.attn_q),+ _resolve(layer, mapping.attn_k),+ _resolve(layer, mapping.attn_v),+ ]+ _fuse_ln_linear(attn_ln, attn_linears)+ _reset_ln_params(attn_ln)++ pre_head_ln = _resolve(model, mapping.pre_head_ln)+ lm_head = _resolve(model, mapping.lm_head)+ _fuse_ln_linear(pre_head_ln, [lm_head])+ _reset_ln_params(pre_head_ln)+++# ---------------------------------------------------------------------------+# Unified weight rotation (full or grouped)+# ---------------------------------------------------------------------------++[email protected]_mode()+def _rotate_weights(+ model,+ mapping: RotationMapping,+ use_fast_had: bool = True,+ group_size: int = None,+ compute_device: torch.device = None,+ had_dict: dict = None,+ preset: str = None,+ fuse_online_to_weight: bool = True,+) -> None:+ """Apply Hadamard rotation to all weights.++ Args:+ group_size: ``None`` → full Hadamard rotation.+ ``int`` → block-diagonal rotation with this block size.+ compute_device: Device to run Hadamard computation on (e.g. ``"cuda:0"``).+ Weights are moved there temporarily and moved back afterwards.+ If ``None``, auto-detects GPU availability.+ allow_online_rotation: If ``True`` (default), apply extra input-side+ Hadamard rotations on ``down_proj`` and the OV pair (``v_proj``+ output + ``o_proj`` input) that require compensating online hooks+ at inference time. If ``False``, skip those extra rotations so+ that **no** online hooks are needed.+ had_dict: Normalized ``dict[int, Tensor]`` of custom Hadamard matrices+ (keyed by dimension). Only used in grouped mode.+ preset: Rotation preset name (``"quarot_hadamard"``, ``"hadamard"``,+ ``"random_hadamard"``, or ``None``).++ * ``"quarot_hadamard"``: fusable (residual-stream) rotations use+ ``fast_hadamard_transform`` / random Hadamard; non-fusable+ (online-paired) rotations and their weight-side counterparts use+ deterministic ``get_hadK``/``matmul_hadU`` so that the online+ hook at inference produces the exact same transform.+ * ``"hadamard"``: all rotations use deterministic ``get_hadK`` /+ ``matmul_hadU``. Full-mode Q is a deterministic Hadamard matrix.+ * ``"random_hadamard"``: all rotations use random Hadamard matrices+ from the global cache (``get_or_create_random_hadamard``).+ Same dimension → same matrix everywhere.+ * ``None``: same behaviour as ``"hadamard"`` (built-in butterfly).+ """+ compute_device = _resolve_compute_device(compute_device)+ config = model.config+ hidden_size = getattr(config, mapping.hidden_size_attr)+ intermediate_size = getattr(config, mapping.intermediate_size_attr)+ num_heads = getattr(config, mapping.num_heads_attr)+ head_dim = mapping.attn_head_dim or (hidden_size // num_heads)++ is_grouped = group_size is not None and group_size > 0+ desc = f"Rotating (group_size={group_size})" if is_grouped else "Rotating"++ # ----- Resolve per-operation Hadamard sources -----+ fused_fast = use_fast_had+ online_fast = False+ if preset == "random_hadamard":+ fused_fast = False++ # -- Matrix resolution --+ had_matrix, _found = _get_custom_had(had_dict, group_size) if is_grouped else (None, False)++ online_had_matrix = had_matrix+ if preset == "random_hadamard" and had_matrix is None:+ had_matrix = get_or_create_random_hadamard(group_size if is_grouped else hidden_size, compute_device)+ online_had_matrix = had_matrix+ if preset == "quarot_hadamard" and is_grouped:+ online_had_matrix = None # force deterministic for online-paired++ # -- Helper: look up cached random matrix for online-paired ops --+ def _online_had(dim):+ """Return cached random matrix for *dim* under random_hadamard, else None."""+ if preset == "random_hadamard":+ return get_or_create_random_hadamard(dim, compute_device)+ return None++ if is_grouped:+ assert hidden_size % group_size == 0, f"group_size={group_size} must divide hidden_size={hidden_size}"+ assert (+ intermediate_size % group_size == 0+ ), f"group_size={group_size} must divide intermediate_size={intermediate_size}"++ # --- Full mode: build Hadamard matrix Q ---+ Q = None+ if not is_grouped:+ if preset == "hadamard":+ Q = deterministic_hadamard_matrix(hidden_size, compute_device)+ else:+ # "random_hadamard", "quarot_hadamard", None — same shape → same matrix+ Q = get_or_create_random_hadamard(hidden_size, compute_device)++ # ---- Top-level: embedding / lm_head ----+ # When fuse_online_to_weight=False, skip embedding and lm_head rotation:+ # each layer is self-contained (weight rotation + online hook cancel out).+ if fuse_online_to_weight:+ embedding = _resolve(model, mapping.embedding)+ if is_grouped:+ _rotate_embedding_grouped(+ embedding, group_size, use_fast_had=fused_fast, compute_device=compute_device, had_matrix=had_matrix+ )+ else:+ dtype = embedding.weight.data.dtype+ dev = embedding.weight.data.device+ cdev = compute_device+ W_ = embedding.weight.data.to(device=cdev, dtype=torch.float64)+ new_W = torch.matmul(W_, Q.to(cdev)).to(device=dev, dtype=dtype)+ del W_+ embedding.weight.data = new_W++ if mapping.positional_embedding is not None:+ pos_emb = _resolve(model, mapping.positional_embedding)+ if is_grouped:+ _rotate_embedding_grouped(+ pos_emb, group_size, use_fast_had=fused_fast, compute_device=compute_device, had_matrix=had_matrix+ )+ else:+ pos_dtype = pos_emb.weight.data.dtype+ pos_dev = pos_emb.weight.data.device+ cdev = compute_device+ P_ = pos_emb.weight.data.to(device=cdev, dtype=torch.float64)+ new_P = torch.matmul(P_, Q.to(cdev)).to(device=pos_dev, dtype=pos_dtype)+ del P_+ pos_emb.weight.data = new_P++ # ---- Top-level: lm_head ----+ lm_head = _resolve(model, mapping.lm_head)+ if is_grouped:+ _rotate_linear_grouped(+ lm_head,+ group_size,+ side="input",+ use_fast_had=fused_fast,+ compute_device=compute_device,+ had_matrix=had_matrix,+ )+ else:+ _rotate_linear_by_Q(lm_head, Q, side="input", compute_device=compute_device)++ gc.collect()+ if torch.cuda.is_available():+ torch.cuda.empty_cache()++ # ---- Per-layer rotation ----+ layers = _resolve(model, mapping.layers_attr)+ for layer in tqdm.tqdm(layers, unit="layer", desc=desc):+ if fuse_online_to_weight:+ # ---- fuse mode: QuaRot-style residual stream rotation ----+ # Q/K/V: only residual Q on input (no online Had stacking, no hook).+ # When Q == online Had (e.g. preset="hadamard"), Q @ Q = I cancels+ # the rotation entirely, destroying quantization benefit.+ # gate/up: only residual Q on input (no online Had stacking, no hook).+ # down_proj: residual Q^T on output + online Had on input (+ hook).+ # v_proj/o_proj: per-head/cross-head Had below (+ hook on o_proj).+ for attr in (mapping.attn_q, mapping.attn_k, mapping.attn_v):+ mod = _resolve(layer, attr)+ if is_grouped:+ _rotate_linear_grouped(+ mod,+ group_size,+ side="input",+ use_fast_had=fused_fast,+ compute_device=compute_device,+ had_matrix=had_matrix,+ )+ else:+ _rotate_linear_by_Q(mod, Q, side="input", compute_device=compute_device)++ # o_proj: residual stream output rotation+ if is_grouped:+ _rotate_linear_grouped(+ _resolve(layer, mapping.attn_o),+ group_size,+ side="output",+ use_fast_had=fused_fast,+ compute_device=compute_device,+ had_matrix=had_matrix,+ )+ else:+ _rotate_linear_by_Q(_resolve(layer, mapping.attn_o), Q, side="output", compute_device=compute_device)++ # gate/up: only residual Q on input+ for attr in mapping.mlp_in:+ mod = _resolve(layer, attr)+ if is_grouped:+ _rotate_linear_grouped(+ mod,+ group_size,+ side="input",+ use_fast_had=fused_fast,+ compute_device=compute_device,+ had_matrix=had_matrix,+ )+ else:+ _rotate_linear_by_Q(mod, Q, side="input", compute_device=compute_device)++ # down_proj: residual output + online input Had+ down_proj = _resolve(layer, mapping.mlp_out)+ if is_grouped:+ _rotate_linear_grouped(+ down_proj,+ group_size,+ side="output",+ use_fast_had=fused_fast,+ compute_device=compute_device,+ had_matrix=had_matrix,+ )+ _rotate_linear_grouped(+ down_proj,+ group_size,+ side="input",+ use_fast_had=online_fast,+ compute_device=compute_device,+ had_matrix=online_had_matrix,+ )+ else:+ _rotate_linear_by_Q(down_proj, Q, side="output", compute_device=compute_device)+ apply_exact_had_to_linear(+ down_proj,+ had_dim=-1,+ output=False,+ use_fast_had=online_fast,+ compute_device=compute_device,+ had_matrix=_online_had(intermediate_size),+ )++ # OV projection: v_proj per-head output + o_proj full/cross-head input+ v_proj = _resolve(layer, mapping.attn_v)+ o_proj = _resolve(layer, mapping.attn_o)+ if is_grouped:+ pass+ else:+ online_head_had = _online_had(head_dim)+ apply_exact_had_to_linear(+ v_proj,+ had_dim=head_dim,+ output=True,+ use_fast_had=online_fast,+ compute_device=compute_device,+ had_matrix=online_head_had,+ )+ if preset == "random_hadamard":+ apply_exact_had_to_linear(+ o_proj,+ had_dim=head_dim,+ output=False,+ use_fast_had=online_fast,+ compute_device=compute_device,+ had_matrix=online_head_had,+ )+ apply_cross_head_had_to_linear(+ o_proj,+ num_heads,+ head_dim,+ use_fast_had=online_fast,+ compute_device=compute_device,+ had_matrix=_online_had(num_heads),+ )+ else:+ apply_exact_had_to_linear(+ o_proj,+ had_dim=-1,+ output=False,+ use_fast_had=online_fast,+ compute_device=compute_device,+ )++ else:+ # ---- unfused mode: no residual rotation, only input-side Had ----+ # Each layer gets Had fused on input side + compensating hook → equivalent.+ # No embedding/lm_head rotation. No self-cancelling pair.+ # v_proj treated same as Q/K (input Had only, no per-head/cross-head).++ # Q/K/V: input-side Had on hidden_size+ for attr in (mapping.attn_q, mapping.attn_k, mapping.attn_v):+ mod = _resolve(layer, attr)+ if is_grouped:+ _rotate_linear_grouped(+ mod,+ group_size,+ side="input",+ use_fast_had=online_fast,+ compute_device=compute_device,+ had_matrix=online_had_matrix,+ )+ else:+ apply_exact_had_to_linear(+ mod,+ had_dim=-1,+ output=False,+ use_fast_had=online_fast,+ compute_device=compute_device,+ had_matrix=_online_had(hidden_size),+ )++ # o_proj: input-side Had on hidden_size (full Had, not cross-head)+ o_proj = _resolve(layer, mapping.attn_o)+ if is_grouped:+ _rotate_linear_grouped(+ o_proj,+ group_size,+ side="input",+ use_fast_had=online_fast,+ compute_device=compute_device,+ had_matrix=online_had_matrix,+ )+ else:+ apply_exact_had_to_linear(+ o_proj,+ had_dim=-1,+ output=False,+ use_fast_had=online_fast,+ compute_device=compute_device,+ had_matrix=_online_had(hidden_size),+ )++ # gate/up: input-side Had on hidden_size+ for attr in mapping.mlp_in:+ mod = _resolve(layer, attr)+ if is_grouped:+ _rotate_linear_grouped(+ mod,+ group_size,+ side="input",+ use_fast_had=online_fast,+ compute_device=compute_device,+ had_matrix=online_had_matrix,+ )+ else:+ apply_exact_had_to_linear(+ mod,+ had_dim=-1,+ output=False,+ use_fast_had=online_fast,+ compute_device=compute_device,+ had_matrix=_online_had(hidden_size),+ )++ # down_proj: input-side Had on intermediate_size+ down_proj = _resolve(layer, mapping.mlp_out)+ if is_grouped:+ _rotate_linear_grouped(+ down_proj,+ group_size,+ side="input",+ use_fast_had=online_fast,+ compute_device=compute_device,+ had_matrix=online_had_matrix,+ )+ else:+ apply_exact_had_to_linear(+ down_proj,+ had_dim=-1,+ output=False,+ use_fast_had=online_fast,+ compute_device=compute_device,+ had_matrix=_online_had(intermediate_size),+ )++ # Per-layer cleanup: drop fp64 temporaries and CUDA caching allocator+ # blocks so peak memory stays at ~1 layer's worth instead of accumulating+ # across all 32+ decoder layers (was the main cause of 33 GB RAM on 8B).+ gc.collect()+ if torch.cuda.is_available():+ torch.cuda.empty_cache()+++# ---------------------------------------------------------------------------+# Unified online hook registration+# ---------------------------------------------------------------------------+++def _register_online_hooks(+ model,+ mapping: RotationMapping,+ fp32_had: bool = False,+ use_fast_had: bool = True,+ group_size: int = None,+ had_dict: dict = None,+ preset: str = None,+ fuse_online_to_weight: bool = True,+):+ """Register online Hadamard pre-forward hooks on ``down_proj`` and ``o_proj``.++ Online hooks must use the **same** Hadamard matrix that was applied to the+ weight-side counterpart during ``_rotate_weights``. For ``quarot_hadamard``+ this is always the deterministic ``get_hadK``/``matmul_hadU`` path+ (``use_fast_had=False``). For ``"random_hadamard"`` it is the random matrix that+ was generated once and stored in ``had_dict``.++ Args:+ group_size: ``None`` → full Hadamard hooks (original QuaRot).+ ``int`` → per-group Hadamard hooks.+ had_dict: Normalized ``dict[int, Tensor]`` of custom Hadamard matrices.+ preset: Rotation preset name.+ Returns:+ list of hook handles.+ """+ config = model.config+ num_heads = getattr(config, mapping.num_heads_attr)+ hidden_size = getattr(config, mapping.hidden_size_attr)+ intermediate_size = getattr(config, mapping.intermediate_size_attr)+ head_dim = mapping.attn_head_dim or (hidden_size // num_heads)++ is_grouped = group_size is not None and group_size > 0++ # Online hooks always use deterministic (fixed) Hadamard — never fast_had+ # for quarot_hadamard; for "random_hadamard" they use the same random matrix+ # that was cached in had_dict by _rotate_weights.+ online_fast = False++ # -- Matrix resolution (must match the *online-paired* matrix used by+ # _rotate_weights for down_proj input / OV pair). Variable name kept in+ # sync with _rotate_weights to make any future drift obvious.+ online_had_matrix, _ = _get_custom_had(had_dict, group_size) if is_grouped else (None, False)+ if preset == "random_hadamard" and online_had_matrix is None:+ online_had_matrix = get_or_create_random_hadamard(group_size if is_grouped else hidden_size)+ if preset == "quarot_hadamard" and is_grouped:+ online_had_matrix = None++ # -- Helper: look up cached random matrix for online-paired hooks --+ def _online_had(dim):+ if preset == "random_hadamard":+ return get_or_create_random_hadamard(dim)+ return None++ mlp_out_suffix = mapping.mlp_out.split(".")[-1]+ attn_o_suffix = mapping.attn_o.split(".")[-1]++ # Suffixes for Q/K/V and gate/up (for online input Had hooks)+ attn_qkv_suffixes = set(attr.split(".")[-1] for attr in (mapping.attn_q, mapping.attn_k, mapping.attn_v))+ mlp_in_suffixes = set(attr.split(".")[-1] for attr in mapping.mlp_in)++ # --- Build hook factories ---+ def _make_down_proj_hook():+ if is_grouped:+ return GroupOnlineHadamardHook(+ group_size=group_size, fp32_had=fp32_had, use_fast_had=online_fast, had_matrix=online_had_matrix+ )+ online_mat = _online_had(intermediate_size)+ if online_mat is not None:+ return FullOnlineHadamardHook(+ had_K=None, K=None, fp32_had=fp32_had, use_fast_had=online_fast, had_matrix=online_mat+ )+ had_K, K = get_hadK(intermediate_size)+ return FullOnlineHadamardHook(had_K=had_K, K=K, fp32_had=fp32_had, use_fast_had=online_fast)++ def _make_hidden_had_hook():+ """Full Had hook on hidden_size (for Q/K/V and gate/up input)."""+ if is_grouped:+ return GroupOnlineHadamardHook(+ group_size=group_size, fp32_had=fp32_had, use_fast_had=online_fast, had_matrix=online_had_matrix+ )+ online_mat = _online_had(hidden_size)+ if online_mat is not None:+ return FullOnlineHadamardHook(+ had_K=None, K=None, fp32_had=fp32_had, use_fast_had=online_fast, had_matrix=online_mat+ )+ had_K, K = get_hadK(hidden_size)+ return FullOnlineHadamardHook(had_K=had_K, K=K, fp32_had=fp32_had, use_fast_had=online_fast)++ def _make_o_proj_hook():+ online_mat = _online_had(num_heads)+ if online_mat is not None:+ return CrossHeadOnlineHadamardHook(+ had_K=None,+ K=None,+ head_dim=head_dim,+ fp32_had=fp32_had,+ use_fast_had=online_fast,+ had_matrix=online_mat,+ )+ had_K, K = get_hadK(num_heads)+ return CrossHeadOnlineHadamardHook(+ had_K=had_K,+ K=K,+ head_dim=head_dim,+ fp32_had=fp32_had,+ use_fast_had=online_fast,+ )++ # --- Register ---+ handles = []++ for name, module in model.named_modules():+ if not isinstance(module, torch.nn.Linear):+ continue+ suffix = name.split(".")[-1]++ if name.endswith(mlp_out_suffix):+ # down_proj: full Had on intermediate_size input+ h = module.register_forward_pre_hook(_make_down_proj_hook())+ handles.append(h)+ elif name.endswith(attn_o_suffix):+ if fuse_online_to_weight and not is_grouped:+ # o_proj: cross-head Had on input (fused mode, full only)+ h = module.register_forward_pre_hook(_make_o_proj_hook())+ handles.append(h)+ elif not fuse_online_to_weight:+ # o_proj: full Had on hidden_size input (unfused mode, matches weight rotation)+ h = module.register_forward_pre_hook(_make_hidden_had_hook())+ handles.append(h)+ elif suffix in attn_qkv_suffixes:+ if not fuse_online_to_weight:+ # Q/K/V: full Had on hidden_size input (unfused mode only).+ # In fused mode Q/K/V only have residual Q on weight (no online Had),+ # and activations come pre-rotated from residual stream → no hook needed.+ h = module.register_forward_pre_hook(_make_hidden_had_hook())+ handles.append(h)+ elif suffix in mlp_in_suffixes:+ if not fuse_online_to_weight:+ # gate/up: full Had on hidden_size input (unfused mode only).+ # Same reasoning as Q/K/V above.+ h = module.register_forward_pre_hook(_make_hidden_had_hook())+ handles.append(h)++ return handles+++# ---------------------------------------------------------------------------+# Public API+# ---------------------------------------------------------------------------+++def apply_rotation_transform(+ model,+ group_size: int = None,+ allow_online_rotation: bool = True,+ rotation_matrix: Union[str, torch.Tensor, Dict[int, torch.Tensor], None] = None,+ compute_device: torch.device | str = None,+ fp32_had: bool = False,+ fuse_online_to_weight: bool = None,+):+ """Fuse layer norms, rotate weights, and register online Hadamard hooks.++ This is the single entry point for applying Hadamard inplace rotation.+ The model architecture is auto-detected via ``model.config.model_type``.++ Args:+ model: A HuggingFace CausalLM model (LLaMA-2/3, Qwen-3, etc.).+ fp32_had: Whether to compute the online Hadamard transform in fp32.+ group_size: If ``None`` (default), use full-dimension Hadamard rotation.+ compute_device: Device to run Hadamard computation on.+ allow_online_rotation: If ``True`` (default), apply online Hadamard+ rotations on ``down_proj`` input and the OV pair.+ rotation_matrix: Rotation matrix selection (``"hadamard"``,+ ``"random_hadamard"``, ``"quarot_hadamard"``, Tensor, dict, or None).+ fuse_online_to_weight: If ``True`` (default), fuse online Hadamard+ rotation into weights (down_proj input, v_proj output, o_proj input)+ and register compensating online hooks. If ``False``, skip+ embedding/lm_head rotation; each linear layer is self-contained+ with input-side Had on weight + compensating online hook on+ activation. No v_proj cross-head or inner-head rotation.++ Returns:+ list of hook handles."""+ if fuse_online_to_weight is None:+ if model.config.model_type in MAPPING_REGISTRY or model.__class__.__name__ in MAPPING_REGISTRY:+ fuse_online_to_weight = True+ else:+ fuse_online_to_weight = False+ had_dict, use_fast_had, preset = _normalize_rotation_matrix(rotation_matrix, group_size)+ compute_device = _resolve_compute_device(compute_device)++ if use_fast_had:+ from auto_round.utils import logger++ try:+ import fast_hadamard_transform # noqa: F401++ if group_size is None:+ logger.warning(+ "fast_hadamard_transform uses a different Hadamard matrix than the "+ "default implementation. Please ensure consistency between training "+ "and inference. This will be refined later."+ )+ except ImportError:+ logger.warning("Importing fast_hadamard_transform failed, falling back to default implementation.")+ use_fast_had = False++ mapping = infer_mapping_from_model(model)++ _untie_word_embeddings(model, mapping)++ if _uses_layernorm_with_mean(model, mapping):+ _subtract_embedding_mean(model, mapping)++ _fuse_layer_norms(model, mapping)++ if _uses_layernorm_with_mean(model, mapping):+ layers = _resolve(model, mapping.layers_attr)+ for layer in layers:+ _bake_mean_into_linear(_resolve(layer, mapping.attn_o))+ _bake_mean_into_linear(_resolve(layer, mapping.mlp_out))+ _replace_layernorms_with_rmsnorm(model)++ _rotate_weights(+ model,+ mapping,+ use_fast_had=use_fast_had,+ group_size=group_size,+ compute_device=compute_device,+ had_dict=had_dict,+ preset=preset,+ fuse_online_to_weight=fuse_online_to_weight,+ )++ handles = []+ if fuse_online_to_weight or allow_online_rotation:+ handles = _register_online_hooks(+ model,+ mapping,+ fp32_had=fp32_had,+ use_fast_had=use_fast_had,+ group_size=group_size,+ had_dict=had_dict,+ preset=preset,+ fuse_online_to_weight=fuse_online_to_weight,+ )++ return model, handles+++# ---------------------------------------------------------------------------+# Quick smoke test+# ---------------------------------------------------------------------------++if __name__ == "__main__":+ from transformers import AutoModelForCausalLM, AutoTokenizer++ model_name = "/models/opt-125m"+ tokenizer = AutoTokenizer.from_pretrained(model_name)+ model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype=torch.float16, device_map="auto")+ model.to("cuda")+ text = "There is a girl who likes adventure,"+ inputs = tokenizer(text, return_tensors="pt").to(model.device)+ print(tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0]))++ apply_rotation_transform(+ model, group_size=-1, allow_online_rotation=True, rotation_matrix="random_hadamard", fuse_online_to_weight=False+ )+ model.to("cuda")+ text = "There is a girl who likes adventure,"+ inputs = tokenizer(text, return_tensors="pt").to(model.device)+ print(tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0]))++ model_name = "/models/Qwen3-8B"+ tokenizer = AutoTokenizer.from_pretrained(model_name)+ model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype=torch.float16, device_map="auto")+ apply_rotation_transform(model, group_size=-1, allow_online_rotation=True, fuse_online_to_weight=True)+ model.to("cuda")+ text = "There is a girl who likes adventure,"+ inputs = tokenizer(text, return_tensors="pt").to(model.device)+ print(tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0]))++ from transformers import AutoModelForCausalLM, AutoTokenizer++ model_name = "/models/Meta-Llama-3.1-8B-Instruct"+ tokenizer = AutoTokenizer.from_pretrained(model_name)+ model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype=torch.float16, device_map="auto")+ apply_rotation_transform(model, fuse_online_to_weight=True, group_size=32)+ model.to("cuda")+ text = "There is a girl who likes adventure,"+ inputs = tokenizer(text, return_tensors="pt").to(model.device)+ print(tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0]))+ #+ # model_name = "/models/Llama-2-7b-chat-hf"+ # tokenizer = AutoTokenizer.from_pretrained(model_name)+ # model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype=torch.float16, device_map="auto")+ # apply_hadamard_rotation(model)+ # model.to("cuda")+ # text = "There is a girl who likes adventure,"+ # inputs = tokenizer(text, return_tensors="pt").to(model.device)+ # print(tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0]))diff --git a/auto_round/algorithms/transforms/rotation/inplace/hooks.py b/auto_round/algorithms/transforms/rotation/inplace/hooks.pynew file mode 100644index 000000000..2b3d26c6e--- /dev/null+++ b/auto_round/algorithms/transforms/rotation/inplace/hooks.py@@ -0,0 +1,786 @@+# # Copyright (C) 2026 Intel Corporation+# # SPDX-License-Identifier: Apache-2.0++"""Online Hadamard transform hooks.++After weight rotation, down_proj and o_proj require an online Hadamard+transform on their *input activations* at inference time. This module+provides the hooks and a helper to register them on the model.+"""++import math++import torch+import torch.nn as nn++try:+ import fast_hadamard_transform+except ImportError:+ fast_hadamard_transform = None+++def _resolve_compute_device(compute_device) -> torch.device:+ """Return *compute_device* if explicitly given, otherwise auto-detect GPU.++ When ``compute_device`` is ``None`` the function checks for CUDA / XPU+ availability and returns the first accelerator it finds so that heavy+ matrix operations are offloaded to GPU even when the model weights live+ on CPU. Falls back to ``torch.device("cpu")`` when no accelerator is+ present.+ """+ if compute_device is not None:+ return torch.device(compute_device) if not isinstance(compute_device, torch.device) else compute_device+ if torch.cuda.is_available():+ return torch.device("cuda:0")+ if hasattr(torch, "xpu") and torch.xpu.is_available():+ return torch.device("xpu:0")+ return torch.device("cpu")+++BUILTIN_ROTATION_PRESETS = {"quarot_hadamard", "hadamard", "random_hadamard"}++# Global cache for random Hadamard matrices keyed by dimension.+# Ensures the same shape always returns the exact same random matrix within+# a process, across all calls to ``_rotate_weights`` / ``_register_online_hooks``.+_RANDOM_HADAMARD_CACHE: dict = {}+++def get_or_create_random_hadamard(dim: int, device=None) -> torch.Tensor:+ """Return a random Hadamard matrix for *dim*, creating and caching it if needed.++ The matrix is cached globally in ``_RANDOM_HADAMARD_CACHE`` so that every+ caller that requests the same *dim* receives the identical matrix.+ """+ if dim in _RANDOM_HADAMARD_CACHE:+ mat = _RANDOM_HADAMARD_CACHE[dim]+ if device is not None:+ mat = mat.to(device)+ return mat+ mat = random_hadamard_matrix(dim, device or torch.device("cpu"))+ _RANDOM_HADAMARD_CACHE[dim] = mat+ return mat+++def clear_random_hadamard_cache():+ """Clear the global random Hadamard matrix cache.++ Call this when you want subsequent ``random_hadamard`` preset runs to+ generate fresh random matrices (e.g. between independent experiments).+ """+ _RANDOM_HADAMARD_CACHE.clear()+++def _normalize_rotation_matrix(rotation_matrix, group_size):+ """Normalize ``rotation_matrix`` into a ``(had_dict, use_fast_had, preset)`` tuple.++ Accepted inputs:+ * ``None`` → ``(None, False, None)`` — use built-in butterfly ``matmul_hadU``.+ * ``"quarot_hadamard"`` → ``(None, True, "quarot_hadamard")`` — fusable+ rotations use ``fast_hadamard_transform`` (random); non-fusable+ (online-paired) rotations use deterministic ``get_hadK``/``matmul_hadU``.+ * ``"hadamard"`` → ``(None, False, "hadamard")`` — all rotations use+ deterministic ``get_hadK``/``matmul_hadU``.+ * ``"random_hadamard"`` → ``(None, False, "random_hadamard")`` — all rotations use+ ``random_hadamard_matrix``.+ * A ``torch.Tensor`` of shape ``(n, n)`` → ``({n: tensor}, False, None)``.+ * A ``dict[int, Tensor]`` → ``(dict, False, None)`` — returned as-is.++ Returns:+ ``(had_dict, use_fast_had, preset)``++ Raises:+ ValueError: if a non-``str`` *rotation_matrix* is given but+ *group_size* is not a positive integer, or an unknown preset.+ """+ if rotation_matrix is None:+ return None, False, None++ if isinstance(rotation_matrix, str):+ if rotation_matrix not in BUILTIN_ROTATION_PRESETS:+ raise ValueError(+ f"Unknown rotation_matrix preset '{rotation_matrix}'. "+ f"Supported presets: {BUILTIN_ROTATION_PRESETS}."+ )+ if rotation_matrix == "quarot_hadamard":+ return None, True, "quarot_hadamard"+ elif rotation_matrix == "hadamard":+ return None, False, "hadamard"+ else: # "random_hadamard"+ return None, False, "random_hadamard"++ is_grouped = group_size is not None and group_size > 0+ if not is_grouped and not isinstance(rotation_matrix, dict):+ raise ValueError(+ "rotation_matrix (Tensor/dict) can only be used with a positive group_size. "+ f"Got group_size={group_size}."+ )++ if isinstance(rotation_matrix, torch.Tensor):+ assert (+ rotation_matrix.ndim == 2 and rotation_matrix.shape[0] == rotation_matrix.shape[1]+ ), f"rotation_matrix must be square, got shape {rotation_matrix.shape}"+ return {rotation_matrix.shape[0]: rotation_matrix}, False, None++ if isinstance(rotation_matrix, dict):+ for k, t in rotation_matrix.items():+ assert (+ isinstance(t, torch.Tensor) and t.ndim == 2 and t.shape[0] == t.shape[1]+ ), f"rotation_matrix[{k}] must be a square tensor, got shape {t.shape}"+ return rotation_matrix, False, None++ raise TypeError(+ f"rotation_matrix must be a Tensor, dict[int, Tensor], str, or None. " f"Got {type(rotation_matrix)}."+ )+++def _get_custom_had(had_dict, size):+ """Look up a custom Hadamard matrix for *size* from the normalized dict.++ Returns ``(had_tensor, True)`` if found, ``(None, False)`` otherwise.+ """+ if had_dict is None:+ return None, False+ if size in had_dict:+ return had_dict[size], True+ return None, False+++# ---------------------------------------------------------------------------+# Hook implementations+# ---------------------------------------------------------------------------+++class FullOnlineHadamardHook(nn.Module):+ """Pre-forward hook: full Hadamard on the entire last dimension (for ``down_proj``)."""++ def __init__(self, had_K, K, fp32_had=False, use_fast_had=True, had_matrix=None):+ super().__init__()+ self.custom_had = had_matrix is not None+ if had_matrix is not None:+ self.register_buffer("had_matrix", had_matrix)+ self.had_K = None+ self.K = None+ else:+ if had_K is not None:+ self.register_buffer("had_K", had_K)+ else:+ self.had_K = None+ self.K = K+ self.fp32_had = fp32_had+ self.use_fast_had = use_fast_had++ def __call__(self, module: nn.Module, args):+ x = args[0] if isinstance(args, tuple) else args+ x_dtype = x.dtype++ if self.custom_had:+ H = self.had_matrix.to(device=x.device, dtype=x.dtype)+ if self.fp32_had:+ H = self.had_matrix.to(device=x.device).float()+ x = (x.float() @ H.T).to(x_dtype)+ else:+ x = x @ H.T+ elif self.fp32_had:+ x = matmul_hadU_cuda(x.float(), self.had_K, self.K, use_fast_had=self.use_fast_had).to(x_dtype)+ else:+ x = matmul_hadU_cuda(x, self.had_K, self.K, use_fast_had=self.use_fast_had)++ if isinstance(args, tuple):+ return (x,) + args[1:]+ return x+++class CrossHeadOnlineHadamardHook(nn.Module):+ """Pre-forward hook: **cross-head** Hadamard on the ``num_heads`` dimension+ (for ``o_proj``).++ After offline rotation:+ - ``v_proj`` absorbed a per-head (within-head) Hadamard on ``head_dim``.+ - ``o_proj`` absorbed a full Hadamard on ``hidden_size``.++ Since ``H_full = H_cross ⊗ H_within`` (Kronecker decomposition) and the+ within-head part is already cancelled by ``v_proj`` through the attention+ path (``H_within² = I``), the online hook only needs to apply the residual+ **cross-head** Hadamard (``H_cross ⊗ I``):++ * reshape ``(*, hidden_size)`` → ``(*, num_heads, head_dim)``+ * transpose → ``(*, head_dim, num_heads)``+ * Hadamard on the **num_heads** axis (last dim)+ * transpose back and reshape+ """++ def __init__(self, had_K, K, head_dim, fp32_had=False, use_fast_had=True, had_matrix=None):+ """+ Args:+ had_K: Hadamard sub-matrix from ``get_hadK(num_heads)``.+ K: Block size from ``get_hadK(num_heads)``.+ head_dim: ``hidden_size // num_attention_heads``.+ fp32_had: Compute in fp32.+ use_fast_had: If True use fast_hadamard_transform; if False use matmul_hadU.+ had_matrix: Optional custom rotation matrix of shape ``(num_heads, num_heads)``.+ """+ super().__init__()+ self.custom_had = had_matrix is not None+ if had_matrix is not None:+ self.register_buffer("had_matrix", had_matrix)+ self.had_K = None+ self.K = None+ else:+ if had_K is not None:+ self.register_buffer("had_K", had_K)+ else:+ self.had_K = None+ self.K = K+ self.had_dim = head_dim+ self.fp32_had = fp32_had+ self.use_fast_had = use_fast_had++ def __call__(self, module: nn.Module, args):+ x = args[0] if isinstance(args, tuple) else args+ x_dtype = x.dtype++ if self.fp32_had:+ x = x.float()++ init_shape = x.shape+ num_heads = init_shape[-1] // self.had_dim++ if self.custom_had:+ H = self.had_matrix.to(device=x.device, dtype=x.dtype)+ # reshape (*, hidden) → (*, num_heads, head_dim), transpose → (*, head_dim, num_heads)+ x = x.reshape(-1, num_heads, self.had_dim).transpose(1, 2)+ # apply H on last dim (num_heads): x @ H.T+ x = (x @ H.T).transpose(1, 2)+ elif self.use_fast_had and fast_hadamard_transform is not None and self.K == 1:+ x = fast_hadamard_transform.hadamard_transform(+ x.reshape(-1, num_heads, self.had_dim).transpose(1, 2),+ scale=1 / math.sqrt(num_heads),+ ).transpose(1, 2)+ else:+ # Fallback: use matmul_hadU (pure butterfly + had_K, no fast_hadamard_transform)+ x = x.reshape(-1, num_heads, self.had_dim).transpose(1, 2)+ x = matmul_hadU(x.contiguous())+ x = x.transpose(1, 2)++ if self.fp32_had:+ x = x.to(x_dtype)+ x = x.reshape(init_shape)++ if isinstance(args, tuple):+ return (x,) + args[1:]+ return x+++# ---------------------------------------------------------------------------+# Registration helper+# ---------------------------------------------------------------------------+++def register_online_had_hooks(model, mapping=None, fp32_had=False, use_fast_had=True):+ """Register online Hadamard pre-forward hooks on ``down_proj`` and ``o_proj``.++ * **down_proj** (``online_full_had``): full Hadamard on ``intermediate_size``.+ Compensates ``apply_exact_had_to_linear(down_proj, had_dim=-1, output=False)``.++ * **o_proj** (``online cross-head had``): cross-head Hadamard on ``num_heads``.+ Compensates the residual after v_proj's within-head Hadamard cancels.++ Args:+ model: A HuggingFace model whose weights have already been rotated.+ mapping: A :class:`RotationMapping` (auto-inferred if ``None``).+ fp32_had: Whether to compute the Hadamard transform in fp32.+ use_fast_had: If True use fast_hadamard_transform; if False use matmul_hadU.++ Returns:+ list of hook handles (call ``handle.remove()`` to detach).+ """+ if mapping is None:+ from auto_round.algorithms.transforms.rotation.inplace.model_config import infer_mapping_from_model++ mapping = infer_mapping_from_model(model)++ config = model.config+ num_heads = getattr(config, mapping.num_heads_attr)+ hidden_size = getattr(config, mapping.hidden_size_attr)+ intermediate_size = getattr(config, mapping.intermediate_size_attr)+ head_dim = mapping.attn_head_dim or (hidden_size // num_heads)++ # down_proj: full Hadamard on intermediate_size+ had_K_full, K_full = get_hadK(intermediate_size)++ # o_proj: cross-head Hadamard on num_heads+ had_K_head, K_head = get_hadK(num_heads)++ # Identify target module suffixes from mapping+ mlp_out_suffix = mapping.mlp_out.split(".")[-1] # e.g. "down_proj"+ attn_o_suffix = mapping.attn_o.split(".")[-1] # e.g. "o_proj"++ handles = []+ for name, module in model.named_modules():+ if name.endswith(mlp_out_suffix) and isinstance(module, nn.Linear):+ hook = FullOnlineHadamardHook(+ had_K=had_K_full,+ K=K_full,+ fp32_had=fp32_had,+ use_fast_had=use_fast_had,+ )+ h = module.register_forward_pre_hook(hook)+ handles.append(h)+ elif name.endswith(attn_o_suffix) and isinstance(module, nn.Linear):+ hook = CrossHeadOnlineHadamardHook(+ had_K=had_K_head,+ K=K_head,+ head_dim=head_dim,+ fp32_had=fp32_had,+ use_fast_had=use_fast_had,+ )+ h = module.register_forward_pre_hook(hook)+ handles.append(h)++ return handles+++def is_pow2(n):+ return (n & (n - 1) == 0) and (n > 0)+++# Adapted from https://github.com/Cornell-RelaxML/quip-sharp/blob/main/lib/utils/matmul_had.py+def get_hadK(n: int, transpose=False) -> (torch.Tensor, int):+ hadK, K = None, None++ if is_pow2(n):+ K = 1+ return hadK, K+ else:+ from auto_round.algorithms.transforms.rotation.utils.math import _fetch_hadamard_divisor++ hadK = _fetch_hadamard_divisor(n, torch.float, torch.device("cpu"))+ if transpose:+ hadK = hadK.T+ if hadK is not None:+ return hadK, 1 if is_pow2(hadK.shape[0]) else hadK.shape[0]+ assert is_pow2(n)+++def matmul_hadU(X, transpose=False):+ n = X.shape[-1]+ hadK, K = get_hadK(n, transpose)+ input = X.clone().view(-1, n, 1)+ output = input.clone()+ while input.shape[1] > K:+ input = input.view(input.shape[0], input.shape[1] // 2, 2, input.shape[2])+ output = output.view(input.shape)+ output[:, :, 0, :] = input[:, :, 0, :] + input[:, :, 1, :]+ output[:, :, 1, :] = input[:, :, 0, :] - input[:, :, 1, :]+ output = output.view(input.shape[0], input.shape[1], -1)+ input, output = (output, input)+ del output++ if K > 1:+ # Do not explicitly repeat - OOM+ # input = torch.bmm(+ # hadK.repeat(len(input), 1, 1).to(input.device).to(input.dtype), input)+ # Use bcast instead+ input = hadK.view(1, K, K).to(input) @ input++ return input.view(X.shape) / torch.tensor(n).sqrt()+++def matmul_hadUt(X):+ return matmul_hadU(X, transpose=True)+++def random_hadamard_matrix(size, device):+ # See https://cornell-relaxml.github.io/quip-sharp/ , Section "Randomized Hadamard Transformation"+ Q = torch.randint(low=0, high=2, size=(size,)).to(torch.float64)+ Q = Q * 2 - 1+ Q = torch.diag(Q)+ return matmul_hadU(Q).to(device)+++def deterministic_hadamard_matrix(size, device):+ """Build a deterministic Hadamard matrix of the given *size*.++ Applies the butterfly ``matmul_hadU`` to an identity matrix so that the+ result is purely determined by ``get_hadK`` (no random sign flips).+ """+ Q = torch.eye(size, dtype=torch.float64)+ return matmul_hadU(Q).to(device)+++def matmul_hadU_cuda(X, hadK, K, use_fast_had=True):+ n = X.shape[-1]+ if not use_fast_had or fast_hadamard_transform is None:+ return matmul_hadU(X)+ if K == 1:+ return fast_hadamard_transform.hadamard_transform(X.contiguous(), 1.0 / torch.tensor(n).sqrt())+ # if transpose:+ # hadK = hadK.T.contiguous()+ input = X.view(*X.shape[:-1], K, n // K)+ input = fast_hadamard_transform.hadamard_transform(input.contiguous(), 1.0 / torch.tensor(n).sqrt())+ input = hadK.to(input.device).to(input.dtype) @ input+ return input.reshape(X.shape)+++def matmul_hadUt_cuda(X, hadK, K, use_fast_had=True):+ return matmul_hadU_cuda(X, hadK, K, use_fast_had=use_fast_had)+++def apply_exact_had_to_linear(+ module, had_dim=-1, output=False, use_fast_had=True, compute_device=None, had_matrix=None+):+ """Apply Hadamard rotation to a Linear layer's weight in-place.++ Args:+ module: ``nn.Linear`` layer.+ had_dim: Dimension of each Hadamard block (``-1`` for full dimension).+ output: If ``True`` rotate the output (row) side; otherwise input (col).+ use_fast_had: Use ``fast_hadamard_transform`` when available.+ compute_device: Device to run computation on.+ had_matrix: Optional custom rotation matrix. When ``had_dim == -1``+ this should be a square tensor whose size equals+ ``out_features`` (output) or ``in_features`` (input). When+ ``had_dim > 0`` the size should equal ``had_dim``.+ """+ assert isinstance(module, torch.nn.Linear)+ in_features, out_features = module.in_features, module.out_features++ if had_dim != -1 and had_matrix is None:+ assert is_pow2(had_dim), "Hadamard dimension must be a power of 2!"++ W_ = module.weight.data+ dtype = W_.dtype+ dev = W_.device+ init_shape = W_.shape+ compute_dev = _resolve_compute_device(compute_device)+ W_ = W_.double().to(compute_dev)++ if had_matrix is not None:+ H = had_matrix.to(device=compute_dev, dtype=torch.float64)+ if had_dim == -1:+ # Full-dimension custom matrix+ if output:+ # W.T = H @ W.T → W = (H @ W.T).T = W @ H.T+ W_ = W_ @ H.T+ else:+ # W = H @ W (rotate input columns: W_new[i,:] = sum H[i,j]*W[j,:])+ # Actually for input side: W_new = W @ H (each row is rotated)+ W_ = W_ @ H.T+ else:+ # Per-block custom matrix+ if output:+ W_ = W_.t()+ transposed_shape = W_.shape+ flat = W_.reshape(-1, had_dim)+ W_ = (flat @ H.T).reshape(transposed_shape).t()+ else:+ flat = W_.reshape(-1, had_dim)+ W_ = (flat @ H.T).reshape(init_shape)+ elif had_dim == -1:+ if output:+ had_K, K = get_hadK(out_features)+ W_ = matmul_hadU_cuda(W_.t(), had_K, K, use_fast_had=use_fast_had).t()+ if not output:+ had_K, K = get_hadK(in_features)+ W_ = matmul_hadU_cuda(W_, had_K, K, use_fast_had=use_fast_had)+ else:+ # Apply Hadamard to the last had_dim chunks of the weights+ if output:+ W_ = W_.t()+ transposed_shape = W_.shape+ if use_fast_had and fast_hadamard_transform is not None:+ W_ = (+ fast_hadamard_transform.hadamard_transform(+ W_.reshape(-1, transposed_shape[-1] // had_dim, had_dim), scale=1 / math.sqrt(had_dim)+ )+ .reshape(transposed_shape)+ .t()+ )+ else:+ W_ = matmul_hadU(W_.reshape(-1, had_dim)).reshape(transposed_shape).t()+ else:+ if use_fast_had and fast_hadamard_transform is not None:+ n = W_.shape[1]+ W_ = fast_hadamard_transform.hadamard_transform(+ W_.reshape(-1, n // had_dim, had_dim), scale=1 / math.sqrt(had_dim)+ ).reshape(init_shape)+ else:+ W_ = matmul_hadU(W_.reshape(-1, had_dim)).reshape(init_shape)+ module.weight.data = W_.to(device=dev, dtype=dtype)+++def apply_cross_head_had_to_linear(+ module, num_heads, head_dim, use_fast_had=True, compute_device=None, had_matrix=None+):+ """Apply a cross-head Hadamard rotation to a Linear layer's input side.++ The operation is equivalent to ``(H_cross ⊗ I_head_dim)`` applied to the+ input columns:++ * Reshape columns ``(hidden_size,)`` → ``(num_heads, head_dim)``+ * Transpose → ``(head_dim, num_heads)``+ * Hadamard on the ``num_heads`` axis+ * Transpose back and reshape++ This mirrors what :class:`CrossHeadOnlineHadamardHook` does at runtime.++ Args:+ module: ``nn.Linear`` layer whose ``in_features == num_heads * head_dim``.+ num_heads: Number of attention heads.+ head_dim: Per-head dimension.+ use_fast_had: Use ``fast_hadamard_transform`` when available.+ compute_device: Device to run computation on.+ had_matrix: Optional custom rotation matrix of shape ``(num_heads, num_heads)``.+ """+ assert isinstance(module, torch.nn.Linear)+ W_ = module.weight.data+ dtype = W_.dtype+ dev = W_.device+ compute_dev = _resolve_compute_device(compute_device)+ W_ = W_.double().to(compute_dev)++ out_f = W_.shape[0]+ # W shape: (out_features, hidden_size) where hidden_size = num_heads * head_dim+ # Reshape columns: (out_f, num_heads, head_dim)+ W_ = W_.reshape(out_f, num_heads, head_dim)+ # Transpose last two dims: (out_f, head_dim, num_heads)+ W_ = W_.transpose(1, 2).contiguous()++ if had_matrix is not None:+ H = had_matrix.to(device=compute_dev, dtype=torch.float64)+ # Apply H on last dim (num_heads): flat @ H.T+ flat = W_.reshape(-1, num_heads)+ W_ = (flat @ H.T).reshape(out_f, head_dim, num_heads)+ elif use_fast_had and fast_hadamard_transform is not None and is_pow2(num_heads):+ W_ = fast_hadamard_transform.hadamard_transform(W_, scale=1.0 / math.sqrt(num_heads))+ else:+ W_ = matmul_hadU(W_.reshape(-1, num_heads)).reshape(out_f, head_dim, num_heads)++ # Transpose back: (out_f, num_heads, head_dim) → (out_f, hidden_size)+ W_ = W_.transpose(1, 2).contiguous().reshape(out_f, num_heads * head_dim)+ module.weight.data = W_.to(device=dev, dtype=dtype)+++# ---------------------------------------------------------------------------+# Grouped (block-diagonal) Hadamard utilities+# ---------------------------------------------------------------------------+++class OnlineHadamardPostHook(nn.Module):+ """Forward hook (post-hook) adapter: wraps a pre-hook-style Hadamard+ transform to apply it on the layer's **output** instead of input.++ Used for v_proj per-head Hadamard on the output side when online+ rotation is not fused into weights.+ """++ def __init__(self, pre_hook):+ super().__init__()+ self.pre_hook = pre_hook++ def __call__(self, module, input, output):+ result = self.pre_hook(module, (output,))+ if isinstance(result, tuple):+ return result[0]+ return result+++class GroupOnlineHadamardHook(nn.Module):+ """Pre-forward hook: block-diagonal Hadamard with fixed ``group_size`` on last dim.++ Reshapes ``(*, D)`` → ``(*, D // group_size, group_size)``, applies Hadamard+ per group, then reshapes back. Much cheaper than a full-dimension Hadamard.+ """++ def __init__(self, group_size, fp32_had=False, use_fast_had=True, had_matrix=None):+ super().__init__()+ self.group_size = group_size+ self.fp32_had = fp32_had+ self.use_fast_had = use_fast_had+ self.custom_had = had_matrix is not None++ if had_matrix is not None:+ self.register_buffer("had_matrix", had_matrix)+ self.had_K = None+ self.K = None+ elif not is_pow2(group_size):+ had_K, K = get_hadK(group_size)+ if had_K is not None:+ self.register_buffer("had_K", had_K)+ else:+ self.had_K = None+ self.K = K+ else:+ self.had_K = None+ self.K = 1++ def __call__(self, module: nn.Module, args):+ x = args[0] if isinstance(args, tuple) else args+ x_dtype = x.dtype+ init_shape = x.shape+ gs = self.group_size++ if self.fp32_had:+ x = x.float()++ # Reshape: (*, D) → (*, D//gs, gs)+ x = x.reshape(*init_shape[:-1], init_shape[-1] // gs, gs)++ if self.custom_had:+ H = self.had_matrix.to(device=x.device, dtype=x.dtype)+ flat = x.reshape(-1, gs)+ x = (flat @ H.T).reshape(*init_shape[:-1], init_shape[-1] // gs, gs)+ elif self.use_fast_had and fast_hadamard_transform is not None and self.K == 1:+ x = fast_hadamard_transform.hadamard_transform(x, scale=1.0 / math.sqrt(gs))+ else:+ x = x.reshape(-1, gs)+ x = matmul_hadU(x)+ x = x.reshape(*init_shape[:-1], init_shape[-1] // gs, gs)++ x = x.reshape(init_shape)++ if self.fp32_had:+ x = x.to(x_dtype)++ if isinstance(args, tuple):+ return (x,) + args[1:]+ return x+++def _apply_grouped_had_to_weight(W, group_size, side="input", use_fast_had=True, had_matrix=None):+ """Apply block-diagonal Hadamard to a weight matrix.++ Args:+ W: Weight tensor, shape (out_features, in_features).+ group_size: Block size for the Hadamard rotation.+ side: ``'input'`` rotates columns (in_features dim),+ ``'output'`` rotates rows (out_features dim).+ use_fast_had: Use fast_hadamard_transform if available.+ had_matrix: Optional custom Hadamard matrix of shape ``(gs, gs)``+ to use instead of the built-in Hadamard.++ Returns:+ Rotated weight tensor.+ """+ gs = group_size+ dtype = W.dtype+ W = W.double()++ def _had_on_last_dim(X):+ """Apply Hadamard on the last dimension (size gs) of X shaped (..., gs)."""+ if had_matrix is not None:+ H = had_matrix.to(device=X.device, dtype=X.dtype)+ # X: (..., gs) → batch matmul with H^T → X @ H^T+ flat = X.reshape(-1, gs)+ return (flat @ H.T).reshape(X.shape)+ if use_fast_had and fast_hadamard_transform is not None and is_pow2(gs):+ return fast_hadamard_transform.hadamard_transform(X, scale=1.0 / math.sqrt(gs))+ orig_shape = X.shape+ return matmul_hadU(X.reshape(-1, gs)).reshape(orig_shape)++ if side == "input":+ out_f, in_f = W.shape+ W = W.reshape(out_f, in_f // gs, gs)+ W = _had_on_last_dim(W)+ W = W.reshape(out_f, in_f)+ else:+ out_f, in_f = W.shape+ Wt = W.t().contiguous()+ Wt = Wt.reshape(in_f, out_f // gs, gs)+ Wt = _had_on_last_dim(Wt)+ W = Wt.reshape(in_f, out_f).t().contiguous()++ return W.to(dtype)+++def _rotate_linear_grouped(module, group_size, side="input", use_fast_had=True, compute_device=None, had_matrix=None):+ """Apply block-diagonal Hadamard rotation to a Linear layer's weight.++ Args:+ module: ``nn.Linear`` layer.+ group_size: Block size.+ side: ``'input'`` or ``'output'``.+ use_fast_had: Use fast_hadamard_transform.+ compute_device: Device to run computation on. If None, auto-detects GPU.+ had_matrix: Optional custom Hadamard matrix of shape ``(gs, gs)``.+ """+ dtype = module.weight.data.dtype+ dev = module.weight.data.device+ compute_dev = _resolve_compute_device(compute_device)+ W = module.weight.data.to(device=compute_dev, dtype=torch.float64)+ W = _apply_grouped_had_to_weight(W, group_size, side=side, use_fast_had=use_fast_had, had_matrix=had_matrix)+ module.weight.data = W.to(device=dev, dtype=dtype)++ if side == "output" and module.bias is not None:+ bias = module.bias.data.to(device=compute_dev, dtype=torch.float64)+ gs = group_size+ bias = bias.reshape(-1, gs)+ if had_matrix is not None:+ H = had_matrix.to(device=compute_dev, dtype=torch.float64)+ bias = (bias @ H.T).reshape(-1)+ elif use_fast_had and fast_hadamard_transform is not None and is_pow2(gs):+ bias = (+ fast_hadamard_transform.hadamard_transform(bias.unsqueeze(0), scale=1.0 / math.sqrt(gs))+ .squeeze(0)+ .reshape(-1)+ )+ else:+ bias = matmul_hadU(bias).reshape(-1)+ module.bias.data = bias.to(device=dev, dtype=dtype)+++def _rotate_embedding_grouped(embedding, group_size, use_fast_had=True, compute_device=None, had_matrix=None):+ """Apply block-diagonal Hadamard rotation to an Embedding layer.++ Embedding weight: (vocab, hidden_size) → rotate on hidden_size (columns).+ """+ dtype = embedding.weight.data.dtype+ dev = embedding.weight.data.device+ compute_dev = _resolve_compute_device(compute_device)+ W = embedding.weight.data.to(device=compute_dev, dtype=torch.float64)+ W = _apply_grouped_had_to_weight(W, group_size, side="input", use_fast_had=use_fast_had, had_matrix=had_matrix)+ new_W = W.to(device=dev, dtype=dtype)+ del W+ embedding.weight.data = new_W+++def register_online_had_hooks_grouped(model, mapping, group_size, fp32_had=False, use_fast_had=True):+ """Register per-group online Hadamard hooks on ``down_proj`` and ``o_proj``.++ In grouped mode:+ - **down_proj**: block-diagonal Hadamard on ``intermediate_size`` with ``group_size``.+ - **o_proj**: block-diagonal Hadamard on ``hidden_size`` with ``group_size``.++ Args:+ model: HuggingFace model with rotated weights.+ mapping: RotationMapping.+ group_size: Block size for block-diagonal Hadamard.+ fp32_had: Compute in fp32.+ use_fast_had: Use fast_hadamard_transform.++ Returns:+ list of hook handles.+ """+ mlp_out_suffix = mapping.mlp_out.split(".")[-1]+ attn_o_suffix = mapping.attn_o.split(".")[-1]++ handles = []+ for name, module in model.named_modules():+ if name.endswith(mlp_out_suffix) and isinstance(module, nn.Linear):+ hook = GroupOnlineHadamardHook(+ group_size=group_size,+ fp32_had=fp32_had,+ use_fast_had=use_fast_had,+ )+ h = module.register_forward_pre_hook(hook)+ handles.append(h)+ elif name.endswith(attn_o_suffix) and isinstance(module, nn.Linear):+ hook = GroupOnlineHadamardHook(+ group_size=group_size,+ fp32_had=fp32_had,+ use_fast_had=use_fast_had,+ )+ h = module.register_forward_pre_hook(hook)+ handles.append(h)++ return handlesdiff --git a/auto_round/algorithms/transforms/rotation/inplace/model_config.py b/auto_round/algorithms/transforms/rotation/inplace/model_config.pynew file mode 100644index 000000000..3ecbf9b69--- /dev/null+++ b/auto_round/algorithms/transforms/rotation/inplace/model_config.py@@ -0,0 +1,169 @@+# # Copyright (C) 2026 Intel Corporation+# # SPDX-License-Identifier: Apache-2.0++"""Model architecture mapping for Hadamard rotation.++Each :class:`RotationMapping` describes *where* the rotation-relevant modules+live inside a model. Currently supports LLaMA-2, LLaMA-3, and Qwen-3 (dense).++New architectures can be supported by calling :func:`register_mapping`.+"""++from __future__ import annotations++from dataclasses import dataclass, field+from typing import Dict, List, Optional++from auto_round.utils import logger++__all__ = [+ "RotationMapping",+ "register_mapping",+ "get_mapping",+ "infer_mapping_from_model",+ "MAPPING_REGISTRY",+]+++# ---------------------------------------------------------------------------+# Mapping dataclass+# ---------------------------------------------------------------------------+++@dataclass+class RotationMapping:+ """Declarative description of a transformer architecture for Hadamard rotation.++ Attribute names follow the dot-path convention relative to the model or+ each decoder layer.++ Config attribute names (read from ``model.config``):+ num_heads_attr, hidden_size_attr, intermediate_size_attr+ head_dim_override – explicit head dim (skip hidden_size // num_heads)+ """++ # -- top-level modules (dot-path from model root) --+ embedding: str = "model.embed_tokens"+ lm_head: str = "lm_head"+ positional_embedding: Optional[str] = None # e.g. "model.decoder.embed_positions" for OPT++ # -- layers container (dot-path from model root) --+ layers_attr: str = "model.layers"++ # -- per-layer: attention (dot-path from each layer) --+ attn_input_ln: str = "input_layernorm"+ attn_q: str = "self_attn.q_proj"+ attn_k: str = "self_attn.k_proj"+ attn_v: str = "self_attn.v_proj"+ attn_o: str = "self_attn.o_proj"++ # -- per-layer: MLP (dot-path from each layer) --+ mlp_input_ln: str = "post_attention_layernorm"+ mlp_in: List[str] = field(default_factory=lambda: ["mlp.up_proj", "mlp.gate_proj"])+ mlp_out: str = "mlp.down_proj"++ # -- final norm (dot-path from model root) --+ pre_head_ln: str = "model.norm"++ # -- head dim override (None = hidden_size // num_heads) --+ attn_head_dim: Optional[int] = None++ # -- config attr names --+ num_heads_attr: str = "num_attention_heads"+ hidden_size_attr: str = "hidden_size"+ intermediate_size_attr: str = "intermediate_size"+++# ---------------------------------------------------------------------------+# Helper: resolve a dot-path attribute on a module+# ---------------------------------------------------------------------------+++def _resolve(root, dot_path: str):+ """Resolve ``'a.b.c'`` to ``root.a.b.c``."""+ obj = root+ for attr in dot_path.split("."):+ obj = getattr(obj, attr)+ return obj+++# ---------------------------------------------------------------------------+# Registry+# ---------------------------------------------------------------------------++MAPPING_REGISTRY: Dict[str, RotationMapping] = {}+++def register_mapping(key: str, mapping: RotationMapping) -> RotationMapping:+ """Register a :class:`RotationMapping` under *key* (model_type or architecture)."""+ MAPPING_REGISTRY[key] = mapping+ return mapping+++def get_mapping(key: str) -> RotationMapping:+ """Look up a mapping by *key*; fall back to default if not found."""+ if key in MAPPING_REGISTRY:+ return MAPPING_REGISTRY[key]+ logger.warning(f"No rotation mapping registered for '{key}', " "falling back to default (LLaMA-like) mapping.")+ return RotationMapping()+++def infer_mapping_from_model(model) -> RotationMapping:+ """Return the best :class:`RotationMapping` for *model*.++ Tries ``model.config.model_type`` first, then ``model.__class__.__name__``.+ """+ model_type = getattr(getattr(model, "config", None), "model_type", "")+ if model_type in MAPPING_REGISTRY:+ return MAPPING_REGISTRY[model_type]++ arch = model.__class__.__name__+ if arch in MAPPING_REGISTRY:+ return MAPPING_REGISTRY[arch]++ logger.warning(+ f"Unrecognised architecture '{arch}' (model_type='{model_type}'). "+ "Falling back to default (LLaMA-like) mapping."+ )+ return RotationMapping()+++# ===================================================================+# Built-in mappings+# ===================================================================++# LLaMA-2 / LLaMA-3 / Mistral / Yi — all share the same layout+_default = RotationMapping()++register_mapping("llama", _default)+register_mapping("LlamaForCausalLM", _default)++# Qwen-3 dense — identical layout to LLaMA+register_mapping("qwen3", _default)+register_mapping("Qwen3ForCausalLM", _default)++# Qwen-2 / Qwen-2.5 dense — identical layout to LLaMA+register_mapping("qwen2", _default)+register_mapping("Qwen2ForCausalLM", _default)++# ---- OPT ----+# OPT uses standard LayerNorm (with bias, subtracts mean),+# different module names, and tied lm_head ↔ embedding weights.+_opt = RotationMapping(+ embedding="model.decoder.embed_tokens",+ lm_head="lm_head",+ positional_embedding="model.decoder.embed_positions",+ layers_attr="model.decoder.layers",+ attn_input_ln="self_attn_layer_norm",+ attn_q="self_attn.q_proj",+ attn_k="self_attn.k_proj",+ attn_v="self_attn.v_proj",+ attn_o="self_attn.out_proj",+ mlp_input_ln="final_layer_norm",+ mlp_in=["fc1"],+ mlp_out="fc2",+ pre_head_ln="model.decoder.final_layer_norm",+ intermediate_size_attr="ffn_dim",+)+register_mapping("opt", _opt)+register_mapping("OPTForCausalLM", _opt)diff --git a/auto_round/algorithms/transforms/rotation/patch.py b/auto_round/algorithms/transforms/rotation/patch.pynew file mode 100644index 000000000..632ec9682--- /dev/null+++ b/auto_round/algorithms/transforms/rotation/patch.py@@ -0,0 +1,197 @@+# Copyright (c) 2026 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+"""Monkey-patching helpers to inject Hadamard transforms into calibration wrappers.++During AutoRound calibration (``need_calibration=True``) the weight is re-+quantised at every forward pass. These patches insert the Hadamard rotation+into :class:`~auto_round.wrapper.WrapperLinear` and+:class:`~auto_round.wrapper.WrapperWALayer` so the transform is applied+transparently inside the tuning loop.++Each patch is idempotent: calling it twice has no effect.+"""++from __future__ import annotations++import torch+import transformers++from auto_round.export.export_to_autoround.qlinear_fp import QuantLinear, pack_fp4_to_uint8+from auto_round.wrapper import WrapperLinear, WrapperWALayer++__all__ = [+ "patch_wrapperlinear_to_apply_transform",+ "patch_wrapperwalayer_forward_to_apply_transform",+ "patch_quantlinear",+]+++def patch_wrapperlinear_to_apply_transform(+ w_transform: torch.nn.Module,+ inp_transform: torch.nn.Module,+) -> None:+ """Inject *w_transform* and *inp_transform* into :class:`WrapperLinear`.++ After this call, every ``WrapperLinear`` instance will:++ * Apply *w_transform* to the weight before quantisation (``_qdq_weight``).+ * Apply *inp_transform* to the activation before quantisation (``_qdq_act``).++ The patch is written at the **class** level and is therefore global – it+ affects all future instances as well. A guard flag ``_hadamard_patched``+ prevents double-patching.+ """+ if getattr(WrapperLinear, "_hadamard_patched", False):+ return++ _orig_qdq_weight = WrapperLinear._qdq_weight++ def _qdq_weight_patched(self, value, min_scale, max_scale):+ if self.orig_layer.bits >= 16:+ # Keep original behaviour for >=16-bit quantisation.+ return _orig_qdq_weight(self, value, min_scale, max_scale)++ if getattr(self, "applied_weight_hadamard", None) is None:+ with torch.no_grad():+ weight = self.orig_layer.weight+ if weight.device.type == "meta":+ weight = self.orig_layer.get_weight().to(self.device)++ is_conv1d = type(self.orig_layer) is transformers.pytorch_utils.Conv1D+ if is_conv1d:+ weight = weight.t().contiguous()+ new_weight = w_transform(weight).to(self.device)+ if is_conv1d:+ new_weight = new_weight.t().contiguous()+ self.orig_layer.weight.data.copy_(new_weight)+ self.applied_weight_hadamard = True++ return _orig_qdq_weight(self, value, min_scale, max_scale)++ _orig_qdq_act = WrapperLinear._qdq_act++ def _qdq_act_patched(self, x, act_min_scale=torch.tensor(1.0), act_max_scale=torch.tensor(1.0), act_max=None):+ x = inp_transform(x)++ return _orig_qdq_act(self, x, act_min_scale=act_min_scale, act_max_scale=act_max_scale, act_max=act_max)++ WrapperLinear._qdq_weight = _qdq_weight_patched+ WrapperLinear._qdq_act = _qdq_act_patched+ WrapperLinear._hadamard_patched = True+++def patch_wrapperwalayer_forward_to_apply_transform(+ inp_transform: torch.nn.Module,+) -> None:+ """Inject *inp_transform* into :class:`WrapperWALayer`.forward.++ After this call every ``WrapperWALayer`` will rotate its input activation+ before the activation quantisation step. Idempotent via+ ``_hadamard_forward_patched`` guard.+ """+ if getattr(WrapperWALayer, "_hadamard_forward_patched", False):+ return++ _orig_forward = WrapperWALayer.forward++ def _forward_patched(self, x):+ act_max = self.orig_layer.act_max if hasattr(self.orig_layer, "act_max") else None+ x = inp_transform(x)+ x, _, _ = self.orig_layer.act_quant_func(+ x,+ bits=self.orig_layer.act_bits,+ group_size=self.orig_layer.act_group_size,+ scale_dtype=self.orig_layer.scale_dtype,+ q_scale_thresh=self.orig_layer.q_scale_thresh,+ data_type=self.orig_layer.act_data_type,+ tensor_max=act_max,+ )+ return self.orig_layer.forward(x)++ WrapperWALayer.forward = _forward_patched+ WrapperWALayer._hadamard_forward_patched = True+++def patch_quantlinear(w_transform) -> None:+ """Patch :class:`QuantLinear` so random Hadamard matrices are saved when packing.++ Only needed for ``random_hadamard`` where the rotation matrix must be+ serialised alongside the quantised weights for correct inference.+ Idempotent via ``_pack_patched`` guard.+ """+ if getattr(QuantLinear, "_pack_patched", False):+ return++ from auto_round.data_type.nvfp import cast_to_fp4, get_reciprocal+ from auto_round.data_type.utils import reshape_pad_tensor_by_group_size, revert_tensor_by_pad+ from auto_round.utils import get_packing_device++ E8M0_EXPONENT_BIAS = 127+ E8M0_EXPONENT_NAN_VAL = 255++ def _pack_patched(+ self,+ linear,+ scales,+ zeros=None,+ g_idx=None,+ global_scale=None,+ input_global_scale=None,+ device=None,+ ):+ device = get_packing_device(device)+ if getattr(linear, "bias", None) is not None:+ self.bias = linear.bias.detach().to(torch.float16)++ W = linear.weight.data.detach().to(device)+ if type(linear) is torch.nn.Conv2d:+ W = W.flatten(1)+ if type(linear) is transformers.pytorch_utils.Conv1D:+ W = W.t()++ tensor, orig_shape, pad_len = reshape_pad_tensor_by_group_size(W, self.group_size)+ scales = scales.to(device)+ if self.is_nv:+ assert global_scale is not None and global_scale.numel() == 1+ global_scale = global_scale.reshape([1]).to(device)+ scaled_tensor = tensor.to(global_scale.dtype) * get_reciprocal(+ scales.reshape(tensor.shape[0], -1) * get_reciprocal(global_scale)+ )+ scaled_tensor.clamp_(-6.0, 6.0)+ scaled_tensor = cast_to_fp4(scaled_tensor)+ else:+ scaled_tensor = tensor / (2 ** scales.reshape(tensor.shape[0], -1))+ scaled_tensor = revert_tensor_by_pad(scaled_tensor, orig_shape=orig_shape, pad_len=pad_len)+ if self.is_mx:+ final_scale = (scales + E8M0_EXPONENT_BIAS).clamp(0, E8M0_EXPONENT_NAN_VAL).to(torch.uint8)+ else:+ final_scale = scales.to(torch.float8_e4m3fn)++ self.weight_scale = final_scale+ if self.bits == 8:+ self.weight = scaled_tensor.to(torch.float8_e4m3fn)+ else:+ self.weight_packed = pack_fp4_to_uint8(scaled_tensor)++ if global_scale is not None:+ self.weight_global_scale = global_scale.to(torch.float32).to(device)+ if input_global_scale is not None:+ self.input_global_scale = input_global_scale.to(torch.float32).to(device).reshape([1])++ # add transform weight+ self.register_buffer("hadamard_matrix", w_transform.weight.to(device))+ return++ QuantLinear.pack = _pack_patched+ QuantLinear._pack_patched = Truediff --git a/auto_round/algorithms/transforms/rotation/transforms.py b/auto_round/algorithms/transforms/rotation/transforms.pynew file mode 100644index 000000000..78dcd0d6b--- /dev/null+++ b/auto_round/algorithms/transforms/rotation/transforms.py@@ -0,0 +1,172 @@+# Copyright (c) 2026 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+"""Concrete ``torch.nn.Module`` implementations of Hadamard transforms.++:class:`HadamardTransform` – block-diagonal Hadamard (deterministic).+:class:`RandomHadamardTransform` – randomly signed Hadamard.+:func:`build_hadamard_transform` – factory that selects the right class.+"""++from __future__ import annotations++import inspect+import math+from typing import Any, Callable, Dict++import torch+import torch.nn as nn++from auto_round.algorithms.transforms.rotation.utils.math import (+ deterministic_hadamard_matrix,+ random_hadamard_matrix,+)+from auto_round.algorithms.transforms.rotation.utils.matrix import apply_transform_weight++__all__ = [+ "HadamardTransform",+ "RandomHadamardTransform",+ "HADAMARDS",+ "build_hadamard_transform",+]+++def _filter_kwargs(fn: Callable, kwargs: Dict[str, Any]) -> Dict[str, Any]:+ """Return only the keyword arguments accepted by *fn*."""+ accepted = inspect.signature(fn).parameters.keys()+ return {k: v for k, v in kwargs.items() if k in accepted}+++class HadamardTransform(nn.Module):+ """Block-diagonal deterministic Hadamard rotation.++ The rotation matrix ``W`` (stored as a frozen ``nn.Parameter``) is+ constructed once from :func:`deterministic_hadamard_matrix` and+ normalised by ``1 / sqrt(block_size)``.++ Args:+ block_size: Size of each Hadamard block (must be a power of 2).+ device: Device to place the weight on.+ precision: Dtype for the weight tensor.+ location: ``"weight"`` (default) or ``"input"`` – controls the+ orientation of the multiplication in :meth:`forward`.+ module_type: ``type(module)`` passed to+ :func:`~utils.matrix.apply_transform_weight`.+ inverse: If ``True``, use transposed orientation (for activation+ transforms that are the inverse of the weight transform).+ """++ def __init__(+ self,+ block_size: int | None = 32,+ device: torch.device | None = None,+ precision: torch.dtype | None = None,+ location: str = "weight",+ module_type: type[nn.Module] = nn.Linear,+ inverse: bool = False,+ ) -> None:+ super().__init__()+ self.size = block_size if block_size is not None else 32+ self.scale = 1.0 / math.sqrt(self.size)+ self.location = location+ self.module_type = module_type+ self.inverse = inverse+ self.weight = self._build_weight(self.size, device, precision)++ def _build_weight(+ self,+ size: int,+ device: torch.device | None,+ precision: torch.dtype | None,+ ) -> nn.Parameter:+ data = deterministic_hadamard_matrix(size, precision, device) * self.scale+ return nn.Parameter(data, requires_grad=False)++ def forward(self, x: torch.Tensor) -> torch.Tensor:+ ori_shape = x.shape+ x = x.view(-1, self.size)+ out = apply_transform_weight(+ self.weight.to(x.device),+ x.to(dtype=self.weight.dtype),+ self.location,+ self.module_type,+ )+ return out.to(x.dtype).view(ori_shape)+++class RandomHadamardTransform(HadamardTransform):+ """Randomly signed Hadamard rotation.++ Extends :class:`HadamardTransform` with a seeded random diagonal so the+ same seed always produces the same rotation matrix.++ Args:+ seed: Integer seed for the internal ``torch.Generator``.+ generator: Pre-built ``torch.Generator`` (overrides *seed* if given).+ *args, **kwargs: Forwarded to :class:`HadamardTransform`.+ """++ def __init__(+ self,+ *args: Any,+ seed: int | None = None,+ generator: torch.Generator | None = None,+ **kwargs: Any,+ ) -> None:+ if generator is not None:+ self.generator = generator+ else:+ self.generator = torch.Generator()+ if seed is not None:+ self.generator.manual_seed(seed)+ super().__init__(*args, **kwargs)++ def _build_weight(+ self,+ size: int,+ device: torch.device | None,+ precision: torch.dtype | None,+ ) -> nn.Parameter:+ data = random_hadamard_matrix(size, precision, device, self.generator) * self.scale+ if self.inverse:+ data = data.T+ return nn.Parameter(data, requires_grad=False)+++# ---------------------------------------------------------------------------+# Registry and factory+# ---------------------------------------------------------------------------++#: Maps ``hadamard_type`` strings to their transform classes.+HADAMARDS: dict[str, type[HadamardTransform]] = {+ "hadamard": HadamardTransform,+ "random_hadamard": RandomHadamardTransform,+}+++def build_hadamard_transform(hadamard_type: str, **kwargs: Any) -> HadamardTransform:+ """Instantiate the correct :class:`HadamardTransform` subclass.++ Args:+ hadamard_type: Key into :data:`HADAMARDS` (``"hadamard"`` or+ ``"random_hadamard"``).+ **kwargs: Forwarded to the transform constructor after filtering+ out unsupported keys.++ Returns:+ A new :class:`HadamardTransform` instance.+ """+ if hadamard_type not in HADAMARDS:+ raise ValueError(f"Unknown hadamard_type: {hadamard_type!r}. " f"Available: {sorted(HADAMARDS)}")+ cls = HADAMARDS[hadamard_type]+ return cls(**_filter_kwargs(cls.__init__, kwargs))diff --git a/auto_round/algorithms/transforms/rotation/utils/__init__.py b/auto_round/algorithms/transforms/rotation/utils/__init__.pynew file mode 100644index 000000000..0b3bf2eae--- /dev/null+++ b/auto_round/algorithms/transforms/rotation/utils/__init__.py@@ -0,0 +1,2 @@+# # Copyright (C) 2026 Intel Corporation+# # SPDX-License-Identifier: Apache-2.0diff --git a/auto_round/experimental/transform/utils/hadamards.safetensors b/auto_round/algorithms/transforms/rotation/utils/hadamards.safetensorssimilarity index 100%rename from auto_round/experimental/transform/utils/hadamards.safetensorsrename to auto_round/algorithms/transforms/rotation/utils/hadamards.safetensorsdiff --git a/auto_round/algorithms/transforms/rotation/utils/math.py b/auto_round/algorithms/transforms/rotation/utils/math.pynew file mode 100644index 000000000..14b15ce0d--- /dev/null+++ b/auto_round/algorithms/transforms/rotation/utils/math.py@@ -0,0 +1,144 @@+# Copyright (c) 2026 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+"""Hadamard matrix construction utilities.++Provides ``deterministic_hadamard_matrix`` (Sylvester construction) and+``random_hadamard_matrix`` (loaded from a precomputed safetensors file).+"""++# note that hadamard matrix multiplication reuses code from+# https://github.com/vllm-project/compressed-tensors/blob/main/src/compressed_tensors/transform/utils/hadamard.py++from __future__ import annotations++import math+from pathlib import Path++import torch+from safetensors import safe_open++__all__ = ["deterministic_hadamard_matrix", "random_hadamard_matrix", "is_pow2"]++# Precomputed Hadamard matrices for non-power-of-2 sizes.+_HADAMARD_MATRICES_PATH: Path = Path(__file__).parent / "hadamards.safetensors"+++def deterministic_hadamard_matrix(+ size: int,+ dtype: torch.dtype = torch.bfloat16,+ device: torch.device = torch.device("cpu"),+) -> torch.Tensor:+ """Construct an ``(size × size)`` Hadamard matrix via Sylvester's construction.++ ``size`` must be a power of 2.++ Adapted from https://github.com/scipy/scipy/blob/v1.15.2/scipy/linalg/_special_matrices.py++ Args:+ size: Order of the matrix; must be a power of 2.+ dtype: Output dtype.+ device: Output device.++ Returns:+ Hadamard tensor of shape ``(size, size)``.+ """+ if size <= 0:+ raise ValueError("Cannot construct Hadamard matrix with size <= 0")+ log2 = int(math.log2(size))+ if size != 2**log2:+ raise ValueError("Deterministic Hadamard requires size == 2^n")++ H = torch.tensor([[1]], dtype=dtype, device=device)+ for _ in range(log2):+ H = torch.vstack((torch.hstack((H, H)), torch.hstack((H, -H))))+ return H+++def random_hadamard_matrix(+ size: int,+ dtype: torch.dtype = torch.bfloat16,+ device: torch.device = torch.device("cpu"),+ gen: torch.Generator | None = None,+) -> torch.Tensor:+ """Create a randomly signed Hadamard matrix of order *size*.++ Supports non-powers-of-2 by reading a precomputed base matrix from+ ``hadamards.safetensors`` and composing it with a random ±1 diagonal.++ Adapted from https://github.com/facebookresearch/SpinQuant/blob/main/utils/hadamard_utils.py++ Args:+ size: Dimension of the matrix.+ dtype: Output dtype.+ device: Output device.+ gen: Optional seeded ``torch.Generator`` for reproducibility.++ Returns:+ Randomly signed Hadamard tensor of shape ``(size, size)``.+ """+ Q = torch.randint(0, 2, (size,), generator=gen, dtype=dtype).to(device)+ Q = Q * 2 - 1+ return _matmul_hadU(torch.diag(Q))+++def is_pow2(n: int) -> bool:+ """Return ``True`` iff *n* is a positive power of two."""+ return n > 0 and (n & (n - 1)) == 0+++# ---------------------------------------------------------------------------+# Internal helpers+# ---------------------------------------------------------------------------+++def _fetch_hadamard_divisor(+ n: int,+ dtype: torch.dtype,+ device: torch.device = torch.device("cpu"),+ file_path: Path = _HADAMARD_MATRICES_PATH,+) -> torch.Tensor | None:+ """Return the largest precomputed Hadamard divisor ``k`` of *n* such that+ ``n / k`` is a power of two, or ``None`` if no such entry exists."""+ open_device = torch.device("cpu") if device.type == "meta" else device+ with safe_open(str(file_path), framework="pt", device=str(open_device)) as f:+ divisors = sorted((int(key) for key in f.keys()), reverse=True)+ for divisor in divisors:+ if n % divisor == 0 and is_pow2(n // divisor):+ return f.get_tensor(str(divisor)).to(dtype=dtype, device=device)+ return None+++def _matmul_hadU(X: torch.Tensor) -> torch.Tensor:+ """Multiply *X* (a diagonal matrix) by the appropriate Hadamard matrix."""+ size = X.size(0)+ dtype = X.dtype+ device = X.device++ hadK = _fetch_hadamard_divisor(size, dtype, device=device)+ if hadK is None:+ raise ValueError(f"Cannot construct random Hadamard matrix of size {size}")+ K = hadK.size(0)++ inp = X.clone().view(-1, size, 1)+ out = inp.clone()+ while inp.shape[1] > K:+ inp = inp.view(inp.shape[0], inp.shape[1] // 2, 2, inp.shape[2])+ out = out.view(inp.shape)+ out[:, :, 0, :] = inp[:, :, 0, :] + inp[:, :, 1, :]+ out[:, :, 1, :] = inp[:, :, 0, :] - inp[:, :, 1, :]+ out = out.view(inp.shape[0], inp.shape[1], -1)+ inp, out = out, inp+ assert inp.shape[1] == K+ del out+ return (hadK.view(1, K, K).to(inp) @ inp).view(X.shape)diff --git a/auto_round/algorithms/transforms/rotation/utils/matrix.py b/auto_round/algorithms/transforms/rotation/utils/matrix.pynew file mode 100644index 000000000..c8c723d83--- /dev/null+++ b/auto_round/algorithms/transforms/rotation/utils/matrix.py@@ -0,0 +1,102 @@+# Copyright (c) 2026 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+"""Linear-algebra helpers for applying weight/activation rotation matrices.++Note: ``apply_transform_weight`` reuses ideas from+https://github.com/vllm-project/compressed-tensors/blob/main/src/compressed_tensors/transform/utils/matrix.py+"""++from __future__ import annotations++import torch++__all__ = ["apply_transform_weight", "multihead_matmul"]+++def apply_transform_weight(+ transform_weight: torch.Tensor,+ value: torch.Tensor,+ location: str,+ module_type: type[torch.nn.Module],+) -> torch.Tensor:+ """Apply *transform_weight* to *value* according to *location*.++ The mathematical relationship for a ``torch.nn.Linear`` layer:++ .. code-block:: none++ y = x W.T (standard linear)+ yh = (x V) (U.T W Vi.T).T (rotated linear)++ where *V* is the input-side rotation and *U* the output-side rotation.++ Args:+ transform_weight: The rotation matrix to apply.+ value: The tensor to rotate (weight or activation).+ location: ``"input"`` or ``"weight"``.+ module_type: ``type(module)`` – determines how the weight transform+ is oriented.++ Returns:+ Rotated tensor with the same shape as *value*.+ """+ if location == "input":+ return multihead_matmul(value, transform_weight)++ if module_type is torch.nn.Linear:+ return multihead_matmul(value, transform_weight.T)++ raise NotImplementedError(+ f"apply_transform_weight: unsupported location={location!r} " f"with module_type={module_type}"+ )+++def multihead_matmul(A: torch.Tensor, B: torch.Tensor) -> torch.Tensor:+ """Block-diagonal matrix multiplication over the last two dimensions.++ Handles the case where *A* and *B* have different sizes in their inner+ dimension by treating the smaller matrix as a repeated block-diagonal.++ For example, if ``A.shape[-1] == 2 * B.shape[-2]``, this is equivalent to::++ A @ block_diag(B, B)++ Args:+ A: Left-hand tensor.+ B: Right-hand tensor.++ Returns:+ Result of the generalised matrix multiplication.++ Raises:+ ValueError: If the inner dimensions are not evenly divisible.+ """+ a_inner = A.shape[-1]+ b_inner = B.shape[-2]++ if a_inner > b_inner:+ if a_inner % b_inner != 0:+ raise ValueError(f"multihead_matmul: A.shape[-1]={a_inner} is not divisible " f"by B.shape[-2]={b_inner}")+ num_heads = a_inner // b_inner+ A = A.unflatten(-1, (num_heads, b_inner))+ return (A @ B).flatten(-2, -1)++ if a_inner < b_inner:+ if b_inner % a_inner != 0:+ raise ValueError(f"multihead_matmul: B.shape[-2]={b_inner} is not divisible " f"by A.shape[-1]={a_inner}")+ num_heads = b_inner // a_inner+ B = B.unflatten(-2, (num_heads, a_inner))+ return (A @ B).flatten(-3, -2)++ return A @ Bdiff --git a/auto_round/algorithms/transforms/rotation/utils/triton/__init__.py b/auto_round/algorithms/transforms/rotation/utils/triton/__init__.pynew file mode 100644index 000000000..0b3bf2eae--- /dev/null+++ b/auto_round/algorithms/transforms/rotation/utils/triton/__init__.py@@ -0,0 +1,2 @@+# # Copyright (C) 2026 Intel Corporation+# # SPDX-License-Identifier: Apache-2.0diff --git a/auto_round/algorithms/transforms/rotation/utils/triton/mxfp4.py b/auto_round/algorithms/transforms/rotation/utils/triton/mxfp4.pynew file mode 100644index 000000000..c26413248--- /dev/null+++ b/auto_round/algorithms/transforms/rotation/utils/triton/mxfp4.py@@ -0,0 +1,192 @@+# Copyright (c) 2026 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.++# Refer code here:+# https://github.com/IST-DASLab/FP-Quant/blob/master/inference_lib/src/fp_quant/module/triton/mxfp4.py++import torch+import triton # pylint: disable=E0401+import triton.language as tl # pylint: disable=E0401+++ configs=[+ triton.Config({"BLOCK_SIZE": 32 * 32}),+ triton.Config({"BLOCK_SIZE": 64 * 32}),+ triton.Config({"BLOCK_SIZE": 128 * 32}),+ triton.Config({"BLOCK_SIZE": 256 * 32}),+ triton.Config({"BLOCK_SIZE": 512 * 32}),+ ],+ key=[],+)+def mxfp4_forward_kernel(+ x_ptr,+ hadamard_matrix_ptr,+ output_ptr,+ clip_mask_ptr,+ n_elements: tl.constexpr,+ hadamard_dim: tl.constexpr,+ group_size: tl.constexpr,+ gaussian_scale: tl.constexpr,+ quest: tl.constexpr,+ BLOCK_SIZE: tl.constexpr,+):+ offsets_hadamard = tl.arange(0, hadamard_dim * hadamard_dim)+ hadamard_matrix = tl.load(hadamard_matrix_ptr + offsets_hadamard).reshape(hadamard_dim, hadamard_dim)++ # load x+ pid = tl.program_id(0)+ start_idx = pid * BLOCK_SIZE+ offsets = start_idx + tl.arange(0, BLOCK_SIZE)+ mask = offsets < n_elements+ x_flat = tl.load(x_ptr + offsets, mask=mask)++ # hadamard transform+ x = tl.reshape(x_flat, (BLOCK_SIZE // hadamard_dim, hadamard_dim))+ x_had = tl.dot(x, hadamard_matrix)++ # group+ x_had_grouped = tl.reshape(x_had, (BLOCK_SIZE // group_size, group_size))++ # scale+ # quest=True: per-group Gaussian-based scale = gaussian_scale * std+ # quest=False: per-group max-abs-based scale, adjusted to FP4 range+ if quest:+ mean_squared = tl.sum(x_had_grouped * x_had_grouped, axis=-1, keep_dims=True) / group_size+ mean = tl.sum(x_had_grouped, axis=-1, keep_dims=True) / group_size+ std = tl.sqrt(mean_squared - mean * mean)+ scales = gaussian_scale * std + 1e-8+ shared_exps = tl.exp2(tl.floor(tl.log2(scales)))+ x_had_scaled = x_had_grouped / shared_exps+ else:+ scales = tl.max(tl.abs(x_had_grouped), axis=-1, keep_dims=True)+ shared_exps = tl.exp2(tl.floor(tl.log2(scales)) - 2) / (3 / 4)+ x_had_scaled = x_had_grouped / shared_exps++ # quantize+ # Map abs(x) to FP4 levels {0, 0.5, 1, 1.5, 2, 3, 4, 6}+ x_had_scaled_abs = tl.abs(x_had_scaled)+ x_had_scaled_sign = tl.where(+ x_had_scaled > 0,+ 1,+ -1,+ )++ x_fp4 = (+ tl.where(+ x_had_scaled_abs > 5,+ 6,+ tl.where(+ x_had_scaled_abs > 3.5,+ 4,+ tl.where(+ x_had_scaled_abs > 2.5,+ 3,+ tl.where(+ x_had_scaled_abs > 1.75,+ 2,+ tl.where(+ x_had_scaled_abs > 1.25,+ 1.5,+ tl.where(+ x_had_scaled_abs > 0.75,+ 1,+ tl.where(+ x_had_scaled_abs > 0.25,+ 0.5,+ 0,+ ),+ ),+ ),+ ),+ ),+ ),+ )+ * x_had_scaled_sign+ )+ if clip_mask_ptr is not None:+ tl.store(+ clip_mask_ptr + offsets,+ tl.reshape(x_had_scaled_abs < 6, (BLOCK_SIZE,)),+ mask=mask,+ )++ # dequantize+ x_dequantized = x_fp4 * shared_exps++ # Reshape back to flat form for storage+ x_dequantized_flat = tl.reshape(x_dequantized, (BLOCK_SIZE,))++ # store+ tl.store(output_ptr + offsets, x_dequantized_flat, mask=mask)+++def mxfp4_forward_kernel_wrapper(+ x,+ hadamard_matrix,+ return_clip_mask=False,+ quest=False,+ gaussian_scale=3 / 4,+):+ """+ Refer code here:+ https://github.com/IST-DASLab/FP-Quant/blob/master/inference_lib/src/fp_quant/module/triton/mxfp4.py+ Apply Hadamard transform + group-wise FP4 quantize/dequantize on x.++ Note:+ The output is still in the Hadamard-transformed space (no inverse Hadamard is applied).+ """+ # Pick a device — we require CUDA+ device = x.device+ if device.type != "cuda":+ raise RuntimeError(+ f"mxfp4_forward_kernel_wrapper requires a CUDA tensor for 'x', "+ f"but got device '{device.type}'. Please move inputs to CUDA before calling."+ )++ # Ensure hadamard_matrix is on the same CUDA device+ if hadamard_matrix.device != device:+ hadamard_matrix = hadamard_matrix.to(device)++ # Make sure inputs are contiguous+ x = x.contiguous()+ hadamard_matrix = hadamard_matrix.contiguous()++ # Create output tensors on CUDA+ output = torch.empty_like(x, device=device)+ if return_clip_mask:+ clip_mask = torch.empty_like(x, dtype=torch.bool, device=device).contiguous()+ else:+ clip_mask = None++ # Get total number of elements and calculate grid for launching the kernel+ n_elements = x.numel()+ grid = lambda meta: (triton.cdiv(n_elements, meta["BLOCK_SIZE"]),)++ # Launch kernel – no need for `with torch.device(...)`+ mxfp4_forward_kernel[grid](+ x_ptr=x,+ hadamard_matrix_ptr=hadamard_matrix,+ output_ptr=output,+ clip_mask_ptr=clip_mask,+ n_elements=n_elements,+ hadamard_dim=hadamard_matrix.shape[-1],+ group_size=32,+ gaussian_scale=gaussian_scale,+ quest=quest,+ )++ return output, clip_maskdiff --git a/auto_round/autoround.py b/auto_round/autoround.pyindex 3f46d40fc..b2bec2651 100644--- a/auto_round/autoround.py+++ b/auto_round/autoround.py@@ -17,6 +17,7 @@ from typing import TYPE_CHECKING, Optional, Unionimport torch+import auto_round.envs as envsfrom auto_round.compressors import (AdamCompressor,BaseCompressor,@@ -33,6 +34,9 @@ from auto_round.utils import is_diffusion_model, is_mllm_modelif TYPE_CHECKING:from auto_round.auto_scheme.gen_auto_scheme import AutoScheme+# Default to new architecture; set AR_DISABLE_NEW_ARCH=true/1 to force old architecture.+NEW_ARCH = not envs.AR_DISABLE_NEW_ARCH+class AutoRound:"""Automatic weight rounding (Signed Gradient Descent) for LLM quantization@@ -53,7 +57,7 @@ class AutoRound:enable_torch_compile (bool): Whether to enable torch.compile for quant blocks/layers."""- SKIP_ARGS = ("local_args", "kwargs", "cls", "model_cls", "dynamic_compressor", "extra_config", "enable_adam")+ SKIP_ARGS = ("local_args", "kwargs", "cls", "model_cls", "dynamic_compressor", "extra_config")bits: int | Nonegroup_size: int | tuple | None@@ -160,6 +164,11 @@ class AutoRound:local_args = {k: v for k, v in locals().items() if k not in cls.SKIP_ARGS}+ if NEW_ARCH:+ from auto_round.compressors_new.entry import AutoRoundCompatible++ return AutoRoundCompatible(**local_args, **kwargs)+model_cls = []has_multimodal_assets = kwargs.get("processor") is not None or kwargs.get("image_processor") is not None@@ -238,21 +247,23 @@ class AutoRound:for key in input_others.keys():if "positional_inputs" in key:continue- if (key not in share_cache_keys or len(indices) == 1) and not isinstance(- input_others[key], (str, bool, type(None))- ):- current_input_others[key] = None- if input_others[key] is not None:- current_input_others[key] = [input_others[key][i] for i in indices]- if len(indices) == 1:- current_input_others[key] = current_input_others[key][0]- else:- try:- current_input_others[key] = torch.cat(current_input_others[key], dim=0)- except TypeError as err:- logger.warning_once("Please check the model cache inputs or try setting batch_size to 1.")- else:+ # Shared cache keys (e.g. position_embeddings, position_ids, cache_position) are stored+ # directly as-is (not wrapped in a per-sample list) when batch_size > 1. Indexing such+ # values by sample index would incorrectly decompose them (e.g. (cos, sin)[0] == cos).+ # Always pass them through unchanged.+ if key in share_cache_keys or isinstance(input_others[key], (str, bool, type(None))):current_input_others[key] = input_others[key]+ elif input_others[key] is not None:+ current_input_others[key] = [input_others[key][i] for i in indices]+ if len(indices) == 1:+ current_input_others[key] = current_input_others[key][0]+ else:+ try:+ current_input_others[key] = torch.cat(current_input_others[key], dim=0)+ except TypeError as err:+ logger.warning_once("Please check the model cache inputs or try setting batch_size to 1.")+ else:+ current_input_others[key] = Nonereturn current_input_ids, current_input_othersdiff --git a/auto_round/calibration/__init__.py b/auto_round/calibration/__init__.pynew file mode 100644index 000000000..14a492441--- /dev/null+++ b/auto_round/calibration/__init__.py@@ -0,0 +1,13 @@+# Copyright (c) 2026 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.diff --git a/auto_round/calibration/utils.py b/auto_round/calibration/utils.pynew file mode 100644index 000000000..523561aef--- /dev/null+++ b/auto_round/calibration/utils.py@@ -0,0 +1,63 @@+# Copyright (c) 2026 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.++import torch++from auto_round.modeling.fused_moe.replace_modules import materialize_model_, safe_to_cpu_+from auto_round.utils import (+ is_quantized_input_module,+)+++def _infer_last_cache_name(block_names, layer_names=None, requested_last_cache_name=None):+ """The latest required cache layer for early-stop forward.++ If there are multiple cache targets, return ``None`` and let runtime+ hooks stop only after all targets are observed in real forward execution.+ """+ if layer_names is None:+ layer_names = []++ if requested_last_cache_name is not None:+ return requested_last_cache_name++ cache_targets = list(block_names) + list(layer_names)+ if len(cache_targets) == 1:+ return cache_targets[0]++ # return None here to enable the logic in _should_stop_cache_forward+ return None+++def _update_inputs(inputs: dict, q_inputs: dict) -> tuple[dict, torch.Tensor]:+ from auto_round.context.model import ModelContext++ model_context = ModelContext()+ if model_context.is_diffusion:+ # flux transformer model's blocks will update hidden_states and encoder_hidden_states+ input_id_str = [key for key in inputs.keys() if "hidden_state" in key]+ if q_inputs is not None:+ q_inputs = {k: q_inputs.pop(k, None) for k in input_id_str}+ return inputs, q_inputs++ keys = inputs.keys()+ input_id_str = [key for key in keys if key.startswith("hidden_state")]+ if len(input_id_str) != 1:+ raise RuntimeError(+ "hidden_states arg mismatch error," "please raise an issue in https://github.com/intel/auto-round/issues"+ )+ inputs["input_ids"] = inputs.pop(input_id_str[0], None)+ if q_inputs is not None:+ q_inputs = q_inputs.pop(input_id_str[0], None)+ return inputs, q_inputsdiff --git a/auto_round/compressors/base.py b/auto_round/compressors/base.pyindex 985ccfc46..0503c8235 100644--- a/auto_round/compressors/base.py+++ b/auto_round/compressors/base.py@@ -33,6 +33,7 @@ from tqdm import tqdmfrom transformers import AutoConfig, set_seedfrom auto_round import envs+from auto_round.algorithms.quantization.sign_round.sign_sgd import SignSGDfrom auto_round.auto_scheme.gen_auto_scheme import AutoSchemefrom auto_round.compressors.shard_writer import shard_writerfrom auto_round.compressors.utils import (@@ -71,7 +72,6 @@ from auto_round.schemes import (preset_name_to_scheme,scheme_to_preset_name,)-from auto_round.sign_sgd import SignSGDfrom auto_round.special_model_handler import get_predefined_fixed_attr, get_predefined_ignore_layers, update_modulefrom auto_round.utils import (INNER_SUPPORTED_LAYER_TYPES,@@ -1865,6 +1865,14 @@ class BaseCompressor(object):if self.low_cpu_mem_usage and self.is_immediate_packing:self.is_immediate_saving = True+ if self.low_cpu_mem_usage and not self.is_immediate_packing:+ logger.info(+ "`low_cpu_mem_usage` is only supported when `immediate_packing` is True. "+ "Setting `low_cpu_mem_usage` to False."+ )+ self.low_cpu_mem_usage = False+ self.is_immediate_saving = False+if self.low_cpu_mem_usage and self.is_immediate_packing:if formats[0].is_gguf():logger.warning(@@ -2679,10 +2687,10 @@ class BaseCompressor(object):Processed data or None"""new_data = data- if batch_size <= 1:- return new_dataif data_name in self.shared_cache_keys:return None+ if batch_size <= 1:+ return new_dataif "alibi" in data_name:if isinstance(data, torch.Tensor):alibi = data@@ -2736,7 +2744,7 @@ class BaseCompressor(object):continueif key not in self.inputs[name].keys(): # initializationdata = to_device(kwargs[key], device=torch.device("cpu"))- if data is None or (self.batch_size > 1 and key in self.shared_cache_keys):+ if data is None or key in self.shared_cache_keys:self.inputs[name][key] = datacontinueif self.batch_size <= 1:@@ -3743,21 +3751,23 @@ class BaseCompressor(object):for key in input_others.keys():if "positional_inputs" in key:continue- if (key not in share_cache_keys or len(indices) == 1) and not isinstance(- input_others[key], (str, bool, type(None))- ):- current_input_others[key] = None- if input_others[key] is not None:- current_input_others[key] = [input_others[key][i] for i in indices]- if len(indices) == 1:- current_input_others[key] = current_input_others[key][0]- else:- try:- current_input_others[key] = torch.cat(current_input_others[key], dim=0)- except TypeError as err:- logger.warning_once("Please check the model cache inputs or try setting batch_size to 1.")- else:+ # Shared cache keys (e.g. position_embeddings, position_ids, cache_position) are stored+ # directly as-is (not wrapped in a per-sample list) when batch_size > 1. Indexing such+ # values by sample index would incorrectly decompose them (e.g. (cos, sin)[0] == cos).+ # Always pass them through unchanged.+ if key in share_cache_keys or isinstance(input_others[key], (str, bool, type(None))):current_input_others[key] = input_others[key]+ elif input_others[key] is not None:+ current_input_others[key] = [input_others[key][i] for i in indices]+ if len(indices) == 1:+ current_input_others[key] = current_input_others[key][0]+ else:+ try:+ current_input_others[key] = torch.cat(current_input_others[key], dim=0)+ except TypeError as err:+ logger.warning_once("Please check the model cache inputs or try setting batch_size to 1.")+ else:+ current_input_others[key] = Nonereturn current_input_ids, current_input_othersdiff --git a/auto_round/compressors/utils.py b/auto_round/compressors/utils.pyindex 7b6f04545..fb6c8a0e6 100644--- a/auto_round/compressors/utils.py+++ b/auto_round/compressors/utils.py@@ -28,7 +28,7 @@ from torch.amp import autocastfrom auto_round.export.export_to_gguf.config import GGML_QUANT_SIZES, GGUF_CONFIG, GGUF_INNER_CONFIG, QK_K, ModelTypefrom auto_round.logger import loggerfrom auto_round.schemes import QuantizationScheme, get_gguf_scheme, preset_name_to_scheme-from auto_round.utils import check_to_quantized, to_standard_regex+from auto_round.utils import check_to_quantized, infer_bits_by_data_type, to_standard_regexclass BackendDataType(str, Enum):@@ -225,30 +225,6 @@ def collect_best_params(block, cache_device="cpu"):return params-def infer_bits_by_data_type(data_type: str):- """Infer bits by data_type-- Args:- data_type (str): data_type-- Returns:- int: bits inferred by data_type, None means cannot infer correct bits by data_type- """- from auto_round.utils import SUPPORTED_DTYPES-- if data_type is None:- return 16- for supported_dtype in SUPPORTED_DTYPES:- if data_type.startswith(supported_dtype) and len(data_type) > len(supported_dtype):- ##first check the following two bits- suc_2str = data_type[len(supported_dtype) : len(supported_dtype) + 2]- if str.isdigit(suc_2str):- return int(suc_2str)- if str.isdigit(data_type[len(supported_dtype)]):- return int(data_type[len(supported_dtype)])- return None--def _get_safetensor_layer_names_not_in_model(model, all_module_names: list) -> list:"""Collect layer names from safetensor files that are not loaded into the model.diff --git a/auto_round/compressors_new/__init__.py b/auto_round/compressors_new/__init__.pynew file mode 100644index 000000000..66dbe5de9--- /dev/null+++ b/auto_round/compressors_new/__init__.py@@ -0,0 +1,53 @@+# Copyright (c) 2026 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.++# Lazy imports to avoid circular dependencies+# Users should import from specific modules instead of this __init__.py++from typing import TYPE_CHECKING++if TYPE_CHECKING:+ from auto_round.compressors_new.calib import CalibCompressor, CalibratedRTNCompressor+ from auto_round.compressors_new.entry import AutoRoundCompatible, AutoRound+ from auto_round.compressors_new.zero_shot import ZeroShotCompressor++__all__ = [+ "AutoRound",+ "CalibCompressor",+ "CalibratedRTNCompressor",+ "ZeroShotCompressor",+ "AutoRoundCompatible",+]+++def __getattr__(name):+ """Lazy import to avoid circular dependencies."""+ if name == "AutoRound" or name == "AutoRoundCompatible":+ from auto_round.compressors_new.entry import AutoRound, AutoRoundCompatible++ if name == "AutoRound":+ return AutoRound+ return AutoRoundCompatible+ elif name in ("CalibCompressor", "CalibratedRTNCompressor"):+ from auto_round.compressors_new.calib import CalibCompressor, CalibratedRTNCompressor++ return {+ "CalibCompressor": CalibCompressor,+ "CalibratedRTNCompressor": CalibratedRTNCompressor,+ }[name]+ elif name == "ZeroShotCompressor":+ from auto_round.compressors_new.zero_shot import ZeroShotCompressor++ return ZeroShotCompressor+ raise AttributeError(f"module '{__name__}' has no attribute '{name}'")diff --git a/auto_round/compressors_new/architecture_visualization.py b/auto_round/compressors_new/architecture_visualization.pynew file mode 100644index 000000000..7ff020733--- /dev/null+++ b/auto_round/compressors_new/architecture_visualization.py@@ -0,0 +1,332 @@+# # Copyright (C) 2026 Intel Corporation+# # SPDX-License-Identifier: Apache-2.0+"""+New Architecture Visualization - Mixin Pattern Combination Table++Demonstrates all possible combinations of model types and compression algorithms.+"""+++def print_architecture_table():+ """Print architecture combination table"""++ print("\n" + "=" * 110)+ print("AutoRound New Architecture - Mixin Pattern Combination Table")+ print("=" * 110 + "\n")++ print(f"{'Model Type':<15} {'Config Type':<20} {'AutoRound (dynamic class)':<40} {'Base classes':<35}")+ print("-" * 110)++ # LLM combinations+ print(f"{'LLM':<15} {'SignRoundConfig':<20} {'CalibCompressor':<40} {'CalibCompressor':<35}")+ print(f"{'LLM':<15} {'RTNConfig':<20} {'CalibratedRTNCompressor':<40} {'CalibratedRTNCompressor':<35}")+ print(f"{'LLM':<15} {'RTNConfig':<20} {'ZeroShotCompressor':<40} {'ZeroShotCompressor':<35}")++ print()++ # MLLM combinations (dynamic classes created in entry.py)+ print(f"{'MLLM':<15} {'SignRoundConfig':<20} {'MLLMCalibCompressor':<40} {'MLLMMixin + CalibCompressor':<35}")+ print(+ f"{'MLLM':<15} {'RTNConfig':<20} {'MLLMCalibratedRTNCompressor':<40} "+ f"{'MLLMMixin + CalibratedRTNCompressor':<35}"+ )+ print(f"{'MLLM':<15} {'RTNConfig':<20} {'MLLMZeroShotCompressor':<40} {'MLLMMixin + ZeroShotCompressor':<35}")++ print()++ # Diffusion combinations (dynamic classes created in entry.py)+ print(+ f"{'Diffusion':<15} {'SignRoundConfig':<20} {'DiffusionCalibCompressor':<40} "+ f"{'DiffusionMixin + CalibCompressor':<35}"+ )+ print(+ f"{'Diffusion':<15} {'RTNConfig':<20} {'DiffusionCalibratedRTNCompressor':<40} "+ f"{'DiffusionMixin + CalibratedRTNCompressor':<35}"+ )+ print(+ f"{'Diffusion':<15} {'RTNConfig':<20} {'DiffusionZeroShotCompressor':<40} "+ f"{'DiffusionMixin + ZeroShotCompressor':<35}"+ )++ print("\n" + "=" * 110 + "\n")+++def print_mixin_explanation():+ """Print Mixin pattern explanation"""++ print("=" * 110)+ print("Mixin Pattern Explanation")+ print("=" * 110 + "\n")++ print("✨ Core Components:")+ print("-" * 110)+ print(" 1. MLLMMixin - MLLM features (processor, template, quant_nontext_module, etc.)")+ print(" 2. DiffusionMixin - Diffusion features (pipeline loading, guidance_scale, etc.)")+ print(" 3. CalibCompressor - AutoRoundCompatible: gradient-based calibration quantization")+ print(" 4. CalibratedRTNCompressor - RTN with importance-matrix (imatrix) or act calibration")+ print(" 5. ZeroShotCompressor - Zero-shot RTN (no calibration data needed)")++ print("\n🎯 Combination Approach:")+ print("-" * 110)+ print(" Dynamic classes created on-the-fly inside AutoRound.__new__():")+ print(" class MLLMCalibCompressor(MLLMMixin, CalibCompressor): pass")+ print(" class MLLMCalibratedRTNCompressor(MLLMMixin, CalibratedRTNCompressor): pass")+ print(" class MLLMZeroShotCompressor(MLLMMixin, ZeroShotCompressor): pass")++ print("\n💡 Advantages:")+ print("-" * 110)+ print(" ✓ Flexible Combination: Any model type can be combined with any compression algorithm")+ print(" ✓ Code Reuse: Mixin code is written once and reused across all compression algorithms")+ print(" ✓ Clear Separation: Model-specific logic (Mixin) and compression algorithm are independent")+ print(" ✓ Easy Extension: Add new model types without touching existing compressor code")++ print("\n" + "=" * 110 + "\n")+++def print_post_init_flow():+ """Print the post_init execution flow"""++ print("=" * 110)+ print("BaseCompressor.post_init() Execution Flow")+ print("=" * 110 + "\n")++ print("""+BaseCompressor.post_init()+│+├─ Step 1: Resolve formats (str → list[OutputFormat])+│ └─ get_formats(self.formats, self)+│+├─ Step 2: Apply format-specific model patches+│ └─ model_context.apply_patches(formats)+│ ├─ _patch_custom_moe_modules() # e.g. Qwen3VL MoE top_k fix+│ ├─ update_module(model, formats) # add gguf_pack_linear etc.+│ └─ assign global_name to all modules+│+├─ Step 3: Setup quantizer on the patched model+│ └─ quantizer = BaseQuantizers.from_config(config)+│ └─ quantizer.post_init()+│ ├─ get ModelContext / CompressContext singletons+│ ├─ _parse_scheme() → resolve final quant attrs+│ ├─ get_block_names(quant_vision=quant_nontext_module)+│ ├─ find_matching_blocks() → quant_block_list+│ └─ back-fill to_quant_block_names if it was None+│+└─ Step 4: Setup device map, torch compile, offloader+ """)++ print("=" * 110 + "\n")+++def print_usage_examples():+ """Print usage examples"""++ print("=" * 110)+ print("Usage Examples")+ print("=" * 110 + "\n")++ print("Example 1: MLLM + AutoRoundCompatible (gradient-based)")+ print("-" * 110)+ print("""+from auto_round.compressors_new.entry import AutoRound+from auto_round.algorithms.quantization.sign_round.config import SignRoundConfig++config = SignRoundConfig(scheme="W4A16", iters=200, nsamples=128)+compressor = AutoRound(+ config=config,+ model="/models/Qwen2-VL-2B-Instruct",+ processor=processor,+ template="qwen2_vl",+ quant_nontext_module=False, # set True to also quantize vision encoder+)+# Dynamically creates: class MLLMCalibCompressor(MLLMMixin, CalibCompressor)+ """)++ print("\nExample 2: MLLM + RTN with imatrix")+ print("-" * 110)+ print("""+from auto_round.algorithms.quantization.rtn.config import RTNConfig++config = RTNConfig(scheme="W4A16")+compressor = AutoRound(+ config=config,+ model="/models/Qwen2-VL-2B-Instruct",+ format="gguf_k", # gguf_k triggers CalibratedRTNCompressor+ processor=processor,+)+# Dynamically creates: class MLLMCalibratedRTNCompressor(MLLMMixin, CalibratedRTNCompressor)+ """)++ print("\nExample 3: Diffusion + AutoRoundCompatible")+ print("-" * 110)+ print("""+config = SignRoundConfig(scheme="W4A16", iters=200)+compressor = AutoRound(+ config=config,+ model="/models/stable-diffusion-2-1",+ guidance_scale=7.5,+)+# Dynamically creates: class DiffusionCalibCompressor(DiffusionMixin, CalibCompressor)+ """)++ print("\n" + "=" * 110 + "\n")+++def print_mro_example():+ """Print MRO (Method Resolution Order) example"""++ print("=" * 110)+ print("Method Resolution Order (MRO) Example")+ print("=" * 110 + "\n")++ print("For class MLLMCalibCompressor(MLLMMixin, CalibCompressor):")+ print("-" * 110)+ print("""+MLLMCalibCompressor (dynamic, created in AutoRound.__new__)+ └─> MLLMMixin+ └─> CalibCompressor+ └─> BaseCompressor+ └─> object++Execution order when calling __init__():+ 1. MLLMCalibCompressor.__init__() → not defined, falls through+ 2. MLLMMixin.__init__()+ - Save MLLM-specific attrs: processor, template, quant_nontext_module, …+ - kwargs.setdefault("quant_nontext_module", quant_nontext_module)+ - Call super().__init__() → enters CalibCompressor+ 3. CalibCompressor.__init__() → BaseCompressor.__init__()+ - pops quant_nontext_module from kwargs+ - Creates ModelContext(…, quant_nontext_module=quant_nontext_module)+ - ModelContext.__init__ eagerly loads the model+ - Creates CompressContext singleton++MLLMCalibCompressor instance has:+ ✓ MLLM features from MLLMMixin (processor, template, calib() override)+ ✓ Calibration compression from CalibCompressor+ ✓ Model/context management from BaseCompressor+ """)++ print("=" * 110 + "\n")+++def print_decision_tree():+ """Print decision tree"""++ print("=" * 110)+ print("AutoRound Creation Decision Tree")+ print("=" * 110 + "\n")++ print("""+AutoRound.__new__(config, model, format, **kwargs)+│+├─ Step 1: Detect model type+│ model_type = detect_model_type(model)+│ ├─ is_diffusion_model() → "diffusion"+│ ├─ is_mllm_model() → "mllm"+│ └─ else → "llm"+│+├─ isinstance(config, SignRoundConfig)+│ ├─ model_type == "mllm"+│ │ └─> class MLLMCalibCompressor(MLLMMixin, CalibCompressor)+│ ├─ model_type == "diffusion"+│ │ └─> class DiffusionCalibCompressor(DiffusionMixin, CalibCompressor)+│ └─ model_type == "llm"+│ └─> CalibCompressor+│+└─ isinstance(config, RTNConfig)+ │+ ├─ enable_imatrix OR needs_act_calib → CalibratedRTNCompressor path+ │ ├─ gguf_k format → enable_imatrix = True+ │ ├─ symmetric int RTN → enable_imatrix = True+ │ ├─ static activation quantization → needs_act_calib = True+ │ │+ │ ├─ model_type == "mllm"+ │ │ └─> class MLLMCalibratedRTNCompressor(MLLMMixin, CalibratedRTNCompressor)+ │ ├─ model_type == "diffusion"+ │ │ └─> class DiffusionCalibratedRTNCompressor(DiffusionMixin, CalibratedRTNCompressor)+ │ └─ model_type == "llm"+ │ └─> CalibratedRTNCompressor+ │+ └─ else (zero-shot) → ZeroShotCompressor path+ ├─ model_type == "mllm"+ │ └─> class MLLMZeroShotCompressor(MLLMMixin, ZeroShotCompressor)+ ├─ model_type == "diffusion"+ │ └─> class DiffusionZeroShotCompressor(DiffusionMixin, ZeroShotCompressor)+ └─ model_type == "llm"+ └─> ZeroShotCompressor+ """)++ print("=" * 110 + "\n")+++def print_quantizer_interface():+ """Print the BaseQuantizers interface contract"""++ print("=" * 110)+ print("BaseQuantizers Interface - Name-based quantize_block / quantize_layer")+ print("=" * 110 + "\n")++ print("""+All quantizers use module *names* (str) instead of module objects.+The module is retrieved internally via get_module(model, name).++ BaseQuantizers (abstract)+ ├─ quantize_block(block_name: Union[str, list[str]], input_ids, input_others, **kwargs)+ │ str → get_module(model, block_name)+ │ list[str] → WrapperMultiblock([get_module(model, n) for n in block_name])+ │ (used when nblocks > 1 in CalibCompressor)+ │+ └─ quantize_layer(layer_name: str, **kwargs)+ → get_module(model, layer_name)++ Implementations:+ ├─ RTNQuantizer.quantize_block(block_name: str)+ ├─ OptimizedRTNQuantizer.quantize_block(block_name: str, input_ids, input_others)+ └─ SignRoundQuantizer.quantize_block(block_name: Union[str, list[str]], input_ids, input_others)+ """)++ print("=" * 110 + "\n")+++def main():+ """Run all visualizations"""++ print_architecture_table()+ print_mixin_explanation()+ print_post_init_flow()+ print_usage_examples()+ print_mro_example()+ print_decision_tree()+ print_quantizer_interface()++ print("=" * 110)+ print("🎉 New architecture supports 9 combinations (3 model types × 3 compression algorithms)")+ print(" CalibratedRTNCompressor (was ImatrixCompressor) lives in calib.py")+ print("=" * 110)+++if __name__ == "__main__":+ main()++ print(f"{'LLM':<15} {'RTNConfig':<20} {'RTN (zero-shot)':<20} {'ZeroShotCompressor':<35}")++ print()++ # MLLM combinations+ print(f"{'MLLM':<15} {'SignRoundConfig':<20} {'AutoRoundCompatible':<20} {'MLLMCalibCompressor':<35}")+ print(f"{'':<15} {'':<20} {'':<20} {' = MLLMMixin + CalibCompressor':<35}")+ print(f"{'MLLM':<15} {'RTNConfig':<20} {'RTN + imatrix':<20} {'MLLMImatrixCompressor':<35}")+ print(f"{'':<15} {'':<20} {'':<20} {' = MLLMMixin + ImatrixCompressor':<35}")+ print(f"{'MLLM':<15} {'RTNConfig':<20} {'RTN (zero-shot)':<20} {'MLLMZeroShotCompressor':<35}")+ print(f"{'':<15} {'':<20} {'':<20} {' = MLLMMixin + ZeroShotCompressor':<35}")++ print()++ # Diffusion combinations+ print(f"{'Diffusion':<15} {'SignRoundConfig':<20} {'AutoRoundCompatible':<20} {'DiffusionCalibCompressor':<35}")+ print(f"{'':<15} {'':<20} {'':<20} {' = DiffusionMixin + CalibCompressor':<35}")+ print(f"{'Diffusion':<15} {'RTNConfig':<20} {'RTN + imatrix':<20} {'DiffusionImatrixCompressor':<35}")+ print(f"{'':<15} {'':<20} {'':<20} {' = DiffusionMixin + ImatrixCompressor':<35}")+ print(f"{'Diffusion':<15} {'RTNConfig':<20} {'RTN (zero-shot)':<20} {'DiffusionZeroShotCompressor':<35}")+ print(f"{'':<15} {'':<20} {'':<20} {' = DiffusionMixin + ZeroShotCompressor':<35}")++ print("\n" + "=" * 100 + "\n")diff --git a/auto_round/compressors_new/base.py b/auto_round/compressors_new/base.pynew file mode 100644index 000000000..37f5726e8--- /dev/null+++ b/auto_round/compressors_new/base.py@@ -0,0 +1,1310 @@+# Copyright (c) 2026 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+import copy+import gc+import os+import sys+from dataclasses import asdict, dataclass, fields+from typing import Any, Optional, Union++import torch+from transformers import set_seed++from auto_round.algorithms.alg_config import AlgConfig+from auto_round.algorithms.quantization import BaseQuantizers, QuantizationConfig+from auto_round.algorithms.transforms import (+ BaseRotationConfig,+ apply_rotation,+)+from auto_round.compressors.utils import is_mx_fp, is_nv_fp+from auto_round.compressors_new.shard_writer import ShardWriter+from auto_round.compressors_new.utils import _get_save_folder_name, block_forward, set_layer_config+from auto_round.context.compress import CompressContext+from auto_round.context.model import ModelContext+from auto_round.formats import OutputFormat, get_formats+from auto_round.logger import logger+from auto_round.schemes import (+ QuantizationScheme,+ _handle_special_schemes,+ _parse_scheme,+ get_gguf_scheme,+ preset_name_to_scheme,+)+from auto_round.special_model_handler import get_predefined_fixed_attr, get_predefined_ignore_layers+from auto_round.utils import (+ INNER_SUPPORTED_LAYER_TYPES,+ SUPPORTED_LAYER_TYPES,+ TORCH_VERSION_AT_LEAST_2_6,+ compile_func,+ compress_layer_names,+ convert_dtype_str2torch,+ extract_block_names_to_str,+ find_matching_blocks,+ get_block_names,+ is_debug_mode,+ is_hpex_available,+ is_quantized_input_module,+ memory_monitor,+)+from auto_round.utils.device import (+ _force_trim_malloc,+ get_major_device,+ patch_xpu_sdpa_drop_causal_mask,+ set_non_auto_device_map,+)+from auto_round.utils.offload import OffloadManager+from auto_round.wrapper import wrapper_block+++@dataclass+class SerializedCompressorConfig:+ bits: Optional[int] = None+ act_bits: Optional[int] = None+ data_type: Optional[str] = None+ act_data_type: Optional[str] = None+ group_size: Optional[int] = None+ act_group_size: Optional[int] = None+ sym: Optional[bool] = None+ act_sym: Optional[bool] = None+ act_dynamic: Optional[bool] = None+ amp: Optional[bool] = None+ batch_size: Optional[int] = None+ enable_minmax_tuning: Optional[bool] = True+ enable_norm_bias_tuning: Optional[bool] = False+ enable_quanted_input: Optional[bool] = True+ gradient_accumulate_steps: Optional[int] = None+ iters: Optional[int] = None+ lr: Optional[float] = None+ low_gpu_mem_usage: Optional[bool] = None+ minmax_lr: Optional[float] = None+ nsamples: Optional[int] = None+ quant_block_list: Optional[list[str]] = None+ regex_config: Optional[dict[str, Any]] = None+ scale_dtype: Optional[str] = None+ seqlen: Optional[int] = None+ supported_types: Optional[list[str]] = SUPPORTED_LAYER_TYPES+ static_attention_dtype: Optional[str] = None+ static_kv_dtype: Optional[str] = None+ super_bits: Optional[int] = None+ super_group_size: Optional[int] = None+ to_quant_block_names: Optional[list[str]] = None+ transform_configs: Optional[list[dict[str, Any]]] = None+++class BaseCompressor(object):+ need_calib: bool = True+ compress_context: CompressContext = None+ model_context: ModelContext = None+ shard_writer: ShardWriter = None+ supported_types = SUPPORTED_LAYER_TYPES+ inner_supported_types = INNER_SUPPORTED_LAYER_TYPES++ # ── Scheme state (populated during resolve_scheme / _scheme_post_init) ──+ is_auto_scheme: bool = False+ orig_scheme = None+ scheme = None+ scale_dtype = None+ layer_config = None+ has_qlayer_outside_block: bool = False+ regex_config: dict = None+ quant_block_list: list = None+ to_quant_block_names = None+ ignore_layers: str = ""+ quant_lm_head: bool = False+ _scheme_resolved: bool = False+ scheme_generator = None++ def __init__(+ self,+ config: Union[AlgConfig, list[AlgConfig]],+ model: Union[torch.nn.Module, str],+ tokenizer=None,+ platform="hf",+ format=None,+ scheme="W4A16",+ low_gpu_mem_usage: bool = False,+ device_map: Union[str, torch.device, int, dict] = 0,+ enable_torch_compile: bool = False,+ seed: int = 42,+ low_cpu_mem_usage: bool = True,+ layer_config=None,+ nsamples: int = None,+ seqlen: int = None,+ **kwargs,+ ):+ self.quantize_config = None+ self.transform_configs: list[BaseRotationConfig] = []+ _config_list = config if isinstance(config, list) else [config]+ for _cfg in _config_list:+ if isinstance(_cfg, QuantizationConfig):+ self.quantize_config = _cfg+ elif isinstance(_cfg, BaseRotationConfig):+ self.transform_configs.append(_cfg)+ assert self.quantize_config is not None, "QuantizationConfig is required for Compressor"++ # Compressor-level calibration/layer params (do not live in QuantizationConfig).+ self.layer_config = layer_config+ self.nsamples = nsamples if nsamples is not None else 128+ self.seqlen = seqlen if seqlen is not None else 2048++ # Scheme is passed directly to the compressor, not stored in QuantizationConfig.+ self.scheme = scheme++ # TODO: refactor calibration+ self.calibration = None++ self.formats = format++ # Extra/legacy kwargs for backward compatibility+ # Major version releases may pack them with extra configuration options+ amp = kwargs.pop("amp", True)+ nblocks = kwargs.pop("nblocks", 1)+ disable_deterministic_algorithms = kwargs.pop("disable_deterministic_algorithms", True)+ enable_deterministic_algorithms = kwargs.pop("enable_deterministic_algorithms", False)++ self._offloader = OffloadManager(enabled=low_cpu_mem_usage, mode="offload", offload_dir_prefix="compressor")++ # Model related+ model_dtype = kwargs.pop("model_dtype", None)+ trust_remote_code = kwargs.pop("trust_remote_code") if "trust_remote_code" in kwargs else True+ quant_nontext_module = kwargs.pop("quant_nontext_module", False)++ self.static_attention_dtype = kwargs.pop("static_attention_dtype", None)+ # Attention static dtype+ if self.static_attention_dtype is not None:+ logger.warning("The static attention dtype is experimental and currently has limited support.")+ # KV cache, this one does not affect tuning but will collect some infos during tuning+ self.static_kv_dtype = kwargs.pop("static_kv_dtype", None)+ if self.static_kv_dtype is not None:+ logger.warning("The static kv is experimental and currently has limited support.")++ if kwargs:+ logger.warning(+ f"unrecognized keys {list(kwargs.keys())} were passed. "+ "Please check them. If you use old api, just ignore this warning."+ )+ if "CUBLAS_WORKSPACE_CONFIG" not in os.environ:+ os.environ["CUBLAS_WORKSPACE_CONFIG"] = ":4096:8"+ # Deprecated, default not to use torch.use_deterministic_algorithms+ if not disable_deterministic_algorithms or enable_deterministic_algorithms:+ if not disable_deterministic_algorithms:+ logger.warning(+ "default not use deterministic_algorithms. disable_deterministic_algorithms is deprecated,"+ " please use enable_deterministic_algorithms instead. "+ )++ torch.use_deterministic_algorithms(True, warn_only=False)+ else:+ torch.use_deterministic_algorithms(True, warn_only=True)++ # XPU SDPA workaround: drop pure causal masks so FLASH backend is used,+ # and set torch.use_deterministic_algorithms(False)+ # instead of MATH (avoids ~10x peak-VRAM blow-up during block tuning).+ patch_xpu_sdpa_drop_causal_mask()++ device = kwargs.pop("device", None)+ if device is not None:+ logger.warning("`device` is deprecated, please use `device_map` instead")++ # Tuning hyperparameters+ self.seed = seed+ set_seed(self.seed)++ self.nblocks = nblocks++ self.enable_torch_compile = enable_torch_compile++ # Whether to pack the layer immediately after tuning+ # Managed via self.compress_context.is_immediate_packing / is_immediate_saving++ torch.set_printoptions(precision=3, sci_mode=True)++ if is_hpex_available():+ logger.info("habana_frameworks is available, import htcore explicitly.")+ import habana_frameworks.torch.core as htcore # pylint: disable=E0401++ # Reset both context singletons before creating fresh instances so that+ # consecutive AutoRound creations don't inherit stale config from earlier ones.+ CompressContext.reset_context()+ ModelContext.reset_context()++ # Resolve the device eagerly so ModelContext can be created before+ # CompressContext. Creating ModelContext first places the large model+ # allocation early in the heap, matching the OLD arch allocation order+ # and reducing C-heap fragmentation (which is amplified on HPU).+ _device = get_major_device(device_map if device_map is not None else 0)++ self.model_context = ModelContext(+ model,+ tokenizer=tokenizer,+ platform=platform,+ model_dtype=model_dtype,+ trust_remote_code=trust_remote_code,+ amp=amp,+ need_calib=self.need_calib,+ device=_device,+ formats=self.formats,+ is_act_quantize=self.quantize_config.is_act_quantize,+ quant_nontext_module=quant_nontext_module,+ )+ # Alternatively, you can use CompressContext.create_context+ self.compress_context = CompressContext(+ low_cpu_mem_usage,+ low_gpu_mem_usage,+ device_map,+ enable_torch_compile,+ formats=self.formats,+ static_kv_dtype=self.static_kv_dtype,+ static_attention_dtype=self.static_attention_dtype,+ )+ self.shard_writer = None++ # scale_dtype is resolved in quantizer.resolve_scheme() after scheme resolution,+ # so it is not initialized here to avoid premature evaluation with an unresolved scheme.++ # Flag for post_init idempotency. Set to False here so post_init() can be called+ # either via quantize_and_save() (preferred, outside inference_mode) or directly+ # from quantize() as a fallback for non-AutoScheme cases.+ self._post_init_done = False++ # Apply torch compile adjustments eagerly so that ar.enable_torch_compile+ # reflects the correct value immediately after construction (not only after post_init).+ self._adjust_torch_compile(enable_torch_compile)+ self.compress_context.enable_torch_compile = self.enable_torch_compile++ self.blocks_requiring_input_ids = []+ self.has_variable_block_shape = False+ fixed_attr = get_predefined_fixed_attr(self.model_context.model) or {}+ for key, value in fixed_attr.items():+ setattr(self, key, value)++ # ── Scheme resolution ─────────────────────────────────────────────────────++ def resolve_scheme(self, model_context=None, compress_context=None, dataset: str = None) -> None:+ """Phase-1 init: resolve scheme and bind config attrs (no model structure needed).++ Must be called BEFORE ``get_formats()`` and BEFORE ``_scheme_post_init()``.+ Idempotent: safe to call multiple times.+ """+ if self._scheme_resolved:+ return++ if model_context is not None:+ self.model_context = model_context+ if compress_context is not None:+ self.compress_context = compress_context+ if dataset is not None:+ self.dataset = dataset++ scheme_fields = {f.name for f in fields(QuantizationScheme)}+ user_scheme_overrides = {+ k: getattr(self.quantize_config, k)+ for k in scheme_fields+ if getattr(self.quantize_config, k, None) is not None+ }+ default_scheme, self.is_auto_scheme, final_attrs = _parse_scheme(self.scheme, user_scheme_overrides)++ for key, value in final_attrs.items():+ setattr(self.quantize_config, key, value)+ if hasattr(self, key):+ setattr(self, key, value)+ self.quantize_config.check_config()++ self.orig_scheme = copy.deepcopy(self.scheme)+ self.scheme = default_scheme++ gguf_scheme_name = get_gguf_scheme(self.scheme)+ if self.scale_dtype is None:+ self.scale_dtype = "fp32" if gguf_scheme_name else "fp16"+ self.scale_dtype = convert_dtype_str2torch(self.scale_dtype)++ self._scheme_resolved = True++ def _scheme_post_init(self) -> None:+ """Phase-4 init: build layer config on the patched model.++ Requires ``resolve_scheme()`` to have been called first.+ Must be called AFTER ``model_context.apply_patches()``.+ """+ assert self._scheme_resolved, (+ "resolve_scheme() must be called before _scheme_post_init(). "+ "BaseCompressor.post_init() does this automatically."+ )++ enable_gguf_official_mixed = not self.is_auto_scheme++ if self.quant_block_list is None:+ quant_nontext_module = getattr(self.model_context, "quant_nontext_module", False)+ all_blocks = get_block_names(self.model_context.model, quant_vision=quant_nontext_module)+ self.quant_block_list = find_matching_blocks(+ self.model_context.model, all_blocks, self.to_quant_block_names+ )+ if self.to_quant_block_names is None and self.quant_block_list:+ self.to_quant_block_names = extract_block_names_to_str(self.quant_block_list)+ self.quantize_config.to_quant_block_names = self.to_quant_block_names++ self.configure_layer_config(enable_gguf_official_mixed=enable_gguf_official_mixed)++ def _gen_auto_scheme(self) -> dict[str, dict]:+ """Generate per-layer config via AutoScheme delta-loss selection."""+ if self.model_context.is_mllm:+ # AutoScheme on a VLM only scores the language tower (the block+ # walker in delta_loss already skips vision/audio sub-trees) and+ # uses a pure-text calibration dataset by default, falling back to+ # the multimodal dataloader if the VLM rejects text-only forward.+ logger.info(+ "AutoScheme on multimodal LLM: scoring the language tower only "+ "with text-only calibration (multimodal dataloader will be used "+ "as a fallback if needed)."+ )++ if is_quantized_input_module(self.model_context.model):+ raise NotImplementedError("AutoScheme does not currently support quantized input models (e.g., FP8).")++ all_dtypes = []+ all_gguf = True+ for option in self.orig_scheme.options:+ dtype = "int"+ if isinstance(option, str):+ if not option.lower().startswith("gguf"):+ all_gguf = False+ option = preset_name_to_scheme(option)+ else:+ all_gguf = False++ if isinstance(option, QuantizationScheme):+ dtype = option.data_type+ elif isinstance(option, dict):+ dtype = option.get("data_type", "int")++ all_dtypes.append(dtype)++ unique_dtypes = set(all_dtypes)+ if len(unique_dtypes) > 1 and not all_gguf:+ logger.warning(+ "Models with mixed data_types "+ "cannot yet be exported to real formats except GGUF. "+ "Please save the model using the `fake` format for now."+ )++ layer_config, self.has_qlayer_outside_block, self.regex_config = set_layer_config(+ self.model_context.model,+ self.layer_config,+ self.scheme,+ self.scale_dtype,+ self.supported_types,+ self.inner_supported_types,+ self.quant_block_list,+ self.ignore_layers,+ self.quant_lm_head,+ enable_gguf_official_mixed=False,+ is_mllm=self.model_context.is_mllm,+ )+ quant_layer_names = layer_config.keys()++ # ---- VLM: peel non-text sub-trees AutoScheme should not score ---- #+ nontext_skipped_layers: dict[str, dict] = {}+ if self.model_context.is_mllm:+ from auto_round.utils import get_block_names++ quant_nontext = getattr(self, "quant_nontext_module", False)+ scoreable_blocks = get_block_names(self.model_context.model, quant_vision=quant_nontext)+ scoreable_block_prefixes = tuple(blk for group in scoreable_blocks for blk in group)+ if quant_nontext:+ peel_markers = ("audio", "speech", "wav", "waveform")+ tower_label = "language+vision"+ peel_label = "audio/speech"+ else:+ peel_markers = (+ "vision",+ "visual",+ "image",+ "img",+ "audio",+ "speech",+ "wav",+ "waveform",+ )+ tower_label = "language"+ peel_label = "vision/audio"++ def _is_scoreable_layer(name: str) -> bool:+ if any(name == p or name.startswith(p + ".") for p in scoreable_block_prefixes):+ return True+ lname = name.lower()+ return not any(marker in lname for marker in peel_markers)++ scoreable_layer_config = {}+ for name, cfg in layer_config.items():+ if _is_scoreable_layer(name):+ scoreable_layer_config[name] = cfg+ else:+ nontext_skipped_layers[name] = cfg++ if nontext_skipped_layers:+ logger.info(+ "AutoScheme (VLM): scoring %d %s-tower layers; "+ "%d %s-tower layers kept at their original 16-bit configuration.",+ len(scoreable_layer_config),+ tower_label,+ len(nontext_skipped_layers),+ peel_label,+ )+ layer_config = scoreable_layer_config+ quant_layer_names = layer_config.keys()++ scheme_keys = {f.name for f in fields(QuantizationScheme)}+ fixed_layer_scheme_new = {+ k: {key: v[key] for key in scheme_keys & v.keys()}+ for k, v in layer_config.items()+ if v.get("fixed_by_user", False)+ }++ from auto_round.auto_scheme.gen_auto_scheme import GenScheme++ if (+ not self.compress_context.enable_torch_compile+ and self.quantize_config.super_bits is None+ and not self.orig_scheme.low_gpu_mem_usage+ ):+ logger.warning("we strongly recommend to set `enable_torch_compile` to True for AutoScheme to save VRAM")+ self.scheme_generator = GenScheme(+ self.orig_scheme,+ self.model_context.model,+ quant_layer_names,+ fixed_layer_scheme_new,+ self.dataset,+ device_map=self.compress_context.device_map,+ tokenizer=self.model_context.tokenizer,+ enable_torch_compile=self.compress_context.enable_torch_compile,+ processor=self.model_context.processor,+ )+ layer_config = self.scheme_generator.get_layer_config()+ # Re-attach vision/audio-tower layers we peeled off earlier so the+ # downstream quantization pipeline sees the complete layer map.+ if nontext_skipped_layers:+ allowed_keys = {f.name for f in fields(QuantizationScheme)} | {+ "fixed_by_user",+ "scale_dtype",+ "scheme",+ }+ for name, cfg in nontext_skipped_layers.items():+ clean_cfg = {k: v for k, v in cfg.items() if k in allowed_keys} if isinstance(cfg, dict) else cfg+ layer_config.setdefault(name, clean_cfg)+ return layer_config++ def configure_layer_config(self, enable_gguf_official_mixed: bool | None = True) -> None:+ """Build ``self.layer_config`` from the resolved scheme on the patched model."""+ is_gguf_format = (f := getattr(self.compress_context, "formats", None)) is not None and "gguf" in f+ predefined_ignore_layers = get_predefined_ignore_layers(self.model_context.model)+ compressed_predefined_ignore_layers = compress_layer_names(predefined_ignore_layers)+ if not is_gguf_format:+ predefined_ignore_layers = get_predefined_ignore_layers(self.model_context.model)+ if predefined_ignore_layers:+ logger.info(f"Using predefined ignore_layers: {compressed_predefined_ignore_layers}")+ tmp_str = ",".join(predefined_ignore_layers)+ if self.ignore_layers == "":+ self.ignore_layers = tmp_str+ else:+ self.ignore_layers += "," + tmp_str++ if self.is_auto_scheme:+ self.layer_config = self._gen_auto_scheme()+ else:+ self.layer_config = _handle_special_schemes(+ self.orig_scheme,+ self.layer_config,+ self.model_context.model,+ supported_types=SUPPORTED_LAYER_TYPES,+ inner_supported_types=INNER_SUPPORTED_LAYER_TYPES,+ quant_lm_head=self.quant_lm_head,+ mllm=self.model_context.is_mllm,+ )+ _gguf_orig_fmt = getattr(self, "_gguf_original_format_name", None)+ if _gguf_orig_fmt and "_MIXED" in _gguf_orig_fmt.upper():+ self.layer_config = _handle_special_schemes(+ _gguf_orig_fmt.lower(),+ self.layer_config,+ self.model_context.model,+ supported_types=SUPPORTED_LAYER_TYPES,+ inner_supported_types=INNER_SUPPORTED_LAYER_TYPES,+ quant_lm_head=self.quant_lm_head,+ mllm=self.model_context.is_mllm,+ )++ fill_default_value = not self.is_auto_scheme+ self.layer_config, self.has_qlayer_outside_block, self.regex_config = set_layer_config(+ self.model_context.model,+ self.layer_config,+ self.scheme,+ self.scale_dtype,+ SUPPORTED_LAYER_TYPES,+ INNER_SUPPORTED_LAYER_TYPES,+ self.quant_block_list,+ self.ignore_layers,+ self.quant_lm_head,+ enable_gguf_official_mixed=enable_gguf_official_mixed,+ is_mllm=self.model_context.is_mllm,+ fill_default_value=fill_default_value,+ gguf_format_name=getattr(self, "_gguf_format_name", None),+ )++ # ─────────────────────────────────────────────────────────────────────────++ @property+ def mllm(self):+ return self.model_context.is_mllm++ @property+ def diffusion(self):+ return self.model_context.is_diffusion++ def _adjust_torch_compile(self, enable_torch_compile: bool) -> None:+ """Sets the torch compile configuration for the tuning."""+ self.enable_torch_compile = enable_torch_compile++ # Determine fp8 / nvfp4 intent from raw config before scheme resolution.+ cfg = self.quantize_config+ raw_scheme = self.scheme if isinstance(self.scheme, str) else ""+ raw_dt = (cfg.data_type or "").lower()+ raw_adt = (cfg.act_data_type or "").lower()+ raw_scheme_upper = raw_scheme.upper()++ is_raw_nv_fp = "nv_fp" in raw_dt or "nv_fp" in raw_adt or "NVFP" in raw_scheme_upper+ is_raw_fp8 = (+ "fp8" in raw_dt+ or "fp8" in raw_adt+ or "FP8" in raw_scheme_upper+ or ("fp" in raw_dt and getattr(cfg, "bits", 16) == 8)+ or ("fp" in raw_adt and getattr(cfg, "act_bits", 16) == 8)+ )++ act_bits = getattr(cfg, "act_bits", 16) or 16+ if (+ not self.enable_torch_compile+ and TORCH_VERSION_AT_LEAST_2_6+ and act_bits > 8+ and not is_debug_mode()+ and not is_raw_fp8+ and self.need_calib+ ):+ logger.info(+ "%s",+ "'enable_torch_compile' is set to `False` by default. "+ "Enabling it can reduce tuning cost by 20%, but it might throw an exception.",+ )+ # On HPU, we rely on torch.compile to speed up the model execution.+ if self.enable_torch_compile and is_raw_fp8 and not is_hpex_available():+ self.enable_torch_compile = False+ logger.warning("reset enable_torch_compile to `False` as fp8 is enabled")+ # TODO: fix https://github.com/intel/auto-round/issues/1109+ if self.enable_torch_compile and is_raw_nv_fp:+ self.enable_torch_compile = False+ logger.warning("reset enable_torch_compile to `False` as nvfp4 is enabled")+ super_group_size = getattr(cfg, "super_group_size", None)+ enable_alg_ext = getattr(cfg, "enable_alg_ext", False)+ if self.enable_torch_compile and super_group_size is not None and enable_alg_ext:+ self.enable_torch_compile = False+ logger.warning("reset enable_torch_compile to `False` as super_group_size is set for algorithm extension")++ def _get_calibration_dataset(self) -> str:+ """Resolve calibration dataset: self.dataset > AutoScheme.dataset > default."""+ dataset = self.__dict__.get("dataset", None)+ if dataset:+ return dataset+ from auto_round.auto_scheme.gen_auto_scheme import AutoScheme++ scheme = self.scheme+ if isinstance(scheme, AutoScheme) and scheme.dataset:+ return scheme.dataset+ return "NeelNanda/pile-10k"++ def post_init(self) -> None:+ """One-time initialization that requires a loaded model.++ Call this OUTSIDE any ``@torch.inference_mode()`` context when using+ AutoScheme – delta-loss selection needs autograd (backward pass).+ ``quantize_and_save()`` does this automatically before entering the+ inference-mode quantize loop.++ Delegates to five ordered pipeline phases; see each ``_resolve_scheme``,+ ``_resolve_formats``, ``_patch_model``, ``_build_layer_config``, and+ ``_hardware_setup`` for the precise preconditions and postconditions.+ """+ if self._post_init_done:+ return++ self._resolve_scheme()++ # After scheme resolution, is_act_quantize is known. When activation+ # quantization is enabled and the model is in float16, convert to+ # bfloat16 to match the old arch. This also detaches any parameter+ # tensors that are still backed by safetensors' mmap, preventing+ # per-block RSS growth (~14 MB/block) when .to(device) page-faults+ # the underlying file pages into physical memory.+ if self.quantize_config.is_act_quantize and self.model_context.amp_dtype == torch.float16:+ logger.warning("force to use bf16 for quantization tuning when enabling activation quantization")+ self.model_context.amp_dtype = torch.bfloat16+ if self.model_context.model.dtype != torch.bfloat16:+ self.model_context.model = self.model_context.model.to(torch.bfloat16)++ self._resolve_formats()+ self._patch_model()+ self._build_layer_config()++ # Reclaim temporaries from Phases 1-4 (scheme resolution, format+ # parsing, model patching, layer-config walk) before Phase 5+ # allocates hardware/compile objects. This compacts the heap so that+ # the fragmentation gap between live and freed blocks is minimised.+ gc.collect()+ _force_trim_malloc()++ self._hardware_setup()++ # Final trim after all init phases.+ gc.collect()+ _force_trim_malloc()++ self._post_init_done = True++ # ── Pipeline phase methods ────────────────────────────────────────────────++ def _resolve_scheme(self) -> None:+ """Phase 1 – Scheme resolution and quantizer construction.++ Preconditions:+ - ``self.quantize_config`` is a valid :class:`QuantizationConfig`.++ Work performed:+ - Seeds scheme-related attrs (``scale_dtype``, ``ignore_layers``,+ ``quant_lm_head``, ``to_quant_block_names``) from ``quantize_config``.+ - Calls :meth:`resolve_scheme` to derive ``data_type``, ``bits``,+ ``sym``, ``scale_dtype`` etc. and write them back to both ``self``+ and ``self.quantize_config``.+ - Constructs ``self.quantizer`` from the now-resolved config and wires+ it to the current model / context.+ - Binds ``self.wrapper_block`` for later use in quantizers.++ Postconditions:+ - ``self.scheme`` and ``self.quantize_config`` carry resolved scheme attrs.+ - ``self.quantizer`` is ready; calibration params (``seqlen``,+ ``nsamples``) are synced.+ """+ cfg = self.quantize_config+ self.scale_dtype = cfg.scale_dtype+ # self.layer_config is already set from __init__ (direct compressor param).+ self.ignore_layers = cfg.ignore_layers+ self.quant_lm_head = cfg.quant_lm_head+ self.to_quant_block_names = cfg.to_quant_block_names++ # Resolve the scheme (pure config work: sets data_type / bits / sym /+ # scale_dtype etc. on both self and self.quantize_config).+ self.resolve_scheme(+ model_context=self.model_context,+ compress_context=self.compress_context,+ dataset=self._get_calibration_dataset(),+ )++ # Create the quantizer now that the config holds resolved values.+ self.quantizer = BaseQuantizers.from_config(self.quantize_config)+ self.quantizer.model_context = self.model_context+ self.quantizer.compress_context = self.compress_context+ self.quantizer.model = self.model_context.model+ self.quantizer.scale_dtype = self.scale_dtype+ # Sync compressor-owned calibration params to quantizer.+ self.quantizer.seqlen = self.seqlen+ self.quantizer.nsamples = self.nsamples+ self.wrapper_block = wrapper_block++ def _resolve_formats(self) -> None:+ """Phase 2 – Format resolution, GGUF attr sync, and rotation application.++ Preconditions:+ - Phase 1 complete: ``self.quantizer`` is initialised and the scheme+ is resolved (``data_type``, ``bits``, ``sym`` etc. are final).++ Work performed:+ - Converts a string ``self.formats`` to a list of+ :class:`~auto_round.formats.OutputFormat` objects via+ :func:`~auto_round.formats.get_formats`.+ - Initialises :class:`~auto_round.compressors_new.shard_writer.ShardWriter`+ when formats are present.+ - **(2b)** Detects GGUF-driven attribute mutations (``bits``, ``sym``,+ ``data_type``, ``group_size``, etc.) that ``gguf_args_check`` may+ have written onto ``self`` inside ``get_formats``, syncs them to+ ``self.quantizer``, and rebuilds ``self.scheme`` accordingly.+ - Merges any GGUF-injected entries into ``self.layer_config``.+ - **(2d)** Applies rotation transforms from ``self.transform_configs``.++ Postconditions:+ - ``self.formats`` is a list (or ``None``).+ - ``self.compress_context.formats`` mirrors ``self.formats``.+ - ``self.quantizer`` carries the GGUF-adjusted scheme attrs.+ - ``self.scheme`` is consistent with the final quantization attrs.+ - All rotation transforms have been applied to ``self.model_context.model``.+ """+ # get_formats() inspects data_type / bits etc. that were just resolved.+ if isinstance(self.formats, str):+ self.formats = get_formats(self.formats, self)+ if self.formats is not None:+ self.compress_context.formats = self.formats+ ShardWriter.reset()+ # Defer ShardWriter construction to _ensure_shard_writer() to avoid+ # heap fragmentation during post_init (parameter iteration).++ # Snapshot the user-specified layer_config before GGUF processing may+ # add extra entries, so we can distinguish them later in step 2b.+ _pre_gguf_layer_config = copy.copy(self.layer_config) or {}++ # ── 2b: propagate GGUF-adjusted attrs back to quantizer ──────────────+ # gguf_args_check (called inside get_formats) may have overridden+ # bits / sym / data_type / super_bits / super_group_size / group_size+ # on *this* BaseCompressor object. The quantizer stored its own copies+ # from Phase 1 (resolve_scheme), so we must sync them now, before+ # _scheme_post_init() builds the layer_config in Phase 4.+ _gguf_forwarded_attrs = (+ "bits",+ "sym",+ "data_type",+ "super_bits",+ "super_group_size",+ "group_size",+ "act_bits",+ "scale_dtype",+ )+ _any_gguf_attr_changed = False+ for _attr in _gguf_forwarded_attrs:+ if _attr in self.__dict__ and hasattr(self.quantizer, _attr):+ if _attr not in ("scale_dtype", "act_bits") and getattr(self.quantizer, _attr) != self.__dict__[_attr]:+ _any_gguf_attr_changed = True+ setattr(self.quantizer, _attr, self.__dict__[_attr])+ # If gguf_args_check changed scheme attrs, rebuild the scheme on both+ # the compressor (SchemeMixin) and the quantizer so that+ # configure_layer_config() and set_layer_config() use the correct+ # default_dict and gguf_name.+ if _any_gguf_attr_changed:+ from auto_round.schemes import PRESET_SCHEMES+ from auto_round.schemes import QuantizationScheme as _QS++ # Prefer to derive the scheme directly from the gguf format name to+ # avoid ambiguity (e.g. Q4_K_S and Q4_K_M share identical weight attrs).+ _gguf_preset_scheme = None+ _gguf_fmt_name = None+ _gguf_original_fmt_name = None+ for _fmt in self.formats or []:+ # GGUFFormat (outer) has output_format="gguf" but backend.output_format="gguf:q4_k_m"+ # GGUFFormat (inner/standalone) has output_format="gguf:q4_k_m"+ _of = getattr(_fmt, "output_format", "")+ if "gguf" in str(_of):+ if str(_of) == "gguf":+ # outer GGUFFormat: full format in _original_format (e.g. "gguf:q2_k_mixed")+ # or backend.output_format (e.g. "gguf:q2_k_s" after _mixed → _s conversion)+ _orig = getattr(_fmt, "_original_format", None)+ if _orig:+ _gguf_original_fmt_name = str(_orig).upper()+ _backend = getattr(_fmt, "backend", None)+ _of = getattr(_backend, "output_format", _of) if _backend is not None else _of+ _preset_key = str(_of).upper()+ if _preset_key in PRESET_SCHEMES:+ _gguf_preset_scheme = PRESET_SCHEMES[_preset_key]+ _gguf_fmt_name = _preset_key+ break+ if _gguf_preset_scheme is not None:+ # Update scheme on both compressor and quantizer.+ self.scheme = _gguf_preset_scheme+ # Store the exact gguf format name so configure_layer_config /+ # set_layer_config can use it directly, avoiding Q4_K_S / Q4_K_M ambiguity.+ self._gguf_format_name = _gguf_fmt_name+ # Store original format name (may include _mixed) for _handle_special_schemes+ if _gguf_original_fmt_name:+ self._gguf_original_format_name = _gguf_original_fmt_name+ else:+ _new_scheme_dict = {f.name: getattr(self, f.name, None) for f in fields(_QS)}+ _new_scheme = _QS.from_dict({k: v for k, v in _new_scheme_dict.items() if v is not None})+ self.scheme = _new_scheme++ _gguf_layer_cfg = {+ k: v for k, v in (self.__dict__.get("layer_config") or {}).items() if k not in (_pre_gguf_layer_config)+ }+ if _gguf_layer_cfg:+ if self.layer_config is None:+ self.layer_config = {}+ for _lname, _lval in _gguf_layer_cfg.items():+ self.layer_config.setdefault(_lname, _lval)++ # ── 2d: apply rotation transforms ────────────────────────────────────+ if self.transform_configs:+ logger.info("Applying Hadamard transform to the model.")+ for rotation_cfg in self.transform_configs:+ self.model_context.model = apply_rotation(+ self.model_context.model,+ rotation_cfg,+ data_type=self.quantize_config.data_type,+ )++ def _patch_model(self) -> None:+ """Phase 3 – Model structure patching.++ Preconditions:+ - Phase 2 complete: ``self.formats`` is resolved so that+ ``apply_patches`` can inspect format-specific requirements.++ Work performed:+ - Delegates to :meth:`~auto_round.context.model.ModelContext.apply_patches`+ which may replace or merge layers (e.g. MoE expert merging, adding+ static-kv wrappers) to produce the final model topology.++ Postconditions:+ - ``self.model_context.model`` reflects the definitive topology that+ :meth:`_build_layer_config` will walk.+ """+ # apply_patches() may replace layers (e.g. MoE expert merging); must+ # happen before configure_layer_config() so it sees the final topology.+ self.model_context.apply_patches(self.formats)++ def _build_layer_config(self) -> None:+ """Phase 4 – Layer-config construction and quantizer sync.++ Preconditions:+ - Phase 3 complete: model topology is final.+ - ``self.scheme`` and all scheme-resolved attrs are consistent with+ the (possibly GGUF-adjusted) values set in Phase 2.++ Work performed:+ - Calls :meth:`_scheme_post_init` which walks the patched model to+ build ``self.layer_config``, ``self.quant_block_list``, etc.+ On the AutoScheme path this also runs delta-loss forward/backward+ passes to select per-layer schemes.+ - Syncs the fully-resolved ``layer_config`` and related attrs to+ ``self.quantizer`` so quantization methods have the complete view.++ Postconditions:+ - ``self.layer_config`` is fully populated.+ - ``self.quantizer`` mirrors ``layer_config``, ``has_qlayer_outside_block``,+ ``regex_config``, ``quant_block_list``, ``to_quant_block_names``,+ ``scale_dtype``, and ``ignore_layers``.+ """+ # configure_layer_config() walks the patched model; _gen_auto_scheme()+ # (AutoScheme path) runs delta-loss forward+backward passes.+ self._scheme_post_init()++ # Sync the fully-resolved scheme state to the quantizer so that+ # quantization methods (quantize_block, quantize_layer, etc.) have+ # access to layer_config, scale_dtype, quant_block_list, etc.+ self.quantizer.layer_config = self.layer_config+ self.quantizer.has_qlayer_outside_block = self.has_qlayer_outside_block+ self.quantizer.regex_config = self.regex_config+ self.quantizer.quant_block_list = self.quant_block_list+ self.quantizer.to_quant_block_names = self.to_quant_block_names+ self.quantizer.scale_dtype = self.scale_dtype+ self.quantizer.ignore_layers = self.ignore_layers++ def _hardware_setup(self) -> None:+ """Phase 5 – Hardware and compile configuration.++ Preconditions:+ - Phase 4 complete: ``layer_config`` is built and+ ``has_qlayer_outside_block`` is known.+ - ``self.quantize_config.data_type`` is the final resolved value+ (needed by :meth:`_adjust_torch_compile`).++ Work performed:+ - Applies the device map via :func:`~auto_round.utils.device.set_non_auto_device_map`.+ - Re-evaluates ``torch.compile`` eligibility now that ``data_type`` is+ resolved and writes the result back to ``compress_context``.+ - Selects ``self.block_forward`` (compiled or plain).+ - Resets the offload manager when ``low_cpu_mem_usage`` is active.+ - Disables ``self.inplace`` when quantized layers live outside+ transformer blocks (incompatible with in-place rewriting).+ - Calls :meth:`_adjust_immediate_packing_and_saving` to decide whether+ layers should be packed / written immediately after each block.++ Postconditions:+ - ``self.block_forward`` is ready for use.+ - ``compress_context.enable_torch_compile`` is final.+ - ``self.inplace`` and ``compress_context.is_immediate_packing`` /+ ``compress_context.is_immediate_saving`` are set to their definitive values.+ """+ set_non_auto_device_map(self.model_context.model, self.compress_context.device_map)+ # Re-evaluate torch.compile eligibility now that data_type is resolved.+ self._adjust_torch_compile(self.enable_torch_compile)+ self.compress_context.enable_torch_compile = self.enable_torch_compile+ # Apply the same act-quantization / alg-ext guard as+ # _resolve_block_forward() so we never compile when hooks are present.+ cfg = self.quantize_config+ _needs_plain_forward = (cfg.is_act_quantize and (not cfg.act_dynamic or cfg.is_act_nv_fp)) or getattr(+ cfg, "enable_alg_ext", False+ )+ # Only compile block_forward when it will actually be used (calibration path).+ # For zero-shot compressors (need_calib=False), block_forward is never called,+ # so skipping compilation avoids unnecessary HPU workspace allocation.+ if self.enable_torch_compile and not _needs_plain_forward and self.need_calib:+ self.block_forward = compile_func(block_forward, self.compress_context.device)+ else:+ self.block_forward = block_forward+ if self.compress_context.low_cpu_mem_usage:+ self._offloader.reset()++ # Disable inplace when quantized layers live outside transformer blocks.+ if self.has_qlayer_outside_block and self.need_calib:+ self.inplace = False++ if not hasattr(self, "formats"):+ logger.warning("this API is deprecated, please use `quantize_and_save` instead")+ else:+ self._adjust_immediate_packing_and_saving()++ # backward compatible with the legacy API+ def __getattr__(self, name: str) -> Any:+ if name in self.__dict__:+ return self.__dict__[name]++ for obj in ["quantizer", "quantize_config", "model_context", "compress_context"]:+ if obj not in self.__dict__:+ continue+ obj = object.__getattribute__(self, obj)+ try:+ return object.__getattribute__(obj, name)+ except AttributeError:+ continue++ raise AttributeError(f"'{type(self).__name__}' object has no attribute '{name}'")++ @property+ def optimizer(self):+ """Return the actual optimizer class, converting string to class for backward compat.++ Old API stored ``self.optimizer = torch.optim.AdamW`` (the class itself).+ New arch stores the optimizer name as a string in ``quantize_config.optimizer``.+ This property converts it so that ``ar.optimizer == torch.optim.AdamW`` works.+ """+ if self.quantize_config is None:+ return None+ opt = getattr(self.quantize_config, "optimizer", None)+ if opt is None:+ # Default to AdamW when enable_adam=True and no explicit optimizer was set+ if getattr(self.quantize_config, "enable_adam", False):+ return torch.optim.AdamW+ return None+ if isinstance(opt, str):+ return getattr(torch.optim, opt, None)+ return opt++ def _adjust_immediate_packing_and_saving(self):+ from auto_round.algorithms.quantization.rtn.config import RTNConfig++ if self.formats is None:+ return++ formats = getattr(self, "formats", [])+ if len(formats) == 1 and not formats[0].is_fake() and self.inplace:+ self.compress_context.is_immediate_packing = True++ if self.has_qlayer_outside_block and self.need_calib:+ self.compress_context.is_immediate_packing = False++ if not ("causallm" in self.model_context.model.__class__.__name__.lower() and not self.model_context.is_mllm):+ # TODO For tied keys, there may some issues, we haven't not verified this+ tied_weight_keys = getattr(self.model_context.model, "_tied_weight_keys", {})+ if len(tied_weight_keys) > 1:+ self.compress_context.is_immediate_saving = False+ if self.compress_context.low_cpu_mem_usage:+ logger.warning("reset low_cpu_mem_usage to False due to tied weights")+ return+ if len(tied_weight_keys) == 1:+ key = list(tied_weight_keys.keys())[0]+ if "lm_head" not in key:+ self.compress_context.is_immediate_saving = False+ if self.compress_context.low_cpu_mem_usage:+ logger.warning("reset low_cpu_mem_usage to False due to tied weights")+ return++ if self.compress_context.low_cpu_mem_usage and self.compress_context.is_immediate_packing:+ self.compress_context.is_immediate_saving = True++ if self.compress_context.low_cpu_mem_usage and self.compress_context.is_immediate_packing:+ if formats[0].is_gguf():+ logger.warning(+ "`low_cpu_mem_usage` is not fully supported for gguf format. "+ "Setting `low_cpu_mem_usage` to False."+ )+ self.compress_context.low_cpu_mem_usage = False+ self.compress_context.is_immediate_saving = False+ elif (+ self.has_qlayer_outside_block+ and getattr(self, "disable_opt_rtn", None)+ and isinstance(self.quantize_config, RTNConfig)+ ):+ logger.info(+ "Keeping `low_cpu_mem_usage` enabled in RTN mode (iters=0): "+ "RTN path uses blockwise quantization and supports per-block offloading."+ )+ elif self.has_qlayer_outside_block and not isinstance(self.quantize_config, RTNConfig):+ logger.warning(+ "`low_cpu_mem_usage` is not fully supported "+ "when there are quantized layers outside blocks and optimized RTN is disabled. "+ "Setting low_cpu_mem_usage to False."+ )+ self.compress_context.low_cpu_mem_usage = False+ self.compress_context.is_immediate_saving = False++ if self.compress_context.is_immediate_saving and not (+ "int" in self.quantize_config.data_type+ or is_nv_fp(self.quantize_config.data_type)+ or is_mx_fp(self.quantize_config.data_type)+ ):+ logger.warning("immediate_saving is only supported for int/nv_fp/mx_fp quantization, set to False")+ self.compress_context.is_immediate_saving = False++ if self.output_dir is None:+ self.compress_context.is_immediate_saving = False++ # Create ShardWriter eagerly only when immediate saving is active+ # (it interleaves with the quantize loop). Otherwise keep it deferred+ # until save_quantized() to avoid heap fragmentation during init.+ if self.compress_context.is_immediate_saving:+ self._ensure_shard_writer()++ def _ensure_shard_writer(self):+ """Lazily create ShardWriter if it hasn't been created yet."""+ if self.shard_writer is None and self.formats is not None:+ self.shard_writer = ShardWriter(self.model_context.model, bits=8)++ def quantize(self) -> tuple[torch.nn.Module, dict[str, Any]]:+ """Quantize the model and return the quantized model along with layer configurations.The entry of AutoRound.+ Returns:+ The quantized model and layer configurations.+ """+ raise NotImplementedError("quantize method must be implemented in subclass")++ def save_quantized(+ self,+ output_dir: str = None,+ format: Union[str, list[OutputFormat]] = None,+ inplace: bool = True,+ return_folders=False,+ **kwargs,+ ) -> torch.nn.Module:+ """Save the quantized model to the specified output directory in the specified format.++ Args:+ output_dir (str, optional): The directory to save the quantized model. Defaults to None.+ format (str, optional): The format in which to save the model. Defaults to "auto_round".+ inplace (bool, optional): Whether to modify the model in place. Defaults to True.+ **kwargs: Additional keyword arguments specific to the export format.++ Returns:+ object: The compressed model object.+ """+ self.output_dir = output_dir+ if output_dir is not None:+ self.compress_context.output_dir = output_dir+ if format is not None:+ if isinstance(format, str) and getattr(self, "formats", None) is None:+ logger.warning(+ f"save_quantized with format is deprecated and will be deleted in auto_round version 1.0."+ f" Please use Compressor(format='{format}' instead)."+ )+ self.formats = get_formats(format, self)+ self.compress_context.formats = self.formats++ if not self.model_context.quantized:+ logger.warning("please run autoround.quantize first")+ return+ folders = []+ if self.formats is None:+ logger.info("format is not set, using default auto_round format.")+ self.formats = "auto_round"+ if isinstance(self.formats, str):+ self.formats = get_formats(self.formats, self)+ self.compress_context.formats = self.formats+ for format in self.formats:+ save_folder = _get_save_folder_name(format)+ if self.act_bits <= 8 and format.is_fake():+ logger.warning(+ "Support for exporting activation quantization is limited. "+ "Please ensure that your configuration is supported."+ )++ serialization_dict = asdict(SerializedCompressorConfig())+ for key in serialization_dict:+ serialization_dict[key] = getattr(self, key, serialization_dict[key])+ from auto_round.version import __version__++ serialization_dict["autoround_version"] = __version__+ if serialization_dict.get("to_quant_block_names") is None and self.quantizer.quant_block_list:+ serialization_dict["to_quant_block_names"] = extract_block_names_to_str(self.quantizer.quant_block_list)+ if "scale_dtype" in serialization_dict.keys():+ serialization_dict["scale_dtype"] = str(serialization_dict["scale_dtype"])++ compressed_model = format.save_quantized(+ save_folder,+ model=self.model_context.model,+ layer_config=self.quantizer.layer_config,+ inplace=inplace,+ tokenizer=self.model_context.tokenizer,+ device=self.compress_context.device,+ serialization_dict=serialization_dict,+ **kwargs,+ )+ folders.append(save_folder)++ if return_folders:+ if len(folders) == 1:+ folders = folders[0]+ return compressed_model, folders+ else:+ return compressed_model++ def _get_export_dir(self, output_dir: str, format_str: str) -> str:+ """Derive a descriptive export directory from model name and quantization config.++ Must be called after ``post_init()`` so that scheme-resolved attrs+ (bits, group_size, data_type, etc.) are available on ``self.quantize_config``.++ Mirrors the logic previously in ``__main__.py`` so callers only need to+ pass the base ``output_dir`` and the format string.+ """+ # Diffusion models use save_quantized from DiffusionMixin which manages its own+ # directory layout (model_index.json + per-component subdirs). Appending a+ # scheme-derived suffix here would place files one level too deep.+ if getattr(self, "diffusion", False):+ return output_dir++ model_name = (getattr(self.model_context.model, "name_or_path", "") or "").rstrip("/")+ cfg = self.quantize_config+ group_size = cfg.group_size+ bits = cfg.bits+ data_type = cfg.data_type or "int"+ act_bits = cfg.act_bits or 16+ act_data_type = cfg.act_data_type or "float"++ is_gguf = "gguf" in (format_str or "")+ last = model_name.split("/")[-1].strip(".")++ if last == "" and not is_gguf:+ # model path is just '.' or './' – put inside output_dir with suffix+ if group_size <= 0:+ suffix = f"afp{act_bits}" if "fp" in act_data_type else f"a{act_bits}"+ else:+ suffix = f"g{group_size}"+ return os.path.join(output_dir, f"w{bits}{suffix}")++ if last == "" and is_gguf:+ return output_dir++ if is_gguf:+ return os.path.join(output_dir, model_name.split("/")[-1] + "-gguf")++ # Normal case: derive suffix from group_size / act config+ if isinstance(group_size, tuple):+ assert len(group_size) == 2, f"Only support 2D group_size, but got {group_size}"+ suffix = f"g{group_size[0]}x{group_size[1]}"+ elif group_size <= 0:+ suffix = f"afp{act_bits}" if "fp" in act_data_type else f"a{act_bits}"+ else:+ suffix = f"g{group_size}"++ prefix = data_type.lower().replace("_", "") if "int" not in data_type or "mx" in data_type else ""+ return os.path.join(+ output_dir,+ model_name.split("/")[-1] + (f"-{prefix}" if prefix else "") + f"-w{bits}{suffix}",+ )++ def quantize_and_save(+ self, output_dir: str = "tmp_autoround", format: str = None, inplace: bool = True, **kwargs+ ) -> tuple[torch.nn.Module, dict[str, Any]]:+ """Quantizes the model and saves it in the specified format(s).++ This function checks the validity of the requested format(s), quantizes+ the model accordingly, and saves it to the specified output directory.+ If multiple formats are provided, the model is saved separately for each format.++ Args:+ output_dir (str, optional): The directory where the quantized model+ will be saved. Defaults to "tmp_autoround".+ format (str, optional): The quantization format(s) to use, separated+ by commas if multiple. Defaults to "auto_round".+ inplace (bool, optional): Whether to modify the model in place if only+ one format is used. Defaults to True.+ **kwargs: Additional arguments for the quantization and saving process.++ Returns:+ model: A qdq model or packed model based on the configurations+ folders: The folder paths where the quantized models are saved.++ Raises:+ ValueError: If an unsupported format is specified.+ """+ # Validate and process the specified formats+ self.output_dir = output_dir+ self.compress_context.output_dir = output_dir++ # check and update the format based on the current configuration+ if format and self.formats is None:+ logger.warning(+ f"quantize_and_save with format is deprecated and will be deleted in auto_round version 1.0."+ f" Please use Compressor(format='{format}' instead)."+ )+ self.formats = format+ if self.formats is None:+ logger.info("format is not set, using default auto_round format.")+ self.formats = "auto_round"++ # If multiple formats are specified, enforce inplace=False+ if len(self.formats.split(",")) > 1:+ inplace = False+ self.inplace = kwargs.get("inplace", inplace)+ kwargs.pop("inplace", None)++ # Perform model quantization+ # IMPORTANT: post_init() must run outside any @torch.inference_mode() context+ # because AutoScheme's delta-loss selection requires gradient tracking.+ self.post_init()+ # If post_init() was called manually before quantize_and_save() (e.g. ar.post_init()+ # in tests), _resolve_formats saw formats=None and was a no-op. Now that we have set+ # self.formats to a default string above, resolve it into OutputFormat objects so that+ # quantize() and save_quantized() receive proper objects, not a raw string.+ if isinstance(self.formats, str):+ self.formats = get_formats(self.formats, self)+ self.compress_context.formats = self.formats+ # Derive descriptive export dir after post_init so scheme-resolved attrs are available.+ _fmt_str = format or (self.formats if isinstance(self.formats, str) else "")+ output_dir = self._get_export_dir(output_dir, _fmt_str)+ self.output_dir = output_dir+ self.compress_context.output_dir = output_dir+ if self.static_attention_dtype is not None:+ from auto_round.experimental.attention import attention_quant_ctx++ with attention_quant_ctx(self.model_context.model, static_attention_dtype=self.static_attention_dtype):+ self.quantize()+ self.model_context.quantized = True+ elif self.static_kv_dtype is not None:+ from auto_round.experimental.kv_cache import kvcache_quant_context++ with kvcache_quant_context(self.model_context.model, static_kv_dtype=self.static_kv_dtype):+ self.quantize()+ self.model_context.quantized = True+ else:+ self.quantize()+ self.model_context.quantized = True++ # Ensure ShardWriter is ready before saving (deferred from post_init).+ self._ensure_shard_writer()++ # Save the quantized model in the specified format_list+ model, folders = self.save_quantized(output_dir, inplace=inplace, return_folders=True, **kwargs)+ memory_monitor.log_summary()++ return model, foldersdiff --git a/auto_round/compressors_new/calib.py b/auto_round/compressors_new/calib.pynew file mode 100644index 000000000..dea8233df--- /dev/null+++ b/auto_round/compressors_new/calib.py@@ -0,0 +1,1684 @@+# Copyright (c) 2026 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+import copy+import gc+import time+import traceback+from functools import partial+from typing import Any, Callable, Optional, Union++import accelerate+import torch+from accelerate.big_modeling import dispatch_model, infer_auto_device_map+from accelerate.utils import get_balanced_memory, get_max_memory+from tqdm import tqdm++from auto_round import envs+from auto_round.algorithms.alg_config import AlgConfig+from auto_round.calibration.utils import (+ _infer_last_cache_name,+ _update_inputs,+)+from auto_round.compressors_new.base import BaseCompressor+from auto_round.compressors_new.utils import (+ _get_quantized_layer_names_outside_blocks,+ check_skippable_keywords,+ immediate_pack,+ init_cache,+ is_nv_fp,+ is_static_wfp8afp8,+ reset_params,+)+from auto_round.logger import logger+from auto_round.modeling.fused_moe.replace_modules import materialize_model_, safe_to_cpu_+from auto_round.utils import (+ SUPPORTED_LAYER_TYPES,+ check_seqlen_compatible,+ check_to_quantized,+ clear_memory,+ compress_layer_names,+ convert_module_to_hp_if_necessary,+ flatten_list,+ get_block_names,+ get_module,+ hook_ngram_embeddings_on_cpu,+ is_auto_device_mapping,+ is_quantized_input_module,+ memory_monitor,+ mv_module_from_gpu,+ set_amax_for_all_moe_layers,+ set_module,+ to_device,+ to_dtype,+ wrap_block_forward_positional_to_kwargs,+)+from auto_round.utils.device import (+ _force_trim_malloc,+ parse_available_devices,+)+from auto_round.wrapper import WrapperLinear, WrapperMultiblock+++class CalibCompressor(BaseCompressor):+ need_calib: bool = True++ def __init__(+ self,+ config: Union[AlgConfig, list[AlgConfig]],+ model: Union[torch.nn.Module, str],+ tokenizer=None,+ platform="hf",+ format=None,+ dataset: Union[str, list, tuple, torch.utils.data.DataLoader] = "NeelNanda/pile-10k",+ iters: int = 200,+ low_gpu_mem_usage: bool = False,+ device_map: Union[str, torch.device, int, dict] = 0,+ enable_torch_compile: bool = False,+ seed: int = 42,+ low_cpu_mem_usage: bool = True,+ **kwargs,+ ):+ self.dataset = dataset+ self.iters = iters+ super().__init__(+ config=config,+ model=model,+ tokenizer=tokenizer,+ platform=platform,+ format=format,+ low_gpu_mem_usage=low_gpu_mem_usage,+ device_map=device_map,+ enable_torch_compile=enable_torch_compile,+ seed=seed,+ low_cpu_mem_usage=low_cpu_mem_usage,+ **kwargs,+ )+ if iters == 0:+ self.lr = 5e-3++ @torch.no_grad()+ def try_cache_inter_data_gpucpu(self, block_names, nsamples, layer_names=None, last_cache_name=None):+ """Attempts to cache intermediate data on GPU, if failed, then using CPU.++ Args:+ block_names (list): List of block names to cache data for.+ nsamples (int): Number of samples to use for caching.+ layer_names (list, optional): List of layer names to cache data for. Defaults to [].+ last_cache_name (str, optional): Name of the last cache. Defaults to None.++ Returns:+ all_inputs: Cached intermediate data.++ Raises:+ Exception: If caching on GPU fails, switches to CPU and caches there.+ """+ if is_quantized_input_module(self.model_context.model):+ layer_names = []+ if layer_names is None:+ layer_names = []++ block_names = flatten_list(block_names)+ self.blocks_requiring_input_ids = [data if isinstance(data, str) else data[0] for data in block_names]++ calibrate_on_cpu = False+ cannot_calibrate_on_cpu = False+ if self.compress_context.low_gpu_mem_usage or (+ len(block_names) == 1+ and len(layer_names) == 0+ and not self.quantizer.has_qlayer_outside_block+ and (last_cache_name is None or last_cache_name in block_names)+ and not getattr(self, "mllm", False)+ ):+ # low_gpu_mem_usage or calibrate only the embedding layer, which is also very fast on CPU+ calibrate_on_cpu = True+ try:+ all_inputs = self.cache_inter_data(+ block_names, nsamples, layer_names=[], last_cache_name=last_cache_name+ )+ except NotImplementedError as error:+ error_msg = str(error)+ if "flash_attn::" in error_msg and "CPU" in error_msg:+ cannot_calibrate_on_cpu = True # fallback to GPU when flash attention is not supported on CPU+ else:+ raise error++ if not calibrate_on_cpu or cannot_calibrate_on_cpu:+ try:+ if any(p.device.type == "meta" for p in self.model_context.model.parameters()):+ materialize_model_(self.model_context.model)++ if (+ hasattr(self.model_context.model, "hf_device_map")+ and len(self.model_context.model.hf_device_map) > 1+ ):+ self.model_context.model = dispatch_model(+ self.model_context.model, device_map=self.model_context.model.hf_device_map+ )+ else:+ # Change this if new device is supported+ if str(self.model_context.model.device) == "cpu" and (+ not self.compress_context.device.startswith("hpu")+ ):+ # type(self.model_context.model._no_split_modules) changes from list to set+ # when transformers > 5.0+ no_split_modules = list(getattr(self.model_context.model, "_no_split_modules", []))+ devices = parse_available_devices(self.compress_context.device_map)++ max_memory = get_max_memory()+ new_max_memory = {}+ if "cpu" not in devices:+ devices.append("cpu")+ for device in devices:+ if ":" in device:+ device = int(device.split(":")[-1])+ elif device == "cpu":+ device = "cpu"+ elif isinstance(device, str):+ device = 0+ else:+ raise ValueError(+ f"Unsupported device {device} in device_map: {self.compress_context.device_map}"+ )+ if device not in max_memory:+ # Skip devices that aee not reported by accelerate's max_memory.+ # This is expected when a device is unavailable or cannot provide memory info.+ continue+ # Use 90% of the reported max memory to leave headroom for activations,+ # temporary tensors, other processes, and allocator fragmentation, reducing+ # the chance of runtime OOM while still utilizing most available memory.+ new_max_memory[device] = max_memory[device] * 0.9++ # If non-CPU devices were requested but none survived, fall back to CPU caching+ # via the OOM handler below, avoiding unnecessary dispatch overhead.+ requested_non_cpu = any((d != "cpu") for d in devices)+ has_non_cpu_memory = any((k != "cpu") for k in new_max_memory)+ if requested_non_cpu and not has_non_cpu_memory:+ raise torch.OutOfMemoryError(+ "No non-CPU device available in accelerate's reported memory. "+ "Falling back to CPU caching."+ )++ # Keep ngram_embeddings on CPU+ has_ngram_embeddings, raw_ngram_embeddings = hook_ngram_embeddings_on_cpu(+ self.model_context.model+ )+ new_max_memory = get_balanced_memory(+ self.model_context.model,+ max_memory=new_max_memory,+ no_split_module_classes=no_split_modules,+ )+ if hasattr(self.model_context.model, "tie_weights"):+ self.model_context.model.tie_weights()+ device_map = infer_auto_device_map(+ self.model_context.model,+ max_memory=new_max_memory,+ no_split_module_classes=no_split_modules,+ )+ if len(devices) > 1 and "cpu" in device_map.values():+ logger.warning(+ "Some layers are offloaded to cpu, which may severely impact calibration speed."+ " Please consider using more cards."+ )++ try:++ self.model_context.model = dispatch_model(self.model_context.model, device_map=device_map)+ if has_ngram_embeddings:+ self.model_context.model.model.ngram_embeddings = raw_ngram_embeddings+ except ValueError as e:+ if "offload_dir" in e.__str__():+ logger.warning(+ f"Due to insufficient resources, disk is used to store the model."+ f" `offload_dir={envs.AR_WORK_SPACE}`"+ )+ self.model_context.model = dispatch_model(+ self.model_context.model, device_map=device_map, offload_dir=envs.AR_WORK_SPACE+ )+ else:+ raise+ else:++ self.model_context.model = self.model_context.model.to(self.compress_context.device)++ all_inputs = self.cache_inter_data(+ block_names, nsamples, layer_names=layer_names, last_cache_name=last_cache_name+ )+ if (+ hasattr(self.model_context.model, "hf_device_map")+ and len(self.model_context.model.hf_device_map) > 1+ ):+ accelerate.hooks.remove_hook_from_submodules(self.model_context.model)++ except torch.OutOfMemoryError as e:+ if cannot_calibrate_on_cpu:+ raise e+ cuda_error_msg = traceback.format_exc()+ try:+ logger.info("switch to cpu to cache block inputs")+ self.compress_context.cache_device = torch.device("cpu")+ if self.quantizer.has_qlayer_outside_block or self.__class__.__name__ == "AutoRoundMLLM":+ logger.warning(+ "we recommend using more GPUs in calibration."+ " Otherwise, some layers may fall back to `rtn` mode, which can affect accuracy."+ )+ accelerate.hooks.remove_hook_from_submodules(self.model_context.model)+ self.model_context.model = mv_module_from_gpu(self.model_context.model)+ clear_memory(device_list=self.compress_context.device_list)+ # Important change after v0.51, on cpu, we use rtn mode for layers in layer_names+ all_inputs = self.cache_inter_data(+ block_names, nsamples, layer_names=[], last_cache_name=last_cache_name+ )+ except Exception as e:+ logger.error(cuda_error_msg)+ raise+ return all_inputs++ @torch.no_grad()+ def cache_inter_data(self, block_names, nsamples, layer_names=None, last_cache_name=None):+ """Save the inputs of block_name for calibration.++ This method temporarily replaces the forward method of the model to capture+ the inputs passing through the specified block. It then calibrates the model+ using a specified number of samples. Finally, it restores the original forward+ method and returns the inputs for the specified block.+ Args:+ block_names (list): The names of the blocks for which inputs are to be saved.+ layer_names (list):The names of the layers for which inputs are to be saved.+ nsamples (int): The number of samples to use for calibration.+ last_cache_name (str, optional): The name of the last layer to be cached,+ we could break the forward in this layer to save time++ Returns:+ dict: A dictionary containing the inputs for the specified block.+ """+ if layer_names is None:+ layer_names = []++ if not self._post_init_done:+ self.post_init()++ if hasattr(self, "quantizer") and hasattr(self.quantizer, "attention_mask"):+ self.quantizer.attention_mask = []++ self.inputs = {}+ block_names = flatten_list(block_names)+ self.to_cached_layers = block_names + layer_names++ tmp_dtype = None # TODO delete this as most model is not fp32 now+ ## have bug if block name is not the first block+ if (len(block_names) > 1 or len(layer_names) > 0) and self.compress_context.low_gpu_mem_usage:+ tmp_dtype = self.model_context.model.dtype+ if self.model_context.amp:+ if self.model_context.model.dtype != self.model_context.model.dtype:+ self.model_context.model = self.model_context.model.to(torch.bfloat16)+ else:+ self.model_context.model = self.model_context.model.to(torch.float32) ##model on cpu++ self.last_cache_name = _infer_last_cache_name(block_names, layer_names, last_cache_name)+ self._cache_target_set = set(self.to_cached_layers)+ self._cache_seen_targets = set()+ calib_bs = self.quantizer.batch_size+ self.hook_handles = []+ self._replace_forward()+ try:+ self.calib(nsamples, calib_bs)+ finally:+ # Use finally to recover_forward and delattr in case of that+ # self.calib raises NotImplementedError, such as: flash_attn on CPU.+ self.model_context.recover_forward()+ for attr in ("last_cache_name", "_cache_target_set", "_cache_seen_targets", "to_cached_layers"):+ if hasattr(self, attr):+ delattr(self, attr)+ # Release calibration dataloader to free tokenized sample tensors+ if hasattr(self, "dataloader"):+ del self.dataloader+ res = self.inputs+ if tmp_dtype is not None:+ self.model_context.model = self.model_context.model.to(tmp_dtype)++ return res++ @torch.no_grad()+ def calib(self, nsamples, bs):+ """Perform calibration for quantization.++ This method calibrates the model for quantization by processing a specified+ number of samples from the calibration dataset. It ensures that the data is+ properly formatted and feeds it to the model. If the number of samples processed+ is less than the specified number, it logs a warning. If no samples are processed,+ it logs an error and exits.+ Args:+ nsamples (int): The number of samples to use for calibration.+ bs (int): The number of samples to use for calibration+ """+ from auto_round.calib_dataset import get_dataloader++ need_attention_mask = True+ if isinstance(self.dataset, str):+ need_attention_mask = False # all supported datasets does not use pad+ dataset = self.dataset.replace(" ", "") ##remove all whitespaces++ # slow here+ self.dataloader = get_dataloader(+ self.model_context.tokenizer,+ self.seqlen,+ dataset,+ self.seed,+ bs,+ self.nsamples,+ )+ else:+ self.dataloader = self.dataset+ total_cnt = 0+ if self.dataloader.__class__.__name__ == "BatchEncoding":+ self.dataloader = [self.dataloader.data]++ for data in self.dataloader:+ if data.__class__.__name__ == "BatchEncoding":+ data = data.data+ if data is None:+ continue+ if isinstance(data, torch.Tensor):+ input_ids = data.to(self.model.device)+ data_new = input_ids+ elif isinstance(data, str):+ if self.model_context.tokenizer is None:+ logger.error("please provide tokenizer for string input")+ exit(-1)+ data = self.model_context.tokenizer(+ data, truncation=True, max_length=self.seqlen, return_tensors="pt"+ ).data+ data_new = {}+ for key in data.keys():+ data_new[key] = data[key].to(self.model.device)+ input_ids = data_new["input_ids"]+ elif isinstance(data, tuple) or isinstance(data, list):+ data_new = to_device(data, self.model.device)+ input_ids = data_new[0]+ else:+ data_new = {}+ for key in data.keys():+ data_new[key] = to_device(data[key], self.model.device)+ if key == "images":+ data_new[key] = to_dtype(data_new[key], self.model.dtype)+ input_ids = data_new["input_ids"]+ if input_ids.shape[-1] < self.seqlen:+ continue+ if need_attention_mask:+ if (+ isinstance(data_new, dict)+ and "attention_mask" in data_new+ and data_new["attention_mask"] is not None+ ):+ new_attention_mask = data_new["attention_mask"]+ elif (+ self.model_context.tokenizer is not None+ and hasattr(self.model_context.tokenizer, "pad_token")+ and self.model_context.tokenizer.pad_token is not None+ ):+ new_attention_mask = (input_ids != self.model_context.tokenizer.pad_token_id).to(torch.long)+ else:+ # Default all ones+ new_attention_mask = torch.ones_like(input_ids, dtype=torch.long)++ # For each sample, check if there are trailing repeated tokens+ # If so, set the mask of the last token to 0+ batch_size, seq_len = input_ids.shape+ for i in range(batch_size):+ last_token = input_ids[i, -1]+ # Check for trailing repeats+ j = seq_len - 2+ repeated = False+ while j >= 0 and input_ids[i, j] == last_token:+ repeated = True+ new_attention_mask[i, j] = 0+ j -= 1+ # If there was at least one repeat, set last token mask to 0+ if repeated:+ new_attention_mask[i, -1] = 0++ # Workaround: some models treat an all-1 attention mask as equivalent to None and+ # will internally replace it with None for block inputs, which can cause tensor+ # concatenation / shape-mismatch issues in downstream code. To avoid providing an+ # all-1 mask, we force the last token in each sequence to be masked out (set to 0)+ # so that the mask is never "all ones". This means the model will not attend to the+ # last position, so the impact on accuracy is minimal as basically equivalent to dropping a single token+ new_attention_mask[:, -1] = 0++ if not hasattr(self.quantizer, "attention_mask"):+ self.quantizer.attention_mask = []+ self.quantizer.attention_mask.extend(list(torch.split(new_attention_mask, 1, dim=0)))+ else:+ new_attention_mask = None+ try:+ kwargs = {"use_cache": False}+ if new_attention_mask is not None and not (isinstance(data_new, dict) and "attention_mask" in data_new):+ kwargs["attention_mask"] = new_attention_mask++ if isinstance(data_new, torch.Tensor):+ self.model(data_new, **kwargs)+ elif isinstance(data_new, tuple) or isinstance(data_new, list):+ self.model(*data_new, **kwargs)+ else:+ self.model(**data_new, **kwargs)+ except NotImplementedError as error:+ error_msg = str(error)+ # Raise NotImplementedError to fallback to CUDA device+ if "flash_attn::" in error_msg and "CPU" in error_msg:+ raise NotImplementedError(+ "Could not run 'flash_attn::_flash_attn_varlen_forward'"+ " with arguments from the 'CPU' backend."+ )+ else:+ pass+ except RuntimeError as error:+ error_msg = str(error)+ if "The expanded size of the tensor" in str(error_msg) and "must match the existing size" in error_msg:+ check_seqlen_compatible(self.seqlen, self.model_context.tokenizer, self.model)+ logger.warning(+ "When quantization encounters tensor shape mismatch error, "+ "you can try to avoid it with batch_size=1"+ )+ raise error+ except Exception as error:+ raise error++ total_cnt += input_ids.shape[0] if len(input_ids.shape) > 1 else 1+ if total_cnt >= nsamples:+ break+ if total_cnt == 0:+ logger.error(+ f"no data has been cached, please provide more data with sequence length "+ f">={self.seqlen} in the dataset or decease the sequence length"+ )+ exit(-1)+ elif total_cnt < nsamples:+ logger.warning_once(+ f"An insufficient number of samples likely reduces the accuracy of the quantized model. "+ f"Target samples count is {nsamples}, while valid samples count is {total_cnt}"+ )++ @torch.no_grad()+ def _get_block_forward_func(self, name: str) -> Callable:+ """Gets the forward function.++ Args:+ name (str): The name of the function.+ Returns:+ function: The forward function.+ """++ def post_process_cache_data(batch_size, data, data_name):+ """+ Processes store data for batch handling, reshaping if necessary.++ Args:+ batch_size (int): The size of the batch.+ data: The data value to store, potentially for caching.+ data_name (str): Name of the data.++ Returns:+ Processed data or None+ """+ new_data = data+ if data_name in self.model_context.shared_cache_keys:+ return None+ if batch_size <= 1:+ return new_data+ if "alibi" in data_name:+ if isinstance(data, torch.Tensor):+ alibi = data+ alibi = alibi.reshape(batch_size, -1, alibi.shape[1], alibi.shape[2])+ new_data = alibi+ return new_data++ def forward(m, hidden_states=None, *positional_inputs, **kwargs):+ """Rewrite forward function, process and collect input data.++ Args:+ hidden_states (torch.Tensor): The hidden states tensor.+ *positional_inputs: Variable number of positional arguments.+ **kwargs: Variable number of keyword arguments.++ Returns:+ NotImplementedError: Getting the first layer inputs and then raise the error to save runtime.+ """+ if name not in self.inputs:+ self.inputs[name] = {}+ init_cache(positional_inputs, self.inputs[name])++ if self.quantizer.batch_dim is None:+ self.quantizer.batch_dim = 0+ if hidden_states is not None and self.quantizer.batch_size > 1:+ if hidden_states.shape[0] > self.quantizer.batch_size:+ self.quantizer.batch_dim = 1+ if len(hidden_states.shape) > 1 and hidden_states.shape[1] > self.quantizer.batch_size:+ logger.error(+ "this model has not been supported, "+ "please raise an issue in https://github.com/intel/auto-round/issues"+ " or try to set the `batch_size` to 1 and "+ "`gradient_accumulate_steps` to your current batch size."+ )+ exit(-1)++ if hidden_states is not None:+ kwargs["hidden_states"] = hidden_states++ for key in kwargs.keys():+ if (+ isinstance(kwargs[key], torch.Tensor)+ or isinstance(kwargs[key], list)+ or isinstance(kwargs[key], tuple)+ ):+ if (+ self.has_variable_block_shape+ and name not in self.blocks_requiring_input_ids+ and key == "hidden_states"+ ):+ continue+ if key not in self.inputs[name].keys(): # initialization+ data = to_device(kwargs[key], device=torch.device("cpu"))+ if data is None or key in self.model_context.shared_cache_keys:+ self.inputs[name][key] = data+ continue+ if self.quantizer.batch_size <= 1:+ self.inputs[name][key] = [data]+ else:+ data = post_process_cache_data(self.quantizer.batch_size, data, key)+ if isinstance(data, torch.Tensor):+ self.inputs[name][key] = list(torch.split(data, 1, dim=self.quantizer.batch_dim))+ else:+ self.inputs[name][key] = [data]+ else: # append cache inputs+ new_data = post_process_cache_data(self.quantizer.batch_size, kwargs[key], key)+ if new_data is None: # shareable args or NoneType+ continue+ new_data = to_device(new_data, device=torch.device("cpu"))+ if self.quantizer.batch_size <= 1:+ self.inputs[name][key].append(new_data)+ else:+ if isinstance(new_data, torch.Tensor):+ self.inputs[name][key].extend(+ list(torch.split(new_data, 1, dim=self.quantizer.batch_dim))+ )+ else:+ self.inputs[name][key].append(new_data)+ elif isinstance(kwargs[key], (str, bool, type(None))):+ if key not in self.inputs[name].keys():+ self.inputs[name][key] = kwargs[key]+ else:+ # Parameters not to be cached+ if check_skippable_keywords(key):+ logger.warning_once(+ f"Please note that '{key}' key" " is not currently used in quantization fine-tuning."+ )+ reset_params(self.inputs[name])++ if self._should_stop_cache_forward(name):+ raise NotImplementedError+ else:+ if hidden_states is not None:+ kwargs.pop("hidden_states")+ return m.orig_forward(hidden_states, *positional_inputs, **kwargs)+ else:+ # Currently only for Llama-3.2-Vision-Instruct Series+ return m.orig_forward(*positional_inputs, **kwargs)++ return forward++ @torch.no_grad()+ def _get_cache_data_hook_for_layer(self, name):+ """A forward hook to save input max of a module+ :param name: the module name+ :return: A hook function."""++ def cache_input_hook(module, inputs, outputs):+ input = inputs+ if isinstance(inputs, tuple) or isinstance(input, list):+ input = inputs[0]+ if name in self.inputs:+ self.inputs[name].extend(list(torch.split(input.to("cpu"), 1, dim=0)))+ else:+ self.inputs[name] = list(torch.split(input.to("cpu"), 1, dim=0))++ if self._should_stop_cache_forward(name):+ raise NotImplementedError++ return cache_input_hook++ def _replace_forward(self):+ """Replaces the forward function."""++ def register_hook(n, m, hook_handles):+ if n in self.to_cached_layers and type(m) not in SUPPORTED_LAYER_TYPES: ##block+ m.orig_forward = m.forward+ m.forward = partial(self._get_block_forward_func(n), m)+ elif n in self.to_cached_layers: ##linear layer or conv1d layer+ hook_func = self._get_cache_data_hook_for_layer(n)+ hook_handle = m.register_forward_hook(hook_func)+ hook_handles.append(hook_handle)++ self.model_context.replace_forward(register_hook)++ def _should_stop_cache_forward(self, name: str) -> bool:+ """Determine whether current forward pass can stop after caching `name`."""+ if name == self.last_cache_name:+ return True++ if self.last_cache_name is not None:+ return False++ if not hasattr(self, "_cache_target_set") or not hasattr(self, "_cache_seen_targets"):+ return False++ if name in self._cache_target_set:+ self._cache_seen_targets.add(name)++ if not self._cache_target_set.issubset(self._cache_seen_targets):+ return False++ # Lock the last cache name after the first full forward pass.+ self.last_cache_name = name+ return True++ def _preprocess_block_inputs(self, inputs, first_input_name="input_ids"):+ input_ids, input_others = self._split_inputs(inputs, first_input_name)+ clear_memory(device_list=self.compress_context.device_list)+ tmp_dtype = self.model_context.amp_dtype if self.model_context.amp else torch.float32+ if input_ids is not None:+ input_ids = to_device(input_ids, self.compress_context.cache_device)+ input_ids = to_dtype(input_ids, tmp_dtype)+ input_others = to_device(input_others, self.compress_context.cache_device)++ for key in input_others.keys():+ if isinstance(input_others[key], torch.Tensor) and (+ input_others[key].dtype == torch.float16 or input_others[key].dtype == torch.bfloat16+ ):+ input_others[key] = input_others[key].to(tmp_dtype)+ elif isinstance(input_others[key], list):+ for i in range(len(input_others[key])):+ to_dtype(input_others[key][i], tmp_dtype)+ return input_ids, input_others++ def _split_inputs(self, inputs: dict, first_input_name: str) -> tuple[torch.Tensor, dict]:+ if self.model_context.is_diffusion:+ input_id_str = [key for key in inputs.keys() if "hidden_state" in key]+ input_ids = {k: inputs.pop(k, None) for k in input_id_str}+ input_others = inputs+ return input_ids, input_others+ input_ids = inputs.get(first_input_name, None)+ inputs.pop(first_input_name, None)+ input_others = inputs+ return input_ids, input_others++ def normalize_decoding_layer_inputs_(self, decoding_layer_inputs: list[tuple[tuple[Any, dict[str, Any]]]]) -> None:+ """Replay captured decoding-layer calls to populate ``self.inputs``.++ Converts the raw ``(args, kwargs)`` tuples captured by LLM-Compressor's+ input hook into the ``self.inputs`` dict format expected by+ :meth:`quantize_block`. The logic mirrors the old-arch implementation in+ ``compressors/base.py``.++ Args:+ decoding_layer_inputs:+ A list of entries captured by a forward hook on the decoding layer.+ Each element is a tuple whose first item is ``(args, kwargs)``.+ """+ first_block_name = self.quant_block_list[0][0]++ class _FakeDecodingLayer(torch.nn.Module):++ def forward(self, *args, **kwargs):+ return args, kwargs++ fake_layer = _FakeDecodingLayer()+ fake_layer.orig_forward = fake_layer.forward+ fake_layer.forward = partial(self._get_block_forward_func(first_block_name), fake_layer)++ self.inputs = {}+ self.last_cache_name = None+ for step_input in decoding_layer_inputs:+ args, kwargs = step_input[0]+ fake_layer(*args, **kwargs)++ def quantize_block(+ self,+ block: torch.nn.Module,+ inputs: tuple,+ q_input: Union[torch.Tensor, dict, None] = None,+ device: Union[str, torch.device] = "cpu",+ auto_offload: bool = True,+ ):+ """Quantize a single decoded block of the model (public API for LLM-Compressor).++ This method is the new-arch equivalent of the old ``BaseCompressor.quantize_block``+ (see ``compressors/base.py``). It is primarily consumed by LLM-Compressor:+ https://github.com/vllm-project/llm-compressor/pull/1994++ The method normalizes the raw decoding-layer inputs provided by LLM-Compressor,+ runs the full infrastructure pipeline (device placement, act-max collection,+ reference-output caching) for the given *block*, delegates the pure-algorithm+ weight optimization to ``self.quantizer.quantize_block``, then returns the+ quantized-block outputs.++ Args:+ block: The transformer block (decoder layer) to quantize.+ inputs: Raw decoding-layer inputs captured by LLM-Compressor's hook.+ Format: list of ``((args, kwargs),)`` tuples as produced by the hook.+ q_input: Optional quantized input from the previous block. ``None`` on+ the first block.+ device: Target device for quantization (e.g. ``"cuda:0"``).+ auto_offload: When *True*, use the device-map-aware offloading path;+ otherwise move ``block`` directly to ``device``.++ Returns:+ tuple: ``(q_outputs, reference_output)`` where *q_outputs* is the+ block's output after quantization (or ``None`` when+ ``enable_quanted_input`` is ``False``), and *reference_output* is the+ full-precision reference output collected before optimization.+ """+ assert not self.mllm and not self.diffusion, (+ f"Currently, {self.__class__.__name__} does not support quantize_block " "for MLLM / diffusion models."+ )++ # Ensure post_init has been called (sets up model_context, compress_context,+ # quantizer, layer_config, etc.).+ if not self._post_init_done:+ self.post_init()++ self.normalize_decoding_layer_inputs_(inputs)+ block_inputs = self.inputs[self.quant_block_list[0][0]]+ input_ids, input_others = self._preprocess_block_inputs(block_inputs, "hidden_states")++ # ── Infrastructure: materialize, dtype convert, device placement ──────+ materialize_model_(block)+ convert_module_to_hp_if_necessary(block, self.model_context.amp_dtype, device)++ if auto_offload:+ if (+ is_auto_device_mapping(self.compress_context.device_map)+ and len(self.compress_context.device_list) > 1+ and not self.model_context.is_diffusion+ ):+ from auto_round.utils.device import set_auto_device_map_for_block_with_tuning++ card_0_in_high_risk, loss_device = set_auto_device_map_for_block_with_tuning(+ block,+ self.compress_context.device_map,+ input_ids,+ self.compress_context.low_gpu_mem_usage,+ self.quantizer.batch_size,+ device,+ )+ else:+ block = block.to(device)+ card_0_in_high_risk, loss_device = False, device+ else:+ card_0_in_high_risk, loss_device = False, device++ if len(self.compress_context.device_list) > 1 and auto_offload:+ from accelerate.hooks import AlignDevicesHook, add_hook_to_module++ for n, m in block.named_modules():+ if len(list(m.children())) != 0 or not hasattr(m, "tuning_device"):+ continue+ add_hook_to_module(m, AlignDevicesHook(m.tuning_device, io_same_device=True), True)++ # ── Infrastructure: collect reference output and act_max ──────────────+ bs = self.quantizer.batch_size * self.quantizer.infer_bs_coeff+ if q_input is None:+ hook_handles = self.quantizer._register_act_max_hook(block)+ reference_output = self.quantizer._get_block_outputs(block, input_ids, input_others, bs)+ for h in hook_handles:+ h.remove()+ else:+ reference_output = self.quantizer._get_block_outputs(block, input_ids, input_others, bs)+ hook_handles = self.quantizer._register_act_max_hook(block)+ if hook_handles:+ self.quantizer._get_block_outputs(block, q_input, input_others, bs, save_output=False)+ for h in hook_handles:+ h.remove()+ if input_ids is not q_input:+ clear_memory(input_ids, device_list=self.compress_context.device_list)+ else:+ clear_memory(device_list=self.compress_context.device_list)+ input_ids = q_input++ # ── Pure algorithm: delegates to quantizer ────────────────────────────+ mid_iter_mem_check = self.compress_context.low_gpu_mem_usage and card_0_in_high_risk+ self.quantizer.quantize_block(+ block,+ input_ids,+ input_others,+ reference_output,+ loss_device=loss_device,+ mid_iter_mem_check=mid_iter_mem_check,+ )++ # ── MoE scale alignment for FP8 dispatch efficiency ────────────────+ if is_nv_fp(self.quantizer.act_data_type) or is_static_wfp8afp8(self.quantizer):+ set_amax_for_all_moe_layers(block, attr_name="act_max")++ # ── Collect quantized-block outputs ───────────────────────────────────+ if self.quantizer.enable_quanted_input:+ q_outputs = self.quantizer._get_block_outputs(block, input_ids, input_others, bs)+ else:+ q_outputs = None++ # ── Cleanup ───────────────────────────────────────────────────────────+ if len(self.compress_context.device_list) > 1:+ accelerate.hooks.remove_hook_from_submodules(block)+ mv_module_from_gpu(block)++ return q_outputs, reference_output++ def _quantize_blocks(+ self,+ model: torch.nn.Module,+ inputs: dict,+ block_names: list,+ q_input: torch.Tensor = None,+ nblocks: int = 1,+ pbar: tqdm = None,+ input_others_extra_blocks: dict = None,+ ):+ """Quantize and dequantize the weights of the specified blocks in the model.++ Args:+ model: The PyTorch model to be quantized.+ inputs: The input data for quantization.+ block_names: The names of the blocks to be quantized and dequantized.+ nblocks: The number of blocks to quantize and dequantize.+ device: The device for quantization and dequantization.++ Returns:+ None+ """+ clear_memory(device_list=self.compress_context.device_list)+ for n, m in model.named_parameters():+ m.requires_grad_(False)++ input_ids, input_others = self._preprocess_block_inputs(inputs)++ if pbar is None:+ pbar = tqdm(range(0, len(block_names), nblocks))++ for i in range(0, len(block_names), nblocks):+ if input_others_extra_blocks and block_names[i] in input_others_extra_blocks:+ input_others = input_others_extra_blocks[block_names[i]]+ _, input_others = self._preprocess_block_inputs(input_others)+ input_others_extra_blocks.pop(block_names[i])+ if i != 0:+ pbar.update(1)+ if nblocks == 1:+ n = block_names[i]+ pbar.set_description(f"Quantizing {n}")+ m = get_module(model, n)+ else:+ names = block_names[i : min(i + nblocks, len(block_names))]+ pbar.set_description(f"Quantizing [{i + 1}-{min(i + nblocks, len(block_names))}]/{len(block_names)}")+ modules = [get_module(model, n) for n in names]+ m = WrapperMultiblock(modules)++ if self.compress_context.low_cpu_mem_usage:+ if nblocks == 1:+ self._offloader.reload(model, n)+ else:+ self._offloader.reload(model, names)++ block_name_or_names = n if nblocks == 1 else names++ # ── Infrastructure: materialize, dtype convert, device placement ──+ materialize_model_(m)+ convert_module_to_hp_if_necessary(m, self.model_context.amp_dtype, self.compress_context.device)++ if is_auto_device_mapping(self.compress_context.device_map) and len(self.compress_context.device_list) > 1:+ from auto_round.utils.device import set_auto_device_map_for_block_with_tuning++ card_0_in_high_risk, loss_device = set_auto_device_map_for_block_with_tuning(+ m,+ self.compress_context.device_map,+ input_ids,+ self.compress_context.low_gpu_mem_usage,+ self.quantizer.batch_size,+ self.compress_context.device,+ )+ else:+ m = m.to(self.compress_context.device)+ card_0_in_high_risk, loss_device = False, self.compress_context.device++ if len(self.compress_context.device_list) > 1:+ from accelerate.hooks import AlignDevicesHook, add_hook_to_module++ for _n, _mod in m.named_modules():+ if len(list(_mod.children())) != 0 or not hasattr(_mod, "tuning_device"):+ continue+ add_hook_to_module(_mod, AlignDevicesHook(_mod.tuning_device, io_same_device=True), True)++ # ── Infrastructure: collect reference output and act_max ──────────+ bs = self.quantizer.batch_size * self.quantizer.infer_bs_coeff+ if q_input is None:+ hook_handles = self.quantizer._register_act_max_hook(m)+ reference_output = self.quantizer._get_block_outputs(m, input_ids, input_others, bs)+ for h in hook_handles:+ h.remove()+ else:+ reference_output = self.quantizer._get_block_outputs(m, input_ids, input_others, bs)+ hook_handles = self.quantizer._register_act_max_hook(m)+ if hook_handles:+ self.quantizer._get_block_outputs(m, q_input, input_others, bs, save_output=False)+ for h in hook_handles:+ h.remove()++ # ── Infrastructure: swap q_input ──────────────────────────────────+ if q_input is not None:+ if input_ids is not q_input:+ clear_memory(input_ids, device_list=self.compress_context.device_list)+ else:+ clear_memory(device_list=self.compress_context.device_list)+ input_ids = q_input++ # ── Pure algorithm: delegates to quantizer ────────────────────────+ mid_iter_mem_check = self.compress_context.low_gpu_mem_usage and card_0_in_high_risk+ self.quantizer.quantize_block(+ m,+ input_ids,+ input_others,+ reference_output,+ loss_device=loss_device,+ mid_iter_mem_check=mid_iter_mem_check,+ )++ # ── MoE scale alignment for FP8 dispatch efficiency ────────────────+ if is_nv_fp(self.quantizer.act_data_type) or is_static_wfp8afp8(self.quantizer):+ set_amax_for_all_moe_layers(m, attr_name="act_max")++ # ── Infrastructure: collect q_outputs if needed ───────────────────+ if self.quantizer.enable_quanted_input:+ q_input = self.quantizer._get_block_outputs(m, input_ids, input_others, bs)+ else:+ q_input = None++ # ── Infrastructure: hook removal, device cleanup, logging ─────────+ if len(self.compress_context.device_list) > 1:+ accelerate.hooks.remove_hook_from_submodules(m)+ mv_module_from_gpu(m)+ if self.enable_torch_compile:+ torch._dynamo.reset()+ self.quantizer._invalidate_block_forward_cache()+ # Keep old-arch semantics: the next block's FP reference input comes+ # from the current block's reference output, while q_input (when+ # enabled) is only used as the quantized-input companion for the+ # next block.+ next_input_ids = reference_output+ clear_memory(+ input_ids if input_ids is not next_input_ids else None, device_list=self.compress_context.device_list+ )+ memory_monitor.log_summary()++ # ── Infrastructure: immediate_pack / shard write ──────────────────+ if self.compress_context.is_immediate_packing:+ for _n, _mod in m.named_modules():+ if hasattr(_mod, "bits") and check_to_quantized(_mod):+ from auto_round.compressors_new.utils import immediate_pack as _immediate_pack++ _immediate_pack(_mod.global_name, self.quantizer.layer_config)++ input_ids = next_input_ids++ if self.compress_context.is_immediate_saving:+ self.shard_writer.write(m, is_finalize=False)++ if self.compress_context.low_cpu_mem_usage and not self.compress_context.is_immediate_saving:+ if nblocks == 1:+ self._offloader(model, n, overwrite=True)+ else:+ for name in names:+ self._offloader(model, name, overwrite=True)+ if pbar is not None:+ pbar.update(1)++ if not self.compress_context.is_immediate_saving:+ self.model = mv_module_from_gpu(self.model)+ for n, m in self.model.named_modules():+ if hasattr(m, "name"):+ delattr(m, "name")++ del q_input+ del input_ids+ del input_others+ del inputs++ clear_memory(device_list=self.compress_context.device_list)++ def quantize(self) -> tuple[torch.nn.Module, dict[str, Any]]:+ """Quantize the model and return the quantized model along with layer configurations.The entry of AutoRound.+ Returns:+ The quantized model and layer configurations.+ """+ self.post_init()++ # Reclaim heap fragmentation from init/post_init before the memory-intensive quantize loop.+ gc.collect()+ _force_trim_malloc()++ self._check_compatibility()++ if bool(self.quantizer.quant_block_list):+ all_blocks = self.quantizer.quant_block_list+ else:+ all_blocks = get_block_names(self.model_context.model)++ if len(all_blocks) == 0:+ logger.warning("could not find blocks, exit with original model")+ return self.model_context.model, self.quantizer.layer_config++ layer_names = _get_quantized_layer_names_outside_blocks(+ model=self.model_context.model,+ layer_config=self.quantizer.layer_config,+ supported_types=SUPPORTED_LAYER_TYPES,+ quant_block_list=self.quantizer.quant_block_list,+ )+ if not self.has_variable_block_shape:+ to_cache_block_names = [block[0] for block in all_blocks]+ else:+ to_cache_block_names = flatten_list(all_blocks)+ if len(layer_names) > 0:+ logger.info(+ "Starting to cache block inputs. This may be slow due to external block layers: %s", layer_names+ )+ else:+ logger.info("start to cache block inputs")+ all_inputs = self.try_cache_inter_data_gpucpu(+ to_cache_block_names,+ self.nsamples,+ layer_names,+ )+ self.inputs = all_inputs+ is_quantized_embedding = self._quantize_embedding_layer()+ clear_memory(device_list=self.compress_context.device_list)+ all_q_inputs = None+ if is_quantized_embedding:+ all_inputs = copy.deepcopy(self.inputs)+ clear_memory(self.inputs, device_list=self.compress_context.device_list)+ all_q_inputs = self.try_cache_inter_data_gpucpu(to_cache_block_names, self.nsamples, layer_names)+ # Remove accelerate dispatch hooks before moving parameters.+ # hf_device_map is kept for reference but hooks are no longer needed.+ if hasattr(self.model_context.model, "hf_device_map") and len(self.model_context.model.hf_device_map) > 1:+ accelerate.hooks.remove_hook_from_submodules(self.model_context.model)+ self.model_context.model = mv_module_from_gpu(self.model_context.model)+ clear_memory(device_list=self.compress_context.device_list)+ logger.info("caching done")+ if self.compress_context.low_cpu_mem_usage:+ if self.model_context.is_model_patched and not self.compress_context.is_immediate_saving:+ self._offloader(self.model_context.model, all_blocks, clear_memory=True, device_list=self.device_list)+ if not self._offloader.enabled:+ self.compress_context.low_cpu_mem_usage = False+ else:+ self.compress_context.low_cpu_mem_usage = False+ if len(all_blocks) > 1:+ pbar = tqdm(range(0, sum([len(i) for i in all_blocks]), self.nblocks))+ else:+ pbar = tqdm(range(0, len(all_blocks[0]), self.nblocks)) # move the alg warning outside pbar++ start_time = time.time()+ for block_names in all_blocks:+ inputs = all_inputs[block_names[0]]+ all_inputs.pop(block_names[0])+ q_inputs = None+ if all_q_inputs is not None:+ q_inputs = all_q_inputs[block_names[0]]+ all_q_inputs.pop(block_names[0])++ inputs, q_inputs = _update_inputs(inputs, q_inputs)++ clear_memory(self.inputs, device_list=self.compress_context.device_list)++ if "input_ids" in inputs.keys():+ total_samples = len(inputs["input_ids"])+ if total_samples < self.quantizer.batch_size:+ self.quantizer.batch_size = total_samples+ logger.warning(f"force the train batch size to {total_samples}")++ self._quantize_blocks(+ self.model_context.model,+ inputs,+ block_names,+ q_input=q_inputs if q_inputs is not None else None,+ nblocks=self.nblocks,+ pbar=pbar,+ input_others_extra_blocks=all_inputs,+ )+ if self.compress_context.is_immediate_packing and len(self.formats) != 1:+ raise ValueError(+ f"Expected exactly one packing format when 'immediate_packing' is True, "+ f"but got {len(self.formats)} formats."+ )+ pbar.set_description("Quantizing done")+ pbar.close()+ if self.compress_context.low_cpu_mem_usage:+ self._offloader.reload(self.model_context.model)+ self._quantize_layers(layer_names, all_inputs)++ convert_module_to_hp_if_necessary(+ self.model_context.model, self.model_context.amp_dtype, self.compress_context.device, to_cpu=True+ )+ if self.compress_context.is_immediate_saving:+ self.shard_writer.write(is_finalize=True)++ end_time = time.time()+ cost_time = end_time - start_time+ logger.info(f"quantization tuning time {cost_time}")++ # Dump a summary+ quantized_layers = []+ unquantized_layers = []+ for n, m in self.model_context.model.named_modules():+ if isinstance(m, tuple(SUPPORTED_LAYER_TYPES)):+ if check_to_quantized(m):+ quantized_layers.append(n)+ else:+ unquantized_layers.append(n)+ elif hasattr(m, "scales") or hasattr(m, "scale"): # packing_immediately+ quantized_layers.append(n)+ summary_info = (+ f"Summary: quantized {len(quantized_layers)}/{len(quantized_layers) + len(unquantized_layers)} in the model"+ )+ if len(unquantized_layers) > 0:+ compressed_unquantized_layers = compress_layer_names(unquantized_layers)+ summary_info += f", unquantized layers: {compressed_unquantized_layers}"+ logger.info(summary_info)++ self.model_context.quantized = True+ return self.model_context.model, self.quantizer.layer_config++ def _quantize_layers(self, layer_names: list, layer_inputs: dict) -> None:+ """Quantizes specified layers based on inputs and configuration.++ Args:+ layer_names (list): list of layer names to quantize.+ layer_inputs (dict): Dictionary mapping layer names to input data.++ Returns:+ None+ """+ # TODO currently we take all the layers outside blocks as post block layers which is not optimal+ # if there is no input for layer, we use rtn++ for layer_name in copy.deepcopy(layer_names):+ if layer_name not in layer_inputs:+ if self.act_bits < 16 and not self.act_dynamic:+ if "lm_head" in layer_name:+ logger.warning_once(+ "Static activation quantization for lm_head is not fully supported yet. "+ "If lm_head calibration inputs are missing, activation scale may fall back to unit scale "+ "or quantization may be skipped."+ )+ # Activation quantization requires collected inputs+ msg_prefix = (+ f"Activation max hook for layer '{layer_name}' is unavailable due to "+ f"insufficient collected inputs. "+ )+ if "fp8_e5m2" in self.act_data_type:+ logger.warning(msg_prefix + "Please notes that unit scale is used for this layer.")+ else:+ logger.warning(+ msg_prefix + "Static activation quantization is not supported or ineffective, "+ "Skipping quantization for this layer."+ )+ layer_names.remove(layer_name)+ continue+ logger.info(f"using rtn to quantize {layer_name}")+ from auto_round.data_type import QUANT_FUNC_WITH_DTYPE++ layer = get_module(self.model, layer_name)+ layer = layer.to(self.compress_context.device)+ layer = convert_module_to_hp_if_necessary(+ layer, self.model_context.amp_dtype, self.compress_context.device+ )+ set_module(self.model, layer_name, layer)++ wrapper_layer = WrapperLinear(+ layer,+ enable_round_tuning=False,+ enable_minmax_tuning=False,+ enable_norm_bias_tuning=False,+ enable_torch_compile=self.enable_torch_compile,+ device=self.compress_context.device,+ disable_opt_rtn=self.disable_opt_rtn,+ )+ new_layer = wrapper_layer.unwrapper({})+ set_module(self.model, layer_name, new_layer)+ layer.cpu()+ layer_names.remove(layer_name)+ if len(layer_names) == 0:+ memory_monitor.update()+ memory_monitor.log_summary()+ return+ q_layer_inputs = None+ enable_quanted_input = self.enable_quanted_input+ has_gguf = False++ if hasattr(self, "formats") and self.formats is not None:+ has_gguf = any(format_.is_gguf() for format_ in self.formats)+ if has_gguf and self.compress_context.is_immediate_packing:+ enable_quanted_input = False++ if hasattr(self.model, "hf_device_map") and len(self.model.hf_device_map) > 1 and enable_quanted_input:+ dispatch_model(self.model, self.model.hf_device_map)++ if enable_quanted_input:+ logger.info("starting to cache layer inputs for %s, this may be quite slow ", layer_names)+ q_layer_inputs = self.try_cache_inter_data_gpucpu([], self.nsamples, layer_names=layer_names)+ if hasattr(self.model, "hf_device_map") and len(self.model.hf_device_map) > 1:+ accelerate.hooks.remove_hook_from_submodules(+ self.model+ ) # self.model.hf_device_map has not been changed+ if not self.compress_context.is_immediate_saving:+ self.model = mv_module_from_gpu(self.model)+ clear_memory(device_list=self.compress_context.device_list)+ quant_layer = self.quantizer.quantize_layer_outside_block+ for layer_name in layer_names:+ layer_input = layer_inputs[layer_name]+ layer_input = to_device(layer_input, self.compress_context.cache_device)+ q_layer_input = q_layer_inputs.get(layer_name, None) if q_layer_inputs is not None else None+ q_layer_input = to_device(q_layer_input, self.compress_context.cache_device)+ quant_layer(layer_name, layer_input, q_layer_input, device=self.compress_context.device)+ if self.compress_context.is_immediate_packing:+ immediate_pack(layer_name, self.quantizer.layer_config)++ if self.compress_context.is_immediate_saving:+ m = get_module(self.model, layer_name)+ self.shard_writer.write(m, name=layer_name, is_finalize=False)+ del layer_input+ clear_memory(q_layer_input, device_list=self.compress_context.device_list)+ memory_monitor.log_summary()++ def _check_compatibility(self) -> None:+ """Checks compatibility of the configurations and model."""+ if (+ self.seqlen is not None+ and hasattr(self.model_context.model, "config")+ and hasattr(self.model_context.model.config, "max_position_embeddings")+ ):+ if self.model_context.model.config.max_position_embeddings < self.seqlen:+ logger.warning(+ f"Change sequence length to {self.model_context.model.config.max_position_embeddings} "+ "due to the limitation of max_position_embeddings"+ )+ self.seqlen = min(self.seqlen, self.model_context.model.config.max_position_embeddings)++ if self.seqlen is not None and hasattr(self.model_context.tokenizer, "model_max_length"):+ if self.model_context.tokenizer.model_max_length < self.seqlen:+ logger.warning(+ f"Change sequence length to {self.model_context.tokenizer.model_max_length} "+ "due to the limitation of model_max_length. "+ "You can also try to increase the model_max_length to avoid this issue."+ )+ self.seqlen = min(self.seqlen, self.model_context.tokenizer.model_max_length)++ if self.group_size == 0 and "fp8" not in self.data_type:+ logger.warning("`group_size==0` is not supported for data_type other than fp8 ")++ if (+ self.bits <= 2+ and (self.iters < 1000 or not getattr(self.quantize_config, "enable_alg_ext", False))+ and self.super_group_size is None+ ):+ logger.warning(+ "for bits <= 2, it is recommended to enable `auto-round-best` " "and turn on `--enable_alg_ext` "+ )+++class CalibratedRTNCompressor(CalibCompressor):+ """CalibCompressor variant for iters=0 RTN that needs calibration data.++ Handles two cases that require forward passes through the model:+ - Weight quantization with imatrix (importance-matrix statistics for+ improved RTN accuracy on INT / weight-only schemes).+ - Activation quantization with static scales (e.g. NVFP4, FP8_STATIC)+ where per-tensor or per-channel scale factors must be collected before+ the actual quantization step.++ Both cases use OptimizedRTNQuantizer and need a calibration dataset,+ which is why they cannot be handled by the zero-shot (no-data) path.+ """++ need_calib: bool = True++ def __init__(+ self,+ config: AlgConfig,+ model: torch.nn.Module,+ **kwargs,+ ):+ kwargs["iters"] = 0+ super().__init__(+ config,+ model,+ **kwargs,+ )++ def _quantize_via_rtn_blockwise(self) -> None:+ """Quantize model layers block by block using cached inputs and imatrix."""++ all_blocks = self.quantizer.quant_block_list or get_block_names(self.model)+ if not all_blocks:+ raise ValueError("Could not find any blocks. Check the model or quant_block_list.")++ if not self.has_variable_block_shape:+ to_cache_block_names = [block[0] for block in all_blocks]+ else:+ to_cache_block_names = flatten_list(all_blocks)+ layer_names = _get_quantized_layer_names_outside_blocks(+ model=self.model_context.model,+ layer_config=self.quantizer.layer_config,+ supported_types=SUPPORTED_LAYER_TYPES,+ quant_block_list=self.quantizer.quant_block_list,+ )+ if (+ self.quantize_config.is_act_quantize+ and (not self.quantize_config.act_dynamic or len(layer_names) > 0)+ or self.has_variable_block_shape+ ):+ if len(layer_names) > 0:+ logger.warning(+ "quantize layers outside blocks for static activation quantizaiton"+ " will significantly increase calibration time"+ )+ all_inputs = self.try_cache_inter_data_gpucpu(to_cache_block_names, self.nsamples, layer_names)+ else:+ all_inputs = self.cache_inter_data(to_cache_block_names, self.nsamples)++ # Clear hooks for multi-GPU setups+ if hasattr(self.model_context.model, "hf_device_map") and len(self.model_context.model.hf_device_map) > 1:+ accelerate.hooks.remove_hook_from_submodules(self.model_context.model)++ pbar = tqdm(range(sum(len(block) for block in all_blocks)))++ for block_names in all_blocks:+ first_block = block_names[0]+ inputs = all_inputs.pop(first_block)+ input_keys = [k for k in inputs if k.startswith("hidden_state")]+ if len(input_keys) != 1:+ raise RuntimeError(+ "hidden_states arg mismatch. Please file an issue at https://github.com/intel/auto-round/issues"+ )+ inputs["input_ids"] = inputs.pop(input_keys[0])++ clear_memory(self.inputs, device_list=self.compress_context.device_list)++ total_samples = len(inputs["input_ids"])+ if total_samples < self.quantize_config.batch_size:+ self.quantize_config.batch_size = total_samples+ logger.warning(f"Forcing batch size to {total_samples}")++ tmp_dtype = self.model_context.amp_dtype if self.model_context.amp else torch.float32++ input_ids = to_device(inputs.pop("input_ids"), self.compress_context.cache_device)+ input_ids = [id_.to(tmp_dtype) for id_ in input_ids]++ def process_input_others(input_others):+ input_others = to_device(input_others, self.compress_context.cache_device)+ for key, val in input_others.items():+ if isinstance(val, torch.Tensor) and val.dtype in (torch.float16, torch.bfloat16):+ input_others[key] = val.to(tmp_dtype)+ elif isinstance(val, list):+ input_others[key] = [to_dtype(v, tmp_dtype) for v in val]+ return input_others++ input_others = inputs+ input_others = process_input_others(input_others)+ for block_name in block_names:+ if block_name in all_inputs.keys():+ input_others = all_inputs[block_name]+ input_others = process_input_others(input_others)+ all_inputs.pop(block_name)+ pbar.set_description(f"Quantizing {block_name}")+ block = get_module(self.model_context.model, block_name)++ # ── Infrastructure: materialize, dtype convert, device placement ──+ materialize_model_(block)+ block.to("cpu")+ block = convert_module_to_hp_if_necessary(+ block, dtype=self.model_context.amp_dtype, device=self.compress_context.device+ )+ if (+ is_auto_device_mapping(self.compress_context.device_map)+ and len(self.compress_context.device_list) > 1+ ):+ from auto_round.utils.device import set_auto_device_map_for_block_with_tuning++ set_auto_device_map_for_block_with_tuning(+ block,+ self.compress_context.device_map,+ input_ids,+ self.compress_context.low_gpu_mem_usage,+ self.quantizer.batch_size,+ self.compress_context.device,+ )+ if len(self.compress_context.device_list) > 1:+ from accelerate.hooks import AlignDevicesHook, add_hook_to_module++ for _, _mod in block.named_modules():+ if len(list(_mod.children())) != 0 or not hasattr(_mod, "tuning_device"):+ continue+ add_hook_to_module(_mod, AlignDevicesHook(_mod.tuning_device, io_same_device=True), True)+ else:+ block = block.to(self.compress_context.device)++ # ── Infrastructure: register act_max hook and run forward pass ──+ hook_handles = self.quantizer._register_act_max_hook(block)+ input_ids = self.quantizer._get_block_outputs(+ block,+ input_ids,+ input_others,+ self.quantizer.batch_size * self.quantizer.infer_bs_coeff,+ )+ for h in hook_handles:+ h.remove()++ if len(self.compress_context.device_list) > 1:+ accelerate.hooks.remove_hook_from_submodules(block)++ if self.compress_context.low_gpu_mem_usage:+ block.to("cpu")+ self.compress_context.clear_memory()++ # ── Pure algorithm ────────────────────────────────────────────+ self.quantizer.quantize_block(block)++ # ── Infrastructure: cleanup ───────────────────────────────────+ mv_module_from_gpu(block)++ if self.compress_context.low_cpu_mem_usage and not self.compress_context.is_immediate_saving:+ self._offloader(self.model_context.model, block_name)+ if block_name == block_names[-1]:+ clear_memory(input_ids, device_list=self.compress_context.device_list)+ else:+ clear_memory(device_list=self.compress_context.device_list)++ memory_monitor.log_summary()+ pbar.update(1)+ pbar.close()+ # Process remaining layers not in blocks+ # Collect names of quantizable layers not belonging to any block+ remain_layer_names = []+ block_name_set = set(name for block in all_blocks for name in block)+ for n, m in self.model_context.model.named_modules():+ if not check_to_quantized(m):+ continue+ # Skip if this layer is part of any block (by prefix match)+ if any(n == block_name or n.startswith(f"{block_name}.") for block_name in block_name_set):+ continue+ remain_layer_names.append(n)++ for name in remain_layer_names:+ dtype = None+ if self.super_group_size is not None:+ dtype = torch.float32+ self.quantizer.quantize_layer_outside_block(name, dtype=dtype)+ # clear_memory(device_list=self.compress_context.device_list)+ # if self.compress_context.is_immediate_saving:+ # shard_writer(self, is_finalize=True)++ def _quant_rtn_with_imatrix(self) -> None:+ """Performs RTN quantization using input activation statistics (imatrix).++ This method accumulates per-channel second-moment activation statistics (imatrix)+ via forward hooks and uses them to perform RTN quantization. If CUDA memory runs out,+ it falls back to CPU-based blockwise quantization.++ Returns:+ None+ """+ logger.info("start to compute imatrix")++ # Load dataset+ from auto_round.calib_dataset import get_dataloader++ if isinstance(self.dataset, str):+ if self.model_context.tokenizer is None:+ raise ValueError("A tokenizer must be set for the model when using a dataset string.")+ dataset_name = self.dataset.replace(" ", "")+ self.dataloader = get_dataloader(+ self.model_context.tokenizer,+ self.seqlen,+ dataset_name,+ self.seed,+ self.quantize_config.batch_size,+ self.nsamples,+ )+ else:+ self.dataloader = self.dataset++ model = self.model_context.model++ # Dispatch multi-GPU model if necessary+ if hasattr(model, "hf_device_map") and len(model.hf_device_map) > 1:+ dispatch_model(model, model.hf_device_map)++ def register_act_hook(model):+ """Registers hooks to accumulate activation squared norms into `imatrix`."""++ def get_imatrix_hook(module, input, output):+ input = input[0] if isinstance(input, (tuple, list)) else input+ flattened = input.reshape(-1, input.shape[-1]).to(torch.float32)+ squared = torch.sum(torch.pow(flattened, 2), dim=0).to(torch.float32)++ if not hasattr(module, "imatrix"):+ module.imatrix = squared+ module.imatrix_cnt = input.shape[0]+ else:+ module.imatrix += squared.to(module.imatrix.device)+ module.imatrix_cnt += input.shape[0]++ hook_handles = []+ for name, module in model.named_modules():+ if type(module) in SUPPORTED_LAYER_TYPES and check_to_quantized(module):+ hook = module.register_forward_hook(get_imatrix_hook)+ hook_handles.append(hook)+ return hook_handles++ hooks = register_act_hook(model)++ try:+ if hasattr(model, "hf_device_map") and len(model.hf_device_map) > 1:+ import accelerate++ accelerate.hooks.remove_hook_from_submodules(model)+ safe_to_cpu_(model)+ clear_memory(device_list=self.compress_context.device_list)+ self._quantize_via_rtn_blockwise()+ except torch.OutOfMemoryError:+ cuda_error_msg = traceback.format_exc()+ try:+ logger.error(cuda_error_msg)+ # Final fallback: warn and use CPU-only quantization+ logger.warning(+ "Fallback to CPU. "+ "Consider enabling `low_gpu_mem_usage` or using more GPUs via `--device 0,1,2,3`."+ )+ safe_to_cpu_(model)+ clear_memory(device_list=self.compress_context.device_list)+ if hasattr(model, "hf_device_map") and len(model.hf_device_map) > 1:+ import accelerate++ accelerate.hooks.remove_hook_from_submodules(model)++ orig_device = self.compress_context.device+ self.compress_context.device = "cpu"+ self._quantize_via_rtn_blockwise()+ self.compress_context.device = orig_device+ except Exception as e:+ raise+ finally:+ # Always remove hooks+ for hook in hooks:+ hook.remove()++ def quantize(self):+ """Quantize all modules in the model using RTN (Round-To-Nearest) strategy.++ If the target format includes GGUF with `k`, and optimized RTN is enabled,+ blockwise quantization with input caching and imatrix is used.++ Returns:+ tuple[nn.Module, Dict[str, Any]]: The quantized model and the layer configuration.+ """+ # post_init must be called OUTSIDE @torch.inference_mode() because+ # AutoScheme delta-loss selection requires autograd (backward pass).+ self.post_init()+ return self._quantize_impl()++ # Use no_grad instead of inference_mode+ # https://github.com/intel/auto-round/issues/1620+ @torch.no_grad()+ def _quantize_impl(self):++ formats = getattr(self, "formats", None) or []+ if not (any(fmt.is_gguf() for fmt in formats) or self.super_bits is not None):+ self._quantize_embedding_layer() # leave to gguf itself to handle++ # Release memory+ clear_memory(device_list=self.compress_context.device_list)++ enable_imatrix = False+ if not getattr(self, "disable_opt_rtn", True):+ formats = getattr(self, "formats", None) or []+ has_gguf_k = (+ any(fmt.is_gguf() and "k" in fmt.output_format for fmt in formats) or self.super_bits is not None+ )+ if has_gguf_k:+ enable_imatrix = True+ elif self.data_type == "int" and self.sym and self.bits < 8:+ enable_imatrix = True++ if enable_imatrix:+ self._quant_rtn_with_imatrix()+ else:+ self._quantize_via_rtn_blockwise()++ convert_module_to_hp_if_necessary(+ self.model_context.model,+ self.model_context.amp_dtype,+ self.compress_context.device,+ )+ if self.compress_context.low_cpu_mem_usage:+ self._offloader.reload(self.model_context.model)+ if self.compress_context.is_immediate_saving:+ self.shard_writer.write(is_finalize=True)++ self.model_context.quantized = True+ return self.model_context.model, self.quantizer.layer_configdiff --git a/auto_round/compressors_new/config.py b/auto_round/compressors_new/config.pynew file mode 100644index 000000000..dd028b6cd--- /dev/null+++ b/auto_round/compressors_new/config.py@@ -0,0 +1,296 @@+# Copyright (c) 2025 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+from __future__ import annotations++from dataclasses import dataclass, fields+from typing import Any, Callable, Optional, Union++import torch+++class ExtraConfig:+ """Class for extra or legacy configs."""++ _model_config = None+ _scheme_config = None+ _tuning_config = None+ _mllm_config = None+ _diffusion_config = None++ def __init__(+ self,+ # tuning+ amp: bool = True,+ disable_opt_rtn: bool | None = None,+ enable_alg_ext: bool = False,+ enable_minmax_tuning: bool = True,+ enable_norm_bias_tuning: bool = False,+ enable_quanted_input: bool = True,+ enable_deterministic_algorithms: bool = False,+ lr: float = None,+ lr_scheduler: Callable = None,+ minmax_lr: float = None,+ nblocks: int = 1,+ to_quant_block_names: Union[str, list, None] = None,+ scale_dtype: str = "fp16",+ # scheme+ bits: int = None,+ group_size: int = None,+ sym: bool = None,+ data_type: str = None,+ act_bits: int = None,+ act_group_size: int = None,+ act_sym: bool = None,+ act_data_type: str = None,+ act_dynamic: bool = None,+ super_bits: int = None,+ super_group_size: int = None,+ static_kv_dtype: Union[str, torch.dtype] = None,+ quant_lm_head: bool = False,+ ignore_layers: str = None,+ # mllm+ processor: Callable = None,+ image_processor: Callable = None,+ quant_nontext_module: bool = False,+ extra_data_dir: str = None,+ template: str = None,+ # diffusion+ guidance_scale: float = 7.5,+ num_inference_steps: int = 50,+ generator_seed: int = None,+ ):+ """Initialize++ Args:+ amp (bool): Whether to use automatic mixed precision (default is True).+ disable_opt_rtn (bool, optional): Disable RTN-mode optimization (iters=0). Defaults to True.+ enable_alg_ext (bool, optional): Enable algorithm extension (primarily for INT2). Defaults to False.+ enable_minmax_tuning (bool, optional): Enable weight min-max tuning. Defaults to True.+ enable_norm_bias_tuning (bool): Whether to enable fast norm/layer_bias tuning.+ enable_quanted_input (bool): Whether to use quantized input data (default is True).+ enable_deterministic_algorithms (bool): Whether to use deterministic_algorithms.+ lr (float): The learning rate (default is 0.005).+ lr_scheduler: The learning rate scheduler to be used.+ minmax_lr (float): The learning rate for min-max tuning (default is None).+ nblocks (int): Number of blocks (default is 1).+ quant_lm_head (bool): Whether to quant lm_head.+ to_quant_block_names (str|list): Names of quantitative blocks, please use commas to separate them.+ scale_dtype (str): The data type of quantization scale to be used (default is "float16"), different kernels+ bits (int, optional): Weight quantization bits. Defaults to 4.+ group_size (int, optional): Weight quantization group size. Defaults to 128.+ sym (bool, optional): Symmetric weight quantization. Defaults to True.+ data_type (str, optional): Weight data type string, e.g., "int". Defaults to "int".+ act_bits (int, optional): Activation quantization bits. Defaults to 16.+ act_group_size (int, optional): Activation group size. Defaults to None.+ act_sym (bool, optional): Symmetric activation quantization. Defaults to None.+ act_data_type (str, optional): Activation data type; inherits weight dtype if None and act_bits < 16.+ act_dynamic (bool, optional): Dynamic activation quantization. Defaults to True.+ super_bits (int): number of scale and mins quant bits for double quant.+ super_group_size (int): the number of super group size when use double quant.+ static_kv_dtype (str): The data type of kv-cache to be used.+ processor: Any multi-modal model will require an object to encode or+ decode the data that groups several modalities (among text, vision and audio).+ image_processor: Image processor for special model like llava.+ quant_nontext_module: Whether to quantize nontext module.+ extra_data_dir: The path of extra data such as images, audio and videos.+ template: The path or name of template used to specify process for different MLLMs.+ guidance_scale (float): Control how much the image generation process follows the text prompt.+ The more it is, the more closely it follows the prompt (default is 7.5).+ num_inference_steps (int): The reference number of denoising steps (default is 50).+ generator_seed (int): A seed that controls the initial noise for image generation (default is None).+ """+ self.tuning_config = TuningExtraConfig(+ amp=amp,+ disable_opt_rtn=disable_opt_rtn,+ enable_alg_ext=enable_alg_ext,+ enable_minmax_tuning=enable_minmax_tuning,+ enable_norm_bias_tuning=enable_norm_bias_tuning,+ enable_quanted_input=enable_quanted_input,+ enable_deterministic_algorithms=enable_deterministic_algorithms,+ lr=lr,+ lr_scheduler=lr_scheduler,+ minmax_lr=minmax_lr,+ nblocks=nblocks,+ to_quant_block_names=to_quant_block_names,+ scale_dtype=scale_dtype,+ )+ self.scheme_config = SchemeExtraConfig(+ bits=bits,+ group_size=group_size,+ sym=sym,+ data_type=data_type,+ act_bits=act_bits,+ act_group_size=act_group_size,+ act_sym=act_sym,+ act_data_type=act_data_type,+ act_dynamic=act_dynamic,+ super_bits=super_bits,+ super_group_size=super_group_size,+ static_kv_dtype=static_kv_dtype,+ quant_lm_head=quant_lm_head,+ ignore_layers=ignore_layers,+ )+ self.mllm_config = MLLMExtraConfig(+ processor=processor,+ image_processor=image_processor,+ quant_nontext_module=quant_nontext_module,+ extra_data_dir=extra_data_dir,+ template=template,+ )+ self.diffusion_config = DiffusionExtraConfig(+ guidance_scale=guidance_scale,+ num_inference_steps=num_inference_steps,+ generator_seed=generator_seed,+ )++ @property+ def tuning_config(self):+ return self._tuning_config++ @tuning_config.setter+ def tuning_config(self, config: TuningExtraConfig):+ assert isinstance(+ config, TuningExtraConfig+ ), f"tuning_config should be ModelExtraConfig, but got {config.__class__.__name__}"+ self._tuning_config = config++ @property+ def scheme_config(self):+ return self._scheme_config++ @scheme_config.setter+ def scheme_config(self, config: SchemeExtraConfig):+ assert isinstance(+ config, SchemeExtraConfig+ ), f"scheme_config should be SchemeExtraConfig, but got {config.__class__.__name__}"+ self._scheme_config = config++ @property+ def mllm_config(self):+ return self._mllm_config++ @mllm_config.setter+ def mllm_config(self, config: MLLMExtraConfig):+ if config is None:+ self._mllm_config = None+ else:+ assert isinstance(+ config, MLLMExtraConfig+ ), f"mllm_config should be MLLMExtraConfig, but got {config.__class__.__name__}"+ self._mllm_config = config++ @property+ def diffusion_config(self):+ return self._diffusion_config++ @diffusion_config.setter+ def diffusion_config(self, config: DiffusionExtraConfig):+ if config is None:+ self._diffusion_config = None+ else:+ assert isinstance(+ config, DiffusionExtraConfig+ ), f"diffusion_config should be DiffusionExtraConfig, but got {config.__class__.__name__}"+ self._diffusion_config = config++ def to_dict(self):+ output_dict = {}+ for config in self.__dict__.values():+ if config:+ output_dict.update(config.to_dict())+ return output_dict+++@dataclass+class BaseExtraConfig:++ @classmethod+ def get_attributes(cls: "BaseExtraConfig") -> list[str]:+ return [field.name for field in fields(cls)]++ def __getitem__(self, key: str):+ if key not in self.get_attributes():+ raise KeyError(f"{key} is not a valid attribute")+ return getattr(self, key)++ def __setitem__(self, key: str, value: None | int | str):+ if key not in self.get_attributes():+ raise KeyError(f"{key} is not a valid attribute")+ setattr(self, key, value)++ def __contains__(self, item):+ return item in self.get_attributes()++ def to_dict(self):+ return self.__dict__++ def is_default(self):+ for field in fields(self):+ default_value = field.default+ current_value = getattr(self, field.name)+ if current_value != default_value:+ return False+ return True+++@dataclass+class TuningExtraConfig(BaseExtraConfig):+ amp: bool = True+ disable_opt_rtn: bool | None = None+ enable_alg_ext: bool = False+ enable_minmax_tuning: bool = True+ enable_norm_bias_tuning: bool = False+ enable_quanted_input: bool = True+ enable_deterministic_algorithms: bool = False+ lr: float = None+ lr_scheduler: Callable = None+ minmax_lr: float = None+ nblocks: int = 1+ to_quant_block_names: Union[str, list, None] = None+ scale_dtype: str = "fp16"+++@dataclass+class SchemeExtraConfig(BaseExtraConfig):+ bits: int = None+ group_size: int = None+ sym: bool = None+ data_type: str = None+ act_bits: int = None+ act_group_size: int = None+ act_sym: bool = None+ act_data_type: str = None+ act_dynamic: bool = None+ super_bits: int = None+ super_group_size: int = None+ static_kv_dtype: Union[str, torch.dtype] = None+ static_attention_dtype: Union[str, torch.dtype] = None+ quant_lm_head: bool = False+ ignore_layers: str = None+++@dataclass+class MLLMExtraConfig(BaseExtraConfig):+ processor: Callable = None+ image_processor: Callable = None+ quant_nontext_module: bool = False+ extra_data_dir: str = None+ template: str = None+++@dataclass+class DiffusionExtraConfig(BaseExtraConfig):+ guidance_scale: float = 7.5+ num_inference_steps: int = 50+ generator_seed: int = Nonediff --git a/auto_round/compressors_new/diffusion_mixin.py b/auto_round/compressors_new/diffusion_mixin.pynew file mode 100644index 000000000..515839c1f--- /dev/null+++ b/auto_round/compressors_new/diffusion_mixin.py@@ -0,0 +1,271 @@+# Copyright (c) 2026 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+import os+from typing import Union++import torch+from tqdm import tqdm++from auto_round.logger import logger+from auto_round.utils.model import wrap_block_forward_positional_to_kwargs+++class DiffusionMixin:+ """Diffusion-specific functionality mixin.++ This mixin adds diffusion model-specific functionality to any compressor+ (CalibCompressor, ZeroShotCompressor, ImatrixCompressor, etc). It handles+ diffusion models (like Stable Diffusion, FLUX) that require special pipeline+ handling and data generation logic.++ Can be combined with:+ - CalibCompressor (for AutoRound with calibration)+ - ImatrixCompressor (for RTN with importance matrix)+ - ZeroShotCompressor (for basic RTN)++ Diffusion-specific parameters:+ guidance_scale: Control how much image generation follows text prompt+ num_inference_steps: Reference number of denoising steps+ generator_seed: Seed for initial noise generation++ Design note:+ ``ModelContext._load_model()`` loads the diffusion pipeline and sets+ ``model_context.pipe`` and ``model_context.model`` (the unet/transformer).+ This mixin reads ``self.model_context.pipe`` directly during calibration and+ saving so that ``model_context`` remains the single source of truth.+ """++ def __init__(self, *args, guidance_scale=7.5, num_inference_steps=50, generator_seed=None, **kwargs):+ # Store diffusion-specific attributes+ self.guidance_scale = guidance_scale+ self.num_inference_steps = num_inference_steps+ self.generator_seed = generator_seed++ # Mirror old-arch DiffusionCompressor.__init__: when iters > 0, diffusion calibration+ # cannot use batch_size > 1 for non-text modules; fold the extra batch into+ # gradient_accumulate_steps so the effective sample count is unchanged.+ # The authoritative batch_size lives on the AlgConfig (args[0]); kwargs may also+ # carry it from AutoRoundCompatible. Patch BOTH (same pattern as MLLMMixin).+ iters = kwargs.get("iters", None)+ _alg_cfg = args[0] if args else None+ if iters is None and _alg_cfg is not None:+ cfgs = _alg_cfg if isinstance(_alg_cfg, list) else [_alg_cfg]+ for cfg in cfgs:+ if hasattr(cfg, "iters") and cfg.iters is not None:+ iters = cfg.iters+ break+ if iters is None:+ iters = 200++ if iters > 0:+ batch_size = kwargs.get("batch_size", None)+ if batch_size is None and _alg_cfg is not None:+ cfgs = _alg_cfg if isinstance(_alg_cfg, list) else [_alg_cfg]+ for cfg in cfgs:+ if hasattr(cfg, "batch_size") and cfg.batch_size is not None:+ batch_size = cfg.batch_size+ break+ if batch_size is not None and batch_size != 1:+ grad_acc = kwargs.get("gradient_accumulate_steps", 1)+ if _alg_cfg is not None:+ cfgs = _alg_cfg if isinstance(_alg_cfg, list) else [_alg_cfg]+ for cfg in cfgs:+ if hasattr(cfg, "gradient_accumulate_steps") and cfg.gradient_accumulate_steps is not None:+ grad_acc = cfg.gradient_accumulate_steps+ break+ new_grad_acc = batch_size * grad_acc+ kwargs["gradient_accumulate_steps"] = new_grad_acc+ kwargs["batch_size"] = 1+ if _alg_cfg is not None:+ cfgs = _alg_cfg if isinstance(_alg_cfg, list) else [_alg_cfg]+ for cfg in cfgs:+ if hasattr(cfg, "batch_size"):+ cfg.batch_size = 1+ if hasattr(cfg, "gradient_accumulate_steps"):+ cfg.gradient_accumulate_steps = new_grad_acc+ logger.warning(+ f"reset batch_size({batch_size}) to 1 and "+ f"gradient_accumulate_steps to {new_grad_acc} "+ f"because batch_size={batch_size} cannot be used for calibrating non-text modules."+ )++ # Call parent class __init__ (will be CalibCompressor, ImatrixCompressor, etc)+ super().__init__(*args, **kwargs)++ # Mirror old-arch DiffusionCompressor._align_device_and_dtype: unconditionally+ # cast the full diffusion pipeline (VAE, text encoder, etc.) to the transformer's+ # dtype so that calibration's pipe(...) call doesn't crash with dtype mismatches+ # when the transformer is force-cast to bf16 for activation quantization.+ # Note: pipe.dtype only reflects the primary component, so an equality check would+ # miss mixed-dtype pipelines where e.g. the VAE is still float32.+ pipe = getattr(self.model_context, "pipe", None)+ model = getattr(self.model_context, "model", None)+ if pipe is not None and model is not None:+ is_nextstep = hasattr(model, "config") and getattr(model.config, "model_type", None) == "nextstep"+ if not is_nextstep:+ pipe.to(model.dtype)++ def _get_block_forward_func(self, name: str):+ """Diffusion models pass positional args; wrap the base forward func accordingly.++ The MRO guarantees that super() resolves to CalibCompressor._get_block_forward_func,+ mirroring the old-arch pattern in compressors/diffusion/compressor.py.+ """+ return wrap_block_forward_positional_to_kwargs(super()._get_block_forward_func(name))++ def _should_stop_cache_forward(self, name: str) -> bool:+ """Diffusion models must run all denoising steps to collect enough inputs.++ Mirrors old-arch DiffusionCompressor._should_stop_cache_forward which always+ returns False so the pipeline never exits early after the first block hit.+ Without this, CalibCompressor._should_stop_cache_forward would stop after the+ first inference step, yielding only nsamples inputs instead of+ nsamples * num_inference_steps.+ """+ return False++ @torch.no_grad()+ def calib(self, nsamples, bs):+ """Perform diffusion-specific calibration for quantization.++ Override parent's calib method to use diffusion dataset loading logic.+ The diffusion pipeline is read from ``self.model_context.pipe``.+ """+ from auto_round.compressors.diffusion.dataset import get_diffusion_dataloader++ pipe = self.model_context.pipe+ if pipe is None:+ raise ValueError(+ "Diffusion pipeline not found in model_context. " "Ensure the model was loaded as a diffusion model."+ )++ logger.warning(+ "Diffusion model will catch nsamples * num_inference_steps inputs, "+ "you can reduce nsamples or num_inference_steps if OOM or take too much time."+ )+ if isinstance(self.dataset, str):+ dataset = self.dataset.replace(" ", "")+ self.dataloader, self.batch_size, self.gradient_accumulate_steps = get_diffusion_dataloader(+ dataset=dataset,+ bs=self.batch_size,+ seed=self.seed,+ nsamples=self.nsamples,+ gradient_accumulate_steps=self.gradient_accumulate_steps,+ )+ else:+ self.dataloader = self.dataset+ total_cnt = 0++ total = nsamples if not hasattr(self.dataloader, "len") else min(nsamples, len(self.dataloader))++ if (+ hasattr(self.model, "hf_device_map")+ and len(self.model.hf_device_map) > 1+ and pipe.device != self.model.device+ and torch.device(self.model.device).type in ["cuda", "xpu"]+ ):+ logger.error(+ "Diffusion model is activated sequential model offloading, it will crash during moving to GPU/XPU. "+ "Please use model path for quantization or "+ "move the pipeline object to GPU/XPU before passing them into API."+ )+ exit(-1)++ if pipe.device != self.model.device:+ pipe.to(self.model.device)+ with tqdm(range(1, total + 1), desc="cache block inputs") as pbar:+ for ids, prompts in self.dataloader:+ if isinstance(prompts, tuple):+ prompts = list(prompts)+ try:+ pipe(+ prompts,+ guidance_scale=self.guidance_scale,+ num_inference_steps=self.num_inference_steps,+ generator=(+ None+ if self.generator_seed is None+ else torch.Generator(device=pipe.device).manual_seed(self.generator_seed)+ ),+ )+ except NotImplementedError:+ pass+ except Exception as error:+ raise error+ step = len(prompts)+ total_cnt += step+ pbar.update(step)+ if total_cnt >= nsamples:+ break+ if total_cnt == 0:+ logger.error(+ f"no data has been cached, please provide more data with sequence length >={self.seqlen} in the "+ f"dataset or decease the sequence length"+ )+ exit(-1)+ elif total_cnt < nsamples:+ logger.warning(+ f"Insufficient number of samples collected may affect the quantization. "+ f"target samples count is {nsamples}, while valid samples count is {total_cnt}"+ )+ if total_cnt < self.batch_size:+ raise ValueError(+ f"valid samples is less than batch_size({self.batch_size}),"+ " please adjust self.batch_size or seqlen."+ )+ max_len = (total_cnt // self.batch_size) * self.batch_size+ for k, v in self.inputs.items():+ for key in v:+ if isinstance(v[key], list) and len(v[key]) == total_cnt:+ self.inputs[k][key] = v[key][:max_len]++ # torch.cuda.empty_cache()++ def save_quantized(self, output_dir=None, format="auto_round", inplace=True, **kwargs):+ """Save the quantized model to the specified output directory in the specified format.++ Args:+ output_dir (str, optional): The directory to save the quantized model. Defaults to None.+ format (str, optional): The format in which to save the model. Defaults to "auto_round".+ inplace (bool, optional): Whether to modify the model in place. Defaults to True.+ **kwargs: Additional keyword arguments specific to the export format.++ Returns:+ object: The compressed model object.+ """+ if output_dir is None:+ return super().save_quantized(output_dir, format=format, inplace=inplace, **kwargs)++ pipe = self.model_context.pipe+ compressed_model = None+ for name in pipe.components.keys():+ val = getattr(pipe, name)+ sub_module_path = (+ os.path.join(output_dir, name) if os.path.basename(os.path.normpath(output_dir)) != name else output_dir+ )+ if (+ hasattr(val, "config")+ and hasattr(val.config, "_name_or_path")+ and val.config._name_or_path == self.model.config._name_or_path+ ):+ compressed_model = super().save_quantized(+ output_dir=sub_module_path if not self.compress_context.is_immediate_saving else output_dir,+ format=format,+ inplace=inplace,+ **kwargs,+ )+ elif val is not None and hasattr(val, "save_pretrained"):+ val.save_pretrained(sub_module_path)+ pipe.config.save_pretrained(output_dir)+ return compressed_modeldiff --git a/auto_round/compressors_new/entry.py b/auto_round/compressors_new/entry.pynew file mode 100644index 000000000..7d519e007--- /dev/null+++ b/auto_round/compressors_new/entry.py@@ -0,0 +1,525 @@+# # Copyright (C) 2026 Intel Corporation+# # SPDX-License-Identifier: Apache-2.0++import os+from typing import Any, Callable, Optional, Union++import torch++from auto_round.algorithms.alg_config import AlgConfig+from auto_round.algorithms.quantization.rtn.config import RTNConfig+from auto_round.algorithms.quantization.sign_round.config import SignRoundConfig+from auto_round.algorithms.transforms.rotation.config import RotationConfig as _NewArchRotationConfig+from auto_round.auto_scheme.gen_auto_scheme import AutoScheme+from auto_round.compressors_new.calib import CalibCompressor, CalibratedRTNCompressor+from auto_round.compressors_new.utils import check_need_act_calibration+from auto_round.compressors_new.zero_shot import ZeroShotCompressor+from auto_round.logger import logger+from auto_round.schemes import QuantizationScheme, _parse_scheme+++def _preview_resolved_attrs(config, scheme=None) -> dict:+ """Resolve scheme attributes without mutating config, for routing decisions.++ Called in ``AutoRound.__new__`` before the concrete compressor class is+ chosen. ``SchemeMixin.resolve_scheme()`` will do the authoritative+ resolution later; this is just a lightweight preview so routing logic+ (``enable_imatrix``, ``needs_act_calib``, etc.) can use the correct values+ even when the user specified only ``scheme=`` without explicit bit/dtype args.++ Returns:+ dict: resolved attributes (may be empty if scheme cannot be previewed).+ """+ if isinstance(scheme, AutoScheme):+ # AutoScheme needs model info — cannot preview, rely on raw config attrs+ return {}+ scheme_attr_names = QuantizationScheme.get_attributes()+ user_overrides = {k: getattr(config, k) for k in scheme_attr_names if getattr(config, k, None) is not None}+ try:+ _, _, final_attrs = _parse_scheme(scheme, user_overrides)+ return final_attrs+ except Exception:+ return {}+++def _eager_validate_scheme(config, scheme=None) -> None:+ """Eagerly validate scheme/config constraints at construction time.++ Mirrors the old-arch ``_check_configs()`` call in ``BaseCompressor.__init__``.+ Raises ``ValueError`` or ``NotImplementedError`` immediately if the scheme+ contains config-only invalid combinations (e.g. tuple group_size with non-fp8+ weight dtype) so that callers get a fast failure rather than a deferred error+ buried inside ``post_init()``.++ ``AutoScheme`` is skipped because it requires model information.+ """+ if isinstance(scheme, AutoScheme):+ return++ scheme_attr_names = QuantizationScheme.get_attributes()+ user_overrides = {k: getattr(config, k) for k in scheme_attr_names if getattr(config, k, None) is not None}+ try:+ _, _, final_attrs = _parse_scheme(scheme, user_overrides)+ except (ValueError, NotImplementedError):+ raise+ except Exception:+ return # Other parse errors are deferred to post_init++ import copy++ temp_config = copy.copy(config)+ for key, value in final_attrs.items():+ setattr(temp_config, key, value)+ temp_config.check_config() # raises ValueError / NotImplementedError if invalid+++# ---------------------------------------------------------------------------+# Compressor-class registry+# ---------------------------------------------------------------------------+# Maps (model_type, base_class_name) → combined class, created lazily.+_COMPRESSOR_REGISTRY: dict[tuple[str, str], type] = {}+++def _get_compressor_class(model_type: str, base_cls: type) -> type:+ """Return the compressor class for *base_cls* wired with the right model-type Mixin.++ For ``model_type == "llm"`` the bare *base_cls* is returned unchanged.+ For ``"mllm"`` and ``"diffusion"`` the corresponding Mixin is prepended via+ :func:`type` and the result is cached in ``_COMPRESSOR_REGISTRY`` so that+ each ``(model_type, base_cls)`` pair is created at most once per process.+ """+ if model_type == "llm":+ return base_cls+ key = (model_type, base_cls.__name__)+ if key in _COMPRESSOR_REGISTRY:+ return _COMPRESSOR_REGISTRY[key]+ if model_type == "mllm":+ from auto_round.compressors_new.mllm_mixin import MLLMMixin++ mixin = MLLMMixin+ elif model_type == "diffusion":+ from auto_round.compressors_new.diffusion_mixin import DiffusionMixin++ mixin = DiffusionMixin+ else:+ return base_cls+ combined = type(f"{model_type.capitalize()}{base_cls.__name__}", (mixin, base_cls), {})+ _COMPRESSOR_REGISTRY[key] = combined+ return combined+++def is_weight_scheme(scheme):+ if isinstance(scheme, str):+ return scheme.upper().startswith("W")+ if isinstance(scheme, dict):+ return all(isinstance(s, str) and s.upper().startswith("W") for s in scheme.values())+ if isinstance(scheme, AutoScheme):+ opts = scheme.options+ if isinstance(opts, (list, tuple)):+ return all(isinstance(s, str) and s.upper().startswith("W") for s in opts)+ if isinstance(opts, str):+ return opts.upper().startswith("W")+ return False+++def detect_model_type(model):+ """Detect the type of model (LLM, MLLM, or Diffusion).++ Args:+ model: Model instance or model path string++ Returns:+ str: "mllm", "diffusion", or "llm"+ """+ from auto_round.utils import is_diffusion_model, is_mllm_model++ # Check if it's a diffusion model first (more specific)+ if is_diffusion_model(model):+ return "diffusion"++ # Check if it's an MLLM+ if is_mllm_model(model):+ return "mllm"++ # Default to standard LLM+ return "llm"+++class AutoRound(object):+ SKIP_ARGS = ("local_args", "kwargs", "cls", "alg_configs", "quant_config", "quant_configs")++ # Mapping from string alias to config class (and optional defaults override).+ _CONFIG_ALIASES: dict[str, type] = {+ "sign_round": SignRoundConfig,+ "signround": SignRoundConfig,+ "rtn": RTNConfig,+ "hadamard": _NewArchRotationConfig,+ }++ @classmethod+ def _resolve_config(cls, config: Union[str, AlgConfig, list]) -> Union[AlgConfig, list[AlgConfig]]:+ """Convert string alias(es) to the corresponding config instance(s) with default parameters."""+ if isinstance(config, str):+ key = config.strip().lower()+ if key not in cls._CONFIG_ALIASES:+ raise ValueError(f"Unknown config alias '{config}'. " f"Supported: {list(cls._CONFIG_ALIASES.keys())}")+ return cls._CONFIG_ALIASES[key]()+ if isinstance(config, list):+ return [cls._resolve_config(c) for c in config]+ return config++ def __new__(+ cls,+ alg_configs: Union[str, AlgConfig, list[Union[str, AlgConfig]]],+ model: Union[torch.nn.Module, str],+ tokenizer=None,+ platform="hf",+ format=None,+ scheme="W4A16",+ low_gpu_mem_usage: bool = False,+ device_map: Union[str, torch.device, int, dict] = 0,+ enable_torch_compile: bool = False,+ seed: int = 42,+ low_cpu_mem_usage: bool = True,+ layer_config=None,+ nsamples: int = None,+ seqlen: int = None,+ **kwargs,+ ):+ from auto_round.algorithms.quantization.config import QuantizationConfig++ # Resolve string alias(es) to config instance(s) before routing.+ alg_configs = cls._resolve_config(alg_configs)++ # Extract the single QuantizationConfig from a list; validate at most one exists.+ if isinstance(alg_configs, list):+ quant_configs = [c for c in alg_configs if isinstance(c, QuantizationConfig)]+ if len(quant_configs) == 0:+ raise ValueError("At least one QuantizationConfig (SignRoundConfig / RTNConfig) is required.")+ if len(quant_configs) > 1:+ raise ValueError(+ f"Only one QuantizationConfig is allowed, but got {len(quant_configs)}: "+ f"{[type(c).__name__ for c in quant_configs]}"+ )+ quant_config = quant_configs[0]+ else:+ quant_config = alg_configs++ # Eagerly validate scheme constraints that do not require model info.+ # This mirrors old-arch _check_configs() called at __init__ time so that+ # callers get ValueError/NotImplementedError on construction, not deferred.+ _eager_validate_scheme(quant_config, scheme)++ # using different compressor base on AlgConfigs+ local_args = {k: v for k, v in locals().items() if k not in cls.SKIP_ARGS}++ # Detect model type to determine if we need special compressor+ model_type = detect_model_type(model)++ # If the user explicitly passes processor/image_processor, treat as MLLM even if+ # auto-detection missed it (mirrors the has_multimodal_assets check in autoround.py).+ has_multimodal_assets = kwargs.get("processor") is not None or kwargs.get("image_processor") is not None+ if has_multimodal_assets and model_type != "mllm":+ model_type = "mllm"++ if isinstance(quant_config, SignRoundConfig):+ return _get_compressor_class(model_type, CalibCompressor)(alg_configs, **local_args, **kwargs)++ elif isinstance(quant_config, RTNConfig):+ enable_imatrix = False+ disable_opt_rtn = getattr(quant_config, "disable_opt_rtn", False)+ # If disable_opt_rtn was not explicitly set and scheme is W8A16/W8A8,+ # auto-disable optimization to improve efficiency.+ if getattr(quant_config, "orig_disable_opt_rtn", None) is None:+ if isinstance(scheme, str) and scheme.upper() in ["W8A16", "W8A8"]:+ logger.warning("`disable_opt_rtn` is turned on for W8A16/W8A8 quantization to improve efficiency.")+ disable_opt_rtn = True+ quant_config.disable_opt_rtn = True+ if not disable_opt_rtn:+ has_gguf_k = "gguf" in format.lower() and "_k" in format.lower() if format else False+ if has_gguf_k:+ enable_imatrix = True+ else:+ # Resolve scheme attrs for routing (config hasn't been through+ # SchemeMixin yet; user may have specified only scheme="W4A16").+ _resolved = _preview_resolved_attrs(quant_config, scheme)+ _sym = _resolved.get("sym", getattr(quant_config, "sym", None))+ _data_type = _resolved.get("data_type", getattr(quant_config, "data_type", "") or "")+ _bits = _resolved.get("bits", getattr(quant_config, "bits", None))+ if _sym is not None and _sym is False:+ enable_imatrix = False+ elif _data_type == "int" and (_bits is None or _bits < 8):+ enable_imatrix = True+ elif is_weight_scheme(scheme):+ enable_imatrix = True+ else:+ _resolved = {}++ _resolved = _resolved if not disable_opt_rtn else _preview_resolved_attrs(quant_config, scheme)+ _act_bits = _resolved.get("act_bits", getattr(quant_config, "act_bits", None))+ _act_data_type = _resolved.get("act_data_type", getattr(quant_config, "act_data_type", None))+ _act_dynamic = _resolved.get("act_dynamic", getattr(quant_config, "act_dynamic", None))+ _is_act_quantize = _act_bits is not None and _act_bits <= 8+ needs_act_calib = _is_act_quantize and check_need_act_calibration(+ _act_dynamic,+ _act_data_type,+ _act_bits if _act_bits is not None else 16,+ static_kv_dtype=kwargs.get("static_kv_dtype"),+ static_attention_dtype=kwargs.get("static_attention_dtype"),+ )++ # AutoScheme always requires calibration data for delta-loss based+ # scheme selection, regardless of whether imatrix is needed.+ from auto_round.auto_scheme.gen_auto_scheme import AutoScheme as _AutoScheme++ is_auto_scheme = isinstance(scheme, _AutoScheme)++ if enable_imatrix or needs_act_calib or is_auto_scheme:+ quant_config._alg_cls = "OptimizedRTNQuantizer"+ return _get_compressor_class(model_type, CalibratedRTNCompressor)(alg_configs, **local_args, **kwargs)+ else:+ quant_config._alg_cls = "RTNQuantizer"+ return _get_compressor_class(model_type, ZeroShotCompressor)(alg_configs, **local_args, **kwargs)+++class AutoRoundCompatible:+ """AutoRoundCompatible wrapper class for backward compatibility.++ This class provides the same API as the old AutoRoundCompatible class but internally+ uses the new AutoRound architecture with Mixin pattern.++ Args:+ model: Model object or model name to load+ tokenizer: Tokenizer for text processing+ platform: Platform to download model ("hf" or "model_scope")+ scheme: Quantization scheme (str, dict, or QuantizationScheme)+ layer_config: Layer-wise quantization config+ dataset: Calibration data+ iters: Optimization iterations+ seqlen: Calibration sequence length+ nsamples: Number of calibration samples+ batch_size: Calibration batch size+ gradient_accumulate_steps: Gradient accumulation steps+ low_gpu_mem_usage: Lower GPU memory mode+ device_map: Device map for each module+ enable_torch_compile: Enable torch.compile+ seed: Random seed+ low_cpu_mem_usage: Lower CPU memory mode+ **kwargs: Additional arguments (bits, group_size, sym, etc.)++ Example:+ >>> # Old API - still works+ >>> from auto_round.compressors_new.entry import AutoRoundCompatible+ >>> autoround = AutoRoundCompatible(+ ... model="/models/opt-125m",+ ... bits=4,+ ... group_size=128,+ ... iters=200,+ ... )+ >>> quantized_model, layer_config = autoround.quantize()+ """++ SKIP_ARGS = ("local_args", "kwargs", "cls", "config")++ bits: int | None+ group_size: int | None+ sym: bool | None+ data_type: str | None+ act_bits: int | None+ act_group_size: int | None+ act_sym: bool | None+ act_data_type: str | None+ act_dynamic: bool | None+ super_bits: int | None+ super_group_size: int | None++ @staticmethod+ def _pop_config_kwargs(kwargs: dict[str, Any]) -> tuple[dict[str, Any], dict[str, Any]]:+ """Extract old-API config kwargs and split them by config type."""+ common_keys = (+ "ignore_layers",+ "quant_lm_head",+ "scale_dtype",+ "super_bits",+ "super_group_size",+ "to_quant_block_names",+ )+ auto_round_only_keys = (+ "nblocks",+ "enable_alg_ext",+ "lr_scheduler",+ "not_use_best_mse",+ "dynamic_max_gap",+ "optimizer",+ "enable_adam",+ "momentum",+ )+ common_kwargs = {}+ auto_round_kwargs = {}+ for key in common_keys:+ if key in kwargs:+ common_kwargs[key] = kwargs.pop(key)+ for key in auto_round_only_keys:+ if key in kwargs:+ auto_round_kwargs[key] = kwargs.pop(key)+ return common_kwargs, auto_round_kwargs++ def __new__(+ cls,+ model: Union[torch.nn.Module, str],+ tokenizer=None,+ platform: str = "hf",+ scheme: Union[str, dict, QuantizationScheme, AutoScheme] = "W4A16",+ layer_config: dict[str, Union[str, dict, QuantizationScheme]] = None,+ dataset: Union[str, list, tuple, torch.utils.data.DataLoader] = "NeelNanda/pile-10k",+ iters: int = 200,+ seqlen: int = 2048,+ nsamples: int = 128,+ batch_size: int = 8,+ gradient_accumulate_steps: int = 1,+ low_gpu_mem_usage: bool = False,+ device_map: Union[str, torch.device, int, dict] = 0,+ enable_torch_compile: bool = False,+ seed: int = 42,+ low_cpu_mem_usage: bool = True,+ **kwargs,+ ):+ """Create AutoRoundCompatible instance using new AutoRound architecture.++ This method translates old AutoRoundCompatible API to new AutoRound API.+ """+ from auto_round.utils import is_diffusion_model, is_mllm_model++ common_config_kwargs, auto_round_config_kwargs = cls._pop_config_kwargs(kwargs)++ # Extract quantization parameters from kwargs or use defaults+ bits = kwargs.pop("bits", None)+ group_size = kwargs.pop("group_size", None)+ sym = kwargs.pop("sym", None)+ data_type = kwargs.pop("data_type", None)+ act_bits = kwargs.pop("act_bits", None)+ act_group_size = kwargs.pop("act_group_size", None)+ act_sym = kwargs.pop("act_sym", None)+ act_data_type = kwargs.pop("act_data_type", None)+ act_dynamic = kwargs.pop("act_dynamic", None)++ # Decide which algorithm to use+ if iters == 0:+ # RTN mode+ disable_opt_rtn = kwargs.pop("disable_opt_rtn", None)+ config = RTNConfig(+ bits=bits,+ group_size=group_size,+ sym=sym,+ data_type=data_type,+ act_bits=act_bits,+ act_group_size=act_group_size,+ act_sym=act_sym,+ act_data_type=act_data_type,+ act_dynamic=act_dynamic,+ disable_opt_rtn=disable_opt_rtn,+ # for optRTN+ batch_size=batch_size,+ **common_config_kwargs,+ )+ else:+ # AutoRoundCompatible mode+ lr = kwargs.pop("lr", None)+ minmax_lr = kwargs.pop("minmax_lr", None)+ enable_minmax_tuning = kwargs.pop("enable_minmax_tuning", True)+ enable_norm_bias_tuning = kwargs.pop("enable_norm_bias_tuning", False)+ enable_quanted_input = kwargs.pop("enable_quanted_input", True)++ config = SignRoundConfig(+ iters=iters,+ batch_size=batch_size,+ gradient_accumulate_steps=gradient_accumulate_steps,+ bits=bits,+ group_size=group_size,+ sym=sym,+ data_type=data_type,+ act_bits=act_bits,+ act_group_size=act_group_size,+ act_sym=act_sym,+ act_data_type=act_data_type,+ act_dynamic=act_dynamic,+ lr=lr,+ minmax_lr=minmax_lr,+ enable_minmax_tuning=enable_minmax_tuning,+ enable_norm_bias_tuning=enable_norm_bias_tuning,+ enable_quanted_input=enable_quanted_input,+ **common_config_kwargs,+ **auto_round_config_kwargs,+ )++ # Determine output format if specified+ format = kwargs.pop("format", None)++ # Extract rotation_config (old-API kwarg) and thread it into alg_configs.+ # In old arch this was a standalone keyword arg; the new arch passes rotation+ # transforms as part of the alg_configs list. All backends (auto / inplace /+ # transform) are dispatched inside ``HadamardRotation.apply_to_model``.+ _rotation_config_raw = kwargs.pop("rotation_config", None)+ if _rotation_config_raw is not None:+ if isinstance(_rotation_config_raw, _NewArchRotationConfig):+ _rc = _rotation_config_raw+ elif isinstance(_rotation_config_raw, dict):+ _rc = _NewArchRotationConfig.model_validate(_rotation_config_raw)+ else:+ # str alias ("default", "random_hadamard", …) -> default config+ _rc = _NewArchRotationConfig()+ config = [config, _rc]++ # Extract MLLM-specific parameters+ processor = kwargs.pop("processor", None)+ image_processor = kwargs.pop("image_processor", None)+ template = kwargs.pop("template", None)+ extra_data_dir = kwargs.pop("extra_data_dir", None)+ quant_nontext_module = kwargs.pop("quant_nontext_module", False)++ # Extract Diffusion-specific parameters+ guidance_scale = kwargs.pop("guidance_scale", 7.5)+ num_inference_steps = kwargs.pop("num_inference_steps", 50)+ generator_seed = kwargs.pop("generator_seed", None)++ # Check model type for logging+ if is_mllm_model(model, platform=platform):+ logger.info("Using MLLM mode for multimodal model (new architecture).")+ elif is_diffusion_model(model):+ logger.info("Using Diffusion mode for diffusion model (new architecture).")+ else:+ logger.info("Using LLM mode (new architecture).")++ # Create AutoRound instance using new architecture+ compressor = AutoRound(+ alg_configs=config,+ model=model,+ tokenizer=tokenizer,+ platform=platform,+ format=format,+ scheme=scheme,+ dataset=dataset,+ iters=iters,+ low_gpu_mem_usage=low_gpu_mem_usage,+ device_map=device_map,+ enable_torch_compile=enable_torch_compile,+ seed=seed,+ low_cpu_mem_usage=low_cpu_mem_usage,+ layer_config=layer_config,+ nsamples=nsamples,+ seqlen=seqlen,+ # MLLM parameters+ processor=processor,+ image_processor=image_processor,+ template=template,+ extra_data_dir=extra_data_dir,+ quant_nontext_module=quant_nontext_module,+ # Diffusion parameters+ guidance_scale=guidance_scale,+ num_inference_steps=num_inference_steps,+ generator_seed=generator_seed,+ # Pass remaining kwargs+ **kwargs,+ )++ return compressordiff --git a/auto_round/compressors_new/mllm_mixin.py b/auto_round/compressors_new/mllm_mixin.pynew file mode 100644index 000000000..ada1eacf5--- /dev/null+++ b/auto_round/compressors_new/mllm_mixin.py@@ -0,0 +1,281 @@+# Copyright (c) 2026 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.++import torch++from auto_round.logger import logger+from auto_round.utils import to_device+++class MLLMMixin:+ """MLLM-specific functionality mixin.++ This mixin adds MLLM-specific functionality to any compressor (CalibCompressor,+ ZeroShotCompressor, ImatrixCompressor, etc). It handles multi-modal models+ (vision-language models) that require special data loading and processing logic.++ Can be combined with:+ - CalibCompressor (for AutoRound with calibration)+ - ImatrixCompressor (for RTN with importance matrix)+ - ZeroShotCompressor (for basic RTN)++ MLLM-specific parameters:+ processor: Multi-modal processor override (normally loaded by ModelContext)+ image_processor: Image processor override (e.g. for LLaVA)+ template: Template name for processing different MLLMs+ extra_data_dir: Path to extra data (images, audio, videos)+ quant_nontext_module: Whether to quantize non-text modules++ Design note:+ ``ModelContext._load_model()`` is responsible for loading the model and its+ associated artifacts (processor, tokenizer, image_processor). This mixin+ reads those artifacts from ``self.model_context`` during calibration.+ If the caller passes explicit ``processor`` / ``image_processor`` overrides,+ they are written into ``model_context`` after ``super().__init__()`` so that+ ``model_context`` remains the single source of truth.+ """++ def __init__(+ self,+ *args,+ processor=None,+ image_processor=None,+ template=None,+ extra_data_dir=None,+ quant_nontext_module=False,+ **kwargs,+ ):+ self.template = template+ self.extra_data_dir = extra_data_dir+ self.quant_nontext_module = quant_nontext_module+ self.template_obj = None++ # Pass quant_nontext_module to ModelContext so get_block_names can include vision blocks+ kwargs.setdefault("quant_nontext_module", quant_nontext_module)++ # Mirror old arch: reset batch_size to 1 when quantizing non-text modules,+ # because vision encoder blocks have non-standard hidden_states shapes that+ # break batch_dim detection, and image collation fails with batch_size > 1.+ if quant_nontext_module:+ # batch_size may come from kwargs (placed there by AutoRoundCompatible local_args)+ # or from the AlgConfig object in args[0] (the authoritative source for quantizer.batch_size).+ # We must update both so that quantizer.batch_size is also reset to 1.+ batch_size = kwargs.get("batch_size", None)+ _alg_cfg = args[0] if args else None+ if batch_size is None and _alg_cfg is not None:+ cfgs = _alg_cfg if isinstance(_alg_cfg, list) else [_alg_cfg]+ for cfg in cfgs:+ if hasattr(cfg, "batch_size") and cfg.batch_size is not None:+ batch_size = cfg.batch_size+ break+ if batch_size is not None and batch_size != 1:+ grad_acc = kwargs.get("gradient_accumulate_steps", 1)+ new_grad_acc = batch_size * grad_acc+ kwargs["gradient_accumulate_steps"] = new_grad_acc+ kwargs["batch_size"] = 1+ # Also patch the AlgConfig object so that BaseCompressor.quantize_config.batch_size == 1+ if _alg_cfg is not None:+ cfgs = _alg_cfg if isinstance(_alg_cfg, list) else [_alg_cfg]+ for cfg in cfgs:+ if hasattr(cfg, "batch_size"):+ cfg.batch_size = 1+ if hasattr(cfg, "gradient_accumulate_steps"):+ cfg.gradient_accumulate_steps = new_grad_acc+ logger.warning(+ f"reset batch_size({batch_size}) to 1 and "+ f"gradient_accumulate_steps to {new_grad_acc} "+ f"because batch_size={batch_size} cannot be used for calibrating non-text modules."+ )++ # super().__init__() creates model_context, which eagerly loads the model and+ # populates model_context.processor / image_processor / tokenizer.+ super().__init__(*args, **kwargs)++ # Apply user-provided overrides into model_context (single source of truth).+ if processor is not None:+ self.model_context.processor = processor+ if image_processor is not None:+ self.model_context.image_processor = image_processor++ @torch.no_grad()+ def calib(self, nsamples, bs):+ """Perform MLLM-specific calibration for quantization.++ Override parent's calib method to use MLLM dataset loading logic.+ All multimodal artifacts are read from ``self.model_context``.+ """+ from transformers import PreTrainedModel++ from auto_round.compressors.mllm.dataset import get_mllm_dataloader+ from auto_round.compressors.mllm.template import get_template+ from auto_round.special_model_handler import MISTRAL_3_2_MODELS++ mc = self.model_context+ processor = mc.processor+ image_processor = mc.image_processor+ tokenizer = mc.tokenizer++ # Handle template selection+ if isinstance(mc.model, PreTrainedModel):+ model_type = getattr(mc.model.config, "model_type", None)+ if model_type == "llava" and self.template is None:+ self.template = "default"++ if hasattr(mc.model, "name_or_path"):+ name = mc.model.name_or_path+ if any([m in name for m in MISTRAL_3_2_MODELS]):+ self.template = "mistral3_2"++ template_name = self.template+ if template_name is None and hasattr(mc.model.config, "model_type"):+ template_name = mc.model.config.model_type+ if template_name is None:+ template_name = "default"++ self.template_obj = get_template(+ template_name,+ model=mc.model,+ tokenizer=tokenizer,+ processor=processor,+ image_processor=image_processor,+ use_rtn=getattr(self.quantize_config, "iters", None) == 0,+ quiet=not self.quant_nontext_module,+ )++ logger.info(f"Using MLLM template: {template_name}")++ dataset = self.dataset.replace(" ", "") if isinstance(self.dataset, str) else self.dataset+ if dataset is None:+ dataset = self.template_obj.default_dataset++ if isinstance(self.dataset, str):+ dataset = self.dataset.replace(" ", "")+ # Mirror old arch __init__: switch text-only dataset to MLLM dataset when+ # quant_nontext_module=True, as text datasets cannot calibrate vision modules.+ from auto_round.calib_dataset import CALIB_DATASETS++ if self.quant_nontext_module and dataset in CALIB_DATASETS:+ logger.warning(+ "Text only dataset cannot be used for calibrating non-text modules,"+ " switching to liuhaotian/llava_conv_58k"+ )+ dataset = "liuhaotian/llava_conv_58k"+ (+ self.dataloader,+ self.batch_size,+ self.seqlen,+ self.gradient_accumulate_steps,+ ) = get_mllm_dataloader(+ template=self.template_obj,+ model=mc.model,+ tokenizer=tokenizer,+ processor=processor,+ image_processor=image_processor,+ dataset=dataset,+ extra_data_dir=self.extra_data_dir,+ seqlen=self.seqlen,+ bs=bs,+ seed=self.seed,+ nsamples=nsamples,+ quant_nontext_module=self.quant_nontext_module,+ )+ else:+ self.dataloader = self.dataset++ # Process data through the model for calibration+ total_cnt = 0+ for data in self.dataloader:+ if data is None:+ continue++ try:+ if isinstance(data, str):+ # List-of-strings dataset: process through template → model inputs+ processed = self.template_obj.processor.get_input(+ text=data, images=None, max_length=self.seqlen, squeeze=False+ )+ data_new = {k: to_device(v, mc.model.device) for k, v in processed.items()}+ elif isinstance(data, dict) and "text" in data:+ # FakeDataLoader-style {"text": ..., "image": ...}: process through template+ text = data["text"]+ if isinstance(text, dict):+ text = [text]+ input_text = self.template_obj._encode(text)+ processed = self.template_obj.processor.get_input(+ text=input_text,+ images=data.get("image", None),+ max_length=self.seqlen,+ squeeze=False,+ )+ data_new = {}+ for key, value in processed.items():+ tensor_val = value if isinstance(value, torch.Tensor) else torch.as_tensor(value)+ data_new[key] = to_device(tensor_val, mc.model.device)+ elif isinstance(data, dict):+ data_new = {+ key: value.to(mc.model.device) if isinstance(value, torch.Tensor) else value+ for key, value in data.items()+ }+ else:+ data_new = data++ if isinstance(data_new, dict):+ mc.model(**data_new)+ else:+ mc.model(data_new)+ except NotImplementedError:+ pass+ except Exception as e:+ logger.warning(f"Calibration forward pass failed: {e}")+ continue++ total_cnt += bs+ if total_cnt >= nsamples:+ break++ if total_cnt == 0:+ logger.error("no data has been cached, please provide more data")+ exit(-1)+ elif total_cnt < nsamples:+ logger.warning(f"Insufficient number of samples: required {nsamples}, but only {total_cnt} were processed.")++ def save_quantized(self, output_dir=None, format="auto_round", inplace=True, **kwargs):+ """Save the quantized model to the specified output directory in the specified format.++ Args:+ output_dir (str, optional): The directory to save the quantized model. Defaults to None.+ format (str, optional): The format in which to save the model. Defaults to "auto_round".+ inplace (bool, optional): Whether to modify the model in place. Defaults to True.+ **kwargs: Additional keyword arguments specific to the export format.++ Returns:+ object: The compressed model object.+ """+ mc = self.model_context+ processor = mc.processor+ image_processor = mc.image_processor+ tokenizer = mc.tokenizer++ if processor is not None and not hasattr(processor, "chat_template"):+ processor.chat_template = None+ compressed_model = super().save_quantized(+ output_dir=output_dir,+ format=format,+ inplace=inplace,+ processor=processor,+ image_processor=image_processor,+ quant_nontext_module=self.quant_nontext_module if hasattr(self, "quant_nontext_module") else False,+ **kwargs,+ )+ return compressed_modeldiff --git a/auto_round/compressors_new/shard_writer.py b/auto_round/compressors_new/shard_writer.pynew file mode 100644index 000000000..dbdd2cc86--- /dev/null+++ b/auto_round/compressors_new/shard_writer.py@@ -0,0 +1,329 @@+# Copyright (c) 2026 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.++import json+import os+from collections import OrderedDict++import torch++from auto_round.compressors_new.utils import _get_save_folder_name+from auto_round.context.compress import CompressContext+from auto_round.context.model import ModelContext+from auto_round.logger import logger+from auto_round.utils import get_lm_head_name, get_module+++class ShardWriter:+ """+ Handles shard-saving of model parameters to disk with memory management.+ """++ _instance = None+ _initialized = False++ model = None+ lm_head_name = None++ def __new__(cls, *args, **kwargs):+ if cls._instance is None:+ cls._instance = super().__new__(cls)+ cls._instance._data = {}+ return cls._instance++ def __init__(+ self,+ model,+ bits,+ max_shard_size=None,+ safe_serialization=True,+ ):+ if ShardWriter._initialized:+ return+ self.model = model+ self.lm_head_name = get_lm_head_name(self.model)+ total_params = sum(p.numel() for p in self.model.parameters())+ # Heuristic estimate of model size in GB used to choose a default max_shard_size:+ # - total_params * rounder.bits -> total number of bits in all parameters+ # - // 8 -> convert bits to bytes+ # - // 1e9 -> approx convert bytes to GB (1e9 bytes ~= 1 GB)+ # - final // 10 -> apply a safety margin so default shards are+ # smaller than the full model; this intentionally+ # underestimates size before clamping below.+ max_split_num = 10+ model_size = int(total_params * bits // 1e9 // 8 + max_split_num - 1) / max_split_num+ model_size = max(1, min(int(model_size), 5))++ # Configuration+ max_shard_size = max_shard_size or f"{model_size}GB"+ self.max_shard_size = self._parse_size(max_shard_size)+ self.safe_serialization = safe_serialization++ # Internal State+ self.use_safetensors = self._check_safetensors()+ self.shard_suffix = "safetensors" if self.use_safetensors else "bin"+ self.current_shard_tensors = OrderedDict()+ self.current_shard_size = 0+ self.shard_meta = [] # List of {tmp_file: str, params: list}+ self.global_weight_map = {}+ self.shard_counter = 0++ # Persistent set of all parameter names already flushed to a shard file.+ # Maintained incrementally in _flush_shard to avoid O(N^2) rebuilds in _add_tensor.+ self._all_saved = set()++ # Stats+ self.total_param_elems = 0+ self.total_param_size_bytes = 0+ self.skipped_meta_tensors = []++ ShardWriter._initialized = True++ @property+ def output_dir(self) -> str:+ """Derive the output directory from the current CompressContext at access time.++ Reading from context rather than caching the path at construction time ensures+ the ShardWriter always uses the final export directory even if+ ``CompressContext.output_dir`` is updated after the ShardWriter was created+ (e.g. by ``_get_export_dir()`` in ``quantize_and_save()``).+ """+ compress_context = CompressContext.get_context()+ formats = compress_context.formats+ base_dir = _get_save_folder_name(formats[0])+ subfolder = getattr(self.model, "_autoround_pipeline_subfolder", None)+ if subfolder:+ base_dir = os.path.join(base_dir, subfolder)+ return os.path.join(base_dir, "")++ @classmethod+ def reset(cls):+ """Reset the singleton state so the next instantiation creates a fresh ShardWriter."""+ cls._initialized = False+ cls._instance = None++ @classmethod+ def get_shard_writer(cls, *args, **kwargs):+ """Return the current singleton instance, or None if not yet initialized.++ Callers that require a valid writer should guard the result with+ ``if self.compress_context.is_immediate_saving`` before use.+ """+ return cls._instance++ def _parse_size(self, size_str: str) -> int:+ if isinstance(size_str, int):+ return size_str+ s = size_str.strip().upper()+ units = {"GB": 1024**3, "MB": 1024**2, "KB": 1024, "B": 1}+ for unit, mult in units.items():+ if s.endswith(unit):+ return int(float(s[: -len(unit)]) * mult)+ return int(s)++ def _check_safetensors(self) -> bool:+ if self.safe_serialization:+ try:+ import safetensors.torch++ return True+ except ImportError:+ logger.warning("safetensors not installed; falling back to torch.save.")+ return False++ def save_module(self, m: torch.nn.Module, name: str = None):+ """Extracts and accumulates tensors from a module."""+ prefix = name if name is not None else getattr(m, "global_name", "model")+ sd = m.state_dict()++ for k, v in sd.items():+ if not isinstance(v, torch.Tensor):+ continue+ param_name = f"{prefix}.{k}"+ self._add_tensor(param_name, v)++ def _add_tensor(self, name: str, tensor: torch.Tensor):+ if isinstance(tensor, torch.Tensor) and tensor.device.type == "meta":+ self.skipped_meta_tensors.append(name)+ return++ # Guard against duplicate saving of the same parameter+ if name in self._all_saved or name in self.current_shard_tensors:+ return++ t_size = tensor.nbytes+ self.total_param_elems += tensor.numel()+ self.total_param_size_bytes += t_size+ tensor = tensor.detach().cpu()+ # If single tensor exceeds limit, flush current, save it solo, then continue+ if t_size > self.max_shard_size:+ self._flush_shard()+ self.current_shard_tensors[name] = tensor+ self.current_shard_size = t_size+ self._flush_shard()+ # If adding exceeds limit, flush first+ elif self.current_shard_size + t_size > self.max_shard_size and self.current_shard_size > 0:+ self._flush_shard()+ self.current_shard_tensors[name] = tensor+ self.current_shard_size = t_size+ else:+ self.current_shard_tensors[name] = tensor+ self.current_shard_size += t_size++ def _handle_tied_weights(self):+ """+ Detects tied weights in the current shard and ensures they are only saved once.+ This is done by tracking storage pointers of tensors and skipping duplicates.+ """+ storage_map = set()+ filtered_tensors = {}++ for name, tensor in self.current_shard_tensors.items():+ if not isinstance(tensor, torch.Tensor):+ filtered_tensors[name] = tensor+ continue++ ptr = tensor.untyped_storage().data_ptr() + tensor.storage_offset() * tensor.element_size()+ if ptr not in storage_map:+ storage_map.add(ptr)+ filtered_tensors[name] = tensor+ self.current_shard_tensors = filtered_tensors++ def _flush_shard(self):+ if not self.current_shard_tensors:+ return++ self.shard_counter += 1+ output_dir = self.output_dir+ os.makedirs(output_dir, exist_ok=True)+ tmp_name = f"model-shard-{self.shard_counter:05d}.{self.shard_suffix}"+ tmp_path = os.path.join(output_dir, tmp_name)+ self._handle_tied_weights()++ if self.use_safetensors:+ from safetensors.torch import save_file++ # Ensure tensors are contiguous in-place to avoid duplicating them in a separate dict,+ # which can increase peak RAM usage during saving.+ for k, v in list(self.current_shard_tensors.items()):+ if isinstance(v, torch.Tensor) and not v.is_contiguous():+ self.current_shard_tensors[k] = v.contiguous()+ save_file(self.current_shard_tensors, tmp_path)+ else:+ torch.save(self.current_shard_tensors, tmp_path)++ saved_params = list(self.current_shard_tensors.keys())+ self.shard_meta.append({"tmp_file": tmp_name, "params": saved_params, "dir": output_dir})+ self._all_saved.update(saved_params)++ # Offload logic: move modules to meta device once all params are saved+ self._offload_to_meta(saved_params)++ self.current_shard_tensors = OrderedDict()+ self.current_shard_size = 0++ def _offload_to_meta(self, saved_params):+ """Attempts to move fully saved modules to the 'meta' device to free RAM."""+ for param_full_name in saved_params:+ module_path = param_full_name.rsplit(".", 1)[0]++ module = get_module(self.model, module_path)+ # Check if all parameters of this module are now in '_all_saved'+ if (+ module is not None+ and isinstance(module, torch.nn.Module)+ and all(f"{module_path}.{k}" in self._all_saved for k in module.state_dict().keys())+ ):+ module.to("meta")++ def finalize(self):+ """Saves remaining weights, renames files, and writes the index JSON."""+ # 1. Capture remaining weights not yet saved+ full_sd = self.model.state_dict()+ tie_word_embeddings = False+ if hasattr(self.model, "config") and hasattr(self.model.config, "tie_word_embeddings"):+ tie_word_embeddings = self.model.config.tie_word_embeddings++ finalize_skipped_meta_tensors = []+ for pname, tensor in full_sd.items():+ if pname in self._all_saved:+ continue+ if tensor.device.type == "meta":+ continue+ layer_name = ".".join(pname.split(".")[:-1])+ if self.lm_head_name is not None and layer_name == self.lm_head_name and tie_word_embeddings:+ lm_head_module = get_module(self.model, self.lm_head_name)+ lm_head_module.to("meta") # Must to meta, otherwise model's saver will dump it again+ continue+ self._add_tensor(pname, tensor.detach().to("cpu"))++ self._flush_shard()++ total_skipped = len(self.skipped_meta_tensors) + len(finalize_skipped_meta_tensors)+ if total_skipped > 0:+ examples = (self.skipped_meta_tensors + finalize_skipped_meta_tensors)[:5]++ # 2. Rename temp files to HF standard and map weights+ if self.shard_counter == 0:+ logger.warning("No tensors saved.")+ return++ output_dir = self.output_dir+ for idx, meta in enumerate(self.shard_meta, start=1):+ shard_dir = meta.get("dir", output_dir)+ old_path = os.path.join(shard_dir, meta["tmp_file"])+ new_name = (+ f"model.{self.shard_suffix}"+ if self.shard_counter == 1+ else f"model-{idx:05d}-of-{self.shard_counter:05d}.{self.shard_suffix}"+ )+ new_path = os.path.join(shard_dir, new_name)+ os.rename(old_path, new_path)+ for p in meta["params"]:+ self.global_weight_map[p] = new_name++ # 3. Write Index JSON+ index_ext = "safetensors.index.json" if self.use_safetensors else "bin.index.json"+ index_path = os.path.join(output_dir, f"model.{index_ext}")++ index_data = {+ "metadata": {+ "format": "safetensors" if self.use_safetensors else "pytorch",+ "total_shards": self.shard_counter,+ "total_parameters": int(self.total_param_elems),+ "total_size": int(self.total_param_size_bytes),+ },+ "weight_map": self.global_weight_map,+ }++ if self.shard_counter > 1:+ with open(index_path, "w", encoding="utf-8") as f:+ json.dump(index_data, f, indent=2)++ logger.info(f"model has been saved to {self.output_dir}")++ @torch.no_grad()+ def write(self, m: torch.nn.Module = None, name: str = None, is_finalize: bool = False):+ if m is None and name is None and not is_finalize and not is_finalize:+ raise ValueError("Must specify either name or m")+ if m is None and name is not None:+ m = get_module(self.model, name)+ # Perform the save+ if m is not None:+ self.save_module(m, name)++ if is_finalize:+ self.finalize()+ ShardWriter._initialized = False+ ShardWriter._instance = Nonediff --git a/auto_round/compressors_new/utils.py b/auto_round/compressors_new/utils.pynew file mode 100644index 000000000..fcebb45d7--- /dev/null+++ b/auto_round/compressors_new/utils.py@@ -0,0 +1,1246 @@+# Copyright (c) 2025 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+import copy+import json+import os+import random+import re+import sys+from dataclasses import asdict, fields+from enum import Enum+from typing import TYPE_CHECKING, Callable, Union++import torch+import transformers+from torch.amp import autocast++from auto_round.export.export_to_gguf.config import GGML_QUANT_SIZES, GGUF_CONFIG, GGUF_INNER_CONFIG, QK_K, ModelType+from auto_round.logger import logger+from auto_round.utils import check_to_quantized, get_layer_names_in_block, get_module, to_standard_regex++if TYPE_CHECKING:+ from auto_round.schemes import QuantizationScheme+++class BackendDataType(str, Enum):+ STANDARD_FP = "fp"+ MX_FP = "mx_fp"+ NV_FP = "nv_fp"+ MX_INT = "mx_int"+++def is_standard_fp(backend):+ backend = backend.lower()+ return BackendDataType.STANDARD_FP in backend and not is_mx_fp(backend) and not is_nv_fp(backend)+++def is_mx_fp(backend):+ backend = backend.lower()+ return BackendDataType.MX_FP in backend+++def is_mx_int(backend):+ backend = backend.lower()+ return BackendDataType.MX_INT in backend+++def is_nv_fp(backend):+ backend = backend.lower()+ return BackendDataType.NV_FP in backend+++def is_wint_woq(ar):+ """Returns True for integer weight-only quantization with non-quantized activations (`act_bits >= 16`)."""+ return "int" in ar.data_type and ar.act_bits >= 16 and ar.super_group_size is None+++def is_wint_a16(ar):+ """Backward-compatible alias for `is_wint_woq()`."""+ return is_wint_woq(ar)+++def _is_weight_fp8_activation_static_fp8(+ bit: int, group_size: int, sym: bool, data_type: str, act_dynamic: bool+) -> bool:+ return bit == 8 and group_size == -1 and sym and data_type == "fp" and not act_dynamic+++def is_wfp8afp8(ar):+ if (+ ("fp8" in ar.act_data_type or ("fp" in ar.act_data_type and ar.act_bits == 8))+ and ("fp8" in ar.data_type or ("fp" in ar.data_type and ar.bits == 8))+ and is_standard_fp(ar.act_data_type)+ and is_standard_fp(ar.data_type)+ ):+ return True+ else:+ return False+++def is_wint8aint8(ar):+ if ("int8" in ar.act_data_type or ("int" in ar.act_data_type and ar.act_bits == 8)) and (+ "int8" in ar.data_type or ("int" in ar.data_type and ar.bits == 8)+ ):+ return True+ else:+ return False+++def is_static_wfp8afp8(ar_or_format: Union[str, Callable]) -> bool:+ if isinstance(ar_or_format, str):+ return "fp8_static" in ar_or_format.lower()+ if ar_or_format.act_dynamic:+ return False+ if is_wfp8afp8(ar_or_format):+ return True+ return False+++def is_dynamic_wint8aint8(ar_or_format: Union[str, Callable]) -> bool:+ if isinstance(ar_or_format, str):+ return "int8_w8a8" in ar_or_format.lower()+ if not ar_or_format.act_dynamic:+ return False+ if is_wint8aint8(ar_or_format):+ return True+ return False+++def is_dynamic_afp8(ar_or_format: Callable) -> bool:+ return ar_or_format.act_dynamic and ar_or_format.act_data_type.startswith("fp") and ar_or_format.act_bits == 8+++def is_block_wfp8(ar_or_format: Callable) -> bool:+ return (+ isinstance(ar_or_format.group_size, tuple)+ and len(ar_or_format.group_size) == 2+ and ar_or_format.data_type.startswith("fp")+ and ar_or_format.bits == 8+ )+++def block_forward(+ block: torch.nn.Module,+ input_ids: torch.Tensor,+ input_others: dict,+ amp: bool = False,+ amp_dtype: torch.dtype = torch.float16,+ device: torch.device = torch.device("cpu"),+ output_return_id: int = 0,+) -> Union[torch.Tensor, dict]:+ """Performs a forward pass through a block with the given inputs.++ Args:+ block: The block to perform the forward pass on.+ input_ids: The input IDs.+ input_others: A dictionary containing other input data.+ amp: A boolean indicating whether to use automatic mixed precision.+ amp_dtype: The data type for automatic mixed precision.+ device: The target device.+ output_return_id: if the output has more than one tenor, return the specified idx tensor.++ Returns:+ output: The output of the forward pass.+ """+ from auto_round.utils.model import to_device++ if input_ids.device != device:+ input_ids = to_device(input_ids, device)+ input_others = to_device(input_others, device)+ input_tuple = input_others.pop("positional_inputs", None)+ if "alibi" in input_others.keys() and input_others["alibi"] is not None:+ alibi = input_others["alibi"]+ input_others["alibi"] = alibi.reshape(-1, alibi.shape[2], alibi.shape[3])+ if amp:+ with autocast(device_type=str(device).split(":")[0], dtype=amp_dtype): # pragma: no cover+ output = block(input_ids, *input_tuple, **input_others)+ else:+ output = block(input_ids, *input_tuple, **input_others)+ if isinstance(output_return_id, int) and (isinstance(output, list) or isinstance(output, tuple)):+ output = output[output_return_id]+ return output+++def check_skippable_keywords(key):+ """+ Prints a reminder if a key is not stored during quantization fine-tuning.+ """+ skippable_cache_keys = ("past_key_value",)+ for cache_key in skippable_cache_keys:+ if cache_key not in key:+ return True+ return False+++def check_need_act_calibration(+ is_act_dynamic: Union[bool, None],+ act_data_type: Union[str, None] = None,+ act_bits: Union[int, None] = 16,+ static_kv_dtype: Union[str, None] = None,+ static_attention_dtype: Union[str, None] = None,+) -> bool:+ if static_kv_dtype is not None or static_attention_dtype is not None:+ return True+ if act_bits is None or act_bits > 8:+ return False+ # None is dynamic+ if is_act_dynamic is not None and not is_act_dynamic:+ return True+ if act_data_type is not None and "static" in act_data_type:+ return True+ return False+++def collect_best_params(block, cache_device="cpu"):+ """Collect the best parameters from the block to the specified device."""+ params = {}+ if hasattr(block, "orig_layer"):+ for key in block.params.keys():+ params[key] = block.params[key].data.to(cache_device, copy=True)+ else:+ for n, m in block.named_modules():+ if hasattr(m, "orig_layer"):+ params[n] = {}+ for key in m.params.keys():+ params[n][key] = m.params[key].data.to(cache_device, copy=True)+ return params+++def infer_bits_by_data_type(data_type: str):+ """Infer bits by data_type++ Args:+ data_type (str): data_type++ Returns:+ int: bits inferred by data_type, None means cannot infer correct bits by data_type+ """+ from auto_round.utils import SUPPORTED_DTYPES++ if data_type is None:+ return 16+ for supported_dtype in SUPPORTED_DTYPES:+ if data_type.startswith(supported_dtype) and len(data_type) > len(supported_dtype):+ ##first check the following two bits+ suc_2str = data_type[len(supported_dtype) : len(supported_dtype) + 2]+ if str.isdigit(suc_2str):+ return int(suc_2str)+ if str.isdigit(data_type[len(supported_dtype)]):+ return int(data_type[len(supported_dtype)])+ return None+++def _get_safetensor_layer_names_not_in_model(model, all_module_names: list) -> list:+ """Collect layer names from safetensor files that are not loaded into the model.++ Some tensors (e.g. MTP layers) exist in the original checkpoint but are not+ instantiated by ``transformers``. This function discovers them so that regex+ patterns in ``layer_config`` can still match them.++ Returns:+ List of layer names (the path without the ``.weight`` suffix) for weight+ tensors present in the safetensor files but absent from *all_module_names*.+ """+ name_or_path = None+ if hasattr(model, "config") and hasattr(model.config, "name_or_path"):+ name_or_path = model.config.name_or_path+ if not name_or_path:+ return []++ if not os.path.isdir(name_or_path):+ try:+ from auto_round.utils.model import download_hf_model++ name_or_path = download_hf_model(name_or_path)+ except Exception as e:+ logger.debug(f"Could not resolve source model path to check for missing tensors: {e}")+ return []++ try:+ from safetensors import safe_open+ except ImportError:+ return []++ # Build tensor-name list from the safetensors index or single file+ source_index_file = os.path.join(name_or_path, "model.safetensors.index.json")+ source_single_file = os.path.join(name_or_path, "model.safetensors")++ tensor_names: list = []+ if os.path.exists(source_index_file):+ with open(source_index_file) as f:+ src_index = json.load(f)+ tensor_names = list(src_index["weight_map"].keys())+ elif os.path.exists(source_single_file):+ with safe_open(source_single_file, framework="pt", device="cpu") as f:+ tensor_names = list(f.keys())+ else:+ return []++ module_name_set = set(all_module_names)+ extra_layer_names = []+ for tensor_name in tensor_names:+ if not tensor_name.endswith(".weight"):+ continue+ layer_name = tensor_name[: -len(".weight")]+ if layer_name not in module_name_set:+ extra_layer_names.append(layer_name)+ return extra_layer_names+++def set_layer_config(+ model: torch.nn.Module,+ layer_config: dict[str, Union[str, dict, "QuantizationScheme"]],+ default_scheme: Union[str, "QuantizationScheme"],+ default_scale_dtype: torch.dtype | str,+ supported_types: tuple,+ inner_supported_types: tuple,+ quant_block_list=None,+ ignore_layers: str = "",+ quant_lm_head: bool = False,+ enable_gguf_official_mixed: bool = True,+ is_mllm: bool = False,+ fill_default_value=True,+ gguf_format_name: str = None,+) -> tuple[dict, bool, dict]:+ """+ Normalize, validate, and expand layer-specific quantization configs.+ Returns (final_layer_config, has_quant_layer_outside_block)+ """++ from auto_round.schemes import QuantizationScheme, get_gguf_scheme, preset_name_to_scheme+ from auto_round.utils.model import get_layer_names_in_block, get_lm_head_name, get_module, is_separate_lm_head++ # ---- helpers -------------------------------------------------+ def dispatch_layer_config(layer_config: dict[str, dict]) -> None:+ """Assign scheme values as attributes to matched modules."""+ for layer_name, scheme in layer_config.items():+ module = get_module(model, layer_name)+ if module is None:+ # Layer exists in safetensor files but is not loaded into the model+ # (e.g. MTP layers that transformers does not instantiate). Skip.+ continue+ for attr, value in scheme.items():+ setattr(module, attr, value)++ def normalize_item(item: Union[str, dict, "QuantizationScheme"], layer_name: str) -> dict:+ """Convert config entry into dict and validate keys."""+ if isinstance(item, str):+ config = asdict(preset_name_to_scheme(item.upper()))+ elif isinstance(item, QuantizationScheme):+ config = asdict(item)+ elif isinstance(item, dict):+ # "in_blocks" is an internal bookkeeping key injected by LLM-Compressor;+ # silently drop it before validation.+ item = {k: v for k, v in item.items() if k != "in_blocks"}+ invalid = set(item) - set(scheme_keys + ("fixed_by_user", "scale_dtype"))+ if invalid:+ raise ValueError(+ f"Invalid keys {invalid} in layer_config for '{layer_name}'. " f"Allowed keys: {scheme_keys}"+ )+ config = dict(item)+ else:+ raise TypeError(+ f"Unsupported type for layer_config[{layer_name}]: {type(item)}. "+ f"Expected str, dict, or QuantizationScheme."+ )+ # Clean up+ config = {k: v for k, v in config.items() if v is not None}+ config["fixed_by_user"] = True+ return config++ # ---- main logic ----------------------------------------------+ extra_scheme_keys = ("scale_dtype",)+ scheme_keys = tuple(f.name for f in fields(QuantizationScheme)) + ("scale_dtype",)+ layer_config = copy.deepcopy(layer_config) or {}+ ignore_layer_patterns = set()+ if ignore_layers:+ ignore_layers = ignore_layers.replace(" ", "").split(",")+ ignore_layers = [name + "." if name[-1].isdigit() else name for name in ignore_layers]+ ignore_layer_patterns = set(ignore_layers)++ # 1. ignore_layers -> force 16+ for name in get_fp_layer_names(model, ignore_layers):+ layer_config[name] = {+ "bits": 16,+ "act_bits": 16,+ "data_type": "float",+ "act_data_type": "float",+ "fixed_by_user": True,+ }++ # 2. normalize+ layer_config = {k: normalize_item(v, k) for k, v in layer_config.items()}++ # 3. infer missing bits+ for cfg in layer_config.values():+ if "data_type" in cfg and "bits" not in cfg:+ if (b := infer_bits_by_data_type(cfg["data_type"])) is not None:+ cfg["bits"] = b+ if "act_data_type" in cfg and "act_bits" not in cfg:+ if (b := infer_bits_by_data_type(cfg["act_data_type"])) is not None:+ cfg["act_bits"] = b++ # 4. fill defaults+ if isinstance(default_scheme, str):+ default_dict = asdict(preset_name_to_scheme(default_scheme.upper()))+ else:+ default_dict = asdict(default_scheme)+ default_dict["scale_dtype"] = default_scale_dtype++ # In AutoScheme with mixed gguf:q4_k_m, the super_group_size of gguf:q8_0 layer is None,+ # which should not be filled by default q4km again+ for cfg in layer_config.values():+ for key in scheme_keys:+ if fill_default_value:+ cfg.setdefault(key, copy.deepcopy(default_dict.get(key)))+ else:+ if key in extra_scheme_keys:+ cfg.setdefault(key, copy.deepcopy(default_dict.get(key)))+ else:+ cfg.setdefault(key, None)++ # 5. collect supported modules+ embedding_types = (torch.nn.Embedding,)+ gguf_name = gguf_format_name or get_gguf_scheme(default_scheme)+ if gguf_name:+ if torch.nn.Embedding not in supported_types:+ supported_types = (*supported_types, torch.nn.Embedding)++ # for some Embedding which type() is not torch.nn.Embedding+ # for example: transformers.models.gemma3.modeling_gemma3.Gemma3TextScaledWordEmbedding+ model_module_name = model.__class__.__module__+ module_cls = sys.modules[model_module_name]+ for name in module_cls.__dict__:+ if name.endswith("Embedding") and not name.endswith("RotaryEmbedding"):+ embedding_types = (*embedding_types, getattr(module_cls, name))+ supported_types = (*supported_types, *embedding_types)++ all_supported_layer_names, embedding_layer_names = [], []+ all_module_names = []+ for n, m in model.named_modules():+ all_module_names.append(n)+ # cleanup stale attributes+ for key in scheme_keys:+ if hasattr(m, key):+ delattr(m, key)+ if type(m) not in supported_types and m.__class__.__name__ not in inner_supported_types:+ continue+ all_supported_layer_names.append(n)+ if isinstance(m, embedding_types) or m.__class__.__name__.endswith("Embedding"):+ embedding_layer_names.append(n)++ # Also include layer names from safetensor files not loaded into the model+ # (e.g. MTP layers that transformers does not instantiate).+ safetensor_only_names = _get_safetensor_layer_names_not_in_model(model, all_module_names)++ # 6. expand regex configs+ regex_config = {}+ for name in list(layer_config.keys()):+ if name in all_supported_layer_names:+ continue+ if name in all_module_names:+ m = get_module(model, name)+ if len(list(m.children())) == 0 and type(m) not in supported_types:+ val = layer_config.pop(name)+ if name in ignore_layer_patterns:+ # Keep unsupported ignore_layers entries so export can serialize+ # them into regex-based extra_config for loaders like vLLM INC.+ regex_config[name] = val+ else:+ logger.warning(+ f"'{name}' exists in the model but is not a supported quantization target "+ f"in the current scheme, ignoring its setting in `layer_config`"+ )+ continue++ regex = re.compile(to_standard_regex(name))+ matched = [ln for ln in all_supported_layer_names if regex.search(ln)]+ safetensor_only_matched = [ln for ln in safetensor_only_names if regex.search(ln)]+ # skip it for mtp layers not loaded in transformers+ if not matched and not safetensor_only_matched:+ # type(mlp.gate) is Qwen3VLMoeTextTopKRouter instead of Linear+ logger.warning_once(+ f"Layer name or regex '{name}' in layer_config does not match any supported layers. "+ + "Please check for typos or update the regex pattern, ignore it for now"+ )+ val = layer_config.pop(name)+ regex_config[name] = val # keep regex config+ for match in matched:+ layer_config[match] = val++ # 7. lm_head+ lm_head_name = get_lm_head_name(model)+ tie_word_embeddings = False+ if hasattr(model, "config") and hasattr(model.config, "tie_word_embeddings"):+ tie_word_embeddings = model.config.tie_word_embeddings++ if lm_head_name in layer_config:+ quant_lm_head = True++ if quant_lm_head and tie_word_embeddings and not gguf_name:+ quant_lm_head = False+ logger.warning(+ "reset `quant_lm_head` to false as quantizing " "lm_head with tied weights has not been supported currently"+ )++ if lm_head_name not in layer_config and quant_lm_head:+ layer_config[lm_head_name] = copy.deepcopy(default_dict)++ if not quant_lm_head and not gguf_name:+ layer_config.pop(lm_head_name, None)++ # 8. enforce shape divisibility for int weight-only+ if default_dict["data_type"] == "int" and default_dict["act_bits"] >= 16 and not gguf_name:+ for n, m in model.named_modules():+ if type(m) in supported_types or m.__class__.__name__ in inner_supported_types:+ if m.weight.shape[0] % 32 or m.weight.shape[1] % 32:+ layer_config.setdefault(n, copy.deepcopy(default_dict))+ layer_config[n].update({"bits": 16, "data_type": "fp", "fixed_by_user": True})+ # logger.warning_once(f"{n} skipped quantization (shape not divisible by 32).")+ # enforce shape divisibility for mxfp/nvfp+ if (is_nv_fp(default_dict["data_type"]) or is_mx_fp(default_dict["data_type"])) and not gguf_name:+ for n, m in model.named_modules():+ if type(m) in supported_types or m.__class__.__name__ in inner_supported_types:+ if m.weight.shape[1] % default_dict["group_size"]:+ layer_config.setdefault(n, copy.deepcopy(default_dict))+ layer_config[n].update(+ {"bits": 16, "data_type": "fp", "act_bits": 16, "act_data_type": "fp", "fixed_by_user": True}+ )+ logger.warning_once(+ f"{n} skipped quantization (shape not divisible by {default_dict['group_size']})."+ )++ # 9. block layers: mark as in_blocks=True+ for name in get_layer_names_in_block(model, supported_types, quant_block_list, inner_supported_types):+ if name not in layer_config:+ layer_config[name] = copy.deepcopy(default_dict)+ layer_config[name]["fixed_by_user"] = False+ layer_config[name]["in_blocks"] = True++ # ---- restore: ensure missing in_blocks are set to False and compute flag ----+ has_qlayer_outside_block = False+ for cfg in layer_config.values():+ if "in_blocks" not in cfg:+ cfg["in_blocks"] = False+ # mark layer outside block+ if not cfg["in_blocks"] and check_to_quantized(cfg):+ has_qlayer_outside_block = True++ # 10. GGUF handling+ if not gguf_name:+ dispatch_layer_config(layer_config)+ return layer_config, has_qlayer_outside_block, regex_config++ # embed + lm_head defaults for gguf+ tie_word_embeddings &= not is_separate_lm_head(model)+ if lm_head_name not in layer_config and not tie_word_embeddings:+ cfg = GGUF_INNER_CONFIG[GGUF_CONFIG[gguf_name.lower()]["lm_head"]]+ cfg = {**cfg, "fixed_by_user": False, "scale_dtype": default_scale_dtype}+ layer_config[lm_head_name] = cfg+ has_qlayer_outside_block = True+ for emd_name in embedding_layer_names:+ if emd_name in layer_config:+ continue+ if not tie_word_embeddings:+ cfg = GGUF_INNER_CONFIG[GGUF_CONFIG[gguf_name.lower()]["embedding"]]+ else:+ cfg = GGUF_INNER_CONFIG[GGUF_CONFIG[gguf_name.lower()]["lm_head"]]+ cfg = {**cfg, "fixed_by_user": False, "scale_dtype": default_scale_dtype}+ layer_config[emd_name] = cfg++ if enable_gguf_official_mixed:+ model_type = ModelType.MMPROJ if is_mllm else ModelType.TEXT+ layer_config, _ = get_layer_config_by_gguf_format(layer_config, gguf_name.lower(), model, model_type)++ dispatch_layer_config(layer_config)+ return layer_config, has_qlayer_outside_block, regex_config+++def _use_more_bits(i_layer: int, n_layer: int):+ return (i_layer < n_layer // 8) or (i_layer >= 7 * n_layer // 8) or ((i_layer - n_layer // 8) % 3 == 2)+++def _search_gguf_type(gguf_type):+ if gguf_type in GGUF_INNER_CONFIG:+ return gguf_type+ pattern = re.compile("gguf:q([0-9]{1,})_[01k]")+ bits = re.search(pattern, gguf_type)+ if not bits:+ raise KeyError(f"{gguf_type} is not a correct gguf type, please check")++ for suffix in ["_k", "_0", "_1"]:+ if gguf_type.endswith(suffix):+ continue+ if (tmp_type := re.sub("_[01k]", suffix, gguf_type)) in GGUF_INNER_CONFIG:+ return tmp_type+ return None+++def gguf_type_fallback(gguf_type: str) -> str:+ gguf_type = gguf_type.lower()+ if gguf_type in ("gguf:q2_k", "gguf:q3_k", "gguf:q4_k"):+ gguf_type = "gguf:q5_0"+ elif gguf_type == "gguf:q5_k":+ gguf_type = "gguf:q5_0"+ elif gguf_type == "gguf:q6_k":+ gguf_type = "gguf:q8_0"+ return gguf_type+++def get_gguf_qtype_by_layer_config(layer_config):+ import gguf # pylint: disable=E0401++ if layer_config["bits"] >= 16:+ return None+ bits = layer_config["bits"]+ super_bits = layer_config.get("super_bits", None)+ sym = layer_config["sym"]+ group_size = layer_config.get("group_size", None)+ super_group_size = layer_config.get("super_group_size", None)+ if bits == 2 and super_bits == 4 and not sym and group_size == 16 and super_group_size == 16:+ return gguf.GGMLQuantizationType.Q2_K+ if bits == 3 and super_bits == 6 and sym and group_size == 16 and super_group_size == 16:+ return gguf.GGMLQuantizationType.Q3_K+ if bits == 4:+ if super_bits is not None and super_bits == 6 and not sym and group_size == 32 and super_group_size == 8:+ return gguf.GGMLQuantizationType.Q4_K+ if super_bits is None and sym and group_size == 32:+ return gguf.GGMLQuantizationType.Q4_0+ if super_bits is None and not sym and group_size == 32:+ return gguf.GGMLQuantizationType.Q4_1+ if bits == 5:+ if super_bits == 6 and not sym and group_size == 32 and super_group_size == 8:+ return gguf.GGMLQuantizationType.Q5_K+ if super_bits is None and sym and group_size == 32:+ return gguf.GGMLQuantizationType.Q5_0+ if super_bits is None and not sym and group_size == 32:+ return gguf.GGMLQuantizationType.Q5_1+ if bits == 6 and super_bits == 8 and group_size == 16 and super_group_size == 16:+ return gguf.GGMLQuantizationType.Q6_K+ if bits == 8 and sym and group_size == 32:+ return gguf.GGMLQuantizationType.Q8_0+ raise ValueError("Unknown layer config")+++def _get_digital_in_layer_name(layer_name):+ pattern = re.compile(r"([a-zA-Z]+\.){1,}(\d+)")+ res = re.search(pattern, layer_name)+ if res:+ return int(res[2])+ else:+ return None+++def _gguf_type_fallback(gguf_type: str) -> str:+ gguf_type = gguf_type.lower()+ if gguf_type in ("gguf:q2_k", "gguf:q3_k", "gguf:q4_k"):+ gguf_type = "gguf:q5_0"+ elif gguf_type == "gguf:q5_k":+ gguf_type = "gguf:q5_0"+ elif gguf_type == "gguf:q6_k":+ gguf_type = "gguf:q8_0"+ return gguf_type+++##https://github.com/ggml-org/llama.cpp/blob/9e31bec4fd53634c9e5b04650488a09a055f5dab/src/llama-quant.cpp#L129+def get_layer_config_by_gguf_format(layer_config, target_gguf_format: str, model, model_type=ModelType.TEXT):+ # # TODO: support for other format later+ # target_gguf_format = next((fmt for fmt in gguf_format if fmt != "fake"), None)++ import gguf # pylint: disable=E0401++ from auto_round.schemes import QuantizationScheme, get_gguf_scheme+ from auto_round.utils.common import MM_KEYS, LazyImport+ from auto_round.utils.model import get_lm_head_name, get_module++ # from auto_round.export.export_to_gguf.convert import ModelBase, get_model_architecture+ convert_hf_to_gguf = LazyImport("auto_round.export.export_to_gguf.convert_hf_to_gguf")++ try:+ model_architecture = convert_hf_to_gguf.get_model_architecture(+ hparams=model.config.to_dict(), model_type=model_type+ )+ except AttributeError as e:+ raise ImportError(+ "Please use the latest gguf-py, you can use the following command to install it:\n"+ "git clone https://github.com/ggml-org/llama.cpp.git && cd llama.cpp/gguf-py"+ " && pip install . sentencepiece"+ )+ try:+ if model_type != ModelType.TEXT:+ model_class_vision = convert_hf_to_gguf.ModelBase.from_model_architecture(+ model_architecture, model_type=model_type+ )+ model_class = convert_hf_to_gguf.ModelBase.from_model_architecture(+ model_architecture, model_type=ModelType.TEXT+ )++ except NotImplementedError:+ return layer_config, {}++ n_layer = None+ if model_type != ModelType.TEXT:+ n_layer_vision = None+ for name in ["n_layers", "num_hidden_layers", "n_layer", "num_layers", "depth"]:+ if hasattr(model.config, name):+ n_layer = getattr(model.config, name)+ if model_type != ModelType.TEXT:+ if n_layer is not None and hasattr(model.config, "text_config"):+ if hasattr(getattr(model.config, "text_config"), name):+ n_layer = getattr(getattr(model.config, "text_config"), name)+ for config_name in ["vision_config", "vision_encoder"]:+ if hasattr(model.config, config_name):+ if hasattr(getattr(model.config, config_name), name):+ n_layer_vision = getattr(getattr(model.config, config_name), name)+ break+ if n_layer and n_layer_vision:+ break++ if n_layer is None:+ return layer_config, {}++ tensor_map = gguf.get_tensor_name_map(model_class.model_arch, n_layer)+ if model_type != ModelType.TEXT:+ tensor_map_vision = gguf.get_tensor_name_map(model_class_vision.model_arch, n_layer_vision)++ def _set_config(config, target_config):+ for k, v in target_config.items():+ if isinstance(config, dict):+ config[k] = v+ else:+ setattr(config, k, v)+ return config++ gguf_format_config = {}+ lm_head_name = get_lm_head_name(model)+ inner_gguf_format = GGUF_CONFIG[target_gguf_format]["mostly"]+ # ggml_type = getattr(gguf.GGMLQuantizationType,inner_gguf_format.split(":")[-1].upper())+ block_size = GGML_QUANT_SIZES[inner_gguf_format.split(":")[-1].lower()][0]+ tie_word_embeddings = True+ if hasattr(model, "config") and hasattr(model.config, "tie_word_embeddings"):+ tie_word_embeddings = model.config.tie_word_embeddings++ n_gqa = 1+ if (+ hasattr(model, "config")+ and hasattr(model.config, "num_attention_heads")+ and hasattr(model.config, "num_key_value_heads")+ ):+ n_gqa = model.config.num_attention_heads // model.config.num_key_value_heads+ n_expert = 0+ for name in ["num_experts", "num_local_experts", "n_routed_experts"]:+ if hasattr(model.config, name):+ n_expert = getattr(model.config, name)++ i_attention_wv = 0+ i_ffn_down = 0+ layer_config_copy = copy.deepcopy(layer_config)+ base_target_bits = None+ if inner_gguf_format.startswith("gguf:q") and len(inner_gguf_format) >= 7 and (inner_gguf_format[6]).isdigit():+ base_target_bits = int(inner_gguf_format[6])++ for layer_name, config in layer_config_copy.items():+ if not check_to_quantized(config):+ continue+ # Reset target_bits each iteration to prevent lm_head/embedding settings+ # from bleeding into subsequent block layers and bypassing their special logic.+ target_bits = base_target_bits+ new_type = GGUF_CONFIG[target_gguf_format]["mostly"]+ layer = get_module(model, layer_name)+ if type(layer) == transformers.pytorch_utils.Conv1D:+ input_features = layer.weight.shape[0]+ else:+ input_features = layer.weight.shape[-1]+ i_layer = _get_digital_in_layer_name(layer_name)++ if lm_head_name is not None and layer_name == lm_head_name:+ target_bits = int(re.search("gguf:q([0-9]{1,})_[01k]", GGUF_CONFIG[target_gguf_format]["lm_head"]).group(1))+ if isinstance(layer, torch.nn.Embedding):+ target_bits = int(+ re.search("gguf:q([0-9]{1,})_[01k]", GGUF_CONFIG[target_gguf_format]["embedding"]).group(1)+ )++ if model_type != ModelType.TEXT and any([key in layer_name for key in MM_KEYS]):+ gguf_name = tensor_map_vision.get_name(layer_name)+ if gguf_name is None:+ for key in MM_KEYS:+ gguf_name = tensor_map_vision.get_name(layer_name.replace(f".{key}", ""))+ if gguf_name is not None:+ break+ else:+ gguf_name = tensor_map.get_name(layer_name)+ if gguf_name is None:+ gguf_name = tensor_map.get_name(layer_name.replace(".language_model", ""))+ bits_index = 6+ if config.get("fixed_by_user", False):+ if "bits" not in config:+ logger.warning(+ f"Setting layer_config requires providing bits, {layer_name} has not bits,"+ f" using bits={target_bits} instead."+ )+ new_type = new_type[:bits_index] + target_bits + new_type[bits_index + 1 :]+ else:+ config_tmp = config.copy()+ scheme_keys = [f.name for f in fields(QuantizationScheme)]+ for key in config.keys():+ if key not in scheme_keys:+ config_tmp.pop(key, None)+ matched_scheme = get_gguf_scheme(QuantizationScheme.from_dict(config_tmp)) # check matched+ if not matched_scheme:+ if config.get("super_group_size", None) is not None or config.get("super_bits", None) is not None:+ new_type = new_type[:bits_index] + str(config["bits"]) + "_k"+ if new_type not in GGUF_INNER_CONFIG:+ prefix_idx = 0 if config.get("sym", True) else 1+ new_type = new_type[:bits_index] + str(config["bits"]) + f"_{prefix_idx}"+ if new_type not in GGUF_INNER_CONFIG:+ new_type = new_type[:bits_index] + str(config["bits"]) + f"_{1-prefix_idx}"+ if new_type not in GGUF_INNER_CONFIG:+ raise ValueError(+ f"the setting in layer_config {layer_name} "+ f"could not match any supported gguf format, please have a check."+ )++ new_type = new_type[:bits_index] + str(config["bits"]) + new_type[bits_index + 1 :]+ new_type = _search_gguf_type(new_type)+ if new_type is None:+ raise ValueError(f"invalid bit setting for {layer_name}")+ elif target_bits is not None and "bits" in config and config["bits"] != target_bits:+ new_type = new_type[:bits_index] + str(config["bits"]) + new_type[bits_index + 1 :]+ new_type = _search_gguf_type(new_type)+ if new_type is None:+ raise ValueError(f"invalid bit setting for {layer_name}")+ elif lm_head_name is not None and layer_name == lm_head_name and not tie_word_embeddings:+ if gguf.MODEL_ARCH.FALCON == model_class.model_arch or input_features % block_size != 0:+ new_type = "gguf:q8_0"+ elif "lm_head" in GGUF_CONFIG[target_gguf_format]:+ new_type = GGUF_CONFIG[target_gguf_format]["lm_head"]+ elif new_type != "gguf:q8_0":+ new_type = "gguf:q6_k"+ elif lm_head_name is not None and layer_name == lm_head_name and tie_word_embeddings:+ # new_type = GGUF_CONFIG[target_gguf_format]["lm_head"]+ continue+ elif isinstance(layer, torch.nn.Embedding):+ if "embedding" in GGUF_CONFIG[target_gguf_format]:+ new_type = GGUF_CONFIG[target_gguf_format]["embedding"]+ elif gguf_name is None:+ pass+ # attn_v+ elif "attn_v" in gguf_name:+ if target_gguf_format == "gguf:q2_k":+ new_type = "gguf:q4_k" if n_gqa >= 4 else "gguf:q3_k"+ elif target_gguf_format == "gguf:q2_k_s" and n_gqa >= 4:+ new_type = "gguf:q4_k"+ elif target_gguf_format == "gguf:q3_k_m":+ new_type = "gguf:q5_k" if i_attention_wv < 2 else "gguf:q4_k"+ elif target_gguf_format == "gguf:q3_k_l":+ new_type = "gguf:q5_k"+ elif (target_gguf_format == "gguf:q4_k_m" or target_gguf_format == "gguf:q5_k_m") and _use_more_bits(+ i_layer, n_layer+ ):+ new_type = "gguf:q6_k"+ elif target_gguf_format == "gguf:q4_k_s" and i_attention_wv < 4:+ new_type = "gguf:q5_k"+ ##TODO check which models are be grouped into to LLM_TYPE_70B+ # if (qs.model.type == LLM_TYPE_70B) {+ # // In the 70B model we have 8 heads sharing the same attn_v weights.+ # As a result, the attn_v.weight tensor is+ # // 8x smaller compared to attn_q.weight.Hence, we can get a nice boost in quantization accuracy with+ # // nearly negligible increase in model size by quantizing this tensor with more bits:+ # if+ # (new_type == GGML_TYPE_Q3_K | | new_type == GGML_TYPE_Q4_K)+ # new_type = GGML_TYPE_Q5_K;+ # }+ if n_expert == 8:+ new_type = "gguf:q8_k"+ i_attention_wv += 1++ elif "attn_k" in gguf_name:+ if n_expert == 8:+ new_type = "gguf:q8_0"+ # ffn_down+ elif "ffn_down" in gguf_name:+ if target_gguf_format == "gguf:q2_k":+ new_type = "gguf:q3_k"+ elif target_gguf_format == "gguf:q2_k_s":+ if i_layer < n_layer / 8:+ new_type = "gguf:q4_k"+ elif target_gguf_format == "gguf:q3_k_m":+ if i_layer < n_layer / 16:+ new_type = "gguf:q5_k"+ elif gguf.MODEL_ARCH.FALCON == model_class.model_arch or _use_more_bits(i_layer, n_layer):+ new_type = "gguf:q4_k"+ else:+ new_type = "gguf:q3_k"+ elif target_gguf_format == "gguf:q3_k_l":+ if gguf.MODEL_ARCH.FALCON == model_class.model_arch:+ new_type = "gguf:q4_k"+ else:+ new_type = "gguf:q5_k"+ elif target_gguf_format == "gguf:q4_k_m":+ if gguf.MODEL_ARCH.FALCON == model_class.model_arch:+ if i_layer < n_layer // 16:+ new_type = "gguf:q6_k"+ elif _use_more_bits(i_layer, n_layer):+ new_type = "gguf:q5_k"+ else:+ new_type = "gguf:q4_k"+ else:+ if _use_more_bits(i_layer, n_layer):+ new_type = "gguf:q6_k"+ elif target_gguf_format == "gguf:q5_k_m" and _use_more_bits(i_layer, n_layer):+ new_type = "gguf:q6_k"+ elif (+ target_gguf_format == "gguf:q4_k_s"+ and model_class.model_arch != gguf.MODEL_ARCH.FALCON+ and i_layer < n_layer / 8+ ):+ new_type = "gguf:q5_k"+ elif (target_gguf_format == "gguf:q4_0" or target_gguf_format == "gguf:q5_0") and i_layer < n_layer / 8:+ if target_gguf_format == "gguf:q4_0":+ new_type = "gguf:q4_1"+ else:+ new_type = "gguf:q5_1"+ i_ffn_down += 1++ # attn_output+ elif "attn_output" in gguf_name:+ if gguf.MODEL_ARCH.FALCON != model_class.model_arch:+ if n_expert == 8:+ if target_gguf_format in (+ "gguf:q2_k",+ "gguf:q3_k_s",+ "gguf:q3_k_m",+ "gguf:q4_k_s",+ "gguf:q4_k_m",+ "gguf:q5_k",+ ):+ new_type = "gguf:q5_k"+ elif target_gguf_format == "gguf:q2_k":+ new_type = "gguf:q3_k"+ elif target_gguf_format == "gguf:q3_k_m":+ new_type = "gguf:q4_k"+ elif target_gguf_format == "gguf:q3_k_l":+ new_type = "gguf:q5_k"+ else:+ if target_gguf_format == "gguf:q3_k_l":+ new_type = "gguf:q4_k"+ # attn_qkv+ elif "attn_qkv" in gguf_name:+ if target_gguf_format in ("gguf:q3_k_m", "gguf:q3_k_l"):+ new_type = "gguf:q4_k"+ elif target_gguf_format == "gguf:q4_k_m":+ new_type = "gguf:q5_k"+ elif target_gguf_format == "gguf:q5_k_m":+ new_type = "gguf:q5_k"+ new_block_size = GGML_QUANT_SIZES[new_type.split(":")[-1].lower()][0]+ if input_features % new_block_size != 0:+ new_type = _gguf_type_fallback(new_type)+ new_block_size = GGML_QUANT_SIZES[new_type.split(":")[-1].lower()][0]+ if input_features % new_block_size != 0:+ new_type = "gguf:bf16"+ logger.warning(+ f"fallback {layer_name} to {new_type}, "+ f"because input_features({input_features}) % block_size({block_size}) != 0"+ )+ # for deepseek v2+ if layer_name.endswith("kv_b_proj") and new_type.endswith("_k") and "Deepseek" in model.config.architectures[0]:+ fallback = False++ # calc if need fallback+ qk_nope_head_dim = model.config.qk_nope_head_dim+ kv_b_shape = get_module(model, layer_name).weight.shape++ if (+ qk_nope_head_dim < QK_K+ or qk_nope_head_dim % QK_K != 0+ or kv_b_shape[-1] < QK_K+ or kv_b_shape[-1] % QK_K != 0+ ):+ fallback = True+ if fallback:+ tmp_type = _gguf_type_fallback(new_type)+ logger.warning_once(+ f"self_attn.kv_b_proj does not support the use of {new_type}, replace it with {tmp_type}"+ )+ new_type = tmp_type++ target_config = GGUF_INNER_CONFIG[new_type]++ _set_config(layer_config[layer_name], target_config)+ _set_config(layer, target_config)+ gguf_format_config[layer_name] = new_type++ return layer_config, gguf_format_config+++def get_fp_layer_names(model: torch.nn.Module, ignore_layers: str):+ """Identifies and returns layers in the model to exclude from quantization.++ This function processes a comma-separated list of fully precision (FP) layers,+ matches them to the names of layers in the model, and returns a list of such+ layers to exclude from quantization.++ Args:+ model (torch.nn.Module): The model whose layers will be inspected.+ ignore_layers (str): A comma-separated string of layer names to be excluded+ from quantization. Whitespace is ignored in this string.++ Returns:+ list: A list of layer names that match the specified FP layers or are+ subcomponents of those layers.+ """+ from auto_round.utils import SUPPORTED_LAYER_TYPES++ if not ignore_layers:+ return []++ all_layer_names = []+ for n, m in model.named_modules():+ if type(m) in SUPPORTED_LAYER_TYPES:+ all_layer_names.append(n)+ not_to_quantized_layers = []++ for fp_layer in ignore_layers:+ if fp_layer == "":+ continue+ if fp_layer in all_layer_names:+ not_to_quantized_layers.append(fp_layer)+ continue+ for name in all_layer_names:+ if fp_layer in name:+ not_to_quantized_layers.append(name)+ not_to_quantized_layers.extend(ignore_layers) # keep regex name for later use+ logger.trace(f"not_to_quantized_layers: {not_to_quantized_layers}")+ return not_to_quantized_layers+++def get_shared_keys(model):+ """+ Retrieves shared keys from the model's state dictionary.++ Args:+ model (torch.nn.Module): The model to retrieve shared keys from.++ Returns:+ tuple: tuple of shared keys.+ """+ from auto_round.special_model_handler import SPECIAL_SHARED_CACHE_KEYS+ from auto_round.utils import SHARED_CACHE_KEYS++ shared_keys = SHARED_CACHE_KEYS+ shared_keys += SPECIAL_SHARED_CACHE_KEYS.get(model.__class__.__name__, ())+ return shared_keys+++def init_cache(positional_inputs, inputs):+ """+ Initializes special model inputs by adding positional inputs if missing.++ Args:+ positional_inputs (list): List of positional inputs to add to inputs.+ inputs (dict): Dictionary of model inputs.++ Modifies:+ inputs (dict): Adds "positional_inputs" key if not present.+ """+ from auto_round.utils.model import to_device++ if "positional_inputs" not in inputs: # for chatglm Series+ inputs["positional_inputs"] = []+ for idx, item in enumerate(positional_inputs):+ inputs["positional_inputs"] = to_device(positional_inputs)+++def reset_params(inputs):+ """+ Resets specific input parameters to avoid saving the key-value cache during fine-tuning.++ Args:+ inputs (dict): Dictionary of model inputs.++ Modifies:+ inputs (dict): Sets "use_cache" to False if the key is present.+ """+ if "use_cache" in inputs.keys(): # Not storing kv cache+ inputs["use_cache"] = False+++class IndexSampler:+ """A cyclic sampler that returns shuffled index batches.++ This sampler maintains internal state so that each call to `next_batch()`+ continues from where it left off. When the remaining number of samples is+ less than `batch_size`, the sampler reshuffles all indices and starts from+ the beginning, discarding the last incomplete batch.++ Attributes:+ nsamples (int): Total number of samples.+ batch_size (int): Number of indices to return in each batch.+ index (int): Current position in the index list.+ indices (List[int]): Shuffled list of indices.+ """++ def __init__(self, nsamples: int, batch_size: int) -> None:+ """Initializes the sampler.++ Args:+ nsamples (int): Total number of samples (must be >= batch_size).+ batch_size (int): Number of indices per batch.++ Raises:+ ValueError: If batch_size is not in the range (0, nsamples].+ """+ if batch_size <= 0 or batch_size > nsamples:+ raise ValueError("batch_size must be > 0 and <= nsamples")++ self.nsamples: int = nsamples+ self.batch_size: int = batch_size+ self.index: int = 0++ self.indices: list[int] = list(range(nsamples))+ random.shuffle(self.indices)++ def next_batch(self) -> list[int]:+ """Returns the next batch of shuffled indices.++ If the remaining indices are fewer than `batch_size`, the sampler+ reshuffles the entire list and starts from the beginning.++ Returns:+ list[int]: A list of size `batch_size` containing sample indices.+ """+ if self.index + self.batch_size > self.nsamples:+ random.shuffle(self.indices)+ self.index = 0++ batch = self.indices[self.index : self.index + self.batch_size]+ self.index += self.batch_size+ return batch+++def _get_quantized_layer_names_outside_blocks(model, layer_config, supported_types, quant_block_list) -> list:+ """Gets the names of quantized layers outside blocks in the model.++ Returns:+ list: List of layer names outside blocks.+ """+ if layer_config is None or len(layer_config) == 0:+ return []++ layer_names = []+ all_layers_in_block = get_layer_names_in_block(model, supported_types, quant_block_list)++ for key in layer_config.keys():+ if key in all_layers_in_block:+ continue+ layer = get_module(model, key)+ if layer is None:+ logger.error(f"could not find layer {key} in the model, exit...")+ exit(-1)+ if type(layer) in supported_types and check_to_quantized(layer_config[key]):+ layer_names.append(key)++ return layer_names+++def _get_diffusion_save_folder_name(format) -> str:+ """Generates the save folder name based on the provided format string.++ If there are multiple formats to handle, the function creates a subfolder+ named after the format string with special characters replaced. If there's+ only one format, it returns the original output directory directly.++ Args:+ format_str (str): The format identifier (e.g., 'gguf:q2_k_s').++ Returns:+ str: The path to the folder where results should be saved.+ """+ from auto_round.context.compress import CompressContext+ from auto_round.context.model import ModelContext++ compress_context = CompressContext.get_context()+ model_context = ModelContext.get_context()++ # Replace special characters to make the folder name filesystem-safe+ sanitized_format = format.get_backend_name().replace(":", "-").replace("_", "-")++ formats = compress_context.formats+ # Use a subfolder only if there are multiple formats+ if len(formats) > 1:+ return (+ os.path.join(compress_context.output_dir, sanitized_format, "transformer")+ if compress_context.is_immediate_saving+ else os.path.join(compress_context.output_dir, sanitized_format, "transformer")+ )++ # if use is_immediate_saving, we need to save model in self.output_dir/transformer folder+ return (+ os.path.join(compress_context.output_dir, "transformer")+ if compress_context.is_immediate_saving+ else compress_context.output_dir+ )+++def _get_save_folder_name(format, *args, **kwargs) -> str:+ """Generates the save folder name based on the provided format string.++ If there are multiple formats to handle, the function creates a subfolder+ named after the format string with special characters replaced. If there's+ only one format, it returns the original output directory directly.++ Args:+ format_str (str): The format identifier (e.g., 'gguf:q2_k_s').++ Returns:+ str: The path to the folder where results should be saved.+ """+ from auto_round.context.compress import CompressContext+ from auto_round.context.model import ModelContext++ compress_context = CompressContext.get_context()+ model_context = ModelContext.get_context()+ if model_context.is_diffusion:+ return _get_diffusion_save_folder_name(format)+ # Replace special characters to make the folder name filesystem-safe+ sanitized_format = format.get_backend_name().replace(":", "-").replace("_", "-")++ # Use a subfolder only if there are multiple formats+ if len(compress_context.formats) > 1:+ return os.path.join(compress_context.output_dir, sanitized_format)++ return compress_context.output_dir+++def immediate_pack(name: str, layer_config: dict):+ from auto_round.context.compress import CompressContext+ from auto_round.context.model import ModelContext++ compress_context = CompressContext.get_context()+ model_context = ModelContext.get_context()++ if not compress_context.is_immediate_packing:+ return+ compress_context.formats[0].immediate_pack(+ name=name,+ model=model_context.model,+ device=compress_context.device,+ output_dir=_get_save_folder_name(compress_context.formats[0]),+ layer_config=layer_config,+ tokenizer=model_context.tokenizer,+ mllm=model_context.is_mllm,+ processor=getattr(model_context, "processor", None),+ image_processor=getattr(model_context, "image_processor", None),+ quant_nontext_module=getattr(model_context, "quant_nontext_module", False),+ )diff --git a/auto_round/compressors_new/zero_shot.py b/auto_round/compressors_new/zero_shot.pynew file mode 100644index 000000000..977244639--- /dev/null+++ b/auto_round/compressors_new/zero_shot.py@@ -0,0 +1,257 @@+# Copyright (c) 2026 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+import copy+from typing import Any, Union++import torch+from tqdm import tqdm++from auto_round.algorithms.alg_config import AlgConfig+from auto_round.compressors_new.base import BaseCompressor+from auto_round.compressors_new.utils import is_nv_fp, is_static_wfp8afp8+from auto_round.logger import logger+from auto_round.modeling.fused_moe.replace_modules import materialize_model_+from auto_round.utils import (+ check_to_quantized,+ clear_memory,+ convert_module_to_hp_if_necessary,+ flatten_list,+ get_block_names,+ get_lm_head_name,+ get_module,+ global_state,+ memory_monitor,+ mv_module_from_gpu,+ set_amax_for_all_moe_layers,+ set_module,+)+++class ZeroShotCompressor(BaseCompressor):+ need_calib: bool = False++ def __init__(+ self,+ config: Union[AlgConfig, list[AlgConfig]],+ model: Union[torch.nn.Module, str],+ tokenizer=None,+ platform="hf",+ format=None,+ low_gpu_mem_usage: bool = False,+ device_map: Union[str, torch.device, int, dict] = 0,+ enable_torch_compile: bool = False,+ enable_alg_ext: bool = False,+ seed: int = 42,+ low_cpu_mem_usage: bool = True,+ **kwargs,+ ):+ super().__init__(+ config=config,+ model=model,+ tokenizer=tokenizer,+ platform=platform,+ format=format,+ device_map=device_map,+ low_gpu_mem_usage=low_gpu_mem_usage,+ enable_torch_compile=enable_torch_compile,+ enable_alg_ext=enable_alg_ext,+ seed=seed,+ low_cpu_mem_usage=low_cpu_mem_usage,+ **kwargs,+ )+ self.lr = 5e-3++ def quantize_block(+ self,+ block: torch.nn.Module,+ inputs: tuple,+ q_input: Union[torch.Tensor, dict, None] = None,+ device: Union[str, torch.device] = "cpu",+ auto_offload: bool = True,+ ):+ """Quantize a single block via RTN (public API for LLM-Compressor).++ ZeroShotCompressor does not need calibration data, so ``inputs`` and+ ``q_input`` are accepted for interface compatibility but not used for+ algorithm purposes. The block is materialized, converted to the target+ dtype, moved to ``device``, and quantized in-place via RTN.++ Returns:+ tuple: ``(None, None)`` — RTN does not produce reference outputs.+ """+ assert not self.mllm and not self.diffusion, (+ f"Currently, {self.__class__.__name__} does not support quantize_block " "for MLLM / diffusion models."+ )++ if not self._post_init_done:+ self.post_init()++ materialize_model_(block)+ convert_module_to_hp_if_necessary(block, self.model_context.amp_dtype, device)+ block = block.to(device)++ self.quantizer.quantize_block(block)++ # ── MoE scale alignment for FP8 dispatch efficiency ────────────────+ if is_nv_fp(self.quantizer.act_data_type) or is_static_wfp8afp8(self.quantizer):+ set_amax_for_all_moe_layers(block, attr_name="act_max")++ mv_module_from_gpu(block)+ return None, None++ # Use no_grad instead of inference_mode+ # https://github.com/intel/auto-round/issues/1620+ @torch.no_grad()+ def quantize(self) -> tuple[torch.nn.Module, dict[str, Any]]:+ """Quantize the model and return the quantized model along with layer configurations.The entry of AutoRound.+ Returns:+ The quantized model and layer configurations.+ """++ self.post_init()++ formats = self.formats if isinstance(self.formats, list) else []+ if not (any(fmt.is_gguf() for fmt in formats) or self.super_bits is not None):+ self._quantize_embedding_layer() # leave to gguf itself to handle++ # Release memory+ clear_memory(device_list=self.device_list)++ # By default, we go with layer-wise way if no replacement happened.+ # In RTN mode (iters == 0), force blockwise quantization to avoid+ # full-model materialization and linear CPU RAM growth.+ use_blockwise_quantization = global_state.replaced_module_count > 0+ if not use_blockwise_quantization:+ logger.info(+ "RTN mode detected (iters=0): force blockwise quantization to avoid "+ "layer-wise full-model materialization."+ )+ use_blockwise_quantization = True+ tied_weights_keys = getattr(self.model, "_tied_weights_keys", [])+ if tied_weights_keys is None:+ tied_weights_keys = []+ if isinstance(tied_weights_keys, dict):+ tied_weights_values = list(tied_weights_keys.values())+ else:+ tied_weights_values = list(tied_weights_keys)+ tied_weights_layers = [".".join(val.split(".")[:-1]) for val in tied_weights_values] # rm weight/bias+ # In fact, we should detect whether it is is_separate_lm_head, to simplify, we don't do it+ if getattr(self, "formats", None) and self.formats[0].is_gguf():+ lm_head_name = get_lm_head_name(self.model)+ if lm_head_name is not None:+ tied_weights_layers.append(lm_head_name)++ if use_blockwise_quantization: # The ram usage is a little higher++ all_blocks = self.quant_block_list or get_block_names(self.model)+ pbar = tqdm(range(sum(len(block) for block in all_blocks)))+ for block_names in all_blocks:+ for block_name in block_names:+ pbar.set_description(f"Quantizing {block_name}")+ block = get_module(self.model, block_name)++ # ── Infrastructure: materialize ───────────────────────────+ materialize_model_(block)++ # ── Pure algorithm ────────────────────────────────────────+ self.quantizer.quantize_block(block)++ # ── MoE scale alignment for FP8 dispatch efficiency ────────────────+ if is_nv_fp(self.quantizer.act_data_type) or is_static_wfp8afp8(self.quantizer):+ set_amax_for_all_moe_layers(block, attr_name="act_max")++ # ── Infrastructure: shard write / device cleanup ──────────+ if self.compress_context.is_immediate_saving:+ # Save non-quantized leaf modules (e.g. norms, embeddings in block).+ for _n, m in block.named_modules():+ if (+ not any(m.children())+ and len(m.state_dict()) > 0+ and hasattr(m, "global_name")+ and m.global_name not in tied_weights_layers+ and not check_to_quantized(m)+ ):+ set_module(self.model, m.global_name, copy.deepcopy(m))+ self.shard_writer.write(name=m.global_name)+ get_module(self.model, m.global_name).to("meta")+ m.to("meta")+ # Write at block scope for any remaining params/buffers.+ self.shard_writer.write(name=block_name)+ block.to("meta")+ else:+ mv_module_from_gpu(block)+ if self.low_cpu_mem_usage:+ self._offloader(self.model, block_name)++ clear_memory(device_list=self.device_list)+ memory_monitor.log_summary()+ pbar.update(1)+ cnt = 1+ remain_layer_names = []+ block_name_set = set(name for block in all_blocks for name in block)+ for n, m in self.model_context.model.named_modules():+ if not check_to_quantized(m):+ continue+ # Skip if this layer is part of any block (by prefix match)+ if any(n == block_name or n.startswith(f"{block_name}.") for block_name in block_name_set):+ continue+ remain_layer_names.append(n)+ for name in remain_layer_names:+ logger.info(f"Quantizing remaining layer {name} on CPU.")+ self.quantizer.quantize_layer(name)+ cnt += 1+ if cnt % 10 == 0:+ clear_memory(device_list=self.device_list)+ memory_monitor.log_summary()+ else:+ all_to_quantized_module_names: list[str] = [+ n for n, m in self.model.named_modules() if check_to_quantized(m)+ ]+ all_to_quantized_module_names = all_to_quantized_module_names+ materialize_model_(self.model)+ self.model.to("cpu")+ block_names_cnt = len(flatten_list(get_block_names(self.model, True)))+ clear_mem_freq = len(all_to_quantized_module_names) // block_names_cnt+ cnt = 0+ pbar = tqdm(all_to_quantized_module_names)++ for n, m in self.model.named_modules():+ if hasattr(m, "global_name") and m.global_name in all_to_quantized_module_names:+ pbar.set_description(f"Quantizing {m.global_name}")+ self.quantizer.quantize_layer(m.global_name)+ cnt += 1+ pbar.update()+ if cnt % clear_mem_freq == 0:+ clear_memory(device_list=self.device_list)+ memory_monitor.log_summary()++ elif (+ not any(m.children())+ and len(m.state_dict()) > 0+ and n not in tied_weights_layers+ and self.compress_context.is_immediate_saving+ ):+ set_module(self.model, n, copy.deepcopy(m))+ self.shard_writer.write(name=n)+ m.to("meta")++ # Convert remaining fp8+ convert_module_to_hp_if_necessary(self.model, self.amp_dtype, self.device)+ if self.low_cpu_mem_usage:+ self._offloader.reload(self.model)+ if self.compress_context.is_immediate_saving:+ self.shard_writer.write(is_finalize=True)++ self.model_context.quantized = True+ return self.model, self.layer_configdiff --git a/auto_round/context/__init__.py b/auto_round/context/__init__.pynew file mode 100644index 000000000..14a492441--- /dev/null+++ b/auto_round/context/__init__.py@@ -0,0 +1,13 @@+# Copyright (c) 2026 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.diff --git a/auto_round/context/base.py b/auto_round/context/base.pynew file mode 100644index 000000000..e3f75fb8f--- /dev/null+++ b/auto_round/context/base.py@@ -0,0 +1,61 @@+# Copyright (c) 2026 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.++from auto_round.logger import logger+++class AutoSkipInitMeta(type):++ def __new__(mcs, name, bases, namespace):+ if "__init__" in namespace:+ original_init = namespace["__init__"]++ def wrapped_init(self, *args, **kwargs):+ if getattr(self, "_singleton_skip_init", False):+ return+ original_init(self, *args, **kwargs)+ self._singleton_skip_init = True++ namespace["__init__"] = wrapped_init++ namespace["_instances"] = {}+ return super().__new__(mcs, name, bases, namespace)++ def __call__(cls, *args, **kwargs):+ if cls not in cls._instances:+ instance = cls.__new__(cls, *args, **kwargs)+ cls._instances[cls] = instance+ instance.__init__(*args, **kwargs)++ return cls._instances[cls]+++class BaseContext(metaclass=AutoSkipInitMeta):+ _instances = {}++ def __init__(self):+ logger.info(f"{self.__class__.__name__} context initialized.")++ @classmethod+ def get_context(cls):+ assert cls in cls._instances, f"{cls.__name__} context has not been created yet."+ return cls._instances.get(cls)++ @classmethod+ def create_context(cls, *args, **kwargs):+ return cls(*args, **kwargs)++ @classmethod+ def reset_context(cls):+ cls._instances.pop(cls, None)diff --git a/auto_round/context/compress.py b/auto_round/context/compress.pynew file mode 100644index 000000000..5b92a8b7c--- /dev/null+++ b/auto_round/context/compress.py@@ -0,0 +1,72 @@+# Copyright (c) 2026 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+from typing import Any, Callable, Optional, Union++import torch++from auto_round.context.base import BaseContext+from auto_round.utils.device import (+ clear_memory,+ clear_memory_if_reached_threshold,+ get_major_device,+ parse_available_devices,+ set_auto_device_map_for_block_with_tuning,+ set_non_auto_device_map,+)++__all__ = ["CompressContext"]+++class CompressContext(BaseContext):++ def __init__(+ self,+ low_cpu_mem_usage: bool = True,+ low_gpu_mem_usage: bool = False,+ device_map: Union[str, torch.device, int, dict] = 0,+ enable_torch_compile: bool = False,+ is_immediate_packing: bool = False,+ is_immediate_saving: bool = False,+ formats: Union[list, str] = None,+ output_dir: str = "./compressed_models",+ static_kv_dtype: Optional[torch.dtype] = None,+ static_attention_dtype: Optional[torch.dtype] = None,+ **kwargs,+ ):+ super().__init__()+ self.low_cpu_mem_usage = low_cpu_mem_usage+ self.low_gpu_mem_usage = low_gpu_mem_usage+ self.formats = formats+ self.output_dir = output_dir+ if device_map is None:+ device_map = 0+ self.device_map = device_map+ if isinstance(self.device_map, str):+ self.device_map = self.device_map.replace(" ", "")+ self.device_list = parse_available_devices(self.device_map)+ self.device = get_major_device(self.device_map)++ self.cache_device = torch.device("cpu") if low_gpu_mem_usage else self.device++ self.enable_torch_compile = enable_torch_compile+ self.immediate_packing = is_immediate_packing+ self.is_immediate_packing = is_immediate_packing+ self.is_immediate_saving = is_immediate_saving+ self.static_kv_dtype = static_kv_dtype+ self.static_attention_dtype = static_attention_dtype++ def clear_memory(self, tensor=None):+ """Clear GPU/CPU memory only when ``low_gpu_mem_usage`` is enabled."""+ if self.low_gpu_mem_usage:+ clear_memory(tensor, device_list=self.device_list)diff --git a/auto_round/context/model.py b/auto_round/context/model.pynew file mode 100644index 000000000..49648997e--- /dev/null+++ b/auto_round/context/model.py@@ -0,0 +1,292 @@+# Copyright (c) 2026 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.++import gc+import importlib+from typing import Any, Callable, Optional, Union++import torch+from packaging import version+from transformers import AutoConfig++from auto_round import envs+from auto_round.compressors.utils import get_shared_keys+from auto_round.context.base import BaseContext+from auto_round.logger import logger+from auto_round.modeling.unfused_moe import apply_model_monkey_patches+from auto_round.special_model_handler import _handle_special_model, update_module+from auto_round.utils import (+ CpuInfo,+ check_and_mark_quantized_module,+ diffusion_load_model,+ is_diffusion_model,+ is_mllm_model,+ is_moe_model,+ is_moe_model_via_config,+ llm_load_model,+ mllm_load_model,+ unsupported_meta_device,+)+from auto_round.utils.device import _force_trim_malloc++__all__ = ["ModelContext"]++_CUSTOM_MOE_REPLACEMENT_MODULES = {+ "gpt_oss": "auto_round.modeling.fused_moe.gpt_oss",+}+++class ModelContext(BaseContext):+ _is_initialized = False++ # model_related+ _model_loaded = False+ _init_model = False+ hook_handles = []++ def __init__(+ self,+ model=None,+ tokenizer=None,+ platform="hf",+ model_dtype=None,+ trust_remote_code=True,+ amp=True,+ need_calib=True,+ device="cpu",+ formats=None,+ is_act_quantize=False,+ quant_nontext_module=False,+ ):+ super().__init__()+ self.quantized = False+ self.is_mllm = False+ self.is_diffusion = False+ self.is_model_patched = False+ self.is_moe_model = False++ assert model is not None, "model must be provided for ModelContext"+ self.model = model+ self.tokenizer = tokenizer+ self.device = device++ # MLLM / diffusion artifacts – always present so callers need no getattr guards.+ # _load_model() will populate the ones that are relevant to the model type.+ self.processor = None+ self.image_processor = None+ self.pipe = None++ if envs.AR_USE_MODELSCOPE:+ platform = "model_scope"+ self.platform = platform+ self.model_dtype = model_dtype+ self.trust_remote_code = trust_remote_code+ self.amp = amp+ self.need_calib = need_calib+ self.quant_nontext_module = quant_nontext_module++ # Load model and run basic initialization eagerly so the model is ready+ # by the time BaseCompressor.post_init() runs.+ self._load_model()++ if unsupported_meta_device(self.model):+ raise RuntimeError(+ "AutoRound does not support parameters on meta device. "+ "Please use more GPUs by setting `--device 0,1,2,3` or just place the model on CPU."+ )+ check_and_mark_quantized_module(self.model)+ self.model = self.model.eval()+ self.shared_cache_keys = get_shared_keys(self.model)++ self.is_moe_model = is_moe_model(self.model)+ self._import_custom_moe_replacements(getattr(self.model, "config", None))++ self._set_amp_dtype()+ if is_act_quantize and self.amp_dtype == torch.float16:+ logger.warning("force to use bf16 for quantization tuning when enabling activation quantization")+ self.amp_dtype = torch.bfloat16+ if self.model.dtype != torch.bfloat16:+ self.model = self.model.to(torch.bfloat16)+ else:+ logger.info(f"using {self.model.dtype} for quantization tuning")++ # Reclaim C heap fragmentation left by model/tokenizer loading so+ # that the quantize loop starts from a tighter RSS baseline.+ gc.collect()+ _force_trim_malloc()++ def _load_model(self):+ if is_mllm_model(self.model, platform=self.platform):+ self.is_mllm = True+ if isinstance(self.model, str):+ self.model, self.processor, self.tokenizer, self.image_processor = mllm_load_model(+ self.model, platform=self.platform, device="cpu", model_dtype=self.model_dtype+ )+ elif is_diffusion_model(self.model):+ self.is_diffusion = True+ self.pipe, self.model = diffusion_load_model(+ self.model, platform=self.platform, device="cpu", model_dtype=self.model_dtype+ )+ elif isinstance(self.model, str):+ config: Optional[AutoConfig] = None+ try:+ config = AutoConfig.from_pretrained(self.model, trust_remote_code=self.trust_remote_code)+ self._import_custom_moe_replacements(config)+ except (OSError, EnvironmentError) as e:+ logger.debug(+ "Failed to load config via AutoConfig.from_pretrained for %s: %s. "+ "Proceeding without config-based checks.",+ self.model,+ e,+ )++ self.is_model_patched = apply_model_monkey_patches(+ model_name=self.model, trust_remote_code=self.trust_remote_code+ )+ import transformers++ if (+ not self.is_model_patched+ and config is not None+ and is_moe_model_via_config(config)+ and version.parse(transformers.__version__) >= version.parse("5.0.0")+ ):+ from auto_round.modeling.fused_moe.replace_modules import BUILTIN_MODULES++ model_type = getattr(config, "model_type", None)+ if model_type is not None and model_type not in BUILTIN_MODULES:+ logger.warning(+ "This MoE model has not been optimized by AutoRound yet, which may result in high RAM usage, "+ "Please consider submitting an issue to https://github.com/intel/auto-round/issues"+ )++ # Reclaim temporary HTTP/config objects from model type detection+ # and AutoConfig loading before the large model allocation. This+ # reduces heap fragmentation especially on HPU where habana internal+ # allocations amplify fragmentation into persistent RSS growth.+ gc.collect()+ _force_trim_malloc()++ self.model, self.tokenizer = llm_load_model(+ self.model,+ platform=self.platform,+ device="cpu", # always load cpu first+ model_dtype=self.model_dtype,+ trust_remote_code=self.trust_remote_code,+ )+ elif self.tokenizer is None and not self.is_diffusion and self.need_calib:+ raise ValueError("A tokenizer must be set for non-str model input")++ self._model_loaded = True++ def _import_custom_moe_replacements(self, model_or_config) -> None:+ model_type = getattr(model_or_config, "model_type", None)+ module_name = _CUSTOM_MOE_REPLACEMENT_MODULES.get(model_type)+ if module_name is None:+ return++ module = importlib.import_module(module_name)+ from auto_round.modeling.fused_moe.replace_modules import BUILTIN_MODULES++ BUILTIN_MODULES.setdefault(model_type, module)+ logger.debug(f"Loaded custom MoE replacement module for {model_type}")++ def _patch_custom_moe_modules(self) -> None:+ model_type = getattr(getattr(self.model, "config", None), "model_type", None)+ if model_type != "qwen3_vl_moe":+ return++ for module in self.model.modules():+ if module.__class__.__name__ != "Qwen3VLMoeTextSparseMoeBlock":+ continue+ if hasattr(module, "top_k"):+ continue++ gate = getattr(module, "gate", None)+ top_k = getattr(gate, "top_k", None)+ if top_k is not None:+ setattr(module, "top_k", top_k)++ def _set_amp_dtype(self) -> None:+ """Sets the automatic mixed precision (AMP) data type for the model based on the device and configuration."""+ self.amp_dtype = torch.bfloat16+ if self.model.dtype != torch.float32:+ self.amp_dtype = self.model.dtype+ if self.device == "cpu" or "hpu" in self.device:+ self.amp_dtype = torch.bfloat16+ if self.amp:+ if self.device == "cpu" and not CpuInfo().bf16:+ self.amp = False+ self.amp_dtype = torch.float32+ self.model = self.model.to(torch.float32)+ logger.warning(+ f"amp is set to FALSE as the current {self.device} device does not support the 'bf16' data type."+ )+ else:+ if self.model.dtype != self.amp_dtype:+ self.model = self.model.to(self.amp_dtype)+ else:+ self.amp_dtype = torch.float32+ self.model = self.model.to(torch.float32)++ def apply_patches(self, formats):+ """Apply format-specific model structure patches.++ Must be called after formats are resolved (list[OutputFormat]) and before+ BaseQuantizers.post_init() so that configure_layer_config() operates on the+ final model structure (post update_module). Eliminates the need for a+ subsequent refresh_quantizer_for_initialized_model() call.+ """+ # It is best to modify the model structure in the quantize function and check the format,+ # because it may cause the gguf format to not be exported normally.+ self._patch_custom_moe_modules()+ self.model = update_module(+ self.model, formats=formats, trust_remote_code=self.trust_remote_code, cleanup_original=False+ )+ self.model = _handle_special_model(self.model)++ # Temporary names must be assigned after handle_moe_model;+ # placing them earlier would cause them to be removed when the module is replaced.+ for n, m in self.model.named_modules():+ m.global_name = n++ if self.amp and self.model.dtype != self.amp_dtype:+ self.model = self.model.to(self.amp_dtype)++ self._init_model = True+ self._is_initialized = True++ def replace_forward(self, register_hook):+ """Replaces the forward function.+ register_hook(layer_name, module, hook_handles)+ """+ assert self._init_model, "should load and initialize model first"+ hook_handles = []++ for n, m in self.model.named_modules():+ register_hook(n, m, hook_handles)++ self.hook_handles = hook_handles++ def recover_forward(self):+ """Recovers the forward function."""+ assert self._init_model, "should load and initialize model first"++ for n, m in self.model.named_modules():+ if hasattr(m, "orig_forward"):+ m.forward = m.orig_forward+ delattr(m, "orig_forward")+ for hook_handle in self.hook_handles:+ hook_handle.remove()+ self.hook_handles = []diff --git a/auto_round/envs.py b/auto_round/envs.pyindex a1a0e4f0d..e216a27e2 100644--- a/auto_round/envs.py+++ b/auto_round/envs.py@@ -34,6 +34,7 @@ environment_variables: dict[str, Callable[[], Any]] = {"AR_DISABLE_DATASET_SUBPROCESS": lambda: os.getenv("AR_DISABLE_DATASET_SUBPROCESS", "0").lower() in ("1", "true"),"AR_DISABLE_COPY_MTP_WEIGHTS": lambda: os.getenv("AR_DISABLE_COPY_MTP_WEIGHTS", "0").lower()in ("1", "true", "yes"),+ "AR_DISABLE_NEW_ARCH": lambda: os.getenv("AR_DISABLE_NEW_ARCH", "0").lower() in ("1", "true", "yes"),"AR_ACT_SCALE": lambda: float(os.getenv("AR_ACT_SCALE", "1.0")),"AR_ENABLE_ACT_MINMAX_TUNING": lambda: os.getenv("AR_ENABLE_ACT_MINMAX_TUNING", "0").lower()in ("1", "true", "yes"),@@ -79,8 +80,8 @@ def set_config(**kwargs):for key, value in kwargs.items():if key in environment_variables:# Convert value to appropriate string format- if key == "AR_USE_MODELSCOPE":- # Handle boolean values for AR_USE_MODELSCOPE+ if key in ("AR_USE_MODELSCOPE", "AR_DISABLE_NEW_ARCH"):+ # Handle boolean values for boolean env flagsstr_value = "true" if value in [True, "True", "true", "1", 1] else "false"else:# For other variables, convert to stringdiff --git a/auto_round/eval/evaluation.py b/auto_round/eval/evaluation.pyindex b3136c32f..e0af76396 100644--- a/auto_round/eval/evaluation.py+++ b/auto_round/eval/evaluation.py@@ -388,8 +388,11 @@ def run_model_evaluation(model, tokenizer, autoround, folders, formats, device_sevaluate_diffusion_model(args, autoround=autoround, model=model)return- # Check if evaluation is needed for language models- eval_folder = folders[-1] if folders else None+ # Check if evaluation Compressoris needed for language models+ if isinstance(folders, list):+ eval_folder = folders[-1] if folders else None+ else:+ eval_folder = foldersif args.tasks is None or args.tasks == "" or eval_folder is None:returndiff --git a/auto_round/experimental/apply_rotation_transform.py b/auto_round/experimental/apply_rotation_transform.pyindex fc7dbf297..520f6b2fb 100644--- a/auto_round/experimental/apply_rotation_transform.py+++ b/auto_round/experimental/apply_rotation_transform.py@@ -1,154 +1,12 @@-# Copyright (c) 2026 Intel Corporation-#-# Licensed under the Apache License, Version 2.0 (the "License");-# you may not use this file except in compliance with the License.-# You may obtain a copy of the License at-#-# http://www.apache.org/licenses/LICENSE-2.0+# # Copyright (C) 2026 Intel Corporation+# # SPDX-License-Identifier: Apache-2.0+"""Backward-compat re-export shim.-"""Unified entry point for Hadamard rotation/transform.--Two backend implementations exist:--* ``inplace`` – :mod:`auto_round.experimental.rotation_inplace`- QuaRot-style residual-stream rotation. Works for any weight/activation- dtype. Optionally fuses the online Hadamard into weights- (``fuse_online_to_weight=True``).-* ``transform`` – :mod:`auto_round.experimental.transform`- Per-Linear weight + activation Hadamard with a fused triton kernel.- Only supports MXFP4 / NVFP4 and **cannot** fuse online to weight.--Routing is controlled by :class:`RotationConfig.backend`:-- "inplace" -> always inplace- "transform" -> always transform (validates dtype + no-fuse)- "auto" -> if user asked to fuse -> inplace- elif data_type is mx_fp / nv_fp -> transform- else -> inplace+The canonical implementation now lives in+:mod:`auto_round.algorithms.transforms.rotation.dispatcher`."""-from __future__ import annotations--from typing import Any, Union--import torch--import auto_round.envs as envs-from auto_round.compressors.utils import is_mx_fp, is_nv_fp-from auto_round.experimental.transform.rotation_config import RotationConfig-from auto_round.experimental.utils import normalize_rotation_config-from auto_round.utils import logger--__all__ = ["apply_hadamard_rotation", "resolve_hadamard_backend"]---def _to_config(- rotation_config: Union[str, dict, RotationConfig, None],- data_type: str,-) -> RotationConfig:- """Normalise *rotation_config* and return a :class:`RotationConfig` instance."""- cfg_dict = normalize_rotation_config(rotation_config, data_type)- if isinstance(cfg_dict, RotationConfig):- return cfg_dict- return RotationConfig.model_validate(cfg_dict or {})---def resolve_hadamard_backend(config: RotationConfig, data_type: str) -> str:- """Resolve the actual backend (``"inplace"`` / ``"transform"``) from config."""- requested = config.backend- fuse_requested = bool(config.fuse_online_to_weight)- allow_online_rotation: bool = config.allow_online_rotation-- if requested == "inplace":- return "inplace"-- transform_backend_name = "transform"- if requested == "transform":- if fuse_requested:- raise ValueError(- f"backend='{transform_backend_name}' does not support fuse_online_to_weight=True. "- "Use backend='inplace' (or backend='auto' with fuse_online_to_weight=True) instead."- )- if not (is_mx_fp(data_type) or is_nv_fp(data_type)):- raise ValueError(- f"backend='{transform_backend_name}' only supports MXFP4 / NVFP4 (got data_type={data_type!r}). "- "Use backend='inplace' or backend='auto' for other dtypes."- )- if not allow_online_rotation:- raise ValueError(f"backend='{transform_backend_name}' only supports `allow_online_rotation`=True")-- return "transform"-- # backend == "auto"- if fuse_requested:- return "inplace"- if is_mx_fp(data_type) or is_nv_fp(data_type):- return "transform"- return "inplace"---def apply_hadamard_rotation(- model: torch.nn.Module,- rotation_config: Union[str, dict, RotationConfig, None],- data_type: str,- compute_device: torch.device | str = None,-) -> (torch.nn.Module, Any):- """Apply Hadamard rotation/transform to *model*, dispatching by backend.-- Args:- model: Target model.- rotation_config: ``str`` / ``dict`` / :class:`RotationConfig` / ``None``.- See :class:`RotationConfig` for fields.- data_type: Quantization data type (e.g. ``"mx_fp"``, ``"nv_fp"``,- ``"int"``, ``"fp"``).- compute_device: Device for inplace-backend computation. Ignored by- the transform backend.-- Returns:- The same model (for chaining); also stored on ``model.rotation_config``.- """- config = _to_config(rotation_config, data_type)- backend = resolve_hadamard_backend(config, data_type)-- # Resolve fuse flag: explicit > env var > default(True)- fuse_online_to_weight = config.fuse_online_to_weight- if config.fuse_online_to_weight is not None:- fuse_online_to_weight = bool(config.fuse_online_to_weight)- elif envs.AR_FUSE_ONLINE_ROTATION:- fuse_online_to_weight = bool(envs.AR_FUSE_ONLINE_ROTATION)-- logger.info(- f"Applying Hadamard (backend={backend}, "- f"data_type={data_type}, fuse_online_to_weight={fuse_online_to_weight if backend == 'inplace' else False})."- )-- if backend == "inplace":- logger.warning("this backend does not support real exporting, please export the model to fake format")- from auto_round.experimental.rotation_inplace import apply_rotation_transform-- # block_size -> group_size (None / -1 / 0 means full-dimension)- bs = config.block_size- group_size = bs if (bs is not None and bs > 0) else None-- model, hooks = apply_rotation_transform(- model,- group_size=group_size,- allow_online_rotation=config.allow_online_rotation,- rotation_matrix=config.hadamard_type,- fuse_online_to_weight=fuse_online_to_weight,- compute_device=compute_device,- )- # Stash for downstream (export / serialization). Plain dict so JSON- # serialization (HF save_pretrained -> config.json) round-trips.- setattr(model, "rotation_config", config.model_dump() if hasattr(config, "model_dump") else config)- return model, hooks-- elif backend == "transform":- supported_hadamard_types = ("hadamard", "random_hadamard")- if config.hadamard_type not in supported_hadamard_types:- raise ValueError("this backend only supports hadamard or random_hadamard")- from auto_round.experimental.transform.apply import apply_rotation_transform-- return apply_rotation_transform(model, config, data_type=data_type)- else:- raise ValueError(f"Unsupported Hadamard backend {backend!r}")+from auto_round.algorithms.transforms.rotation.dispatcher import ( # noqa: F401+ apply_hadamard_rotation,+ resolve_hadamard_backend,+)diff --git a/auto_round/experimental/rotation_inplace/__init__.py b/auto_round/experimental/rotation_inplace/__init__.pyindex 8cdef31b0..07b3d40c8 100644--- a/auto_round/experimental/rotation_inplace/__init__.py+++ b/auto_round/experimental/rotation_inplace/__init__.py@@ -1,5 +1,14 @@# # Copyright (C) 2026 Intel Corporation# # SPDX-License-Identifier: Apache-2.0+"""Backward-compat re-export shim.-from auto_round.experimental.rotation_inplace.apply_rotation_transform import apply_rotation_transform-from auto_round.experimental.rotation_inplace.utils import clear_random_hadamard_cache+The canonical implementation now lives in+:mod:`auto_round.algorithms.transforms.rotation.inplace`.+"""++from auto_round.algorithms.transforms.rotation.inplace.apply import ( # noqa: F401+ apply_rotation_transform,+)+from auto_round.algorithms.transforms.rotation.inplace.hooks import ( # noqa: F401+ clear_random_hadamard_cache,+)diff --git a/auto_round/experimental/rotation_inplace/apply_rotation_transform.py b/auto_round/experimental/rotation_inplace/apply_rotation_transform.pyindex 1052b036f..86e429c10 100644--- a/auto_round/experimental/rotation_inplace/apply_rotation_transform.py+++ b/auto_round/experimental/rotation_inplace/apply_rotation_transform.py@@ -1,882 +1,12 @@# # Copyright (C) 2026 Intel Corporation# # SPDX-License-Identifier: Apache-2.0+"""Backward-compat re-export shim.-"""Hadamard inplace rotation — public API and rotation primitives.--Supports LLaMA-2, LLaMA-3, Qwen-3 (and any model with the same layout).-The entry point is :func:`apply_hadamard_rotation`.+The canonical implementation now lives in+:mod:`auto_round.algorithms.transforms.rotation.inplace.apply`."""-import gc-import typing-from typing import Dict, Union--import torch-import tqdm--from auto_round.experimental.rotation_inplace.model_config import (- MAPPING_REGISTRY,- RotationMapping,- _resolve,- infer_mapping_from_model,-)-from auto_round.experimental.rotation_inplace.utils import (- CrossHeadOnlineHadamardHook,- FullOnlineHadamardHook,- GroupOnlineHadamardHook,- _get_custom_had,- _normalize_rotation_matrix,- _resolve_compute_device,- _rotate_embedding_grouped,- _rotate_linear_grouped,- apply_cross_head_had_to_linear,- apply_exact_had_to_linear,- deterministic_hadamard_matrix,- get_hadK,- get_or_create_random_hadamard,+from auto_round.algorithms.transforms.rotation.inplace.apply import * # noqa: F401, F403+from auto_round.algorithms.transforms.rotation.inplace.apply import ( # noqa: F401+ apply_rotation_transform,)--# ----------------------------------------------------------------------------# Low-level primitives (model-agnostic via RotationMapping)-# ------------------------------------------------------------------------------def _fuse_ln_linear(- layernorm: torch.nn.Module,- linear_layers: typing.Iterable[torch.nn.Linear],-) -> None:- """Fuse the linear operations in LayerNorm into adjacent linear blocks."""- for linear in linear_layers:- linear_dtype = linear.weight.dtype- dev = linear.weight.device-- W_ = linear.weight.data.double()- ln_weight = layernorm.weight.double().to(dev)- linear.weight.data = (W_ * ln_weight).to(linear_dtype)-- if hasattr(layernorm, "bias") and layernorm.bias is not None:- if linear.bias is None:- linear.bias = torch.nn.Parameter(torch.zeros(linear.out_features, dtype=torch.float64, device=dev))- ln_bias = layernorm.bias.double().to(dev)- linear.bias.data = linear.bias.data.double() + torch.matmul(W_, ln_bias)- linear.bias.data = linear.bias.data.to(linear_dtype)---def _reset_ln_params(layernorm: torch.nn.Module) -> None:- """Reset LayerNorm to identity: weight=1, bias=0."""- layernorm.weight.data.fill_(1.0)- if hasattr(layernorm, "bias") and layernorm.bias is not None:- layernorm.bias.data.fill_(0.0)---def _rotate_linear_by_Q(module: torch.nn.Linear, Q: torch.Tensor, side: str, compute_device=None) -> None:- """Apply rotation *Q* to a Linear layer's weight (and bias if present).-- Args:- side: ``'input'`` → W = W @ Q (rotate input side)- ``'output'`` → W = Q^T @ W (rotate output side)- compute_device: Device to run computation on. If None, auto-detects GPU.- """- dtype = module.weight.data.dtype- dev = module.weight.data.device- cdev = _resolve_compute_device(compute_device)- W_ = module.weight.data.to(device=cdev, dtype=torch.float64)- Q_ = Q.to(device=cdev)- if side == "input":- new_W = torch.matmul(W_, Q_).to(device=dev, dtype=dtype)- else:- new_W = torch.matmul(Q_.T, W_).to(device=dev, dtype=dtype)- # Release fp64 copy before assigning back so peak memory ≈ 1× weight + 1× rotated.- del W_- module.weight.data = new_W- if side == "output" and module.bias is not None:- b = module.bias.data.to(device=cdev, dtype=torch.float64)- new_b = torch.matmul(Q_.T, b).to(device=dev, dtype=dtype)- del b- module.bias.data = new_b- del Q_---def _untie_word_embeddings(model, mapping: RotationMapping) -> None:- """Break tied weights between lm_head and embedding if they share the same tensor."""- embedding = _resolve(model, mapping.embedding)- lm_head = _resolve(model, mapping.lm_head)-- if lm_head.weight.data_ptr() != embedding.weight.data_ptr():- return-- lm_head.weight = torch.nn.Parameter(lm_head.weight.data.clone())- if hasattr(model.config, "tie_word_embeddings"):- model.config.tie_word_embeddings = False---def _uses_layernorm_with_mean(model, mapping: RotationMapping) -> bool:- """Check whether the model uses standard LayerNorm (which subtracts mean)."""- layers = _resolve(model, mapping.layers_attr)- first_ln = _resolve(layers[0], mapping.attn_input_ln)- return isinstance(first_ln, torch.nn.LayerNorm)---def _bake_mean_into_linear(linear: torch.nn.Linear) -> None:- """Subtract column-wise mean from a Linear layer's weight (and mean from bias)."""- linear_dtype = linear.weight.dtype- W_ = linear.weight.data.double()- linear.weight.data = (W_ - W_.mean(dim=-2, keepdim=True)).to(linear_dtype)- if linear.bias is not None:- b_ = linear.bias.data.double()- linear.bias.data = (b_ - b_.mean()).to(linear_dtype)---def _subtract_embedding_mean(model, mapping: RotationMapping) -> None:- """Subtract per-row mean from the embedding weight matrix."""- W = _resolve(model, mapping.embedding)- dtype = W.weight.data.dtype- W_ = W.weight.data.to(dtype=torch.float64)- W.weight.data = (W_ - W_.mean(dim=-1, keepdim=True)).to(dtype=dtype)-- if mapping.positional_embedding is not None:- P = _resolve(model, mapping.positional_embedding)- p_dtype = P.weight.data.dtype- P_ = P.weight.data.to(dtype=torch.float64)- P.weight.data = (P_ - P_.mean(dim=-1, keepdim=True)).to(dtype=p_dtype)---class _RMSNorm(torch.nn.Module):- """RMS Normalization (no mean subtraction)."""-- def __init__(self, dim: int, eps: float = 1e-5):- super().__init__()- self.eps = eps- self.register_buffer("weight", torch.ones(dim))-- def forward(self, x: torch.Tensor) -> torch.Tensor:- rms = torch.sqrt(x.pow(2).mean(dim=-1, keepdim=True) + self.eps)- return x / rms * self.weight---def _replace_layernorms_with_rmsnorm(model) -> None:- """Replace all ``nn.LayerNorm`` modules with ``_RMSNorm``."""- replacements = []- for name, module in model.named_modules():- if isinstance(module, torch.nn.LayerNorm):- replacements.append((name, module))-- for name, module in replacements:- parts = name.rsplit(".", 1)- if len(parts) == 2:- parent = _resolve(model, parts[0])- attr = parts[1]- else:- parent = model- attr = parts[0]- rms = _RMSNorm(module.normalized_shape[0], eps=module.eps)- rms = rms.to(device=module.weight.device, dtype=module.weight.dtype)- setattr(parent, attr, rms)---# ----------------------------------------------------------------------------# High-level steps driven by RotationMapping-# ------------------------------------------------------------------------------def _fuse_layer_norms(model, mapping: RotationMapping) -> None:- """Fuse all LayerNorm parameters into adjacent Linear layers."""- layers = _resolve(model, mapping.layers_attr)-- for layer in layers:- mlp_ln = _resolve(layer, mapping.mlp_input_ln)- mlp_linears = [_resolve(layer, p) for p in mapping.mlp_in]- _fuse_ln_linear(mlp_ln, mlp_linears)- _reset_ln_params(mlp_ln)-- attn_ln = _resolve(layer, mapping.attn_input_ln)- attn_linears = [- _resolve(layer, mapping.attn_q),- _resolve(layer, mapping.attn_k),- _resolve(layer, mapping.attn_v),- ]- _fuse_ln_linear(attn_ln, attn_linears)- _reset_ln_params(attn_ln)-- pre_head_ln = _resolve(model, mapping.pre_head_ln)- lm_head = _resolve(model, mapping.lm_head)- _fuse_ln_linear(pre_head_ln, [lm_head])- _reset_ln_params(pre_head_ln)---# ----------------------------------------------------------------------------# Unified weight rotation (full or grouped)-# -----------------------------------------------------------------------------[email protected]_mode()-def _rotate_weights(- model,- mapping: RotationMapping,- use_fast_had: bool = True,- group_size: int = None,- compute_device: torch.device = None,- had_dict: dict = None,- preset: str = None,- fuse_online_to_weight: bool = True,-) -> None:- """Apply Hadamard rotation to all weights.-- Args:- group_size: ``None`` → full Hadamard rotation.- ``int`` → block-diagonal rotation with this block size.- compute_device: Device to run Hadamard computation on (e.g. ``"cuda:0"``).- Weights are moved there temporarily and moved back afterwards.- If ``None``, auto-detects GPU availability.- allow_online_rotation: If ``True`` (default), apply extra input-side- Hadamard rotations on ``down_proj`` and the OV pair (``v_proj``- output + ``o_proj`` input) that require compensating online hooks- at inference time. If ``False``, skip those extra rotations so- that **no** online hooks are needed.- had_dict: Normalized ``dict[int, Tensor]`` of custom Hadamard matrices- (keyed by dimension). Only used in grouped mode.- preset: Rotation preset name (``"quarot_hadamard"``, ``"hadamard"``,- ``"random_hadamard"``, or ``None``).-- * ``"quarot_hadamard"``: fusable (residual-stream) rotations use- ``fast_hadamard_transform`` / random Hadamard; non-fusable- (online-paired) rotations and their weight-side counterparts use- deterministic ``get_hadK``/``matmul_hadU`` so that the online- hook at inference produces the exact same transform.- * ``"hadamard"``: all rotations use deterministic ``get_hadK`` /- ``matmul_hadU``. Full-mode Q is a deterministic Hadamard matrix.- * ``"random_hadamard"``: all rotations use random Hadamard matrices- from the global cache (``get_or_create_random_hadamard``).- Same dimension → same matrix everywhere.- * ``None``: same behaviour as ``"hadamard"`` (built-in butterfly).- """- compute_device = _resolve_compute_device(compute_device)- config = model.config- hidden_size = getattr(config, mapping.hidden_size_attr)- intermediate_size = getattr(config, mapping.intermediate_size_attr)- num_heads = getattr(config, mapping.num_heads_attr)- head_dim = mapping.attn_head_dim or (hidden_size // num_heads)-- is_grouped = group_size is not None and group_size > 0- desc = f"Rotating (group_size={group_size})" if is_grouped else "Rotating"-- # ----- Resolve per-operation Hadamard sources ------ fused_fast = use_fast_had- online_fast = False- if preset == "random_hadamard":- fused_fast = False-- # -- Matrix resolution --- had_matrix, _found = _get_custom_had(had_dict, group_size) if is_grouped else (None, False)-- online_had_matrix = had_matrix- if preset == "random_hadamard" and had_matrix is None:- had_matrix = get_or_create_random_hadamard(group_size if is_grouped else hidden_size, compute_device)- online_had_matrix = had_matrix- if preset == "quarot_hadamard" and is_grouped:- online_had_matrix = None # force deterministic for online-paired-- # -- Helper: look up cached random matrix for online-paired ops --- def _online_had(dim):- """Return cached random matrix for *dim* under random_hadamard, else None."""- if preset == "random_hadamard":- return get_or_create_random_hadamard(dim, compute_device)- return None-- if is_grouped:- assert hidden_size % group_size == 0, f"group_size={group_size} must divide hidden_size={hidden_size}"- assert (- intermediate_size % group_size == 0- ), f"group_size={group_size} must divide intermediate_size={intermediate_size}"-- # --- Full mode: build Hadamard matrix Q ---- Q = None- if not is_grouped:- if preset == "hadamard":- Q = deterministic_hadamard_matrix(hidden_size, compute_device)- else:- # "random_hadamard", "quarot_hadamard", None — same shape → same matrix- Q = get_or_create_random_hadamard(hidden_size, compute_device)-- # ---- Top-level: embedding / lm_head ----- # When fuse_online_to_weight=False, skip embedding and lm_head rotation:- # each layer is self-contained (weight rotation + online hook cancel out).- if fuse_online_to_weight:- embedding = _resolve(model, mapping.embedding)- if is_grouped:- _rotate_embedding_grouped(- embedding, group_size, use_fast_had=fused_fast, compute_device=compute_device, had_matrix=had_matrix- )- else:- dtype = embedding.weight.data.dtype- dev = embedding.weight.data.device- cdev = compute_device- W_ = embedding.weight.data.to(device=cdev, dtype=torch.float64)- new_W = torch.matmul(W_, Q.to(cdev)).to(device=dev, dtype=dtype)- del W_- embedding.weight.data = new_W-- if mapping.positional_embedding is not None:- pos_emb = _resolve(model, mapping.positional_embedding)- if is_grouped:- _rotate_embedding_grouped(- pos_emb, group_size, use_fast_had=fused_fast, compute_device=compute_device, had_matrix=had_matrix- )- else:- pos_dtype = pos_emb.weight.data.dtype- pos_dev = pos_emb.weight.data.device- cdev = compute_device- P_ = pos_emb.weight.data.to(device=cdev, dtype=torch.float64)- new_P = torch.matmul(P_, Q.to(cdev)).to(device=pos_dev, dtype=pos_dtype)- del P_- pos_emb.weight.data = new_P-- # ---- Top-level: lm_head ----- lm_head = _resolve(model, mapping.lm_head)- if is_grouped:- _rotate_linear_grouped(- lm_head,- group_size,- side="input",- use_fast_had=fused_fast,- compute_device=compute_device,- had_matrix=had_matrix,- )- else:- _rotate_linear_by_Q(lm_head, Q, side="input", compute_device=compute_device)-- gc.collect()- if torch.cuda.is_available():- torch.cuda.empty_cache()-- # ---- Per-layer rotation ----- layers = _resolve(model, mapping.layers_attr)- for layer in tqdm.tqdm(layers, unit="layer", desc=desc):- if fuse_online_to_weight:- # ---- fuse mode: QuaRot-style residual stream rotation ----- # Q/K/V: only residual Q on input (no online Had stacking, no hook).- # When Q == online Had (e.g. preset="hadamard"), Q @ Q = I cancels- # the rotation entirely, destroying quantization benefit.- # gate/up: only residual Q on input (no online Had stacking, no hook).- # down_proj: residual Q^T on output + online Had on input (+ hook).- # v_proj/o_proj: per-head/cross-head Had below (+ hook on o_proj).- for attr in (mapping.attn_q, mapping.attn_k, mapping.attn_v):- mod = _resolve(layer, attr)- if is_grouped:- _rotate_linear_grouped(- mod,- group_size,- side="input",- use_fast_had=fused_fast,- compute_device=compute_device,- had_matrix=had_matrix,- )- else:- _rotate_linear_by_Q(mod, Q, side="input", compute_device=compute_device)-- # o_proj: residual stream output rotation- if is_grouped:- _rotate_linear_grouped(- _resolve(layer, mapping.attn_o),- group_size,- side="output",- use_fast_had=fused_fast,- compute_device=compute_device,- had_matrix=had_matrix,- )- else:- _rotate_linear_by_Q(_resolve(layer, mapping.attn_o), Q, side="output", compute_device=compute_device)-- # gate/up: only residual Q on input- for attr in mapping.mlp_in:- mod = _resolve(layer, attr)- if is_grouped:- _rotate_linear_grouped(- mod,- group_size,- side="input",- use_fast_had=fused_fast,- compute_device=compute_device,- had_matrix=had_matrix,- )- else:- _rotate_linear_by_Q(mod, Q, side="input", compute_device=compute_device)-- # down_proj: residual output + online input Had- down_proj = _resolve(layer, mapping.mlp_out)- if is_grouped:- _rotate_linear_grouped(- down_proj,- group_size,- side="output",- use_fast_had=fused_fast,- compute_device=compute_device,- had_matrix=had_matrix,- )- _rotate_linear_grouped(- down_proj,- group_size,- side="input",- use_fast_had=online_fast,- compute_device=compute_device,- had_matrix=online_had_matrix,- )- else:- _rotate_linear_by_Q(down_proj, Q, side="output", compute_device=compute_device)- apply_exact_had_to_linear(- down_proj,- had_dim=-1,- output=False,- use_fast_had=online_fast,- compute_device=compute_device,- had_matrix=_online_had(intermediate_size),- )-- # OV projection: v_proj per-head output + o_proj full/cross-head input- v_proj = _resolve(layer, mapping.attn_v)- o_proj = _resolve(layer, mapping.attn_o)- if is_grouped:- pass- else:- online_head_had = _online_had(head_dim)- apply_exact_had_to_linear(- v_proj,- had_dim=head_dim,- output=True,- use_fast_had=online_fast,- compute_device=compute_device,- had_matrix=online_head_had,- )- if preset == "random_hadamard":- apply_exact_had_to_linear(- o_proj,- had_dim=head_dim,- output=False,- use_fast_had=online_fast,- compute_device=compute_device,- had_matrix=online_head_had,- )- apply_cross_head_had_to_linear(- o_proj,- num_heads,- head_dim,- use_fast_had=online_fast,- compute_device=compute_device,- had_matrix=_online_had(num_heads),- )- else:- apply_exact_had_to_linear(- o_proj,- had_dim=-1,- output=False,- use_fast_had=online_fast,- compute_device=compute_device,- )-- else:- # ---- unfused mode: no residual rotation, only input-side Had ----- # Each layer gets Had fused on input side + compensating hook → equivalent.- # No embedding/lm_head rotation. No self-cancelling pair.- # v_proj treated same as Q/K (input Had only, no per-head/cross-head).-- # Q/K/V: input-side Had on hidden_size- for attr in (mapping.attn_q, mapping.attn_k, mapping.attn_v):- mod = _resolve(layer, attr)- if is_grouped:- _rotate_linear_grouped(- mod,- group_size,- side="input",- use_fast_had=online_fast,- compute_device=compute_device,- had_matrix=online_had_matrix,- )- else:- apply_exact_had_to_linear(- mod,- had_dim=-1,- output=False,- use_fast_had=online_fast,- compute_device=compute_device,- had_matrix=_online_had(hidden_size),- )-- # o_proj: input-side Had on hidden_size (full Had, not cross-head)- o_proj = _resolve(layer, mapping.attn_o)- if is_grouped:- _rotate_linear_grouped(- o_proj,- group_size,- side="input",- use_fast_had=online_fast,- compute_device=compute_device,- had_matrix=online_had_matrix,- )- else:- apply_exact_had_to_linear(- o_proj,- had_dim=-1,- output=False,- use_fast_had=online_fast,- compute_device=compute_device,- had_matrix=_online_had(hidden_size),- )-- # gate/up: input-side Had on hidden_size- for attr in mapping.mlp_in:- mod = _resolve(layer, attr)- if is_grouped:- _rotate_linear_grouped(- mod,- group_size,- side="input",- use_fast_had=online_fast,- compute_device=compute_device,- had_matrix=online_had_matrix,- )- else:- apply_exact_had_to_linear(- mod,- had_dim=-1,- output=False,- use_fast_had=online_fast,- compute_device=compute_device,- had_matrix=_online_had(hidden_size),- )-- # down_proj: input-side Had on intermediate_size- down_proj = _resolve(layer, mapping.mlp_out)- if is_grouped:- _rotate_linear_grouped(- down_proj,- group_size,- side="input",- use_fast_had=online_fast,- compute_device=compute_device,- had_matrix=online_had_matrix,- )- else:- apply_exact_had_to_linear(- down_proj,- had_dim=-1,- output=False,- use_fast_had=online_fast,- compute_device=compute_device,- had_matrix=_online_had(intermediate_size),- )-- # Per-layer cleanup: drop fp64 temporaries and CUDA caching allocator- # blocks so peak memory stays at ~1 layer's worth instead of accumulating- # across all 32+ decoder layers (was the main cause of 33 GB RAM on 8B).- gc.collect()- if torch.cuda.is_available():- torch.cuda.empty_cache()---# ----------------------------------------------------------------------------# Unified online hook registration-# ------------------------------------------------------------------------------def _register_online_hooks(- model,- mapping: RotationMapping,- fp32_had: bool = False,- use_fast_had: bool = True,- group_size: int = None,- had_dict: dict = None,- preset: str = None,- fuse_online_to_weight: bool = True,-):- """Register online Hadamard pre-forward hooks on ``down_proj`` and ``o_proj``.-- Online hooks must use the **same** Hadamard matrix that was applied to the- weight-side counterpart during ``_rotate_weights``. For ``quarot_hadamard``- this is always the deterministic ``get_hadK``/``matmul_hadU`` path- (``use_fast_had=False``). For ``"random_hadamard"`` it is the random matrix that- was generated once and stored in ``had_dict``.-- Args:- group_size: ``None`` → full Hadamard hooks (original QuaRot).- ``int`` → per-group Hadamard hooks.- had_dict: Normalized ``dict[int, Tensor]`` of custom Hadamard matrices.- preset: Rotation preset name.- Returns:- list of hook handles.- """- config = model.config- num_heads = getattr(config, mapping.num_heads_attr)- hidden_size = getattr(config, mapping.hidden_size_attr)- intermediate_size = getattr(config, mapping.intermediate_size_attr)- head_dim = mapping.attn_head_dim or (hidden_size // num_heads)-- is_grouped = group_size is not None and group_size > 0-- # Online hooks always use deterministic (fixed) Hadamard — never fast_had- # for quarot_hadamard; for "random_hadamard" they use the same random matrix- # that was cached in had_dict by _rotate_weights.- online_fast = False-- # -- Matrix resolution (must match the *online-paired* matrix used by- # _rotate_weights for down_proj input / OV pair). Variable name kept in- # sync with _rotate_weights to make any future drift obvious.- online_had_matrix, _ = _get_custom_had(had_dict, group_size) if is_grouped else (None, False)- if preset == "random_hadamard" and online_had_matrix is None:- online_had_matrix = get_or_create_random_hadamard(group_size if is_grouped else hidden_size)- if preset == "quarot_hadamard" and is_grouped:- online_had_matrix = None-- # -- Helper: look up cached random matrix for online-paired hooks --- def _online_had(dim):- if preset == "random_hadamard":- return get_or_create_random_hadamard(dim)- return None-- mlp_out_suffix = mapping.mlp_out.split(".")[-1]- attn_o_suffix = mapping.attn_o.split(".")[-1]-- # Suffixes for Q/K/V and gate/up (for online input Had hooks)- attn_qkv_suffixes = set(attr.split(".")[-1] for attr in (mapping.attn_q, mapping.attn_k, mapping.attn_v))- mlp_in_suffixes = set(attr.split(".")[-1] for attr in mapping.mlp_in)-- # --- Build hook factories ---- def _make_down_proj_hook():- if is_grouped:- return GroupOnlineHadamardHook(- group_size=group_size, fp32_had=fp32_had, use_fast_had=online_fast, had_matrix=online_had_matrix- )- online_mat = _online_had(intermediate_size)- if online_mat is not None:- return FullOnlineHadamardHook(- had_K=None, K=None, fp32_had=fp32_had, use_fast_had=online_fast, had_matrix=online_mat- )- had_K, K = get_hadK(intermediate_size)- return FullOnlineHadamardHook(had_K=had_K, K=K, fp32_had=fp32_had, use_fast_had=online_fast)-- def _make_hidden_had_hook():- """Full Had hook on hidden_size (for Q/K/V and gate/up input)."""- if is_grouped:- return GroupOnlineHadamardHook(- group_size=group_size, fp32_had=fp32_had, use_fast_had=online_fast, had_matrix=online_had_matrix- )- online_mat = _online_had(hidden_size)- if online_mat is not None:- return FullOnlineHadamardHook(- had_K=None, K=None, fp32_had=fp32_had, use_fast_had=online_fast, had_matrix=online_mat- )- had_K, K = get_hadK(hidden_size)- return FullOnlineHadamardHook(had_K=had_K, K=K, fp32_had=fp32_had, use_fast_had=online_fast)-- def _make_o_proj_hook():- online_mat = _online_had(num_heads)- if online_mat is not None:- return CrossHeadOnlineHadamardHook(- had_K=None,- K=None,- head_dim=head_dim,- fp32_had=fp32_had,- use_fast_had=online_fast,- had_matrix=online_mat,- )- had_K, K = get_hadK(num_heads)- return CrossHeadOnlineHadamardHook(- had_K=had_K,- K=K,- head_dim=head_dim,- fp32_had=fp32_had,- use_fast_had=online_fast,- )-- # --- Register ---- handles = []-- for name, module in model.named_modules():- if not isinstance(module, torch.nn.Linear):- continue- suffix = name.split(".")[-1]-- if name.endswith(mlp_out_suffix):- # down_proj: full Had on intermediate_size input- h = module.register_forward_pre_hook(_make_down_proj_hook())- handles.append(h)- elif name.endswith(attn_o_suffix):- if fuse_online_to_weight and not is_grouped:- # o_proj: cross-head Had on input (fused mode, full only)- h = module.register_forward_pre_hook(_make_o_proj_hook())- handles.append(h)- elif not fuse_online_to_weight:- # o_proj: full Had on hidden_size input (unfused mode, matches weight rotation)- h = module.register_forward_pre_hook(_make_hidden_had_hook())- handles.append(h)- elif suffix in attn_qkv_suffixes:- if not fuse_online_to_weight:- # Q/K/V: full Had on hidden_size input (unfused mode only).- # In fused mode Q/K/V only have residual Q on weight (no online Had),- # and activations come pre-rotated from residual stream → no hook needed.- h = module.register_forward_pre_hook(_make_hidden_had_hook())- handles.append(h)- elif suffix in mlp_in_suffixes:- if not fuse_online_to_weight:- # gate/up: full Had on hidden_size input (unfused mode only).- # Same reasoning as Q/K/V above.- h = module.register_forward_pre_hook(_make_hidden_had_hook())- handles.append(h)-- return handles---# ----------------------------------------------------------------------------# Public API-# ------------------------------------------------------------------------------def apply_rotation_transform(- model,- group_size: int = None,- allow_online_rotation: bool = True,- rotation_matrix: Union[str, torch.Tensor, Dict[int, torch.Tensor], None] = None,- compute_device: torch.device | str = None,- fp32_had: bool = False,- fuse_online_to_weight: bool = None,-):- """Fuse layer norms, rotate weights, and register online Hadamard hooks.-- This is the single entry point for applying Hadamard inplace rotation.- The model architecture is auto-detected via ``model.config.model_type``.-- Args:- model: A HuggingFace CausalLM model (LLaMA-2/3, Qwen-3, etc.).- fp32_had: Whether to compute the online Hadamard transform in fp32.- group_size: If ``None`` (default), use full-dimension Hadamard rotation.- compute_device: Device to run Hadamard computation on.- allow_online_rotation: If ``True`` (default), apply online Hadamard- rotations on ``down_proj`` input and the OV pair.- rotation_matrix: Rotation matrix selection (``"hadamard"``,- ``"random_hadamard"``, ``"quarot_hadamard"``, Tensor, dict, or None).- fuse_online_to_weight: If ``True`` (default), fuse online Hadamard- rotation into weights (down_proj input, v_proj output, o_proj input)- and register compensating online hooks. If ``False``, skip- embedding/lm_head rotation; each linear layer is self-contained- with input-side Had on weight + compensating online hook on- activation. No v_proj cross-head or inner-head rotation.-- Returns:- list of hook handles."""- if fuse_online_to_weight is None:- if model.config.model_type in MAPPING_REGISTRY or model.__class__.__name__ in MAPPING_REGISTRY:- fuse_online_to_weight = True- else:- fuse_online_to_weight = False- had_dict, use_fast_had, preset = _normalize_rotation_matrix(rotation_matrix, group_size)- compute_device = _resolve_compute_device(compute_device)-- if use_fast_had:- from auto_round.utils import logger-- try:- import fast_hadamard_transform # noqa: F401-- if group_size is None:- logger.warning(- "fast_hadamard_transform uses a different Hadamard matrix than the "- "default implementation. Please ensure consistency between training "- "and inference. This will be refined later."- )- except ImportError:- logger.warning("Importing fast_hadamard_transform failed, falling back to default implementation.")- use_fast_had = False-- mapping = infer_mapping_from_model(model)-- _untie_word_embeddings(model, mapping)-- if _uses_layernorm_with_mean(model, mapping):- _subtract_embedding_mean(model, mapping)-- _fuse_layer_norms(model, mapping)-- if _uses_layernorm_with_mean(model, mapping):- layers = _resolve(model, mapping.layers_attr)- for layer in layers:- _bake_mean_into_linear(_resolve(layer, mapping.attn_o))- _bake_mean_into_linear(_resolve(layer, mapping.mlp_out))- _replace_layernorms_with_rmsnorm(model)-- _rotate_weights(- model,- mapping,- use_fast_had=use_fast_had,- group_size=group_size,- compute_device=compute_device,- had_dict=had_dict,- preset=preset,- fuse_online_to_weight=fuse_online_to_weight,- )-- handles = []- if fuse_online_to_weight or allow_online_rotation:- handles = _register_online_hooks(- model,- mapping,- fp32_had=fp32_had,- use_fast_had=use_fast_had,- group_size=group_size,- had_dict=had_dict,- preset=preset,- fuse_online_to_weight=fuse_online_to_weight,- )-- return model, handles---# ----------------------------------------------------------------------------# Quick smoke test-# -----------------------------------------------------------------------------if __name__ == "__main__":- from transformers import AutoModelForCausalLM, AutoTokenizer-- model_name = "/models/opt-125m"- tokenizer = AutoTokenizer.from_pretrained(model_name)- model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype=torch.float16, device_map="auto")- model.to("cuda")- text = "There is a girl who likes adventure,"- inputs = tokenizer(text, return_tensors="pt").to(model.device)- print(tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0]))-- apply_rotation_transform(- model, group_size=-1, allow_online_rotation=True, rotation_matrix="random_hadamard", fuse_online_to_weight=False- )- model.to("cuda")- text = "There is a girl who likes adventure,"- inputs = tokenizer(text, return_tensors="pt").to(model.device)- print(tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0]))-- model_name = "/models/Qwen3-8B"- tokenizer = AutoTokenizer.from_pretrained(model_name)- model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype=torch.float16, device_map="auto")- apply_rotation_transform(model, group_size=-1, allow_online_rotation=True, fuse_online_to_weight=True)- model.to("cuda")- text = "There is a girl who likes adventure,"- inputs = tokenizer(text, return_tensors="pt").to(model.device)- print(tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0]))-- from transformers import AutoModelForCausalLM, AutoTokenizer-- model_name = "/models/Meta-Llama-3.1-8B-Instruct"- tokenizer = AutoTokenizer.from_pretrained(model_name)- model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype=torch.float16, device_map="auto")- apply_rotation_transform(model, fuse_online_to_weight=True, group_size=32)- model.to("cuda")- text = "There is a girl who likes adventure,"- inputs = tokenizer(text, return_tensors="pt").to(model.device)- print(tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0]))- #- # model_name = "/models/Llama-2-7b-chat-hf"- # tokenizer = AutoTokenizer.from_pretrained(model_name)- # model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype=torch.float16, device_map="auto")- # apply_hadamard_rotation(model)- # model.to("cuda")- # text = "There is a girl who likes adventure,"- # inputs = tokenizer(text, return_tensors="pt").to(model.device)- # print(tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0]))diff --git a/auto_round/experimental/rotation_inplace/model_config.py b/auto_round/experimental/rotation_inplace/model_config.pyindex 3ecbf9b69..35078cd28 100644--- a/auto_round/experimental/rotation_inplace/model_config.py+++ b/auto_round/experimental/rotation_inplace/model_config.py@@ -1,169 +1,9 @@# # Copyright (C) 2026 Intel Corporation# # SPDX-License-Identifier: Apache-2.0+"""Backward-compat re-export shim.-"""Model architecture mapping for Hadamard rotation.--Each :class:`RotationMapping` describes *where* the rotation-relevant modules-live inside a model. Currently supports LLaMA-2, LLaMA-3, and Qwen-3 (dense).--New architectures can be supported by calling :func:`register_mapping`.+The canonical implementation now lives in+:mod:`auto_round.algorithms.transforms.rotation.inplace.model_config`."""-from __future__ import annotations--from dataclasses import dataclass, field-from typing import Dict, List, Optional--from auto_round.utils import logger--__all__ = [- "RotationMapping",- "register_mapping",- "get_mapping",- "infer_mapping_from_model",- "MAPPING_REGISTRY",-]---# ----------------------------------------------------------------------------# Mapping dataclass-# ------------------------------------------------------------------------------@dataclass-class RotationMapping:- """Declarative description of a transformer architecture for Hadamard rotation.-- Attribute names follow the dot-path convention relative to the model or- each decoder layer.-- Config attribute names (read from ``model.config``):- num_heads_attr, hidden_size_attr, intermediate_size_attr- head_dim_override – explicit head dim (skip hidden_size // num_heads)- """-- # -- top-level modules (dot-path from model root) --- embedding: str = "model.embed_tokens"- lm_head: str = "lm_head"- positional_embedding: Optional[str] = None # e.g. "model.decoder.embed_positions" for OPT-- # -- layers container (dot-path from model root) --- layers_attr: str = "model.layers"-- # -- per-layer: attention (dot-path from each layer) --- attn_input_ln: str = "input_layernorm"- attn_q: str = "self_attn.q_proj"- attn_k: str = "self_attn.k_proj"- attn_v: str = "self_attn.v_proj"- attn_o: str = "self_attn.o_proj"-- # -- per-layer: MLP (dot-path from each layer) --- mlp_input_ln: str = "post_attention_layernorm"- mlp_in: List[str] = field(default_factory=lambda: ["mlp.up_proj", "mlp.gate_proj"])- mlp_out: str = "mlp.down_proj"-- # -- final norm (dot-path from model root) --- pre_head_ln: str = "model.norm"-- # -- head dim override (None = hidden_size // num_heads) --- attn_head_dim: Optional[int] = None-- # -- config attr names --- num_heads_attr: str = "num_attention_heads"- hidden_size_attr: str = "hidden_size"- intermediate_size_attr: str = "intermediate_size"---# ----------------------------------------------------------------------------# Helper: resolve a dot-path attribute on a module-# ------------------------------------------------------------------------------def _resolve(root, dot_path: str):- """Resolve ``'a.b.c'`` to ``root.a.b.c``."""- obj = root- for attr in dot_path.split("."):- obj = getattr(obj, attr)- return obj---# ----------------------------------------------------------------------------# Registry-# -----------------------------------------------------------------------------MAPPING_REGISTRY: Dict[str, RotationMapping] = {}---def register_mapping(key: str, mapping: RotationMapping) -> RotationMapping:- """Register a :class:`RotationMapping` under *key* (model_type or architecture)."""- MAPPING_REGISTRY[key] = mapping- return mapping---def get_mapping(key: str) -> RotationMapping:- """Look up a mapping by *key*; fall back to default if not found."""- if key in MAPPING_REGISTRY:- return MAPPING_REGISTRY[key]- logger.warning(f"No rotation mapping registered for '{key}', " "falling back to default (LLaMA-like) mapping.")- return RotationMapping()---def infer_mapping_from_model(model) -> RotationMapping:- """Return the best :class:`RotationMapping` for *model*.-- Tries ``model.config.model_type`` first, then ``model.__class__.__name__``.- """- model_type = getattr(getattr(model, "config", None), "model_type", "")- if model_type in MAPPING_REGISTRY:- return MAPPING_REGISTRY[model_type]-- arch = model.__class__.__name__- if arch in MAPPING_REGISTRY:- return MAPPING_REGISTRY[arch]-- logger.warning(- f"Unrecognised architecture '{arch}' (model_type='{model_type}'). "- "Falling back to default (LLaMA-like) mapping."- )- return RotationMapping()---# ===================================================================-# Built-in mappings-# ===================================================================--# LLaMA-2 / LLaMA-3 / Mistral / Yi — all share the same layout-_default = RotationMapping()--register_mapping("llama", _default)-register_mapping("LlamaForCausalLM", _default)--# Qwen-3 dense — identical layout to LLaMA-register_mapping("qwen3", _default)-register_mapping("Qwen3ForCausalLM", _default)--# Qwen-2 / Qwen-2.5 dense — identical layout to LLaMA-register_mapping("qwen2", _default)-register_mapping("Qwen2ForCausalLM", _default)--# ---- OPT -----# OPT uses standard LayerNorm (with bias, subtracts mean),-# different module names, and tied lm_head ↔ embedding weights.-_opt = RotationMapping(- embedding="model.decoder.embed_tokens",- lm_head="lm_head",- positional_embedding="model.decoder.embed_positions",- layers_attr="model.decoder.layers",- attn_input_ln="self_attn_layer_norm",- attn_q="self_attn.q_proj",- attn_k="self_attn.k_proj",- attn_v="self_attn.v_proj",- attn_o="self_attn.out_proj",- mlp_input_ln="final_layer_norm",- mlp_in=["fc1"],- mlp_out="fc2",- pre_head_ln="model.decoder.final_layer_norm",- intermediate_size_attr="ffn_dim",-)-register_mapping("opt", _opt)-register_mapping("OPTForCausalLM", _opt)+from auto_round.algorithms.transforms.rotation.inplace.model_config import * # noqa: F401, F403diff --git a/auto_round/experimental/rotation_inplace/utils.py b/auto_round/experimental/rotation_inplace/utils.pyindex 04bb18981..4ddd80d48 100644--- a/auto_round/experimental/rotation_inplace/utils.py+++ b/auto_round/experimental/rotation_inplace/utils.py@@ -1,786 +1,9 @@# # Copyright (C) 2026 Intel Corporation# # SPDX-License-Identifier: Apache-2.0+"""Backward-compat re-export shim.-"""Online Hadamard transform hooks.--After weight rotation, down_proj and o_proj require an online Hadamard-transform on their *input activations* at inference time. This module-provides the hooks and a helper to register them on the model.+The canonical implementation now lives in+:mod:`auto_round.algorithms.transforms.rotation.inplace.hooks`."""-import math--import torch-import torch.nn as nn--try:- import fast_hadamard_transform-except ImportError:- fast_hadamard_transform = None---def _resolve_compute_device(compute_device) -> torch.device:- """Return *compute_device* if explicitly given, otherwise auto-detect GPU.-- When ``compute_device`` is ``None`` the function checks for CUDA / XPU- availability and returns the first accelerator it finds so that heavy- matrix operations are offloaded to GPU even when the model weights live- on CPU. Falls back to ``torch.device("cpu")`` when no accelerator is- present.- """- if compute_device is not None:- return torch.device(compute_device) if not isinstance(compute_device, torch.device) else compute_device- if torch.cuda.is_available():- return torch.device("cuda:0")- if hasattr(torch, "xpu") and torch.xpu.is_available():- return torch.device("xpu:0")- return torch.device("cpu")---BUILTIN_ROTATION_PRESETS = {"quarot_hadamard", "hadamard", "random_hadamard"}--# Global cache for random Hadamard matrices keyed by dimension.-# Ensures the same shape always returns the exact same random matrix within-# a process, across all calls to ``_rotate_weights`` / ``_register_online_hooks``.-_RANDOM_HADAMARD_CACHE: dict = {}---def get_or_create_random_hadamard(dim: int, device=None) -> torch.Tensor:- """Return a random Hadamard matrix for *dim*, creating and caching it if needed.-- The matrix is cached globally in ``_RANDOM_HADAMARD_CACHE`` so that every- caller that requests the same *dim* receives the identical matrix.- """- if dim in _RANDOM_HADAMARD_CACHE:- mat = _RANDOM_HADAMARD_CACHE[dim]- if device is not None:- mat = mat.to(device)- return mat- mat = random_hadamard_matrix(dim, device or torch.device("cpu"))- _RANDOM_HADAMARD_CACHE[dim] = mat- return mat---def clear_random_hadamard_cache():- """Clear the global random Hadamard matrix cache.-- Call this when you want subsequent ``random_hadamard`` preset runs to- generate fresh random matrices (e.g. between independent experiments).- """- _RANDOM_HADAMARD_CACHE.clear()---def _normalize_rotation_matrix(rotation_matrix, group_size):- """Normalize ``rotation_matrix`` into a ``(had_dict, use_fast_had, preset)`` tuple.-- Accepted inputs:- * ``None`` → ``(None, False, None)`` — use built-in butterfly ``matmul_hadU``.- * ``"quarot_hadamard"`` → ``(None, True, "quarot_hadamard")`` — fusable- rotations use ``fast_hadamard_transform`` (random); non-fusable- (online-paired) rotations use deterministic ``get_hadK``/``matmul_hadU``.- * ``"hadamard"`` → ``(None, False, "hadamard")`` — all rotations use- deterministic ``get_hadK``/``matmul_hadU``.- * ``"random_hadamard"`` → ``(None, False, "random_hadamard")`` — all rotations use- ``random_hadamard_matrix``.- * A ``torch.Tensor`` of shape ``(n, n)`` → ``({n: tensor}, False, None)``.- * A ``dict[int, Tensor]`` → ``(dict, False, None)`` — returned as-is.-- Returns:- ``(had_dict, use_fast_had, preset)``-- Raises:- ValueError: if a non-``str`` *rotation_matrix* is given but- *group_size* is not a positive integer, or an unknown preset.- """- if rotation_matrix is None:- return None, False, None-- if isinstance(rotation_matrix, str):- if rotation_matrix not in BUILTIN_ROTATION_PRESETS:- raise ValueError(- f"Unknown rotation_matrix preset '{rotation_matrix}'. "- f"Supported presets: {BUILTIN_ROTATION_PRESETS}."- )- if rotation_matrix == "quarot_hadamard":- return None, True, "quarot_hadamard"- elif rotation_matrix == "hadamard":- return None, False, "hadamard"- else: # "random_hadamard"- return None, False, "random_hadamard"-- is_grouped = group_size is not None and group_size > 0- if not is_grouped and not isinstance(rotation_matrix, dict):- raise ValueError(- "rotation_matrix (Tensor/dict) can only be used with a positive group_size. "- f"Got group_size={group_size}."- )-- if isinstance(rotation_matrix, torch.Tensor):- assert (- rotation_matrix.ndim == 2 and rotation_matrix.shape[0] == rotation_matrix.shape[1]- ), f"rotation_matrix must be square, got shape {rotation_matrix.shape}"- return {rotation_matrix.shape[0]: rotation_matrix}, False, None-- if isinstance(rotation_matrix, dict):- for k, t in rotation_matrix.items():- assert (- isinstance(t, torch.Tensor) and t.ndim == 2 and t.shape[0] == t.shape[1]- ), f"rotation_matrix[{k}] must be a square tensor, got shape {t.shape}"- return rotation_matrix, False, None-- raise TypeError(- f"rotation_matrix must be a Tensor, dict[int, Tensor], str, or None. " f"Got {type(rotation_matrix)}."- )---def _get_custom_had(had_dict, size):- """Look up a custom Hadamard matrix for *size* from the normalized dict.-- Returns ``(had_tensor, True)`` if found, ``(None, False)`` otherwise.- """- if had_dict is None:- return None, False- if size in had_dict:- return had_dict[size], True- return None, False---# ----------------------------------------------------------------------------# Hook implementations-# ------------------------------------------------------------------------------class FullOnlineHadamardHook(nn.Module):- """Pre-forward hook: full Hadamard on the entire last dimension (for ``down_proj``)."""-- def __init__(self, had_K, K, fp32_had=False, use_fast_had=True, had_matrix=None):- super().__init__()- self.custom_had = had_matrix is not None- if had_matrix is not None:- self.register_buffer("had_matrix", had_matrix)- self.had_K = None- self.K = None- else:- if had_K is not None:- self.register_buffer("had_K", had_K)- else:- self.had_K = None- self.K = K- self.fp32_had = fp32_had- self.use_fast_had = use_fast_had-- def __call__(self, module: nn.Module, args):- x = args[0] if isinstance(args, tuple) else args- x_dtype = x.dtype-- if self.custom_had:- H = self.had_matrix.to(device=x.device, dtype=x.dtype)- if self.fp32_had:- H = self.had_matrix.to(device=x.device).float()- x = (x.float() @ H.T).to(x_dtype)- else:- x = x @ H.T- elif self.fp32_had:- x = matmul_hadU_cuda(x.float(), self.had_K, self.K, use_fast_had=self.use_fast_had).to(x_dtype)- else:- x = matmul_hadU_cuda(x, self.had_K, self.K, use_fast_had=self.use_fast_had)-- if isinstance(args, tuple):- return (x,) + args[1:]- return x---class CrossHeadOnlineHadamardHook(nn.Module):- """Pre-forward hook: **cross-head** Hadamard on the ``num_heads`` dimension- (for ``o_proj``).-- After offline rotation:- - ``v_proj`` absorbed a per-head (within-head) Hadamard on ``head_dim``.- - ``o_proj`` absorbed a full Hadamard on ``hidden_size``.-- Since ``H_full = H_cross ⊗ H_within`` (Kronecker decomposition) and the- within-head part is already cancelled by ``v_proj`` through the attention- path (``H_within² = I``), the online hook only needs to apply the residual- **cross-head** Hadamard (``H_cross ⊗ I``):-- * reshape ``(*, hidden_size)`` → ``(*, num_heads, head_dim)``- * transpose → ``(*, head_dim, num_heads)``- * Hadamard on the **num_heads** axis (last dim)- * transpose back and reshape- """-- def __init__(self, had_K, K, head_dim, fp32_had=False, use_fast_had=True, had_matrix=None):- """- Args:- had_K: Hadamard sub-matrix from ``get_hadK(num_heads)``.- K: Block size from ``get_hadK(num_heads)``.- head_dim: ``hidden_size // num_attention_heads``.- fp32_had: Compute in fp32.- use_fast_had: If True use fast_hadamard_transform; if False use matmul_hadU.- had_matrix: Optional custom rotation matrix of shape ``(num_heads, num_heads)``.- """- super().__init__()- self.custom_had = had_matrix is not None- if had_matrix is not None:- self.register_buffer("had_matrix", had_matrix)- self.had_K = None- self.K = None- else:- if had_K is not None:- self.register_buffer("had_K", had_K)- else:- self.had_K = None- self.K = K- self.had_dim = head_dim- self.fp32_had = fp32_had- self.use_fast_had = use_fast_had-- def __call__(self, module: nn.Module, args):- x = args[0] if isinstance(args, tuple) else args- x_dtype = x.dtype-- if self.fp32_had:- x = x.float()-- init_shape = x.shape- num_heads = init_shape[-1] // self.had_dim-- if self.custom_had:- H = self.had_matrix.to(device=x.device, dtype=x.dtype)- # reshape (*, hidden) → (*, num_heads, head_dim), transpose → (*, head_dim, num_heads)- x = x.reshape(-1, num_heads, self.had_dim).transpose(1, 2)- # apply H on last dim (num_heads): x @ H.T- x = (x @ H.T).transpose(1, 2)- elif self.use_fast_had and fast_hadamard_transform is not None and self.K == 1:- x = fast_hadamard_transform.hadamard_transform(- x.reshape(-1, num_heads, self.had_dim).transpose(1, 2),- scale=1 / math.sqrt(num_heads),- ).transpose(1, 2)- else:- # Fallback: use matmul_hadU (pure butterfly + had_K, no fast_hadamard_transform)- x = x.reshape(-1, num_heads, self.had_dim).transpose(1, 2)- x = matmul_hadU(x.contiguous())- x = x.transpose(1, 2)-- if self.fp32_had:- x = x.to(x_dtype)- x = x.reshape(init_shape)-- if isinstance(args, tuple):- return (x,) + args[1:]- return x---# ----------------------------------------------------------------------------# Registration helper-# ------------------------------------------------------------------------------def register_online_had_hooks(model, mapping=None, fp32_had=False, use_fast_had=True):- """Register online Hadamard pre-forward hooks on ``down_proj`` and ``o_proj``.-- * **down_proj** (``online_full_had``): full Hadamard on ``intermediate_size``.- Compensates ``apply_exact_had_to_linear(down_proj, had_dim=-1, output=False)``.-- * **o_proj** (``online cross-head had``): cross-head Hadamard on ``num_heads``.- Compensates the residual after v_proj's within-head Hadamard cancels.-- Args:- model: A HuggingFace model whose weights have already been rotated.- mapping: A :class:`RotationMapping` (auto-inferred if ``None``).- fp32_had: Whether to compute the Hadamard transform in fp32.- use_fast_had: If True use fast_hadamard_transform; if False use matmul_hadU.-- Returns:- list of hook handles (call ``handle.remove()`` to detach).- """- if mapping is None:- from auto_round.experimental.rotation_inplace.model_config import infer_mapping_from_model-- mapping = infer_mapping_from_model(model)-- config = model.config- num_heads = getattr(config, mapping.num_heads_attr)- hidden_size = getattr(config, mapping.hidden_size_attr)- intermediate_size = getattr(config, mapping.intermediate_size_attr)- head_dim = mapping.attn_head_dim or (hidden_size // num_heads)-- # down_proj: full Hadamard on intermediate_size- had_K_full, K_full = get_hadK(intermediate_size)-- # o_proj: cross-head Hadamard on num_heads- had_K_head, K_head = get_hadK(num_heads)-- # Identify target module suffixes from mapping- mlp_out_suffix = mapping.mlp_out.split(".")[-1] # e.g. "down_proj"- attn_o_suffix = mapping.attn_o.split(".")[-1] # e.g. "o_proj"-- handles = []- for name, module in model.named_modules():- if name.endswith(mlp_out_suffix) and isinstance(module, nn.Linear):- hook = FullOnlineHadamardHook(- had_K=had_K_full,- K=K_full,- fp32_had=fp32_had,- use_fast_had=use_fast_had,- )- h = module.register_forward_pre_hook(hook)- handles.append(h)- elif name.endswith(attn_o_suffix) and isinstance(module, nn.Linear):- hook = CrossHeadOnlineHadamardHook(- had_K=had_K_head,- K=K_head,- head_dim=head_dim,- fp32_had=fp32_had,- use_fast_had=use_fast_had,- )- h = module.register_forward_pre_hook(hook)- handles.append(h)-- return handles---def is_pow2(n):- return (n & (n - 1) == 0) and (n > 0)---# Adapted from https://github.com/Cornell-RelaxML/quip-sharp/blob/main/lib/utils/matmul_had.py-def get_hadK(n: int, transpose=False) -> (torch.Tensor, int):- hadK, K = None, None-- if is_pow2(n):- K = 1- return hadK, K- else:- from auto_round.experimental.transform.utils.hadamard import _fetch_hadamard_divisor-- hadK = _fetch_hadamard_divisor(n, torch.float, torch.device("cpu"))- if transpose:- hadK = hadK.T- if hadK is not None:- return hadK, 1 if is_pow2(hadK.shape[0]) else hadK.shape[0]- assert is_pow2(n)---def matmul_hadU(X, transpose=False):- n = X.shape[-1]- hadK, K = get_hadK(n, transpose)- input = X.clone().view(-1, n, 1)- output = input.clone()- while input.shape[1] > K:- input = input.view(input.shape[0], input.shape[1] // 2, 2, input.shape[2])- output = output.view(input.shape)- output[:, :, 0, :] = input[:, :, 0, :] + input[:, :, 1, :]- output[:, :, 1, :] = input[:, :, 0, :] - input[:, :, 1, :]- output = output.view(input.shape[0], input.shape[1], -1)- input, output = (output, input)- del output-- if K > 1:- # Do not explicitly repeat - OOM- # input = torch.bmm(- # hadK.repeat(len(input), 1, 1).to(input.device).to(input.dtype), input)- # Use bcast instead- input = hadK.view(1, K, K).to(input) @ input-- return input.view(X.shape) / torch.tensor(n).sqrt()---def matmul_hadUt(X):- return matmul_hadU(X, transpose=True)---def random_hadamard_matrix(size, device):- # See https://cornell-relaxml.github.io/quip-sharp/ , Section "Randomized Hadamard Transformation"- Q = torch.randint(low=0, high=2, size=(size,)).to(torch.float64)- Q = Q * 2 - 1- Q = torch.diag(Q)- return matmul_hadU(Q).to(device)---def deterministic_hadamard_matrix(size, device):- """Build a deterministic Hadamard matrix of the given *size*.-- Applies the butterfly ``matmul_hadU`` to an identity matrix so that the- result is purely determined by ``get_hadK`` (no random sign flips).- """- Q = torch.eye(size, dtype=torch.float64)- return matmul_hadU(Q).to(device)---def matmul_hadU_cuda(X, hadK, K, use_fast_had=True):- n = X.shape[-1]- if not use_fast_had or fast_hadamard_transform is None:- return matmul_hadU(X)- if K == 1:- return fast_hadamard_transform.hadamard_transform(X.contiguous(), 1.0 / torch.tensor(n).sqrt())- # if transpose:- # hadK = hadK.T.contiguous()- input = X.view(*X.shape[:-1], K, n // K)- input = fast_hadamard_transform.hadamard_transform(input.contiguous(), 1.0 / torch.tensor(n).sqrt())- input = hadK.to(input.device).to(input.dtype) @ input- return input.reshape(X.shape)---def matmul_hadUt_cuda(X, hadK, K, use_fast_had=True):- return matmul_hadU_cuda(X, hadK, K, use_fast_had=use_fast_had)---def apply_exact_had_to_linear(- module, had_dim=-1, output=False, use_fast_had=True, compute_device=None, had_matrix=None-):- """Apply Hadamard rotation to a Linear layer's weight in-place.-- Args:- module: ``nn.Linear`` layer.- had_dim: Dimension of each Hadamard block (``-1`` for full dimension).- output: If ``True`` rotate the output (row) side; otherwise input (col).- use_fast_had: Use ``fast_hadamard_transform`` when available.- compute_device: Device to run computation on.- had_matrix: Optional custom rotation matrix. When ``had_dim == -1``- this should be a square tensor whose size equals- ``out_features`` (output) or ``in_features`` (input). When- ``had_dim > 0`` the size should equal ``had_dim``.- """- assert isinstance(module, torch.nn.Linear)- in_features, out_features = module.in_features, module.out_features-- if had_dim != -1 and had_matrix is None:- assert is_pow2(had_dim), "Hadamard dimension must be a power of 2!"-- W_ = module.weight.data- dtype = W_.dtype- dev = W_.device- init_shape = W_.shape- compute_dev = _resolve_compute_device(compute_device)- W_ = W_.double().to(compute_dev)-- if had_matrix is not None:- H = had_matrix.to(device=compute_dev, dtype=torch.float64)- if had_dim == -1:- # Full-dimension custom matrix- if output:- # W.T = H @ W.T → W = (H @ W.T).T = W @ H.T- W_ = W_ @ H.T- else:- # W = H @ W (rotate input columns: W_new[i,:] = sum H[i,j]*W[j,:])- # Actually for input side: W_new = W @ H (each row is rotated)- W_ = W_ @ H.T- else:- # Per-block custom matrix- if output:- W_ = W_.t()- transposed_shape = W_.shape- flat = W_.reshape(-1, had_dim)- W_ = (flat @ H.T).reshape(transposed_shape).t()- else:- flat = W_.reshape(-1, had_dim)- W_ = (flat @ H.T).reshape(init_shape)- elif had_dim == -1:- if output:- had_K, K = get_hadK(out_features)- W_ = matmul_hadU_cuda(W_.t(), had_K, K, use_fast_had=use_fast_had).t()- if not output:- had_K, K = get_hadK(in_features)- W_ = matmul_hadU_cuda(W_, had_K, K, use_fast_had=use_fast_had)- else:- # Apply Hadamard to the last had_dim chunks of the weights- if output:- W_ = W_.t()- transposed_shape = W_.shape- if use_fast_had and fast_hadamard_transform is not None:- W_ = (- fast_hadamard_transform.hadamard_transform(- W_.reshape(-1, transposed_shape[-1] // had_dim, had_dim), scale=1 / math.sqrt(had_dim)- )- .reshape(transposed_shape)- .t()- )- else:- W_ = matmul_hadU(W_.reshape(-1, had_dim)).reshape(transposed_shape).t()- else:- if use_fast_had and fast_hadamard_transform is not None:- n = W_.shape[1]- W_ = fast_hadamard_transform.hadamard_transform(- W_.reshape(-1, n // had_dim, had_dim), scale=1 / math.sqrt(had_dim)- ).reshape(init_shape)- else:- W_ = matmul_hadU(W_.reshape(-1, had_dim)).reshape(init_shape)- module.weight.data = W_.to(device=dev, dtype=dtype)---def apply_cross_head_had_to_linear(- module, num_heads, head_dim, use_fast_had=True, compute_device=None, had_matrix=None-):- """Apply a cross-head Hadamard rotation to a Linear layer's input side.-- The operation is equivalent to ``(H_cross ⊗ I_head_dim)`` applied to the- input columns:-- * Reshape columns ``(hidden_size,)`` → ``(num_heads, head_dim)``- * Transpose → ``(head_dim, num_heads)``- * Hadamard on the ``num_heads`` axis- * Transpose back and reshape-- This mirrors what :class:`CrossHeadOnlineHadamardHook` does at runtime.-- Args:- module: ``nn.Linear`` layer whose ``in_features == num_heads * head_dim``.- num_heads: Number of attention heads.- head_dim: Per-head dimension.- use_fast_had: Use ``fast_hadamard_transform`` when available.- compute_device: Device to run computation on.- had_matrix: Optional custom rotation matrix of shape ``(num_heads, num_heads)``.- """- assert isinstance(module, torch.nn.Linear)- W_ = module.weight.data- dtype = W_.dtype- dev = W_.device- compute_dev = _resolve_compute_device(compute_device)- W_ = W_.double().to(compute_dev)-- out_f = W_.shape[0]- # W shape: (out_features, hidden_size) where hidden_size = num_heads * head_dim- # Reshape columns: (out_f, num_heads, head_dim)- W_ = W_.reshape(out_f, num_heads, head_dim)- # Transpose last two dims: (out_f, head_dim, num_heads)- W_ = W_.transpose(1, 2).contiguous()-- if had_matrix is not None:- H = had_matrix.to(device=compute_dev, dtype=torch.float64)- # Apply H on last dim (num_heads): flat @ H.T- flat = W_.reshape(-1, num_heads)- W_ = (flat @ H.T).reshape(out_f, head_dim, num_heads)- elif use_fast_had and fast_hadamard_transform is not None and is_pow2(num_heads):- W_ = fast_hadamard_transform.hadamard_transform(W_, scale=1.0 / math.sqrt(num_heads))- else:- W_ = matmul_hadU(W_.reshape(-1, num_heads)).reshape(out_f, head_dim, num_heads)-- # Transpose back: (out_f, num_heads, head_dim) → (out_f, hidden_size)- W_ = W_.transpose(1, 2).contiguous().reshape(out_f, num_heads * head_dim)- module.weight.data = W_.to(device=dev, dtype=dtype)---# ----------------------------------------------------------------------------# Grouped (block-diagonal) Hadamard utilities-# ------------------------------------------------------------------------------class OnlineHadamardPostHook(nn.Module):- """Forward hook (post-hook) adapter: wraps a pre-hook-style Hadamard- transform to apply it on the layer's **output** instead of input.-- Used for v_proj per-head Hadamard on the output side when online- rotation is not fused into weights.- """-- def __init__(self, pre_hook):- super().__init__()- self.pre_hook = pre_hook-- def __call__(self, module, input, output):- result = self.pre_hook(module, (output,))- if isinstance(result, tuple):- return result[0]- return result---class GroupOnlineHadamardHook(nn.Module):- """Pre-forward hook: block-diagonal Hadamard with fixed ``group_size`` on last dim.-- Reshapes ``(*, D)`` → ``(*, D // group_size, group_size)``, applies Hadamard- per group, then reshapes back. Much cheaper than a full-dimension Hadamard.- """-- def __init__(self, group_size, fp32_had=False, use_fast_had=True, had_matrix=None):- super().__init__()- self.group_size = group_size- self.fp32_had = fp32_had- self.use_fast_had = use_fast_had- self.custom_had = had_matrix is not None-- if had_matrix is not None:- self.register_buffer("had_matrix", had_matrix)- self.had_K = None- self.K = None- elif not is_pow2(group_size):- had_K, K = get_hadK(group_size)- if had_K is not None:- self.register_buffer("had_K", had_K)- else:- self.had_K = None- self.K = K- else:- self.had_K = None- self.K = 1-- def __call__(self, module: nn.Module, args):- x = args[0] if isinstance(args, tuple) else args- x_dtype = x.dtype- init_shape = x.shape- gs = self.group_size-- if self.fp32_had:- x = x.float()-- # Reshape: (*, D) → (*, D//gs, gs)- x = x.reshape(*init_shape[:-1], init_shape[-1] // gs, gs)-- if self.custom_had:- H = self.had_matrix.to(device=x.device, dtype=x.dtype)- flat = x.reshape(-1, gs)- x = (flat @ H.T).reshape(*init_shape[:-1], init_shape[-1] // gs, gs)- elif self.use_fast_had and fast_hadamard_transform is not None and self.K == 1:- x = fast_hadamard_transform.hadamard_transform(x, scale=1.0 / math.sqrt(gs))- else:- x = x.reshape(-1, gs)- x = matmul_hadU(x)- x = x.reshape(*init_shape[:-1], init_shape[-1] // gs, gs)-- x = x.reshape(init_shape)-- if self.fp32_had:- x = x.to(x_dtype)-- if isinstance(args, tuple):- return (x,) + args[1:]- return x---def _apply_grouped_had_to_weight(W, group_size, side="input", use_fast_had=True, had_matrix=None):- """Apply block-diagonal Hadamard to a weight matrix.-- Args:- W: Weight tensor, shape (out_features, in_features).- group_size: Block size for the Hadamard rotation.- side: ``'input'`` rotates columns (in_features dim),- ``'output'`` rotates rows (out_features dim).- use_fast_had: Use fast_hadamard_transform if available.- had_matrix: Optional custom Hadamard matrix of shape ``(gs, gs)``- to use instead of the built-in Hadamard.-- Returns:- Rotated weight tensor.- """- gs = group_size- dtype = W.dtype- W = W.double()-- def _had_on_last_dim(X):- """Apply Hadamard on the last dimension (size gs) of X shaped (..., gs)."""- if had_matrix is not None:- H = had_matrix.to(device=X.device, dtype=X.dtype)- # X: (..., gs) → batch matmul with H^T → X @ H^T- flat = X.reshape(-1, gs)- return (flat @ H.T).reshape(X.shape)- if use_fast_had and fast_hadamard_transform is not None and is_pow2(gs):- return fast_hadamard_transform.hadamard_transform(X, scale=1.0 / math.sqrt(gs))- orig_shape = X.shape- return matmul_hadU(X.reshape(-1, gs)).reshape(orig_shape)-- if side == "input":- out_f, in_f = W.shape- W = W.reshape(out_f, in_f // gs, gs)- W = _had_on_last_dim(W)- W = W.reshape(out_f, in_f)- else:- out_f, in_f = W.shape- Wt = W.t().contiguous()- Wt = Wt.reshape(in_f, out_f // gs, gs)- Wt = _had_on_last_dim(Wt)- W = Wt.reshape(in_f, out_f).t().contiguous()-- return W.to(dtype)---def _rotate_linear_grouped(module, group_size, side="input", use_fast_had=True, compute_device=None, had_matrix=None):- """Apply block-diagonal Hadamard rotation to a Linear layer's weight.-- Args:- module: ``nn.Linear`` layer.- group_size: Block size.- side: ``'input'`` or ``'output'``.- use_fast_had: Use fast_hadamard_transform.- compute_device: Device to run computation on. If None, auto-detects GPU.- had_matrix: Optional custom Hadamard matrix of shape ``(gs, gs)``.- """- dtype = module.weight.data.dtype- dev = module.weight.data.device- compute_dev = _resolve_compute_device(compute_device)- W = module.weight.data.to(device=compute_dev, dtype=torch.float64)- W = _apply_grouped_had_to_weight(W, group_size, side=side, use_fast_had=use_fast_had, had_matrix=had_matrix)- module.weight.data = W.to(device=dev, dtype=dtype)-- if side == "output" and module.bias is not None:- bias = module.bias.data.to(device=compute_dev, dtype=torch.float64)- gs = group_size- bias = bias.reshape(-1, gs)- if had_matrix is not None:- H = had_matrix.to(device=compute_dev, dtype=torch.float64)- bias = (bias @ H.T).reshape(-1)- elif use_fast_had and fast_hadamard_transform is not None and is_pow2(gs):- bias = (- fast_hadamard_transform.hadamard_transform(bias.unsqueeze(0), scale=1.0 / math.sqrt(gs))- .squeeze(0)- .reshape(-1)- )- else:- bias = matmul_hadU(bias).reshape(-1)- module.bias.data = bias.to(device=dev, dtype=dtype)---def _rotate_embedding_grouped(embedding, group_size, use_fast_had=True, compute_device=None, had_matrix=None):- """Apply block-diagonal Hadamard rotation to an Embedding layer.-- Embedding weight: (vocab, hidden_size) → rotate on hidden_size (columns).- """- dtype = embedding.weight.data.dtype- dev = embedding.weight.data.device- compute_dev = _resolve_compute_device(compute_device)- W = embedding.weight.data.to(device=compute_dev, dtype=torch.float64)- W = _apply_grouped_had_to_weight(W, group_size, side="input", use_fast_had=use_fast_had, had_matrix=had_matrix)- new_W = W.to(device=dev, dtype=dtype)- del W- embedding.weight.data = new_W---def register_online_had_hooks_grouped(model, mapping, group_size, fp32_had=False, use_fast_had=True):- """Register per-group online Hadamard hooks on ``down_proj`` and ``o_proj``.-- In grouped mode:- - **down_proj**: block-diagonal Hadamard on ``intermediate_size`` with ``group_size``.- - **o_proj**: block-diagonal Hadamard on ``hidden_size`` with ``group_size``.-- Args:- model: HuggingFace model with rotated weights.- mapping: RotationMapping.- group_size: Block size for block-diagonal Hadamard.- fp32_had: Compute in fp32.- use_fast_had: Use fast_hadamard_transform.-- Returns:- list of hook handles.- """- mlp_out_suffix = mapping.mlp_out.split(".")[-1]- attn_o_suffix = mapping.attn_o.split(".")[-1]-- handles = []- for name, module in model.named_modules():- if name.endswith(mlp_out_suffix) and isinstance(module, nn.Linear):- hook = GroupOnlineHadamardHook(- group_size=group_size,- fp32_had=fp32_had,- use_fast_had=use_fast_had,- )- h = module.register_forward_pre_hook(hook)- handles.append(h)- elif name.endswith(attn_o_suffix) and isinstance(module, nn.Linear):- hook = GroupOnlineHadamardHook(- group_size=group_size,- fp32_had=fp32_had,- use_fast_had=use_fast_had,- )- h = module.register_forward_pre_hook(hook)- handles.append(h)-- return handles+from auto_round.algorithms.transforms.rotation.inplace.hooks import * # noqa: F401, F403diff --git a/auto_round/experimental/transform/apply.py b/auto_round/experimental/transform/apply.pyindex aafd4c1b0..e0aa77e13 100644--- a/auto_round/experimental/transform/apply.py+++ b/auto_round/experimental/transform/apply.py@@ -1,196 +1,13 @@# # Copyright (C) 2026 Intel Corporation# # SPDX-License-Identifier: Apache-2.0+"""Backward-compat re-export shim.-import torch-import tqdm+The canonical implementation now lives in+:mod:`auto_round.algorithms.transforms.rotation.apply`.+"""-from auto_round.experimental.qmodules.base import QModuleBase-from auto_round.experimental.transform.hadamards import build_hadamard_transform-from auto_round.experimental.transform.rotation_config import RotationConfig-from auto_round.experimental.utils import is_triton_kernel_available, normalize_rotation_config+from auto_round.algorithms.transforms.rotation.apply import ( # noqa: F401+ apply_rotation_transform,+)__all__ = ["apply_rotation_transform"]---def apply_rotation_transform(- model: torch.nn.Module,- config: str | dict | RotationConfig | None,- location: str = "weight",- use_tqdm=True,- desc=None,- data_type="mx_fp",-):- """- Apply a transform configuration to a model.-- Weight and activation transforms are attached as submodules and are- triggered via PyTorch hooks.-- :param model: Model to which the transform configuration will be applied.- :param config: Transform configuration to apply. Supported values are:- * ``str``: A named/preset transform configuration. In this case,- resolved to a concrete quantization/transform configuration.- * ``dict``: A raw configuration mapping that will be normalized- (via :func:`normalize_rotation_config`) and then passed to- :class:`TransformConfig`.- * :class:`TransformConfig`: An existing configuration instance.- This will be used to construct the final configuration after- normalization.- * ``None``: Uses the default behavior of- :func:`_normalize_rotation_config` (for example, inferring a- configuration from ``data_type`` or other project defaults), if- supported.- :param data_type: quantization data type.- :param use_tqdm: If ``True``, wrap the per-module application in a- tqdm progress bar.- :param desc: Optional description string to show in the tqdm progress- bar. If ``None``, a description will be derived from- ``config.transform_type``.- """-- config = normalize_rotation_config(config, data_type)- if not isinstance(config, RotationConfig):- config = RotationConfig(**config)-- modules_config = [- (name, module, config)- for name, module in model.named_modules()- if isinstance(module, torch.nn.Linear) or isinstance(module, QModuleBase)- ]-- desc = f"Applying {config.hadamard_type} transforms" if desc is None else desc- for name, module, config in tqdm.tqdm(modules_config, desc=desc, disable=(not use_tqdm)):- if "lm_head" in name:- continue- _apply_to_module(model, module, config, location, data_type)-- # attach config to model for compression/serialization. Use a plain dict so- # that downstream HF `save_pretrained` -> JSON works (RotationConfig is a- # pydantic model and is not directly JSON serializable).- setattr(model, "rotation_config", config.model_dump() if hasattr(config, "model_dump") else config)- hooks = None-- return model, hooks---def _apply_to_module(- model: torch.nn.Module,- module: torch.nn.Module,- config: RotationConfig,- location: str = "weight",- data_type: str = "mx_fp",-):- """- Create transforms and apply them to the module-- :param model: model which module belongs to- :param module: target module to apply transforms to- """-- # create transform as submodule- hadamard_name = config.hadamard_type-- if location == "input":-- # activation needs transpose- input_hadamard_transform = build_hadamard_transform(- **config.model_dump(),- location="input",- inverse=True,- device="cpu",- precision=module.dtype, # for online activation, the transform dtype maybe bfloat16/float16.- )-- if config.hadamard_type != "random_hadamard":- hadamard_weight = input_hadamard_transform.weight- else:- hadamard_weight = None-- if is_triton_kernel_available(data_type):- from auto_round.experimental.transform.triton.mxfp4 import mxfp4_forward_kernel_wrapper-- def input_hook(self, args):- input = args[0]- # transform(input)- orig_shape = input.shape- orig_dtype = input.dtype- x_flat = input.contiguous().flatten(end_dim=-2)- qdq_input, _ = mxfp4_forward_kernel_wrapper(- x_flat,- (- hadamard_weight.to(orig_dtype)- if hadamard_weight is not None- else self.hadamard_matrix.T.to(orig_dtype)- ), # this matrix from w_transform, needs transpose- )- return qdq_input.reshape(orig_shape).to(orig_dtype)-- # for fused transform + quantization kernel- module.pre_dequantized_input = True- module.register_forward_pre_hook(input_hook, prepend=True)- else:-- from auto_round.experimental.transform.utils.matrix import _multihead_matmul-- def input_hook(self, args):- input = args[0]-- ori_shape = input.shape- orig_dtype = input.dtype-- if hadamard_weight is not None:- input = input.view(-1, hadamard_weight.shape[0])- return (- (_multihead_matmul(input, hadamard_weight.to(input.device).to(orig_dtype)))- .view(ori_shape)- .to(orig_dtype)- )- else:- input = input.view(-1, self.hadamard_matrix.shape[0])- return (- (_multihead_matmul(input, self.hadamard_matrix.T.to(orig_dtype))).view(ori_shape).to(orig_dtype)- )-- # for fused transform + quantization kernel- module.pre_dequantized_input = False- module.register_forward_pre_hook(input_hook, prepend=True)-- elif location == "weight":- # eagerly apply transformation to weight- # fuse transform into weight- assert hasattr(module, "weight")-- weight_hadamard_transform = build_hadamard_transform(- **config.model_dump(),- location="weight",- device=module.weight.device,- )-- # need save random hadamard matrix needed when inference- if config.hadamard_type == "random_hadamard":- # for saving transform weight- from auto_round.experimental.transform.patch_modules import patch_quantlinear-- patch_quantlinear(weight_hadamard_transform)-- # for autoround tuning: weight not tuning- # for rtn: weight transformed before saving- from auto_round.experimental.transform.patch_modules import (- patch_wrapperlinear_to_apply_transform,- patch_wrapperwalayer_forward_to_apply_transform,- )-- input_hadamard_transform = build_hadamard_transform(- **config.model_dump(),- location="input",- inverse=True,- device=module.weight.device,- precision=module.weight.dtype, # for online activation, the transform dtype maybe bfloat16/float16.- )-- patch_wrapperlinear_to_apply_transform(weight_hadamard_transform, input_hadamard_transform)- patch_wrapperwalayer_forward_to_apply_transform(input_hadamard_transform)-- else:- # TODO: apply transform to output/q/k- raise NotImplementedError()diff --git a/auto_round/experimental/transform/hadamards.py b/auto_round/experimental/transform/hadamards.pyindex 42c47818d..83efa91b6 100644--- a/auto_round/experimental/transform/hadamards.py+++ b/auto_round/experimental/transform/hadamards.py@@ -1,143 +1,24 @@-# Copyright (c) 2026 Intel Corporation-#-# Licensed under the Apache License, Version 2.0 (the "License");-# you may not use this file except in compliance with the License.-# You may obtain a copy of the License at-#-# http://www.apache.org/licenses/LICENSE-2.0-#-# Unless required by applicable law or agreed to in writing, software-# distributed under the License is distributed on an "AS IS" BASIS,-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.-# See the License for the specific language governing permissions and-# limitations under the License.--import inspect-import math-from typing import Any, Callable, Dict--import torch-import torch.nn as nn--from auto_round.experimental.transform.utils.hadamard import deterministic_hadamard_matrix, random_hadamard_matrix-from auto_round.experimental.transform.utils.matrix import apply_transform_weight---def filter_kwarg_dict(fn_or_method: Callable, kwarg_dict: Dict[str, Any]) -> Dict[str, Any]:- fn_or_method_keys = inspect.signature(fn_or_method).parameters.keys()- return {k: v for k, v in kwarg_dict.items() if k in fn_or_method_keys}---class HadamardTransform(nn.Module):-- def __init__(- self,- block_size: int = 32,- device: torch.device = None,- precision: torch.dtype = torch.float32,- location: str = "weight",- module_type: type[torch.nn.Module] = torch.nn.Linear,- inverse: bool = False,- ):- """Initialize a Hadamard transform module.-- Args:- block_size: Size of each Hadamard block. The input tensor is reshaped- to ``(-1, block_size)`` before applying the transform.- device: Device on which to create the Hadamard matrix.- precision: Data type used for the Hadamard matrix weights, using float32 as default.- location: Target location used by ``apply_transform_weight`` when- applying the transform.- module_type: Module type associated with the transform application,- typically ``torch.nn.Linear``.- inverse: Whether to build the inverse form of the transform.- """-- super().__init__()- self.size = block_size- self.scale = 1 / math.sqrt(self.size)- self.location = location- self.module_type = module_type- self.inverse = inverse- self.weight = self._create_weight(self.size, device, precision)-- def _create_weight(- self,- size: int,- device: torch.device = None,- precision: torch.dtype = torch.float32,- ) -> torch.nn.Parameter:- data = deterministic_hadamard_matrix(size, precision, device) * self.scale- # TODO: implement SpinQuant, which rotation matrix is learnable- return nn.Parameter(data, requires_grad=False)-- def forward(self, x: torch.Tensor):- # Hadamard transform is it own inverse- ori_shape = x.shape- x = x.view(-1, self.size)- return (- (- apply_transform_weight(- self.weight.to(x.device),- x.to(dtype=self.weight.dtype),- self.location,- self.module_type,- )- )- .to(x.dtype)- .view(ori_shape)- )---class RandomHadamardTransform(HadamardTransform):- def __init__(- self,- block_size: int = 32,- device: torch.device = None,- precision: torch.dtype = None,- location: str = "weight",- module_type: type[torch.nn.Module] = torch.nn.Linear,- inverse: bool = False,- seed: int | None = None,- generator: torch.Generator | None = None,- ):- if generator is not None:- self.generator = generator- else:- self.generator = torch.Generator()- if seed is not None:- self.generator.manual_seed(seed)-- super().__init__(- block_size=block_size,- device=device,- precision=precision,- location=location,- module_type=module_type,- inverse=inverse,- )-- def _create_weight(- self,- size: int,- device: torch.device = None,- precision: torch.dtype = None,- ) -> torch.nn.Parameter:- data = random_hadamard_matrix(size, precision, device, self.generator) * self.scale- # activation needs transpose- if self.inverse:- data = data.T- # data = deterministic_hadamard_matrix(size, precision, device) * self.scale- # TODO: implement SpinQuant, which rotation matrix is learnable- return nn.Parameter(data, requires_grad=False)---HADAMARDS = {- "hadamard": HadamardTransform,- "random_hadamard": RandomHadamardTransform,-}---def build_hadamard_transform(hadamard_type: str, **hadamard_kwargs):- hadamard = HADAMARDS[hadamard_type]- return hadamard(**filter_kwarg_dict(hadamard.__init__, hadamard_kwargs))+# # Copyright (C) 2026 Intel Corporation+# # SPDX-License-Identifier: Apache-2.0+"""Backward-compat re-export shim.++The canonical implementation now lives in+:mod:`auto_round.algorithms.transforms.rotation.transforms`.+"""++from auto_round.algorithms.transforms.rotation.transforms import (+ HADAMARDS,+ HadamardTransform,+ RandomHadamardTransform,+)+from auto_round.algorithms.transforms.rotation.transforms import _filter_kwargs as filter_kwarg_dict # noqa: F401+from auto_round.algorithms.transforms.rotation.transforms import (+ build_hadamard_transform,+)++__all__ = [+ "HADAMARDS",+ "HadamardTransform",+ "RandomHadamardTransform",+ "build_hadamard_transform",+]diff --git a/auto_round/experimental/transform/patch_modules.py b/auto_round/experimental/transform/patch_modules.pyindex e0f9adbae..980c28588 100644--- a/auto_round/experimental/transform/patch_modules.py+++ b/auto_round/experimental/transform/patch_modules.py@@ -1,172 +1,19 @@# # Copyright (C) 2026 Intel Corporation# # SPDX-License-Identifier: Apache-2.0--import torch-import transformers--from auto_round.export.export_to_autoround.qlinear_fp import QuantLinear, pack_fp4_to_uint8-from auto_round.wrapper import WrapperLinear, WrapperWALayer---def patch_wrapperlinear_to_apply_transform(w_transform, inp_transform):- """- Globally monkey-patch WrapperLinear._qdq_weight and WrapperLinear._qdq_act so that it applies- a weight and activation transform before quantization.-- e.g. by apply_transform() before wrapper_block().- """-- if getattr(WrapperLinear, "_hadamard_patched", False):- return-- orig_qdq_weight = WrapperLinear._qdq_weight-- def _qdq_weight_patched(self, value, min_scale, max_scale):- """- # If no transform attached, fall back to original behavior- if not hasattr(self.orig_layer, transform_attr):- return orig_qdq_weight(self, value, min_scale, max_scale)- """-- if self.orig_layer.bits >= 16:- # keep original behavior for >=16bit to avoid changing semantics unexpectedly- return orig_qdq_weight(self, value, min_scale, max_scale)-- if getattr(self, "applied_weight_hadamard", None) is None:- with torch.no_grad():- weight = self.orig_layer.weight- if weight.device.type == "meta":- weight = self.orig_layer.get_weight().to(self.device)-- is_conv1d = type(self.orig_layer) == transformers.pytorch_utils.Conv1D- if is_conv1d:- weight = weight.t().continuous()- new_weight = w_transform(weight).to(self.device)- if is_conv1d:- new_weight = weight.t().continuous()- self.orig_layer.weight.data.copy_(new_weight)- self.applied_weight_hadamard = True-- return orig_qdq_weight(self, value, min_scale, max_scale)-- orig_qdq_act = WrapperLinear._qdq_act-- def _qdq_act_patched(self, x, act_min_scale, act_max_scale, act_max=None):-- x = inp_transform(x)-- return orig_qdq_act(self, x, act_min_scale, act_max_scale, act_max)-- WrapperLinear._qdq_weight = _qdq_weight_patched- WrapperLinear._qdq_act = _qdq_act_patched- WrapperLinear._hadamard_patched = True---def patch_wrapperwalayer_forward_to_apply_transform(inp_transform):- """- Globally monkey-patch WrapperWALayer.forward so that it applies- a activation transform before quantization.-- e.g. by apply_transform() before wrapper_block().- """-- if getattr(WrapperWALayer, "_hadamard_forward_patched", False):- return-- orig_forward = WrapperWALayer.forward-- def _forward_patched(self, x):- """- # If no transform attached, fall back to original behavior- if not hasattr(self.orig_layer, transform_attr):- return orig_forward(self, x)- """-- act_max = self.orig_layer.act_max if hasattr(self.orig_layer, "act_max") else None-- # transform = getattr(self.orig_layer, transform_attr)- x = inp_transform(x)-- x, _, _ = self.orig_layer.act_quant_func(- x,- bits=self.orig_layer.act_bits,- group_size=self.orig_layer.act_group_size,- scale_dtype=self.orig_layer.scale_dtype,- q_scale_thresh=self.orig_layer.q_scale_thresh,- data_type=self.orig_layer.act_data_type,- tensor_max=act_max,- )- return self.orig_layer.forward(x)-- WrapperWALayer.forward = _forward_patched- WrapperWALayer._hadamard_forward_patched = True---def patch_quantlinear(w_transform):- """ """-- if getattr(QuantLinear, "_pack_patched", False):- return-- from auto_round.data_type.nvfp import cast_to_fp4, get_reciprocal- from auto_round.data_type.utils import reshape_pad_tensor_by_group_size, revert_tensor_by_pad- from auto_round.utils import get_packing_device-- E8M0_EXPONENT_BIAS = 127- E8M0_EXPONENT_NAN_VAL = 255-- def _pack_patched(- self, linear, scales, zeros=None, g_idx=None, global_scale=None, input_global_scale=None, device=None- ):- device = get_packing_device(device)- if getattr(linear, "bias", None) is not None:- self.bias = linear.bias.detach().to(torch.float16)-- W = linear.weight.data.detach().to(device)- if type(linear) == torch.nn.Conv2d:- W = W.flatten(1)- if type(linear) == transformers.pytorch_utils.Conv1D:- W = W.t()-- tensor, orig_shape, pad_len = reshape_pad_tensor_by_group_size(W, self.group_size)- scales = scales.to(device)- if self.is_nv:- assert global_scale is not None and global_scale.numel() == 1- global_scale = global_scale.reshape([1])- global_scale = global_scale.to(device)- scaled_tensor = tensor.to(global_scale.dtype) * get_reciprocal(- scales.reshape(tensor.shape[0], -1) * get_reciprocal(global_scale)- )- scaled_tensor.clamp_(-6.0, 6.0)- scaled_tensor = cast_to_fp4(scaled_tensor)- else:- scaled_tensor = tensor / (2 ** scales.reshape(tensor.shape[0], -1))- scaled_tensor = revert_tensor_by_pad(scaled_tensor, orig_shape=orig_shape, pad_len=pad_len)- if self.is_mx:- final_scale = (scales + E8M0_EXPONENT_BIAS).clamp(0, E8M0_EXPONENT_NAN_VAL).to(torch.uint8)- else:- final_scale = scales.to(torch.float8_e4m3fn)-- self.weight_scale = final_scale- # self.weight = get_compressed_weight(scaled_tensor, self.bits, self.data_type) ## TODO- if self.bits == 8:- compress_dtype = torch.float8_e4m3fn- self.weight = scaled_tensor.to(compress_dtype)-- else:- compress_dtype = torch.uint8- self.weight_packed = pack_fp4_to_uint8(scaled_tensor)-- if global_scale is not None:- self.weight_global_scale = global_scale.to(torch.float32).to(device)-- if input_global_scale is not None:- # TODO: the shape of `input_global_scale` is [] in some cases — need to investigate why.- self.input_global_scale = input_global_scale.to(torch.float32).to(device).reshape([1])-- # add transform weight- self.register_buffer("hadamard_matrix", w_transform.weight.to(device))- return-- QuantLinear.pack = _pack_patched- QuantLinear._pack_patched = True+"""Backward-compat re-export shim.++The canonical implementation now lives in+:mod:`auto_round.algorithms.transforms.rotation.patch`.+"""++from auto_round.algorithms.transforms.rotation.patch import ( # noqa: F401+ patch_quantlinear,+ patch_wrapperlinear_to_apply_transform,+ patch_wrapperwalayer_forward_to_apply_transform,+)++__all__ = [+ "patch_quantlinear",+ "patch_wrapperlinear_to_apply_transform",+ "patch_wrapperwalayer_forward_to_apply_transform",+]diff --git a/auto_round/experimental/transform/rotation_config.py b/auto_round/experimental/transform/rotation_config.pyindex dfcfb5d45..fad2ede62 100644--- a/auto_round/experimental/transform/rotation_config.py+++ b/auto_round/experimental/transform/rotation_config.py@@ -1,61 +1,11 @@# # Copyright (C) 2026 Intel Corporation# # SPDX-License-Identifier: Apache-2.0+"""Backward-compat re-export shim.-from typing import Optional+The canonical ``RotationConfig`` schema now lives in+:mod:`auto_round.algorithms.transforms.rotation.config`.+"""-from pydantic import BaseModel, Field, field_validator+from auto_round.algorithms.transforms.rotation.config import RotationConfig # noqa: F401__all__ = ["RotationConfig"]---class RotationConfig(BaseModel):- """- Unified configuration for Hadamard rotation/transform applied to a model.-- Two implementation paths are supported:-- * ``backend="inplace"`` -> ``auto_round.experimental.rotation_inplace``- QuaRot-style residual-stream / per-layer rotation. Supports any- weight/activation dtype (incl. INT4/INT8/FPx). Can optionally fuse- the online Hadamard into weights (``fuse_online_to_weight=True``).- * ``backend="transform"`` -> ``auto_round.experimental.transform``- Per-Linear weight + activation Hadamard with a fused triton kernel.- **Only supports MXFP4 / NVFP4** and **cannot fuse online to weight.**- * ``backend="auto"`` (default)- - If ``fuse_online_to_weight=True`` -> inplace (fused).- - Else if ``data_type`` is MX-FP / NV-FP -> transform.- - Otherwise -> inplace (unfused).-- Notes:- * ``block_size`` is the group/block size for grouped Hadamard.- For ``backend="inplace"`` it is forwarded as ``group_size`` (``None``- / ``-1`` means full-dimension Hadamard).- """-- # ---- shared ----- backend: str = Field(default="auto")- block_size: Optional[int] = Field(default=None)- hadamard_type: str = Field(default="hadamard")-- # ---- inplace-only ----- fuse_online_to_weight: Optional[bool] = Field(default=None)- allow_online_rotation: bool = Field(default=True)-- # for random hadamard transform (transform path)- random_seed: bool = Field(default=False, exclude=True)-- @field_validator("backend")- @classmethod- def validate_backend(cls, v: str) -> str:- allowed = {"auto", "inplace", "transform"}- if v not in allowed:- raise ValueError(f"Unsupported backend: {v}. Supported values: {sorted(allowed)}")- return v-- @field_validator("hadamard_type")- @classmethod- def validate_hadamard_type(cls, v: str) -> str:- allowed = {"hadamard", "random_hadamard", "quarot_hadamard"}- if v not in allowed:- raise ValueError(f"Unsupported hadamard_type: {v}. Supported values: {sorted(allowed)}")- return vdiff --git a/auto_round/experimental/transform/triton/mxfp4.py b/auto_round/experimental/transform/triton/mxfp4.pyindex 8028c167b..3cc26a7d0 100644--- a/auto_round/experimental/transform/triton/mxfp4.py+++ b/auto_round/experimental/transform/triton/mxfp4.py@@ -1,197 +1,14 @@-# Copyright (c) 2026 Intel Corporation-#-# Licensed under the Apache License, Version 2.0 (the "License");-# you may not use this file except in compliance with the License.-# You may obtain a copy of the License at-#-# http://www.apache.org/licenses/LICENSE-2.0-#-# Unless required by applicable law or agreed to in writing, software-# distributed under the License is distributed on an "AS IS" BASIS,-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.-# See the License for the specific language governing permissions and-# limitations under the License.+# # Copyright (C) 2026 Intel Corporation+# # SPDX-License-Identifier: Apache-2.0+"""Backward-compat re-export shim.-# Refer code here:-# https://github.com/IST-DASLab/FP-Quant/blob/master/inference_lib/src/fp_quant/module/triton/mxfp4.py+The canonical implementation now lives in+:mod:`auto_round.algorithms.transforms.rotation.utils.triton.mxfp4`.+"""-import torch-import triton # pylint: disable=E0401-import triton.language as tl # pylint: disable=E0401--- configs=[- triton.Config({"BLOCK_SIZE": 32 * 32}),- triton.Config({"BLOCK_SIZE": 64 * 32}),- triton.Config({"BLOCK_SIZE": 128 * 32}),- triton.Config({"BLOCK_SIZE": 256 * 32}),- triton.Config({"BLOCK_SIZE": 512 * 32}),- ],- key=[],+from auto_round.algorithms.transforms.rotation.utils.triton.mxfp4 import ( # noqa: F401+ mxfp4_forward_kernel,+ mxfp4_forward_kernel_wrapper,)-def mxfp4_forward_kernel(- x_ptr,- hadamard_matrix_ptr,- output_ptr,- clip_mask_ptr,- n_elements: tl.constexpr,- hadamard_dim: tl.constexpr,- group_size: tl.constexpr,- gaussian_scale: tl.constexpr,- quest: tl.constexpr,- BLOCK_SIZE: tl.constexpr,-):- offsets_hadamard = tl.arange(0, hadamard_dim * hadamard_dim)- hadamard_matrix = tl.load(hadamard_matrix_ptr + offsets_hadamard).reshape(hadamard_dim, hadamard_dim)-- # load x- pid = tl.program_id(0)- start_idx = pid * BLOCK_SIZE- offsets = start_idx + tl.arange(0, BLOCK_SIZE)- mask = offsets < n_elements- x_flat = tl.load(x_ptr + offsets, mask=mask)-- # hadamard transform- x = tl.reshape(x_flat, (BLOCK_SIZE // hadamard_dim, hadamard_dim))- x_had = tl.dot(x, hadamard_matrix)-- # group- x_had_grouped = tl.reshape(x_had, (BLOCK_SIZE // group_size, group_size))-- # scale- # quest=True: per-group Gaussian-based scale = gaussian_scale * std- # quest=False: per-group max-abs-based scale, adjusted to FP4 range- if quest:- mean_squared = tl.sum(x_had_grouped * x_had_grouped, axis=-1, keep_dims=True) / group_size- mean = tl.sum(x_had_grouped, axis=-1, keep_dims=True) / group_size- std = tl.sqrt(mean_squared - mean * mean)- scales = gaussian_scale * std + 1e-8- shared_exps = tl.exp2(tl.floor(tl.log2(scales)))- x_had_scaled = x_had_grouped / shared_exps- else:- scales = tl.max(tl.abs(x_had_grouped), axis=-1, keep_dims=True)- shared_exps = tl.exp2(tl.floor(tl.log2(scales)) - 2) / (3 / 4)- x_had_scaled = x_had_grouped / shared_exps-- # quantize- # Map abs(x) to FP4 levels {0, 0.5, 1, 1.5, 2, 3, 4, 6}- x_had_scaled_abs = tl.abs(x_had_scaled)- x_had_scaled_sign = tl.where(- x_had_scaled > 0,- 1,- -1,- )-- x_fp4 = (- tl.where(- x_had_scaled_abs > 5,- 6,- tl.where(- x_had_scaled_abs > 3.5,- 4,- tl.where(- x_had_scaled_abs > 2.5,- 3,- tl.where(- x_had_scaled_abs > 1.75,- 2,- tl.where(- x_had_scaled_abs > 1.25,- 1.5,- tl.where(- x_had_scaled_abs > 0.75,- 1,- tl.where(- x_had_scaled_abs > 0.25,- 0.5,- 0,- ),- ),- ),- ),- ),- ),- )- * x_had_scaled_sign- )- if clip_mask_ptr is not None:- tl.store(- clip_mask_ptr + offsets,- tl.reshape(x_had_scaled_abs < 6, (BLOCK_SIZE,)),- mask=mask,- )-- # dequantize- x_dequantized = x_fp4 * shared_exps-- # Reshape back to flat form for storage- x_dequantized_flat = tl.reshape(x_dequantized, (BLOCK_SIZE,))-- # store- tl.store(output_ptr + offsets, x_dequantized_flat, mask=mask)---def mxfp4_forward_kernel_wrapper(- x,- hadamard_matrix,- return_clip_mask=False,- quest=False,- gaussian_scale=3 / 4,-):- """- Refer code here:- https://github.com/IST-DASLab/FP-Quant/blob/master/inference_lib/src/fp_quant/module/triton/mxfp4.py- Apply Hadamard transform + group-wise FP4 quantize/dequantize on x.-- Note:- The output is still in the Hadamard-transformed space (no inverse Hadamard is applied).- """- # Pick a device — we require CUDA- device = x.device- if device.type != "cuda":- raise RuntimeError(- f"mxfp4_forward_kernel_wrapper requires a CUDA tensor for 'x', "- f"but got device '{device.type}'. Please move inputs to CUDA before calling."- )-- # Ensure hadamard_matrix is on the same CUDA device- if hadamard_matrix.device != device:- hadamard_matrix = hadamard_matrix.to(device)-- dtype = hadamard_matrix.dtype-- if x.dtype != dtype:- x = x.to(dtype)-- # Make sure inputs are contiguous- x = x.contiguous()- hadamard_matrix = hadamard_matrix.contiguous()-- # Create output tensors on CUDA- output = torch.empty_like(x, device=device)- if return_clip_mask:- clip_mask = torch.empty_like(x, dtype=torch.bool, device=device).contiguous()- else:- clip_mask = None-- # Get total number of elements and calculate grid for launching the kernel- n_elements = x.numel()- grid = lambda meta: (triton.cdiv(n_elements, meta["BLOCK_SIZE"]),)-- # Launch kernel – no need for `with torch.device(...)`- mxfp4_forward_kernel[grid](- x_ptr=x,- hadamard_matrix_ptr=hadamard_matrix,- output_ptr=output,- clip_mask_ptr=clip_mask,- n_elements=n_elements,- hadamard_dim=hadamard_matrix.shape[-1],- group_size=32,- gaussian_scale=gaussian_scale,- quest=quest,- )- return output, clip_mask+__all__ = ["mxfp4_forward_kernel", "mxfp4_forward_kernel_wrapper"]diff --git a/auto_round/experimental/transform/utils/hadamard.py b/auto_round/experimental/transform/utils/hadamard.pyindex 320ae1832..8bed8a7c1 100644--- a/auto_round/experimental/transform/utils/hadamard.py+++ b/auto_round/experimental/transform/utils/hadamard.py@@ -1,151 +1,18 @@# # Copyright (C) 2026 Intel Corporation# # SPDX-License-Identifier: Apache-2.0+"""Backward-compat re-export shim.-import math-from pathlib import Path--import torch-from safetensors import safe_open--REPO_PATH = Path(__file__).parent / "hadamards.safetensors"+The canonical implementation now lives in+:mod:`auto_round.algorithms.transforms.rotation.utils.math`.+"""+from auto_round.algorithms.transforms.rotation.utils.math import _HADAMARD_MATRICES_PATH as REPO_PATH # noqa: F401+from auto_round.algorithms.transforms.rotation.utils.math import (+ _fetch_hadamard_divisor,+ _matmul_hadU,+ deterministic_hadamard_matrix,+ is_pow2,+ random_hadamard_matrix,+)__all__ = ["random_hadamard_matrix", "deterministic_hadamard_matrix", "is_pow2"]---# note that hadamard matrix multiplication reuses the code from-# https://github.com/vllm-project/compressed-tensors/blob/main/src/compressed_tensors/transform/utils/hadamard.py---def deterministic_hadamard_matrix(- size: int,- dtype: torch.dtype = torch.bfloat16,- device: torch.device = torch.device("cpu"),-) -> torch.Tensor:- """- Construct an n-by-n Hadamard matrix, using Sylvester's construction.- `n` must be a power of 2.-- Adapted from https://github.com/scipy/scipy/blob/v1.15.2/scipy/linalg/_special_matrices.py # noqa: E501-- :param size: order of the matrix, must be a power of 2- :param dtype: data type of matrix- :param device: device to construct matrix on- :return: hadamard matrix of size `size`- """- if size <= 0:- raise ValueError("Cannot construct deterministic hadamard of size <= 0")-- log2 = int(math.log2(size))- if size != 2**log2:- raise ValueError("Cannot construct deterministic hadamard of size != 2^n")-- H = torch.tensor([[1]], dtype=dtype, device=device)-- # Sylvester's construction- for _ in range(log2):- H = torch.vstack((torch.hstack((H, H)), torch.hstack((H, -H))))-- return H---def random_hadamard_matrix(- size: int,- dtype: torch.dtype = torch.bfloat16,- device: torch.device = torch.device("cpu"),- gen: torch.Generator | None = None,-) -> torch.Tensor:- """- Produces a randomly generated Hadamard matrix. Differs from- `deterministic_hadamard_matrix` in that this function supports non powers of 2- and randomization using a seeded generator-- Adapted from https://github.com/facebookresearch/SpinQuant/blob/main/utils/hadamard_utils.py # noqa: E501- Known matrices were retrieved from N. J. A. Sloane's Library of Hadamard Matrices- http://www.neilsloane.com/hadamard/ # noqa: E501-- :param size: The dimension of the hamadard matrix- :param dtype: data type of matrix- :param device: device to construct matrix on- :param gen: Optional generator random values- :return: randomly generated hadamard matrix- """- Q = torch.randint(low=0, high=2, size=(size,), generator=gen, device="cpu") # cpu- Q = Q.to(device=device, dtype=dtype)- Q = Q * 2 - 1- Q = torch.diag(Q)- return _matmul_hadU(Q)---def is_pow2(n: int) -> bool:- """- Check if a number is a power of 2-- :param n: number to check- :return: True iff `n` is a power of 2- """- return n > 0 and (n & (n - 1) == 0)---def _fetch_hadamard_divisor(- n: int,- dtype: torch.dtype,- device: torch.device = torch.device("cpu"),- file_path: str = REPO_PATH,-) -> torch.Tensor | None:- """- Fetch a known hadamard matrix from the given file path. The returned matrix will- be of of size `k` such that `n / k` is a power of two. Return None if no such- matrix exists.-- Note: This function reopens the safetensors file every time it is called.- This is technically inefficient, but a very small runtime cost and simpler- than forcing callers to manage the file open context-- :param n: size of known hadamard matrix- :param dtype: data type to move fetched hadamard to- :param device: device to move fetched hadamard to- :return: a known hadamard matrix of size `n` if one exists, else None- """- open_device = torch.device("cpu") if device.type == "meta" else device- with safe_open(file_path, framework="pt", device=str(open_device)) as file:- divisors = sorted((int(key) for key in file.keys()), reverse=True)- for divisor in divisors:- if n % divisor == 0 and is_pow2(n // divisor):- return file.get_tensor(str(divisor)).to(dtype=dtype, device=device)-- return None---def _matmul_hadU(X: torch.Tensor) -> torch.Tensor:- size = X.size(0)- dtype = X.dtype- device = X.device-- # Check if we have the determined hadamard matrix- hadK = _fetch_hadamard_divisor(size, dtype, device=device)- if hadK is None:- raise ValueError(f"Cannot construct random hadamard matrix of size {size}")- K = hadK.size(0)-- # Reshape diag matrix with randomized -1/+1- input = X.clone().view(-1, size, 1)- output = input.clone()- while input.shape[1] > K:- input = input.view(input.shape[0], input.shape[1] // 2, 2, input.shape[2])- output = output.view(input.shape)- output[:, :, 0, :] = input[:, :, 0, :] + input[:, :, 1, :]- output[:, :, 1, :] = input[:, :, 0, :] - input[:, :, 1, :]- output = output.view(input.shape[0], input.shape[1], -1)- input, output = (output, input)- assert input.shape[1] == K- del output-- # Do not explicitly repeat - OOM- # input = torch.bmm(- # hadK.repeat(len(input), 1, 1).to(input.device).to(input.dtype), input)- # Use bcast instead- input = hadK.view(1, K, K).to(input) @ input-- # normalize- return input.view(X.shape)diff --git a/auto_round/experimental/transform/utils/matrix.py b/auto_round/experimental/transform/utils/matrix.pyindex 46d684e80..184d91055 100644--- a/auto_round/experimental/transform/utils/matrix.py+++ b/auto_round/experimental/transform/utils/matrix.py@@ -1,98 +1,17 @@# # Copyright (C) 2026 Intel Corporation# # SPDX-License-Identifier: Apache-2.0+"""Backward-compat re-export shim.-import torch+The canonical implementation now lives in+:mod:`auto_round.algorithms.transforms.rotation.utils.matrix`.+"""-__all__ = ["apply_transform_weight"]--# note that apply_transform_weight reuses some code from-# https://github.com/vllm-project/compressed-tensors/blob/main/src/compressed_tensors/transform/utils/matrix.py---def apply_transform_weight(- transform_weight: torch.Tensor,- value: torch.Tensor,- location: str,- module_type: type[torch.nn.Module],-) -> torch.Tensor:- """- Using the transform location, apply the transform_weight to the- given value wrt linear weights. For more info on input and output transforms,- see `TransformLocation`-- The following explains how weights should be applied to values according to location-- let x be input activation- W be weight,- yh, xh, Wh be transformed output, input, weight-- note that- y = (x W.T) // torch.nn.Linear-- Choose values for yh, xh, and Wh which incorporate matrix transforms-- let V, Vi be transform matrices on input side- U, Ui be transform matrices on output side-- pick xh = (x V)- Wh = (U.T W Vi.T)- yh = (y U)+from auto_round.algorithms.transforms.rotation.utils.matrix import ( # noqa: F401+ apply_transform_weight,+ multihead_matmul,+)- The following shows that `yh = (xh) (Wh).T` for the chosen values of yh, xh, and Wh+# Old private name kept for backward compatibility.+_multihead_matmul = multihead_matmul- (xh) (Wh).T = (x V) (U.T W Vi.T).T- = (x V) (Vi W.T U) // transpose matrix product identity- = (x W.T) U- = y U- = yh-- :param transform_weight: transform weight to apply- :param value: value to apply transform_weight to- :param location: determines how weight should be applied- :param model_type: result of type(module), passed in to determine application of- weight transform- :return: value after transform_weight has been applied- """-- if location == "input":- return _multihead_matmul(value, transform_weight)-- if module_type == torch.nn.Linear:- return _multihead_matmul(value, transform_weight.T)---def _multihead_matmul(A: torch.Tensor, B: torch.Tensor) -> torch.Tensor:- """- Performs A @ B for last two dims of two matrices A and B that possibly- have different shapes, as is the case in multi-headed dimension. If- shapes are different, this is equivalent to converting the last two dims- of the smaller matrix into a block-diagonal matrix with the same shape as- the last two dims of the larger matrix.-- E.g. if A is half the size of B, this function will perform- [[A ] @ B- [ A]]-- If B is a third of the size of A, this function will perform- A @ [[B ]- [ B ]- [ B]]-- This function will error out if the shapes are not evenly divisible-- :param A: left-hand tensor- :param B: right-hand tensor- :return: result- """- if A.shape[-1] > B.shape[-2]:- head_dim = B.shape[-2]- num_heads = A.shape[-1] // head_dim- A = A.unflatten(-1, (num_heads, head_dim))- return (A @ B).flatten(-2, -1)- elif A.shape[-1] < B.shape[-2]:- head_dim = A.shape[-1]- num_heads = B.shape[-2] // head_dim- B = B.unflatten(-2, (num_heads, head_dim))- return (A @ B).flatten(-3, -2)- else:- return A @ B+__all__ = ["apply_transform_weight"]diff --git a/auto_round/experimental/utils.py b/auto_round/experimental/utils.pyindex ea1dade78..80e830bed 100644--- a/auto_round/experimental/utils.py+++ b/auto_round/experimental/utils.py@@ -138,84 +138,21 @@ def is_triton_kernel_available(data_type: str) -> bool:def dump_group_size_to_rotation_config(rotation_config: str | dict | RotationConfig, group_size: int):- rotation_dict = to_dict_rotation_config(rotation_config)- if rotation_dict.get("block_size", None) is None:- rotation_dict["block_size"] = group_size- return rotation_dict+ from auto_round.algorithms.transforms.rotation.config import dump_group_size_to_rotation_config as _impl++ return _impl(rotation_config, group_size)def to_dict_rotation_config(rotation_config: str | dict | RotationConfig):- if isinstance(rotation_config, str):- key = rotation_config.strip()- if not key:- return {}+ from auto_round.algorithms.transforms.rotation.config import to_dict_rotation_config as _impl- if key == "default":- cfg_dict = {"hadamard_type": "hadamard"}- else:- cfg_dict = {"hadamard_type": key}- elif isinstance(rotation_config, RotationConfig):- cfg_dict = rotation_config.model_dump()- else:- cfg_dict = dict(rotation_config)- return cfg_dict+ return _impl(rotation_config)def normalize_rotation_config(rotation_config: str | dict | RotationConfig | None, data_type: str) -> dict[str, Any]:- """- Normalize and validate `rotation_config`.-- Supported input types:- - None -> {}- - dict -> validated via RotationConfig- - RotationConfig -> validated & converted to dict- - str -> shorthand for `hadamard_type` in HADAMARDS keys-- Additional behavior:- - If block_size is not set by user:- - mx_fp -> default block_size to 32- - nv_fp -> default block_size to 16- - other data types -> emit a warning- - If block_size is set but does not match the recommended value:- - mx_fp expects 32- - nv_fp expects 16- - emit a warning- """+ from auto_round.algorithms.transforms.rotation.config import normalize_rotation_config as _impl- def _apply_data_type_block_size(cfg_dict: dict[str, Any], block_size_explicitly_set: bool) -> dict[str, Any]:- block_size = cfg_dict.get("block_size")-- if not block_size_explicitly_set or block_size is None:- if is_mx_fp(data_type):- cfg_dict["block_size"] = 32- logger.warning("block_size is not set for data_type 'mx_fp'; defaulting to 32.")- elif is_nv_fp(data_type):- cfg_dict["block_size"] = 16- logger.warning("block_size is not set for data_type 'nv_fp'; defaulting to 16.")- else:- logger.warning(- f"block_size is not set and cannot be inferred for data_type {data_type!r}; "- "please set block_size explicitly in rotation_config if needed."- )- else:- if is_mx_fp(data_type) and block_size != 32:- logger.warning(f"data_type is 'mx_fp' but block_size={block_size}; recommended value is 32.")- elif is_nv_fp(data_type) and block_size != 16:- logger.warning(f"data_type is 'nv_fp' but block_size={block_size}; recommended value is 16.")-- return cfg_dict-- # 1) None -> {}- if rotation_config is None:- return {}-- rotation_dict = to_dict_rotation_config(rotation_config)- block_size_explicitly_set = "block_size" in rotation_dict- cfg_dict = _apply_data_type_block_size(rotation_dict, block_size_explicitly_set)- try:- return RotationConfig.model_validate(cfg_dict).model_dump()- except Exception as e:- raise ValueError(f"Invalid RotationConfig: {e}") from e+ return _impl(rotation_config, data_type)def check_supported_schemes(scheme: str):diff --git a/auto_round/export/export_to_autogptq/export.py b/auto_round/export/export_to_autogptq/export.pyindex 5594b5979..c46b8c947 100644--- a/auto_round/export/export_to_autogptq/export.py+++ b/auto_round/export/export_to_autogptq/export.py@@ -205,7 +205,7 @@ def save_quantized_as_autogptq(# --- 1️⃣ Extract inputs & configs ---quantization_config = serialization_dict- quant_block_list = serialization_dict.get("quant_block_list", get_block_names(model))+ quant_block_list = serialization_dict.get("quant_block_list") or get_block_names(model)processor = kwargs.get("processor")image_processor = kwargs.get("image_processor")safe_serialization = kwargs.get("safe_serialization", True)@@ -249,7 +249,7 @@ def save_quantized_as_autogptq(continue# Handle block layersif in_blocks or (block_name_to_quantize and check_start_with_block_name(layer_name, block_name_to_quantize)):- neq_keys = check_neq_config(cfg, **{k: quantization_config[k] for k in scheme_keys})+ neq_keys = check_neq_config(cfg, **{k: quantization_config.get(k) for k in scheme_keys})if neq_keys:if matches_any_regex(layer_name, regex_config):continuediff --git a/auto_round/export/export_to_autoround/export.py b/auto_round/export/export_to_autoround/export.pyindex 60613b7e8..94a439975 100644--- a/auto_round/export/export_to_autoround/export.py+++ b/auto_round/export/export_to_autoround/export.py@@ -12,7 +12,6 @@# See the License for the specific language governing permissions and# limitations under the License.-import copyimport functoolsimport inspect@@ -285,7 +284,7 @@ def save_quantized_as_autoround(elif cfg["in_blocks"] or (block_name_to_quantize is not None and check_start_with_block_name(layer_name, block_name_to_quantize)):- neq_keys = check_neq_config(cfg, **{k: quantization_config[k] for k in scheme_keys})+ neq_keys = check_neq_config(cfg, **{k: quantization_config.get(k) for k in scheme_keys})if len(neq_keys) > 0:extra_config[layer_name] = {}for key in neq_keys:@@ -296,7 +295,7 @@ def save_quantized_as_autoround(if regex_config is not None:for name, cfg in regex_config.items():regex_name = to_standard_regex(name)- neq_keys = check_neq_config(cfg, **{k: quantization_config[k] for k in scheme_keys})+ neq_keys = check_neq_config(cfg, **{k: quantization_config.get(k) for k in scheme_keys})if len(neq_keys) > 0:extra_config[regex_name] = {}for key in neq_keys:diff --git a/auto_round/export/export_to_autoround/export_to_fp8.py b/auto_round/export/export_to_autoround/export_to_fp8.pyindex 90c228583..200e10a94 100644--- a/auto_round/export/export_to_autoround/export_to_fp8.py+++ b/auto_round/export/export_to_autoround/export_to_fp8.py@@ -228,7 +228,7 @@ def save_quantized_as_autoround(elif cfg["in_blocks"] or (block_name_to_quantize is not None and check_start_with_block_name(layer_name, block_name_to_quantize)):- neq_keys = check_neq_config(cfg, **{k: quantization_config[k] for k in scheme_keys})+ neq_keys = check_neq_config(cfg, **{k: quantization_config.get(k) for k in scheme_keys})if len(neq_keys) > 0:extra_config[layer_name] = {}for key in neq_keys:diff --git a/auto_round/export/export_to_autoround/export_to_nvfp_mx.py b/auto_round/export/export_to_autoround/export_to_nvfp_mx.pyindex 506a6a85d..6c2926cfb 100644--- a/auto_round/export/export_to_autoround/export_to_nvfp_mx.py+++ b/auto_round/export/export_to_autoround/export_to_nvfp_mx.py@@ -214,7 +214,7 @@ def save_quantized_as_fp(elif cfg["in_blocks"] or (block_name_to_quantize is not None and check_start_with_block_name(layer_name, block_name_to_quantize)):- neq_keys = check_neq_config(cfg, **{k: quantization_config[k] for k in scheme_keys})+ neq_keys = check_neq_config(cfg, **{k: quantization_config.get(k) for k in scheme_keys})if len(neq_keys) > 0:extra_config[layer_name] = {}for key in neq_keys:@@ -225,7 +225,7 @@ def save_quantized_as_fp(if regex_config is not None:for name, cfg in regex_config.items():regex_name = to_standard_regex(name)- neq_keys = check_neq_config(cfg, **{k: quantization_config[k] for k in scheme_keys})+ neq_keys = check_neq_config(cfg, **{k: quantization_config.get(k) for k in scheme_keys})if len(neq_keys) > 0:extra_config[regex_name] = {}for key in neq_keys:diff --git a/auto_round/export/export_to_gguf/export.py b/auto_round/export/export_to_gguf/export.pyindex c3a06c2ad..5f6dd8453 100644--- a/auto_round/export/export_to_gguf/export.py+++ b/auto_round/export/export_to_gguf/export.py@@ -146,11 +146,7 @@ def pack_gguf_layer():"""Export the model to gguf format."""global gguf_model_instance_global- # if output_dir is not None and os.path.exists(output_dir):- # logger.warning_once(f"{output_dir} already exists, this may cause model conflict")if "gguf_model_instance_global" not in globals():- config = model.config-gguf_model_instance_global = [create_model_class(output_dir,diff --git a/auto_round/export/export_to_llmcompressor/config.py b/auto_round/export/export_to_llmcompressor/config.pyindex 546b9addb..62017ec73 100644--- a/auto_round/export/export_to_llmcompressor/config.py+++ b/auto_round/export/export_to_llmcompressor/config.py@@ -28,16 +28,17 @@from auto_round.utils import logger-def check_compressed_tensors_supported(): # pragma: no cover+def check_compressed_tensors_supported(raise_error: bool = False): # pragma: no covertry:import compressed_tensors # noqa: F401return Trueexcept ImportError:- logger.error(- "Please install compressed-tensors via 'pip install compressed-tensors'" " to save as llm-compressor format"- )- exit(-1)+ msg = "Please install compressed-tensors via 'pip install compressed-tensors' to save as llm-compressor format"+ logger.error(msg)+ if raise_error:+ raise ImportError(msg) from None+ return Falseif check_compressed_tensors_supported():@@ -64,7 +65,7 @@ def initialize_quantization(scheme, targets=["Linear"], config_groups=None, kv_ckv_cache_scheme = kv_cache_schemeignore = ignoreusing_mxfp4_for_mxfp8 = False- check_compressed_tensors_supported()+ check_compressed_tensors_supported(raise_error=True)if scheme is not None and config_groups is not None:raise ValueError("Please specify either `scheme` or `config_groups`")diff --git a/auto_round/export/export_to_llmcompressor/export_to_fp.py b/auto_round/export/export_to_llmcompressor/export_to_fp.pyindex b33bbb951..b18d9b149 100644--- a/auto_round/export/export_to_llmcompressor/export_to_fp.py+++ b/auto_round/export/export_to_llmcompressor/export_to_fp.py@@ -272,7 +272,7 @@ def save_quantized_as_fp(ignore.append(lm_head_name)# get llm-compressor format config- check_compressed_tensors_supported()+ check_compressed_tensors_supported(raise_error=True)# Detect mixed precision by grouping quantized layers by (bits, data_type)scheme_groups = {} # (bits, data_type) -> list of layer namesdiff --git a/auto_round/export/export_to_llmcompressor/export_to_static_fp.py b/auto_round/export/export_to_llmcompressor/export_to_static_fp.pyindex c733c4510..3f5ee91f3 100644--- a/auto_round/export/export_to_llmcompressor/export_to_static_fp.py+++ b/auto_round/export/export_to_llmcompressor/export_to_static_fp.py@@ -153,7 +153,7 @@ def save_quantized_as_static_fp(pack_layer(name, model, serialization_dict.get("data_type", "fp8"), device)# Get llm-compressor format config- check_compressed_tensors_supported()+ check_compressed_tensors_supported(raise_error=True)from compressed_tensors.quantization import ( # pylint: disable=E0401QuantizationArgs,QuantizationConfig,diff --git a/auto_round/export/utils.py b/auto_round/export/utils.pyindex a71aad2ab..c740791ef 100644--- a/auto_round/export/utils.py+++ b/auto_round/export/utils.py@@ -15,6 +15,7 @@ import jsonimport osimport shutil+import torchimport torch.nn as nnimport auto_round.envs as envs@@ -26,6 +27,21 @@ from auto_round.utils import ()+def _save_model_configs(model: nn.Module, save_dir: str) -> None:+ if hasattr(model, "config") and model.config is not None:+ model.config.save_pretrained(save_dir)++ if hasattr(model, "generation_config") and model.generation_config is not None:+ model.generation_config.save_pretrained(save_dir)+++def _state_dict_has_meta_tensor(model: nn.Module) -> bool:+ for tensor in model.state_dict().values():+ if isinstance(tensor, torch.Tensor) and tensor.device.type == "meta":+ return True+ return False++def is_local_pipeline_model_dir(model_dir: str) -> bool:if not model_dir or not os.path.isdir(model_dir):return False@@ -184,13 +200,17 @@ def save_model(os.makedirs(save_dir, exist_ok=True)if unsupported_meta_device(model):- if hasattr(model, "config") and model.config is not None:- model.config.save_pretrained(save_dir)-- if hasattr(model, "generation_config") and model.generation_config is not None:- model.generation_config.save_pretrained(save_dir)+ _save_model_configs(model, save_dir)else:- model.save_pretrained(save_dir, max_shard_size=max_shard_size, safe_serialization=safe_serialization)+ has_meta_tensor = _state_dict_has_meta_tensor(model)+ if has_meta_tensor:+ logger.info(+ "Detected meta tensors in state_dict after shard-based saving; skipping model.save_pretrained and "+ "saving configs only."+ )+ _save_model_configs(model, save_dir)+ else:+ model.save_pretrained(save_dir, max_shard_size=max_shard_size, safe_serialization=safe_serialization)# Allow disabling copy_missing_tensors_from_source via env var AR_DISABLE_COPY_MTP_WEIGHTS, default enabledif not envs.AR_DISABLE_COPY_MTP_WEIGHTS:@@ -330,7 +350,9 @@ def filter_quantization_config(quantization_config):"scale_dtype": "torch.float16","seqlen": 2048,}- iters = quantization_config.get("iters", 200)+ iters = quantization_config.get("iters")+ if iters is None:+ iters = 0default_dict["lr"] = 1.0 / iters if iters > 0 else 5e-3default_dict["minmax_lr"] = default_dict["lr"]@@ -349,13 +371,15 @@ def filter_quantization_config(quantization_config):quantization_config.pop("act_sym", None)quantization_config.pop("act_group_size", None)- clean_list = ("supported_types", "quant_block_list")+ clean_list = ("supported_types", "quant_block_list", "transform_configs")for key in list(quantization_config.keys()):if callable(key):quantization_config.pop(key)elif isinstance(quantization_config[key], (list, tuple)):if any([callable(item) for item in quantization_config[key]]):quantization_config.pop(key)+ elif len(quantization_config[key]) == 0:+ quantization_config.pop(key)if key in clean_list and key in quantization_config:quantization_config.pop(key)return quantization_configdiff --git a/auto_round/formats.py b/auto_round/formats.pyindex cd7f076ea..3a19fe2d6 100644--- a/auto_round/formats.py+++ b/auto_round/formats.py@@ -45,7 +45,9 @@ from auto_round.schemes import (get_gguf_scheme,)from auto_round.utils import (+ INNER_SUPPORTED_LAYER_TYPES,SUPPORTED_FORMATS,+ SUPPORTED_LAYER_TYPES,check_to_quantized,compress_layer_names,copy_python_files_from_model_cache,@@ -158,7 +160,7 @@ def _check_divisible_by_32(ar):skipped_layers = []if default_dict["data_type"] == "int" and default_dict["act_bits"] >= 16:for n, m in ar.model.named_modules():- if type(m) in ar.supported_types or m.__class__.__name__ in ar.inner_supported_types:+ if type(m) in SUPPORTED_LAYER_TYPES or m.__class__.__name__ in INNER_SUPPORTED_LAYER_TYPES:if m.weight.shape[0] % 32 or m.weight.shape[1] % 32:if ar.layer_config is None:ar.layer_config = {}@@ -378,7 +380,7 @@ class LLMCompressorFormat(OutputFormat):if is_nv_fp(ar.data_type) or is_mx_fp(ar.data_type):from auto_round.export.export_to_llmcompressor import check_compressed_tensors_supported- check_compressed_tensors_supported()+ check_compressed_tensors_supported(raise_error=True)self.backend = LLMCompressorFormat(ar.data_type, ar)elif is_dynamic_afp8(ar) and is_block_wfp8(ar):self.backend = LLMCompressorFormat(AutoRoundExportFormat.FP8_BLOCK.value, ar)@@ -690,7 +692,7 @@ class AutoAWQFormat(OutputFormat):if not awq_supported:logger.warning(f"The AutoAWQ format may not be supported due to {info}")if ar.bits != 4:- raise ValueError("The AWQ format only supports W4 quantization ")+ raise ValueError(f"auto_awq format support quantization scheme with W4A16 but got bits={ar.bits}")if self.backend is None:_check_divisible_by_32(ar)@@ -759,6 +761,7 @@ class GGUFFormat(OutputFormat):def __init__(self, format: str, ar: BaseCompressor):if format.startswith("gguf:"):+ self._original_format = format # preserve "gguf:q2_k_mixed" etc. for Phase 2bself.gguf_args_check(ar, format, model_type=ModelType.TEXT)if ar.mllm:self.gguf_args_check(ar, format, model_type=ModelType.MMPROJ)@@ -794,14 +797,14 @@ class GGUFFormat(OutputFormat):return Truedef check_and_reset_format(self, ar):- if ar.iters != 0 and ar.bits != 3 and not ar.enable_alg_ext:+ if getattr(ar, "iters", 0) != 0 and ar.bits != 3 and not ar.enable_alg_ext:logger.warning_once("`iters=0` is recommended when exporting to current GGUF format"" or add `enable_alg_ext` for better accuracy with much more tuning cost."" Please refer to https://github.com/intel/auto-round/tree/main/docs/gguf_alg_ext_acc.md"" for the accuracy results.")- elif ar.bits >= 8 and ar.iters != 0:+ elif ar.bits >= 8 and getattr(ar, "iters", 0) != 0:logger.warning_once("`iters=0` is recommended for bits>=8")if getattr(ar, "quant_nontext_module", False):diff --git a/auto_round/modeling/fused_moe/gpt_oss.py b/auto_round/modeling/fused_moe/gpt_oss.pyindex 959e01356..da68a57fc 100644--- a/auto_round/modeling/fused_moe/gpt_oss.py+++ b/auto_round/modeling/fused_moe/gpt_oss.py@@ -12,7 +12,6 @@# See the License for the specific language governing permissions and# limitations under the License.-import torchimport transformersfrom packaging import version@@ -33,6 +32,7 @@ from auto_round.utils import clear_memory, unsupported_meta_deviceclass GPTOssSingleExpert(nn.Module):+def __init__(self, hidden_size: int, intermediate_size: int, dtype: torch.dtype | None = None):super().__init__()self.hidden_size = hidden_size@@ -101,17 +101,26 @@ class SequentialGPTOSSMoE(ReplacementModuleBase):B, T, H = hidden_states.shapex = hidden_states.reshape(-1, H)- # Use the original router (it returns scores and indices already softmaxed over top-k)- router_scores, router_indices = self.router(x) # scores: [tokens, E], indices: [tokens, k]+ # Use the original router (it returns logits, scores and indices)+ router_out = self.router(x)+ if len(router_out) == 3:+ _, router_scores, router_indices = router_out+ else:+ router_scores, router_indices = router_outfinal_hidden_states = self.shared_expert(x) if self.shared_expert is not None else torch.zeros_like(x)num_all_tokens, total_num_experts = x.size(0), self.num_expertsmask_weights = torch.zeros((num_all_tokens, total_num_experts), dtype=x.dtype, device=x.device)- topk_ids, experts_mask = router_indices, router_scores- topk_ids = topk_ids.to(torch.int64)+ topk_ids = router_indices.to(torch.int64)mask_weights.scatter_(-1, topk_ids, 1)+ # Build per-expert routing score matrix: shape (num_experts, num_tokens)+ # experts_mask[e, t] = router score of expert e for token t (0 if not selected)+ expert_score_matrix = torch.zeros_like(mask_weights)+ expert_score_matrix.scatter_(-1, topk_ids, router_scores)+ expert_score_matrix = expert_score_matrix.transpose(0, 1) # (num_experts, num_tokens)+mask_weights = mask_weights[:num_all_tokens, :total_num_experts]mask_weights = mask_weights.transpose(0, 1)@@ -124,7 +133,7 @@ class SequentialGPTOSSMoE(ReplacementModuleBase):mask_weight = mask_weights[expert_idx].unsqueeze(1)current_state_static = x * mask_weightexpert_output = self.experts[expert_idx](current_state_static)- expert_output = expert_output * experts_mask[expert_idx].unsqueeze(1)+ expert_output = expert_output * expert_score_matrix[expert_idx].unsqueeze(1)final_hidden_states += expert_outputreturn final_hidden_states.view(B, T, H), router_scores.view(B * T, -1)diff --git a/auto_round/modeling/unfused_moe/__init__.py b/auto_round/modeling/unfused_moe/__init__.pyindex 3b9511731..a112a2a15 100644--- a/auto_round/modeling/unfused_moe/__init__.py+++ b/auto_round/modeling/unfused_moe/__init__.py@@ -145,7 +145,10 @@ def get_file_path_via_model_name(model_or_path: str, file_name):def pre_check_config(model_name: str | torch.nn.Module, trust_remote_code: bool = True):if isinstance(model_name, str):- config = AutoConfig.from_pretrained(model_name, trust_remote_code=trust_remote_code)+ try:+ config = AutoConfig.from_pretrained(model_name, trust_remote_code=trust_remote_code)+ except (OSError, EnvironmentError, ValueError):+ return Falseelif isinstance(model_name, torch.nn.Module):config = getattr(model_name, "config", None)if config is None:diff --git a/auto_round/schemes.py b/auto_round/schemes.pyindex c3bec7ebf..a298575aa 100644--- a/auto_round/schemes.py+++ b/auto_round/schemes.py@@ -13,15 +13,19 @@# limitations under the License.import copyfrom copy import deepcopy-from dataclasses import dataclass, fields-from typing import Optional, Union+from dataclasses import asdict, dataclass, fields+from typing import TYPE_CHECKING, Any, Optional, Unionimport torchfrom auto_round.logger import logger+from auto_round.utils import SUPPORTED_DTYPES, infer_bits_by_data_type__all__ = ["QuantizationScheme", "get_gguf_scheme", "preset_name_to_scheme"]+if TYPE_CHECKING:+ from auto_round.auto_scheme.gen_auto_scheme import AutoScheme+@dataclassclass QuantizationScheme:@@ -40,7 +44,9 @@ class QuantizationScheme:@classmethoddef from_dict(cls, config: dict):- return cls(**config)+ field_names = {f.name for f in fields(cls)}+ filtered_config = {k: v for k, v in config.items() if k in field_names}+ return cls(**filtered_config)@classmethoddef get_attributes(cls: "QuantizationScheme") -> list[str]:@@ -132,6 +138,147 @@ def is_preset_scheme(name: str) -> bool:return name.upper() in PRESET_SCHEMES+def _reconcile_bits_and_dtype(config: dict, prefix: str = ""):+ """+ Harmonizes 'bits' and 'data_type' for weights or activations.+ Ensures internal consistency by prioritizing data_type inference.+ """++ dt_key = f"{prefix}data_type"+ bits_key = f"{prefix}bits"++ if config.get(dt_key) is None:+ return++ # Infer the correct bit-width based on the data_type string+ inferred_bits = infer_bits_by_data_type(config[dt_key])++ if inferred_bits is not None and inferred_bits < 16:+ # Check for conflict between user-specified bits and inferred bits+ if inferred_bits != config.get(bits_key):+ logger.warning(f"'{dt_key}' does not match '{bits_key}'. " f"Resetting '{bits_key}' to {inferred_bits}.")+ config[bits_key] = inferred_bits++ # Normalize data_type (e.g., 'mx_fp4' -> 'mx')+ for supported in SUPPORTED_DTYPES:+ if config[dt_key] == f"{supported}{inferred_bits}":+ config[dt_key] = supported+ break+++def _override_scheme_with_user_specify(+ scheme: Union[str, dict, QuantizationScheme], user_scheme_overrides: dict[str, Any], return_str=True+) -> Union[str, QuantizationScheme]:+ """+ Updates a base quantization scheme with user-provided overrides.+ Handles GGUF formatting and synchronizes weight/activation parameters.+ """+ # 1. GGUF special handling: map data_type suffix to GGUF scheme names+ dt_override = user_scheme_overrides.get("data_type", "")+ if (+ isinstance(scheme, QuantizationScheme) or (isinstance(scheme, str) and not scheme.startswith("gguf"))+ ) and dt_override.endswith("_dq"):+ if "bits" not in user_scheme_overrides:+ raise KeyError(f"Must specify 'bits' when using data_type={dt_override}")++ bits = user_scheme_overrides["bits"]+ suffix = "k" if bits == 6 else "k_s"+ scheme = f"gguf:q{bits}_{suffix}"++ # 2. Convert input scheme to a dictionary for processing+ if isinstance(scheme, QuantizationScheme):+ scheme_dict = asdict(scheme)+ elif isinstance(scheme, str):+ normalized_name = scheme.strip("'\" ").upper()+ if normalized_name.startswith("GGUF") and len(user_scheme_overrides) > 0:+ logger.warning_once(+ "When using GGUF scheme, user-specified overrides will be ignored to ensure format compatibility."+ )+ user_scheme_overrides = {}+ # If no overrides exist, return the normalized string immediately+ if not user_scheme_overrides and return_str:+ return normalized_name+ scheme_dict = asdict(preset_name_to_scheme(normalized_name))+ else:+ scheme_dict = scheme.copy()++ # 3. Apply overrides and define default behaviors+ scheme_dict.update(user_scheme_overrides)++ if scheme_dict.get("act_dynamic") is None:+ scheme_dict["act_dynamic"] = True++ # 4. Reconcile weight settings (bits vs data_type)+ _reconcile_bits_and_dtype(scheme_dict)++ # 5. Fallback logic: Inherit activation settings from weight settings+ scheme_dict["act_group_size"] = (+ scheme_dict.get("act_group_size")+ if scheme_dict.get("act_group_size") is not None+ else scheme_dict.get("group_size")+ )+ scheme_dict["act_bits"] = scheme_dict.get("act_bits") or 16+ scheme_dict["act_sym"] = (+ scheme_dict.get("act_sym") if scheme_dict.get("act_sym") is not None else scheme_dict.get("sym")+ )++ # 6. Activation data_type logic+ if scheme_dict.get("act_data_type") is None:+ is_supported = scheme_dict["data_type"] in SUPPORTED_DTYPES+ if is_supported and scheme_dict["act_bits"] < 16:+ scheme_dict["act_data_type"] = scheme_dict["data_type"]+ logger.info(f"Activation adopting weight data_type: {scheme_dict['data_type']}")+ else:+ scheme_dict["act_data_type"] = "float"++ # 7. Reconcile activation settings+ _reconcile_bits_and_dtype(scheme_dict, prefix="act_")++ return QuantizationScheme.from_dict(scheme_dict)+++def _parse_scheme(+ scheme: Union[str, dict, QuantizationScheme, "AutoScheme"], user_scheme_overrides: dict[str, Any]+) -> tuple[Union[str, QuantizationScheme], bool]:+ """+ Parses the final scheme.+ """+ from auto_round.auto_scheme.gen_auto_scheme import AutoScheme++ is_auto_scheme = isinstance(scheme, AutoScheme)+ if is_auto_scheme:+ if not scheme.options:+ raise ValueError("AutoScheme options cannot be empty")+ else:+ for option in scheme.options:+ if isinstance(option, str):+ if "mixed" in option:+ raise ValueError(f"Mixed option {option} is not supported")++ # Map user overrides across all auto-scheme options+ scheme.options = [_override_scheme_with_user_specify(opt, user_scheme_overrides) for opt in scheme.options]++ # Select the primary scheme for attribute binding (skipping BF16)+ default_scheme = scheme.options[0]+ for opt in scheme.options:+ if opt == "BF16":+ continue+ if isinstance(opt, QuantizationScheme):+ if opt.bits < 16 or (opt.act_bits and opt.act_bits < 16):+ default_scheme = opt+ break+ else:+ default_scheme = _override_scheme_with_user_specify(scheme, user_scheme_overrides)++ # Extract attributes from the chosen default_scheme+ if isinstance(default_scheme, str):+ final_attrs = _override_scheme_with_user_specify(default_scheme, user_scheme_overrides, return_str=False)+ final_attrs = asdict(final_attrs)+ else:+ final_attrs = asdict(default_scheme)+ return default_scheme, is_auto_scheme, final_attrs++W4A16 = QuantizationScheme.from_dict({"bits": 4,@@ -182,7 +329,6 @@ W2A16G64 = QuantizationScheme.from_dict(})-W2A16G32 = QuantizationScheme.from_dict({"bits": 2,@@ -252,7 +398,6 @@ MXFP4_RCEIL = QuantizationScheme.from_dict(})-MXFP8 = QuantizationScheme.from_dict({"bits": 8,@@ -377,7 +522,6 @@ INT4 = QuantizationScheme.from_dict(})-# For AutoScheme 16 bits optionsBF16 = QuantizationScheme.from_dict({@@ -482,6 +626,18 @@ def get_gguf_scheme(scheme: Union[str, QuantizationScheme]) -> str:return schemeif isinstance(scheme, str):return ""+ # AutoScheme is a lazy placeholder whose concrete option is resolved later+ # (after model loading). It is never a GGUF scheme itself.+ from auto_round.auto_scheme.gen_auto_scheme import AutoScheme++ if isinstance(scheme, AutoScheme):+ # options is always a list after AutoScheme.__post_init__.+ # If the primary option is a GGUF scheme, propagate it so that+ # scale_dtype defaults to fp32 (GGUF convention).+ primary = scheme.options[0] if scheme.options else None+ if isinstance(primary, str) and primary.upper().startswith("GGUF"):+ return primary+ return ""for key, val in PRESET_SCHEMES.items():# For q40 or q4_1 we only support it with str scheme, otherwise it will be matched incorrectly with W4G32if not key.upper().startswith("GGUF") or ("0" in key or "1" in key):diff --git a/auto_round/special_model_handler.py b/auto_round/special_model_handler.pyindex 1a336ee6f..264c77ab8 100644--- a/auto_round/special_model_handler.py+++ b/auto_round/special_model_handler.py@@ -191,19 +191,20 @@ def _patch_gemma4_model(model):def _handle_special_model(model):- if hasattr(model, "config") and model.config.model_type == "deepseek_vl_v2":+ model_type = getattr(getattr(model, "config", None), "model_type", None)+ if model_type == "deepseek_vl_v2":from functools import partialmodel.forward = partial(_deepseek_vl2_forward, model)- if hasattr(model, "config") and model.config.model_type == "qwen2_5_omni":+ if model_type == "qwen2_5_omni":from functools import partialmodel.forward = partial(_qwen2_5_omni_forward, model)- if hasattr(model, "config") and model.config.model_type == "qwen3_omni_moe":+ if model_type == "qwen3_omni_moe":from functools import partialmodel.forward = partial(_qwen3_omni_moe_forward, model)- if hasattr(model, "config") and model.config.model_type == "gemma4":+ if hasattr(model, "config") and model_type == "gemma4":import transformersfrom packaging import versiondiff --git a/auto_round/utils/common.py b/auto_round/utils/common.pyindex d49999225..865134b8b 100644--- a/auto_round/utils/common.py+++ b/auto_round/utils/common.py@@ -1031,3 +1031,27 @@ def compress_layer_names(names: list) -> str:parts.extend(singles)parts.sort()return ", ".join(parts)+++def infer_bits_by_data_type(data_type: str):+ """Infer bits by data_type++ Args:+ data_type (str): data_type++ Returns:+ int: bits inferred by data_type, None means cannot infer correct bits by data_type+ """+ from auto_round.utils import SUPPORTED_DTYPES++ if data_type is None:+ return 16+ for supported_dtype in SUPPORTED_DTYPES:+ if data_type.startswith(supported_dtype) and len(data_type) > len(supported_dtype):+ ##first check the following two bits+ suc_2str = data_type[len(supported_dtype) : len(supported_dtype) + 2]+ if str.isdigit(suc_2str):+ return int(suc_2str)+ if str.isdigit(data_type[len(supported_dtype)]):+ return int(data_type[len(supported_dtype)])+ return Nonediff --git a/auto_round/utils/device.py b/auto_round/utils/device.pyindex a4727a996..c05a17e79 100644--- a/auto_round/utils/device.py+++ b/auto_round/utils/device.py@@ -40,7 +40,6 @@ DEVICE_ENVIRON_VARIABLE_MAPPING = {"hpu": "HABANA_VISIBLE_MODULES",}-# Note on HPU usage:# There are two modes available for enabling auto-round on HPU:# 1. Compile Mode@@ -360,7 +359,16 @@ def detect_device(device: Union[None, str, int, torch.device] = None) -> str:return device-def get_device_and_parallelism(device: Union[str, torch.device, int]) -> tuple[str, bool]:+def get_device_and_parallelism(device: Union[str, torch.device, int, dict]) -> tuple[str, bool]:+ if device is None:+ device = detect_device(device)+ return device, False+ if isinstance(device, dict):+ unique_devices = set(device.values())+ if len(unique_devices) == 1:+ device = next(iter(unique_devices))+ else:+ device = "auto"if isinstance(device, str):if device in ["cuda", "xpu", "hpu"]:device = detect_device(device)@@ -481,6 +489,7 @@ class fake_triton_for_hpu(ContextDecorator):# Create and inject fake triton moduleclass FakeTriton:+def __getattr__(self, name):return None@@ -646,6 +655,25 @@ def _clear_memory_for_cpu_and_cuda(_malloc_trim_counter = 0+def _force_trim_malloc() -> None:+ """Unconditionally release glibc heap pages back to the OS on Linux.++ Unlike :func:`_maybe_trim_malloc`, this ignores the call-count throttle and+ always invokes ``malloc_trim(0)``. Use at critical lifecycle boundaries+ (end of model loading, end of post_init, start of quantize loop) where a+ one-time trim has a meaningful impact on peak RSS.+ """+ if os.name != "posix":+ return+ if os.environ.get("AR_ENABLE_MALLOC_TRIM", "1") != "1":+ return+ try:+ libc = ctypes.CDLL("libc.so.6")+ libc.malloc_trim(0)+ except Exception:+ pass++def _maybe_trim_malloc() -> None:"""Optionally release glibc heap pages back to OS on Linux.@@ -678,6 +706,7 @@ def _maybe_trim_malloc() -> None:class ClearMemory:+def __init__(self, device_list: list | tuple | None = None):self.device_list = device_list@@ -689,6 +718,13 @@ class ClearMemory:from auto_round.utils.device import is_hpex_availableif is_hpex_available():+ # Clear CPU-side references so Python can reclaim them.+ if isinstance(tensor, list):+ for i in range(len(tensor)):+ tensor[i] = None+ tensor = None+ gc.collect()+ _force_trim_malloc()memory_monitor.update_hpu(device_list)returnelse:@@ -1789,6 +1825,7 @@ def dump_mem_usage(msg: str = "", log_level: str = "info"):"""Decorator to dump memory usage before and after a function call."""def decorator(func):+@functools.wraps(func)def wrapper(*args, **kwargs):memory_monitor.update_cpu()diff --git a/auto_round/utils/model.py b/auto_round/utils/model.pyindex 210905e84..04a8b0395 100644--- a/auto_round/utils/model.py+++ b/auto_round/utils/model.py@@ -18,7 +18,7 @@ import osimport refrom collections import UserDictfrom pathlib import Path-from typing import Union+from typing import TYPE_CHECKING, Unionimport psutilimport torch@@ -28,7 +28,6 @@ from packaging import versionfrom auto_round import envsfrom auto_round.export.export_to_gguf.config import ModelTypefrom auto_round.logger import logger-from auto_round.schemes import QuantizationSchemefrom auto_round.utils.common import monkey_patch_modelfrom auto_round.utils.weight_handler import (_dequant_fp8_linear_weight,@@ -39,6 +38,9 @@ from auto_round.utils.weight_handler import (FIX_MISTRAL_REGEX_MODEL_TYPE_LIST = ["longcat_next"]+if TYPE_CHECKING:+ from auto_round.schemes import QuantizationScheme+def clean_module_parameter(submodule: torch.nn.Module, param_name: str) -> None:"""This function is recommended to be used instead of module.weight = None.@@ -762,7 +764,10 @@ def is_mllm_model(model_or_path: Union[str, torch.nn.Module], platform: str = Nomodel_path = model_or_path if isinstance(model_or_path, str) else model_or_path.name_or_path# For dummy model, model_path could be "".- if model_path and not os.path.isdir(model_path):+ # Only try to download if the path looks like a HF repo id (not a local filesystem path).+ # Skip download for absolute paths or relative paths that contain current/parent dir markers.+ _is_local_path = os.path.isabs(model_path) or model_path.startswith("./") or model_path.startswith("../")+ if model_path and not os.path.isdir(model_path) and not _is_local_path:model_path = download_or_get_path(model_path, platform=platform)if isinstance(model_path, str):@@ -1129,6 +1134,7 @@ def check_to_quantized(config):bool: True if the configuration is valid for quantization (bits <= 8),False otherwise."""+ from auto_round.schemes import QuantizationSchemeif isinstance(config, (dict, QuantizationScheme)):bits = config.get("bits", None)diff --git a/auto_round/wrapper.py b/auto_round/wrapper.pyindex 9b329624e..93324334b 100644--- a/auto_round/wrapper.py+++ b/auto_round/wrapper.py@@ -744,7 +744,12 @@ class WrapperMultiblock(torch.nn.Module):def wrapper_block(- block, enable_minmax_tuning, enable_norm_bias_tuning, enable_torch_compile=False, device="cpu", **kwargs+ block,+ enable_minmax_tuning,+ enable_norm_bias_tuning,+ enable_torch_compile=False,+ device="cpu",+ **kwargs,):"""Wraps the layers in the given block with a custom Wrapper module.diff --git a/benchmark_both.py b/benchmark_both.pynew file mode 100644index 000000000..320f18b11--- /dev/null+++ b/benchmark_both.py@@ -0,0 +1,83 @@+"""Quick A/B benchmark: old (compressors) vs new (compressors_new) architecture.++Uses AR_DISABLE_NEW_ARCH env-var to toggle. Runs each configuration in a+subprocess to avoid cross-contamination, with a warmup run to fill OS page cache.+"""++import json+import os+import subprocess+import sys+import time++MODEL = "Qwen/Qwen3-0.6B"+ITERS = "200"+SCHEME = "W4A16"+DEVICE = "cuda:0"++CMD_TEMPLATE = [+ sys.executable,+ "-m",+ "auto_round",+ "--model_name",+ MODEL,+ "--scheme",+ SCHEME,+ "--iters",+ ITERS,+ "--device",+ DEVICE,+]+++def run_once(label: str, env_override: dict) -> float:+ env = {**os.environ, **env_override}+ print(f"\n{'='*60}")+ print(f" Running: {label}")+ print(f" AR_DISABLE_NEW_ARCH={env.get('AR_DISABLE_NEW_ARCH', 'unset')}")+ print(f"{'='*60}", flush=True)+ t0 = time.perf_counter()+ proc = subprocess.run(CMD_TEMPLATE, env=env, capture_output=True, text=True)+ elapsed = time.perf_counter() - t0+ if proc.returncode != 0:+ print(f"STDERR:\n{proc.stderr[-2000:]}")+ raise RuntimeError(f"{label} failed with rc={proc.returncode}")+ print(f" {label}: {elapsed:.1f}s")+ return elapsed+++def main():+ # Warmup: fill OS page cache & JIT caches+ print("Warmup run (old arch)...")+ run_once("warmup", {"AR_DISABLE_NEW_ARCH": "1"})++ # Interleaved runs to reduce bias+ results = {"old": [], "new": []}+ for trial in range(2):+ if trial % 2 == 0:+ first, second = ("old", "1"), ("new", "0")+ else:+ first, second = ("new", "0"), ("old", "1")+ for label, flag in [first, second]:+ t = run_once(f"{label} (trial {trial+1})", {"AR_DISABLE_NEW_ARCH": flag})+ results[label].append(t)++ # Summary+ print("\n" + "=" * 60)+ print("SUMMARY")+ print("=" * 60)+ for arch in ["old", "new"]:+ times = results[arch]+ avg = sum(times) / len(times)+ print(f" {arch}: {[f'{t:.1f}' for t in times]} avg={avg:.1f}s")+ old_avg = sum(results["old"]) / len(results["old"])+ new_avg = sum(results["new"]) / len(results["new"])+ diff_pct = (new_avg - old_avg) / old_avg * 100+ print(f"\n Diff: {diff_pct:+.1f}% (new vs old)")+ print(f" {'PASS' if abs(diff_pct) < 5 else 'FAIL'} (threshold: ±5%)")++ json.dump(results, open("benchmark_results/latest.json", "w"), indent=2)+++if __name__ == "__main__":+ main()diff --git a/performance_ut.sh b/performance_ut.shnew file mode 100644index 000000000..12e78906b--- /dev/null+++ b/performance_ut.sh@@ -0,0 +1,115 @@+#!/bin/bash+set -euo pipefail++PATTERN='[-a-zA-Z0-9_]*='++for i in "$@"; do+ case $i in+ --model_name=*)+ model_name=$(echo $i | sed "s/${PATTERN}//")+ ;;+ --scheme=*)+ scheme=$(echo $i | sed "s/${PATTERN}//")+ ;;+ *)+ echo "Parameter $i not recognized."+ exit 1+ ;;+ esac+done++readonly WORKSPACE_DIR="/auto-round"+readonly LOG_DIR="${WORKSPACE_DIR}/log_dir"+readonly PERF_SCRIPT_DIR="${WORKSPACE_DIR}/.azure-pipelines/scripts/performance"+readonly BASELINE_GIT_URL="git+https://github.com/intel/auto-round.git"+readonly ITERS=200++log_group_start() { echo "##[group]$1"; }+log_group_end() { echo "##[endgroup]"; }+log_info() { echo -e "[\033[32mINFO\033[0m] $1"; }+log_err() { echo -e "[\033[31mERROR\033[0m] $1" >&2; }++function setup_environment() {+ log_group_start "Set up environment..."++ export TZ='Asia/Shanghai'+ export TQDM_MININTERVAL=60+ export HF_HUB_DISABLE_PROGRESS_BARS=1+ export UV_NO_PROGRESS=1+ export UV_SYSTEM_PYTHON=1++ log_info "Creating log directory: ${LOG_DIR}"+ mkdir -p "${LOG_DIR}"++ log_info "Downloading model: ${model_name}"+ hf download "${model_name}"++ log_group_end+}++function install_auto_round() {+ local install_source=$1+ local mode_name=$2++ log_group_start "Install requirements for [${mode_name}]..."++ (+ cd "${WORKSPACE_DIR}"+ log_info "Uninstalling existing auto-round..."+ uv pip uninstall auto-round || true++ log_info "Installing auto-round from: ${install_source}"+ BUILD_HPU_ONLY=1 uv pip install "${install_source}"+ )++ log_group_end+}++function run_performance_test() {+ local test_mode=$1+ local log_file="${LOG_DIR}/perf_test_${test_mode}.log"++ log_group_start "Run ${test_mode} performance test (${scheme})..."++ (+ cd "${PERF_SCRIPT_DIR}"+ log_info "Executing auto-round for ${scheme}. Logging to ${log_file}"+ auto-round \+ --model_name "${model_name}" \+ --scheme "${scheme}" \+ --iters "${ITERS}" \+ --enable_torch_compile \+ --device hpu \+ --output_dir "./${test_mode}" 2>&1 | tee -a "${log_file}"+ )++ log_group_end+}++function run_performance_check() {+ log_group_start "Check performance results..."++ (+ cd "${PERF_SCRIPT_DIR}"+ log_info "Executing check_performance.py"+ python check_performance.py+ )++ log_group_end+}++function main() {+ setup_environment++ install_auto_round "." "current"+ run_performance_test "current"++ install_auto_round "${BASELINE_GIT_URL}" "baseline"+ run_performance_test "baseline"++ run_performance_check++ log_info "All tasks completed successfully."+}++main "$@"\ No newline at end of filediff --git a/profile_rss_per_block.py b/profile_rss_per_block.pynew file mode 100644index 000000000..66b13ce59--- /dev/null+++ b/profile_rss_per_block.py@@ -0,0 +1,203 @@+"""Granular per-block RSS profiling for peak RAM regression diagnosis.++Instruments both old and new architecture to measure RSS at key points+within the per-block quantization loop.++Usage:+ # New arch:+ python profile_rss_per_block.py+ # Old arch:+ AR_DISABLE_NEW_ARCH=1 python profile_rss_per_block.py+"""++import gc+import os+import resource+import sys+import time+++def rss_mb():+ """Get current RSS in MB (no gc.collect - raw measurement)."""+ return resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / 1024 # KB -> MB on Linux+++def rss_mb_clean():+ """Get current RSS in MB after gc.collect."""+ gc.collect()+ return resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / 1024+++# Use psutil for live RSS (ru_maxrss is peak, not current)+import psutil++_proc = psutil.Process()+++def live_rss_mb():+ """Current RSS in MB (not peak)."""+ return _proc.memory_info().rss / (1024 * 1024)+++def live_rss_mb_clean():+ gc.collect()+ try:+ import ctypes++ libc = ctypes.CDLL("libc.so.6")+ libc.malloc_trim(0)+ except Exception:+ pass+ return _proc.memory_info().rss / (1024 * 1024)+++arch = os.environ.get("AR_DISABLE_NEW_ARCH", "0")+arch_label = "OLD" if arch == "1" else "NEW"+print(f"\n{'='*70}")+print(f" {arch_label} Architecture - Granular Per-Block RSS Profiling")+print(f"{'='*70}")+print(f"Before import RSS: {live_rss_mb():.1f} MB")++# Monkey-patch to add instrumentation+if arch != "1":+ # NEW ARCH: patch CalibCompressor._quantize_single_block+ from auto_round.compressors_new import calib as calib_mod++ _orig_quantize_single_block = calib_mod.CalibCompressor._quantize_single_block+ _orig_quantize_blocks = calib_mod.CalibCompressor._quantize_blocks++ _block_rss_log = []++ def _patched_quantize_single_block(self, model, m, input_ids, input_others, q_input):+ block_idx = len(_block_rss_log)+ rss_before = live_rss_mb()++ result = _orig_quantize_single_block(self, model, m, input_ids, input_others, q_input)++ rss_after_return = live_rss_mb()+ gc.collect()+ rss_after_gc = live_rss_mb()+ try:+ import ctypes++ libc = ctypes.CDLL("libc.so.6")+ libc.malloc_trim(0)+ except Exception:+ pass+ rss_after_trim = live_rss_mb()++ entry = {+ "block": block_idx,+ "before": rss_before,+ "after_return": rss_after_return,+ "after_gc": rss_after_gc,+ "after_trim": rss_after_trim,+ "delta_return": rss_after_return - rss_before,+ "delta_gc": rss_after_gc - rss_before,+ "delta_trim": rss_after_trim - rss_before,+ }+ _block_rss_log.append(entry)+ print(+ f" Block {block_idx:2d}: before={rss_before:.1f} after_ret={rss_after_return:.1f} "+ f"after_gc={rss_after_gc:.1f} after_trim={rss_after_trim:.1f} "+ f"delta_ret={entry['delta_return']:+.1f} delta_trim={entry['delta_trim']:+.1f} MB",+ flush=True,+ )+ return result++ calib_mod.CalibCompressor._quantize_single_block = _patched_quantize_single_block++else:+ # OLD ARCH: patch LLMCompressor._quantize_block+ from auto_round.compressors import base as base_mod++ _orig_quantize_block = base_mod.LLMCompressor._quantize_block++ _block_rss_log = []++ def _patched_quantize_block(self, block, input_ids, input_others, q_input=None, device="cpu", auto_offload=True):+ block_idx = len(_block_rss_log)+ rss_before = live_rss_mb()++ result = _orig_quantize_block(self, block, input_ids, input_others, q_input, device, auto_offload)++ rss_after_return = live_rss_mb()+ gc.collect()+ rss_after_gc = live_rss_mb()+ try:+ import ctypes++ libc = ctypes.CDLL("libc.so.6")+ libc.malloc_trim(0)+ except Exception:+ pass+ rss_after_trim = live_rss_mb()++ entry = {+ "block": block_idx,+ "before": rss_before,+ "after_return": rss_after_return,+ "after_gc": rss_after_gc,+ "after_trim": rss_after_trim,+ "delta_return": rss_after_return - rss_before,+ "delta_gc": rss_after_gc - rss_before,+ "delta_trim": rss_after_trim - rss_before,+ }+ _block_rss_log.append(entry)+ print(+ f" Block {block_idx:2d}: before={rss_before:.1f} after_ret={rss_after_return:.1f} "+ f"after_gc={rss_after_gc:.1f} after_trim={rss_after_trim:.1f} "+ f"delta_ret={entry['delta_return']:+.1f} delta_trim={entry['delta_trim']:+.1f} MB",+ flush=True,+ )+ return result++ base_mod.LLMCompressor._quantize_block = _patched_quantize_block++print(f"After import RSS: {live_rss_mb():.1f} MB")++from auto_round import AutoRound++print(f"After AutoRound import RSS: {live_rss_mb():.1f} MB")++import shutil++save_dir = "/tmp/profile_rss_output"+shutil.rmtree(save_dir, ignore_errors=True)++print("\nCreating AutoRound instance...")+ar = AutoRound(+ model="Qwen/Qwen3-0.6B",+ scheme="FP8_STATIC",+ iters=200,+ nsamples=128,+ enable_torch_compile=True,+)+print(f"After init RSS: {live_rss_mb():.1f} MB")+print(f"After init RSS (clean): {live_rss_mb_clean():.1f} MB")++print("\nStarting quantize_and_save...\n")+model, folder = ar.quantize_and_save(output_dir=save_dir, format="llm_compressor")++print(f"\n{'='*70}")+print(f" SUMMARY ({arch_label} Architecture)")+print(f"{'='*70}")+print(f"Final RSS: {live_rss_mb():.1f} MB")+print(f"Final RSS (clean): {live_rss_mb_clean():.1f} MB")+print("\nPer-block deltas (after return, after gc+trim):")+for e in _block_rss_log:+ print(+ f" Block {e['block']:2d}: delta_ret={e['delta_return']:+.1f} delta_trim={e['delta_trim']:+.1f} MB "+ f"(abs: {e['after_trim']:.1f} MB)"+ )++# Compute growth rate+if len(_block_rss_log) >= 2:+ first = _block_rss_log[0]["after_trim"]+ last = _block_rss_log[-1]["after_trim"]+ n = len(_block_rss_log) - 1+ print(f"\nGrowth: {first:.1f} -> {last:.1f} MB over {n} blocks = {(last-first)/n:.1f} MB/block avg")++print(f"\nPeak RSS (ru_maxrss): {resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / 1024:.1f} MB")++shutil.rmtree(save_dir, ignore_errors=True)diff --git a/setup.py b/setup.pyindex 1b759fe83..6bb184bcb 100644--- a/setup.py+++ b/setup.py@@ -130,7 +130,6 @@ PKG_INSTALL_CFG = {# python setup.py hpu install###############################################################################-HPU_REQUIREMENTS_FILE = "requirements-hpu.txt"HPU_INSTALL_CFG = {"include_packages": find_packages(@@ -144,7 +143,6 @@ HPU_INSTALL_CFG = {"install_requires": fetch_requirements(HPU_REQUIREMENTS_FILE),}-# Support legacy `python setup.py hpu install` invocation for backward compatibility.# For python -m build / uv build, use the BUILD_HPU_ONLY=1 environment variable instead.if __name__ == "__main__":@@ -186,5 +184,10 @@ if __name__ == "__main__":"License :: OSI Approved :: Apache Software License",],include_package_data=True,- package_data={"": ["mllm/templates/*.json", "experimental/transform/utils/hadamards.safetensors"]},+ package_data={+ "": [+ "mllm/templates/*.json",+ "algorithms/transforms/rotation/utils/hadamards.safetensors",+ ]+ },)diff --git a/test/envs.py b/test/envs.pyindex f998fd4a3..d071b6668 100644--- a/test/envs.py+++ b/test/envs.py@@ -39,6 +39,10 @@ def is_optimum_available():return importlib.util.find_spec("optimum") is not None+def is_compressed_tensors_available():+ return importlib.util.find_spec("compressed_tensors") is not None++def is_ipex_available():try:require_version("intel-extension-for-pytorch>=2.5")@@ -135,6 +139,16 @@ def require_optimum(test_case):return unittest.skipUnless(is_optimum_available(), "test requires optimum")(test_case)+def require_compressed_tensors(test_case):+ """+ Decorator marking a test that requires compressed-tensors.++ These tests are skipped when compressed-tensors isn't installed.++ """+ return unittest.skipUnless(is_compressed_tensors_available(), "test requires compressed-tensors")(test_case)++def require_greater_than_050(test_case):"""Decorator marking a test that requires auto-round>=0.5.0.diff --git a/test/test_ark/test_model.py b/test/test_ark/test_model.pyindex ca0d8b4b6..476dc7ec9 100644--- a/test/test_ark/test_model.py+++ b/test/test_ark/test_model.py@@ -44,14 +44,16 @@ class TestAutoRoundARKBackend:else:autoround = AutoRound(model, tokenizer, bits=bits, group_size=group_size, sym=sym)quantized_model_path = self.save_folder- autoround.quantize_and_save(output_dir=quantized_model_path, format=format) ##will convert to gptq model+ _, saved_folder = autoround.quantize_and_save(+ output_dir=quantized_model_path, format=format+ ) ##will convert to gptq modelquantization_config = AutoRoundConfig(backend="ark")model = AutoModelForCausalLM.from_pretrained(- quantized_model_path, dtype=dtype, device_map=device, quantization_config=quantization_config+ saved_folder, dtype=dtype, device_map=device, quantization_config=quantization_config)- tokenizer = AutoTokenizer.from_pretrained(self.save_folder)+ tokenizer = AutoTokenizer.from_pretrained(saved_folder)model_infer(model, tokenizer)evaluate_accuracy(model, tokenizer, threshold=tar_acc, batch_size=32, limit=limit)torch.xpu.empty_cache()diff --git a/test/test_cpu/advanced/test_low_precision_input_model.py b/test/test_cpu/advanced/test_low_precision_input_model.pyindex 9b0779954..9087e0de0 100644--- a/test/test_cpu/advanced/test_low_precision_input_model.py+++ b/test/test_cpu/advanced/test_low_precision_input_model.py@@ -97,6 +97,6 @@ class TestCompressedTensor:iters=2,nsamples=2,)- ar.quantize_and_save(tmp_path, format="llm_compressor")- model = transformers.AutoModelForCausalLM.from_pretrained(tmp_path)+ _, quantized_model_path = ar.quantize_and_save(tmp_path, format="llm_compressor")+ model = transformers.AutoModelForCausalLM.from_pretrained(quantized_model_path)assert model, "Failed to load the quantized model"diff --git a/test/test_cpu/backends/test_torch_backend.py b/test/test_cpu/backends/test_torch_backend.pyindex 14951f2e6..7e45c69c4 100644--- a/test/test_cpu/backends/test_torch_backend.py+++ b/test/test_cpu/backends/test_torch_backend.py@@ -41,23 +41,25 @@ class TestAutoRoundTorchBackend:dataset=dataloader,)quantized_model_path = self.save_folder- autoround.quantize_and_save(output_dir=quantized_model_path, format="auto_round:gptqmodel")+ _, quantized_model_path = autoround.quantize_and_save(+ output_dir=quantized_model_path, format="auto_round:gptqmodel"+ )quantization_config = AutoRoundConfig(backend="torch")model = AutoModelForCausalLM.from_pretrained(quantized_model_path, dtype=torch.float16, device_map="cpu", quantization_config=quantization_config)- tokenizer = AutoTokenizer.from_pretrained(self.save_folder)+ tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)model_infer(model, tokenizer)evaluate_accuracy(model, tokenizer, threshold=0.35, batch_size=16, limit=10)torch.cuda.empty_cache()model = AutoModelForCausalLM.from_pretrained(- self.save_folder, dtype=torch.bfloat16, device_map="cpu", quantization_config=quantization_config+ quantized_model_path, dtype=torch.bfloat16, device_map="cpu", quantization_config=quantization_config)- tokenizer = AutoTokenizer.from_pretrained(self.save_folder)+ tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)model_infer(model, tokenizer)evaluate_accuracy(model, tokenizer, threshold=0.35, batch_size=16, limit=10)torch.cuda.empty_cache()@@ -77,14 +79,16 @@ class TestAutoRoundTorchBackend:dataset=dataloader,)quantized_model_path = self.save_folder- autoround.quantize_and_save(output_dir=quantized_model_path, format="auto_round") ##will convert to gptq model+ _, quantized_model_path = autoround.quantize_and_save(+ output_dir=quantized_model_path, format="auto_round"+ ) ##will convert to gptq modelquantization_config = AutoRoundConfig(backend="torch")model = AutoModelForCausalLM.from_pretrained(quantized_model_path, dtype=torch.float16, device_map="auto", quantization_config=quantization_config)- tokenizer = AutoTokenizer.from_pretrained(self.save_folder)+ tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)model_infer(model, tokenizer)evaluate_accuracy(model, tokenizer, threshold=0.28, batch_size=32, limit=1000)torch.cuda.empty_cache()diff --git a/test/test_cpu/core/test_autoround.py b/test/test_cpu/core/test_autoround.pyindex dacd7bf4a..6ed626cd8 100644--- a/test/test_cpu/core/test_autoround.py+++ b/test/test_cpu/core/test_autoround.py@@ -20,6 +20,7 @@ from ...helpers import (class TestAutoRound:+@classmethoddef setup_class(self):model_name = opt_name_or_path@@ -372,14 +373,14 @@ class TestAutoRound:bits, group_size, sym = 4, 128, Trueautoround = AutoRound(model, tokenizer, bits=bits, group_size=group_size, sym=sym, iters=0, nsamples=1)quantized_model_path = self.save_folder- autoround.quantize_and_save(output_dir=quantized_model_path, format="auto_round")+ _, quantized_model_path = autoround.quantize_and_save(output_dir=quantized_model_path, format="auto_round")model = AutoModelForCausalLM.from_pretrained(- self.save_folder,+ quantized_model_path,torch_dtype=torch.float16,device_map="auto",)- tokenizer = AutoTokenizer.from_pretrained(self.save_folder)+ tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)model_infer(model, tokenizer)def test_embed_quant(self, tiny_opt_model_path, dataloader):@@ -426,7 +427,9 @@ class TestAutoRound:autoround.quantize()quantized_model_path = self.save_folder- autoround.save_quantized(output_dir=quantized_model_path, format="auto_round", inplace=True)+ _, quantized_model_path = autoround.save_quantized(+ output_dir=quantized_model_path, format="auto_round", inplace=True, return_folders=True+ )model = AutoModelForCausalLM.from_pretrained(quantized_model_path, device_map="cpu")tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)@@ -457,7 +460,9 @@ class TestAutoRound:autoround.quantize()quantized_model_path = self.save_folder- autoround.save_quantized(output_dir=quantized_model_path, format="auto_awq", inplace=True)+ _, quantized_model_path = autoround.save_quantized(+ output_dir=quantized_model_path, format="auto_awq", inplace=True, return_folders=True+ )quantization_config = AutoRoundConfig()model = AutoModelForCausalLM.from_pretrained(@@ -492,7 +497,9 @@ class TestAutoRound:autoround.quantize()quantized_model_path = self.save_folder- autoround.save_quantized(output_dir=quantized_model_path, format="auto_gptq", inplace=True)+ _, quantized_model_path = autoround.save_quantized(+ output_dir=quantized_model_path, format="auto_round", inplace=True, return_folders=True+ )quantization_config = AutoRoundConfig()model = AutoModelForCausalLM.from_pretrained(@@ -527,7 +534,9 @@ class TestAutoRound:autoround.quantize()quantized_model_path = self.save_folder- autoround.save_quantized(output_dir=quantized_model_path, format="auto_round", inplace=True)+ _, quantized_model_path = autoround.save_quantized(+ output_dir=quantized_model_path, format="auto_round", inplace=True, return_folders=True+ )quantization_config = AutoRoundConfig()model = AutoModelForCausalLM.from_pretrained(@@ -636,8 +645,8 @@ class TestAutoRound:nsamples=1,disable_opt_rtn=True,)- ar.quantize_and_save(output_dir=self.save_folder, format="auto_round")- model = AutoModelForCausalLM.from_pretrained(self.save_folder, device_map="cpu")+ _, quantized_model_path = ar.quantize_and_save(output_dir=self.save_folder, format="auto_round")+ model = AutoModelForCausalLM.from_pretrained(quantized_model_path, device_map="cpu")assert "lm_head" in model.config.quantization_config.extra_configassert model.config.quantization_config.extra_config["lm_head"]["bits"] == 4@@ -653,8 +662,8 @@ class TestAutoRound:disable_opt_rtn=True,layer_config=layer_config,)- ar.quantize_and_save(output_dir=self.save_folder, format="auto_round")- model = AutoModelForCausalLM.from_pretrained(self.save_folder, device_map="cpu")+ _, quantized_model_path = ar.quantize_and_save(output_dir=self.save_folder, format="auto_round")+ model = AutoModelForCausalLM.from_pretrained(quantized_model_path, device_map="cpu")assert "lm_head" in model.config.quantization_config.extra_configassert model.config.quantization_config.extra_config["lm_head"]["bits"] == 4@@ -664,12 +673,6 @@ class TestAutoRound:assert ar.optimizer == torch.optim.AdamWassert ar.mllm- # test old api- from auto_round import AutoRoundMLLM-- ar = AutoRoundMLLM(model_name)- assert ar.mllm-def test_attention_mask_in_dataset(self):from transformers import AutoTokenizerdiff --git a/test/test_cpu/export/test_export.py b/test/test_cpu/export/test_export.pyindex 0f4ec3ce9..4f4c0d177 100644--- a/test/test_cpu/export/test_export.py+++ b/test/test_cpu/export/test_export.py@@ -28,6 +28,7 @@ def _get_folder_size(path: str) -> float:class TestAutoRound:+@classmethoddef setup_class(self):self.model_name = opt_name_or_path@@ -59,7 +60,7 @@ class TestAutoRound:)quantized_model_path = self.save_dir- autoround.quantize_and_save(output_dir=quantized_model_path, format="auto_gptq")+ _, quantized_model_path = autoround.quantize_and_save(output_dir=quantized_model_path, format="auto_gptq")if group_size == -1:continue@@ -86,7 +87,7 @@ class TestAutoRound:dataset=dataloader,)quantized_model_path = self.save_dir- autoround.quantize_and_save(output_dir=quantized_model_path, format="auto_round")+ _, quantized_model_path = autoround.quantize_and_save(output_dir=quantized_model_path, format="auto_round")if group_size == -1:continue@@ -111,7 +112,9 @@ class TestAutoRound:)quantized_model_path = self.save_dir- autoround.quantize_and_save(output_dir=quantized_model_path, format="auto_round:auto_awq")+ _, quantized_model_path = autoround.quantize_and_save(+ output_dir=quantized_model_path, format="auto_round:auto_awq"+ )# quantization_config = AutoRoundConfig(# backend="cpu"@@ -220,7 +223,7 @@ class TestAutoRound:static_kv_dtype=static_kv_dtype,)quantized_model_path = self.save_dir- autoround.quantize_and_save(output_dir=quantized_model_path, format="auto_round")+ _, quantized_model_path = autoround.quantize_and_save(output_dir=quantized_model_path, format="auto_round")f = safe_open(os.path.join(quantized_model_path, "model.safetensors"), framework="pt")assert "model.decoder.layers.8.self_attn.k_proj.input_scale" in f.keys()assert "model.decoder.layers.8.self_attn.k_proj.weight_scale" in f.keys()@@ -280,7 +283,7 @@ class TestAutoRound:act_group_size=0,)quantized_model_path = self.save_dir- autoround.quantize_and_save(output_dir=quantized_model_path, format="auto_round")+ _, quantized_model_path = autoround.quantize_and_save(output_dir=quantized_model_path, format="auto_round")f = safe_open(os.path.join(quantized_model_path, "model.safetensors"), framework="pt")assert "model.decoder.layers.8.self_attn.k_proj.input_scale" in f.keys()@@ -305,7 +308,7 @@ class TestAutoRound:static_attention_dtype="fp8",)quantized_model_path = self.save_dir- autoround.quantize_and_save(output_dir=quantized_model_path, format="auto_round")+ _, quantized_model_path = autoround.quantize_and_save(output_dir=quantized_model_path, format="auto_round")f = safe_open(os.path.join(quantized_model_path, "model.safetensors"), framework="pt")assert "model.decoder.layers.8.self_attn.k_proj.input_scale" in f.keys()assert "model.decoder.layers.8.self_attn.k_proj.weight_scale" in f.keys()@@ -363,7 +366,9 @@ class TestAutoRound:dataset=dataloader,)quantized_model_path = self.save_dir- compressed_model, _ = autoround.quantize_and_save(output_dir=quantized_model_path, format="auto_gptq")+ compressed_model, quantized_model_path = autoround.quantize_and_save(+ output_dir=quantized_model_path, format="auto_gptq"+ )lm_head = compressed_model.lm_headassert hasattr(lm_head, "bits") and lm_head.bits == 4, "Illegal GPTQ quantization for lm_head layer"quantization_config = AutoRoundConfig()@@ -383,6 +388,7 @@ class TestAutoRound:self.model_name,scheme="FP8_STATIC",)+ autoround.post_init()format_list = get_formats("auto_round, llm_compressor, auto_round:llm_compressor", autoround)assert len(format_list) == 3assert format_list[0].output_format == "auto_round"@@ -396,6 +402,7 @@ class TestAutoRound:self.model_name,scheme="W4A16",)+ autoround.post_init()format_list = get_formats("auto_round:auto_awq, auto_gptq", autoround)assert format_list[0].output_format == "auto_round"assert format_list[0].get_backend_name() == "auto_round:auto_awq"@@ -406,6 +413,7 @@ class TestAutoRound:model=self.model_name,scheme="INT8",)+ autoround.post_init()format_list = get_formats("llm_compressor, auto_round:llm_compressor", autoround)assert format_list[0].output_format == "llm_compressor"assert format_list[0].get_backend_name() == "llm_compressor:int8_w8a8"@@ -417,6 +425,7 @@ class TestAutoRound:model=self.model_name,scheme="INT8_W8A8",)+ autoround_old.post_init()format_list_old = get_formats("llm_compressor, auto_round:llm_compressor", autoround_old)assert format_list_old[0].output_format == "llm_compressor"assert format_list_old[0].get_backend_name() == "llm_compressor:int8_w8a8"@@ -433,6 +442,7 @@ class TestAutoRound:group_size=32,sym=True,)+ ar.post_init()with pytest.raises(ValueError, match="auto_awq format support quantization scheme with W4A16 but got bits=2"):get_formats("auto_round:auto_awq", ar)@@ -446,6 +456,7 @@ class TestAutoRound:group_size=32,sym=True,)+ ar.post_init()with pytest.raises(ValueError, match="but got data_type=fp, bits=4"):get_formats("auto_round:llm_compressor", ar)@@ -456,6 +467,7 @@ class TestAutoRound:group_size=256,sym=True,)+ ar.post_init()get_formats("auto_round:auto_awq", ar)def test_autoawq_qwen3_vl_infer(self, dataloader):@@ -469,7 +481,9 @@ class TestAutoRound:dataset=dataloader,)quantized_model_path = self.save_dir- autoround.quantize_and_save(output_dir=quantized_model_path, inplace=False, format="auto_awq")+ _, quantized_model_path = autoround.quantize_and_save(+ output_dir=quantized_model_path, inplace=False, format="auto_awq"+ )# Check items of modules_to_not_convert in quantization configquantization_config_path = f"{quantized_model_path}/quantization_config.json"@@ -508,7 +522,7 @@ class TestAutoRound:scheme=scheme,)quantized_model_path = self.save_dir- autoround.quantize_and_save(output_dir=quantized_model_path, format="llm_compressor")+ _, quantized_model_path = autoround.quantize_and_save(output_dir=quantized_model_path, format="llm_compressor")with safe_open(os.path.join(quantized_model_path, "model.safetensors"), framework="pt") as f:assert "model.decoder.layers.8.self_attn.k_proj.weight_scale" in f.keys()assert f.get_tensor("model.decoder.layers.5.self_attn.v_proj.weight").dtype == torch.int8@@ -536,7 +550,7 @@ class TestAutoRound:sym=sym,)quantized_model_path = self.save_dir- autoround.quantize_and_save(output_dir=quantized_model_path, format="llm_compressor")+ _, quantized_model_path = autoround.quantize_and_save(output_dir=quantized_model_path, format="llm_compressor")with safe_open(os.path.join(quantized_model_path, "model.safetensors"), framework="pt") as f:# weights must be packed as int32 (compressed-tensors stores both int4 and int8 as torch.int32)weight = f.get_tensor("model.decoder.layers.5.self_attn.v_proj.weight_packed")diff --git a/test/test_cpu/export/test_gguf_format.py b/test/test_cpu/export/test_gguf_format.pyindex 945c28db0..9666bb286 100644--- a/test/test_cpu/export/test_gguf_format.py+++ b/test/test_cpu/export/test_gguf_format.py@@ -46,7 +46,9 @@ class TestGGUF:)quantized_model_path = self.save_dir- autoround.quantize_and_save(output_dir=quantized_model_path, inplace=False, format="gguf:q4_0")+ _, quantized_model_path = autoround.quantize_and_save(+ output_dir=quantized_model_path, inplace=False, format="gguf:q4_0"+ )gguf_file = os.listdir(quantized_model_path)[0]assert gguf_file.endswith(".gguf"), "Saved file is not in gguf format"# Accuracy test is covered in test_cuda/export/test_gguf_format.py::TestAutoRound::test_q4_0_accuracy@@ -59,10 +61,12 @@ class TestGGUF:disable_opt_rtn=True,)quantized_model_path = self.save_dir- autoround.quantize_and_save(output_dir=quantized_model_path, inplace=False, format="gguf:q*_1")+ _, quantized_model_path = autoround.quantize_and_save(+ output_dir=quantized_model_path, inplace=False, format="gguf:q*_1"+ )assert autoround.group_size == 32assert not autoround.sym- gguf_file = os.listdir(self.save_dir)[0]+ gguf_file = os.listdir(quantized_model_path)[0]model = AutoModelForCausalLM.from_pretrained(quantized_model_path, gguf_file=gguf_file, device_map="auto")eval_generated_prompt(model, self.tokenizer)@@ -91,7 +95,9 @@ class TestGGUF:disable_opt_rtn=True,)quantized_model_path = self.save_dir- autoround.quantize_and_save(output_dir=quantized_model_path, format="gguf:q4_k_m,fake")+ _, quantized_model_path = autoround.quantize_and_save(+ output_dir=quantized_model_path, format="gguf:q4_k_m,fake"+ )assert autoround.layer_config["model.layers.1.self_attn.v_proj"]["super_group_size"] == 16assert autoround.layer_config["model.layers.1.self_attn.v_proj"]["data_type"] == "int_sym_dq"assert autoround.layer_config["model.layers.0.self_attn.v_proj"]["data_type"] == "int_asym_dq"@@ -134,9 +140,9 @@ class TestGGUF:shutil.rmtree("../../tmp_autoround", ignore_errors=True)def test_vlm_gguf(self, tiny_qwen_vl_model_path):- from auto_round import AutoRoundMLLM+ from auto_round import AutoRound- autoround = AutoRoundMLLM(+ autoround = AutoRound(tiny_qwen_vl_model_path,iters=0,nsamples=8,@@ -144,8 +150,8 @@ class TestGGUF:quant_nontext_module=True,)quantized_model_path = self.save_dir- autoround.quantize_and_save(output_dir=quantized_model_path, format="gguf:q4_0")- assert "mmproj-model.gguf" in os.listdir(self.save_dir)+ _, quantized_model_path = autoround.quantize_and_save(output_dir=quantized_model_path, format="gguf:q4_0")+ assert "mmproj-model.gguf" in os.listdir(quantized_model_path)for file_name in os.listdir(quantized_model_path):file_size = os.path.getsize(os.path.join(quantized_model_path, file_name)) / 1024**2if file_name == "mmproj-model.gguf":@@ -154,9 +160,9 @@ class TestGGUF:assert file_size < 270, f"file size {file_size} MB is too large for non-quantized mmproj-model.gguf"def test_vlm_gguf_wo_quant_nontext_module(self, tiny_qwen_vl_model_path):- from auto_round import AutoRoundMLLM+ from auto_round import AutoRound- autoround = AutoRoundMLLM(+ autoround = AutoRound(tiny_qwen_vl_model_path,iters=0,nsamples=8,@@ -164,8 +170,8 @@ class TestGGUF:quant_nontext_module=False,)quantized_model_path = self.save_dir- autoround.quantize_and_save(output_dir=quantized_model_path, format="gguf:q4_0")- assert "mmproj-model.gguf" in os.listdir(self.save_dir)+ _, quantized_model_path = autoround.quantize_and_save(output_dir=quantized_model_path, format="gguf:q4_0")+ assert "mmproj-model.gguf" in os.listdir(quantized_model_path)for file_name in os.listdir(quantized_model_path):file_size = os.path.getsize(os.path.join(quantized_model_path, file_name)) / 1024**2if file_name == "mmproj-model.gguf":@@ -254,7 +260,7 @@ class TestGGUF:disable_opt_rtn=True,)quantized_model_path = self.save_dir- autoround.quantize_and_save(output_dir=quantized_model_path, format="gguf:q2_k_mixed")+ _, quantized_model_path = autoround.quantize_and_save(output_dir=quantized_model_path, format="gguf:q2_k_mixed")gguf_file = os.listdir(quantized_model_path)[0]file_size = os.path.getsize(os.path.join(quantized_model_path, gguf_file)) / 1024**2assert file_size < 1150, f"file size {file_size} MB is too large for q2_k_mixed format"diff --git a/test/test_cpu/export/test_llmc_format.py b/test/test_cpu/export/test_llmc_format.pyindex 2a6a0a5ec..c00e5e6df 100644--- a/test/test_cpu/export/test_llmc_format.py+++ b/test/test_cpu/export/test_llmc_format.py@@ -10,10 +10,14 @@ from auto_round import AutoRound, AutoSchemefrom auto_round.export.export_to_llmcompressor import export_to_fp as llmc_fp_exportfrom auto_round.export.export_to_llmcompressor import export_to_static_fp as llmc_static_fp_export+from ...envs import is_compressed_tensors_availablefrom ...helpers import forbid_threaded_packing, get_model_path, opt_name_or_path+pytestmark = pytest.mark.skipif(not is_compressed_tensors_available(), reason="test requires compressed-tensors")+class TestLLMC:+@classmethoddef setup_class(self):self.model_name = get_model_path("stas/tiny-random-llama-2")@@ -52,7 +56,7 @@ class TestLLMC:nsamples=2,iters=0,)- autoround.quantize_and_save(tmp_path, format="llm_compressor")+ _, quantized_model_path = autoround.quantize_and_save(tmp_path, format="llm_compressor")# from vllm import LLM# model = LLM(tmp_path)# result = model.generate("Hello my name is")@@ -62,13 +66,13 @@ class TestLLMC:from safetensors import safe_open- config = json.load(open(os.path.join(tmp_path, "config.json")))+ config = json.load(open(os.path.join(quantized_model_path, "config.json")))assert "group_0" in config["quantization_config"]["config_groups"]assert config["quantization_config"]["config_groups"]["group_0"]["input_activations"]["num_bits"] == 8assert config["quantization_config"]["config_groups"]["group_0"]["weights"]["strategy"] == "channel"assert config["quantization_config"]["quant_method"] == "compressed-tensors"- f = safe_open(os.path.join(tmp_path, "model.safetensors"), framework="pt")+ f = safe_open(os.path.join(quantized_model_path, "model.safetensors"), framework="pt")assert len(f.get_tensor("model.decoder.layers.0.fc1.weight_scale").shape) == 2def test_autoround_llmcompressor_fp8(self, tmp_path):@@ -82,11 +86,11 @@ class TestLLMC:nsamples=2,iters=0,)- autoround.quantize_and_save(tmp_path, format="auto_round:llm_compressor")+ _, quantized_model_path = autoround.quantize_and_save(tmp_path, format="auto_round:llm_compressor")import json- config = json.load(open(os.path.join(tmp_path, "config.json")))+ config = json.load(open(os.path.join(quantized_model_path, "config.json")))assert "group_0" in config["quantization_config"]["config_groups"]assert config["quantization_config"]["config_groups"]["group_0"]["input_activations"]["num_bits"] == 8assert config["quantization_config"]["config_groups"]["group_0"]["weights"]["strategy"] == "tensor"@@ -101,7 +105,7 @@ class TestLLMC:disable_opt_rtn=True,scheme=scheme,)- compressed_model, _ = ar.quantize_and_save(output_dir=tmp_path, format="llm_compressor")+ compressed_model, tmp_path = ar.quantize_and_save(output_dir=tmp_path, format="llm_compressor")tmp_layer = compressed_model.model.decoder.layers[1].self_attn.q_projassert (hasattr(tmp_layer, "weight_scale")@@ -135,7 +139,7 @@ class TestLLMC:disable_opt_rtn=True,scheme=scheme,)- ar.quantize_and_save(output_dir=tmp_path, format="llm_compressor")+ _, tmp_path = ar.quantize_and_save(output_dir=tmp_path, format="llm_compressor")model = AutoModelForCausalLM.from_pretrained(tmp_path, torch_dtype="auto", trust_remote_code=True)op = model.model.decoder.layers[0].fc1if op.quantization_scheme.targets != ["Linear"]:diff --git a/test/test_cpu/models/test_conv1d.py b/test/test_cpu/models/test_conv1d.pyindex 8a30b8207..717178e73 100644--- a/test/test_cpu/models/test_conv1d.py+++ b/test/test_cpu/models/test_conv1d.py@@ -39,7 +39,7 @@ class TestQuantizationConv1d:)autoround.quantize()- autoround.save_quantized(self.save_dir)+ _, quantized_model_path = autoround.save_quantized(self.save_dir, return_folders=True)- model = AutoModelForCausalLM.from_pretrained(self.save_dir, device_map="cpu", trust_remote_code=True)+ model = AutoModelForCausalLM.from_pretrained(quantized_model_path, device_map="cpu", trust_remote_code=True)model_infer(model, self.tokenizer)diff --git a/test/test_cpu/models/test_mllm.py b/test/test_cpu/models/test_mllm.pyindex 9fd6c69df..2c0c71bd4 100644--- a/test/test_cpu/models/test_mllm.py+++ b/test/test_cpu/models/test_mllm.py@@ -4,13 +4,14 @@ import shutilimport pytestfrom transformers import AutoModelForImageTextToText, AutoProcessor, AutoTokenizer, Qwen2VLForConditionalGeneration-from auto_round import AutoRoundMLLM+from auto_round import AutoRoundfrom auto_round.utils import get_block_namesfrom ...helpers import get_model_path, opt_name_or_pathclass FakeDataLoader:+def __init__(self):self.batch_size = 1@@ -27,7 +28,8 @@ class FakeDataLoader:yield self.data-class TestAutoRoundMLLM:+class TestAutoRound:+@classmethoddef setup_class(self):self.model_name = get_model_path("Qwen/Qwen2-VL-2B-Instruct")@@ -43,7 +45,7 @@ class TestAutoRoundMLLM:def test_tune(self, tiny_qwen_vl_model_path):bits, group_size = 4, 128- autoround = AutoRoundMLLM(+ autoround = AutoRound(model=tiny_qwen_vl_model_path,bits=bits,group_size=group_size,@@ -64,7 +66,7 @@ class TestAutoRoundMLLM:tiny_qwen_vl_model_path, trust_remote_code=True, device_map="auto")bits, group_size = 4, 128- autoround = AutoRoundMLLM(+ autoround = AutoRound(model,tokenizer,processor=processor,@@ -120,7 +122,7 @@ class TestAutoRoundMLLM:)bits, group_size = 4, 128dataset = ["dataset test", "list test"]- autoround = AutoRoundMLLM(+ autoround = AutoRound(model,tokenizer,processor=processor,@@ -154,7 +156,7 @@ class TestAutoRoundMLLM:)bits, group_size = 4, 128dataset = ["test pure text", "input for mllm"]- autoround = AutoRoundMLLM(+ autoround = AutoRound(model,tokenizer,processor=processor,@@ -215,7 +217,7 @@ class TestAutoRoundMLLM:model_name = tiny_qwen_2_5_vl_model_pathmodel, processor, tokenizer, image_processor = mllm_load_model(model_name)- autoround = AutoRoundMLLM(+ autoround = AutoRound(model,tokenizer,iters=1,@@ -225,15 +227,17 @@ class TestAutoRoundMLLM:processor=processor,image_processor=image_processor,)- autoround.quantize_and_save(self.save_dir, format="auto_round")+ _, quantized_model_path = autoround.quantize_and_save(self.save_dir, format="auto_round")import requestsfrom PIL import Imagefrom transformers import AutoProcessor, AutoTokenizer, Qwen2_5_VLForConditionalGeneration- model = Qwen2_5_VLForConditionalGeneration.from_pretrained(self.save_dir, torch_dtype="auto", device_map="auto")+ model = Qwen2_5_VLForConditionalGeneration.from_pretrained(+ quantized_model_path, torch_dtype="auto", device_map="auto"+ )image_url = "https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen-VL/assets/demo.jpeg"- processor = AutoProcessor.from_pretrained(self.save_dir)+ processor = AutoProcessor.from_pretrained(quantized_model_path)messages = [{"role": "user",@@ -269,7 +273,7 @@ class TestAutoRoundMLLM:model_name = tiny_qwen_2_5_vl_model_pathmodel, processor, tokenizer, image_processor = mllm_load_model(model_name)- autoround = AutoRoundMLLM(+ autoround = AutoRound(model,tokenizer,iters=1,diff --git a/test/test_cpu/models/test_moe_model.py b/test/test_cpu/models/test_moe_model.pyindex ea2e5a6ed..9c25b1d81 100644--- a/test/test_cpu/models/test_moe_model.py+++ b/test/test_cpu/models/test_moe_model.py@@ -24,7 +24,7 @@ def quantize_model(model, output_dir, scheme, iters=0, ignore_layers="self_attn,disable_opt_rtn=disable_opt_rtn,)quantized_model, save_folder = autoround.quantize_and_save(format="auto_round", output_dir=output_dir)- return quantized_model+ return quantized_model, save_folderdef count_modules_by_type(model, target_module_name_or_class):@@ -44,7 +44,9 @@ def count_modules_by_type(model, target_module_name_or_class):def test_gptoss(scheme, tiny_gpt_oss_model_path, tmp_path):config = AutoConfig.from_pretrained(tiny_gpt_oss_model_path, trust_remote_code=True)output_dir = str(tmp_path / "saved")- quantized_model = quantize_model(tiny_gpt_oss_model_path, output_dir, scheme, ignore_layers="self_attn,lm_head")+ quantized_model, save_folder = quantize_model(+ tiny_gpt_oss_model_path, output_dir, scheme, ignore_layers="self_attn,lm_head"+ )# Ensure the quantized model is not Noneassert quantized_model is not None, "Quantized model should not be None."@@ -63,7 +65,7 @@ def test_gptoss(scheme, tiny_gpt_oss_model_path, tmp_path):), f"Expected {config.num_hidden_layers * 3 * config.num_local_experts} QuantLinear modules, found {quant_linear_cnt}."# verify the quantized model can be loaded and run inference- loaded_model = GptOssForCausalLM.from_pretrained(output_dir)+ loaded_model = GptOssForCausalLM.from_pretrained(save_folder)inp = torch.randint(0, 100, (1, 32))with torch.inference_mode():@@ -72,12 +74,14 @@ def test_gptoss(scheme, tiny_gpt_oss_model_path, tmp_path):def test_llama4(tiny_llama4_model_path):output_dir = "./tmp/test_quantized_llama4"- quantized_model = quantize_model(tiny_llama4_model_path, output_dir, "MXFP4", ignore_layers="self_attn,lm_head")+ quantized_model, save_folder = quantize_model(+ tiny_llama4_model_path, output_dir, "MXFP4", ignore_layers="self_attn,lm_head"+ )# Ensure the quantized model is not Noneassert quantized_model is not None, "Quantized model should not be None."- loaded_model = Llama4ForConditionalGeneration.from_pretrained(output_dir)+ loaded_model = Llama4ForConditionalGeneration.from_pretrained(save_folder)inp = torch.randint(0, 100, (1, 32))with torch.inference_mode():@@ -97,9 +101,9 @@ def test_qwen3_vl_moe_mxfp(tiny_qwen3_vl_moe_model_path):disable_opt_rtn=True,ignore_layers="self_attn,lm_head, mlp.gate",)- quantized_model, _ = autoround.quantize_and_save(format="auto_round", output_dir=output_dir)+ quantized_model, quantized_model_path = autoround.quantize_and_save(format="auto_round", output_dir=output_dir)assert quantized_model is not None, "Quantized model should not be None."- loaded_model = Qwen3VLMoeForConditionalGeneration.from_pretrained(output_dir, device_map="cpu")+ loaded_model = Qwen3VLMoeForConditionalGeneration.from_pretrained(quantized_model_path, device_map="cpu")inp = torch.randint(0, 100, (1, 32))with torch.inference_mode():diff --git a/test/test_cpu/models/test_omni_model.py b/test/test_cpu/models/test_omni_model.pyindex ce3f43023..3c5d82548 100644--- a/test/test_cpu/models/test_omni_model.py+++ b/test/test_cpu/models/test_omni_model.py@@ -11,7 +11,6 @@# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.# See the License for the specific language governing permissions and# limitations under the License.-"""Unit tests for Qwen2.5-Omni and Qwen3-Omni-MoE model support.Tests cover:@@ -282,6 +281,8 @@ class TestQwen3OmniMoeReplacement:intermediate = 32 # moe_intermediate_size# Verify thinker expert weights+ # Use equal_nan=True because fused expert parameters may contain+ # uninitialized memory with NaN bit patterns, and NaN != NaN in IEEE 754.for i in range(4):expert = model.thinker.model.layers[0].mlp.experts[i]assert_same_weights(expert.gate_proj.weight.data, thinker_gate_up[i, :intermediate, :])diff --git a/test/test_cpu/quantization/test_act_quantization.py b/test/test_cpu/quantization/test_act_quantization.pyindex 4798e0fe4..6546efd62 100644--- a/test/test_cpu/quantization/test_act_quantization.py+++ b/test/test_cpu/quantization/test_act_quantization.py@@ -108,7 +108,7 @@ class TestAutoRoundAct:layer_config=layer_config,)quantized_model_path = self.save_dir- autoround.quantize_and_save(output_dir=quantized_model_path, format="auto_round")+ _, quantized_model_path = autoround.quantize_and_save(output_dir=quantized_model_path, format="auto_round")model = AutoModelForCausalLM.from_pretrained(quantized_model_path, device_map="cpu")assert "lm_head" not in model.config.quantization_config.extra_config@@ -129,7 +129,7 @@ class TestAutoRoundAct:layer_config=layer_config,)quantized_model_path = self.save_dir- autoround.quantize_and_save(output_dir=quantized_model_path, format="auto_round")+ _, quantized_model_path = autoround.quantize_and_save(output_dir=quantized_model_path, format="auto_round")model = AutoModelForCausalLM.from_pretrained(quantized_model_path, device_map="cpu")kproj_config = model.config.quantization_config.extra_config["model.decoder.layers.1.self_attn.k_proj"]assert "act_bits" in kproj_config.keys() and kproj_config["act_bits"] == 16@@ -148,7 +148,7 @@ class TestAutoRoundAct:layer_config=layer_config,)quantized_model_path = self.save_dir- autoround.quantize_and_save(output_dir=quantized_model_path, format="auto_round")+ _, quantized_model_path = autoround.quantize_and_save(output_dir=quantized_model_path, format="auto_round")model = AutoModelForCausalLM.from_pretrained(quantized_model_path, device_map="cpu")extra_config = model.config.quantization_config.extra_config@@ -178,7 +178,7 @@ class TestAutoRoundAct:layer_config=layer_config,)quantized_model_path = self.save_dir- autoround.quantize_and_save(output_dir=quantized_model_path, format="auto_round")+ _, quantized_model_path = autoround.quantize_and_save(output_dir=quantized_model_path, format="auto_round")from transformers import AutoConfigextra_config = AutoConfig.from_pretrained(quantized_model_path).quantization_config["extra_config"]diff --git a/test/test_cpu/quantization/test_asym.py b/test/test_cpu/quantization/test_asym.pyindex 05f9c1990..0f792a3a1 100644--- a/test/test_cpu/quantization/test_asym.py+++ b/test/test_cpu/quantization/test_asym.py@@ -28,15 +28,15 @@ class TestAutoRoundAsym:ar = AutoRound(tiny_opt_model_path, bits=bits, group_size=group_size, sym=sym, iters=0, seqlen=2, nsamples=1)- ar.quantize_and_save(format="auto_round", output_dir=self.save_folder)+ _, quantized_model_path = ar.quantize_and_save(format="auto_round", output_dir=self.save_folder)model = AutoModelForCausalLM.from_pretrained(- self.save_folder,+ quantized_model_path,torch_dtype="auto",device_map="auto",)- tokenizer = AutoTokenizer.from_pretrained(self.save_folder)+ tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)model_infer(model, tokenizer)def test_asym_bits(self, tiny_opt_model_path):@@ -45,15 +45,15 @@ class TestAutoRoundAsym:ar = AutoRound(tiny_opt_model_path, bits=bits, group_size=group_size, sym=sym, iters=0, seqlen=2, nsamples=1)- ar.quantize_and_save(format="auto_round", output_dir=self.save_folder)+ _, quantized_model_path = ar.quantize_and_save(format="auto_round", output_dir=self.save_folder)model = AutoModelForCausalLM.from_pretrained(- self.save_folder,+ quantized_model_path,torch_dtype="auto",device_map="auto",)- tokenizer = AutoTokenizer.from_pretrained(self.save_folder)+ tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)model_infer(model, tokenizer)# use parameters later@@ -70,13 +70,13 @@ class TestAutoRoundAsym:nsamples=1,disable_opt_rtn=True,)- ar.quantize_and_save(format=format, output_dir=self.save_folder)+ _, quantized_model_path = ar.quantize_and_save(format=format, output_dir=self.save_folder)model = AutoModelForCausalLM.from_pretrained(- self.save_folder,+ quantized_model_path,torch_dtype="auto",device_map="auto",)- tokenizer = AutoTokenizer.from_pretrained(self.save_folder)+ tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)model_infer(model, tokenizer)diff --git a/test/test_cpu/quantization/test_mix_bits.py b/test/test_cpu/quantization/test_mix_bits.pyindex 1fbbfed44..6eab008c0 100644--- a/test/test_cpu/quantization/test_mix_bits.py+++ b/test/test_cpu/quantization/test_mix_bits.py@@ -59,7 +59,7 @@ class TestAutoRound:dataset=dataloader,)quantized_model_path = self.save_dir- autoround.quantize_and_save(output_dir=quantized_model_path, format="auto_gptq")+ _, quantized_model_path = autoround.quantize_and_save(output_dir=quantized_model_path, format="auto_gptq")# test original GPTQModel inferencefrom gptqmodel import GPTQModel@@ -87,7 +87,7 @@ class TestAutoRound:dataset=dataloader,)quantized_model_path = self.save_dir- autoround.quantize_and_save(output_dir=quantized_model_path, format="auto_gptq")+ _, quantized_model_path = autoround.quantize_and_save(output_dir=quantized_model_path, format="auto_gptq")quantization_config = AutoRoundConfig()model = AutoModelForCausalLM.from_pretrained(quantized_model_path, device_map="cpu", quantization_config=quantization_config@@ -114,7 +114,7 @@ class TestAutoRound:layer_config=layer_config,)quantized_model_path = self.save_dir- autoround.quantize_and_save(output_dir=quantized_model_path, format="auto_round")+ _, quantized_model_path = autoround.quantize_and_save(output_dir=quantized_model_path, format="auto_round")config_file = Path(quantized_model_path) / "config.json"with open(config_file, "r", encoding="utf-8") as f:config = json.load(f)@@ -147,7 +147,7 @@ class TestAutoRound:layer_config=layer_config,)quantized_model_path = self.save_dir- autoround.quantize_and_save(output_dir=quantized_model_path, format="auto_awq")+ _, quantized_model_path = autoround.quantize_and_save(output_dir=quantized_model_path, format="auto_awq")quantization_config = AutoRoundConfig()model = AutoModelForCausalLM.from_pretrained(quantized_model_path, device_map="cpu", quantization_config=quantization_config@@ -219,7 +219,9 @@ class TestAutoRound:layer_config=layer_config,)quantized_model_path = self.save_dir- autoround.quantize_and_save(output_dir=quantized_model_path, inplace=False, format="auto_round")+ _, quantized_model_path = autoround.quantize_and_save(+ output_dir=quantized_model_path, inplace=False, format="auto_round"+ )model = AutoModelForCausalLM.from_pretrained(quantized_model_path,torch_dtype="auto",diff --git a/test/test_cpu/quantization/test_mxfp_nvfp.py b/test/test_cpu/quantization/test_mxfp_nvfp.pyindex d0c15ccf8..2b091a068 100644--- a/test/test_cpu/quantization/test_mxfp_nvfp.py+++ b/test/test_cpu/quantization/test_mxfp_nvfp.py@@ -10,6 +10,7 @@ from transformers import AutoModelForCausalLM, AutoRoundConfig, AutoTokenizerfrom auto_round import AutoRoundfrom auto_round.export.export_to_autoround import export_to_nvfp_mx as autoround_nvfp_mx_export+from ...envs import require_compressed_tensorsfrom ...helpers import forbid_threaded_packing, is_model_outputs_similar, transformers_version@@ -79,7 +80,9 @@ class TestAutoRoundFP:layer_config=layer_config,trust_remote_code=False,)- compressed_model, _ = autoround.quantize_and_save(output_dir=self.save_dir, inplace=True, format="auto_round")+ compressed_model, quantized_model_path = autoround.quantize_and_save(+ output_dir=self.save_dir, inplace=True, format="auto_round"+ )lm_head = compressed_model.lm_headassert (hasattr(lm_head, "weight_scale")@@ -88,7 +91,6 @@ class TestAutoRoundFP:and lm_head.weight_packed.dtype is torch.uint8and lm_head.weight_scale.dtype is torch.float8_e4m3fn), "Illegal NVFP4 packing for lm_head layer"- quantized_model_path = self.save_dirassert is_model_outputs_similar(model_name, quantized_model_path)def test_mxfp4_moe_ar(self, tiny_deepseek_v2_model_path, dataloader):@@ -119,6 +121,7 @@ class TestAutoRoundFP:and lm_head.weight_scale.dtype is torch.uint8), "Illegal MXFP4 packing for lm_head layer"+ @require_compressed_tensorsdef test_mxfp4_llmcompressor_format(self, tiny_opt_model_path, dataloader):model_name = tiny_opt_model_pathfrom transformers import AutoConfig@@ -157,6 +160,7 @@ class TestAutoRoundFP:and quantization_config["config_groups"]["group_0"]["weights"]["num_bits"] == 4), f"Invalid MXFP4 quantization configuration: {quantization_config}"+ @require_compressed_tensorsdef test_rtn_mxfp4_llmcompressor_format(self, tiny_opt_model_path, dataloader):model_name = tiny_opt_model_pathfrom transformers import AutoConfig@@ -195,6 +199,7 @@ class TestAutoRoundFP:and quantization_config["config_groups"]["group_0"]["weights"]["num_bits"] == 4), f"Invalid MXFP4 quantization configuration: {quantization_config}"+ @require_compressed_tensorsdef test_mxfp8_llmcompressor_format(self, tiny_opt_model_path, dataloader):model_name = tiny_opt_model_pathfrom transformers import AutoConfig@@ -208,7 +213,9 @@ class TestAutoRoundFP:dataset=dataloader,)quantized_model_path = self.save_dir- compressed_model, _ = autoround.quantize_and_save(output_dir=quantized_model_path, format="llm_compressor")+ compressed_model, quantized_model_path = autoround.quantize_and_save(+ output_dir=quantized_model_path, format="llm_compressor"+ )tmp_layer = compressed_model.model.decoder.layers[1].self_attn.q_projassert (hasattr(tmp_layer, "weight_scale")@@ -230,6 +237,7 @@ class TestAutoRoundFP:0.05 < folder_size_gb < 0.1), f"Quantized model folder size {folder_size_gb:.2f} GB is outside the expected range (0.05~0.1 GB)"+ @require_compressed_tensorsdef test_nvfp4_llmcompressor_format(self, tiny_opt_model_path, dataloader):model_name = tiny_opt_model_pathfrom transformers import AutoConfig@@ -243,7 +251,9 @@ class TestAutoRoundFP:dataset=dataloader,)quantized_model_path = self.save_dir- compressed_model, _ = autoround.quantize_and_save(output_dir=quantized_model_path, format="llm_compressor")+ compressed_model, quantized_model_path = autoround.quantize_and_save(+ output_dir=quantized_model_path, format="llm_compressor"+ )tmp_layer = compressed_model.model.decoder.layers[1].self_attn.q_projassert (hasattr(tmp_layer, "weight_scale")@@ -279,7 +289,9 @@ class TestAutoRoundFP:dataset=dataloader,)quantized_model_path = self.save_dir- compressed_model, _ = autoround.quantize_and_save(output_dir=quantized_model_path, format="auto_round")+ compressed_model, quantized_model_path = autoround.quantize_and_save(+ output_dir=quantized_model_path, format="auto_round"+ )tmp_layer = compressed_model.model.decoder.layers[1].self_attn.q_projassert (hasattr(tmp_layer, "weight_scale")@@ -334,7 +346,9 @@ class TestAutoRoundFP:layer_config=layer_config,)quantized_model_path = self.save_dir- autoround.quantize_and_save(output_dir=quantized_model_path, inplace=True, format="auto_round")+ _, quantized_model_path = autoround.quantize_and_save(+ output_dir=quantized_model_path, inplace=True, format="auto_round"+ )assert is_model_outputs_similar(model_name, quantized_model_path)@pytest.mark.parametrize(@@ -371,7 +385,7 @@ class TestAutoRoundFP:)quantized_model_path = self.save_dir- compressed_model, _ = autoround.quantize_and_save(+ compressed_model, quantized_model_path = autoround.quantize_and_save(output_dir=quantized_model_path,format="auto_round",)diff --git a/test/test_cpu/quantization/test_mxfp_save_load.py b/test/test_cpu/quantization/test_mxfp_save_load.pyindex 25e5a2428..ea7e4478d 100644--- a/test/test_cpu/quantization/test_mxfp_save_load.py+++ b/test/test_cpu/quantization/test_mxfp_save_load.py@@ -61,7 +61,7 @@ def test_e2e_quant_and_load(scheme_name, weight_data_type, act_data_type):# Quantize and save the model to the temporary directoryquantized_model_path = f"{temp_dir}/tmp_autoround"- autoround.quantize_and_save(format="auto_round", output_dir=quantized_model_path)+ _, quantized_model_path = autoround.quantize_and_save(format="auto_round", output_dir=quantized_model_path)# Perform inference with the quantized modelmodel = AutoModelForCausalLM.from_pretrained(diff --git a/test/test_cpu/schemes/test_scheme.py b/test/test_cpu/schemes/test_scheme.pyindex 8af28b7a5..41e9a9f7d 100644--- a/test/test_cpu/schemes/test_scheme.py+++ b/test/test_cpu/schemes/test_scheme.py@@ -11,6 +11,7 @@ from ...helpers import get_model_path, get_tiny_model, opt_name_or_path, qwen_naclass TestAutoRound:+@pytest.fixture(autouse=True)def setup_save_folder(self, tmp_path):self.save_folder = str(tmp_path / "saved")@@ -35,10 +36,12 @@ class TestAutoRound:def test_w4a16(self, tiny_opt_model_path, dataloader):ar = AutoRound(tiny_opt_model_path, scheme="W4A16", nsamples=1, iters=1, seqlen=2, dataset=dataloader)+ ar.post_init()assert ar.bits == 4def test_w2a16_rtn(self, tiny_opt_model_path, dataloader):ar = AutoRound(tiny_opt_model_path, scheme="W2A16", nsamples=1, iters=0, seqlen=2, dataset=dataloader)+ ar.post_init()assert ar.bits == 2def test_w4a16_mixed(self, tiny_qwen_moe_model_path, dataloader):@@ -56,13 +59,13 @@ class TestAutoRound:low_cpu_mem_usage=False,layer_config=layer_config,)- ar.quantize_and_save(self.save_folder)+ _, quantized_model_path = ar.quantize_and_save(self.save_folder)assert ar.bits == 4assert ar.model.model.layers[0].self_attn.q_proj.bits == 8assert ar.model.model.layers[0].self_attn.k_proj.bits == 16assert ar.model.model.layers[0].mlp.experts[0].up_proj.bits == 4# assert ar.model.model.layers[0].mlp.shared_expert.gate_proj.bits == 8 # gate has been added to ignore_layers- model = transformers.AutoModelForCausalLM.from_pretrained(self.save_folder, trust_remote_code=True)+ model = transformers.AutoModelForCausalLM.from_pretrained(quantized_model_path, trust_remote_code=True)assert model is not None, "Model loading failed after quantization with W4A16_MIXED scheme on MoE"def test_w4a16_mixed_mllm(self, tiny_qwen_2_5_vl_model_path, dataloader):@@ -78,8 +81,8 @@ class TestAutoRound:dataset=dataloader,low_cpu_mem_usage=False,)- ar.quantize_and_save(self.save_folder)- model = transformers.Qwen2_5_VLForConditionalGeneration.from_pretrained(self.save_folder)+ _, quantized_model_path = ar.quantize_and_save(self.save_folder)+ model = transformers.Qwen2_5_VLForConditionalGeneration.from_pretrained(quantized_model_path)assert model is not None, "Model loading failed after quantization with W4A16_MIXED scheme on MLLM"assert ar.bits == 4assert ar.model.model.language_model.layers[0].self_attn.q_proj.bits == 16@@ -87,6 +90,7 @@ class TestAutoRound:def test_mxfp4(self, tiny_opt_model_path, dataloader):ar = AutoRound(tiny_opt_model_path, scheme="MXFP4", nsamples=1, iters=1, seqlen=2, dataset=dataloader)+ ar.post_init()assert ar.bits == 4assert ar.act_bits == 4assert ar.data_type == "mx_fp"@@ -94,29 +98,32 @@ class TestAutoRound:def test_mxfp4_rceil(self, tiny_opt_model_path):ar = AutoRound(tiny_opt_model_path, scheme="MXFP4_RCEIL", nsamples=1, iters=1)+ ar.post_init()assert ar.bits == 4assert ar.act_bits == 4assert ar.data_type == "mx_fp"assert ar.act_data_type == "mx_fp_rceil"- ar.quantize_and_save()- model = transformers.AutoModelForCausalLM.from_pretrained("tmp_autoround", trust_remote_code=True)+ _, quantized_model_path = ar.quantize_and_save()+ model = transformers.AutoModelForCausalLM.from_pretrained(quantized_model_path, trust_remote_code=True)assert model is not None, "Model loading failed after quantization with MXFP4 scheme"def test_vlm(self, tiny_qwen_vl_model_path):- from auto_round import AutoRoundMLLM+ from auto_round import AutoRound- ar = AutoRoundMLLM(tiny_qwen_vl_model_path, scheme="W2A16", nsamples=1, iters=1, seqlen=2)+ ar = AutoRound(tiny_qwen_vl_model_path, scheme="W2A16", nsamples=1, iters=1, seqlen=2)+ ar.post_init()assert ar.bits == 2assert ar.act_bits == 16def test_nvfp4(self, tiny_opt_model_path, dataloader):ar = AutoRound(tiny_opt_model_path, scheme="NVFP4", nsamples=1, iters=1, seqlen=2, dataset=dataloader)+ ar.post_init()assert ar.bits == 4assert ar.act_bits == 4assert ar.data_type == "nv_fp"assert ar.act_data_type == "nv_fp4_with_static_gs"- ar.quantize_and_save(self.save_folder)- model = transformers.AutoModelForCausalLM.from_pretrained(self.save_folder, trust_remote_code=True)+ _, quantized_model_path = ar.quantize_and_save(self.save_folder)+ model = transformers.AutoModelForCausalLM.from_pretrained(quantized_model_path, trust_remote_code=True)assert model is not None, "Model loading failed after quantization with NVFP4 scheme"@pytest.mark.parametrize(@@ -198,24 +205,26 @@ class TestAutoRound:def test_fp8_static(self, tiny_opt_model_path):ar = AutoRound(tiny_opt_model_path, scheme="FP8_STATIC", nsamples=1, iters=1)+ ar.post_init()assert ar.bits == 8assert ar.act_bits == 8assert ar.data_type == "fp"assert ar.act_data_type == "fp"assert ar.group_size == -1assert ar.act_dynamic is False- ar.quantize_and_save()- model = transformers.AutoModelForCausalLM.from_pretrained("tmp_autoround", trust_remote_code=True)+ _, quantized_model_path = ar.quantize_and_save()+ model = transformers.AutoModelForCausalLM.from_pretrained(quantized_model_path, trust_remote_code=True)assert model is not None, "Model loading failed after quantization with FP8_STATIC scheme"def test_fp8_static_rtn(self, tiny_opt_model_path):ar = AutoRound(tiny_opt_model_path, scheme="FP8_STATIC", nsamples=1, iters=0, disable_opt_rtn=True)+ ar.post_init()assert ar.bits == 8assert ar.act_bits == 8assert ar.data_type == "fp"assert ar.act_data_type == "fp"assert ar.group_size == -1assert ar.act_dynamic is False- ar.quantize_and_save(self.save_folder)- model = transformers.AutoModelForCausalLM.from_pretrained(self.save_folder, trust_remote_code=True)+ _, quantized_model_path = ar.quantize_and_save(self.save_folder)+ model = transformers.AutoModelForCausalLM.from_pretrained(quantized_model_path, trust_remote_code=True)assert model is not None, "Model loading failed after quantization with FP8_STATIC scheme"diff --git a/test/test_cpu/utils/test_generation.py b/test/test_cpu/utils/test_generation.pyindex 448c3cc40..2acb560a2 100644--- a/test/test_cpu/utils/test_generation.py+++ b/test/test_cpu/utils/test_generation.py@@ -43,7 +43,9 @@ class TestAutoRoundFormatGeneration:)quantized_model_path = self.save_folder- autoround.quantize_and_save(output_dir=quantized_model_path, format="auto_round", inplace=False)+ _, quantized_model_path = autoround.quantize_and_save(+ output_dir=quantized_model_path, format="auto_round", inplace=False+ )model = AutoModelForCausalLM.from_pretrained(quantized_model_path, device_map="cpu")tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)@@ -78,7 +80,7 @@ class TestAutoRoundFormatGeneration:)quantized_model_path = self.save_folder- autoround.quantize_and_save(output_dir=quantized_model_path, format="auto_round")+ _, quantized_model_path = autoround.quantize_and_save(output_dir=quantized_model_path, format="auto_round")model = AutoModelForCausalLM.from_pretrained(quantized_model_path, device_map="auto", trust_remote_code=Truediff --git a/test/test_cuda/advanced/test_multiple_card.py b/test/test_cuda/advanced/test_multiple_card.pyindex ac6d0fb22..abc3bddb6 100644--- a/test/test_cuda/advanced/test_multiple_card.py+++ b/test/test_cuda/advanced/test_multiple_card.py@@ -198,24 +198,24 @@ class TestAutoRound:@multi_carddef test_mllm_device_map(self, tiny_qwen_2_5_vl_model_path):- from auto_round import AutoRoundMLLM+ from auto_round import AutoRounddevice_map = "0,1"- ar = AutoRoundMLLM(tiny_qwen_2_5_vl_model_path, device_map=device_map)+ ar = AutoRound(tiny_qwen_2_5_vl_model_path, device_map=device_map)assert ar.device == "cuda:0"assert ar.device_map == device_mapdevice_map = 1- ar = AutoRoundMLLM(ar.model, ar.tokenizer, processor=ar.processor, device_map=device_map)+ ar = AutoRound(ar.model, ar.tokenizer, processor=ar.processor, device_map=device_map)assert ar.device == "cuda:1"assert ar.device_map == device_mapdevice_map = "auto"- ar = AutoRoundMLLM(ar.model, ar.tokenizer, processor=ar.processor, device_map=device_map)+ ar = AutoRound(ar.model, ar.tokenizer, processor=ar.processor, device_map=device_map)assert ar.device == "cuda"assert ar.device_map == device_mapdevice_map = {"model.language_model.layers": 0, "model.visual.blocks": 1}- ar = AutoRoundMLLM(ar.model, ar.tokenizer, processor=ar.processor, device_map=device_map)+ ar = AutoRound(ar.model, ar.tokenizer, processor=ar.processor, device_map=device_map)assert ar.model.model.language_model.layers[0].self_attn.q_proj.tuning_device == "cuda:0"assert ar.model.model.visual.blocks[0].mlp.gate_proj.tuning_device == "cuda:1"diff --git a/test/test_cuda/algorithms/test_alg_ext.py b/test/test_cuda/algorithms/test_alg_ext.pyindex 17cbe6d90..7aceb84fb 100644--- a/test/test_cuda/algorithms/test_alg_ext.py+++ b/test/test_cuda/algorithms/test_alg_ext.py@@ -48,9 +48,9 @@ class TestAlgExt:def test_2bits(self):model_name = get_model_path("facebook/opt-125m")ar = AutoRound(model=model_name, bits=2, group_size=64, enable_alg_ext=True)- ar.quantize_and_save(self.save_folder)+ _, quantized_model_path = ar.quantize_and_save(self.save_folder)model = AutoModelForCausalLM.from_pretrained(- self.save_folder,+ quantized_model_path,device_map="auto",)diff --git a/test/test_cuda/algorithms/test_auto_scheme.py b/test/test_cuda/algorithms/test_auto_scheme.pyindex 9cde148d7..cd9f85e73 100644--- a/test/test_cuda/algorithms/test_auto_scheme.py+++ b/test/test_cuda/algorithms/test_auto_scheme.py@@ -264,16 +264,16 @@ class TestAutoScheme:model_name = get_model_path("facebook/opt-125m")scheme = AutoScheme(avg_bits=3, options=("W2A16", "W4A16", "W8A16", "BF16"))ar = AutoRound(model=model_name, scheme=scheme)- ar.quantize_and_save(output_dir=self.save_dir)- evaluate_accuracy(self.save_dir, threshold=0.25)+ _, quantized_model_path = ar.quantize_and_save(output_dir=self.save_dir)+ evaluate_accuracy(quantized_model_path, threshold=0.25)@pytest.mark.skip_ci(reason="The evaluation is time-consuming")def test_enable_torch_compile(self):model_name = get_model_path("facebook/opt-125m")scheme = AutoScheme(avg_bits=2, options=("W2A16"), ignore_scale_zp_bits=True)ar = AutoRound(model=model_name, scheme=scheme, enable_torch_compile=True)- ar.quantize_and_save(output_dir=self.save_dir)- evaluate_accuracy(self.save_dir, threshold=0.10)+ _, quantized_model_path = ar.quantize_and_save(output_dir=self.save_dir)+ evaluate_accuracy(quantized_model_path, threshold=0.10)def test_mixed_bits_get_scoring(self):"""Verify that AutoScheme scoring produces accuracy above a known reference threshold for mixed-bitdiff --git a/test/test_cuda/backends/test_exllamav2_backend.py b/test/test_cuda/backends/test_exllamav2_backend.pyindex 4ed812105..db3a15b63 100644--- a/test/test_cuda/backends/test_exllamav2_backend.py+++ b/test/test_cuda/backends/test_exllamav2_backend.py@@ -41,23 +41,25 @@ class TestAutoRoundexllamaBackend:model_path, bits=bits, group_size=group_size, sym=sym, iters=1, seqlen=2, dataset=dataloader)quantized_model_path = self.save_dir- autoround.quantize_and_save(output_dir=quantized_model_path, format="auto_round:gptqmodel")+ _, quantized_model_path = autoround.quantize_and_save(+ output_dir=quantized_model_path, format="auto_round:gptqmodel"+ )quantization_config = AutoRoundConfig(backend="gptqmodel:exllamav2")model = AutoModelForCausalLM.from_pretrained(- self.save_dir, torch_dtype=torch.float16, device_map="auto", quantization_config=quantization_config+ quantized_model_path, torch_dtype=torch.float16, device_map="auto", quantization_config=quantization_config)- tokenizer = AutoTokenizer.from_pretrained(self.save_dir)+ tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)model_infer(model, tokenizer)evaluate_accuracy(model, tokenizer, threshold=0.35, batch_size=16)torch.cuda.empty_cache()model = AutoModelForCausalLM.from_pretrained(- self.save_dir, torch_dtype=torch.bfloat16, device_map="auto", quantization_config=quantization_config+ quantized_model_path, torch_dtype=torch.bfloat16, device_map="auto", quantization_config=quantization_config)- tokenizer = AutoTokenizer.from_pretrained(self.save_dir)+ tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)model_infer(model, tokenizer)evaluate_accuracy(model, tokenizer, threshold=0.35, batch_size=16)torch.cuda.empty_cache()@@ -79,14 +81,16 @@ class TestAutoRoundexllamaBackend:dataset=dataloader,)quantized_model_path = self.save_dir- autoround.quantize_and_save(output_dir=quantized_model_path, format="auto_round") ##will convert to gptq model+ _, quantized_model_path = autoround.quantize_and_save(+ output_dir=quantized_model_path, format="auto_round"+ ) ##will convert to gptq modelquantization_config = AutoRoundConfig(backend="gptq:exllamav2") ## or exllamav2model = AutoModelForCausalLM.from_pretrained(- self.save_dir, torch_dtype=torch.float16, device_map="auto", quantization_config=quantization_config+ quantized_model_path, torch_dtype=torch.float16, device_map="auto", quantization_config=quantization_config)- tokenizer = AutoTokenizer.from_pretrained(self.save_dir)+ tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)model_infer(model, tokenizer)evaluate_accuracy(model, tokenizer, threshold=0.27, batch_size=16)torch.cuda.empty_cache()@@ -108,14 +112,16 @@ class TestAutoRoundexllamaBackend:sym=True,)quantized_model_path = self.save_dir- autoround.quantize_and_save(output_dir=quantized_model_path, format="auto_round") ##will convert to gptq model+ _, quantized_model_path = autoround.quantize_and_save(+ output_dir=quantized_model_path, format="auto_round"+ ) ##will convert to gptq modelquantization_config = AutoRoundConfig(backend="gptq:exllamav2") ## or exllamav2model = AutoModelForCausalLM.from_pretrained(- self.save_dir, torch_dtype=torch.float16, device_map="auto", quantization_config=quantization_config+ quantized_model_path, torch_dtype=torch.float16, device_map="auto", quantization_config=quantization_config)- tokenizer = AutoTokenizer.from_pretrained(self.save_dir)+ tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)model_infer(model, tokenizer)evaluate_accuracy(model, tokenizer, threshold=0.15, batch_size=64)torch.cuda.empty_cache()@@ -134,15 +140,17 @@ class TestAutoRoundexllamaBackend:disable_opt_rtn=True,)quantized_model_path = self.save_dir- autoround.quantize_and_save(output_dir=quantized_model_path, format="auto_round:auto_awq")+ _, quantized_model_path = autoround.quantize_and_save(+ output_dir=quantized_model_path, format="auto_round:auto_awq"+ )quantization_config = AutoRoundConfig(backend="gptqmodel:awq_exllamav2")# test awq bfloat16 inferencemodel = AutoModelForCausalLM.from_pretrained(- self.save_dir, torch_dtype=torch.bfloat16, device_map="auto", quantization_config=quantization_config+ quantized_model_path, torch_dtype=torch.bfloat16, device_map="auto", quantization_config=quantization_config)assert model.dtype == torch.bfloat16, f"Expected model dtype bfloat16, got {model.dtype}"- tokenizer = AutoTokenizer.from_pretrained(self.save_dir)+ tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)# Inference generation checkeval_generated_prompt(model, tokenizer)# Accuracy check@@ -165,14 +173,16 @@ class TestAutoRoundexllamaBackend:disable_opt_rtn=True,)quantized_model_path = self.save_dir- autoround.quantize_and_save(output_dir=quantized_model_path, format="auto_round:auto_awq")+ _, quantized_model_path = autoround.quantize_and_save(+ output_dir=quantized_model_path, format="auto_round:auto_awq"+ )quantization_config = AutoRoundConfig(backend="gptqmodel:awq_exllamav2")model = AutoModelForCausalLM.from_pretrained( # test awq bfloat16 inference- self.save_dir, torch_dtype=torch.bfloat16, device_map="auto", quantization_config=quantization_config+ quantized_model_path, torch_dtype=torch.bfloat16, device_map="auto", quantization_config=quantization_config)- tokenizer = AutoTokenizer.from_pretrained(self.save_dir)+ tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)# Inference generation checkeval_generated_prompt(model, tokenizer)# Accuracy checkdiff --git a/test/test_cuda/backends/test_marlin_backend.py b/test/test_cuda/backends/test_marlin_backend.pyindex ff00315d4..95b71b02b 100644--- a/test/test_cuda/backends/test_marlin_backend.py+++ b/test/test_cuda/backends/test_marlin_backend.py@@ -49,14 +49,14 @@ class TestAutoRoundMarlinBackend:dataset=dataloader,)quantized_model_path = self.save_dir- autoround.quantize_and_save(output_dir=quantized_model_path, format="auto_gptq")+ _, quantized_model_path = autoround.quantize_and_save(output_dir=quantized_model_path, format="auto_gptq")quantization_config = AutoRoundConfig(backend="marlin")model = AutoModelForCausalLM.from_pretrained(- self.save_dir, torch_dtype=torch.float16, device_map="auto", quantization_config=quantization_config+ quantized_model_path, torch_dtype=torch.float16, device_map="auto", quantization_config=quantization_config)- tokenizer = AutoTokenizer.from_pretrained(self.save_dir)+ tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)model_infer(model, tokenizer)evaluate_accuracy(model, tokenizer, threshold=0.27, batch_size=16)torch.cuda.empty_cache()@@ -80,14 +80,17 @@ class TestAutoRoundMarlinBackend:dataset=dataloader,)quantized_model_path = self.save_dir- autoround.quantize_and_save(output_dir=quantized_model_path, format="auto_gptq")+ _, quantized_model_path = autoround.quantize_and_save(output_dir=quantized_model_path, format="auto_gptq")quantization_config = AutoRoundConfig(backend="marlin")model = AutoModelForCausalLM.from_pretrained(- self.save_dir, torch_dtype=torch.float16, device_map="auto", quantization_config=quantization_config+ quantized_model_path,+ torch_dtype=torch.float16,+ device_map="auto",+ quantization_config=quantization_config,)- tokenizer = AutoTokenizer.from_pretrained(self.save_dir)+ tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)model_infer(model, tokenizer)evaluate_accuracy(model, tokenizer, threshold=0.14, batch_size=16)@@ -107,14 +110,17 @@ class TestAutoRoundMarlinBackend:dataset=dataloader,)quantized_model_path = self.save_dir- autoround.quantize_and_save(output_dir=quantized_model_path, format="auto_round")+ _, quantized_model_path = autoround.quantize_and_save(output_dir=quantized_model_path, format="auto_round")quantization_config = AutoRoundConfig(backend="marlin")model = AutoModelForCausalLM.from_pretrained(- self.save_dir, torch_dtype=torch.float16, device_map="auto", quantization_config=quantization_config+ quantized_model_path,+ torch_dtype=torch.float16,+ device_map="auto",+ quantization_config=quantization_config,)- tokenizer = AutoTokenizer.from_pretrained(self.save_dir)+ tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)model_infer(model, tokenizer)evaluate_accuracy(model, tokenizer, threshold=0.14, batch_size=16)@@ -137,7 +143,7 @@ class TestAutoRoundMarlinBackend:## quantization_config = AutoRoundConfig(backend="marlin")# model = AutoModelForCausalLM.from_pretrained(- # self.save_dir,+ # quantized_model_path,# torch_dtype=torch.float16,# device_map="auto",# quantization_config=quantization_config@@ -179,14 +185,16 @@ class TestAutoRoundMarlinBackend:disable_opt_rtn=True,)quantized_model_path = self.save_dir- autoround.quantize_and_save(output_dir=quantized_model_path, format="auto_round:auto_awq")+ _, quantized_model_path = autoround.quantize_and_save(+ output_dir=quantized_model_path, format="auto_round:auto_awq"+ )quantization_config = AutoRoundConfig(backend="gptqmodel:awq_marlin")model = AutoModelForCausalLM.from_pretrained(- self.save_dir, torch_dtype="auto", device_map="cuda:0", quantization_config=quantization_config+ quantized_model_path, torch_dtype="auto", device_map="cuda:0", quantization_config=quantization_config)- tokenizer = AutoTokenizer.from_pretrained(self.save_dir)+ tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)# Inference generation checkeval_generated_prompt(model, tokenizer)# Accuracy check@@ -211,14 +219,16 @@ class TestAutoRoundMarlinBackend:disable_opt_rtn=True,)quantized_model_path = self.save_dir- autoround.quantize_and_save(output_dir=quantized_model_path, format="auto_round:auto_awq")+ _, quantized_model_path = autoround.quantize_and_save(+ output_dir=quantized_model_path, format="auto_round:auto_awq"+ )quantization_config = AutoRoundConfig(backend="gptqmodel:awq_marlin")model = AutoModelForCausalLM.from_pretrained(- self.save_dir, torch_dtype="auto", device_map="cuda:0", quantization_config=quantization_config+ quantized_model_path, torch_dtype="auto", device_map="cuda:0", quantization_config=quantization_config)- tokenizer = AutoTokenizer.from_pretrained(self.save_dir)+ tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)# Inference generation checkeval_generated_prompt(model, tokenizer)# Accuracy checkdiff --git a/test/test_cuda/backends/test_torch_backend.py b/test/test_cuda/backends/test_torch_backend.pyindex dfd2c85eb..71e743f14 100644--- a/test/test_cuda/backends/test_torch_backend.py+++ b/test/test_cuda/backends/test_torch_backend.py@@ -49,14 +49,16 @@ class TestAutoRoundTorchBackend:dataset=dataloader,)quantized_model_path = self.save_dir- autoround.quantize_and_save(output_dir=quantized_model_path, format="auto_round:gptqmodel")+ _, quantized_model_path = autoround.quantize_and_save(+ output_dir=quantized_model_path, format="auto_round:gptqmodel"+ )quantization_config = AutoRoundConfig(backend="torch")model = AutoModelForCausalLM.from_pretrained(quantized_model_path, torch_dtype=torch.float16, device_map="auto", quantization_config=quantization_config)- tokenizer = AutoTokenizer.from_pretrained(self.save_dir)+ tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)model_infer(model, tokenizer)evaluate_accuracy(model, tokenizer, threshold=0.35, batch_size=16)torch.cuda.empty_cache()@@ -79,13 +81,15 @@ class TestAutoRoundTorchBackend:dataset=dataloader,)quantized_model_path = self.save_dir- autoround.quantize_and_save(output_dir=quantized_model_path, format="auto_round") ##will convert to gptq model+ _, quantized_model_path = autoround.quantize_and_save(+ output_dir=quantized_model_path, format="auto_round"+ ) ##will convert to gptq modelquantization_config = AutoRoundConfig(backend="torch")model = AutoModelForCausalLM.from_pretrained(quantized_model_path, torch_dtype=torch.float16, device_map="auto", quantization_config=quantization_config)- tokenizer = AutoTokenizer.from_pretrained(self.save_dir)+ tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)model_infer(model, tokenizer)evaluate_accuracy(model, tokenizer, threshold=0.28, batch_size=16)torch.cuda.empty_cache()@@ -130,7 +134,9 @@ class TestAutoRoundTorchBackend:autoround.quantize()quantized_model_path = self.save_dir- autoround.save_quantized(output_dir=quantized_model_path, inplace=False, format="auto_round")+ _, quantized_model_path = autoround.save_quantized(+ output_dir=quantized_model_path, inplace=False, format="auto_round", return_folders=True+ )device = "auto" ##cpu, hpu, cudafrom transformers import AutoRoundConfig@@ -160,14 +166,16 @@ class TestAutoRoundTorchBackend:disable_opt_rtn=True,)quantized_model_path = self.save_dir- autoround.quantize_and_save(output_dir=quantized_model_path, format="auto_round:auto_awq")+ _, quantized_model_path = autoround.quantize_and_save(+ output_dir=quantized_model_path, format="auto_round:auto_awq"+ )quantization_config = AutoRoundConfig(backend="gptqmodel:awq_torch")model = AutoModelForCausalLM.from_pretrained(- self.save_dir, torch_dtype=torch.float16, device_map="auto", quantization_config=quantization_config+ quantized_model_path, torch_dtype=torch.float16, device_map="auto", quantization_config=quantization_config)- tokenizer = AutoTokenizer.from_pretrained(self.save_dir)+ tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)# Inference generation checkoutput = model_infer(model, tokenizer)assert isinstance(output, str) and len(output.strip()) > 0, "Model failed to generate non-empty output"diff --git a/test/test_cuda/backends/test_triton_backend.py b/test/test_cuda/backends/test_triton_backend.pyindex 6675d1620..fa1c1f152 100644--- a/test/test_cuda/backends/test_triton_backend.py+++ b/test/test_cuda/backends/test_triton_backend.py@@ -35,14 +35,14 @@ class TestAutoRoundTritonBackend:bits, group_size, sym = 2, 32, Falseautoround = AutoRound(model, tokenizer, bits=bits, group_size=group_size, sym=sym)quantized_model_path = self.save_folder- autoround.quantize_and_save(output_dir=quantized_model_path)+ _, quantized_model_path = autoround.quantize_and_save(output_dir=quantized_model_path)quantization_config = AutoRoundConfig(backend="tritonv2")model = AutoModelForCausalLM.from_pretrained(- self.save_folder, torch_dtype=torch.float16, device_map="auto", quantization_config=quantization_config+ quantized_model_path, torch_dtype=torch.float16, device_map="auto", quantization_config=quantization_config)- tokenizer = AutoTokenizer.from_pretrained(self.save_folder)+ tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)model_infer(model, tokenizer)evaluate_accuracy(model, tokenizer, threshold=0.19, batch_size=16)torch.cuda.empty_cache()@@ -65,14 +65,16 @@ class TestAutoRoundTritonBackend:dataset=dataloader,)quantized_model_path = self.save_folder- autoround.quantize_and_save(output_dir=quantized_model_path, format="auto_round:gptqmodel")+ _, quantized_model_path = autoround.quantize_and_save(+ output_dir=quantized_model_path, format="auto_round:gptqmodel"+ )quantization_config = AutoRoundConfig(backend="tritonv2")model = AutoModelForCausalLM.from_pretrained(- self.save_folder, torch_dtype=torch.float16, device_map="auto", quantization_config=quantization_config+ quantized_model_path, torch_dtype=torch.float16, device_map="auto", quantization_config=quantization_config)- tokenizer = AutoTokenizer.from_pretrained(self.save_folder)+ tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)model_infer(model, tokenizer)evaluate_accuracy(model, tokenizer, threshold=0.34, batch_size=16)torch.cuda.empty_cache()@@ -95,23 +97,23 @@ class TestAutoRoundTritonBackend:dataset=dataloader,)quantized_model_path = self.save_folder- autoround.quantize_and_save(output_dir=quantized_model_path)+ _, quantized_model_path = autoround.quantize_and_save(output_dir=quantized_model_path)quantization_config = AutoRoundConfig(backend="tritonv2")model = AutoModelForCausalLM.from_pretrained(- self.save_folder, torch_dtype=torch.float16, device_map="auto", quantization_config=quantization_config+ quantized_model_path, torch_dtype=torch.float16, device_map="auto", quantization_config=quantization_config)- tokenizer = AutoTokenizer.from_pretrained(self.save_folder)+ tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)model_infer(model, tokenizer)evaluate_accuracy(model, tokenizer, threshold=0.26, batch_size=16)torch.cuda.empty_cache()model = AutoModelForCausalLM.from_pretrained(- self.save_folder, torch_dtype=torch.bfloat16, device_map="auto", quantization_config=quantization_config+ quantized_model_path, torch_dtype=torch.bfloat16, device_map="auto", quantization_config=quantization_config)- tokenizer = AutoTokenizer.from_pretrained(self.save_folder)+ tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)model_infer(model, tokenizer)evaluate_accuracy(model, tokenizer, threshold=0.26, batch_size=16)torch.cuda.empty_cache()@@ -125,23 +127,23 @@ class TestAutoRoundTritonBackend:bits, group_size, sym = 4, 256, Trueautoround = AutoRound(model, tokenizer, bits=bits, group_size=group_size, sym=sym, nsamples=1, iters=1)quantized_model_path = self.save_folder- autoround.quantize_and_save(output_dir=quantized_model_path)+ _, quantized_model_path = autoround.quantize_and_save(output_dir=quantized_model_path)quantization_config = AutoRoundConfig(backend="tritonv2")model = AutoModelForCausalLM.from_pretrained(- self.save_folder, torch_dtype=torch.float16, device_map="auto", quantization_config=quantization_config+ quantized_model_path, torch_dtype=torch.float16, device_map="auto", quantization_config=quantization_config)- tokenizer = AutoTokenizer.from_pretrained(self.save_folder)+ tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)model_infer(model, tokenizer)evaluate_accuracy(model, tokenizer, threshold=0.27, batch_size=16)torch.cuda.empty_cache()model = AutoModelForCausalLM.from_pretrained(- self.save_folder, torch_dtype=torch.bfloat16, device_map="auto", quantization_config=quantization_config+ quantized_model_path, torch_dtype=torch.bfloat16, device_map="auto", quantization_config=quantization_config)- tokenizer = AutoTokenizer.from_pretrained(self.save_folder)+ tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)model_infer(model, tokenizer)evaluate_accuracy(model, tokenizer, threshold=0.27, batch_size=16)torch.cuda.empty_cache()@@ -161,23 +163,23 @@ class TestAutoRoundTritonBackend:sym=sym,)quantized_model_path = self.save_folder- autoround.quantize_and_save(output_dir=quantized_model_path)+ _, quantized_model_path = autoround.quantize_and_save(output_dir=quantized_model_path)quantization_config = AutoRoundConfig(backend="tritonv2")model = AutoModelForCausalLM.from_pretrained(- self.save_folder, torch_dtype=torch.float16, device_map="auto", quantization_config=quantization_config+ quantized_model_path, torch_dtype=torch.float16, device_map="auto", quantization_config=quantization_config)- tokenizer = AutoTokenizer.from_pretrained(self.save_folder)+ tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)model_infer(model, tokenizer)evaluate_accuracy(model, tokenizer, threshold=0.18, batch_size=16)torch.cuda.empty_cache()model = AutoModelForCausalLM.from_pretrained(- self.save_folder, torch_dtype=torch.bfloat16, device_map="auto", quantization_config=quantization_config+ quantized_model_path, torch_dtype=torch.bfloat16, device_map="auto", quantization_config=quantization_config)- tokenizer = AutoTokenizer.from_pretrained(self.save_folder)+ tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)model_infer(model, tokenizer)evaluate_accuracy(model, tokenizer, threshold=0.18, batch_size=16)torch.cuda.empty_cache()diff --git a/test/test_cuda/export/test_auto_awq_format.py b/test/test_cuda/export/test_auto_awq_format.pyindex 394b6ab9f..2ee2b9de4 100644--- a/test/test_cuda/export/test_auto_awq_format.py+++ b/test/test_cuda/export/test_auto_awq_format.py@@ -70,7 +70,7 @@ class TestAutoRound:layer_config=layer_config,)quantized_model_path = os.path.join(self.save_dir, "test_export")- autoround.quantize_and_save(output_dir=quantized_model_path, format="auto_awq")+ _, quantized_model_path = autoround.quantize_and_save(output_dir=quantized_model_path, format="auto_awq")# test loading with AutoRoundConfigmodel = AutoModelForCausalLM.from_pretrained(@@ -99,7 +99,7 @@ class TestAutoRound:layer_config=layer_config,)quantized_model_path = "self.save_dir"- autoround.quantize_and_save(output_dir=quantized_model_path, format="auto_awq")+ _, quantized_model_path = autoround.quantize_and_save(output_dir=quantized_model_path, format="auto_awq")quantization_config = AutoRoundConfig()model = AutoModelForCausalLM.from_pretrained(quantized_model_path, device_map="auto", quantization_config=quantization_configdiff --git a/test/test_cuda/export/test_auto_gptq_format.py b/test/test_cuda/export/test_auto_gptq_format.pyindex 474e058d0..f7a2891cf 100644--- a/test/test_cuda/export/test_auto_gptq_format.py+++ b/test/test_cuda/export/test_auto_gptq_format.py@@ -72,7 +72,7 @@ class TestAutoRound:layer_config=layer_config,)quantized_model_path = self.save_dir- autoround.quantize_and_save(output_dir=quantized_model_path, format="auto_gptq")+ _, quantized_model_path = autoround.quantize_and_save(output_dir=quantized_model_path, format="auto_gptq")model = AutoModelForCausalLM.from_pretrained(quantized_model_path, device_map="auto", trust_remote_code=True)tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)diff --git a/test/test_cuda/export/test_auto_round_format.py b/test/test_cuda/export/test_auto_round_format.pyindex ba9862712..863dfc9e9 100644--- a/test/test_cuda/export/test_auto_round_format.py+++ b/test/test_cuda/export/test_auto_round_format.py@@ -53,7 +53,7 @@ class TestAutoRound:)quantized_model_path = self.save_dir- autoround.quantize_and_save(output_dir=quantized_model_path, format="auto_round")+ _, quantized_model_path = autoround.quantize_and_save(output_dir=quantized_model_path, format="auto_round")# Verify loadingmodel = AutoModelForCausalLM.from_pretrained(quantized_model_path, device_map="cuda:0", trust_remote_code=True)@@ -74,7 +74,7 @@ class TestAutoRound:bits, group_size, sym = 4, 128, Trueautoround = AutoRound(model_name, bits=bits, group_size=group_size, sym=sym, layer_config=layer_config)quantized_model_path = self.save_dir- autoround.quantize_and_save(output_dir=quantized_model_path, format="auto_round")+ _, quantized_model_path = autoround.quantize_and_save(output_dir=quantized_model_path, format="auto_round")eval_generated_prompt(quantized_model_path)evaluate_accuracy(quantized_model_path, threshold=0.32, batch_size=16)@@ -92,23 +92,31 @@ class TestAutoRound:sym=sym,)quantized_model_path = self.save_dir- autoround.quantize_and_save(output_dir=quantized_model_path, format="auto_round:auto_awq")+ _, quantized_model_path = autoround.quantize_and_save(+ output_dir=quantized_model_path, format="auto_round:auto_awq"+ )quantization_config = AutoRoundConfig(backend="auto")model = AutoModelForCausalLM.from_pretrained(- self.save_dir, torch_dtype=torch.float16, device_map="cuda:0", quantization_config=quantization_config+ quantized_model_path,+ torch_dtype=torch.float16,+ device_map="cuda:0",+ quantization_config=quantization_config,)- tokenizer = AutoTokenizer.from_pretrained(self.save_dir)+ tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)eval_generated_prompt(model, tokenizer)evaluate_accuracy(model, tokenizer, threshold=0.18, batch_size=16)torch.cuda.empty_cache()model = AutoModelForCausalLM.from_pretrained(- self.save_dir, torch_dtype=torch.bfloat16, device_map="cuda:0", quantization_config=quantization_config+ quantized_model_path,+ torch_dtype=torch.bfloat16,+ device_map="cuda:0",+ quantization_config=quantization_config,)- tokenizer = AutoTokenizer.from_pretrained(self.save_dir)+ tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)eval_generated_prompt(model, tokenizer)@pytest.mark.skip_ci(reason="Time-consuming; Accuracy evaluation")@@ -138,7 +146,7 @@ class TestAutoRound:)quantized_model_path = self.save_dir- autoround.quantize_and_save(output_dir=quantized_model_path)+ _, quantized_model_path = autoround.quantize_and_save(output_dir=quantized_model_path)from transformers import AutoRoundConfig@@ -188,7 +196,9 @@ class TestAutoRound:)quantized_model_path = self.save_dir- autoround.quantize_and_save(output_dir=quantized_model_path, format="auto_round:auto_awq")+ _, quantized_model_path = autoround.quantize_and_save(+ output_dir=quantized_model_path, format="auto_round:auto_awq"+ )model = AutoModelForCausalLM.from_pretrained(quantized_model_path, device_map="auto", trust_remote_code=True)tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)diff --git a/test/test_cuda/export/test_gguf_format.py b/test/test_cuda/export/test_gguf_format.pyindex 1e413970f..fb4069048 100644--- a/test/test_cuda/export/test_gguf_format.py+++ b/test/test_cuda/export/test_gguf_format.py@@ -132,7 +132,7 @@ class TestAutoRound:disable_opt_rtn=True,)quantized_model_path = self.save_dir- autoround.quantize_and_save(output_dir=quantized_model_path, format="gguf:q4_0")+ _, quantized_model_path = autoround.quantize_and_save(output_dir=quantized_model_path, format="gguf:q4_0")file_name = os.listdir(quantized_model_path)[0]file_size = os.path.getsize(os.path.join(quantized_model_path, file_name)) / 1024**2assert abs(file_size - 307) < 5.0@@ -169,11 +169,11 @@ class TestAutoRound:quant_nontext_module=True,)quantized_model_path = self.save_dir- autoround.quantize_and_save(output_dir=quantized_model_path, format="gguf:q4_k_m")- assert "mmproj-model.gguf" in os.listdir(self.save_dir)- for file in os.listdir(self.save_dir):- print(f"{file}: {os.path.getsize(os.path.join(self.save_dir, file)) / 1024**2} MB")- file_size = os.path.getsize(os.path.join(self.save_dir, file)) / 1024**2+ _, quantized_model_path = autoround.quantize_and_save(output_dir=quantized_model_path, format="gguf:q4_k_m")+ assert "mmproj-model.gguf" in os.listdir(quantized_model_path)+ for file in os.listdir(quantized_model_path):+ print(f"{file}: {os.path.getsize(os.path.join(quantized_model_path, file)) / 1024**2} MB")+ file_size = os.path.getsize(os.path.join(quantized_model_path, file)) / 1024**2if "mmproj-model.gguf" in file:assert abs(file_size - 75) < 5.0else:@@ -198,7 +198,7 @@ class TestAutoRound:disable_opt_rtn=True,)quantized_model_path = self.save_dir- autoround.quantize_and_save(output_dir=quantized_model_path, format="gguf:q2_k_mixed")+ _, quantized_model_path = autoround.quantize_and_save(output_dir=quantized_model_path, format="gguf:q2_k_mixed")gguf_file = os.listdir(quantized_model_path)[0]file_size = os.path.getsize(os.path.join(quantized_model_path, gguf_file)) / 1024**2assert abs(file_size - 1236) < 5.0@@ -229,7 +229,7 @@ class TestAutoRound:disable_opt_rtn=True,)quantized_model_path = self.save_dir- autoround.quantize_and_save(output_dir=quantized_model_path, format="gguf:q2_k_s")+ _, quantized_model_path = autoround.quantize_and_save(output_dir=quantized_model_path, format="gguf:q2_k_s")gguf_file = os.listdir(quantized_model_path)[0]gguf_model = GGUFReader(os.path.join(quantized_model_path, gguf_file))ffn_down_type = None@@ -258,5 +258,7 @@ class TestAutoRound:disable_opt_rtn=True,)quantized_model_path = self.save_dir- autoround.quantize_and_save(output_dir=quantized_model_path, inplace=False, format="fake")+ _, quantized_model_path = autoround.quantize_and_save(+ output_dir=quantized_model_path, inplace=False, format="fake"+ )eval_generated_prompt(quantized_model_path)diff --git a/test/test_cuda/export/test_llmc_format.py b/test/test_cuda/export/test_llmc_format.pyindex 232bbb94b..e15223f83 100644--- a/test/test_cuda/export/test_llmc_format.py+++ b/test/test_cuda/export/test_llmc_format.py@@ -7,8 +7,11 @@ import transformersfrom auto_round import AutoRoundfrom auto_round import schemes as ar_schemes+from ...envs import is_compressed_tensors_availablefrom ...helpers import eval_generated_prompt, get_model_path, is_cuda_support_fp8+pytestmark = pytest.mark.skipif(not is_compressed_tensors_available(), reason="test requires compressed-tensors")+class TestAutoRound:@@ -41,7 +44,7 @@ class TestAutoRound:dataset=dataloader,)print(ar.model)- compressed_model, _ = ar.quantize_and_save(output_dir=self.save_dir, format="llm_compressor")+ compressed_model, quantized_model_path = ar.quantize_and_save(output_dir=self.save_dir, format="llm_compressor")tmp_layer = compressed_model.model.layers[1].self_attn.q_projassert (hasattr(tmp_layer, "weight_scale")@@ -50,7 +53,7 @@ class TestAutoRound:and tmp_layer.weight_scale.shape[0] == 2048), "Illegal MXFP4 packing name or data_type or shape"quantization_config = transformers.AutoConfig.from_pretrained(- self.save_dir, trust_remote_code=True+ quantized_model_path, trust_remote_code=True).quantization_configassert (quantization_config["format"] == "mxfp4-pack-quantized"@@ -68,7 +71,9 @@ class TestAutoRound:dataset=dataloader,)quantized_model_path = self.save_dir- compressed_model, _ = autoround.quantize_and_save(output_dir=quantized_model_path, format="llm_compressor")+ compressed_model, quantized_model_path = autoround.quantize_and_save(+ output_dir=quantized_model_path, format="llm_compressor"+ )tmp_layer = compressed_model.model.decoder.layers[1].self_attn.q_projassert (hasattr(tmp_layer, "weight_scale")@@ -98,7 +103,9 @@ class TestAutoRound:disable_opt_rtn=True,)quantized_model_path = self.save_dir- compressed_model, _ = autoround.quantize_and_save(output_dir=quantized_model_path, format="llm_compressor")+ compressed_model, quantized_model_path = autoround.quantize_and_save(+ output_dir=quantized_model_path, format="llm_compressor"+ )tmp_layer = compressed_model.model.layers[1].self_attn.q_projassert hasattr(tmp_layer, "weight_scale")assert tmp_layer.weight.dtype is torch.float8_e4m3fndiff --git a/test/test_cuda/integrations/test_sglang.py b/test/test_cuda/integrations/test_sglang.pyindex 2bee14bd5..196fa2efa 100644--- a/test/test_cuda/integrations/test_sglang.py+++ b/test/test_cuda/integrations/test_sglang.py@@ -10,11 +10,11 @@ import torchfrom auto_round import AutoRound-from ...helpers import get_model_path, opt_name_or_path+from ...helpers import get_model_path, qwen_name_or_pathclass TestAutoRound:- model_name = opt_name_or_path+ model_name = qwen_name_or_path@pytest.fixture(autouse=True)def _save_dir(self, tmp_path):@@ -35,6 +35,20 @@ class TestAutoRound:shutil.rmtree("runs", ignore_errors=True)def _run_sglang_inference(self, model_path: Path):+ # SM 12.x (Blackwell) GPUs require CUDA >= 12.9 for sglang's gptq_marlin_repack JIT kernel.+ # Skip inference when the environment is known to be incompatible.+ if torch.cuda.is_available():+ try:+ major, minor = torch.cuda.get_device_capability()+ if major >= 12:+ cuda_ver = tuple(int(x) for x in (torch.version.cuda or "0.0").split(".")[:2])+ if cuda_ver < (12, 9):+ pytest.skip(+ f"SM {major}.{minor} GPU requires CUDA >= 12.9 for sglang GPTQ JIT kernels "+ f"(installed: CUDA {torch.version.cuda})"+ )+ except Exception:+ passllm = sgl.Engine(model_path=str(model_path), mem_fraction_static=0.5, disable_piecewise_cuda_graph=True, cuda_graph_bs=[1])@@ -58,13 +72,13 @@ class TestAutoRound:dataset=dataloader,)- autoround.quantize_and_save(+ _, quantized_model_path = autoround.quantize_and_save(output_dir=self.save_dir,inplace=True,format="auto_round",)- generated_text = self._run_sglang_inference(self.save_dir)+ generated_text = self._run_sglang_inference(quantized_model_path)print(generated_text)assert "!!!" not in generated_text@@ -73,7 +87,7 @@ class TestAutoRound:layer_config = {"self_attn": {"bits": 8},"lm_head": {"bits": 16},- "fc1": {"bits": 16, "act_bits": 16},+ "mlp": {"bits": 16, "act_bits": 16},}autoround = AutoRound(@@ -85,22 +99,22 @@ class TestAutoRound:layer_config=layer_config,)- autoround.quantize_and_save(+ _, quantized_model_path = autoround.quantize_and_save(output_dir=self.save_dir,inplace=True,format="auto_round",)- config_file = Path(self.save_dir) / "config.json"+ config_file = Path(quantized_model_path) / "config.json"with open(config_file, "r", encoding="utf-8") as f:config = json.load(f)quant_config = config.get("quantization_config", {})extra_config = quant_config.get("extra_config", {})# check extra_config only saved attributes differing from Scheme values- assert "act_bits" not in extra_config[".*fc1.*"].keys()- assert "group_size" not in extra_config[".*fc1.*"].keys()- assert "bits" in extra_config[".*fc1.*"].keys() and extra_config[".*fc1.*"]["bits"] == 16+ assert "act_bits" not in extra_config[".*mlp.*"].keys()+ assert "group_size" not in extra_config[".*mlp.*"].keys()+ assert "bits" in extra_config[".*mlp.*"].keys() and extra_config[".*mlp.*"]["bits"] == 16assert "bits" in extra_config[".*self_attn.*"].keys() and extra_config[".*self_attn.*"]["bits"] == 8- generated_text = self._run_sglang_inference(self.save_dir)+ generated_text = self._run_sglang_inference(quantized_model_path)print(generated_text)assert "!!!" not in generated_text@@ -117,13 +131,13 @@ class TestAutoRound:dataset=dataloader,)- autoround.quantize_and_save(+ _, quantized_model_path = autoround.quantize_and_save(output_dir=self.save_dir,inplace=True,format="auto_round:auto_awq",)- generated_text = self._run_sglang_inference(self.save_dir)+ generated_text = self._run_sglang_inference(quantized_model_path)print(generated_text)assert "!!!" not in generated_textdiff --git a/test/test_cuda/integrations/test_vllm.py b/test/test_cuda/integrations/test_vllm.pyindex 3cef719e5..e9faa3e14 100644--- a/test/test_cuda/integrations/test_vllm.py+++ b/test/test_cuda/integrations/test_vllm.py@@ -21,6 +21,31 @@ from ...helpers import get_model_pathos.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"++def _is_sm12_with_old_cuda() -> bool:+ """Return True when the GPU is SM 12.x (Blackwell) and CUDA < 12.9.++ vLLM's gptq_marlin JIT kernels require CUDA >= 12.9 on SM 12.x devices.+ """+ try:+ import torch++ if not torch.cuda.is_available():+ return False+ major, _ = torch.cuda.get_device_capability()+ if major < 12:+ return False+ cuda_ver = tuple(int(x) for x in (torch.version.cuda or "0.0").split(".")[:2])+ return cuda_ver < (12, 9)+ except Exception:+ return False+++pytestmark = pytest.mark.skipif(+ _is_sm12_with_old_cuda(),+ reason="SM 12.x (Blackwell) GPU requires CUDA >= 12.9 for vLLM GPTQ marlin JIT kernels",+)+MODELS = ["OPEA/Qwen2.5-0.5B-Instruct-int4-sym-inc", ##auto_round:auto_gptq"Intel/Qwen2-0.5B-Instruct-int4-sym-AutoRound", ##auto_round:auto_awq@@ -100,7 +125,7 @@ def test_mixed_llmcompressor_format_vllm(tiny_opt_model_path, dataloader, tmp_palayer_config=layer_config,)quantized_model_path = str(tmp_path / "saved")- autoround.quantize_and_save(output_dir=quantized_model_path, format="llm_compressor")+ _, quantized_model_path = autoround.quantize_and_save(output_dir=quantized_model_path, format="llm_compressor")# verify loading.llm = LLM(@@ -165,9 +190,9 @@ def test_auto_round_awq_format_vllm():iters=1,seqlen=2,)- autoround.quantize_and_save(output_dir=save_dir, format="auto_round:auto_awq")+ _, quantized_model_path = autoround.quantize_and_save(output_dir=save_dir, format="auto_round:auto_awq")sampling_params = SamplingParams(temperature=0.8, top_p=0.95, max_tokens=32)- llm = LLM(model=save_dir, trust_remote_code=True, tensor_parallel_size=1, gpu_memory_utilization=0.7)+ llm = LLM(model=quantized_model_path, trust_remote_code=True, tensor_parallel_size=1, gpu_memory_utilization=0.7)outputs = llm.generate(["The capital of France is"], sampling_params)generated_text = outputs[0].outputs[0].textprint(generated_text)diff --git a/test/test_cuda/models/test_fp8_model.py b/test/test_cuda/models/test_fp8_model.pyindex e8343b0b2..0bbbba1f8 100644--- a/test/test_cuda/models/test_fp8_model.py+++ b/test/test_cuda/models/test_fp8_model.py@@ -32,14 +32,14 @@ class TestAutoRound:def test_small_model_rtn_generation(self, mock_fp8_capable_device, tiny_fp8_qwen_model_path):ar = AutoRound(tiny_fp8_qwen_model_path, iters=0, disable_opt_rtn=True)- ar.quantize_and_save(output_dir=self.save_dir)- model = AutoModelForCausalLM.from_pretrained(self.save_dir, torch_dtype="auto", trust_remote_code=True)- tokenizer = AutoTokenizer.from_pretrained(self.save_dir)+ _, quantized_model_path = ar.quantize_and_save(output_dir=self.save_dir)+ model = AutoModelForCausalLM.from_pretrained(quantized_model_path, torch_dtype="auto", trust_remote_code=True)+ tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)generate_prompt(model, tokenizer)def test_gguf_imatrix(self, mock_fp8_capable_device, tiny_fp8_qwen_model_path):ar = AutoRound(tiny_fp8_qwen_model_path, iters=0)- ar.quantize_and_save(format="gguf:q2_k_s", output_dir=self.save_dir)+ _, quantized_model_path = ar.quantize_and_save(format="gguf:q2_k_s", output_dir=self.save_dir)# from llama_cpp import Llama## gguf_file = os.listdir("saved/Qwen3-0.6B-FP8/-gguf")[0]@@ -47,8 +47,8 @@ class TestAutoRound:# output = llm("There is a girl who likes adventure,", max_tokens=32)# print(output)# shutil.rmtree("./saved", ignore_errors=True)- # model = AutoModelForCausalLM.from_pretrained(self.save_dir, torch_dtype="auto", trust_remote_code=True)- # tokenizer = AutoTokenizer.from_pretrained(self.save_dir)+ # model = AutoModelForCausalLM.from_pretrained(quantized_model_path, torch_dtype="auto", trust_remote_code=True)+ # tokenizer = AutoTokenizer.from_pretrained(quantized_model_path )# text = "There is a girl who likes adventure,"# inputs = tokenizer(text, return_tensors="pt").to(model.device)# print(tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0]))@@ -58,21 +58,21 @@ class TestAutoRound:model_name = get_model_path("Qwen/Qwen3-0.6B-FP8")ar = AutoRound(model=model_name, iters=0)_, folder = ar.quantize_and_save(output_dir=self.save_dir)- evaluate_accuracy(self.save_dir, threshold=0.25)+ evaluate_accuracy(folder, threshold=0.25)@pytest.mark.skip_ci(reason="Triton issue; time-consuming")def test_small_model_iters1(self, mock_fp8_capable_device):model_name = get_model_path("Qwen/Qwen3-0.6B-FP8")ar = AutoRound(model=model_name, iters=1)_, folder = ar.quantize_and_save(output_dir=self.save_dir)- evaluate_accuracy(self.save_dir, threshold=0.25)+ evaluate_accuracy(folder, threshold=0.25)@pytest.mark.skip_ci(reason="Triton issue; time-consuming")def test_medium_model_rtn(self, mock_fp8_capable_device):model_name = get_model_path("Qwen/Qwen3-0.6B-FP8")ar = AutoRound(model=model_name, iters=0)_, folder = ar.quantize_and_save(output_dir=self.save_dir)- evaluate_accuracy(self.save_dir, threshold=0.33)+ evaluate_accuracy(folder, threshold=0.33)@pytest.mark.skip_ci(reason="Triton issue; time-consuming")def test_medium_model_rtn_with_lm_head(self, mock_fp8_capable_device):@@ -80,17 +80,17 @@ class TestAutoRound:layer_config = {"lm_head": {"bits": 4}}ar = AutoRound(model=model_name, iters=0, layer_config=layer_config)_, folder = ar.quantize_and_save(output_dir=self.save_dir)- evaluate_accuracy(self.save_dir, threshold=0.33)+ evaluate_accuracy(folder, threshold=0.33)def test_fp8_model_gguf_q4(self, mock_fp8_capable_device, tiny_fp8_qwen_model_path):from llama_cpp import Llamaar = AutoRound(tiny_fp8_qwen_model_path, iters=0, disable_opt_rtn=True)- ar.quantize_and_save(output_dir=self.save_dir, format="gguf:q4_0")- for file in os.listdir(self.save_dir):+ _, quantized_model_path = ar.quantize_and_save(output_dir=self.save_dir, format="gguf:q4_0")+ for file in os.listdir(quantized_model_path):if file.endswith(".gguf"):gguf_file = file- llm = Llama(f"{self.save_dir}/{gguf_file}", n_gpu_layers=-1)+ llm = Llama(f"{quantized_model_path}/{gguf_file}", n_gpu_layers=-1)output = llm("There is a girl who likes adventure,", max_tokens=32)print(output)@@ -99,11 +99,11 @@ class TestAutoRound:from llama_cpp import Llamaar = AutoRound(tiny_fp8_qwen_model_path, iters=1)- ar.quantize_and_save(output_dir=self.save_dir, format="gguf:q3_k_s")- for file in os.listdir(self.save_dir):+ _, quantized_model_path = ar.quantize_and_save(output_dir=self.save_dir, format="gguf:q3_k_s")+ for file in os.listdir(quantized_model_path):if file.endswith(".gguf"):gguf_file = file- llm = Llama(f"{self.save_dir}/{gguf_file}", n_gpu_layers=-1)+ llm = Llama(f"{quantized_model_path}/{gguf_file}", n_gpu_layers=-1)output = llm("There is a girl who likes adventure,", max_tokens=32)print(output)@@ -113,8 +113,8 @@ class TestAutoRound:model_name = tiny_fp8_qwen_model_pathprint(f"Testing scheme: {scheme}")ar = AutoRound(model_name, iters=0, scheme=scheme, disable_opt_rtn=True, nsamples=2)- ar.quantize_and_save(output_dir=self.save_dir)- model = AutoModelForCausalLM.from_pretrained(self.save_dir, torch_dtype="auto", trust_remote_code=True)+ _, quantized_model_path = ar.quantize_and_save(output_dir=self.save_dir)+ model = AutoModelForCausalLM.from_pretrained(quantized_model_path, torch_dtype="auto", trust_remote_code=True)assert model is not None, f"Failed to load model for scheme {scheme}"@@ -128,9 +128,9 @@ def test_qwen3_fp8_moe_mxfp(tiny_fp8_qwen_moe_model_path, mock_fp8_capable_deviciters=0,low_cpu_mem_usage=False,)- quantized_model, _ = autoround.quantize_and_save(format="auto_round", output_dir=output_dir)+ quantized_model, quantized_model_path = autoround.quantize_and_save(format="auto_round", output_dir=output_dir)assert quantized_model is not None, "Quantized model should not be None."- loaded_model = AutoModelForCausalLM.from_pretrained(output_dir)+ loaded_model = AutoModelForCausalLM.from_pretrained(quantized_model_path)for n, m in quantized_model.named_modules():if m.__class__.__name__ == "QuantLinear":loaded_m = loaded_model.get_submodule(n)diff --git a/test/test_cuda/models/test_mllm.py b/test/test_cuda/models/test_mllm.pyindex 52d09175c..ba79f77f8 100644--- a/test/test_cuda/models/test_mllm.py+++ b/test/test_cuda/models/test_mllm.py@@ -8,7 +8,7 @@ import requestsfrom PIL import Imagefrom transformers import AutoModelForImageTextToText, AutoProcessor, AutoTokenizer, Qwen2VLForConditionalGeneration-from auto_round import AutoRoundMLLM+from auto_round import AutoRoundfrom auto_round.utils import get_block_namesfrom ...envs import require_gptqmodel, require_optimum, require_vlm_env@@ -16,6 +16,7 @@ from ...helpers import get_model_pathclass VisionDataLoader:+def __init__(self):self.batch_size = 1@@ -37,7 +38,8 @@ class VisionDataLoader:@pytest.mark.skip_ci(reason="Only tiny model is suggested")-class TestAutoRoundMLLM:+class TestAutoRound:+@pytest.fixture(autouse=True)def _save_dir(self, tmp_path):self.save_dir = str(tmp_path / "saved")@@ -112,13 +114,13 @@ class TestAutoRoundMLLM:@require_gptqmodel@require_optimumdef test_vlm_tune(self):- from auto_round import AutoRoundMLLM+ from auto_round import AutoRound## load the modelmodel_name = get_model_path("Qwen/Qwen2-VL-2B-Instruct")## quantize the modelbits, group_size, sym = 4, 128, True- autoround = AutoRoundMLLM(model_name, bits=bits, group_size=group_size, sym=sym, iters=1, nsamples=1)+ autoround = AutoRound(model_name, bits=bits, group_size=group_size, sym=sym, iters=1, nsamples=1)autoround.quantize()quantized_model_path = self.save_dir@@ -181,7 +183,7 @@ class TestAutoRoundMLLM:model_path, trust_remote_code=True, device_map="auto", torch_dtype="auto")- autoround = AutoRoundMLLM(+ autoround = AutoRound(model=model,tokenizer=tokenizer,processor=processor,diff --git a/test/test_cuda/models/test_moe_model.py b/test/test_cuda/models/test_moe_model.pyindex 95c61bd55..8ba5e871d 100644--- a/test/test_cuda/models/test_moe_model.py+++ b/test/test_cuda/models/test_moe_model.py@@ -22,10 +22,10 @@ def test_qwen3_5_moe(tiny_qwen35_moe_model_path):seqlen=32,iters=1,)- quantized_model, _ = ar.quantize_and_save(format="auto_round", output_dir=output_dir)+ quantized_model, quantized_model_path = ar.quantize_and_save(format="auto_round", output_dir=output_dir)assert quantized_model is not None, "Quantized model should not be None."- loaded_model = Qwen3_5MoeForConditionalGeneration.from_pretrained(output_dir)+ loaded_model = Qwen3_5MoeForConditionalGeneration.from_pretrained(quantized_model_path)loaded_model.to("cuda")inp = torch.randint(0, 100, (1, 64)).to("cuda")diff --git a/test/test_cuda/models/test_omni_model.py b/test/test_cuda/models/test_omni_model.pyindex b4ccdb81c..d74a4e529 100644--- a/test/test_cuda/models/test_omni_model.py+++ b/test/test_cuda/models/test_omni_model.py@@ -73,10 +73,10 @@ class TestQwen2_5OmniQuantization:for extra_file in ["spk_dict.pt"]:src = os.path.join(tiny_qwen2_5_omni_model_path, extra_file)if os.path.exists(src):- shutil.copy2(src, tmp_path)+ shutil.copy2(src, save_folder)# Reload- loaded_model = Qwen2_5OmniForConditionalGeneration.from_pretrained(tmp_path, device_map="cuda")+ loaded_model = Qwen2_5OmniForConditionalGeneration.from_pretrained(save_folder, device_map="cuda")# Run inference on thinkerinp = torch.randint(0, 100, (1, 64)).to("cuda")@@ -111,7 +111,7 @@ class TestQwen3OmniMoeQuantization:assert quantized_model is not None, "Quantized model should not be None"# Reload- loaded_model = Qwen3OmniMoeForConditionalGeneration.from_pretrained(self.save_dir, device_map="cuda")+ loaded_model = Qwen3OmniMoeForConditionalGeneration.from_pretrained(save_folder, device_map="cuda")# Run inference on thinkerinp = torch.randint(0, 100, (1, 64)).to("cuda")@@ -134,7 +134,7 @@ class TestQwen3OmniMoeQuantization:assert quantized_model is not None, "MXFP4 quantized model should not be None"# Reload and inference- loaded_model = Qwen3OmniMoeForConditionalGeneration.from_pretrained(self.save_dir, device_map="cuda")+ loaded_model = Qwen3OmniMoeForConditionalGeneration.from_pretrained(save_folder, device_map="cuda")inp = torch.randint(0, 100, (1, 64)).to("cuda")with torch.inference_mode():diff --git a/test/test_cuda/quantization/test_asym.py b/test/test_cuda/quantization/test_asym.pyindex 5145cf13d..97ac64288 100644--- a/test/test_cuda/quantization/test_asym.py+++ b/test/test_cuda/quantization/test_asym.py@@ -29,15 +29,15 @@ class TestAutoRoundAsym:def test_asym_group_size_with_tuning(self, group_size, tiny_opt_model_path):bits, sym = 4, Falsear = AutoRound(tiny_opt_model_path, bits=bits, group_size=group_size, sym=sym, iters=1, seqlen=2, nsamples=1)- ar.quantize_and_save(format="auto_round", output_dir=self.save_dir)+ _, quantized_model_path = ar.quantize_and_save(format="auto_round", output_dir=self.save_dir)model = AutoModelForCausalLM.from_pretrained(- self.save_dir,+ quantized_model_path,torch_dtype="auto",device_map="auto",)- tokenizer = AutoTokenizer.from_pretrained(self.save_dir)+ tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)model_infer(model, tokenizer)@pytest.mark.skip_ci(reason="Not necessary since it's covered by backend tests") # skip this test in CI@@ -45,15 +45,15 @@ class TestAutoRoundAsym:def test_asym_bits_with_tuning(self, bits, tiny_opt_model_path):group_size, sym = 128, Falsear = AutoRound(tiny_opt_model_path, bits=bits, group_size=group_size, sym=sym, iters=1, seqlen=2, nsamples=1)- ar.quantize_and_save(format="auto_round", output_dir=self.save_dir)+ _, quantized_model_path = ar.quantize_and_save(format="auto_round", output_dir=self.save_dir)model = AutoModelForCausalLM.from_pretrained(- self.save_dir,+ quantized_model_path,torch_dtype="auto",device_map="auto",)- tokenizer = AutoTokenizer.from_pretrained(self.save_dir)+ tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)model_infer(model, tokenizer)@pytest.mark.skip_ci(reason="Not necessary since it's covered by backend tests") # skip this test in CI@@ -61,17 +61,17 @@ class TestAutoRoundAsym:def test_asym_format_with_tuning(self, format, tiny_opt_model_path):bits, group_size, sym = 4, 128, Falsear = AutoRound(tiny_opt_model_path, bits=bits, group_size=group_size, sym=sym, iters=1, seqlen=2, nsamples=1)- ar.quantize_and_save(format=format, output_dir=self.save_dir)+ _, quantized_model_path = ar.quantize_and_save(format=format, output_dir=self.save_dir)if format == "auto_round:auto_gptq":# Cannot load correctly, skip auto_gptq since it's deprecated.returnmodel = AutoModelForCausalLM.from_pretrained(- self.save_dir,+ quantized_model_path,torch_dtype="auto",device_map="auto",)- tokenizer = AutoTokenizer.from_pretrained(self.save_dir)+ tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)model_infer(model, tokenizer)diff --git a/test/test_cuda/quantization/test_mxfp_nvfp.py b/test/test_cuda/quantization/test_mxfp_nvfp.pyindex 3e40e7288..bdc79075e 100644--- a/test/test_cuda/quantization/test_mxfp_nvfp.py+++ b/test/test_cuda/quantization/test_mxfp_nvfp.py@@ -55,7 +55,7 @@ def test_e2e_quant_and_infer(scheme, tiny_qwen_model_path):# Quantize and save the model to the temporary directoryquantized_model_path = f"{temp_dir}/tmp_autoround_{scheme}"- autoround.quantize_and_save(format="auto_round", output_dir=quantized_model_path)+ _, quantized_model_path = autoround.quantize_and_save(format="auto_round", output_dir=quantized_model_path)# Perform inference with the quantized modelmodel = AutoModelForCausalLM.from_pretrained(@@ -144,7 +144,9 @@ class TestAutoRound:layer_config=layer_config,)quantized_model_path = self.save_dir- autoround.quantize_and_save(output_dir=quantized_model_path, inplace=False, format="auto_round")+ _, quantized_model_path = autoround.quantize_and_save(+ output_dir=quantized_model_path, inplace=False, format="auto_round"+ )model = AutoModelForCausalLM.from_pretrained(quantized_model_path, torch_dtype="auto", device_map="auto")tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)from ...helpers import evaluate_accuracydiff --git a/test/test_cuda/quantization/test_torch_compile.py b/test/test_cuda/quantization/test_torch_compile.pyindex 620ef75b0..e7efe0bf2 100644--- a/test/test_cuda/quantization/test_torch_compile.py+++ b/test/test_cuda/quantization/test_torch_compile.py@@ -64,9 +64,9 @@ class TestTorchCompile:seqlen=16,enable_torch_compile=True,)- autoround.quantize_and_save(output_dir=self.save_dir, format="gguf:q2_k_s")+ _, quantized_model_path = autoround.quantize_and_save(output_dir=self.save_dir, format="gguf:q2_k_s")- saved_files = [f for f in os.listdir(self.save_dir) if f.endswith(".gguf")]+ saved_files = [f for f in os.listdir(quantized_model_path) if f.endswith(".gguf")]assert len(saved_files) > 0, "No GGUF file was generated"shutil.rmtree(self.save_dir, ignore_errors=True)diff --git a/test/test_cuda/transform/test_mxfp4_transform.py b/test/test_cuda/transform/test_mxfp4_transform.pyindex 16511edad..5df09db9f 100644--- a/test/test_cuda/transform/test_mxfp4_transform.py+++ b/test/test_cuda/transform/test_mxfp4_transform.py@@ -38,10 +38,10 @@ class TestAutoRound:scheme=scheme,rotation_config="default",)- compressed_model, _ = ar.quantize_and_save(output_dir=self.save_dir, format="auto_round")+ compressed_model, quantized_model_path = ar.quantize_and_save(output_dir=self.save_dir, format="auto_round")- model = AutoModelForCausalLM.from_pretrained(self.save_dir, torch_dtype="auto", device_map="cuda")- tokenizer = AutoTokenizer.from_pretrained(self.save_dir)+ model = AutoModelForCausalLM.from_pretrained(quantized_model_path, torch_dtype="auto", device_map="cuda")+ tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)from ...helpers import generate_promptgenerate_prompt(model, tokenizer)@@ -57,10 +57,10 @@ class TestAutoRound:scheme=scheme,rotation_config="default",)- compressed_model, _ = ar.quantize_and_save(output_dir=self.save_dir, format="auto_round")+ compressed_model, quantized_model_path = ar.quantize_and_save(output_dir=self.save_dir, format="auto_round")- model = AutoModelForCausalLM.from_pretrained(self.save_dir, torch_dtype="auto", device_map="cuda")- tokenizer = AutoTokenizer.from_pretrained(self.save_dir)+ model = AutoModelForCausalLM.from_pretrained(quantized_model_path, torch_dtype="auto", device_map="cuda")+ tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)from ...helpers import generate_promptgenerate_prompt(model, tokenizer)@@ -76,10 +76,10 @@ class TestAutoRound:scheme=scheme,rotation_config="random_hadamard",)- compressed_model, _ = ar.quantize_and_save(output_dir=self.save_dir, format="auto_round")+ compressed_model, quantized_model_path = ar.quantize_and_save(output_dir=self.save_dir, format="auto_round")- model = AutoModelForCausalLM.from_pretrained(self.save_dir, torch_dtype="auto", device_map="cuda")- tokenizer = AutoTokenizer.from_pretrained(self.save_dir)+ model = AutoModelForCausalLM.from_pretrained(quantized_model_path, torch_dtype="auto", device_map="cuda")+ tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)from ...helpers import generate_promptgenerate_prompt(model, tokenizer)diff --git a/test/test_xpu/test_autoround.py b/test/test_xpu/test_autoround.pyindex 4c4c63678..d03c71444 100644--- a/test/test_xpu/test_autoround.py+++ b/test/test_xpu/test_autoround.py@@ -12,6 +12,7 @@ from ..helpers import get_model_pathclass TestAutoRoundXPU:+@classmethoddef setup_class(self):self.device = "xpu"@@ -47,7 +48,7 @@ class TestAutoRoundXPU:dataset=dataloader,)quantized_model_path = self.save_dir- autoround.quantize_and_save(output_dir=quantized_model_path)+ _, quantized_model_path = autoround.quantize_and_save(output_dir=quantized_model_path)quantization_config = AutoRoundConfig(backend="auto")model = AutoModelForCausalLM.from_pretrained(@@ -78,7 +79,9 @@ class TestAutoRoundXPU:dataset=dataloader,)quantized_model_path = self.save_dir- autoround.quantize_and_save(output_dir=quantized_model_path, format="auto_round:auto_awq")+ _, quantized_model_path = autoround.quantize_and_save(+ output_dir=quantized_model_path, format="auto_round:auto_awq"+ )quantization_config = AutoRoundConfig(backend="auto")model = AutoModelForCausalLM.from_pretrained(@@ -109,7 +112,9 @@ class TestAutoRoundXPU:dataset=dataloader,)quantized_model_path = "./saved"- ar.quantize_and_save(output_dir=quantized_model_path, inplace=True, format="auto_round")+ _, quantized_model_path = ar.quantize_and_save(+ output_dir=quantized_model_path, inplace=True, format="auto_round"+ )# test loadingif scheme not in ["FPW8A16"]: # FPW8A16 group_size is 0@@ -140,7 +145,9 @@ class TestAutoRoundXPU:)quantized_model_path = "./saved"- ar.quantize_and_save(output_dir=quantized_model_path, inplace=True, format="auto_round")+ _, quantized_model_path = ar.quantize_and_save(+ output_dir=quantized_model_path, inplace=True, format="auto_round"+ )quantization_config = AutoRoundConfig(backend="auto")import requests@@ -211,7 +218,9 @@ class TestAutoRoundXPU:dataset=dataloader,)quantized_model_path = "./saved"- ar.quantize_and_save(output_dir=quantized_model_path, inplace=True, format="auto_round")+ _, quantized_model_path = ar.quantize_and_save(+ output_dir=quantized_model_path, inplace=True, format="auto_round"+ )quantization_config = AutoRoundConfig(backend="auto")model = AutoModelForCausalLM.from_pretrained(