PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
refactor and support for multi algs fusion (#1920)
Heng Guodf9299dacec4
diff --git a/.claude/skills/adapt-new-diffusion-model/SKILL.md b/.claude/skills/adapt-new-diffusion-model/SKILL.mdindex 89b36c274..30eec15c9 100644--- a/.claude/skills/adapt-new-diffusion-model/SKILL.md+++ b/.claude/skills/adapt-new-diffusion-model/SKILL.md@@ -53,14 +53,12 @@ def is_diffusion_model(model_or_path):If your model doesn't have `model_index.json`, either create one in the modeldirectory or pass diffusion-specific options through new-architecture-`ExtraConfig` / `AutoRound` kwargs:+`AutoRound` kwargs:```python-from auto_round.compressors.config import ExtraConfig-ar = AutoRound(model,- extra_config=ExtraConfig(num_inference_steps=5),+ num_inference_steps=5,)```@@ -300,7 +298,7 @@ ar = AutoRound(## Checklist-- [ ] `is_diffusion_model()` detects model (or forced via extra_config)+- [ ] `is_diffusion_model()` detects model- [ ] Transformer block class name identified- [ ] `DIFFUSION_OUTPUT_CONFIGS` entry added with correct output tensor names and order- [ ] Pipeline runs without errors during calibrationdiff --git a/.claude/skills/add-vlm-model/SKILL.md b/.claude/skills/add-vlm-model/SKILL.mdindex 8d91c2af2..7b26fc9fa 100644--- a/.claude/skills/add-vlm-model/SKILL.md+++ b/.claude/skills/add-vlm-model/SKILL.md@@ -103,8 +103,8 @@ The new architecture routes multimodal calibration through:- `auto_round/special_model_handler.py` for multimodal block discovery and special forwardsIf your model works with an existing template/processor, prefer passing-`template=...`, `processor=...`, or `image_processor=...` through `AutoRound` /-`ExtraConfig` instead of adding compressor code.+`template=...`, `processor=...`, or `image_processor=...` directly through+`AutoRound` kwargs instead of adding compressor code.## Step 3: Add Calibration Templatediff --git a/auto_round/__init__.py b/auto_round/__init__.pyindex 509885b33..d1eb92d06 100644--- a/auto_round/__init__.py+++ b/auto_round/__init__.py@@ -15,6 +15,15 @@ from auto_round.autoround import AutoRound# support for old apifrom auto_round.autoround import AutoRoundLLM, AutoRoundMLLM, AutoRoundAdam, AutoRoundDiffusion+from auto_round.algorithms.quantization.rtn.config import OptimizedRTNConfig, RTNConfig+from auto_round.algorithms.quantization.sign_round.config import (+ AdamRoundConfig,+ SignRoundConfig,+ SignRoundV2Config,+)+from auto_round.algorithms.transforms.awq.config import AWQConfig+from auto_round.algorithms.transforms.quarot.config import RotationConfig+from auto_round.algorithms.transforms.spinquant.preprocessor import SpinQuantConfigfrom auto_round.schemes import QuantizationSchemefrom auto_round.auto_scheme import AutoSchemefrom auto_round.utils import LazyImport@@ -23,3 +32,22 @@ from auto_round.utils import monkey_patchmonkey_patch()from .version import __version__++__all__ = [+ "__version__",+ "AutoRound",+ "AutoRoundLLM",+ "AutoRoundMLLM",+ "AutoRoundAdam",+ "AutoRoundDiffusion",+ "AutoScheme",+ "QuantizationScheme",+ "RTNConfig",+ "OptimizedRTNConfig",+ "SignRoundConfig",+ "AdamRoundConfig",+ "SignRoundV2Config",+ "AWQConfig",+ "RotationConfig",+ "SpinQuantConfig",+]diff --git a/auto_round/__main__.py b/auto_round/__main__.pyindex 70992f5df..914f5424d 100644--- a/auto_round/__main__.py+++ b/auto_round/__main__.py@@ -11,884 +11,10 @@# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.# See the License for the specific language governing permissions and# limitations under the License.-import argparse-import json-import os-import re-import sys--import torch--from auto_round.auto_scheme import AutoScheme-from auto_round.compressors.base import BaseCompressor-from auto_round.eval.eval_cli import EvalArgumentParser, eval, eval_task_by_task-from auto_round.eval.evaluation import run_model_evaluation-from auto_round.schemes import PRESET_SCHEMES, preset_name_to_scheme-from auto_round.utils import (- clear_memory,- get_device_and_parallelism,- get_model_dtype,- parse_layer_config_arg,-)--RECIPES = {- "default": {"batch_size": 8, "iters": 200, "seqlen": 2048, "nsamples": 128, "lr": None},- "best": {"batch_size": 8, "iters": 1000, "seqlen": 2048, "nsamples": 512, "lr": None},- "light": {"batch_size": 8, "iters": 50, "seqlen": 2048, "nsamples": 128, "lr": 5e-3},- "rtn": {"batch_size": 8, "iters": 0, "seqlen": 2048, "nsamples": 1, "lr": None, "disable_opt_rtn": True},- "opt_rtn": {"batch_size": 8, "iters": 0, "seqlen": 2048, "nsamples": 128, "lr": None, "disable_opt_rtn": False},-}---class BasicArgumentParser(argparse.ArgumentParser):-- def __init__(self, *args, **kwargs):- super().__init__(*args, **kwargs)- self.add_argument(- "model",- default=None,- nargs="?",- help="Path to the pre-trained model or model identifier from huggingface.co/models. "- "Examples: 'facebook/opt-125m', 'bert-base-uncased', or local path like '/path/to/model'",- )- basic = self.add_argument_group("Basic Arguments")- basic.add_argument(- "--model_name",- "--model",- "--model_name_or_path",- default="facebook/opt-125m",- help="Path to the pre-trained model or model identifier from huggingface.co/models. "- "Examples: 'facebook/opt-125m', 'bert-base-uncased', or local path like '/path/to/model'",- )- basic.add_argument("--model_dtype", default=None, help="model dtype used to load the pre-trained model")- basic.add_argument(- "--platform",- default="hf",- help="Platform to load the pre-trained model. Options: [hf, model_scope]."- " hf stands for huggingface and model_scope stands for model scope.",- )- basic.add_argument(- "--scheme",- default="W4A16",- type=str,- # choices=["W4A16", "W2A16", "W3A16", "W8A16", "MXFP4", "MXFP8", "NVFP4", "FPW8A16", "FP8_STATIC"],- help="Quantization scheme to use. "- "W4A16: 4-bit weights with 16-bit activations (default). "- "Other options include W2A16, W3A16, W8A16, W8A8 for different bit widths, "- "and MXFP4/MXFP8/NVFP4 for different data type.",- )- basic.add_argument(- "--algorithm",- default=None,- type=str.lower,- choices=["auto_round", "rtn", "awq"],- help="Quantization algorithm to use. "- "auto_round: SignSGD-based optimization (default when iters > 0). "- "rtn: Round-to-nearest (default when iters == 0). "- "awq: Activation-Aware Weight Quantization (AWQ smoothing + RTN).",- )- basic.add_argument(- "--batch_size",- "--train_bs",- "--bs",- default=None,- type=int,- help="The batch size for tuning/calibration."- "Larger batch sizes may improve stability but require more memory.",- )- basic.add_argument(- "--avg_bits", "--target_bits", default=None, type=float, help="for auto scheme, number of avg weight bits"- )- basic.add_argument(- "--options", default=None, type=str, help="for auto scheme, options for auto scheme, e.g. 'W4A16,W8A16'"- )-- basic.add_argument(- "--iters",- "--iter",- default=None,- type=int,- help="Number of iterations to tune each block. "- "More iterations may lead to better quantization quality but take longer.",- )- basic.add_argument(- "--seqlen",- "--seq_len",- default=None,- type=int,- help="Sequence length of the calibration samples"- "Longer sequences capture more context but use more memory.",- )- basic.add_argument(- "--nsamples",- "--nsample",- default=None,- type=int,- help="Number of calibration samples to use for quantization.",- )- basic.add_argument(- "--device_map",- "--device",- "--devices",- default="0",- type=str,- help="The device to be used for tuning. "- "Currently, device settings support CPU, GPU, and HPU."- "The default is set to cuda:0,"- "allowing for automatic detection and switch to HPU or CPU."- "set --device 0,1,2 to use multiple cards.",- )- basic.add_argument(- "--dataset",- default="NeelNanda/pile-10k",- type=str,- help="Calibration dataset for quantization. "- "Should be a dataset from huggingface datasets or local path. ",- )- basic.add_argument("--seed", default=42, type=int, help="Random seed for reproducibility.")- basic.add_argument("--adam", action="store_true", help="Use Adam optimizer instead of SignSGD.")- basic.add_argument(- "--low_gpu_mem_usage",- action="store_true",- help="Enable memory-efficient mode by offloading intermediate features to CPU. "- "Useful when working with large models that don't fit in GPU memory.",- )- basic.add_argument(- "--low_cpu_mem_usage",- action="store_true",- help=(- "Deprecated: low CPU memory mode is enabled by default. "- "This flag is kept only for backward compatibility and has no effect "- "beyond explicitly re-enabling the default behavior."- ),- )- basic.add_argument(- "--disable_low_cpu_mem_usage",- action="store_true",- help=("Disable low CPU memory mode. " "Use this flag to turn off the default low CPU memory behavior."),- )- basic.add_argument(- "--format",- "--formats",- default="auto_round",- type=str,- help="Output format for the quantized model."- "'auto_round' is the recommended format"- "use command `auto_round list format` to show all supported formats with support scheme.",- )- basic.add_argument(- "--output_dir",- default="./tmp_autoround",- type=str,- help="Directory to save the quantized model and related files",- )- basic.add_argument(- "--not_use_best_mse",- action="store_true",- help="Disable using the iteration with best MSE loss during tuning.",- )- basic.add_argument(- "--enable_torch_compile", action="store_true", help="Enable PyTorch compilation for faster execution. "- )- basic.add_argument(- "--disable_trust_remote_code",- action="store_true",- help="Disable trusting remote code when loading models. "- "Use for security if you don't trust the model source.",- )- tuning = self.add_argument_group("Tuning Arguments")- tuning.add_argument(- "--ignore_scale_zp_bits",- action="store_true",- help="for auto scheme whether ignore scale zp bits calculation ",- )- tuning.add_argument(- "--lr",- default=None,- type=float,- help="Learning rate for tuning. " "If None, automatically sets to 1.0/iters. ",- )- tuning.add_argument(- "--minmax_lr",- default=None,- type=float,- help="Learning rate specifically for min-max tuning. " "If None, uses the same value as --lr. ",- )- tuning.add_argument(- "--momentum",- default=0,- type=float,- help="Momentum factor for the optimizer. Default is 0 (no momentum).",- )- tuning.add_argument(- "--gradient_accumulate_steps",- default=1,- type=int,- help="Number of steps to accumulate gradients before updating weights. "- "Effectively increases batch size without requiring more GPU memory. "- "Useful for large models with limited memory.",- )- tuning.add_argument(- "--nblocks",- default=1,- type=int,- help="Number of blocks to tune simultaneously. "- "Higher values may speed up tuning but require more memory. "- "Recommended to keep at 1 for stability with large models.",- )- tuning.add_argument(- "--scale_dtype",- default=None,- choices=["fp16", "float16", "bf16", "bfloat16", "fp32", "float32"],- help="Data type for quantization scales. "- "fp16/bf16: lower memory, fp32: higher precision. "- "Choose based on your hardware support and accuracy requirements.",- )- tuning.add_argument(- "--disable_amp",- action="store_true",- help="Disable Automatic Mixed Precision (AMP). "- "AMP speeds up training but may affect numerical stability in some cases.",- )- tuning.add_argument(- "--disable_minmax_tuning",- action="store_true",- help="Disable weight min-max range tuning. "- "Not recommended as it may significantly reduce quantization accuracy.",- )- tuning.add_argument(- "--enable_norm_bias_tuning", action="store_true", help="Enable normalization layer bias tuning. "- )- tuning.add_argument(- "--disable_quanted_input",- action="store_true",- help="Use original (non-quantized) inputs for each block instead of"- " quantized outputs from previous blocks. ",- )- tuning.add_argument(- "--to_quant_block_names",- default=None,- type=str,- help="Specific blocks to quantize, separated by commas. "- "Example: 'block1,block2,block3'. "- "If None, all blocks will be quantized.",- )- tuning.add_argument(- "--enable_alg_ext",- action="store_true",- help="Enable experimental algorithms that may provide better quantization results. "- "These are newer methods that might improve accuracy but are less tested.",- )- tuning.add_argument(- "--disable_deterministic_algorithms",- action="store_true",- help="deprecated, disable torch deterministic algorithms.",- )- tuning.add_argument(- "--enable_deterministic_algorithms",- action="store_true",- help="Enable PyTorch deterministic algorithms for reproducible results. ",- )- group_opt_rtn = tuning.add_mutually_exclusive_group()- group_opt_rtn.add_argument(- "--disable_opt_rtn",- action="store_const",- const=True,- dest="disable_opt_rtn",- default=None,- help="Disable optimization for RTN (Round-To-Nearest) mode when iters=0. "- "RTN is fast but less accurate; keeping optimization enabled is recommended.",- )- group_opt_rtn.add_argument(- "--enable_opt_rtn",- action="store_const",- const=False,- dest="disable_opt_rtn",- help="Enable optimization for RTN mode when iters=0.",- )- group_model_free = tuning.add_mutually_exclusive_group()- group_model_free.add_argument(- "--model_free",- action="store_true",- help="Force model-free quantization mode. "- "Downloads and quantizes safetensors files directly using RTN, "- "without loading the full model into memory. "- "Only supports auto_round output format.",- )- group_model_free.add_argument(- "--disable_model_free",- action="store_true",- help="Disable the automatic model-free routing that activates when "- "--iters 0 --disable_opt_rtn is combined with a supported INT WOQ scheme. "- "Use this to force the regular AutoRound flow.",- )-- awq_group = self.add_argument_group("AWQ Arguments")- awq_group.add_argument(- "--duo_scaling",- default=True,- type=lambda s: "both" if s.lower() == "both" else s.lower() in ("true", "1", "yes"),- help="Whether to use both activations and weights for AWQ scaling. "- "Options: true/false/'both'. 'both' searches both modes and picks the best. "- "(default: True).",- )- awq_group.add_argument(- "--n_grid",- default=20,- type=int,- help="Number of grid points for AWQ scaling ratio search (default: 20).",- )-- scheme = self.add_argument_group("Scheme Arguments")- scheme.add_argument("--bits", default=None, type=int, help="Number of bits for weight quantization. ")- scheme.add_argument(- "--group_size",- default=None,- type=lambda s: int(s) if s.lstrip("-").isdigit() else tuple([int(x.strip()) for x in s.split(",")]),- help="Group size for weight quantization.",- )- scheme.add_argument("--asym", action="store_true", help="Use asymmetric quantization instead of symmetric.")- scheme.add_argument(- "--act_asym", action="store_true", help="Use asymmetric quantization for activation instead of symmetric."- )- scheme.add_argument(- "--data_type",- "--dtype",- default=None,- help="Data type for quantization. Options: 'int' for integer, 'mx_fp' for mixed floating-point, etc.",- )- scheme.add_argument(- "--act_bits",- default=None,- type=int,- help="Number of bits for activation quantization. "- "Activation quantization significantly impacts performance and accuracy.",- )- scheme.add_argument(- "--act_group_size",- default=None,- type=int,- help="Group size for activation quantization. " "Similar to weight group size but for activations.",- )- scheme.add_argument(- "--act_data_type", "--act_dtype", default=None, type=str, help="Data type for activation quantization. "- )- scheme.add_argument(- "--disable_act_dynamic", action="store_true", help="Use static instead of dynamic activation quantization. "- )- scheme.add_argument(- "--layer_config",- default=None,- type=str,- help="Per-layer quantization config for missing tensors (e.g., MTP layers) as a JSON string. "- "Keys are name prefixes, values are config dicts with optional bits/group_size/sym. "- 'Example: "{mtp:{bits:8,data_type:int},mtp.fc:{bits:16}}". '- "These settings are saved to extra_config and override the global quantization config.",- )- scheme.add_argument(- "--shared_layers",- type=str,- nargs="+",- action="append",- default=None,- help="[mix-precision] ensure that listed layers are using same data type for quantization",- )- scheme.add_argument(- "--quant_lm_head",- action="store_true",- help="Quantize the lm_head. " "Usually kept in higher precision for better output quality.",- )- scheme.add_argument(- "--ignore_layers",- "--fp_layers",- default="",- type=str,- help="List of layer names to keep in original precision (not quantized). "- "Useful for preserving critical layers. Separate multiple names with commas.",- )- scheme.add_argument(- "--static_kv_dtype",- default=None,- type=str,- choices=["fp8", "float8_e4m3fn"],- help="Data type for static quantize key and value. ",- )-- scheme.add_argument(- "--static_attention_dtype",- default=None,- type=str,- choices=["fp8", "float8_e4m3fn"],- help="Data type for static quantize attention. ",- )- scheme.add_argument(- "--rotation_type",- default=None,- type=str,- choices=["hadamard", "random_hadamard", "inplace_quarot_hadamard", "inplace_hadamard", "inplace_random"],- help="Research feature: applies a rotation (e.g., Hadamard) to reduce activation/weight outliers",- )- scheme.add_argument(- "--rotation_block_size",- default=None,- type=int,- help="Research feature rotation_block_size",- )- gguf = self.add_argument_group("Double Quant Arguments")- gguf.add_argument(- "--super_group_size", default=None, type=int, help="Super group size for double quantization."- )- gguf.add_argument(- "--super_bits",- default=None,- type=int,- help="Number of bits for scale and zero-point quantization in double quantization. ",- )-- ## ======================= eval =======================- eval_args = self.add_argument_group("eval arguments")- eval_args.add_argument(- "--tasks",- "--task",- nargs="?",- const="lambada_openai,hellaswag,winogrande,piqa,mmlu,wikitext,truthfulqa_mc1,"- "openbookqa,boolq,arc_easy,arc_challenge",- default=None,- help="LM-Evaluation-Harness tasks to run. "- "Specify specific tasks like 'mmlu,wikitext' for custom evaluation.",- )- eval_args.add_argument("--eval_bs", default=None, type=int, help="Batch size for evaluation.")- eval_args.add_argument(- "--limit",- type=float,- default=None,- metavar="N|0<N<1",- help="Limit the number of examples per task. "- "Integer: exact number of examples (e.g., 1000). "- "Float between 0-1: fraction of total examples.",- )- eval_args.add_argument(- "--eval_task_by_task", action="store_true", help="Evaluate tasks sequentially instead of batching. "- )- eval_args.add_argument(- "--eval_backend",- default="hf",- type=str,- choices=["hf", "vllm"],- help="Backend to use for model evaluation. Use hf backend for evaluation by default.",- )- eval_args.add_argument(- "--vllm_args",- default=None,- type=str,- help="(for vllm) Custom vllm arguments in format: '--arg1=value1,--arg2=value2'. "- "Example: '--tensor_parallel_size=2,--gpu_memory_utilization=0.9'",- )- eval_args.add_argument(- "--eval_model_dtype",- default=None,- type=str,- help="Torch data type for model loading during evaluation. "- "Options: 'float16', 'bfloat16', 'float32'. "- "Should match your hardware capabilities for best performance.",- )- eval_args.add_argument("--add_bos_token", action="store_true", help="add BOS token")-- ## ======================= MLLM =======================- mllm_args = self.add_argument_group("Multimodal Large Language Model(MLLM) arguments")- mllm_args.add_argument(- "--mllm",- action="store_true",- help="[Deprecated] AutoRound now automatically detects and uses MLLM mode when needed.",- )- mllm_args.add_argument(- "--quant_nontext_module",- action="store_true",- help="Quantize non-text modules (vision/audio/video components). "- "Enables full multimodal model quantization but may affect visual quality.",- )- mllm_args.add_argument(- "--extra_data_dir",- default=None,- type=str,- help="Directory containing multimodal data (images/audio/videos). "- "Can be a single directory or specify types: "- "'image=/path/to/images,video=/path/to/videos,audio=/path/to/audio'. "- "If not found locally, will attempt to download standard datasets.",- )- mllm_args.add_argument(- "--template",- default=None,- type=str,- help="Custom template for building training datasets. "- "Useful for specialized multimodal tasks or custom data formats.",- )-- ## ======================= diffusion model eval =======================- diffusion_args = self.add_argument_group("diffusion model arguments")- diffusion_args.add_argument(- "--prompt_file",- default=None,- type=str,- help="File containing prompts for evaluation, one per line. "- "Use this for batch evaluation with multiple prompts.",- )- diffusion_args.add_argument(- "--prompt",- default=None,- type=str,- help="Single prompt for quick testing. " "Overrides prompt_file if both are specified.",- )- diffusion_args.add_argument(- "--metrics",- "--metric",- default="clip",- help="Evaluation metrics for generated images. "- "'clip': CLIP score measuring text-image alignment. "- "'clip-iqa': CLIP-based image quality assessment. "- "'imagereward': Learned metric for image quality.",- )- diffusion_args.add_argument(- "--image_save_dir",- default="./tmp_image_save",- type=str,- help="Directory to save generated images during evaluation. " "Useful for visual inspection of results.",- )- diffusion_args.add_argument(- "--guidance_scale",- default=7.5,- type=float,- help="Classifier-free guidance scale for diffusion models. "- "Higher values (7-20) make the model follow the prompt more closely. "- "Lower values give more creative/random results.",- )-- diffusion_args.add_argument(- "--num_inference_steps",- default=50,- type=int,- help="Number of denoising steps in the diffusion process. "- "More steps (50-100) usually give better quality but take longer. "- "Fewer steps (10-30) are faster but lower quality.",- )-- diffusion_args.add_argument(- "--generator_seed",- default=None,- type=int,- help="Random seed for image generation reproducibility. "- "Using the same seed produces identical results across runs.",- )---def list_item():- args = argparse.ArgumentParser()- args.add_argument("item", type=str, help="item to list, e.g., format")- args = args.parse_args()- if args.item == "format" or args.item == "formats":- from auto_round.formats import OutputFormat-- print("AutoRound supported output formats and quantization scheme:")- print(OutputFormat.get_support_matrix())---def start(recipe="default"):- recipe = RECIPES[recipe]- parser = BasicArgumentParser()- args = parser.parse_args()- for k, v in recipe.items():- if getattr(args, k) is None:- setattr(args, k, v)- tune(args)---def tune(args):- assert args.model or args.model_name, "[model] or --model MODEL_NAME should be set."- if args.model is None:- args.model = args.model_name- if args.eval_bs is None:- args.eval_bs = "auto"- from transformers import AutoConfig, AutoModelForCausalLM, AutoTokenizer- from transformers.utils.versions import require_version-- if args.tasks is not None:- require_version(- "lm_eval>=0.4.2",- "lm-eval is required for evaluation, please install it with `pip install 'lm-eval>=0.4.2'`",- )-- from auto_round.utils import get_library_version, get_major_device, logger-- if args.low_cpu_mem_usage:- logger.warning(- "`low_cpu_mem_usage` is deprecated and is now enabled by default. "- "To disable it, use `--disable_low_cpu_mem_usage`."- )-- if args.format is None:- args.format = "auto_round"-- formats = args.format.lower().replace(" ", "").split(",")- from auto_round.utils import SUPPORTED_FORMATS-- for format in formats:- if format not in SUPPORTED_FORMATS:- raise ValueError(f"{format} is not supported, we only support {SUPPORTED_FORMATS}")-- if "auto_gptq" in args.format and args.asym is True:- logger.warning(- "the auto_gptq kernel has issues with asymmetric quantization. "- "It is recommended to use sym quantization or --format='auto_round'"- )-- if "marlin" in args.format and args.asym is True:- raise RuntimeError("marlin backend only supports sym quantization, please remove --asym")-- if args.enable_torch_compile:- logger.info(- "`torch.compile` is enabled to reduce tuning costs. "- "If it causes issues, you can disable it by removing `--enable_torch_compile` argument."- )-- model_name = args.model- if model_name[-1] == "/":- model_name = model_name[:-1]- logger.info(f"start to quantize {model_name}")-- from auto_round import AutoRound-- if "bloom" in model_name:- args.low_gpu_mem_usage = False-- if args.quant_lm_head:- for format in formats:- # MLX (native ``mlx`` and ``auto_round:mlx``) supports per-layer- # mixed-bit quantization including lm_head; treat it the same as- # auto_round / fake here.- if "auto_round" not in format and "fake" not in format and "mlx" not in format:- auto_round_formats = [s for s in SUPPORTED_FORMATS if s.startswith("auto_round") or s == "mlx"]- raise ValueError(- f"{format} is not supported for lm-head quantization, please change to {auto_round_formats}"- )-- enable_torch_compile = True if "--enable_torch_compile" in sys.argv else False- sym = None # the default value should be None now- if args.asym: # if the scheme is asym, how to set it to sym is an issue- sym = False- act_dynamic = None- if args.disable_act_dynamic:- act_dynamic = False- scheme = args.scheme.upper()- if scheme not in PRESET_SCHEMES:- raise ValueError(f"{scheme} is not supported. only {PRESET_SCHEMES.keys()} are supported ")- if args.disable_deterministic_algorithms:- logger.warning(- "default not use deterministic_algorithms. disable_deterministic_algorithms is deprecated,"- " please use enable_deterministic_algorithms instead. "- )-- from auto_round.compressors import (- DiffusionExtraConfig,- ExtraConfig,- MLLMExtraConfig,- SchemeExtraConfig,- TuningExtraConfig,- )-- extra_config = ExtraConfig()- tuning_config = TuningExtraConfig(- amp=not args.disable_amp,- disable_opt_rtn=args.disable_opt_rtn,- enable_alg_ext=args.enable_alg_ext,- enable_minmax_tuning=not args.disable_minmax_tuning,- enable_norm_bias_tuning=args.enable_norm_bias_tuning,- enable_quanted_input=not args.disable_quanted_input,- enable_deterministic_algorithms=args.enable_deterministic_algorithms,- lr=args.lr,- minmax_lr=args.minmax_lr,- nblocks=args.nblocks,- to_quant_block_names=args.to_quant_block_names,- scale_dtype=args.scale_dtype,- )- scheme_config = SchemeExtraConfig(- bits=args.bits,- group_size=args.group_size,- sym=sym,- data_type=args.data_type,- act_bits=args.act_bits,- act_group_size=args.act_group_size,- act_data_type=args.act_data_type,- act_dynamic=act_dynamic,- act_sym=None if not args.asym else False,- super_bits=args.super_bits,- super_group_size=args.super_group_size,- quant_lm_head=args.quant_lm_head,- ignore_layers=args.ignore_layers,- static_kv_dtype=args.static_kv_dtype,- static_attention_dtype=args.static_attention_dtype,- )- mllm_config = MLLMExtraConfig(- quant_nontext_module=args.quant_nontext_module, extra_data_dir=args.extra_data_dir, template=args.template- )- diffusion_config = DiffusionExtraConfig(- guidance_scale=args.guidance_scale,- num_inference_steps=args.num_inference_steps,- generator_seed=args.generator_seed,- )- extra_config.tuning_config = tuning_config- extra_config.scheme_config = scheme_config- extra_config.mllm_config = mllm_config- extra_config.diffusion_config = diffusion_config-- layer_config = {}- if args.layer_config:- layer_config = parse_layer_config_arg(args.layer_config)- args.layer_config = layer_config-- low_cpu_mem_usage = True- if args.disable_low_cpu_mem_usage:- low_cpu_mem_usage = False-- if args.avg_bits is not None:- if args.options is None:- raise ValueError("please set --options for auto scheme")- if enable_torch_compile:- logger.warning(- "`enable_torch_compile=True` with AutoScheme may cause compile errors "- "on some models. If so, try removing `--enable_torch_compile`."- )- scheme = AutoScheme(- options=args.options,- avg_bits=args.avg_bits,- shared_layers=args.shared_layers,- ignore_scale_zp_bits=args.ignore_scale_zp_bits,- low_gpu_mem_usage=True, # force it to be True as it uses much smaller vram but similar time cost- low_cpu_mem_usage=low_cpu_mem_usage,- )- rot_config = None- if args.rotation_type:- from auto_round.algorithms.transforms.rotation.config import RotationConfig-- rot_config = RotationConfig(hadamard_type=args.rotation_type, block_size=args.rotation_block_size)-- autoround: BaseCompressor = AutoRound(- model=model_name,- platform=args.platform,- format=args.format,- scheme=scheme,- dataset=args.dataset,- iters=args.iters,- seqlen=args.seqlen,- nsamples=args.nsamples,- batch_size=args.batch_size,- gradient_accumulate_steps=args.gradient_accumulate_steps,- low_gpu_mem_usage=args.low_gpu_mem_usage,- low_cpu_mem_usage=low_cpu_mem_usage,- device_map=args.device_map,- enable_torch_compile=enable_torch_compile,- seed=args.seed,- not_use_best_mse=args.not_use_best_mse,- enable_adam=args.adam,- extra_config=extra_config,- layer_config=layer_config,- model_dtype=args.model_dtype,- momentum=args.momentum,- trust_remote_code=not args.disable_trust_remote_code,- rotation_config=rot_config,- model_free=args.model_free,- disable_model_free=args.disable_model_free,- algorithm=getattr(args, "algorithm", None),- **(- {"duo_scaling": args.duo_scaling, "n_grid": args.n_grid}- if getattr(args, "algorithm", None) == "awq"- else {}- ),- )-- # ======================= Quantize and save model =======================- # Export directory is now derived automatically inside quantize_and_save via- # BaseCompressor._get_export_dir(), so we only need to pass the base output_dir.- model, folders = autoround.quantize_and_save(args.output_dir, format=args.format) # pylint: disable=E1101- tokenizer = autoround.tokenizer # pylint: disable=E1101- clear_memory()-- # ======================= Model evaluation =======================- run_model_evaluation(model, tokenizer, autoround, folders, formats, args)---def setup_eval_parser():- parser = EvalArgumentParser()- args = parser.parse_args()- return args---def run_eval():- from auto_round.logger import logger- from auto_round.utils import is_gguf_model, is_mllm_model-- args = setup_eval_parser()- assert args.model or args.model_name, "[model] or --model MODEL_NAME should be set."-- if args.model is None:- args.model = args.model_name- if "llama" in args.model.lower() and not args.add_bos_token:- logger.warning("set add_bos_token=True for llama model.")- args.add_bos_token = True- if not is_gguf_model(args.model) and is_mllm_model(args.model):- args.mllm = True-- if args.eval_task_by_task:- eval_task_by_task(- model=args.model,- device=args.device_map,- limit=args.limit,- tasks=args.tasks,- batch_size=args.eval_bs,- trust_remote_code=not args.disable_trust_remote_code,- eval_model_dtype=args.eval_model_dtype,- add_bos_token=args.add_bos_token,- )- else:- eval(args)---def run():- if "list" in sys.argv or "--list" in sys.argv:- if "list" in sys.argv:- sys.argv.remove("list")- if "--list" in sys.argv:- sys.argv.remove("--list")- list_item()- exit()- if "--eval" in sys.argv or "eval" in sys.argv:- if "--eval" in sys.argv:- sys.argv.remove("--eval")- if "eval" in sys.argv:- sys.argv.remove("eval")- run_eval()- else:- start()---def run_best():- start("best")---def run_light():- start("light")---def run_rtn():- start("rtn")---def run_opt_rtn():- start("opt_rtn")---def run_mllm():- run()+# Thin shim - all logic lives in auto_round.cli.main.+# This file exists solely to satisfy setup.cfg console_scripts entry points.+from auto_round.cli.main import run, run_best, run_eval, run_light, run_mllm, run_opt_rtn, run_rtn # noqa: F401if __name__ == "__main__":run()diff --git a/auto_round/alg_ext.py b/auto_round/alg_ext.pydeleted file mode 100644index e6137d1a5..000000000--- a/auto_round/alg_ext.py+++ /dev/null@@ -1,872 +0,0 @@-# Copyright (c) 2025 Intel Corporation-#-# Licensed under the Apache License, Version 2.0 (the "License");-# you may not use this file except in compliance with the License.-# You may obtain a copy of the License at-#-# http://www.apache.org/licenses/LICENSE-2.0-#-# Unless required by applicable law or agreed to in writing, software-# distributed under the License is distributed on an "AS IS" BASIS,-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.-# See the License for the specific language governing permissions and-# limitations under the License.--import logging-import types-from contextlib import nullcontext-from functools import lru_cache, partial-from typing import Any, Callable, Union--import torch-import transformers-from torch import autocast-from torch.functional import F--from auto_round import AutoRound-from auto_round.compressors.utils import check_need_act_calibration, is_nv_fp, is_wint4aint4-from auto_round.data_type.int import search_scales-from auto_round.data_type.mxfp import MXFP_FORMAT_CACHE, quant_element-from auto_round.data_type.nvfp import FLOAT4_E2M1_MAX, FLOAT8_E4M3_MAX, ref_nvfp4_quant-from auto_round.data_type.utils import floor_ste, reshape_pad_tensor_by_group_size, revert_tensor_by_pad, round_ste-from auto_round.logger import logger-from auto_round.utils import SUPPORTED_LAYER_TYPES, check_to_quantized, compile_func, get_reciprocal, set_module-from auto_round.wrapper import NORM_MAPPING, WrapperLinear--__all__ = ["wrapper_autoround"]---def wrapper_autoround(cls: AutoRound):- cls._register_act_max_hook = types.MethodType(_register_act_max_hook_ext, cls)- if (- cls.sym- and cls.enable_alg_ext- and cls.super_group_size is None- and ((cls.data_type.startswith("int")) or cls.data_type.startswith("mx") or cls.data_type.startswith("nv"))- ):- if cls.bits > 2 and (not cls.data_type.startswith("mx") or not cls.data_type.startswith("nv")):- logger.warning_once(- "algorithm extension has only undergone limited validation on "- "W2A16,INT4, MXFP4 and NVFP4; use with caution."- )- cls._get_loss = types.MethodType(_get_loss_ext, cls)- setattr(cls, "wrapper_block", wrapper_block_v2)- if cls.data_type.endswith("dq"):- setattr(cls, "wrapper_block", dq_wrapper_block)---def get_abs_top_percent_mask(x: torch.Tensor, percent: float = 1.0):- """- Return a mask for the top `percent` absolute values in x and its inverse.-- Args:- x (torch.Tensor): Input tensor.- percent (float): Percentage of elements to select (0~100).-- Returns:- mask (torch.BoolTensor): True for top `percent` abs elements.- inv_mask (torch.BoolTensor): Inverse of mask.- """- flat = x.view(-1)- k = max(1, int(flat.numel() * percent / 1000))- _, idx = torch.topk(torch.abs(flat), k)-- mask = torch.zeros_like(flat, dtype=torch.bool)- mask[idx] = True- mask = mask.view_as(x)- return mask, ~mask---def _get_loss_ext(- self: AutoRound,- output_q: torch.Tensor,- current_output: torch.Tensor,- indices: torch.Tensor,- mse_loss: Callable,- device: Union[str, torch.device] = "cpu",-):- _, mask = get_abs_top_percent_mask(torch.abs(output_q - current_output))- autocast_ctx = autocast(device_type=str(device).split(":")[0], dtype=self.amp_dtype) if self.amp else nullcontext()- if self.attention_mask:- tmp_attention_mask = [self.attention_mask[i] for i in indices]- tmp_attention_mask = torch.cat(tmp_attention_mask, dim=0).to(device)- tmp_attention_mask.unsqueeze_(-1)-- with autocast_ctx:- loss = torch.mean(- (torch.abs(output_q.to(torch.float32) - current_output.to(torch.float32)) * tmp_attention_mask * mask)- ** 2- )- else:- with autocast_ctx:- loss = torch.mean((torch.abs(output_q.to(torch.float32) - current_output.to(torch.float32)) * mask) ** 2)-- return loss---def quant_tensor_sym(- tensor,- bits=4,- group_size=-1,- v=0,- min_scale=1.0,- max_scale=1.0,- scale_dtype=torch.float16,- tensor_min=None,- tensor_max=None,- q_scale_thresh=1e-5,- init_scale=None,- **kwargs,-):- """Quantize and de-quantize tensor symmetrically (full-range, llama.cpp style).-- ``maxq`` is computed via ``int(2.0 ** (bits - 1))`` so it stays a plain- Python int constant inside the inductor graph and Triton never tries to- lower a ``2 ** SymInt`` through ``libdevice.pow(fp32, i64)``.- """- tensor, orig_shape, pad_len = reshape_pad_tensor_by_group_size(tensor, group_size)- maxq = int(2.0 ** (bits - 1))- scale = init_scale * max_scale.unsqueeze(dim=-1)- int_w = round_ste(tensor / scale + v)- q = torch.clamp(int_w, -maxq, maxq - 1)- qdq_result = (scale * q).to(tensor.dtype)- qdq_result = revert_tensor_by_pad(qdq_result, orig_shape=orig_shape, pad_len=pad_len)- return qdq_result, scale, maxq--[email protected]_mode()-def qdq_mxfp(tensor, max_val, max_norm, emax, ebits, mbits):- shared_exp = torch.where(max_val == 0, torch.ones_like(max_val), torch.log2(max_val))- shared_exp = torch.floor(shared_exp)- scale_emax = (1 << (8 - 1)) - 1- shared_exp = (shared_exp - emax).clamp(min=-scale_emax, max=scale_emax)-- scale = torch.pow(2.0, shared_exp)- tensor = tensor / scale- tensor = torch.clamp(tensor, min=-max_norm, max=max_norm)- tensor = quant_element(tensor, ebits, mbits, max_norm)-- tensor = tensor * scale- return tensor---def mx_init(tensor, bits, qw=None):- data_type = "mx_fp" + str(bits)- ebits, mbits, emax, max_norm, min_norm = MXFP_FORMAT_CACHE[data_type]- tensor = tensor.to(torch.float32)- max_val, _ = torch.max(torch.abs(tensor), dim=-1, keepdim=True)- qdq_t = qdq_mxfp(tensor, max_val, max_norm, emax, ebits, mbits)- best_loss = torch.sum((qdq_t - tensor) ** 2 * qw, dim=-1)- scales = torch.ones_like(max_val)- tmp_scale = 0.5- while tmp_scale < 1.51:- if tmp_scale == 1.0:- continue- max_val_new = max_val * tmp_scale- tmp_qdq_t = qdq_mxfp(tensor, max_val_new, max_norm, emax, ebits, mbits)- loss = torch.sum((tmp_qdq_t - tensor) ** 2 * qw, dim=-1)- replace_id = loss < best_loss- scales[replace_id] = (torch.ones_like(scales) * tmp_scale)[replace_id]- best_loss[replace_id] = loss[replace_id]- tmp_scale += 0.01- return scales---def nv_fp4(tensor, bits=4, group_size=16, v=0, global_scale=None, max_scale=1.0, init_scale=1.0, **kwargs):- orig_dtype = tensor.dtype- tensor, orig_shape, pad_len = reshape_pad_tensor_by_group_size(tensor, group_size)- if global_scale is None:- tensor_max = tensor.abs().max().to(torch.float32)- global_scale = FLOAT8_E4M3_MAX * FLOAT4_E2M1_MAX * get_reciprocal(tensor_max)- global_scale = global_scale.to(device=tensor.device, dtype=torch.float32)- if isinstance(init_scale, torch.Tensor):- init_scale = init_scale.view(-1)- max_scale = max_scale * init_scale- qdq_res, scale = ref_nvfp4_quant(tensor, global_scale, group_size, v, scale_coeff=max_scale)- qdq_res = revert_tensor_by_pad(qdq_res, orig_shape=orig_shape, pad_len=pad_len)- return qdq_res.to(orig_dtype), scale, None---def nv_init(tensor, bits, qw=None):- tensor = tensor.to(torch.float32)- qdq_t, dummy_scale, _ = nv_fp4(tensor, bits=4, group_size=16, v=0, max_scale=1.0)- best_loss = torch.sum((qdq_t - tensor) ** 2 * qw, dim=-1)- scales = torch.ones_like(dummy_scale)- tmp_scale = 0.5- while tmp_scale < 1.51:- if tmp_scale == 1.0:- continue- scales_new = torch.ones_like(dummy_scale) * tmp_scale- tmp_qdq_t, _, _ = nv_fp4(tensor, bits=4, group_size=16, v=0, max_scale=scales_new)- loss = torch.sum((tmp_qdq_t - tensor) ** 2 * qw, dim=-1)- replace_id = loss < best_loss- scales[replace_id] = scales_new[replace_id]- best_loss[replace_id] = loss[replace_id]- tmp_scale += 0.01- return scales---def quant_mx(- tensor,- bits=4,- group_size=-1,- v=0,- max_scale=1.0,- init_scale=1.0,- mantissa_rounding="even",- data_type="mx_fp",- **kwargs,-):- """Quantize the given tensor using the specified parameters.-- This function performs quantization on the `tensor` tensor according to the- given bit width (`bits`), data type (`data_type`), and additional parameters.- The quantization process involves scaling the tensor values and adjusting- the exponent and mantissa to fit within the specified format.-- Args:- tensor (torch.Tensor): The tensor containing the tensors to be quantized.- bits (int): The bit width to be used for quantization.- group_size (int): The group size of sharing scale and exponent.- data_type (str): The data type for quantization (e.g., 'mx_fp4').- v (float): A value used for adjusting the tensors.- max_scale (float or torch.Tensor): The maximum scale to be applied to the tensors.- mantissa_rounding (str): rounding method for mantissa,currently support even,nearest,floor-- Returns:- tuple: A tuple containing the quantized tensors, shared exponent, and None (reserved for future use).-- Raises:- KeyError: If `data_type` is not found in `MXFP_FORMAT_CACHE`.- """- tensor, orig_shape, pad_len = reshape_pad_tensor_by_group_size(tensor, group_size)- data_type = data_type if data_type in MXFP_FORMAT_CACHE else "mx_fp" + str(bits)- ebits, mbits, emax, max_norm, min_norm = MXFP_FORMAT_CACHE[data_type]- orig_dtype = tensor.dtype- tensor = tensor.to(torch.float32)- max_val, _ = torch.max(torch.abs(tensor), dim=-1, keepdim=True)- if isinstance(max_scale, torch.Tensor):- max_val *= init_scale * (max_scale.unsqueeze(dim=-1)).to(tensor.device)- else:- max_val *= init_scale * max_scale-- # shared_exp = torch.log2(shared_exp + FP32_MIN_NORMAL * (shared_exp == 0).type(shared_exp.dtype))- shared_exp = torch.where(max_val == 0, torch.ones_like(max_val), torch.log2(max_val))- shared_exp = floor_ste(shared_exp)- scale_emax = (1 << (8 - 1)) - 1- shared_exp = (shared_exp - emax).clamp(min=-scale_emax, max=scale_emax)-- scale = torch.pow(2.0, shared_exp)- tensor = tensor / scale + v- tensor = torch.clamp(tensor, min=-max_norm, max=max_norm)- tensor = quant_element(tensor, ebits, mbits, max_norm, mantissa_rounding)-- tensor = tensor * scale- tensor = revert_tensor_by_pad(tensor, orig_shape=orig_shape, pad_len=pad_len)- return tensor.to(orig_dtype), shared_exp.to(orig_dtype), None---class WrapperLinearV2(WrapperLinear):- """A wrapper for linear/conv1d layers to enable quantization and tuning.-- This module wraps an existing linear or conv1d layer and provides additional functionality- for quantization, parameter tuning, and activation/bias normalization.-- Args:- orig_layer (torch.nn.Module): The original layer to be wrapped (linear or conv1d).- enable_minmax_tuning (bool): Whether to enable min-max scale tuning.- enable_norm_bias_tuning (bool): Whether to enable normalization and tuning of the bias term.- device (str): Device on which to run computations (e.g., 'cpu' or 'cuda').- """-- def __init__(- self,- orig_layer,- enable_minmax_tuning=True,- enable_norm_bias_tuning=False,- device="cpu",- enable_round_tuning=True,- enable_torch_compile=False,- disable_opt_rtn=True, # TODO does not support it- **kwargs,- ):- """Initializes the WrapperLinear module.-- Args:- orig_layer (torch.nn.Module): The original layer to wrap.- enable_minmax_tuning (bool): Whether to enable min-max scale tuning.- enable_norm_bias_tuning (bool): Whether to enable normalization and tuning for the bias term.- device (str): The computation device, such as 'cpu' or 'cuda'.- """- super(WrapperLinearV2, self).__init__(- orig_layer=orig_layer,- enable_minmax_tuning=enable_minmax_tuning,- enable_norm_bias_tuning=enable_norm_bias_tuning,- device=device,- enable_round_tuning=enable_round_tuning,- enable_torch_compile=enable_torch_compile,- disable_opt_rtn=disable_opt_rtn,- **kwargs,- )-- def _init_tuning_params_and_quant_func(self):- """Initializes tuning parameters and quantization functions.-- This method sets up required parameters and functions for weight quantization,- activation quantization, and bias/normalization.- """- super()._init_tuning_params_and_quant_func()-- orig_weight = getattr(self.orig_layer, "get_weight", lambda: self.orig_layer.weight)()- weight_reshape, _, _ = reshape_pad_tensor_by_group_size(orig_weight.data, self.orig_layer.group_size)- if hasattr(self.orig_layer, "imatrix"): # MOE model may have no imatrix- imatrix = self.orig_layer.imatrix.reshape(1, -1)- imatrix = reshape_pad_tensor_by_group_size(imatrix, self.orig_layer.group_size, val=1e-5)[0].view(1, -1)- imatrix = imatrix.expand(weight_reshape.numel() // imatrix.numel(), -1)- imatrix = imatrix.reshape(weight_reshape.shape)- imatrix = imatrix.to(orig_weight.device)- else:- imatrix = 1.0- if self.orig_layer.data_type.startswith("int"):- self.init_scale = search_scales(weight_reshape, self.orig_layer.bits, imatrix)- self.init_scale = torch.where(- self.init_scale < 0,- torch.clamp(self.init_scale, max=-self.q_scale_thresh),- torch.clamp(self.init_scale, min=self.q_scale_thresh),- )- elif self.orig_layer.data_type.startswith("mx"):- self.init_scale = mx_init(weight_reshape, self.orig_layer.bits, imatrix)- elif self.orig_layer.data_type.startswith("nv"):- self.init_scale = nv_init(weight_reshape, self.orig_layer.bits, imatrix)- else:- self.init_scale = 1.0- self.orig_layer.imatrix = None- delattr(self.orig_layer, "imatrix")-- # self.weight_quant_func, self.data_type = get_quant_func(orig_layer.data_type, orig_layer.bits, orig_layer.sym)- if self.orig_layer.data_type.startswith("int"):- self.weight_quant_func = quant_tensor_sym- elif self.orig_layer.data_type.startswith("mx"):- self.weight_quant_func = quant_mx- elif self.orig_layer.data_type.startswith("nv"):- self.weight_quant_func = nv_fp4- else:- logger.error("unsupported dtype")- exit(-1)- self.data_type = self.orig_layer.data_type- if self.enable_torch_compile:- self.weight_quant_func = compile_func(self.weight_quant_func, self.device)-- def _qdq_weight(self, value, min_scale, max_scale):- """Quantizes and dequantizes weights with tuning parameters.-- Args:- value (torch.Tensor): Value added for rounding for tuning.- min_scale (torch.Tensor): Minimum scale for the min value of quantization.- max_scale (torch.Tensor): Maximum scale for the max value of quantization.-- Returns:- tuple: Quantized weight, scale, and zero point.- """- if self.orig_layer.bits >= 16:- return self.orig_layer.weight, None, None- min_scale.data.clamp_(0, 2.0) # TODO changed- max_scale.data.clamp_(0, 2.0)- weight = self.orig_layer.weight- if weight.device.type == "meta":- weight = self.orig_layer.get_weight().to(self.device)- if isinstance(self.orig_layer, transformers.pytorch_utils.Conv1D):- weight = weight.t()-- quant_kwargs = {}- if hasattr(self.orig_layer, "super_bits"):- quant_kwargs["super_bits"] = self.orig_layer.super_bits- quant_kwargs["super_group_size"] = self.orig_layer.super_group_size-- weight_q, scale, zp = self.weight_quant_func(- weight,- bits=self.orig_layer.bits,- group_size=self.orig_layer.group_size,- v=value,- min_scale=min_scale,- max_scale=max_scale,- scale_dtype=self.orig_layer.scale_dtype,- tensor_min=self.weight_min,- tensor_max=self.weight_max,- data_type=self.data_type,- q_scale_thresh=self.q_scale_thresh,- global_scale=getattr(self, "weight_global_scale", None),- init_scale=self.init_scale,- **quant_kwargs,- )- weight_q = weight_q.to(weight.dtype)- if isinstance(self.orig_layer, transformers.pytorch_utils.Conv1D):- weight_q = weight_q.t()- return weight_q, scale, zp---def wrapper_block_v2(block, enable_minmax_tuning, enable_norm_bias_tuning, device="cpu", **kwargs):- """Wraps the layers in the given block with a custom Wrapper module.-- Args:- block: The input block containing linear and conv1d layers to be wrapped.- enable_minmax_tuning: A boolean indicating whether min-max tuning is enabled.-- Returns:- list: A list of names of the wrapped layers and unwrapped layers.- """- quantized_layers = []- unquantized_layers = []- for n, m in block.named_modules():- if isinstance(m, SUPPORTED_LAYER_TYPES):- if not check_to_quantized(m):- unquantized_layers.append(n)- continue- new_m = WrapperLinearV2(- m,- enable_minmax_tuning=enable_minmax_tuning,- enable_norm_bias_tuning=enable_norm_bias_tuning,- device=device,- **kwargs,- )- set_module(block, n, new_m)- quantized_layers.append(n)-- elif enable_norm_bias_tuning:- if "norm" in m.__class__.__name__.lower():- if m.__class__.__name__ in NORM_MAPPING.keys():- wrapper_layer_class = NORM_MAPPING[m.__class__.__name__]- new_m = wrapper_layer_class(m, device=device)- set_module(block, n, new_m)- elif "RMSNorm" in m.__class__.__name__:- logger.warning_once(- f"use LlamaRMSNorm to wrap {m.__class__.__name__}, please check the correctness yourself"- )- wrapper_layer_class = NORM_MAPPING["LlamaRMSNorm"]- new_m = wrapper_layer_class(m, device=device)- set_module(block, n, new_m)- else:- logger.warning_once(f"{m.__class__.__name__} is not supported")- return quantized_layers, unquantized_layers---def _register_act_max_hook_ext(self, model):-- def get_act_max_hook(module, input, output):- if isinstance(input, (tuple, list)):- input = input[0]- if input.numel() == 0:- return # as no needs for act_max update- input, _, _ = reshape_pad_tensor_by_group_size(input, self.act_group_size)- act_max = torch.max(torch.abs(input), dim=-1).values- if not hasattr(module, "act_max") or module.act_max.numel() == 0:- module.act_max = act_max- else:- act_max = act_max.to(module.act_max.device)- if is_nv_fp(self.act_data_type): ## for nvfp per-tensor input_global_scale calculation usage- module.act_max = torch.max(torch.tensor([act_max.max(), module.act_max.max()], device=act_max.device))- else:- module.act_max = torch.max(act_max, module.act_max)-- def get_imatrix_hook(module, input, output):- input = input[0] if isinstance(input, (tuple, list)) else input- flattened = input.reshape(-1, input.shape[-1]).to(torch.float32)- squared = torch.sum(torch.pow(flattened, 2), dim=0).to(torch.float32)-- if not hasattr(module, "imatrix"):- module.imatrix = squared- else:- module.imatrix += squared.to(module.imatrix.device)-- hook_handles = []-- for n, m in model.named_modules():- if isinstance(m, self.supported_types) and check_to_quantized(m):- if not is_wint4aint4(self): # INT4 no imatrix is much better- hook = m.register_forward_hook(get_imatrix_hook)- hook_handles.append(hook)-- if (- hasattr(m, "act_dynamic")- and check_need_act_calibration(m.act_dynamic, m.act_data_type, m.act_bits)- and check_to_quantized(m)- ):- hook = m.register_forward_hook(get_act_max_hook)- hook_handles.append(hook)- continue-- # for whole model, RTN- if n in self.layer_config:- config = self.layer_config[n]- act_dynamic = config.get("act_dynamic", True)- act_data_type = config.get("act_data_type", None)- act_bits = config.get("act_bits", 16)- if (- config["bits"] <= 8- and check_need_act_calibration(act_dynamic, act_data_type, act_bits)- and check_to_quantized(config)- ):- hook = m.register_forward_hook(get_act_max_hook)- hook_handles.append(hook)- continue- return hook_handles---def _dq_asym_qdq(tensor, scale, wmin, bits, group_size, v=0):- """Pure asym double-quant qdq math given precomputed scale/wmin.-- ``maxq`` is computed via ``int(2.0 ** bits) - 1`` so that any SymInt- handling for ``bits`` does not produce a ``libdevice.pow(fp32, i64)`` call- in Triton (which lacks that overload). The final ``int(...)`` cast keeps- ``maxq`` as a Python int constant inside the compiled graph.- """- tensor, orig_shape, pad_len = reshape_pad_tensor_by_group_size(tensor, group_size)- orig_dtype = tensor.dtype- tensor = tensor.to(torch.float32)- maxq = int(2.0**bits) - 1- inverse_scale = get_reciprocal(scale)- int_w = torch.clamp(round_ste((tensor + wmin) * inverse_scale + v), 0, maxq)- qdq = (scale * int_w - wmin).to(orig_dtype)- qdq = revert_tensor_by_pad(qdq, orig_shape=orig_shape, pad_len=pad_len)- return qdq---def _dq_sym_qdq(tensor, scale, bits, v=0):- """Pure sym double-quant qdq math given precomputed scale.-- ``maxq`` is computed via float ``2.0 ** (bits - 1)`` then cast to- ``int`` to avoid SymInt-driven shifts being lowered through- ``libdevice.pow``.- """- from auto_round.export.export_to_gguf.config import GGML_QUANT_SIZES, QK_K-- group_size = 16- super_group_size = 16- maxq = int(2.0 ** (bits - 1))- tensor, orig_shape, pad_len = reshape_pad_tensor_by_group_size(tensor, group_size)- orig_dtype = tensor.dtype- tensor = tensor.to(torch.float32)- ggml_type = f"q{bits}_k"- block_size, _ = GGML_QUANT_SIZES[ggml_type]- n_blocks = tensor.nelement() // block_size- tensor = tensor.reshape(n_blocks, super_group_size, QK_K // super_group_size)- if isinstance(v, torch.Tensor):- v_r, _, _ = reshape_pad_tensor_by_group_size(v, group_size)- v_r = v_r.reshape(n_blocks, super_group_size, QK_K // super_group_size)- else:- v_r = v- zp = torch.full_like(scale, maxq)- inverse_scale = get_reciprocal(scale)- int_w = round_ste(tensor * inverse_scale + v_r).clip(-maxq, maxq - 1) + maxq- qdq = (scale * (int_w - zp)).to(orig_dtype)- qdq = revert_tensor_by_pad(qdq, orig_shape=orig_shape, pad_len=pad_len)- return qdq---class DQWrapperLinear(WrapperLinear):- """A wrapper for linear/conv1d layers to enable quantization and tuning.-- This module wraps an existing linear or conv1d layer and provides additional functionality- for quantization, parameter tuning, and activation/bias normalization.-- Args:- orig_layer (torch.nn.Module): The original layer to be wrapped (linear or conv1d).- enable_minmax_tuning (bool): Whether to enable min-max scale tuning.- enable_norm_bias_tuning (bool): Whether to enable normalization and tuning of the bias term.- device (str): Device on which to run computations (e.g., 'cpu' or 'cuda').- """-- def __init__(- self,- orig_layer,- enable_minmax_tuning=True,- enable_norm_bias_tuning=False,- device="cpu",- enable_round_tuning=True,- enable_torch_compile=False,- disable_opt_rtn=True,- **kwargs,- ):- """Initializes the WrapperLinear module.-- Args:- orig_layer (torch.nn.Module): The original layer to wrap.- enable_minmax_tuning (bool): Whether to enable min-max scale tuning.- enable_norm_bias_tuning (bool): Whether to enable normalization and tuning for the bias term.- device (str): The computation device, such as 'cpu' or 'cuda'.- """- super(DQWrapperLinear, self).__init__(- orig_layer=orig_layer,- enable_minmax_tuning=enable_minmax_tuning,- enable_norm_bias_tuning=enable_norm_bias_tuning,- device=device,- enable_round_tuning=enable_round_tuning,- enable_torch_compile=enable_torch_compile,- disable_opt_rtn=disable_opt_rtn,- **kwargs,- )- self.prev_scale = None- self.prev_wmin = None- self.prev_d_scale = None- self.prev_d_wmin = None-- def _init_tuning_params_and_quant_func(self):- """Initializes tuning parameters and quantization functions.-- This method sets up required parameters and functions for weight quantization,- activation quantization, and bias/normalization.- """- super()._init_tuning_params_and_quant_func()- p_dtype = torch.float32- # ``search_func`` is kept un-compiled because it contains data-dependent- # control flow (imatrix branches, iterative searches), while- # ``weight_quant_func`` is the compilable pure-math part.- self.search_func = None- self._dq_kind = None- self._is_dq_path = False- if hasattr(self.orig_layer, "super_group_size") and self.orig_layer.super_group_size is not None:- self._is_dq_path = True- from auto_round.data_type.gguf import search_gguf_scale_min_asym, search_gguf_scale_min_sym-- if self.orig_layer.data_type == "int_asym_dq":- self.search_func = search_gguf_scale_min_asym- self.weight_quant_func = _dq_asym_qdq- self._dq_kind = "asym"- else:- self.search_func = search_gguf_scale_min_sym- self.weight_quant_func = _dq_sym_qdq- self._dq_kind = "sym"- elif self.orig_layer.sym:- from auto_round.data_type.int import quant_tensor_sym-- self.weight_quant_func = quant_tensor_sym- else:- from auto_round.data_type.int import quant_tensor_asym-- self.weight_quant_func = quant_tensor_asym- self.data_type = self.orig_layer.data_type- if self.enable_act_quant:- from auto_round.data_type.gguf import quant_tensor_gguf_asym_dq as _gguf_asym_dq- from auto_round.data_type.gguf import quant_tensor_gguf_sym_dq as _gguf_sym_dq-- self.act_quant_func = _gguf_asym_dq if self.orig_layer.act_data_type == "int_asym_dq" else _gguf_sym_dq- if self.enable_torch_compile:- self.act_quant_func = compile_func(self.act_quant_func, self.device)- self._init_params("act_max_scale", p_dtype, (1), 1.0, not self.orig_layer.act_dynamic)-- if self.enable_torch_compile:- self.weight_quant_func = compile_func(self.weight_quant_func, self.device)-- @torch.no_grad()- def _run_search(self, weight, v):- """Run the per-format scale/wmin search separately from the quant func.-- Uses the search routines from ``auto_round.data_type.gguf`` and forwards- the tuning perturbation ``v``. Returns the parameters to feed into the- (compilable) ``weight_quant_func``.- """- from auto_round.data_type.gguf import double_quant_tensor_sym_rtn- from auto_round.export.export_to_gguf.config import GGML_QUANT_SIZES, QK_K-- bits = self.orig_layer.bits- scale_dtype = self.orig_layer.scale_dtype- imatrix = getattr(self.orig_layer, "imatrix", None)-- if self._dq_kind == "asym":- group_size = 16 if bits == 2 else 32- t, _, _ = reshape_pad_tensor_by_group_size(weight.to(torch.float32), group_size)- v_r = v- if isinstance(v, torch.Tensor):- v_r, _, _ = reshape_pad_tensor_by_group_size(v, group_size)- scale, wmin, d_scale, d_wmin = self.search_func(- t,- bits=bits,- scale_dtype=scale_dtype,- imatrix=imatrix,- split_num=1,- v=v_r,- )- # Search funcs are decorated with ``@torch.inference_mode()``; their- # outputs are inference tensors and cannot be saved for backward.- # Clone to detach from inference mode so autograd may use them.- return {- "scale": scale.clone(),- "wmin": wmin.clone(),- "d_scale": d_scale.clone(),- "d_wmin": d_wmin.clone(),- }-- # sym path- group_size = 16- super_group_size = 16- t, _, _ = reshape_pad_tensor_by_group_size(weight.to(torch.float32), group_size)- ggml_type = f"q{bits}_k"- block_size, _ = GGML_QUANT_SIZES[ggml_type]- n_blocks = t.nelement() // block_size- t = t.reshape(n_blocks, super_group_size, QK_K // super_group_size)- v_r = v- if isinstance(v, torch.Tensor):- v_r, _, _ = reshape_pad_tensor_by_group_size(v, group_size)- v_r = v_r.reshape(n_blocks, super_group_size, QK_K // super_group_size)- super_bits = 6 if bits == 3 else 8- scale = self.search_func(t, bits, imatrix, scale_dtype, split_num=1, v=v_r)- scale = scale.to(scale_dtype)- scale = torch.where(torch.abs(scale) < 1e-30, torch.zeros_like(scale), scale)- scale, d_scale = double_quant_tensor_sym_rtn(scale, super_bits)- scale = scale.unsqueeze(-1)- # Clone to escape inference-mode tensors (see asym branch comment).- return {"scale": scale.clone(), "d_scale": d_scale.clone()}-- def _qdq_weight(self, value, min_scale, max_scale, scale_v=None, iter=None):- """Quantizes and dequantizes weights with tuning parameters.-- Args:- value (torch.Tensor): Value added for rounding for tuning.- min_scale (torch.Tensor): Minimum scale for the min value of quantization.- max_scale (torch.Tensor): Maximum scale for the max value of quantization.-- Returns:- tuple: Quantized weight, scale, and zero point.- """- if self.orig_layer.bits >= 16:- return self.orig_layer.weight, None, None- min_scale.data.clamp_(0.5, 1.5)- max_scale.data.clamp_(0.5, 1.5)- weight = self.orig_layer.weight- if weight.device.type == "meta":- weight = self.orig_layer.get_weight().to(self.device)- if isinstance(self.orig_layer, transformers.pytorch_utils.Conv1D):- weight = weight.t()-- if self._is_dq_path:- # Split search (data-dependent, un-compiled) from quant math (compilable).- iter_v = self.cur_iter if (iter is None and hasattr(self, "cur_iter")) else iter- if iter_v is None:- iter_v = 0- need_search = (iter_v % 10 == 0) or (iter_v == -1) or (self.prev_scale is None)- if need_search:- params = self._run_search(weight, value)- self.prev_scale = params["scale"]- self.prev_d_scale = params["d_scale"]- if self._dq_kind == "asym":- self.prev_wmin = params["wmin"]- self.prev_d_wmin = params["d_wmin"]- else:- params = {- "scale": self.prev_scale.detach(),- "d_scale": self.prev_d_scale.detach(),- }- if self._dq_kind == "asym":- params["wmin"] = self.prev_wmin.detach()- params["d_wmin"] = self.prev_d_wmin.detach()-- bits = self.orig_layer.bits- if self._dq_kind == "asym":- group_size = 16 if bits == 2 else 32- weight_q = self.weight_quant_func(- weight,- params["scale"],- params["wmin"],- bits,- group_size,- v=value,- )- scale_out = {"scale": params["scale"], "d_scale": params["d_scale"]}- zp_out = {"wmin": params["wmin"], "d_wmin": params["d_wmin"]}- else:- weight_q = self.weight_quant_func(- weight,- params["scale"],- bits,- v=value,- )- scale_out = {"scale": params["scale"], "d_scale": params["d_scale"]}- zp_out = torch.full_like(params["scale"], int(2.0 ** (bits - 1)))-- weight_q = weight_q.to(weight.dtype)- if isinstance(self.orig_layer, transformers.pytorch_utils.Conv1D):- weight_q = weight_q.t()- return weight_q, scale_out, zp_out-- # Non-dq path: preserve original behavior.- quant_kwargs = {}- if hasattr(self.orig_layer, "super_bits"):- quant_kwargs["super_bits"] = self.orig_layer.super_bits- quant_kwargs["super_group_size"] = self.orig_layer.super_group_size- weight_q, scale, zp = self.weight_quant_func(- weight,- bits=self.orig_layer.bits,- group_size=self.orig_layer.group_size,- v=value,- min_scale=min_scale,- max_scale=max_scale,- scale_dtype=self.orig_layer.scale_dtype,- tensor_min=self.weight_min,- tensor_max=self.weight_max,- data_type=self.data_type,- q_scale_thresh=self.q_scale_thresh,- prev_scale=self.prev_scale,- prev_wmin=self.prev_wmin,- prev_d_scale=self.prev_d_scale,- prev_d_wmin=self.prev_d_wmin,- **quant_kwargs,- )- weight_q = weight_q.to(weight.dtype)- if isinstance(self.orig_layer, transformers.pytorch_utils.Conv1D):- weight_q = weight_q.t()- if isinstance(scale, dict) and "d_scale" in scale and self.prev_scale is None:- self.prev_scale = scale["scale"]- self.prev_d_scale = scale["d_scale"]- if isinstance(zp, dict):- self.prev_wmin = zp["wmin"]- self.prev_d_wmin = zp["d_wmin"]- elif self.prev_scale is None:- self.prev_scale = scale- # self.orig_layer.imatrix = None- return weight_q, scale, zp---def dq_wrapper_block(block, enable_minmax_tuning, enable_norm_bias_tuning, device="cpu", **kwargs):- """Wraps the layers in the given block with a custom Wrapper module.-- Args:- block: The input block containing linear and conv1d layers to be wrapped.- enable_minmax_tuning: A boolean indicating whether min-max tuning is enabled.-- Returns:- list: A list of names of the wrapped layers and unwrapped layers.- """- quantized_layers = []- unquantized_layers = []- for n, m in block.named_modules():- if isinstance(m, SUPPORTED_LAYER_TYPES):- if not check_to_quantized(m):- unquantized_layers.append(n)- continue- new_m = DQWrapperLinear(- m,- enable_minmax_tuning=enable_minmax_tuning,- enable_norm_bias_tuning=enable_norm_bias_tuning,- device=device,- **kwargs,- )- set_module(block, n, new_m)- quantized_layers.append(n)-- elif enable_norm_bias_tuning:- if "norm" in m.__class__.__name__.lower():- if m.__class__.__name__ in NORM_MAPPING.keys():- wrapper_layer_class = NORM_MAPPING[m.__class__.__name__]- new_m = wrapper_layer_class(m, device=device)- set_module(block, n, new_m)- elif "RMSNorm" in m.__class__.__name__:- logger.warning_once(- f"use LlamaRMSNorm to wrap {m.__class__.__name__}, please check the correctness yourself"- )- wrapper_layer_class = NORM_MAPPING["LlamaRMSNorm"]- new_m = wrapper_layer_class(m, device=device)- set_module(block, n, new_m)- else:- logger.warning_once(f"{m.__class__.__name__} is not supported")-- return quantized_layers, unquantized_layersdiff --git a/auto_round/algorithms/base.py b/auto_round/algorithms/base.pyindex 4590536c5..82fb89fd1 100644--- a/auto_round/algorithms/base.py+++ b/auto_round/algorithms/base.py@@ -13,5 +13,88 @@# limitations under the License.+from contextlib import contextmanager+from typing import Any++from auto_round.algorithms.registry import resolve_pipeline_member+from auto_round.schemes import QuantizationScheme++class BaseAlgorithm:pass+++class BasePipelineMember:+ """Shared interface for all members of a quantization pipeline."""++ model_context = None+ compress_context = None+ _scheme_context_fields = set(QuantizationScheme.get_attributes())+ bits: int | None+ group_size: int | tuple | None+ sym: bool | None+ data_type: str | None+ act_bits: int | None+ act_group_size: int | None+ act_sym: bool | None+ act_data_type: str | None+ act_dynamic: bool | None+ super_bits: int | None+ super_group_size: int | None+ super_sym: bool | None+ scale_dtype: str | None++ def __init__(self, config: Any = None) -> None:+ self.config = config+ self.scheme = getattr(config, "scheme", None)++ @classmethod+ def from_config(cls, config: Any) -> "BasePipelineMember":+ """Instantiate the registered implementation class for ``config``."""+ alg_cls = resolve_pipeline_member(config)+ if cls is alg_cls:+ return cls(config)+ return alg_cls(config)++ def bind(self, compressor: Any) -> None:+ """Wire shared context from the owning compressor."""+ self.model_context = compressor.model_context+ self.compress_context = compressor.compress_context+ self.scheme = getattr(compressor, "scheme_context", None)++ def prepare_run(self, compressor: Any) -> None:+ """Model-level preparation called once before block iteration starts."""+ return++ def get_act_calib_policy(self, ctx: Any) -> Any:+ """Return the activation calibration policy for this block."""+ from auto_round.algorithms.pipeline import ActCalibPolicy, CalibTiming, InputSource++ return ActCalibPolicy(when=CalibTiming.SKIP, source=InputSource.FP_CACHE)++ @contextmanager+ def block_forward_hooks(self, ctx: Any) -> Any:+ """Register algorithm-specific forward hooks for the reference forward."""+ yield []++ def finalize_run(self, compressor: Any) -> None:+ """Model-level teardown called once after all blocks are processed."""+ return+++def _make_scheme_property(name):+ def getter(self):+ scheme = getattr(self, "scheme", None)+ return getattr(scheme, name, None) if scheme is not None else None++ def setter(self, value):+ scheme = getattr(self, "scheme", None)+ if scheme is None:+ raise AttributeError(f"{type(self).__name__} has no bound scheme")+ setattr(scheme, name, value)++ return property(getter, setter)+++for _scheme_field in QuantizationScheme.get_attributes():+ setattr(BasePipelineMember, _scheme_field, _make_scheme_property(_scheme_field))diff --git a/auto_round/algorithms/config_resolver.py b/auto_round/algorithms/config_resolver.pynew file mode 100644index 000000000..ad1563abb--- /dev/null+++ b/auto_round/algorithms/config_resolver.py@@ -0,0 +1,162 @@+# Copyright (c) 2026 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.++from collections import defaultdict+from dataclasses import fields+from typing import Any+++def get_algorithm_class(config: Any):+ """Return the registered implementation class for a quantization config."""+ from auto_round.algorithms.registry import normalize_algorithm_config, resolve_pipeline_member++ try:+ return resolve_pipeline_member(normalize_algorithm_config(config))+ except ValueError:+ return None+++def is_preprocessor_config(config: Any) -> bool:+ """Whether *config* resolves to a BaseWeightTransformer implementation."""+ from auto_round.algorithms.transforms.base import BaseWeightTransformer++ alg_cls = get_algorithm_class(config)+ return alg_cls is not None and issubclass(alg_cls, BaseWeightTransformer)+++def is_block_quantizer_config(config: Any) -> bool:+ """Whether *config* resolves to a BaseQuantizer implementation."""+ from auto_round.algorithms.quantization.base import BaseQuantizer++ alg_cls = get_algorithm_class(config)+ return alg_cls is not None and issubclass(alg_cls, BaseQuantizer)+++def split_quantization_configs(configs: list[Any]) -> tuple[list[Any], list[Any]]:+ """Split configs into preprocessor and block-quantizer config lists."""+ preprocessors = []+ block_quantizers = []+ for config in configs:+ if is_preprocessor_config(config):+ preprocessors.append(config)+ elif is_block_quantizer_config(config):+ block_quantizers.append(config)+ return preprocessors, block_quantizers+++def _quantization_configs(configs: list[Any]) -> list[Any]:+ """Return configs that participate in quantization shared-field resolution."""+ from auto_round.algorithms.quantization.config import QuantizationConfig++ return [config for config in configs if isinstance(config, QuantizationConfig)]+++def _public_config_attrs(config: Any) -> dict[str, Any]:+ """Public, data-like attrs used for structural shared config resolution."""+ return {+ key: value+ for key, value in vars(config).items()+ if key != "scheme" and not key.startswith("_") and not callable(value)+ }+++def _format_shared_config_values(field: str, values: list[tuple[Any, Any]]) -> str:+ parts = [f"{type(config).__name__}.{field}={value!r}" for config, value in values]+ return ", ".join(parts)+++def _resolve_shared_scheme_values(configs: list[Any]) -> None:+ from auto_round.schemes import QuantizationScheme++ scheme_fields = {field.name for field in fields(QuantizationScheme)}+ field_values: dict[str, list[tuple[Any, Any]]] = defaultdict(list)+ for config in configs:+ for attr_name in getattr(config, "_user_set_scheme_fields", set()):+ if attr_name not in scheme_fields:+ continue+ value = getattr(config, attr_name, None)+ if value is not None:+ field_values[attr_name].append((config, value))++ for attr_name, values in field_values.items():+ unique_values = []+ for _, value in values:+ if not any(value == existing for existing in unique_values):+ unique_values.append(value)+ if len(unique_values) > 1:+ raise ValueError(+ f"Conflicting shared scheme field {attr_name!r}: "+ f"{_format_shared_config_values(attr_name, values)}. "+ "Use the same value for shared fields or pass scheme arguments through Compressor."+ )+ shared_value = unique_values[0]+ for config in configs:+ if hasattr(config, "scheme") and attr_name not in getattr(config, "_user_set_scheme_fields", set()):+ setattr(config.scheme, attr_name, shared_value)+++def resolve_shared_config_values(configs: list[Any]) -> list[Any]:+ """Merge shared public attrs across quantization configs without naming fields.++ A field is shared when at least two quantization configs define the same+ public attribute. ``None`` means "not set" and inherits from the single+ non-None value, while conflicting non-None values fail fast. Configs that do+ not define a field do not participate in that field.+ """+ quant_configs = _quantization_configs(configs)+ _resolve_shared_scheme_values(quant_configs)+ attrs_by_config = [(config, _public_config_attrs(config)) for config in quant_configs]+ field_to_configs: dict[str, list[Any]] = defaultdict(list)+ for config, attrs in attrs_by_config:+ for attr_name in attrs:+ field_to_configs[attr_name].append(config)++ for attr_name, field_configs in field_to_configs.items():+ if len(field_configs) < 2:+ continue++ field_attrs = [+ (config, attrs[attr_name])+ for config, attrs in attrs_by_config+ if any(config is field_config for field_config in field_configs)+ ]+ non_none_values = [(config, value) for config, value in field_attrs if value is not None]+ unique_values = []+ for _, value in non_none_values:+ if not any(value == existing for existing in unique_values):+ unique_values.append(value)+ if len(unique_values) > 1:+ raise ValueError(+ f"Conflicting shared config field {attr_name!r}: "+ f"{_format_shared_config_values(attr_name, non_none_values)}. "+ "Use the same value for shared fields or leave it unset on secondary algorithms."+ )+ if len(unique_values) == 1:+ shared_value = unique_values[0]+ for config in field_configs:+ if getattr(config, attr_name) is None:+ setattr(config, attr_name, shared_value)+ return configs+++def sync_shared_config_from(source_config: Any, target_configs: list[Any]) -> None:+ """Propagate resolved source values to targets that already define matching attrs."""+ source_attrs = _public_config_attrs(source_config)+ for target in _quantization_configs(target_configs):+ if target is source_config:+ continue+ target_attrs = _public_config_attrs(target)+ for attr_name, source_value in source_attrs.items():+ if attr_name in target_attrs and source_value is not None:+ setattr(target, attr_name, source_value)diff --git a/auto_round/algorithms/pipeline.py b/auto_round/algorithms/pipeline.pynew file mode 100644index 000000000..39141eda9--- /dev/null+++ b/auto_round/algorithms/pipeline.py@@ -0,0 +1,694 @@+# Copyright (c) 2026 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+"""Algorithm Fusion Pipeline abstraction.++This module defines the core data structures and utilities for composing+pre-processing algorithms (AWQ smooth, SmoothQuant, Rotation…) with a+block-quantization algorithm (RTN, SignRound/AutoRound…) into a single+shared-calibration pipeline.++Design invariants (see AWQ_REFACTOR_PLAN.md §0.0 and §3.0):+- ``QuantizationPipeline`` is the *first-class abstraction*; it is NOT just+ AWQ's helper.+- All block-wise scheduling in ``DataDrivenCompressor`` operates against+ ``QuantizationPipeline``, never against a concrete ``AWQQuantizer``.+- Single-algorithm use is expressed as+ ``QuantizationPipeline(preprocessors=[], block_quantizer=q)``, which is+ semantically identical to the current direct-quantizer path.+"""++from __future__ import annotations++from contextlib import ExitStack+from dataclasses import dataclass, field+from enum import IntEnum+from typing import TYPE_CHECKING, Any, Union++import torch++from auto_round.algorithms.config_resolver import (+ get_algorithm_class,+ resolve_shared_config_values,+ split_quantization_configs,+)+from auto_round.utils.device_manager import device_manager++if TYPE_CHECKING: # avoid circular imports at runtime+ import torch++ from auto_round.algorithms.base import BasePipelineMember+ from auto_round.algorithms.quantization.base import BaseQuantizer+ from auto_round.algorithms.quantization.config import QuantizationConfig+ from auto_round.algorithms.transforms.base import BaseWeightTransformer++# ---------------------------------------------------------------------------+# Policy+# ---------------------------------------------------------------------------+++class CalibTiming(IntEnum):+ """When to run the activation calibration hook forward pass.++ SKIP:+ No activation calibration needed (e.g. AWQ W4A16 pure weight-only smooth).+ WITH_REFERENCE:+ Register act-calib hook and run together with the FP reference forward+ (RTN / AutoRound default). When ``source == InputSource.QUANTIZED_INPUT``, a+ separate hook-only forward is run with ``quantized_input`` instead.+ AFTER_PREPROCESS:+ Run a dedicated forward pass *after* all ``pre_quantize_block`` calls+ complete (i.e. after smooth/rotation transforms are applied). Used for+ AWQ W8A8 / static-activation quantization where the activation+ distribution changes after smoothing.+ """++ SKIP = 0+ WITH_REFERENCE = 1+ AFTER_PREPROCESS = 2+++class InputSource(IntEnum):+ """Which tensor to use as the block input for the act-calib forward pass.++ FP_CACHE:+ The original FP calibration activations cached at the start of the run+ (``enable_quanted_input=False`` path, RTN default).+ QUANTIZED_INPUT:+ The quantized output of the previous block used as the next block's input+ (``enable_quanted_input=True`` path, SignRound / AutoRound).+ """++ FP_CACHE = 0+ QUANTIZED_INPUT = 1+++@dataclass+class ActCalibPolicy:+ """Activation calibration policy: when and from what inputs to collect act stats.++ Attributes:+ when: ``CalibTiming`` — controls the scheduling phase.+ source: ``InputSource`` — which tensor feeds the calibration forward.+ Ignored when ``when == CalibTiming.SKIP``.+ """++ when: CalibTiming = CalibTiming.WITH_REFERENCE+ source: InputSource = InputSource.FP_CACHE++ def __post_init__(self) -> None:+ if not isinstance(self.when, CalibTiming):+ try:+ self.when = CalibTiming(self.when)+ except ValueError as exc:+ raise ValueError(f"ActCalibPolicy.when must be a CalibTiming, got {self.when!r}") from exc+ if not isinstance(self.source, InputSource):+ try:+ self.source = InputSource(self.source)+ except ValueError as exc:+ raise ValueError(f"ActCalibPolicy.source must be an InputSource, got {self.source!r}") from exc++ @classmethod+ def no_collection(cls) -> "ActCalibPolicy":+ """Convenience factory: no activation calibration."""+ return cls(when=CalibTiming.SKIP, source=InputSource.FP_CACHE)+++def merge_policies(policies: list["ActCalibPolicy"]) -> "ActCalibPolicy":+ """Merge act-calib policies from multiple pipeline members.++ Rules:+ - ``when``: take the *latest* timing (``SKIP < WITH_REFERENCE < AFTER_PREPROCESS``).+ - ``source``: when two policies share the same ``when`` but differ in ``source``,+ that is a compatibility conflict → raise ``ValueError`` fail-fast.+ - Policies with ``when == CalibTiming.SKIP`` do **not** contribute a source+ constraint.++ Returns:+ A merged :class:`ActCalibPolicy`.+ """+ if not policies:+ return ActCalibPolicy.no_collection()++ merged_when = max(policies, key=lambda p: p.when).when++ if merged_when == CalibTiming.SKIP:+ return ActCalibPolicy.no_collection()++ contributing = [p for p in policies if p.when == merged_when]+ sources = {p.source for p in contributing}+ if len(sources) > 1:+ raise ValueError(+ f"Incompatible act-calib policies: multiple algorithms request "+ f"when={merged_when.name!r} but with different input sources: "+ f"{[s.name for s in sources]}. "+ "Use a compatible combination of algorithms or file an issue."+ )++ return ActCalibPolicy(when=merged_when, source=contributing[0].source)+++# ---------------------------------------------------------------------------+# Context dataclasses+# ---------------------------------------------------------------------------+++@dataclass+class BlockIO:+ """Owns per-block calibration inputs, outputs, and batch forward mechanics."""++ _fp_inputs: Any+ _input_others: dict+ _quantized_inputs: Any = None+ _reference_outputs: Any = None+ _quantized_outputs: Any = None+ _active_source: InputSource = InputSource.FP_CACHE+ batch_dim: int = 0+ seqlen: int = 2048+ shared_cache_keys: tuple = ()+ _quantizer: Any = None+ _block: Any = None++ def _inputs_for(self, source: InputSource):+ if source == InputSource.QUANTIZED_INPUT:+ return self._quantized_inputs+ return self._fp_inputs++ def has_quantized_inputs(self) -> bool:+ return self._quantized_inputs is not None++ @property+ def num_samples(self) -> int:+ input_ids = self._inputs_for(self._active_source)+ if input_ids is None:+ return 0+ return self._num_samples(input_ids)++ def _release_references(+ self,+ *,+ keep_fp_inputs: bool = False,+ keep_quantized_inputs: bool = False,+ keep_reference_outputs: bool = False,+ keep_input_others: bool = False,+ ) -> None:+ """Drop large cached references once a block has finished using them."""+ if not keep_fp_inputs:+ self._fp_inputs = None+ if not keep_quantized_inputs:+ self._quantized_inputs = None+ self._quantized_outputs = None+ if not keep_reference_outputs:+ self._reference_outputs = None+ if not keep_input_others:+ self._input_others = {}++ def finish(self) -> None:+ self._release_references()+ self._quantizer = None+ self._block = None++ def _select_inputs_for_source(self, source: InputSource, indices):+ input_ids = self._inputs_for(source)+ if input_ids is None:+ raise ValueError(f"Input source {source.name} is unavailable for this block.")+ return self._select_inputs(input_ids, self._input_others, indices)++ def forward_block_batch(+ self,+ indices: torch.Tensor,+ *,+ device: Union[str, torch.device],+ cache_device: Union[str, torch.device, None] = None,+ ) -> Any:+ quantizer = self._quantizer+ block = self._block+ if quantizer is None or block is None:+ raise ValueError("BlockIO forward_batch requires bound quantizer and block.")++ input_ids, input_others = self._select_inputs_for_source(self._active_source, indices)+ output = self._run_block(block, quantizer, input_ids, input_others, device)+ output = self._normalize_output_for_loss(output)+ if cache_device is not None:+ output = output.to(cache_device)+ return output++ def _run_block(self, block, quantizer, input_ids, input_others, device):+ return quantizer._resolve_block_forward()(+ block,+ input_ids,+ input_others,+ quantizer.model_context.amp,+ quantizer.model_context.amp_dtype,+ device,+ 0,+ )++ @torch.no_grad()+ def _collect_outputs(self, block, quantizer, *, source: InputSource, batch_size: int, save: bool = True):+ input_ids = self._inputs_for(source)+ if input_ids is None:+ raise ValueError(f"Input source {source.name} is unavailable for this block.")+ outputs = self._init_output_buffer()+ for start, end in self._iter_batch_ranges(input_ids, batch_size):+ indices = torch.arange(start, end).to(torch.long)+ previous_source = self._active_source+ self._active_source = source+ try:+ output = self.forward_block_batch(+ indices, device=device_manager.device, cache_device=quantizer.compress_context.cache_device+ )+ finally:+ self._active_source = previous_source+ if save:+ self._append_output(outputs, output, quantizer)+ quantizer.compress_context.clear_memory()+ return outputs++ @torch.no_grad()+ def collect_reference(self, hooks=None) -> Any:+ _ = hooks+ outputs = self._collect_outputs(+ self._block,+ self._quantizer,+ source=InputSource.FP_CACHE,+ batch_size=self._quantizer.batch_size,+ )+ self._reference_outputs = outputs+ if self._active_source == InputSource.QUANTIZED_INPUT:+ self._fp_inputs = None+ return outputs++ @torch.no_grad()+ def collect_quantized_stats(self, hooks=None) -> Any:+ _ = hooks+ if not self.has_quantized_inputs():+ return None+ return self._collect_outputs(+ self._block,+ self._quantizer,+ source=InputSource.QUANTIZED_INPUT,+ batch_size=self._quantizer.batch_size,+ save=False,+ )++ @torch.no_grad()+ def collect_next_inputs(self) -> Any:+ if self._quantizer is None or not self._quantizer.enable_quanted_input:+ return None+ outputs = self._collect_outputs(+ self._block,+ self._quantizer,+ source=self._active_source,+ batch_size=self._quantizer.batch_size,+ )+ self._quantized_outputs = outputs+ return outputs++ def seed_reference(self, fp_inputs: Any, reference_outputs: Any) -> None:+ self._fp_inputs = fp_inputs+ self._reference_outputs = reference_outputs++ def get_reference_outputs(self, indices: torch.Tensor, *, device=None) -> torch.Tensor:+ if self._reference_outputs is None:+ raise ValueError("Reference outputs have not been collected for this block.")+ output = torch.cat([self._reference_outputs[i] for i in indices], dim=self.batch_dim)+ return output.to(device) if device is not None else output++ def _normalize_output_for_loss(self, output: Any) -> torch.Tensor:+ if isinstance(output, torch.Tensor):+ return output+ if isinstance(output, (tuple, list)) and len(output) == 1 and isinstance(output[0], torch.Tensor):+ return output[0]+ raise TypeError(+ "BlockIO forward must return a tensor or a single-tensor tuple/list after normalization. "+ f"Got {type(output).__name__}."+ )++ def _count_input_elements(self, indices, *, source: InputSource | None = None) -> int:+ source = self._active_source if source is None else source+ input_ids = self._inputs_for(source)+ if isinstance(input_ids, dict):+ current_input_ids = [input_ids["hidden_states"][i] for i in indices]+ else:+ current_input_ids = [input_ids[i] for i in indices]+ return sum(t.numel() for t in current_input_ids)++ def count_batch_elements(self, indices: torch.Tensor) -> int:+ return self._count_input_elements(indices, source=self._active_source)++ def _preprocess_block_inputs(self, input_ids, input_others: dict, block):+ return input_ids, input_others++ def _iter_batch_ranges(self, input_ids, batch_size: int):+ for start in range(0, self._num_samples(input_ids), batch_size):+ end = min(self._num_samples(input_ids), start + batch_size)+ yield start, end++ def _num_samples(self, input_ids) -> int:+ return len(input_ids)++ def _init_output_buffer(self):+ return []++ def _append_output(self, outputs, output, quantizer) -> None:+ if quantizer.batch_size == 1:+ outputs.append(output)+ else:+ outputs.extend(list(torch.split(output, 1, dim=self.batch_dim)))++ def _select_inputs(self, input_ids, input_others: dict, indices):+ if isinstance(input_ids, list):+ current_input_ids = [input_ids[i] for i in indices]+ current_input_ids = torch.cat(current_input_ids, dim=self.batch_dim)+ elif isinstance(input_ids, dict):+ current_input_ids = {}+ for key in input_ids.keys():+ current_input_ids[key] = torch.cat([input_ids[key][i] for i in indices], dim=self.batch_dim)+ else:+ raise TypeError(f"Unsupported input container type: {type(input_ids).__name__}")++ current_input_others = {"positional_inputs": input_others["positional_inputs"]}+ for key in input_others.keys():+ if "positional_inputs" in key:+ continue+ if key in self.shared_cache_keys:+ val = input_others[key]+ if isinstance(val, list) and len(val) == 1:+ current_input_others[key] = val[0]+ elif isinstance(val, list) and len(val) > 1:+ idx = int(indices[0]) if len(indices) == 1 else 0+ current_input_others[key] = val[idx] if idx < len(val) else val[0]+ else:+ current_input_others[key] = val+ elif not isinstance(input_others[key], (str, bool, type(None))):+ current_input_others[key] = [input_others[key][i] for i in indices]+ if len(current_input_others[key]) == 1:+ current_input_others[key] = current_input_others[key][0]+ else:+ current_input_others[key] = torch.cat(current_input_others[key], dim=self.batch_dim)+ else:+ current_input_others[key] = input_others[key]+ return current_input_ids, current_input_others+++class DiffusionBlockIO(BlockIO):+ """BlockIO variant for diffusion blocks with dict inputs and tuple outputs."""++ def __init__(self, *args, output_config: list[str] | None = None, **kwargs) -> None:+ super().__init__(*args, **kwargs)+ self.output_config = output_config or ["hidden_states"]++ def _preprocess_block_inputs(self, input_ids, input_others: dict, block):+ if not isinstance(input_ids, dict):+ return input_ids, input_others+ extra_keys = [key for key in list(input_ids.keys()) if key not in self.output_config]+ for key in extra_keys:+ input_others[key] = input_ids.pop(key)+ return input_ids, input_others++ def _run_block(self, block, quantizer, input_ids, input_others, device):+ if not isinstance(input_ids, dict):+ return super()._run_block(block, quantizer, input_ids, input_others, device)+ input_ids = dict(input_ids)+ input_others = dict(input_others)+ hidden_states = input_ids.pop("hidden_states")+ input_others.update(input_ids)+ return quantizer._resolve_block_forward()(+ block,+ hidden_states,+ input_others,+ quantizer.model_context.amp,+ quantizer.model_context.amp_dtype,+ device,+ None,+ )++ def _num_samples(self, input_ids) -> int:+ return len(input_ids["hidden_states"]) if isinstance(input_ids, dict) else len(input_ids)++ def _normalize_output_for_loss(self, output: Any) -> torch.Tensor:+ if isinstance(output, torch.Tensor):+ return output+ if not isinstance(output, (tuple, list)):+ raise TypeError(+ "DiffusionBlockIO forward must return a tensor or tuple/list of tensors. "+ f"Got {type(output).__name__}."+ )+ if "hidden_states" not in self.output_config:+ raise ValueError(+ "DiffusionBlockIO requires 'hidden_states' in output_config to normalize outputs for loss."+ )+ hidden_state_index = self.output_config.index("hidden_states")+ if hidden_state_index >= len(output):+ raise ValueError(+ f"Diffusion block output has {len(output)} tensors, but hidden_states index is {hidden_state_index}."+ )+ hidden_states = output[hidden_state_index]+ if not isinstance(hidden_states, torch.Tensor):+ raise TypeError("DiffusionBlockIO expected hidden_states to be a tensor after output normalization.")+ return hidden_states+++@dataclass+class BlockContext:+ """Per-block context threaded through the lifecycle hooks.++ Passed to lifecycle methods like ``block_forward_hooks``,+ ``pre_quantize_block``, ``post_quantize_block``, etc.++ ``block_names`` preserves the *scheduling group* (which may contain more+ than one block when ``nblocks > 1``). Pre-processing algorithms that+ only support single-block operation (e.g. AWQ Phase-1) must check+ ``len(block_names) == 1`` in ``prepare_block_group`` and raise+ ``ValueError`` with a user-readable message.+ """++ model: "torch.nn.Module"+ block: "torch.nn.Module"+ block_names: list[str] # scheduling group; len > 1 when nblocks > 1+ block_name: str # = block_names[0] for single-block; descriptive label for multi+ block_index: int # 0-based index within the current all_blocks group+ io: BlockIO+ bs: int = 1+ loss_device: Union[str, "torch.device", None] = None+ device: Union[str, "torch.device", None] = None+ mid_iter_mem_check: bool = False+ is_mllm: bool = False # fail-fast gate for algorithms that don't support MLLM+ is_diffusion: bool = False # fail-fast gate for algorithms that don't support diffusion+ pbar: Any = None+ # Names of FP parameters modified in-place by preprocessors (for example,+ # smooth source norm weights). Populated by pre_quantize_block; read by Compressor+ # during immediate_save to persist FP param changes alongside packed weights.+ # Pipelines without FP-param preprocessors leave this empty, no behavior change.+ modified_fp_params: list = field(default_factory=list)++ def mark_modified_fp_params(self, param_names: list[str]) -> None:+ """Called by preprocessors to declare which FP params were modified in-place."""+ self.modified_fp_params.extend(param_names)++ def collect_reference(self, hooks=None) -> Any:+ return self.io.collect_reference(hooks)++ def collect_quantized_stats(self, hooks=None) -> Any:+ return self.io.collect_quantized_stats(hooks)++ def collect_next_inputs(self) -> Any:+ return self.io.collect_next_inputs()++ def has_quantized_inputs(self) -> bool:+ return self.io.has_quantized_inputs()++ @property+ def num_samples(self) -> int:+ return self.io.num_samples++ def count_batch_elements(self, indices: torch.Tensor) -> int:+ return self.io.count_batch_elements(indices)++ def get_reference_outputs(self, indices: torch.Tensor, *, device=None) -> torch.Tensor:+ return self.io.get_reference_outputs(indices, device=device)++ def forward_block_batch(+ self,+ indices: torch.Tensor,+ *,+ device: Union[str, torch.device],+ cache_device: Union[str, torch.device, None] = None,+ ) -> Any:+ return self.io.forward_block_batch(indices, device=device, cache_device=cache_device)++ def finish(self) -> None:+ self.io.finish()+++# ---------------------------------------------------------------------------+# QuantizationPipeline+# ---------------------------------------------------------------------------+++@dataclass+class QuantizationPipeline:+ """An ordered composition of pre-processing quantizers + one block quantizer.++ The ``preprocessors`` list is order-sensitive: algorithms are applied in+ the listed order (e.g. ``[Rotation, AWQ]``). There must be **exactly one**+ ``block_quantizer`` (the terminal weight-compression step).++ Single-algorithm use:+ ``QuantizationPipeline(preprocessors=[], block_quantizer=rtn_quantizer)``+ is semantically equivalent to the current direct-quantizer path; the+ compressor's existing ``self.quantizer`` call-sites are transparently+ forwarded to ``block_quantizer`` via a ``@property``.+ """++ preprocessors: list["BaseWeightTransformer"] = field(default_factory=list)+ block_quantizer: "BaseQuantizer" = None # type: ignore[assignment]++ def __post_init__(self) -> None:+ if self.block_quantizer is None:+ raise ValueError("QuantizationPipeline requires a non-None block_quantizer.")+ from auto_round.algorithms.quantization.base import BaseQuantizer+ from auto_round.algorithms.transforms.base import BaseWeightTransformer++ for q in self.preprocessors:+ if not isinstance(q, BaseWeightTransformer):+ raise TypeError(+ f"{type(q).__name__} is listed as a preprocessor but does not " f"inherit BaseWeightTransformer."+ )+ if not isinstance(self.block_quantizer, BaseQuantizer):+ raise TypeError(+ f"{type(self.block_quantizer).__name__} is used as block_quantizer but does not "+ f"inherit BaseQuantizer."+ )++ def all(self) -> "list[BasePipelineMember]":+ """Return all members in pipeline order: preprocessors then block_quantizer."""+ return [*self.preprocessors, self.block_quantizer]++ @classmethod+ def from_configs(cls, configs: list, compressor: Any = None) -> "QuantizationPipeline":+ """Construct a ``QuantizationPipeline`` from a list of algorithm config instances.++ Resolution rules:+ 1. If no ``QuantizationConfig`` with a ``BaseQuantizer`` is found in *configs*,+ a default :class:`RTNConfig` is appended automatically.+ 2. If ``compressor`` indicates a diffusion model, :class:`DiffusionMixin` is+ dynamically prepended to each algorithm class's MRO before instantiation,+ activating diffusion-aware method overrides without touching the class definitions.+ 3. Instances of ``BaseWeightTransformer`` go into ``preprocessors`` (in order).+ 4. Exactly one ``BaseQuantizer`` becomes ``block_quantizer``.+ 5. Multiple block-quantization configs raise ``ValueError``.+ 6. If ``compressor`` is provided, every member is bound to it.+ """+ from auto_round.algorithms.quantization.base import BaseQuantizer, DiffusionMixin+ from auto_round.algorithms.quantization.config import QuantizationConfig+ from auto_round.algorithms.transforms.base import BaseWeightTransformer++ is_diffusion = compressor is not None and getattr(compressor.model_context, "is_diffusion", False)+ configs = list(configs)++ # Ensure at least one terminal block quantizer is present; fall back to RTN.+ _, block_quantizer_configs = split_quantization_configs(configs)+ has_quantizer = bool(block_quantizer_configs)+ if not has_quantizer:+ from auto_round.algorithms.quantization.rtn.config import RTNConfig++ configs = list(configs) + [RTNConfig()]++ configs = resolve_shared_config_values(configs)++ def _resolve_cls(cfg):+ alg_cls = get_algorithm_class(cfg)+ if alg_cls is None:+ raise ValueError(f"Unknown algorithm config type {type(cfg).__name__!r}.")+ if is_diffusion and not issubclass(alg_cls, DiffusionMixin):+ alg_cls = type(alg_cls.__name__, (DiffusionMixin, alg_cls), {})+ return alg_cls++ preprocessors = []+ block_quantizers = []++ for cfg in configs:+ if not isinstance(cfg, QuantizationConfig):+ continue+ from auto_round.algorithms.registry import normalize_algorithm_config++ cfg = normalize_algorithm_config(cfg)+ alg_cls = _resolve_cls(cfg)+ q = alg_cls(cfg)+ if compressor is not None:+ q.bind(compressor)+ if isinstance(q, BaseWeightTransformer):+ preprocessors.append(q)+ elif isinstance(q, BaseQuantizer):+ block_quantizers.append(q)+ else:+ raise TypeError(+ f"Algorithm class {type(q).__name__} must inherit either " "BaseWeightTransformer or BaseQuantizer."+ )++ if len(block_quantizers) > 1:+ raise ValueError(+ f"QuantizationPipeline allows exactly one block-quantization config, "+ f"but got {len(block_quantizers)}: "+ f"{[type(q).__name__ for q in block_quantizers]}. "+ "Ensure only one of RTNConfig / SignRoundConfig / etc. is in the pipeline."+ )++ seen_preprocessors = set()+ for preprocessor in preprocessors:+ name = type(preprocessor).__name__+ if name in seen_preprocessors:+ raise ValueError(+ f"Duplicate preprocessor {name} in QuantizationPipeline. "+ "Repeated instances of the same preprocessor are not supported yet."+ )+ seen_preprocessors.add(name)++ return cls(preprocessors=preprocessors, block_quantizer=block_quantizers[0])++ # ── Convenience act-calib helpers ────────────────────────────────────────++ def get_merged_policy(self, ctx: "BlockContext") -> ActCalibPolicy:+ """Compute the merged act-calib policy for the current block."""+ policies = [q.get_act_calib_policy(ctx) for q in self.all()]+ return merge_policies(policies)++ def enter_block_forward_hooks(self, ctx: "BlockContext", fwd_stack: ExitStack) -> list:+ """Enter all pipeline members' ``block_forward_hooks`` into *fwd_stack*.++ Iterates over all members (preprocessors then block_quantizer) in order,+ entering each member's ``block_forward_hooks(ctx)`` context manager into+ the provided :class:`contextlib.ExitStack`.++ Returns the hook handles yielded by the terminal ``block_quantizer``+ so the caller can determine whether any act-calib hooks were registered+ (needed to decide whether a second forward with quantized inputs is required).+ """+ self.enter_preprocessor_hooks(ctx, fwd_stack)+ return self.enter_quantizer_hooks(ctx, fwd_stack)++ def enter_preprocessor_hooks(self, ctx: "BlockContext", fwd_stack: ExitStack) -> None:+ """Enter preprocessor hooks only.++ Preprocessor hooks collect stats from the FP reference forward. They are+ intentionally separate from terminal quantizer hooks so quantizer stats+ can be collected from quantized inputs when required by policy.+ """+ for pre in self.preprocessors:+ fwd_stack.enter_context(pre.block_forward_hooks(ctx))++ def enter_quantizer_hooks(self, ctx: "BlockContext", fwd_stack: ExitStack) -> list:+ """Enter terminal block-quantizer hooks only and return their handles."""+ return fwd_stack.enter_context(self.block_quantizer.block_forward_hooks(ctx))diff --git a/auto_round/algorithms/quantization/__init__.py b/auto_round/algorithms/quantization/__init__.pyindex 5b297df1d..98acde190 100644--- a/auto_round/algorithms/quantization/__init__.py+++ b/auto_round/algorithms/quantization/__init__.py@@ -12,13 +12,33 @@# See the License for the specific language governing permissions and# limitations under the License.-from auto_round.algorithms.quantization.base import BaseQuantizers+from auto_round.algorithms.base import BasePipelineMember+from auto_round.algorithms.quantization.base import BaseQuantizer, DiffusionMixin, RTNLayerFallbackMixinfrom auto_round.algorithms.quantization.config import QuantizationConfig-from auto_round.algorithms.quantization.sign_round.config import SignRoundConfig+from auto_round.algorithms.pipeline import (+ ActCalibPolicy,+ CalibTiming,+ InputSource,+ BlockContext,+ QuantizationPipeline,+ merge_policies,+)+from auto_round.algorithms.quantization.sign_round.config import AdamRoundConfig, SignRoundConfig, SignRoundV2Configfrom auto_round.algorithms.quantization.sign_round.quantizer import SignRoundQuantizerfrom auto_round.algorithms.quantization.sign_roundv2 import SignRoundV2Quantizerfrom auto_round.algorithms.quantization.adam_round.adam import AdamRoundQuantizer-from auto_round.algorithms.quantization.awq.config import AWQConfig-from auto_round.algorithms.quantization.awq.quantizer import AWQQuantizer-from auto_round.algorithms.quantization.rtn.config import RTNConfig+from auto_round.algorithms.quantization.rtn.config import OptimizedRTNConfig, RTNConfigfrom auto_round.algorithms.quantization.rtn.quantizer import RTNQuantizer, OptimizedRTNQuantizer+from auto_round.algorithms.transforms.base import BaseWeightTransformer+++def __getattr__(name):+ if name == "AWQConfig":+ from auto_round.algorithms.transforms.awq.config import AWQConfig++ return AWQConfig+ if name == "AWQQuantizer":+ from auto_round.algorithms.transforms.awq.quantizer import AWQQuantizer++ return AWQQuantizer+ raise AttributeError(f"module {__name__!r} has no attribute {name!r}")diff --git a/auto_round/algorithms/quantization/adam_round/adam.py b/auto_round/algorithms/quantization/adam_round/adam.pyindex b017c4a02..ce5b4eec7 100644--- a/auto_round/algorithms/quantization/adam_round/adam.py+++ b/auto_round/algorithms/quantization/adam_round/adam.py@@ -15,15 +15,18 @@ from typing import Unionimport torch+from auto_round.algorithms.quantization.sign_round.config import AdamRoundConfigfrom auto_round.algorithms.quantization.sign_round.quantizer import SignRoundQuantizer+from auto_round.algorithms.registry import register_pipeline_memberfrom auto_round.schemes import QuantizationSchemefrom auto_round.utils import check_is_cpu, htcore, is_hpex_availablefrom auto_round.utils.device_manager import device_manager+@register_pipeline_member(AdamRoundConfig)class AdamRoundQuantizer(SignRoundQuantizer):- def __init__(self, config):+ def __init__(self, config: AdamRoundConfig) -> None:super().__init__(config)self.momentum = None # AdamW handles momentum internallydiff --git a/auto_round/algorithms/quantization/awq/__init__.py b/auto_round/algorithms/quantization/awq/__init__.pydeleted file mode 100644index 14a492441..000000000--- a/auto_round/algorithms/quantization/awq/__init__.py+++ /dev/null@@ -1,13 +0,0 @@-# Copyright (c) 2026 Intel Corporation-#-# Licensed under the Apache License, Version 2.0 (the "License");-# you may not use this file except in compliance with the License.-# You may obtain a copy of the License at-#-# http://www.apache.org/licenses/LICENSE-2.0-#-# Unless required by applicable law or agreed to in writing, software-# distributed under the License is distributed on an "AS IS" BASIS,-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.-# See the License for the specific language governing permissions and-# limitations under the License.diff --git a/auto_round/algorithms/quantization/awq/config.py b/auto_round/algorithms/quantization/awq/config.pydeleted file mode 100644index 50340eeda..000000000--- a/auto_round/algorithms/quantization/awq/config.py+++ /dev/null@@ -1,77 +0,0 @@-# Copyright (c) 2026 Intel Corporation-#-# Licensed under the Apache License, Version 2.0 (the "License");-# you may not use this file except in compliance with the License.-# You may obtain a copy of the License at-#-# http://www.apache.org/licenses/LICENSE-2.0-#-# Unless required by applicable law or agreed to in writing, software-# distributed under the License is distributed on an "AS IS" BASIS,-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.-# See the License for the specific language governing permissions and-# limitations under the License.--from auto_round.algorithms.quantization.config import QuantizationConfig---class AWQConfig(QuantizationConfig):- """Configuration for AWQ (Activation-Aware Weight Quantization).-- AWQ protects salient weight channels by analyzing activation patterns and- applying channel-wise scaling to reduce quantization error. The scaling- factors are computed offline via a grid search over calibration data.- After smoothing, standard RTN quantization is applied to the adjusted weights.-- Args:- duo_scaling: Use both activations and weights for the scaling factor.- True: always use duo scaling. False: only activation scaling.- "both": search both modes and pick the best.- n_grid: Number of grid points for the scaling-ratio search.- seqlen: Calibration sequence length.- nsamples: Number of calibration samples. Grid search time scales- linearly with nsamples (each batch triggers one parent forward- per grid point).- batch_size: Batch size for calibration forward passes.- apply_smooth: Whether to apply AWQ smoothing (channel scaling).- True: run both smoothing and quantization (default AWQ behavior).- False: skip smoothing, only run RTN quantization.- mappings: Explicit AWQ mappings. Each mapping is a dict with keys- ``smooth_layer`` (str) and ``balance_layers`` (list[str]).- If None, mappings are inferred automatically from the model structure.- **kwargs: Forwarded to ``QuantizationConfig`` (bits, group_size, sym, …).- """-- _alg_cls = "AWQQuantizer"-- def __init__(- self,- *,- duo_scaling: bool | str = True,- n_grid: int = 20,- seqlen: int = 2048,- nsamples: int = 128,- batch_size: int = 8,- apply_smooth: bool = True,- mappings: list[dict] = None,- **kwargs,- ):- super().__init__(**kwargs)-- if isinstance(duo_scaling, str) and duo_scaling != "both":- raise ValueError(f"duo_scaling must be True, False, or 'both', got '{duo_scaling}'")- self.duo_scaling = duo_scaling- self.n_grid = n_grid- self.seqlen = seqlen- self.nsamples = nsamples- self.batch_size = batch_size- self.apply_smooth = apply_smooth- self.mappings = mappings- self.infer_bs_coeff = 1- self.batch_dim = None-- # TODO adjust those args defaults after architecture refactoring:- self.enable_quanted_input = False # AWQ doesn't cascade quantized block outputs- # AWQ uses plain RTN (no iterative optimization) for the quantization step.- self.disable_opt_rtn = True- self.orig_disable_opt_rtn = Truediff --git a/auto_round/algorithms/quantization/awq/quantizer.py b/auto_round/algorithms/quantization/awq/quantizer.pydeleted file mode 100644index 1a6d850e1..000000000--- a/auto_round/algorithms/quantization/awq/quantizer.py+++ /dev/null@@ -1,957 +0,0 @@-# Copyright (c) 2026 Intel Corporation-#-# Licensed under the Apache License, Version 2.0 (the "License");-# you may not use this file except in compliance with the License.-# You may obtain a copy of the License at-#-# http://www.apache.org/licenses/LICENSE-2.0-#-# Unless required by applicable law or agreed to in writing, software-# distributed under the License is distributed on an "AS IS" BASIS,-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.-# See the License for the specific language governing permissions and-# limitations under the License.--"""AWQ (Activation-Aware Weight Quantization) quantizer.--The algorithm:-1. Collects per-channel activation magnitudes during calibration.-2. For each smooth-balance mapping, performs a grid search over scaling ratios- to find the one that minimizes quantization error (output-based loss).-3. Applies the best channel-wise scaling: balance_layer.weight *= scales,- smooth_layer.weight /= scales (or smooth_layer.bias /= scales if 1-D).---Reference implementations:- - AutoAWQ: https://github.com/casper-hansen/AutoAWQ- - llm-compressor: https://github.com/vllm-project/llm-compressor-"""--from __future__ import annotations--import contextlib-import inspect-import traceback--import torch-from tqdm import tqdm--from auto_round.algorithms.quantization.awq.config import AWQConfig-from auto_round.algorithms.quantization.awq.mappings import (- ResolvedMapping,- _extract_block_prefix,- resolve_mappings,-)-from auto_round.algorithms.quantization.base import BaseQuantizers-from auto_round.compressors.shard_writer import ShardWriter-from auto_round.compressors.utils import immediate_pack-from auto_round.data_type.utils import (- get_quant_func,- reshape_pad_tensor_by_group_size,- revert_tensor_by_pad,- update_block_global_scale_if_needed,-)-from auto_round.logger import logger-from auto_round.utils import (- check_to_quantized,- clear_memory,- convert_module_to_hp_if_necessary,- get_module,- set_amax_for_all_moe_layers,- set_module,-)-from auto_round.utils.device_manager import device_manager-from auto_round.wrapper import WrapperLinear-from auto_round.wrapper import WrapperMultiblock as _WrapperMultiblock---class AWQQuantizer(BaseQuantizers):- """AWQ quantizer: activation-aware channel scaling + delegated quantization.-- AWQ is a pre-quantization transform that applies channel-wise scaling to reduce- quantization error. The actual quantization is delegated to an inner- quantizer (currently RTN by default).- """-- def __init__(self, config: AWQConfig):- super().__init__(config)- self.duo_scaling = config.duo_scaling- self.n_grid = config.n_grid- self.apply_smooth = config.apply_smooth-- # Populated during calibration- self._user_mappings = config.mappings- self._resolved_mappings: list[ResolvedMapping] = []- self._block_groups: dict[str, list[ResolvedMapping]] = {}- self._activation_stats: dict[str, list[torch.Tensor]] = {}- # Parent module kwargs cache: parent_module → list of kwargs dicts- self._parent_args_cache: dict[torch.nn.Module, list[dict]] = {}- self._parent_signatures: dict[int, inspect.BoundArguments] = {}- self._smoothing_applied: bool = False-- # Fail fast: validate scheme at construction time- self._check_scheme_compatibility()-- def bind(self, compressor) -> None:- """Wire shared state and validate compressor settings for AWQ compatibility."""- super().bind(compressor)- # Check for unsupported compressor-level args- nblocks = getattr(compressor, "nblocks", 1)- if nblocks > 1:- logger.warning(- f"AWQ does not support nblocks > 1 (got nblocks={nblocks}). "- f"AWQ smoothing resolves activation hooks and mappings per single block prefix. "- f"Falling back to nblocks=1."- )- compressor.nblocks = 1-- def __setattr__(self, name, value):- """Trigger model-level AWQ setup when compress_context is assigned."""- super().__setattr__(name, value)- if name == "compress_context" and value is not None:- self._prepare_model()-- # ── Public API ────────────────────────────────────────────────────────────-- def resolve_all_mappings(self, model: torch.nn.Module) -> dict[str, list[ResolvedMapping]]:- """Resolve all AWQ mappings and group by block prefix.-- Call this once before the block-by-block loop. The resolved mappings- are stored internally and used by ``register_activation_hooks`` and- ``smooth_block`` for per-block filtering.-- Returns:- Dictionary mapping block prefix (e.g. "model.layers.0") to the- list of resolved mappings for that block.- """- self._resolved_mappings = resolve_mappings(model, self._user_mappings)- block_groups: dict[str, list[ResolvedMapping]] = {}- for m in self._resolved_mappings:- prefix = _extract_block_prefix(m.smooth_name)- block_groups.setdefault(prefix, []).append(m)- self._block_groups = block_groups- return block_groups-- def apply_smoothing(- self,- model: torch.nn.Module,- device: torch.device | str | None = None,- ) -> None:- """Apply AWQ smoothing to the model weights.-- This is the core AWQ step: for each mapping, find the best scaling- factor and apply it to smooth/balance layers.-- When *device* is provided the model is assumed to reside on CPU and- only the modules needed for each mapping are temporarily moved to- *device* during the grid search. This dramatically reduces peak VRAM- (~14 GB savings for an 8B model).-- Should be called AFTER calibration data has been collected and BEFORE- RTN quantization.- """- if self._smoothing_applied:- return-- # Resolve mappings- self._resolved_mappings = resolve_mappings(model, self._user_mappings)- if not self._resolved_mappings:- logger.warning("No AWQ mappings resolved; skipping smoothing.")- self._smoothing_applied = True- return-- if not self._activation_stats:- logger.error("No activation statistics collected for AWQ smoothing.")-- logger.info(f"Applying AWQ smoothing to {len(self._resolved_mappings)} mappings...")-- for mapping in tqdm(self._resolved_mappings, desc="AWQ Smoothing"):- if mapping.smooth_name not in self._activation_stats:- logger.warning(f"No activation stats for '{mapping.smooth_name}', skipping this layer.")- continue-- act_sum, act_count = self._activation_stats.pop(mapping.smooth_name)- if act_count == 0:- logger.warning(f"Zero activation count for '{mapping.smooth_name}', skipping this layer.")- continue-- # Mean activation per channel (act_count is a plain int now,- # avoiding a CPU tensor allocation per channel).- x_mean = (act_sum / act_count).to(torch.float32)- del act_sum-- with self._align_modules(mapping, device):- best_scales = self._grid_search_scales(mapping, x_mean)-- if best_scales is not None:- self._apply_scales(mapping, best_scales)-- # Eagerly free per-mapping parent cache to reduce RAM- parent = mapping.parent- self._parent_args_cache.pop(parent, None)-- self._activation_stats.clear()- self._parent_args_cache.clear()- self._parent_signatures.clear()- self._smoothing_applied = True- logger.info("AWQ smoothing complete.")-- def smooth_block(self, block_prefix: str) -> None:- """Apply AWQ smoothing for one block's mappings.-- Assumes activation stats and parent kwargs have already been collected- for this block via ``register_activation_hooks`` + block forward.- The block's modules are expected to be on the compute device, so- no device alignment is needed (unlike ``apply_smoothing`` which uses- ``_align_modules`` for CPU-offloaded models).-- Args:- block_prefix: Block prefix (e.g. "model.layers.0") identifying- which mappings to smooth.- """- block_mappings = [m for m in self._resolved_mappings if _extract_block_prefix(m.smooth_name) == block_prefix]- for mapping in block_mappings:- if mapping.smooth_name not in self._activation_stats:- logger.warning(f"No activation stats for '{mapping.smooth_name}', skipping.")- continue-- act_sum, act_count = self._activation_stats.pop(mapping.smooth_name)- if act_count == 0:- logger.warning(f"Zero activation count for '{mapping.smooth_name}', skipping.")- continue-- x_mean = (act_sum / act_count).to(torch.float32)- del act_sum-- # Block is on the compute device — grid search runs in-place- best_scales = self._grid_search_scales(mapping, x_mean)-- if best_scales is not None:- self._apply_scales(mapping, best_scales)-- # Free parent kwargs cache after ALL mappings for this block are done.- # (All mappings in a block typically share the same parent module;- # popping inside the loop would break output-based loss for- # subsequent mappings.)- seen_parents = set()- for mapping in block_mappings:- pid = id(mapping.parent)- if pid not in seen_parents:- seen_parents.add(pid)- self._parent_args_cache.pop(mapping.parent, None)-- # ── Model-level initialization ─────────────────────────────────────────────-- def _prepare_model(self) -> None:- """One-time model-level AWQ setup: check compatibility and resolve mappings.-- Triggered automatically when model_context is assigned (via __setattr__).- """- from auto_round.algorithms.quantization.awq.mappings import check_model_compatibility-- model = self.model_context.model- report = check_model_compatibility(model, self._user_mappings)- for w in report["warnings"]:- logger.warning(w)- if not report["compatible"]:- model_class = report.get("model_class", "unknown")- raise ValueError(- f"AWQ: no smooth-balance mappings could be resolved for "- f"'{model_class}'. Either the model architecture is not "- f"supported for automatic AWQ mapping detection, or the model "- f"has no repeating transformer block structure. "- f"You can provide explicit mappings via "- f"AWQConfig(mappings=[{{'smooth_layer': '<regex>', "- f"'balance_layers': ['<regex>', ...]}}])."- )-- self.resolve_all_mappings(model)-- # AWQ caches block I/O on CPU — only one block lives on GPU at a time.- self.compress_context.cache_device = torch.device("cpu")-- def _check_scheme_compatibility(self) -> None:- """Validate the quantization scheme against AWQ inference backends."""- bits = self.bits- act_bits = self.act_bits or 16- data_type = self.data_type or "int"-- if "int" not in data_type:- raise ValueError(- f"AWQ requires integer data_type, got '{data_type}'. "- f"AWQ channel scaling is designed for integer quantization "- f"grids. Use algorithm='autoround' for FP8/MXFP quantization."- )-- if act_bits is not None and act_bits < 16 and act_bits != 8:- raise ValueError(- f"AWQ with act_bits={act_bits} is not supported. "- f"No inference kernel exists for W{bits}A{act_bits} in vllm "- f"or sglang. Supported schemes: W4A16 (canonical AWQ) or "- f"W8A8 (compressed_tensors INT8 backend)."- )-- if bits == 4 and act_bits >= 16:- pass- elif bits == 8 and act_bits == 8:- logger.info(- "AWQ with W8A8: AWQ smoothing will be applied, followed by "- "INT8 quantization. This is served by vllm's "- "compressed_tensors backend (cutlass INT8 GEMM), not AWQ "- "kernels."- )- elif bits not in (4, 8):- logger.warning(- f"AWQ with bits={bits}: vllm AWQ kernels only support "- f"bits=4 (AWQ/Marlin) natively. bits=8 is supported via "- f"compressed_tensors. Other bit widths may not have "- f"optimized inference kernels."- )-- # ── Weight quantization (delegated RTN) ─────────────────────────────────────-- def quantize_layer(self, name: str, dtype: torch.dtype = None) -> None:- """Quantize a single layer using RTN after AWQ smoothing has been applied.-- AWQ's quantize_layer is simpler than RTN's because:- - AWQ always uses plain RTN (disable_opt_rtn=True, no imatrix)- - No MoE-specific RTN disabling (AWQ handles MoE via mappings)- - No GGUF special path (AWQ targets AWQ/Marlin inference kernels)-- Args:- name: Fully-qualified module name (e.g. "model.layers.0.self_attn.q_proj").- dtype: Optional dtype to cast the layer to before quantization.- """- m = get_module(self.model, name)- if dtype is not None:- m = m.to(dtype)-- m = convert_module_to_hp_if_necessary(m, self.model_context.amp_dtype, device_manager.device)- set_module(self.model, name, m)- tuning_device = m.tuning_device if hasattr(m, "tuning_device") else device_manager.device-- try:- m = m.to(tuning_device)- m = WrapperLinear(- m,- device=tuning_device,- enable_minmax_tuning=False,- enable_norm_bias_tuning=False,- enable_round_tuning=False,- enable_torch_compile=self.compress_context.enable_torch_compile,- disable_opt_rtn=self.config.disable_opt_rtn,- iters=0,- )- m = m.unwrapper({})- except torch.OutOfMemoryError:- cuda_error_msg = traceback.format_exc()- m = m.orig_layer if hasattr(m, "orig_layer") else m- try:- logger.error(cuda_error_msg)- logger.warning("AWQ quantize_layer: falling back to CPU.")- m.to("cpu")- m = WrapperLinear(- m,- enable_minmax_tuning=False,- enable_norm_bias_tuning=False,- enable_round_tuning=False,- enable_torch_compile=self.compress_context.enable_torch_compile,- disable_opt_rtn=self.config.disable_opt_rtn,- iters=0,- )- m = m.unwrapper({})- except Exception:- raise-- set_module(self.model, name, m)- self._immediate_pack_and_save_module(name)-- def quantize_layer_outside_block(self, *args, **kwargs):- """Quantize layers outside blocks (e.g. lm_head). Delegates to quantize_layer."""- return self.quantize_layer(*args, **kwargs)-- def _immediate_pack_and_save_module(self, module_name: str) -> None:- """Pack and/or save a quantized module if immediate mode is enabled."""- shard_writer = ShardWriter.get_shard_writer()- to_cpu = self.compress_context.low_gpu_mem_usage- module = get_module(self.model, module_name)- if self.compress_context.is_immediate_packing:- immediate_pack(module_name, self.layer_config)- if to_cpu:- module = module.to("cpu")- packed_module = get_module(self.model, module_name)- set_module(self.model, module_name, packed_module.to("cpu"))- else:- if to_cpu:- module = module.to("cpu")- set_module(self.model, module_name, module)- if self.compress_context.is_immediate_saving:- module = get_module(self.model, module_name)- module.to("cpu")- shard_writer.write(module, module_name, False)- module.to("meta")-- # ── Block-level quantization ──────────────────────────────────────────────-- def quantize_block(- self, block: torch.nn.Module, input_ids=None, input_others=None, reference_output=None, **kwargs- ) -> dict:- """AWQ block quantization: collect stats → smooth → quantize.-- The full per-block AWQ pipeline:- 1. Register activation hooks for this block's mappings- 2. Run block forward to collect activation stats + parent kwargs- 3. Apply AWQ smoothing (grid search + scale application)- 4. Collect act_max AFTER smoothing (for activation quantization)- 5. Quantize the smoothed block via RTN-- Args:- block: Module already on the compute device.- input_ids: Calibration inputs for this block.- input_others: Additional kwargs for block forward.- reference_output: FP16 block output (kept for interface consistency).- **kwargs: Must include 'block_name' for mapping lookup.- """- if isinstance(block, _WrapperMultiblock):- raise ValueError(- "AWQ does not support nblocks > 1 (multi-block quantization). "- "Each block must be quantized individually. "- "Please set nblocks=1 when using algorithm='awq'."- )-- block_name = kwargs.get("block_name")- if block_name is None:- # Infer block_name from resolved mappings by matching the block's modules- for prefix, mappings in self._block_groups.items():- if any(m.smooth_layer is mod or m.parent is mod for m in mappings for mod in block.modules()):- block_name = prefix- break- if block_name is None:- raise ValueError(- "AWQQuantizer.quantize_block() could not determine block_name. "- "Pass block_name explicitly or ensure resolved mappings cover this block."- )-- model = self.model- bs = self.batch_size * self.infer_bs_coeff- if self.compress_context.low_gpu_mem_usage:- bs = 1- logger.info("AWQ: low_gpu_mem_usage enabled, setting inference batch size to 1.")-- # Step 1 & 2: AWQ smoothing (optional)- if self.apply_smooth:- awq_hooks = self.register_activation_hooks(model, block_prefix=block_name)- self._get_block_outputs(block, input_ids, input_others, bs, save_output=False)- for h in awq_hooks:- h.remove()-- self.smooth_block(block_name)-- # Step 3: Collect act_max AFTER smoothing- # AWQ smoothing changes internal activations (LayerNorm output /= scales),- # so act_max must be collected post-smoothing. Reset any pre-smoothing- # act_max values first to avoid stale data persisting via max().- act_max_hooks = self.register_calibration_hooks(block, imatrix=False)- if act_max_hooks:- for _name, m in block.named_modules():- if hasattr(m, "act_max"):- del m.act_max- self._get_block_outputs(block, input_ids, input_others, bs, save_output=False)- for h in act_max_hooks:- h.remove()-- # Step 4: Quantize the smoothed block (delegated RTN)- # This is equivalent to RTNQuantizer.quantize_block but inlined here- # to avoid inheritance coupling.- update_block_global_scale_if_needed(block, self.data_type, self.group_size)- if (- self.config.is_act_nv_fp- or self.config.is_static_afp8- or (self.config.is_wfp8afp8 and not self.config.act_dynamic)- ):- set_amax_for_all_moe_layers(block, attr_name="act_max")-- for _name, m in block.named_modules():- if hasattr(m, "global_name") and check_to_quantized(m):- self.quantize_layer(m.global_name)-- return {}-- # ── Module alignment (device onload/offload) ─────────────────────────────-- @contextlib.contextmanager- def _align_modules(- self,- mapping,- device: torch.device | str | None,- ):- """Temporarily move mapping-related modules to *device* for grid search.-- If *device* is None (model already on the compute device), this is a- no-op passthrough.- """- if device is None:- yield- return-- modules_to_move = [mapping.parent, mapping.smooth_layer] + list(mapping.balance_layers)- # Deduplicate (parent may overlap with smooth/balance)- seen = set()- unique = []- for m in modules_to_move:- if id(m) not in seen:- seen.add(id(m))- unique.append(m)-- # Onload- for m in unique:- m.to(device)-- try:- yield- finally:- # Offload back to CPU- for m in unique:- m.to("cpu")- clear_memory()-- # ── Activation statistics collection ──────────────────────────────────────-- def register_activation_hooks(- self,- model: torch.nn.Module,- block_prefix: str | None = None,- ) -> list:- """Register activation hooks for AWQ and return hook handles.-- Hooks are registered for:- 1. Smooth layers: accumulate abs activation sums/counts for x_mean.- 2. Parent modules: cache forward kwargs for output-based loss-- Args:- model: The model (hooks are registered on the actual module- objects, which are shared with blocks obtained via- ``get_module``).- block_prefix: When provided, only register hooks for mappings- whose smooth layer belongs to this block (e.g.- "model.layers.0"). Used in the block-by-block pipeline.-- Should be called by the compressor before calibration, and handles- should be removed after calibration.- """- if not self._resolved_mappings:- self._resolved_mappings = resolve_mappings(model, self._user_mappings)-- mappings = self._resolved_mappings- if block_prefix is not None:- mappings = [m for m in mappings if _extract_block_prefix(m.smooth_name) == block_prefix]-- hooks = []- smooth_names = {m.smooth_name for m in mappings}-- # Hook smooth layers for activation statistics- for name, module in model.named_modules():- if name not in smooth_names:- continue-- def _make_hook(layer_name):- def hook_fn(mod, args, output):- if isinstance(output, tuple):- x = output[0]- else:- x = output- if x is None or x.numel() == 0:- return-- # Compute abs-sum directly on GPU without materialising a- # full float32 intermediate. flatten→abs→sum produces a- # single [channels] tensor; .cpu() immediately frees the- # GPU allocation (aligned with AR's "immediate CPU move"- # pattern from CalibCompressor hooks).- channel_sum = x.detach().float().flatten(0, -2).abs().sum(dim=0).cpu()- count = x.shape[:-1].numel()-- if layer_name not in self._activation_stats:- self._activation_stats[layer_name] = [- torch.zeros_like(channel_sum),- 0,- ]- self._activation_stats[layer_name][0] += channel_sum- self._activation_stats[layer_name][1] += count-- return hook_fn-- h = module.register_forward_hook(_make_hook(name))- hooks.append(h)-- # Hook parent modules to cache kwargs for output-based loss- parent_modules_hooked = set()- for mapping in mappings:- parent = mapping.parent- if id(parent) in parent_modules_hooked:- continue- parent_modules_hooked.add(id(parent))-- self._parent_args_cache[parent] = []-- def _make_parent_hook(parent_module):- def hook_fn(mod, args, kwargs):- # Cache ALL calibration batches for output-based grid- # search loss, use all calibration data without subsampling.- mod_cls_id = id(type(mod))- if mod_cls_id not in self._parent_signatures:- self._parent_signatures[mod_cls_id] = inspect.signature(mod.forward)- sig = self._parent_signatures[mod_cls_id]- bound = sig.bind(*args, **kwargs)- bound.apply_defaults()-- # Infer the parent's compute dtype so that tensors- # upcasted by nn.LayerNorm (float32) are stored in the- # weight dtype (e.g. bfloat16). This avoids repeated- # per-sample casting in the grid search stage.- param = next(mod.parameters(), None)- w_dtype = param.dtype if param is not None else None-- stored = {}- for k, v in bound.arguments.items():- if isinstance(v, torch.Tensor):- v = v.detach()- if w_dtype is not None and v.is_floating_point() and v.dtype != w_dtype:- v = v.to(w_dtype)- stored[k] = v- elif isinstance(v, tuple) and any(isinstance(t, torch.Tensor) for t in v):- # Detach tensors in tuples (e.g. position_embeddings- # = (cos, sin)) to release computation graph refs.- stored[k] = tuple(- (- (- t.detach().to(w_dtype)- if w_dtype and t.is_floating_point() and t.dtype != w_dtype- else t.detach()- )- if isinstance(t, torch.Tensor)- else t- )- for t in v- )- elif hasattr(v, "key_cache"):- # Null out KV cache objects (DynamicCache etc.)- stored[k] = None- else:- stored[k] = v-- self._parent_args_cache[parent_module].append(stored)-- return hook_fn-- h = parent.register_forward_pre_hook(_make_parent_hook(parent), with_kwargs=True)- hooks.append(h)-- return hooks-- # ── Grid search ───────────────────────────────────────────────────────────-- def _get_grid_search_params(self) -> list[tuple[float, bool]]:- """Get grid search parameters (ratio, duo_scaling).-- Returns:- List of (ratio, use_duo_scaling) tuples for the grid search.- """- match self.duo_scaling:- # "both": half grid with duo off, half with duo on- case "both":- n_grid = max(int(self.n_grid / 2), 2)- return [- (grid_idx / (n_grid - 1), duo_scaling)- for grid_idx in range(n_grid)- for duo_scaling in [False, True]- ]- case False:- n_grid = max(self.n_grid, 2)- return [(grid_idx / (n_grid - 1), False) for grid_idx in range(n_grid)]- # True: include identity (0.0, False) as first, then duo points- case True:- n_grid = max(self.n_grid, 3)- return [(0.0, False)] + [(grid_idx / (n_grid - 2), True) for grid_idx in range(n_grid - 1)]- case _:- raise ValueError(f"Found unexpected duo_scaling configuration {self.duo_scaling}")-- @staticmethod- def _compute_layer_means(layers: list[torch.nn.Module], group_size: int) -> torch.Tensor:- """Compute per-channel mean of normalised weights.-- Within each quantization group, weights are normalized by their group max- (so values are on a 0-1 scale), then averaged across all groups to get- per-channel importance.-- Args:- layers: Balance layers whose weights are concatenated.- group_size: Quantization group size. If -1, uses full channel width.-- Returns:- Per-channel mean of normalised weights [in_features].- """- # Concatenate all balance layer weights [total_out, in_features]- weight = torch.cat([m.weight.detach().float() for m in layers], dim=0)- org_shape = weight.shape-- # Determine effective group size- gs = group_size if group_size > 0 else org_shape[1]-- # Pad when needed so AWQ works with layers whose input width is not a- # multiple of group_size, matching grouped RTN quantization behavior.- weight, _, pad_len = reshape_pad_tensor_by_group_size(weight, gs)- # Normalize within each group: abs / max (0-1 scale per group)- w_scale = weight.abs() / (weight.abs().amax(dim=1, keepdim=True) + 1e-6)- # Remove padding, then take mean across output channels.- w_scale = revert_tensor_by_pad(w_scale, orig_shape=org_shape, pad_len=pad_len)- w_mean = w_scale.mean(0)- return w_mean-- @torch.no_grad()- def _grid_search_scales(- self,- mapping: ResolvedMapping,- x_mean: torch.Tensor,- ) -> torch.Tensor | None:- """Find the best scaling ratio via grid search.-- Uses output-based error:- ``L(s) = || fp16_output - Q(W*s) @ (X/s) ||^2``-- For each candidate scaling ratio, applies scales to balance layer- weights, quantize-dequantizes them, runs all cached calibration- batches through the parent module, and computes the output MSE- against the fp16 baseline.-- Returns:- Best scales tensor, or None if no valid scale was found.- """- device = mapping.balance_layers[0].weight.device- x_mean = x_mean.to(device)-- # Compute normalised weight means- group_size = self.group_size if self.group_size > 0 else -1- if self.duo_scaling is not False:- w_mean = self._compute_layer_means(mapping.balance_layers, group_size).to(device)-- # Try to run parent module forward for output-based loss- parent_kwargs_list = self._parent_args_cache.get(mapping.parent, [])- use_parent_forward = len(parent_kwargs_list) > 0-- if use_parent_forward:- # Compute fp16 baseline outputs for loss computation- fp16_outputs = self._run_parent_samples(mapping.parent, parent_kwargs_list)- if not fp16_outputs or all(f.numel() == 0 for f in fp16_outputs):- use_parent_forward = False-- # Save original weights for restoration during grid search- orig_state = {bl: bl.weight.data.clone() for bl in mapping.balance_layers}-- best_error = float("inf")- best_scales = None- best_ratio = -1-- # Pre-resolve quant function once.- ref_layer = mapping.balance_layers[0]- ref_name = getattr(ref_layer, "global_name", None) or ""- ref_cfg = self.layer_config.get(ref_name, {})- try:- cached_quant_func, _ = get_quant_func(- ref_cfg.get("data_type", self.data_type),- ref_cfg.get("bits", self.bits),- ref_cfg.get("sym", self.sym),- disable_opt_rtn=True,- group_size=ref_cfg.get("group_size", self.group_size),- iters=0,- )- except Exception:- cached_quant_func = None-- grid_params = self._get_grid_search_params()-- for ratio, use_duo in grid_params:- # Compute scales- if use_duo:- scales = (x_mean.pow(ratio) / (w_mean.pow(1 - ratio) + 1e-4)).clamp(min=1e-4)- else:- scales = x_mean.pow(ratio).clamp(min=1e-4).view(-1)- scales = scales / (scales.max() * scales.min()).sqrt()- scales[torch.isinf(scales)] = 1- scales[torch.isnan(scales)] = 1- scales_view = scales.view(1, -1).to(device)-- if use_parent_forward:- # Q(W * s) / s: mutate balance layer weights- for bl in mapping.balance_layers:- bl.weight.data.copy_(orig_state[bl] * scales_view)- w_qdq = self._quantize_dequantize_weight(- bl,- bl.weight.data.float(),- quant_func=cached_quant_func,- )- if w_qdq is not None:- bl.weight.data = (w_qdq / scales_view).to(bl.weight.dtype)- else:- bl.weight.data.copy_(orig_state[bl])-- # Collect quantized outputs then compute loss- int_w_outputs = self._run_parent_samples(mapping.parent, parent_kwargs_list)- total_loss = self._compute_loss(fp16_outputs, int_w_outputs)- del int_w_outputs-- # Restore original weights- for bl in mapping.balance_layers:- bl.weight.data.copy_(orig_state[bl])- else:- # Weight-only fallback: || W - Q(W*s)/s ||^2- total_loss = 0.0- for bl in mapping.balance_layers:- w_orig = orig_state[bl].to(device)- w_scaled = w_orig * scales_view- w_qdq = self._quantize_dequantize_weight(- bl,- w_scaled,- quant_func=cached_quant_func,- )- if w_qdq is None:- total_loss = float("inf")- break- w_qdq_unscaled = w_qdq / scales_view- total_loss += (w_orig - w_qdq_unscaled).pow(2).sum().item()-- if total_loss < best_error:- best_error = total_loss- best_scales = scales.clone()- best_ratio = ratio-- if best_ratio < 0:- logger.warning(f"AWQ grid search failed for '{mapping.smooth_name}': " "no finite error found.")- return None-- logger.debug(f"AWQ '{mapping.smooth_name}': best_ratio={best_ratio:.2f}, " f"best_error={best_error:.3e}")- return best_scales-- # ── Parent module forward ─────────────────────────────-- @torch.no_grad()- def _run_parent_samples(- self,- parent: torch.nn.Module,- kwargs_list: list[dict],- ) -> list[torch.Tensor]:- """Run cached samples through the parent module.-- Feeds cached kwargs through the parent forward without any CUDA- synchronisation between batches, so the GPU can pipeline all forwards.- Outputs are kept on-device for loss computation.- """- outputs = []- for stored_kwargs in kwargs_list:- out = parent(**stored_kwargs)- if isinstance(out, tuple):- out = out[0]- outputs.append(out)- return outputs-- @staticmethod- @torch.no_grad()- def _compute_loss(- fp16_outputs: list[torch.Tensor],- int_w_outputs: list[torch.Tensor],- ) -> float:- """Compute normalised MSE between fp16 and quantized outputs."""- device = fp16_outputs[0].device- loss = torch.tensor(0.0, device=device)- num_elements = torch.tensor(0, device=device, dtype=torch.long)-- for fp16_out, int_w_out in zip(fp16_outputs, int_w_outputs):- loss += torch.nn.functional.mse_loss(- fp16_out.float(),- int_w_out.to(fp16_out.device).float(),- reduction="sum",- )- num_elements += fp16_out.numel()-- if num_elements == 0:- return float("inf")- return (loss / num_elements).item()-- def _quantize_dequantize_weight(- self,- layer: torch.nn.Module,- weight: torch.Tensor,- quant_func=None,- ) -> torch.Tensor | None:- """Quantize-dequantize a weight tensor using the layer's config.-- Args:- quant_func: Pre-resolved quantization function. When provided,- skips the ``get_quant_func`` dispatch (avoids redundant- lookups in the inner grid search loop).-- Returns the quantized-then-dequantized weight, or None on failure.- """- layer_name = getattr(layer, "global_name", None) or ""- config = self.layer_config.get(layer_name, {})- bits = config.get("bits", self.bits)- group_size = config.get("group_size", self.group_size)- sym = config.get("sym", self.sym)- data_type = config.get("data_type", self.data_type)-- if quant_func is None:- try:- quant_func, _ = get_quant_func(- data_type,- bits,- sym,- disable_opt_rtn=True, # AWQ always uses plain RTN- group_size=group_size,- iters=0, # Route to rtn_int_sym, not int_sym- )- except Exception:- return None-- if quant_func is None:- return None-- try:- qdq_weight, scale, zp = quant_func(- weight,- bits=bits,- group_size=group_size,- )- return qdq_weight- except Exception:- return None-- # ── Apply scales ──────────────────────────────────────────────────────────-- @torch.no_grad()- def _apply_scales(self, mapping: ResolvedMapping, scales: torch.Tensor) -> None:- """Apply the computed AWQ scales to smooth and balance layers.-- - Balance layers (Linear): weight *= scales (along input channels)- - Smooth layer (LayerNorm/RMSNorm): weight /= scales, bias /= scales- """- for bl in mapping.balance_layers:- device = bl.weight.device- s = scales.to(device).view(1, -1)- bl.weight.data.mul_(s)-- smooth = mapping.smooth_layer- device = smooth.weight.device- s = scales.to(device)-- if smooth.weight.ndim == 1:- # LayerNorm / RMSNorm: 1-D weight (per-channel)- smooth.weight.data.div_(s)- else:- # Edge case: when smooth layer's out_features != balance layer's- # in_features (e.g. fused qkv_proj smoothing o_proj). Scale the- # last output features (aligned with AutoAWQ).- # https://github.com/casper-hansen/AutoAWQ/blob/main/awq/quantize/scale.py#L123- smooth.weight.data[-s.size(0) :].div_(s.view(-1, 1))-- if hasattr(smooth, "bias") and smooth.bias is not None:- smooth.bias.data.div_(s)diff --git a/auto_round/algorithms/quantization/base.py b/auto_round/algorithms/quantization/base.pyindex 790f400f1..15f40d0ea 100644--- a/auto_round/algorithms/quantization/base.py+++ b/auto_round/algorithms/quantization/base.py@@ -11,20 +11,22 @@# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.# See the License for the specific language governing permissions and# limitations under the License.-import importlibimport traceback-from collections import defaultdict-from typing import Union+from contextlib import contextmanager+from typing import Anyimport torch+from auto_round.algorithms.base import BasePipelineMemberfrom auto_round.algorithms.quantization.config import QuantizationConfig-from auto_round.algorithms.quantization.utils import register_act_max_hooks+from auto_round.algorithms.transforms.base import BaseWeightTransformerfrom auto_round.compressors.utils import (block_forward,+ check_need_act_calibration,immediate_pack,)from auto_round.data_type import QUANT_FUNC_WITH_DTYPE+from auto_round.data_type.utils import reshape_pad_tensor_by_group_sizefrom auto_round.logger import loggerfrom auto_round.utils import (INNER_SUPPORTED_LAYER_TYPES,@@ -40,19 +42,129 @@ from auto_round.utils.device_manager import device_managerfrom auto_round.wrapper import WrapperLinear-class BaseQuantizers:- # Class-level attribute declarations for convenient access in quantization methods.- # Scheme-related attrs (layer_config, scale_dtype, has_qlayer_outside_block, etc.)- # are resolved by SchemeMixin in BaseCompressor and synced here after post_init().- model_context = None- compress_context = None- dataset = None- supported_types = SUPPORTED_LAYER_TYPES- inner_supported_types = INNER_SUPPORTED_LAYER_TYPES- enable_alg_ext = False- # Subclasses that support diffusion models should override this with the- # appropriate output key mapping, e.g.:- # DIFFUSION_OUTPUT_CONFIGS = {"FluxTransformerBlock": ["encoder_hidden_states", "hidden_states"]}+class RTNLayerFallbackMixin:+ """Default outside-block/layer quantization via RTN.++ Algorithms that want RTN fallback for embeddings, lm_head, or layers outside+ transformer blocks should inherit this mixin explicitly.+ """++ @torch.no_grad()+ def quantize_layer_via_rtn(self, layer_name: str, disable_opt_rtn: bool | None = None) -> None:+ """Quantize one layer with RTN and handle optional immediate pack/save."""+ layer = get_module(self.model, layer_name)+ layer = convert_module_to_hp_if_necessary(layer, self.model_context.amp_dtype, device_manager.device)+ set_module(self.model, layer_name, layer)+ tuning_device = layer.tuning_device if hasattr(layer, "tuning_device") else device_manager.device+ if (+ self.compress_context.is_immediate_packing+ and self.compress_context.formats[0].is_gguf()+ and not getattr(self.config, "disable_opt_rtn", False)+ ):+ layer = layer.to(tuning_device)+ layer.scale = None+ layer.zp = None+ else:+ try:+ if disable_opt_rtn is None:+ disable_opt_rtn = bool(getattr(self.config, "disable_opt_rtn", False))+ if (+ not disable_opt_rtn+ and getattr(self.config, "orig_disable_opt_rtn", None) is None+ and self.model_context.is_moe_model+ and "expert" in layer.global_name+ and "shared_expert" not in layer.global_name+ and self.config.super_bits is None+ ):+ disable_opt_rtn = True+ logger.warning_once(+ "MoE layer detected: optimized RTN is disabled for efficiency. "+ "Use `--enable_opt_rtn` to force-enable it for MoE layers."+ )+ layer = layer.to(tuning_device)+ layer = WrapperLinear(+ layer,+ device=tuning_device,+ enable_minmax_tuning=False,+ enable_norm_bias_tuning=False,+ enable_round_tuning=False,+ enable_torch_compile=self.compress_context.enable_torch_compile,+ disable_opt_rtn=disable_opt_rtn,+ iters=0,+ )+ layer = layer.unwrapper({})+ except torch.OutOfMemoryError:+ cuda_error_msg = traceback.format_exc()+ layer = layer.orig_layer if hasattr(layer, "orig_layer") else layer+ try:+ logger.error(cuda_error_msg)+ logger.warning("falling back to CPU.")+ layer.to("cpu")+ layer = WrapperLinear(+ layer,+ enable_minmax_tuning=False,+ enable_norm_bias_tuning=False,+ enable_round_tuning=False,+ enable_torch_compile=self.compress_context.enable_torch_compile,+ iters=0,+ )+ layer = layer.unwrapper({})+ except Exception:+ raise+ set_module(self.model, layer_name, layer)+ self._immediate_pack_and_save_module(layer_name)++ def _immediate_pack_and_save_module(self, module_name):+ from auto_round.compressors.shard_writer import ShardWriter++ shard_writer = ShardWriter.get_shard_writer()+ to_cpu = self.compress_context.low_gpu_mem_usage+ module = get_module(self.model, module_name)+ if self.compress_context.is_immediate_packing:+ immediate_pack(module_name, self.layer_config)+ if to_cpu:+ module = module.to("cpu")+ packed_module = get_module(self.model, module_name)+ set_module(self.model, module_name, packed_module.to("cpu"))+ else:+ if to_cpu:+ module = module.to("cpu")+ set_module(self.model, module_name, module)+ if self.compress_context.is_immediate_saving:+ module = get_module(self.model, module_name)+ module.to("cpu")+ shard_writer.write(module, module_name, False)+ module.to("meta")++ def quantize_layer_outside_block(self, layer_name: str, input_ids=None, **kwargs):+ dtype = kwargs.pop("dtype", None)+ if dtype is not None:+ layer = get_module(self.model, layer_name)+ set_module(self.model, layer_name, layer.to(dtype))+ self.quantize_layer_via_rtn(layer_name, **kwargs)+++class DiffusionMixin:+ """Mixin that adds diffusion-model support to a :class:`BaseQuantizer` subclass.++ Attach to any :class:`BaseQuantizer` subclass that needs to quantize+ diffusion models (e.g. Flux, StableAudio, Wan):++ class SignRoundQuantizer(DiffusionMixin, BaseQuantizer): ...++ The mixin overrides :meth:`create_block_io` so diffusion-specific input+ slicing, output mapping, and hidden_states extraction live in+ :class:`DiffusionBlockIO` rather than individual quantizers.++ ``DIFFUSION_OUTPUT_CONFIGS`` maps block class name → ordered list of output+ tensor keys. Extend in subclasses to register new diffusion architectures+ without overriding any method.+ """++ # Map block class name → list of output tensor keys returned by that block.+ # The order of keys must match the order of tensors in the block's return tuple.+ # Subclasses can extend this dict to register new architectures without+ # overriding any method.DIFFUSION_OUTPUT_CONFIGS: dict = {"FluxTransformerBlock": ["encoder_hidden_states", "hidden_states"],"FluxSingleTransformerBlock": ["encoder_hidden_states", "hidden_states"],@@ -62,24 +174,63 @@ class BaseQuantizers:"WanTransformerBlock": ["hidden_states"],}- def __init__(self, config: QuantizationConfig):- self.config = config+ def _get_output_config(self, block: torch.nn.Module) -> list:+ """Return the output key list for *block* from ``DIFFUSION_OUTPUT_CONFIGS``."""+ return self.DIFFUSION_OUTPUT_CONFIGS.get(block.__class__.__name__, ["hidden_states"])++ def create_block_io(self, input_ids, input_others, quantized_input=None, block=None):+ from auto_round.algorithms.pipeline import DiffusionBlockIO, InputSource++ active_source = (+ InputSource.QUANTIZED_INPUT+ if quantized_input is not None and self.enable_quanted_input+ else InputSource.FP_CACHE+ )+ io = DiffusionBlockIO(+ _fp_inputs=input_ids,+ _input_others=input_others,+ _quantized_inputs=quantized_input,+ _active_source=active_source,+ batch_dim=self.batch_dim,+ seqlen=self.seqlen,+ shared_cache_keys=self.model_context.shared_cache_keys,+ _quantizer=self,+ _block=block,+ output_config=self._get_output_config(block),+ )+ io._fp_inputs, io._input_others = io._preprocess_block_inputs(io._fp_inputs, io._input_others, block)+ return io+++class BaseQuantizer(BasePipelineMember):+ """Base class for terminal weight-compression algorithms in a QuantizationPipeline.++ Developers adding a new quantization algorithm should inherit from this+ class and override at minimum :meth:`quantize_block`.++ For diffusion model support, also inherit :class:`DiffusionMixin`:+ ``class MyQuantizer(DiffusionMixin, BaseQuantizer): ...``++ Lifecycle hooks to override as needed:+ - :meth:`prepare_run` – model-level setup (once before all blocks)+ - :meth:`get_act_calib_policy` – activation calibration policy+ - :meth:`block_forward_hooks` – register act-calib hooks (context manager)+ - :meth:`quantize_block` – **must override**: quantize a single block+ - :meth:`quantize_layer_outside_block` – quantize layers outside blocks+ - :meth:`finalize_run` – model-level teardown (once after all blocks)+ """++ # Class-level attribute declarations for convenient access in quantization methods.+ # Scheme-related attrs (layer_config, scale_dtype, has_qlayer_outside_block, etc.)+ # are resolved by SchemeMixin in BaseCompressor and synced here after post_init().+ dataset = None+ supported_types = SUPPORTED_LAYER_TYPES+ inner_supported_types = INNER_SUPPORTED_LAYER_TYPES+ enable_alg_ext = False++ def __init__(self, config: QuantizationConfig) -> None:+ super().__init__(config)self.layer_config = None- self.bits = config.bits- self.group_size = config.group_size- self.sym = config.sym- self.data_type = config.data_type- self.act_bits = config.act_bits- self.act_group_size = config.act_group_size- self.act_sym = config.act_sym- self.act_data_type = config.act_data_type- self.act_dynamic = config.act_dynamic- self.super_bits = config.super_bits- self.super_group_size = config.super_group_size- self.scale_dtype = config.scale_dtype- self.ignore_layers = config.ignore_layers- self.quant_lm_head = config.quant_lm_head- self.to_quant_block_names = config.to_quant_block_names# Calibration-time state lives on a shared# :class:`~auto_round.calibration.state.CalibrationState` instance.# The compressor wires its own instance here in ``_resolve_scheme``;@@ -97,11 +248,11 @@ class BaseQuantizers:# ── Shared CalibrationState forwarders ───────────────────────────────────────@property- def calibration_state(self):+ def calibration_state(self) -> Any:return self._calibration_state@calibration_state.setter- def calibration_state(self, new_state) -> None:+ def calibration_state(self, new_state: Any) -> None:# Compressor-supplied shared instance; just rebind.self._calibration_state = new_state@@ -110,15 +261,15 @@ class BaseQuantizers:return self._calibration_state.attention_mask@attention_mask.setter- def attention_mask(self, value) -> None:+ def attention_mask(self, value: list) -> None:self._calibration_state.attention_mask = value if value is not None else []@property- def batch_dim(self):+ def batch_dim(self) -> int:return self._calibration_state.batch_dim@batch_dim.setter- def batch_dim(self, value) -> None:+ def batch_dim(self, value: int) -> None:self._calibration_state.batch_dim = value@property@@ -126,7 +277,7 @@ class BaseQuantizers:return self._calibration_state.batch_size@batch_size.setter- def batch_size(self, value) -> None:+ def batch_size(self, value: int) -> None:self._calibration_state.batch_size = value@property@@ -137,16 +288,7 @@ class BaseQuantizers:def seqlen(self) -> int:return self._calibration_state.seqlen- @classmethod- def from_config(cls, config: QuantizationConfig):- if cls.__name__ == config._alg_cls:- return cls(config)- else:- module = importlib.import_module("auto_round.algorithms.quantization")- alg_cls = getattr(module, config._alg_cls)- return alg_cls(config)-- def bind(self, compressor) -> None:+ def bind(self, compressor: Any) -> None:"""Wire shared state from the owning compressor.The compressor owns the authoritative ``model_context`` /@@ -156,41 +298,158 @@ class BaseQuantizers:"""self.model_context = compressor.model_contextself.compress_context = compressor.compress_context+ self.scheme = compressor.scheme_contextself.scale_dtype = compressor.scale_dtype# Share the compressor's CalibrationState instance.self._calibration_state = compressor._calibration_state@property- def model(self):+ def model(self) -> torch.nn.Module | None:return self.model_context.model if self.model_context is not None else None@property- def formats(self):+ def formats(self) -> Any:return getattr(self.compress_context, "formats", None)@property- def amp(self):+ def amp(self) -> bool:return getattr(self.model_context, "amp", False)@property- def amp_dtype(self):- import torch-+ def amp_dtype(self) -> torch.dtype:return getattr(self.model_context, "amp_dtype", torch.float32)- def register_calibration_hooks(self, model, *, act_max: bool = True, imatrix: bool = True):- return register_act_max_hooks(self, model) if act_max else []+ # ── Activation-calibration hook infrastructure ───────────────────────────────++ def _register_act_max_hooks(self, model):+ """Register per-module act_max tracking hooks for static activation quantization.++ Internal implementation called by :meth:`block_forward_hooks`.+ Returns a list of hook handles that the caller must remove when done.+ """++ def collect_act_max(module, input, output):+ input = input[0] if isinstance(input, (tuple, list)) else input+ if input.numel() == 0:+ return+ input, _, _ = reshape_pad_tensor_by_group_size(input, self.act_group_size)+ act_max = torch.max(torch.abs(input), dim=-1).values+ if not hasattr(module, "act_max") or module.act_max.numel() == 0:+ module.act_max = act_max+ if self.config.is_act_nv_fp:+ max_val = act_max.max()+ module.act_max = max_val.unsqueeze(0) if max_val.dim() == 0 else max_val+ return++ act_max = act_max.to(module.act_max.device)+ if self.config.is_act_nv_fp:+ max_val = torch.max(act_max.max(), module.act_max.max())+ module.act_max = max_val.unsqueeze(0) if max_val.dim() == 0 else max_val+ else:+ module.act_max = torch.max(act_max, module.act_max)++ def should_collect(name, module):+ if isinstance(module, SUPPORTED_LAYER_TYPES):+ return (+ hasattr(module, "act_dynamic")+ and check_need_act_calibration(module.act_dynamic, module.act_data_type, module.act_bits)+ and check_to_quantized(module)+ )+ if name in self.layer_config:+ config = self.layer_config[name]+ act_dynamic = config.get("act_dynamic", True)+ act_data_type = config.get("act_data_type", None)+ act_bits = config.get("act_bits", 16)+ return (+ config["bits"] <= 8+ and check_need_act_calibration(act_dynamic, act_data_type, act_bits)+ and check_to_quantized(config)+ )+ return False++ handles = []+ if should_collect("", model):+ handles.append(model.register_forward_hook(collect_act_max))+ return handles+ for name, module in model.named_modules():+ if name and should_collect(name, module):+ handles.append(module.register_forward_hook(collect_act_max))+ return handles++ @contextmanager+ def block_forward_hooks(self, ctx: Any) -> Any:+ """Register act-calib forward hooks for the reference forward.++ Implements the :meth:`BasePipelineMember.block_forward_hooks` interface for+ terminal quantizers. Yields the list of hook handles so the caller can+ determine whether any act-calib hooks were registered (used to decide+ whether a second forward with quantized inputs is needed).+ """+ from auto_round.algorithms.pipeline import CalibTiming++ policy = self.get_act_calib_policy(ctx)+ if policy.when == CalibTiming.SKIP:+ yield []+ return+ handles = self._register_act_max_hooks(ctx.block)+ try:+ yield handles+ finally:+ for h in handles:+ h.remove()++ def get_act_calib_policy(self, ctx: Any) -> Any:+ """Return the activation calibration policy for this block.++ Default: ``WITH_REFERENCE + FP_CACHE``, or ``QUANTIZED_INPUT`` when+ ``enable_quanted_input=True`` and a quantized previous-block output is available.+ """+ from auto_round.algorithms.pipeline import ActCalibPolicy, CalibTiming, InputSource++ if ctx.has_quantized_inputs() and self.enable_quanted_input:+ return ActCalibPolicy(when=CalibTiming.WITH_REFERENCE, source=InputSource.QUANTIZED_INPUT)+ return ActCalibPolicy(when=CalibTiming.WITH_REFERENCE, source=InputSource.FP_CACHE)++ def create_block_io(+ self,+ input_ids: Any,+ input_others: dict,+ quantized_input: Any = None,+ block: torch.nn.Module | None = None,+ ) -> Any:+ from auto_round.algorithms.pipeline import BlockIO, InputSource++ active_source = (+ InputSource.QUANTIZED_INPUT+ if quantized_input is not None and self.enable_quanted_input+ else InputSource.FP_CACHE+ )+ io = BlockIO(+ _fp_inputs=input_ids,+ _input_others=input_others,+ _quantized_inputs=quantized_input,+ _active_source=active_source,+ batch_dim=self.batch_dim,+ seqlen=self.seqlen,+ shared_cache_keys=self.model_context.shared_cache_keys,+ _quantizer=self,+ _block=block,+ )+ io._fp_inputs, io._input_others = io._preprocess_block_inputs(io._fp_inputs, io._input_others, block)+ return io++ # ── Embedding quantization ────────────────────────────────────────────────────@torch.inference_mode()- def _quantize_embedding_layer(self):+ def quantize_embedding_layer(self) -> bool:"""Quantizes embedding layers in the model according to the configuration.This method iterates through all modules in the model, identifies embedding- layers specified in `self.quantizer.layer_config`, and applies the appropriate quantization+ layers specified in `self.layer_config`, and applies the appropriate quantizationfunction based on bit precision, grouping strategy, and dtype.Returns:- bool: True if the quantization process completes without critical errors.+ bool: True if any embedding layer was quantized."""is_quantized = Falsefor name, module in self.model_context.model.named_modules():@@ -264,9 +523,9 @@ class BaseQuantizers:return is_quantized- def quantize_block(- self, block: torch.nn.Module, input_ids=None, input_others=None, reference_output=None, **kwargs- ) -> dict:+ # ── Abstract quantization interface ──────────────────────────────────────────++ def quantize_block(self, ctx: Any) -> dict:"""Apply the quantization algorithm to a prepared block.This is the **pure-algorithm** entry point called by the Compressor after@@ -274,121 +533,28 @@ class BaseQuantizers:registration, DDP setup) has been completed.Implementations should:- - Perform the algorithm-specific weight/activation quantization on ``block``.+ - Perform the algorithm-specific weight/activation quantization on ``ctx.block``.- Return a dict of best parameters (may be empty for zero-shot algorithms).Args:- block: Module already placed on the correct device(s).- input_ids: Calibration inputs on cache_device (None for zero-shot RTN).- input_others: Additional inputs (None for zero-shot RTN).- reference_output: FP reference outputs collected by Compressor- (None for algorithms that don't need a reconstruction loss).- **kwargs: Algorithm-specific keyword arguments (e.g. ``loss_device``,- ``card_0_in_high_risk`` for SignRoundQuantizer).+ ctx: Per-block pipeline context. ``ctx.io`` owns calibration inputs,+ quantized inputs, and reference outputs.Returns:dict: Best quantization parameters found, or ``{}`` if not applicable."""- raise NotImplementedError("quantize_block must be implemented in subclasses of BaseQuantizers")-- @torch.no_grad()- def quantize_layer_via_rtn(self, layer_name: str, disable_opt_rtn: bool | None = None) -> None:- """Quantize one layer with RTN and handle optional immediate pack/save."""- layer = get_module(self.model, layer_name)- layer = convert_module_to_hp_if_necessary(layer, self.model_context.amp_dtype, device_manager.device)- set_module(self.model, layer_name, layer)- tuning_device = layer.tuning_device if hasattr(layer, "tuning_device") else device_manager.device- if (- self.compress_context.is_immediate_packing- and self.compress_context.formats[0].is_gguf()- and not getattr(self.config, "disable_opt_rtn", False)- ):- layer = layer.to(tuning_device)- layer.scale = None- layer.zp = None- else:- try:- if disable_opt_rtn is None:- disable_opt_rtn = bool(getattr(self.config, "disable_opt_rtn", False))- if (- not disable_opt_rtn- and getattr(self.config, "orig_disable_opt_rtn", None) is None- and self.model_context.is_moe_model- and "expert" in layer.global_name- and "shared_expert" not in layer.global_name- and self.config.super_bits is None- ):- disable_opt_rtn = True- logger.warning_once(- "MoE layer detected: optimized RTN is disabled for efficiency. "- "Use `--enable_opt_rtn` to force-enable it for MoE layers."- )- layer = layer.to(tuning_device)- layer = WrapperLinear(- layer,- device=tuning_device,- enable_minmax_tuning=False,- enable_norm_bias_tuning=False,- enable_round_tuning=False,- enable_torch_compile=self.compress_context.enable_torch_compile,- disable_opt_rtn=disable_opt_rtn,- iters=0,- )- layer = layer.unwrapper({})- except torch.OutOfMemoryError:- cuda_error_msg = traceback.format_exc()- layer = layer.orig_layer if hasattr(layer, "orig_layer") else layer- try:- logger.error(cuda_error_msg)- logger.warning("falling back to CPU.")- layer.to("cpu")- layer = WrapperLinear(- layer,- enable_minmax_tuning=False,- enable_norm_bias_tuning=False,- enable_round_tuning=False,- enable_torch_compile=self.compress_context.enable_torch_compile,- iters=0,- )- layer = layer.unwrapper({})- except Exception:- raise-- set_module(self.model, layer_name, layer)- self._immediate_pack_and_save_module(layer_name)-- def _immediate_pack_and_save_module(self, module_name):- from auto_round.compressors.shard_writer import ShardWriter+ raise NotImplementedError("quantize_block must be implemented in subclasses of BaseQuantizer")- shard_writer = ShardWriter.get_shard_writer()- to_cpu = self.compress_context.low_gpu_mem_usage- module = get_module(self.model, module_name)- if self.compress_context.is_immediate_packing:- immediate_pack(module_name, self.layer_config)- if to_cpu:- module = module.to("cpu")- packed_module = get_module(self.model, module_name)- set_module(self.model, module_name, packed_module.to("cpu"))- else:- if to_cpu:- module = module.to("cpu")- set_module(self.model, module_name, module)- if self.compress_context.is_immediate_saving:- module = get_module(self.model, module_name)- module.to("cpu")- shard_writer.write(module, module_name, False)- module.to("meta")-- def quantize_layer(self, layer_name: str, **kwargs):+ def quantize_layer(self, layer_name: str, **kwargs) -> None:"""Quantizes a single layer of the model.Args:layer_name (str): The name of the layer to quantize. The layer module isretrieved internally via get_module(model, layer_name)."""- raise NotImplementedError("quantize_layer must be implemented in subclasses of BaseQuantizers")+ raise NotImplementedError("quantize_layer must be implemented in subclasses of BaseQuantizer")- def quantize_layer_outside_block(self, layer_name: str, input_ids=None, **kwargs):+ def quantize_layer_outside_block(self, layer_name: str, input_ids: Any = None, **kwargs) -> Any:"""Quantizes a single layer of the model outside of a block.Args:@@ -396,76 +562,7 @@ class BaseQuantizers:retrieved internally via get_module(model, layer_name).input_ids: Optional calibration inputs for data-driven outside-layer quantization."""- dtype = kwargs.pop("dtype", None)- if dtype is not None:- layer = get_module(self.model, layer_name)- set_module(self.model, layer_name, layer.to(dtype))- self.quantize_layer_via_rtn(layer_name, **kwargs)-- @torch.no_grad()- def _get_block_outputs(- self,- block: torch.nn.Module,- input_ids,- input_others,- bs: int,- save_output: bool = True,- device_override: Union[torch.device, str, None] = None,- ):- """Compute the output of a block for calibration inputs.-- Shared by SignRoundQuantizer and OptimizedRTNQuantizer. Algorithm-specific- block-forward selection (compile vs. plain) is handled here based on- ``enable_alg_ext`` and act-quantization flags.-- Args:- device_override: Override the target device. Used by diffusion with- multi-device dispatch to pass None so block_forward uses the block's- current device instead of forcing a specific device.- """- diffusion_fn = getattr(self, "_get_diffusion_block_outputs", None)- if getattr(self.model_context, "is_diffusion", False):- device = device_override if device_override is not None else device_manager.device- return self._get_diffusion_block_outputs(- block,- input_ids,- input_others,- bs,- device,- self.compress_context.cache_device,- )-- _bf = self._resolve_block_forward()-- output = []- nsamples = len(input_ids)- for i in range(0, nsamples, bs):- end_index = min(nsamples, i + bs)- indices = torch.arange(i, end_index).to(torch.long)- tmp_input_ids, tmp_input_others = self._sampling_inputs(- input_ids,- input_others,- indices,- self.seqlen,- self.batch_dim,- share_cache_keys=self.model_context.shared_cache_keys,- )- tmp_output = _bf(- block,- tmp_input_ids,- tmp_input_others,- self.model_context.amp,- self.model_context.amp_dtype,- device_manager.device,- ).to(self.compress_context.cache_device)- if save_output:- if self.batch_size == 1:- output.append(tmp_output)- else:- output.extend(list(torch.split(tmp_output, 1, dim=self.batch_dim)))- self.compress_context.clear_memory()-- return output+ raise NotImplementedError("quantize_layer_outside_block must be implemented in subclasses or mixins")def _resolve_block_forward(self):"""Resolve and cache the block forward function once.@@ -500,180 +597,13 @@ class BaseQuantizers:self.__dict__.pop("_resolved_block_forward", None)self.__dict__.pop("_compiled_block_forward", None)- def _get_current_q_output(- self,- block: torch.nn.Module,- input_ids,- input_others: dict,- indices,- device,- cache_device: str = "cpu",- ) -> torch.Tensor:- """Compute block output for a mini-batch selected by *indices* (used during training).-- Handles both LLM and diffusion model block formats. Uses the compiled- block_forward when enable_torch_compile is True (same as _get_block_outputs),- matching old-arch behavior where self.block_forward was compiled at init.- """- current_input_ids, current_input_others = self._sampling_inputs(- input_ids,- input_others,- indices,- seqlen=self.seqlen,- batch_dim=self.batch_dim,- share_cache_keys=self.model_context.shared_cache_keys,- )- _bf = self._resolve_block_forward()-- if getattr(self.model_context, "is_diffusion", False):- output_config = self.DIFFUSION_OUTPUT_CONFIGS.get(block.__class__.__name__, ["hidden_states"])- idx = None if "hidden_states" not in output_config else output_config.index("hidden_states")- if isinstance(current_input_ids, dict):- hidden_states = current_input_ids.pop("hidden_states")- current_input_others.update(current_input_ids)- current_input_ids = hidden_states- output_q = _bf(- block,- current_input_ids,- current_input_others,- self.model_context.amp,- self.model_context.amp_dtype,- device,- idx,- )- else:- output_q = _bf(- block,- current_input_ids,- current_input_others,- self.model_context.amp,- self.model_context.amp_dtype,- device,- )- return output_q.to(cache_device)-- @classmethod- @torch.no_grad()- def _sampling_inputs(- cls,- input_ids: Union[list[torch.Tensor], dict],- input_others: dict,- indices,- seqlen: int,- batch_dim: int = 0,- share_cache_keys: tuple = (),- ):- """Sample a mini-batch of calibration inputs by indices.-- Shared by SignRoundQuantizer and OptimizedRTNQuantizer.- """- if isinstance(input_ids, list):- current_input_ids = [input_ids[i] for i in indices]- current_input_ids = torch.cat(current_input_ids, dim=batch_dim)- elif isinstance(input_ids, dict):- current_input_ids = defaultdict(list)- for k in input_ids.keys():- current_input_ids[k].extend([input_ids[k][i] for i in indices])- current_input_ids[k] = torch.cat(current_input_ids[k], dim=batch_dim)-- current_input_others = {"positional_inputs": input_others["positional_inputs"]}- for key in input_others.keys():- if "positional_inputs" in key:- continue- if key in share_cache_keys:- # Shared keys are stored once (not per-sample), often wrapped in a- # 1-element list by the caching hook. Unwrap so the model receives- # the raw value (e.g. (cos, sin) tuple, not [(cos, sin)]).- # Exception: if the hook detected that the "shared" key actually varies- # per sample (e.g. position_embeddings in a VLM visual encoder), it- # upgrades the storage to a per-sample list with >1 elements.- val = input_others[key]- if isinstance(val, list) and len(val) == 1:- current_input_others[key] = val[0]- elif isinstance(val, list) and len(val) > 1:- # Per-sample storage for a nominally-shared key that varies across- # calibration samples (e.g. position_embeddings in Qwen2-VL visual- # encoder blocks where each image has a different patch count).- idx = int(indices[0]) if len(indices) == 1 else 0- current_input_others[key] = val[idx] if idx < len(val) else val[0]- else:- current_input_others[key] = val- elif not isinstance(input_others[key], (str, bool, type(None))):- current_input_others[key] = None- if input_others[key] is not None:- current_input_others[key] = [input_others[key][i] for i in indices]- if len(indices) == 1:- current_input_others[key] = current_input_others[key][0]- else:- try:- current_input_others[key] = torch.cat(current_input_others[key], dim=0)- except TypeError as err:- logger.warning_once("Please check the model cache inputs or try setting batch_size to 1.")- else:- current_input_others[key] = input_others[key]+ def prepare_run(self, compressor: Any) -> None:+ """Model-level preparation (called once before block iteration starts)."""+ return- return current_input_ids, current_input_others+ def finalize_run(self, compressor: Any) -> None:+ """Model-level teardown (called once after all blocks are processed).- @torch.no_grad()- def _get_diffusion_block_outputs(- self,- block: torch.nn.Module,- input_ids: Union[torch.Tensor, dict],- input_others,- bs: int,- device: Union[str, torch.device],- cache_device: Union[str, torch.device],- save_output: bool = True,- ):- """Compute block outputs for diffusion models.-- Uses ``self.DIFFUSION_OUTPUT_CONFIGS`` to map block class names to their- output keys. Subclasses override ``DIFFUSION_OUTPUT_CONFIGS`` to add- support for new diffusion architectures.+ Must be idempotent – the Compressor calls this inside a ``try/finally``."""- output = defaultdict(list)- output_config = self.DIFFUSION_OUTPUT_CONFIGS.get(block.__class__.__name__, ["hidden_states"])- if isinstance(input_ids, dict):- nsamples = len(input_ids["hidden_states"])- else:- nsamples = len(input_ids)-- for i in range(0, nsamples, bs):- end_index = min(nsamples, i + bs)- indices = torch.arange(i, end_index).to(torch.long)- tmp_input_ids, tmp_input_others = self._sampling_inputs(- input_ids,- input_others,- indices,- self.seqlen,- self.batch_dim,- share_cache_keys=self.model_context.shared_cache_keys,- )- if isinstance(tmp_input_ids, dict):- hidden_states = tmp_input_ids.pop("hidden_states")- tmp_input_others.update(tmp_input_ids)- tmp_input_ids = hidden_states-- tmp_output = block_forward(- block,- tmp_input_ids,- tmp_input_others,- self.model_context.amp,- self.model_context.amp_dtype,- device,- None,- )- if isinstance(tmp_output, torch.Tensor):- tmp_output = [tmp_output]- assert len(output_config) == len(tmp_output)- tmp_output = dict(zip(output_config, tmp_output))-- if save_output:- for name, out in tmp_output.items():- if self.batch_size == 1:- output[name].append(out.to(cache_device))- else:- output[name].extend(list(torch.split(out.to(cache_device), 1, dim=self.batch_dim)))- self.compress_context.clear_memory()-- return output+ returndiff --git a/auto_round/algorithms/quantization/config.py b/auto_round/algorithms/quantization/config.pyindex d99219e51..6a0487fdc 100644--- a/auto_round/algorithms/quantization/config.py+++ b/auto_round/algorithms/quantization/config.py@@ -11,11 +11,9 @@# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.# See the License for the specific language governing permissions and# limitations under the License.-from dataclasses import dataclassfrom enum import Enumfrom typing import ClassVar, Union-from auto_round.algorithms.alg_config import AlgConfigfrom auto_round.export.export_to_gguf.config import GGUF_INNER_CONFIGfrom auto_round.logger import loggerfrom auto_round.schemes import QuantizationScheme@@ -29,28 +27,57 @@ class BackendDataType(str, Enum):FP8 = "fp8"-@dataclass(kw_only=True)class QuantizationConfig:- _alg_cls: ClassVar[str] = None-- # quantization args- bits: int = None- group_size: int = None- sym: bool = None- data_type: str = None- act_bits: int = None- act_group_size: int = None- act_sym: bool = None- act_data_type: str = None- act_dynamic: bool = None- super_bits: int = None- super_group_size: int = None- scale_dtype: str = None- ignore_layers: str = ""- quant_lm_head: bool = False- to_quant_block_names: Union[str, list, None] = None-- def __post_init__(self):+ """Common quantization configuration shared by block quantizers.++ Args:+ bits: Weight quantization bit width.+ group_size: Weight quantization group size. Use -1 for per-channel,+ 0 for per-tensor, or a positive integer for grouped quantization.+ sym: Whether to use symmetric weight quantization.+ data_type: Weight quantization data type, such as int, mx_fp,+ nv_fp, or fp8 variants.+ act_bits: Activation quantization bit width.+ act_group_size: Activation quantization group size.+ act_sym: Whether to use symmetric activation quantization.+ act_data_type: Activation quantization data type.+ act_dynamic: Whether activation quantization should be dynamic.+ super_bits: Bit width used for double quantization metadata.+ super_group_size: Group size used for double quantization metadata.+ """++ _scheme_fields: ClassVar[set[str]] = set(QuantizationScheme.get_attributes())++ def __init__(self, *, scheme: QuantizationScheme = None, **kwargs) -> None:+ object.__setattr__(self, "scheme", scheme if scheme is not None else QuantizationScheme.empty())+ object.__setattr__(self, "_user_set_scheme_fields", set())++ unknown = []+ for key, value in kwargs.items():+ if key in self._scheme_fields:+ setattr(self.scheme, key, value)+ self._user_set_scheme_fields.add(key)+ else:+ unknown.append(key)+ if unknown:+ unknown_args = ", ".join(repr(arg) for arg in unknown)+ raise TypeError(f"Unexpected quantization config argument(s): {unknown_args}")++ self._check_partial_config()++ def __getattr__(self, name):+ if name in self._scheme_fields:+ return getattr(self.scheme, name, None)+ raise AttributeError(f"{type(self).__name__!s} object has no attribute {name!r}")++ def __setattr__(self, name, value):+ if name in self._scheme_fields and "scheme" in self.__dict__:+ setattr(self.scheme, name, value)+ self._user_set_scheme_fields.add(name)+ return+ object.__setattr__(self, name, value)++ def _check_partial_config(self):# Run block-wise validation early (at construction time, before model loading).# Scheme resolution is deferred to BaseCompressor.post_init() via SchemeMixin.# Guard with None checks in case the user hasn't explicitly set data_type/bits@@ -156,27 +183,27 @@ class QuantizationConfig:logger.warning("dtype nv_fp should only support group_size of 16 in real deployment")@property- def is_act_quantize(self):+ def is_act_quantize(self) -> bool:return self.act_bits is not None and self.act_bits <= 8@property- def is_nv_fp(self):+ def is_nv_fp(self) -> bool:return self.data_type is not None and BackendDataType.NV_FP in self.data_type@property- def is_act_nv_fp(self):+ def is_act_nv_fp(self) -> bool:return self.act_data_type is not None and BackendDataType.NV_FP in self.act_data_type@property- def is_mx_fp(self):+ def is_mx_fp(self) -> bool:return self.data_type is not None and BackendDataType.MX_FP in self.data_type@property- def is_act_mx_fp(self):+ def is_act_mx_fp(self) -> bool:return self.act_data_type is not None and BackendDataType.MX_FP in self.act_data_type@property- def is_dynamic_wint8aint8(self):+ def is_dynamic_wint8aint8(self) -> bool:if self.act_dynamic:return Trueif self.act_data_type is not None and self.data_type is not None:@@ -187,7 +214,7 @@ class QuantizationConfig:return False@property- def is_standard_fp(self):+ def is_standard_fp(self) -> bool:return (self.data_type is not Noneand BackendDataType.STANDARD_FP in self.data_type@@ -196,7 +223,7 @@ class QuantizationConfig:)@property- def is_act_standard_fp(self):+ def is_act_standard_fp(self) -> bool:return (self.act_data_type is not Noneand BackendDataType.STANDARD_FP in self.act_data_type@@ -205,15 +232,15 @@ class QuantizationConfig:)@property- def is_static_afp8(self):+ def is_static_afp8(self) -> bool:return self.act_data_type is not None and BackendDataType.FP8_STATIC in self.act_data_type@property- def is_static_wfp8afp8(self):+ def is_static_wfp8afp8(self) -> bool:return self.data_type is not None and BackendDataType.FP8_STATIC in self.data_type and self.is_static_afp8@property- def is_wfp8afp8(self):+ def is_wfp8afp8(self) -> bool:if self.act_data_type is None or self.data_type is None:return Falseif (diff --git a/auto_round/algorithms/quantization/registry.py b/auto_round/algorithms/quantization/registry.pynew file mode 100644index 000000000..82a77742c--- /dev/null+++ b/auto_round/algorithms/quantization/registry.py@@ -0,0 +1,11 @@+# # Copyright (C) 2026 Intel Corporation+# # SPDX-License-Identifier: Apache-2.0++from auto_round.algorithms.registry import list_registered_algorithms, register_algorithm, resolve_alg_config+++def register_alg(alias, factory):+ register_algorithm(alias, aliases=(alias,), config_factory=factory)+++__all__ = ["register_alg", "resolve_alg_config", "list_registered_algorithms"]diff --git a/auto_round/algorithms/quantization/rtn/config.py b/auto_round/algorithms/quantization/rtn/config.pyindex 5e27dbccb..8c1f52eec 100644--- a/auto_round/algorithms/quantization/rtn/config.py+++ b/auto_round/algorithms/quantization/rtn/config.py@@ -18,14 +18,22 @@ from auto_round.logger import loggerclass RTNConfig(QuantizationConfig):- _alg_cls = "RTNQuantizer"-def __init__(self,*,disable_opt_rtn: bool = None,**kwargs,- ):+ ) -> None:+ """Initialize an RTN configuration.++ Args:+ disable_opt_rtn: Whether to disable the optimized RTN path.+ ``None`` keeps the default heuristic, True forces plain+ RTN, and False forces the optimized implementation.+ **kwargs: Common quantization arguments forwarded to+ QuantizationConfig, such as bits, group_size, sym,+ data_type, and activation quantization fields.+ """# pop before super().__init__ so it doesn't leak into QuantizationConfig as an unknown kwargenable_opt_rtn = kwargs.pop("enable_opt_rtn", None)super().__init__(**kwargs)@@ -47,5 +55,7 @@ class RTNConfig(QuantizationConfig):)disable_opt_rtn = Falseself.disable_opt_rtn = disable_opt_rtn- if not self.disable_opt_rtn:- self._alg_cls = "OptimizedRTNQuantizer"+++class OptimizedRTNConfig(RTNConfig):+ passdiff --git a/auto_round/algorithms/quantization/rtn/quantizer.py b/auto_round/algorithms/quantization/rtn/quantizer.pyindex 6259afaaf..0a031cfec 100644--- a/auto_round/algorithms/quantization/rtn/quantizer.py+++ b/auto_round/algorithms/quantization/rtn/quantizer.py@@ -11,11 +11,29 @@# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.# See the License for the specific language governing permissions and# limitations under the License.+from collections import defaultdict+from contextlib import contextmanager+from typing import Any, Callable, Optional, Union++import accelerateimport torch-from auto_round.algorithms.quantization.base import BaseQuantizers-from auto_round.algorithms.quantization.rtn.config import RTNConfig-from auto_round.algorithms.quantization.utils import register_imatrix_hooks+from auto_round.algorithms.quantization.base import BaseQuantizer, RTNLayerFallbackMixin+from auto_round.algorithms.quantization.rtn.config import OptimizedRTNConfig, RTNConfig+from auto_round.algorithms.quantization.sign_round.quantizer import SignRoundQuantizer+from auto_round.algorithms.registry import register_pipeline_member+from auto_round.compressors.utils import (+ IndexSampler,+ block_forward,+ check_need_act_calibration,+ check_skippable_keywords,+ collect_best_params,+ get_shared_keys,+ infer_bits_by_data_type,+ init_cache,+ reset_params,+ set_layer_config,+)from auto_round.data_type.utils import update_block_global_scale_if_neededfrom auto_round.utils import (check_to_quantized,@@ -25,15 +43,14 @@ from auto_round.utils import ()-class RTNQuantizer(BaseQuantizers):+@register_pipeline_member(RTNConfig)+class RTNQuantizer(RTNLayerFallbackMixin, BaseQuantizer):- def __init__(self, config: RTNConfig):- BaseQuantizers.__init__(self, config)+ def __init__(self, config: RTNConfig) -> None:+ BaseQuantizer.__init__(self, config)@torch.no_grad()- def quantize_block(- self, block: torch.nn.Module, input_ids=None, input_others=None, reference_output=None, **kwargs- ) -> dict:+ def quantize_block(self, ctx) -> dict:"""Apply zero-shot RTN quantization to a block.Pure-algorithm entry point. Infrastructure (materialize, shard writing,@@ -48,6 +65,7 @@ class RTNQuantizer(BaseQuantizers):Returns:dict: Empty dict (zero-shot RTN has no tunable parameters to return)."""+ block = ctx.blockif (self.config.is_act_nv_fpor self.config.is_static_afp8@@ -72,10 +90,11 @@ class RTNQuantizer(BaseQuantizers):self.quantize_layer_via_rtn(name)+@register_pipeline_member(OptimizedRTNConfig)class OptimizedRTNQuantizer(RTNQuantizer):- def __init__(self, config: RTNConfig):- BaseQuantizers.__init__(self, config)+ def __init__(self, config: RTNConfig) -> None:+ BaseQuantizer.__init__(self, config)self.data_type = config.data_typeself.group_size = config.group_sizeself.infer_bs_coeff = config.infer_bs_coeff@@ -83,16 +102,36 @@ class OptimizedRTNQuantizer(RTNQuantizer):self.enable_alg_ext = True- def register_calibration_hooks(self, model, *, act_max: bool = True, imatrix: bool = True):- hook_handles = super().register_calibration_hooks(model, act_max=act_max, imatrix=imatrix)- if imatrix and self.enable_imatrix:- hook_handles.extend(register_imatrix_hooks(self, model, with_count=True))- return hook_handles+ @contextmanager+ def block_forward_hooks(self, ctx):+ with super().block_forward_hooks(ctx) as hook_handles:+ if self.enable_imatrix:+ hook_handles.extend(self._register_imatrix_hooks(ctx.block, with_count=True))+ yield hook_handles++ def _register_imatrix_hooks(self, model, *, with_count: bool = False):+ def collect_imatrix(module, input, output):+ input = input[0] if isinstance(input, (tuple, list)) else input+ flattened = input.reshape(-1, input.shape[-1]).to(torch.float32)+ squared = torch.sum(torch.pow(flattened, 2), dim=0).to(torch.float32)++ if not hasattr(module, "imatrix"):+ module.imatrix = squared+ if with_count:+ module.imatrix_cnt = input.shape[0]+ return+ module.imatrix += squared.to(module.imatrix.device)+ if with_count:+ module.imatrix_cnt += input.shape[0]++ handles = []+ for _, module in model.named_modules():+ if isinstance(module, self.supported_types) and check_to_quantized(module):+ handles.append(module.register_forward_hook(collect_imatrix))+ return handles@torch.no_grad()- def quantize_block(- self, block: torch.nn.Module, input_ids=None, input_others=None, reference_output=None, **kwargs- ):+ def quantize_block(self, ctx):"""Apply imatrix-informed RTN quantization to a block.Pure-algorithm entry point. Device placement and cleanup are handled@@ -106,6 +145,7 @@ class OptimizedRTNQuantizer(RTNQuantizer):input_others: Unused for optimized RTN.reference_output: Unused for optimized RTN."""+ block = ctx.blockupdate_block_global_scale_if_needed(block, self.data_type, self.group_size)if (self.config.is_act_nv_fpdiff --git a/auto_round/algorithms/quantization/sign_round/config.py b/auto_round/algorithms/quantization/sign_round/config.pyindex 2aad5e08e..a3af1799c 100644--- a/auto_round/algorithms/quantization/sign_round/config.py+++ b/auto_round/algorithms/quantization/sign_round/config.py@@ -11,25 +11,14 @@# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.# See the License for the specific language governing permissions and# limitations under the License.-from typing import Union+from typing import Any, Callable, Unionfrom auto_round.algorithms.quantization.config import QuantizationConfigfrom auto_round.logger import loggerclass SignRoundConfig(QuantizationConfig):- """-- Args:- iters (int): Number of iterations (default is 200).- lr (float): The learning rate (default is 0.005).- minmax_lr (float): The learning rate for min-max tuning (default is None).- lr_scheduler: The learning rate scheduler to be used.- enable_minmax_tuning (bool): Whether to enable min-max tuning (default is True).- enable_norm_bias_tuning (bool): Whether to enable fast norm/layer_bias tuning- """-- _alg_cls = "SignRoundQuantizer"+ """Configuration for SignRound-style block quantization."""def __init__(self,@@ -37,7 +26,7 @@ class SignRoundConfig(QuantizationConfig):iters: int = 200,lr: float = None,minmax_lr: float = None,- lr_scheduler=None,+ lr_scheduler: Callable | None = None,momentum: float = 0.0,nblocks: int = 1,enable_minmax_tuning: bool = True,@@ -50,7 +39,39 @@ class SignRoundConfig(QuantizationConfig):optimizer: str = None,enable_adam: bool = False,**kwargs,- ):+ ) -> None:+ """Initialize a SignRound configuration.++ Args:+ iters: Number of optimization iterations for each quantized+ block.+ lr: Learning rate used by the main rounding optimization.+ If None, a heuristic based on ``iters`` is used.+ minmax_lr: Learning rate used by min-max tuning. If None, it+ falls back to ``lr``.+ lr_scheduler: Optional learning-rate scheduler name or+ scheduler object used by the optimizer.+ momentum: Momentum factor used by the optimizer.+ nblocks: Number of blocks to optimize together.+ enable_minmax_tuning: Whether to tune weight min/max ranges.+ enable_norm_bias_tuning: Whether to tune normalization and+ bias terms.+ gradient_accumulate_steps: Number of gradient accumulation+ steps used per optimization update.+ enable_alg_ext: Whether to enable the experimental SignRound+ extension implementation.+ not_use_best_mse: Whether to skip restoring the best-MSE+ checkpoint during tuning.+ dynamic_max_gap: Maximum dynamic gap used by adaptive tuning+ logic.+ enable_quanted_input: Whether each block should consume the+ quantized output of previous blocks during calibration.+ optimizer: Optional optimizer name override.+ enable_adam: Whether to use the Adam-based SignRound variant.+ **kwargs: Common quantization arguments forwarded to+ QuantizationConfig, such as bits, group_size, sym,+ data_type, and activation quantization fields.+ """super().__init__(**kwargs)self.gradient_accumulate_steps = gradient_accumulate_stepsself.iters = iters@@ -87,11 +108,6 @@ class SignRoundConfig(QuantizationConfig):self.optimizer = optimizerself.enable_adam = enable_adam- if self.enable_adam:- self._alg_cls = "AdamRoundQuantizer"- elif self.enable_alg_ext:- self._alg_cls = "SignRoundV2Quantizer"-def check_configs(self) -> None:"""Checks if the configurations are valid.@@ -106,3 +122,11 @@ class SignRoundConfig(QuantizationConfig):raise ValueError("`nblocks` must be positive")if self.gradient_accumulate_steps <= 0:raise ValueError("`gradient_accumulate_steps` must be positive")+++class AdamRoundConfig(SignRoundConfig):+ pass+++class SignRoundV2Config(SignRoundConfig):+ passdiff --git a/auto_round/algorithms/quantization/sign_round/quantizer.py b/auto_round/algorithms/quantization/sign_round/quantizer.pyindex 13e8fbe8e..ed627eb5a 100644--- a/auto_round/algorithms/quantization/sign_round/quantizer.py+++ b/auto_round/algorithms/quantization/sign_round/quantizer.py@@ -15,15 +15,16 @@ import copyfrom collections import defaultdictfrom contextlib import nullcontextfrom functools import partial-from typing import Any, Callable, Optional, Union+from typing import TYPE_CHECKING, Any, Callable, Optional, Unionimport accelerateimport torchfrom torch import autocast-from auto_round.algorithms.quantization.base import BaseQuantizers+from auto_round.algorithms.quantization.base import BaseQuantizer, RTNLayerFallbackMixinfrom auto_round.algorithms.quantization.sign_round.config import SignRoundConfigfrom auto_round.algorithms.quantization.sign_round.sign_sgd import SignSGD+from auto_round.algorithms.registry import register_pipeline_memberfrom auto_round.compressors.utils import (IndexSampler,block_forward,@@ -34,6 +35,9 @@ from auto_round.compressors.utils import (from auto_round.data_type.utils import reshape_pad_tensor_by_group_size, update_fused_layer_global_scalesfrom auto_round.logger import loggerfrom auto_round.utils import (+ check_to_quantized,+ compile_func,+ convert_module_to_hp_if_necessary,get_module,htcore,is_hpex_available,@@ -47,10 +51,14 @@ from auto_round.utils.device_manager import device_managerfrom auto_round.utils.distributed import setup_ddp_if_needed_from auto_round.wrapper import WrapperLinear, unwrapper_block, unwrapper_layer, wrapper_block+if TYPE_CHECKING:+ from auto_round.algorithms.pipeline import BlockContext-class SignRoundQuantizer(BaseQuantizers):- def __init__(self, config: SignRoundConfig):+@register_pipeline_member(SignRoundConfig)+class SignRoundQuantizer(RTNLayerFallbackMixin, BaseQuantizer):++ def __init__(self, config: SignRoundConfig) -> None:super().__init__(config)self.iters = config.itersself.lr = config.lr@@ -69,28 +77,6 @@ class SignRoundQuantizer(BaseQuantizers):self.optimizer = self._get_optimizer(optimizer=config.optimizer)self.wrapper_block = wrapper_block- def _get_current_output(self, output: list[torch.Tensor], indices: list[int]) -> torch.Tensor:- if self.model_context.is_diffusion:- assert "hidden_states" in output- current_output = [output["hidden_states"][x] for x in indices]- current_output = torch.cat(current_output, dim=self.batch_dim)- return current_output- current_output = [output[x] for x in indices]- current_output = torch.cat(current_output, dim=self.batch_dim)- return current_output-- def _get_current_num_elm(- self,- input_ids: list[torch.Tensor],- indices: list[int],- ) -> int:- if self.model_context.is_diffusion:- current_input_ids = [input_ids["hidden_states"][i] for i in indices]- return sum(id.numel() for id in current_input_ids)-- current_input_ids = [input_ids[i] for i in indices]- return sum(id.numel() for id in current_input_ids)-def _get_non_zero_cnt(self, tensor: list[torch.Tensor], indices: list[int]) -> int:current_tensors = [tensor[i] for i in indices]non_zero_cnt = 0@@ -100,8 +86,8 @@ class SignRoundQuantizer(BaseQuantizers):def _get_loss(self,- output_q: torch.Tensor,- current_output: torch.Tensor,+ pred_output: torch.Tensor,+ ref_output: torch.Tensor,indices: torch.Tensor,mse_loss: Callable,device: Union[str, torch.device] = "cpu",@@ -118,28 +104,18 @@ class SignRoundQuantizer(BaseQuantizers):with autocast_ctx:loss = mse_loss( # pylint: disable=not-callable- (output_q * tmp_attention_mask).to(torch.float32),- (current_output * tmp_attention_mask).to(torch.float32),+ (pred_output * tmp_attention_mask).to(torch.float32),+ (ref_output * tmp_attention_mask).to(torch.float32),)else:with autocast_ctx:loss = mse_loss( # pylint: disable=not-callable- output_q.to(torch.float32), current_output.to(torch.float32)+ pred_output.to(torch.float32), ref_output.to(torch.float32))return loss- def quantize_block(- self,- block: torch.nn.Module,- input_ids: Union[list[torch.Tensor], dict],- input_others: dict,- reference_output,- *,- loss_device: Union[str, torch.device],- mid_iter_mem_check: bool = False,- **kwargs,- ) -> dict:+ def quantize_block(self, ctx: "BlockContext") -> dict:"""Apply the AutoRound optimization algorithm to a block.This is the pure-algorithm entry point. All infrastructure concerns@@ -148,21 +124,17 @@ class SignRoundQuantizer(BaseQuantizers):before and after this call.Args:- block: Module already placed on the correct device(s).- input_ids: Calibration inputs (already on cache_device).- input_others: Additional inputs for the block's forward pass.- reference_output: FP reference outputs collected by the Compressor.- loss_device: Device on which to compute the MSE loss.- mid_iter_mem_check: Pre-evaluated by the Compressor as- ``low_gpu_mem_usage and card_0_in_high_risk``. When True,- triggers mid-iteration memory threshold checks to reduce- fragmentation on the primary GPU.+ ctx: Per-block pipeline context. ``ctx.io`` owns calibration inputs,+ reference outputs, and mini-batch block forwards.Returns:best_params: Best quantization parameters found during optimization.Empty dict if no trainable parameters were found."""+ block = ctx.blockdevice = device_manager.device+ loss_device = ctx.loss_device+ mid_iter_mem_check = ctx.mid_iter_mem_checkquantized_layer_names, unquantized_layer_names = self.wrapper_block(block,@@ -220,10 +192,7 @@ class SignRoundQuantizer(BaseQuantizers):else:lr_schedule = copy.deepcopy(self.lr_scheduler)- if isinstance(input_ids, dict): # input_ids of Flux is dict- nsamples = len(input_ids["hidden_states"])- else:- nsamples = len(input_ids)+ nsamples = ctx.num_sampleslast_best_iter = 0best_loss = torch.finfo(torch.float).maxnum_elm = 1@@ -240,7 +209,7 @@ class SignRoundQuantizer(BaseQuantizers):# We assume the block input and output shape is sameif self.gradient_accumulate_steps != 1 and not self.attention_mask:whole_indices = torch.arange(global_batch_size)- num_elm = self._get_current_num_elm(input_ids, whole_indices)+ num_elm = ctx.count_batch_elements(whole_indices)setup_ddp_if_needed_(self, block, device_manager.device_list)index_sampler = IndexSampler(nsamples, global_batch_size)batch_size = self.batch_size@@ -255,10 +224,11 @@ class SignRoundQuantizer(BaseQuantizers):for batch_start in range(0, len(global_indices), batch_size):indices = global_indices[batch_start : batch_start + batch_size]- current_output = self._get_current_output(reference_output, indices)- current_output = to_device(current_output, loss_device)- output_q = self._get_current_q_output(block, input_ids, input_others, indices, device, loss_device)- loss = self._get_loss(output_q, current_output, indices, mse_loss, device)+ ref_output = ctx.get_reference_outputs(indices, device=loss_device)+ # BlockIO centralizes batch input selection, reference caching,+ # and forwarding for the currently scheduled block (ctx.block).+ pred_output = ctx.forward_block_batch(indices, device=device, cache_device=loss_device)+ loss = self._get_loss(pred_output, ref_output, indices, mse_loss, device)num_elm = 1 if num_elm <= 0 else num_elmtotal_loss += loss.item() / num_elm@@ -330,7 +300,7 @@ class SignRoundQuantizer(BaseQuantizers):Args:layer_name (str): The name of the layer to quantize.- input_ids (list[torch.Tensor], optional): Input data for quantization.+ inputs (torch.Tensor): Input data for quantization.q_inputs (torch.Tensor, optional): Quantized input data. Defaults to None.device (torch.device, optional): The device to use for quantization. Defaults to torch.device("cpu").@@ -371,7 +341,7 @@ class SignRoundQuantizer(BaseQuantizers):static_attention_dtype,):tmp_inputs = q_inputs if q_inputs is not None else input_ids- hook_handles = self.register_calibration_hooks(layer)+ hook_handles = self._register_act_max_hooks(layer)with torch.no_grad():for input in tmp_inputs:layer(input)@@ -433,9 +403,9 @@ class SignRoundQuantizer(BaseQuantizers):if gradient_accumulate_steps != 1 and not self.attention_mask:whole_indices = torch.arange(global_batch_size)if q_inputs is not None:- num_elm = self._get_current_num_elm(q_inputs, whole_indices)+ num_elm = self._count_layer_input_elements(q_inputs, whole_indices)else:- num_elm = self._get_current_num_elm(input_ids, whole_indices)+ num_elm = self._count_layer_input_elements(input_ids, whole_indices)index_sampler = IndexSampler(nsamples, global_batch_size)@@ -529,6 +499,9 @@ class SignRoundQuantizer(BaseQuantizers):)return SignSGD+ def _count_layer_input_elements(self, input_ids, indices: list) -> int:+ return sum(input_ids[i].numel() for i in indices)+def _get_scaler(self):"""Returns scaler, in SignRound, no need to use scaler."""return Nonediff --git a/auto_round/algorithms/quantization/sign_round/sign_sgd.py b/auto_round/algorithms/quantization/sign_round/sign_sgd.pyindex e4b948107..b1fe06025 100644--- a/auto_round/algorithms/quantization/sign_round/sign_sgd.py+++ b/auto_round/algorithms/quantization/sign_round/sign_sgd.py@@ -16,7 +16,7 @@# limitations under the License.-from typing import List, Optional+from typing import Iterable, List, Optional# From PyTorch:#@@ -212,17 +212,17 @@ class SignSGD(Optimizer):def __init__(self,- params,- lr=required,- momentum=0,- dampening=0,- weight_decay=0,- nesterov=False,+ params: Iterable[Tensor],+ lr: float = required,+ momentum: float = 0,+ dampening: float = 0,+ weight_decay: float = 0,+ nesterov: bool = False,*,- maximize=False,+ maximize: bool = False,foreach: Optional[bool] = None,- differentiable=False- ):+ differentiable: bool = False,+ ) -> None:if lr is not required and lr < 0.0:raise ValueError("Invalid learning rate: {}".format(lr))if momentum < 0.0:@@ -320,7 +320,7 @@ def sgd(lr: float,dampening: float,nesterov: bool,- maximize: bool+ maximize: bool,):r"""Functional API that performs SGD algorithm computation.@@ -364,7 +364,7 @@ def _single_tensor_sgd(dampening: float,nesterov: bool,maximize: bool,- has_sparse_grad: bool+ has_sparse_grad: bool,):for i, param in enumerate(params):d_p = d_p_list[i] if not maximize else -d_p_list[i]diff --git a/auto_round/algorithms/quantization/sign_roundv2/quantizer.py b/auto_round/algorithms/quantization/sign_roundv2/quantizer.pyindex b8ff6c9bb..4d272172f 100644--- a/auto_round/algorithms/quantization/sign_roundv2/quantizer.py+++ b/auto_round/algorithms/quantization/sign_roundv2/quantizer.py@@ -12,7 +12,7 @@# See the License for the specific language governing permissions and# limitations under the License.-from contextlib import nullcontext+from contextlib import contextmanager, nullcontextfrom functools import partialfrom typing import Callable, Union@@ -20,9 +20,9 @@ import torchimport transformersfrom torch import autocast-from auto_round.algorithms.quantization.sign_round.config import SignRoundConfig+from auto_round.algorithms.quantization.sign_round.config import SignRoundConfig, SignRoundV2Configfrom auto_round.algorithms.quantization.sign_round.quantizer import SignRoundQuantizer-from auto_round.algorithms.quantization.utils import register_imatrix_hooks+from auto_round.algorithms.registry import register_pipeline_memberfrom auto_round.data_type.gguf import (double_quant_tensor_sym_rtn,quant_tensor_gguf_asym_dq,@@ -134,10 +134,10 @@ class SignRoundOptimizedWrapperLinear(WrapperLinear):class SignRoundDQWrapperLinear(WrapperLinear):minmax_scale_bound = (0.5, 1.5)- def __init__(self, *args, **kwargs):+ def __init__(self, *args, **kwargs) -> None:if "enable_minmax_tuning" in kwargs:logger.warning_once("disable minmax tuning for a little better accuracy and lower cost")- kwargs["enable_minmax_tuning"] = False # a little faster and better+ kwargs["enable_minmax_tuning"] = Falsesuper().__init__(*args, **kwargs)self.prev_scale = Noneself.prev_wmin = None@@ -180,7 +180,6 @@ class SignRoundDQWrapperLinear(WrapperLinear):@torch.no_grad()def _run_search(self, weight, v):- """Per-format scale/wmin search separated from the (compilable) quant func."""from auto_round.export.export_to_gguf.config import GGML_QUANT_SIZES, QK_Kbits = self.orig_layer.bits@@ -201,8 +200,6 @@ class SignRoundDQWrapperLinear(WrapperLinear):split_num=1,v=v_r,)- # Search funcs use ``@torch.inference_mode()``; clone to detach so- # autograd may consume them.return {"scale": scale.clone(),"wmin": wmin.clone(),@@ -210,7 +207,6 @@ class SignRoundDQWrapperLinear(WrapperLinear):"d_wmin": d_wmin.clone(),}- # sym pathgroup_size = 16super_group_size = 16t, _, _ = reshape_pad_tensor_by_group_size(weight.to(torch.float32), group_size)@@ -232,13 +228,12 @@ class SignRoundDQWrapperLinear(WrapperLinear):def _qdq_weight(self, value, min_scale, max_scale):if not self._is_dq_path:- # Non-dq path keeps the original behavior (base class handles it).return super()._qdq_weight(value, min_scale, max_scale)if self.orig_layer.bits >= 16:return self.orig_layer.weight, None, Nonemin_bound, max_bound = self.minmax_scale_bound- min_scale.data.clamp_(min_bound, max_bound) # TODO this one could be deleted+ min_scale.data.clamp_(min_bound, max_bound)max_scale.data.clamp_(min_bound, max_bound)weight = self.orig_layer.weightif weight.device.type == "meta":@@ -246,7 +241,6 @@ class SignRoundDQWrapperLinear(WrapperLinear):if isinstance(self.orig_layer, transformers.pytorch_utils.Conv1D):weight = weight.t()- # Re-search every 10 steps; otherwise reuse the cached search results.iter_v = getattr(self, "cur_iter", 0)need_search = (iter_v == 0) or (iter_v == -1) or (self.prev_scale is None)if need_search:@@ -294,10 +288,11 @@ class SignRoundDQWrapperLinear(WrapperLinear):return weight_q, scale_out, zp_out+@register_pipeline_member(SignRoundV2Config)class SignRoundV2Quantizer(SignRoundQuantizer):"""SignRound variant using the open algorithm-extension path in the new architecture."""- def __init__(self, config: SignRoundConfig):+ def __init__(self, config: SignRoundConfig) -> None:super().__init__(config)self._use_outlier_suppressed_loss = Falselogger.info("using algorithm extension for quantization.")@@ -323,14 +318,14 @@ class SignRoundV2Quantizer(SignRoundQuantizer):def _get_loss(self,- output_q: torch.Tensor,- current_output: torch.Tensor,+ pred_output: torch.Tensor,+ ref_output: torch.Tensor,indices: torch.Tensor,mse_loss: Callable,device: Union[str, torch.device] = "cpu",):if self._use_outlier_suppressed_loss:- loss_diff = torch.abs(output_q - current_output)+ loss_diff = torch.abs(pred_output - ref_output)flat_diff = loss_diff.view(-1)topk = max(1, int(flat_diff.numel() / 1000))_, top_indices = torch.topk(torch.abs(flat_diff), topk)@@ -348,7 +343,7 @@ class SignRoundV2Quantizer(SignRoundQuantizer):with autocast_ctx:return torch.mean((- torch.abs(output_q.to(torch.float32) - current_output.to(torch.float32))+ torch.abs(pred_output.to(torch.float32) - ref_output.to(torch.float32))* tmp_attention_mask* mask)@@ -356,20 +351,34 @@ class SignRoundV2Quantizer(SignRoundQuantizer):)with autocast_ctx:- return torch.mean(- (torch.abs(output_q.to(torch.float32) - current_output.to(torch.float32)) * mask) ** 2- )- return super()._get_loss(output_q, current_output, indices, mse_loss, device)-- def register_calibration_hooks(self, model, *, act_max: bool = True, imatrix: bool = True):- hook_handles = super().register_calibration_hooks(model, act_max=act_max, imatrix=imatrix)- if not imatrix:- return hook_handles-- is_wint4aint4 = ("int4" in self.act_data_type or ("int" in self.act_data_type and self.act_bits == 4)) and (+ return torch.mean((torch.abs(pred_output.to(torch.float32) - ref_output.to(torch.float32)) * mask) ** 2)+ return super()._get_loss(pred_output, ref_output, indices, mse_loss, device)++ @contextmanager+ def block_forward_hooks(self, ctx):+ with super().block_forward_hooks(ctx) as hook_handles:+ if not self._is_wint4aint4():+ hook_handles.extend(self._register_imatrix_hooks(ctx.block))+ yield hook_handles++ def _is_wint4aint4(self):+ return ("int4" in self.act_data_type or ("int" in self.act_data_type and self.act_bits == 4)) and ("int4" in self.data_type or ("int" in self.data_type and self.bits == 4))- if is_wint4aint4:- return hook_handles- hook_handles.extend(register_imatrix_hooks(self, model))- return hook_handles++ def _register_imatrix_hooks(self, model):+ def collect_imatrix(module, input, output):+ input = input[0] if isinstance(input, (tuple, list)) else input+ flattened = input.reshape(-1, input.shape[-1]).to(torch.float32)+ squared = torch.sum(torch.pow(flattened, 2), dim=0).to(torch.float32)++ if not hasattr(module, "imatrix"):+ module.imatrix = squared+ return+ module.imatrix += squared.to(module.imatrix.device)++ handles = []+ for _, module in model.named_modules():+ if isinstance(module, self.supported_types) and check_to_quantized(module):+ handles.append(module.register_forward_hook(collect_imatrix))+ return handlesdiff --git a/auto_round/algorithms/registry.py b/auto_round/algorithms/registry.pynew file mode 100644index 000000000..d3a16e6cb--- /dev/null+++ b/auto_round/algorithms/registry.py@@ -0,0 +1,168 @@+# # Copyright (C) 2026 Intel Corporation+# # SPDX-License-Identifier: Apache-2.0++from __future__ import annotations++import copy+import importlib+from dataclasses import dataclass, field+from typing import TYPE_CHECKING, Any, Callable++if TYPE_CHECKING:+ from auto_round.algorithms.base import BasePipelineMember+++@dataclass+class AlgRegistryEntry:+ name: str+ aliases: tuple[str, ...] = ()+ config_factory: Callable[[], object] | None = None+ cli_handler: type | None = None+ summary: str = ""+ alias_factories: dict[str, Callable[[], object]] = field(default_factory=dict)+++_ALG_REGISTRY: dict[str, AlgRegistryEntry] = {}+_ALIAS_TO_NAME: dict[str, str] = {}+_CONFIG_IMPL_REGISTRY: dict[type, type["BasePipelineMember"]] = {}+_builtin_algorithms_registered = False+_pipeline_members_registered = False+++def _ensure_builtin_algorithms_registered() -> None:+ global _builtin_algorithms_registered+ if _builtin_algorithms_registered:+ return+ importlib.import_module("auto_round.cli.algorithms")+ _builtin_algorithms_registered = True+++def _ensure_pipeline_members_registered() -> None:+ global _pipeline_members_registered+ if _pipeline_members_registered:+ return+ for module_name in (+ "auto_round.algorithms.quantization.rtn.quantizer",+ "auto_round.algorithms.quantization.sign_round.quantizer",+ "auto_round.algorithms.quantization.sign_roundv2.quantizer",+ "auto_round.algorithms.quantization.adam_round.adam",+ "auto_round.algorithms.transforms.awq.quantizer",+ ):+ importlib.import_module(module_name)+ _pipeline_members_registered = True+++def register_algorithm(+ name: str,+ *,+ aliases: tuple[str, ...] = (),+ config_factory: Callable[[], object] | None = None,+ cli_handler: type | None = None,+ summary: str = "",+ alias_factories: dict[str, Callable[[], object]] | None = None,+) -> None:+ key = name.strip().lower()+ entry = _ALG_REGISTRY.get(key)+ if entry is None:+ entry = AlgRegistryEntry(name=key)+ _ALG_REGISTRY[key] = entry++ merged_aliases = tuple(+ dict.fromkeys((entry.aliases or ()) + tuple(a.strip().lower() for a in aliases if a.strip()))+ )+ if config_factory is not None:+ entry.config_factory = config_factory+ if cli_handler is not None:+ entry.cli_handler = cli_handler+ if summary:+ entry.summary = summary+ if alias_factories:+ entry.alias_factories.update({k.strip().lower(): v for k, v in alias_factories.items()})+ entry.aliases = merged_aliases++ _ALIAS_TO_NAME[key] = key+ for alias in merged_aliases:+ _ALIAS_TO_NAME[alias] = key+++def resolve_algorithm_alias(alias: str) -> str | None:+ _ensure_builtin_algorithms_registered()+ return _ALIAS_TO_NAME.get(alias.strip().lower())+++def get_algorithm_entry(name: str) -> AlgRegistryEntry:+ _ensure_builtin_algorithms_registered()+ canonical = resolve_algorithm_alias(name)+ if canonical is None or canonical not in _ALG_REGISTRY:+ raise KeyError(name)+ return _ALG_REGISTRY[canonical]+++def iter_algorithm_entries() -> list[AlgRegistryEntry]:+ _ensure_builtin_algorithms_registered()+ return list(_ALG_REGISTRY.values())+++def resolve_alg_config(alias: str) -> object:+ _ensure_builtin_algorithms_registered()+ canonical = resolve_algorithm_alias(alias)+ if canonical is None:+ raise ValueError(+ f"Unknown algorithm alias '{alias}'. Supported aliases: {sorted(_ALIAS_TO_NAME.keys())}. "+ "If you are adding a new algorithm, register it via auto_round.algorithms.registry.register_algorithm()."+ )++ entry = _ALG_REGISTRY[canonical]+ factory = entry.alias_factories.get(alias.strip().lower(), entry.config_factory)+ if factory is None:+ raise ValueError(f"Algorithm alias '{alias}' is registered but has no config factory.")+ return factory()+++def list_registered_algorithms() -> list[str]:+ _ensure_builtin_algorithms_registered()+ return sorted(_ALIAS_TO_NAME.keys())+++def register_pipeline_member(config_cls: type):+ def _decorator(member_cls: type["BasePipelineMember"]) -> type["BasePipelineMember"]:+ _CONFIG_IMPL_REGISTRY[config_cls] = member_cls+ return member_cls++ return _decorator+++def resolve_pipeline_member(config: object) -> type["BasePipelineMember"]:+ _ensure_pipeline_members_registered()+ config_cls = type(config)+ for cls in config_cls.__mro__:+ member_cls = _CONFIG_IMPL_REGISTRY.get(cls)+ if member_cls is not None:+ return member_cls+ raise ValueError(f"Unknown algorithm config type {config_cls.__name__!r}.")+++def coerce_config_class(config: object, target_cls: type) -> object:+ if type(config) is target_cls:+ return config+ new_config = copy.copy(config)+ if hasattr(config, "scheme") and getattr(config, "scheme", None) is not None:+ new_config.scheme = config.scheme.copy()+ if hasattr(config, "_user_set_scheme_fields"):+ new_config._user_set_scheme_fields = set(getattr(config, "_user_set_scheme_fields", set()))+ new_config.__class__ = target_cls+ return new_config+++def normalize_algorithm_config(config: object) -> object:+ from auto_round.algorithms.quantization.rtn.config import OptimizedRTNConfig, RTNConfig+ from auto_round.algorithms.quantization.sign_round.config import AdamRoundConfig, SignRoundConfig, SignRoundV2Config++ if type(config) is RTNConfig and not getattr(config, "disable_opt_rtn", False):+ return coerce_config_class(config, OptimizedRTNConfig)+ if type(config) is SignRoundConfig:+ if getattr(config, "enable_adam", False):+ return coerce_config_class(config, AdamRoundConfig)+ if getattr(config, "enable_alg_ext", False):+ return coerce_config_class(config, SignRoundV2Config)+ return configdiff --git a/auto_round/algorithms/transforms/__init__.py b/auto_round/algorithms/transforms/__init__.pyindex 5e48cd8c9..17a64d993 100644--- a/auto_round/algorithms/transforms/__init__.py+++ b/auto_round/algorithms/transforms/__init__.py@@ -20,7 +20,7 @@ quantisation step to improve numerical properties.Current algorithms------------------* **hadamard** – Block-diagonal Hadamard rotations (QuaRot / SpinQuant style).- See :mod:`auto_round.algorithms.transforms.rotation`.+ See :mod:`auto_round.algorithms.transforms.quarot`.* **spinquant** – SpinQuant/QuaRot multi-level rotation (R1–R4) with optionalonline hooks, trainable rotations, and known Hadamard matrices for non-pow2.See :mod:`auto_round.algorithms.transforms.spinquant`.@@ -45,6 +45,7 @@ from typing import Anyimport torchfrom auto_round.algorithms.transforms.base import (+ BaseWeightTransformer,BaseRotation,BaseRotationConfig,SerializerMixin,@@ -52,7 +53,7 @@ from auto_round.algorithms.transforms.base import (check_supported_schemes,_ensure_registry_populated,)-from auto_round.algorithms.transforms.rotation import (+from auto_round.algorithms.transforms.quarot import (HadamardRotation,apply_rotation_transform,normalize_rotation_config as _normalize_hadamard_config,@@ -61,6 +62,7 @@ from auto_round.algorithms.transforms.rotation import (__all__ = [# Base interfaces+ "BaseWeightTransformer","BaseRotation","BaseRotationConfig","SerializerMixin",@@ -82,6 +84,18 @@ __all__ = []+def __getattr__(name):+ if name == "AWQConfig":+ from auto_round.algorithms.transforms.awq.config import AWQConfig++ return AWQConfig+ if name == "AWQQuantizer":+ from auto_round.algorithms.transforms.awq.quantizer import AWQQuantizer++ return AWQQuantizer+ raise AttributeError(f"module {__name__!r} has no attribute {name!r}")++def normalize_rotation_config(config: Any,) -> BaseRotationConfig | None:diff --git a/auto_round/algorithms/alg_config.py b/auto_round/algorithms/transforms/awq/__init__.pysimilarity index 76%rename from auto_round/algorithms/alg_config.pyrename to auto_round/algorithms/transforms/awq/__init__.pyindex d9d5f0c75..5637a3e0b 100644--- a/auto_round/algorithms/alg_config.py+++ b/auto_round/algorithms/transforms/awq/__init__.py@@ -12,7 +12,7 @@# See the License for the specific language governing permissions and# limitations under the License.+from auto_round.algorithms.transforms.awq.config import AWQConfig+from auto_round.algorithms.transforms.awq.quantizer import AWQQuantizer-class AlgConfig:- def __init__(self):- pass+__all__ = ["AWQConfig", "AWQQuantizer"]diff --git a/auto_round/algorithms/transforms/awq/config.py b/auto_round/algorithms/transforms/awq/config.pynew file mode 100644index 000000000..c2c580cb4--- /dev/null+++ b/auto_round/algorithms/transforms/awq/config.py@@ -0,0 +1,92 @@+# Copyright (c) 2026 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.++from auto_round.algorithms.quantization.config import QuantizationConfig+++class AWQConfig(QuantizationConfig):+ """Configuration for AWQ (Activation-Aware Weight Quantization).++ AWQ is a **pre-processing** algorithm (``role="preprocess"``). It+ protects salient weight channels by analyzing activation patterns and+ applying channel-wise scaling to reduce quantization error. After+ smoothing, a separate ``block_quantizer`` (RTN, SignRound, …) performs+ the actual weight compression.++ The quantization parameters (``bits``, ``group_size``, ``sym``,+ ``data_type``, …) on this config are used *only* for the internal grid+ search loss calculation (quantize-dequantize during scale selection).+ The definitive quantization parameters for the final weight compression+ step come from the pipeline's ``block_quantizer`` config.+ """++ def __init__(+ self,+ *,+ duo_scaling: bool | str = True,+ n_grid: int = 20,+ seqlen: int = 2048,+ nsamples: int = 128,+ batch_size: int = 8,+ apply_smooth: bool = True,+ mappings: list[dict] | None = None,+ **kwargs,+ ) -> None:+ """Initialize an AWQ configuration.++ Args:+ duo_scaling: Whether AWQ should use activation-aware and+ weight-aware scaling together. Use True to always enable+ duo scaling, False to use activation-only scaling, or+ "both" to search both modes and keep the better result.+ n_grid: Number of grid-search points used when searching the+ AWQ scaling ratio.+ seqlen: Calibration sequence length retained for compatibility+ with standalone AWQ entry points.+ nsamples: Number of calibration samples retained for compatibility+ with standalone AWQ entry points.+ batch_size: Batch size retained for compatibility with standalone+ AWQ entry points.+ apply_smooth: Whether to apply AWQ smoothing before the downstream+ block quantizer.+ mappings: Optional explicit AWQ smooth/balance mappings. Each+ item should contain ``smooth_layer`` and+ ``balance_layers`` entries. If None, mappings are inferred+ automatically from the model structure.+ **kwargs: Common quantization arguments forwarded to+ QuantizationConfig, such as bits, group_size, sym,+ data_type, and activation quantization fields.+ """+ super().__init__(**kwargs)++ if isinstance(duo_scaling, str) and duo_scaling != "both":+ raise ValueError(f"duo_scaling must be True, False, or 'both', got '{duo_scaling!r}'")+ self.duo_scaling = duo_scaling+ self.n_grid = n_grid+ self.seqlen = seqlen+ self.nsamples = nsamples+ self.batch_size = batch_size+ self.apply_smooth = apply_smooth+ self.mappings = mappings+ self.infer_bs_coeff = 1+ self.batch_dim = None+ # NOTE: enable_quanted_input is NOT set here. It belongs to the+ # block_quantizer (RTN/AutoRound), not to AWQ. See §3.7.1.++ def __repr__(self) -> str:+ return (+ f"AWQConfig(duo_scaling={self.duo_scaling!r}, n_grid={self.n_grid}, "+ f"bits={self.bits}, group_size={self.group_size}, sym={self.sym}, "+ f"mappings={'<explicit>' if self.mappings else 'auto'})"+ )diff --git a/auto_round/algorithms/quantization/awq/mappings.py b/auto_round/algorithms/transforms/awq/mappings.pysimilarity index 99%rename from auto_round/algorithms/quantization/awq/mappings.pyrename to auto_round/algorithms/transforms/awq/mappings.pyindex 32617632f..d7f6fd147 100644--- a/auto_round/algorithms/quantization/awq/mappings.py+++ b/auto_round/algorithms/transforms/awq/mappings.py@@ -386,7 +386,7 @@ def _build_hybrid_attention_mappings(model: torch.nn.Module) -> list[AWQMapping]mappings.append(AWQMapping(r"up_proj$", [r"down_proj$"]))- logger.warning(+ logger.info(f"Built dynamic hybrid-attention AWQ mappings: "f"{len(full_indices)} full-attention, {len(linear_indices)} linear-attention, "f"projections={linear_proj_names}, MoE={is_moe}"diff --git a/auto_round/algorithms/transforms/awq/quantizer.py b/auto_round/algorithms/transforms/awq/quantizer.pynew file mode 100644index 000000000..0908ac78c--- /dev/null+++ b/auto_round/algorithms/transforms/awq/quantizer.py@@ -0,0 +1,567 @@+# Copyright (c) 2026 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+"""AWQ (Activation-Aware Weight Quantization) quantizer.++Algorithm:+1. Collect per-channel activation magnitudes during calibration.+2. For each smooth-balance mapping, perform a grid search over scaling ratios+ to find the one that minimises quantization error (output-based loss).+3. Apply the best channel-wise scaling:+ - balance_layer.weight *= scales+ - smooth_layer.weight /= scales (or smooth_layer.bias /= scales if 1-D)+4. Weight compression is delegated to the pipeline's block_quantizer.++Reference implementations:+ - AutoAWQ: https://github.com/casper-hansen/AutoAWQ+ - llm-compressor: https://github.com/vllm-project/llm-compressor+"""++from __future__ import annotations++import gc+import inspect+from contextlib import contextmanager+from typing import TYPE_CHECKING, Any++import torch++from auto_round.algorithms.pipeline import (+ ActCalibPolicy,+ CalibTiming,+ InputSource,+)+from auto_round.algorithms.registry import register_pipeline_member+from auto_round.algorithms.transforms.awq.config import AWQConfig+from auto_round.algorithms.transforms.awq.mappings import (+ ResolvedMapping,+ _extract_block_prefix,+ check_model_compatibility,+ resolve_mappings,+)+from auto_round.algorithms.transforms.base import BaseWeightTransformer+from auto_round.data_type.utils import (+ get_quant_func,+ reshape_pad_tensor_by_group_size,+ revert_tensor_by_pad,+)+from auto_round.logger import logger++if TYPE_CHECKING:+ from auto_round.algorithms.pipeline import BlockContext+++@register_pipeline_member(AWQConfig)+class AWQQuantizer(BaseWeightTransformer):+ """AWQ quantizer: activation-aware weight smoothing pre-processor.++ Inherits :class:`~auto_round.algorithms.transforms.base.BaseWeightTransformer`.+ It smooths block weights in-place; actual weight compression (RTN /+ SignRound) is performed by the pipeline's ``block_quantizer``.+ """++ def __init__(self, config: AWQConfig) -> None:+ super().__init__(config)+ self.duo_scaling: bool | str = config.duo_scaling+ self.n_grid: int = config.n_grid++ self._user_mappings: list[dict] | None = config.mappings++ # Set at runtime by the compressor's post_init() via ``pre.layer_config = self.layer_config``.+ self.layer_config: dict | None = None++ self._resolved_mappings: list[ResolvedMapping] = []+ self._block_mappings: dict[str, list[ResolvedMapping]] = {}++ self._activation_stats: dict[str, list] = {}+ self._parent_args_cache: dict[torch.nn.Module, list[dict]] = {}+ self._parent_signatures: dict[int, inspect.Signature] = {}++ self._finalized: bool = False++ # ── Algorithm Fusion: lifecycle hook implementations ──────────────────────++ def bind(self, compressor) -> None:+ """Wire shared state and force AWQ onto single-block scheduling."""+ super().bind(compressor)+ nblocks = getattr(compressor, "nblocks", 1)+ if nblocks > 1:+ logger.warning(+ "AWQ does not support nblocks > 1 (got nblocks=%s). " "Falling back to nblocks=1.",+ nblocks,+ )+ compressor.nblocks = 1++ def prepare_run(self, compressor) -> None:+ """Validate compatibility, resolve model-wide mappings, and group by block prefix."""+ model = compressor.model_context.model+ report = check_model_compatibility(model, self._user_mappings)+ for warning in report["warnings"]:+ logger.warning(warning)++ # ── Resolve all model-level mappings (name-only, no module caching) ──+ self._resolved_mappings = resolve_mappings(model, self._user_mappings)+ if not self._resolved_mappings:+ raise ValueError(+ "AWQ: no layer mappings were resolved for this model. "+ f"Model class: {type(model).__name__}. "+ "To add support, provide explicit 'mappings' in AWQConfig, or "+ "add an entry to auto_round/algorithms/transforms/awq/mappings.py."+ )++ # Group mappings by block prefix for O(1) lookup during block iteration.+ self._block_mappings = {}+ for m in self._resolved_mappings:+ prefix = _extract_block_prefix(m.smooth_name)+ self._block_mappings.setdefault(prefix, []).append(m)++ if compressor.compress_context is not None:+ compressor.compress_context.cache_device = torch.device("cpu")++ logger.info(+ "AWQ: resolved %d mappings across %d blocks.",+ len(self._resolved_mappings),+ len(self._block_mappings),+ )+ self._finalized = False++ def get_act_calib_policy(self, ctx: "BlockContext"):+ """AWQ W4A16 (weight-only): no activation calibration needed."""+ # AWQ pre-processing does not collect act-calib stats; that is the+ # block_quantizer's concern. For W8A8/static activation, a post-smooth+ # forward may be needed — handled via the block_quantizer's policy.+ return ActCalibPolicy(when=CalibTiming.SKIP, source=InputSource.FP_CACHE)++ @contextmanager+ def block_forward_hooks(self, ctx: "BlockContext"):+ """Register AWQ activation-stats and parent-kwargs hooks.++ Hooks are registered on the *current block's* smooth sources and+ parent modules. All handles are removed when this context manager+ exits (before ``__exit__`` returns), regardless of exceptions.+ """+ handles = []+ block_mappings = self._block_mappings.get(ctx.block_name, [])+ if block_mappings:+ handles = self._register_awq_hooks(ctx.model, ctx.block, ctx.block_name)+ try:+ yield handles+ finally:+ for h in handles:+ h.remove()+ handles.clear()++ def pre_quantize_block(self, ctx: "BlockContext") -> None:+ """Apply AWQ smoothing for this block and mark modified params.++ Called after the reference forward (activation stats collected) and+ before the block quantizer runs.+ """+ if len(ctx.block_names) != 1:+ raise ValueError(f"AWQ requires nblocks=1, got {len(ctx.block_names)} blocks: {ctx.block_names}.")+ block_name = ctx.block_names[0]+ block_mappings = self._block_mappings.get(block_name, [])+ if not block_mappings:+ logger.debug("AWQ: no mappings for block '%s', skipping.", block_name)+ return+ self._smooth_block(block_name, block_mappings)+ modified = []+ for mapping in block_mappings:+ modified.extend(mapping.balance_names)+ modified.append(mapping.smooth_name)+ ctx.mark_modified_fp_params(modified)++ def post_quantize_block(self, ctx: "BlockContext") -> None:+ """Release per-block AWQ caches to free memory."""+ block_mappings = self._block_mappings.get(ctx.block_name, [])+ if not block_mappings:+ return+ for m in block_mappings:+ self._activation_stats.pop(m.smooth_name, None)+ seen_parents: set[int] = set()+ for m in block_mappings:+ pid = id(m.parent)+ if pid not in seen_parents:+ seen_parents.add(pid)+ self._parent_args_cache.pop(m.parent, None)++ def finalize_run(self, compressor) -> None:+ """Idempotent global teardown. Safe to call inside try/finally."""+ if self._finalized:+ return+ self._activation_stats.clear()+ self._parent_args_cache.clear()+ self._parent_signatures.clear()+ self._finalized = True+ logger.debug("AWQ: finalize_quantization complete.")++ # ── Hook registration ─────────────────────────────────────────────────────++ def _register_awq_hooks(+ self,+ model: torch.nn.Module,+ block: torch.nn.Module,+ block_name: str,+ ) -> list:+ """Register activation-stats and parent-kwargs hooks for one block."""+ handles = []+ mappings = self._block_mappings.get(block_name, [])+ smooth_names = {m.smooth_name for m in mappings}++ # ── Smooth-layer activation-stats hooks ───────────────────────────────+ # Priority: smooth source forward_hook (output stats).+ # Each smooth source is hooked exactly once (set de-duplication via name).+ for name, module in block.named_modules():+ full_name = f"{block_name}.{name}" if name else block_name+ if full_name not in smooth_names:+ continue++ def _make_stats_hook(layer_name: str):++ def hook_fn(mod, args, output):+ x = output[0] if isinstance(output, tuple) else output+ if x is None or x.numel() == 0:+ return+ channel_sum = x.detach().float().flatten(0, -2).abs().sum(dim=0).cpu()+ count = x[..., 0].numel()+ if layer_name not in self._activation_stats:+ self._activation_stats[layer_name] = [+ torch.zeros_like(channel_sum),+ 0,+ ]+ self._activation_stats[layer_name][0] += channel_sum+ self._activation_stats[layer_name][1] += count++ return hook_fn++ h = module.register_forward_hook(_make_stats_hook(full_name))+ handles.append(h)++ # ── Parent-kwargs hooks ───────────────────────────────────────────────+ # One forward_pre_hook per unique parent module in the current block.+ parent_modules_hooked: set[int] = set()+ for mapping in mappings:+ parent = mapping.parent+ hook_target = mapping.activation_hook_target+ if hook_target:+ target_parent = dict(model.named_modules()).get(hook_target)+ if target_parent is None:+ logger.warning(+ "AWQ: activation_hook_target '%s' for '%s' was not found; using resolved parent '%s'.",+ hook_target,+ mapping.smooth_name,+ mapping.parent_name,+ )+ else:+ parent = target_parent+ if id(parent) in parent_modules_hooked:+ continue+ parent_modules_hooked.add(id(parent))++ if parent not in self._parent_args_cache:+ self._parent_args_cache[parent] = []++ def _make_parent_hook(parent_module: torch.nn.Module):++ def hook_fn(mod, args, kwargs):+ cls_id = id(type(mod))+ if cls_id not in self._parent_signatures:+ self._parent_signatures[cls_id] = inspect.signature(mod.forward)+ sig = self._parent_signatures[cls_id]+ try:+ bound = sig.bind(*args, **kwargs)+ bound.apply_defaults()+ except TypeError:+ return # signature mismatch; skip this sample++ param = next(mod.parameters(), None)+ w_dtype = param.dtype if param is not None else None++ stored: dict[str, Any] = {}+ for k, v in bound.arguments.items():+ if isinstance(v, torch.Tensor):+ v = v.detach()+ if w_dtype and v.is_floating_point() and v.dtype != w_dtype:+ v = v.to(w_dtype)+ stored[k] = v+ elif isinstance(v, tuple) and any(isinstance(t, torch.Tensor) for t in v):+ stored[k] = tuple(+ (+ t.detach().to(w_dtype)+ if (w_dtype and isinstance(t, torch.Tensor) and t.is_floating_point())+ else (t.detach() if isinstance(t, torch.Tensor) else t)+ )+ for t in v+ )+ elif hasattr(v, "key_cache"):+ stored[k] = None # Null out KV cache objects+ else:+ stored[k] = v++ self._parent_args_cache[parent_module].append(stored)++ return hook_fn++ h = parent.register_forward_pre_hook(_make_parent_hook(parent), with_kwargs=True)+ handles.append(h)++ return handles++ # ── Smoothing (grid search + scale apply) ─────────────────────────────────++ def _smooth_block(self, block_prefix: str, block_mappings: list) -> None:+ """Run grid search and apply AWQ scales for one block."""+ for mapping in block_mappings:+ if mapping.smooth_name not in self._activation_stats:+ logger.warning(+ "AWQ: no activation stats for '%s' in block '%s'; skipping.",+ mapping.smooth_name,+ block_prefix,+ )+ continue++ act_sum, act_count = self._activation_stats[mapping.smooth_name]+ if act_count == 0:+ logger.warning(+ "AWQ: zero activation count for '%s' in block '%s'; skipping.",+ mapping.smooth_name,+ block_prefix,+ )+ continue++ x_mean = (act_sum / act_count).to(torch.float32)+ del act_sum++ best_scales = self._grid_search_scales(mapping, x_mean)+ if best_scales is not None:+ self._apply_scales(mapping, best_scales)++ # Release parent kwargs after ALL mappings for this block are processed.+ seen_parents: set[int] = set()+ for mapping in block_mappings:+ pid = id(mapping.parent)+ if pid not in seen_parents:+ seen_parents.add(pid)+ self._parent_args_cache.pop(mapping.parent, None)++ def _get_grid_search_params(self) -> list[tuple[float, bool]]:+ """Return (ratio, use_duo_scaling) tuples for the grid search."""+ match self.duo_scaling:+ case "both":+ n = max(int(self.n_grid / 2), 2)+ return [(idx / (n - 1), duo) for idx in range(n) for duo in [False, True]]+ case False:+ n = max(self.n_grid, 2)+ return [(idx / (n - 1), False) for idx in range(n)]+ case True:+ n = max(self.n_grid, 3)+ return [(0.0, False)] + [(idx / (n - 2), True) for idx in range(n - 1)]+ case _:+ raise ValueError(f"Unexpected duo_scaling value: {self.duo_scaling!r}")++ @staticmethod+ def _compute_layer_means(layers: list[torch.nn.Module], group_size: int) -> torch.Tensor:+ """Per-channel mean of normalised weights across all balance layers."""+ weight = torch.cat([m.weight.detach().float() for m in layers], dim=0)+ org_shape = weight.shape+ gs = group_size if group_size > 0 else org_shape[1]+ weight, _, pad_len = reshape_pad_tensor_by_group_size(weight, gs)+ w_scale = weight.abs() / (weight.abs().amax(dim=1, keepdim=True) + 1e-6)+ w_scale = revert_tensor_by_pad(w_scale, orig_shape=org_shape, pad_len=pad_len)+ return w_scale.mean(0)++ @torch.no_grad()+ def _grid_search_scales(+ self,+ mapping: ResolvedMapping,+ x_mean: torch.Tensor,+ ) -> torch.Tensor | None:+ """Find the best scaling ratio for *mapping* via output-based loss."""+ device = mapping.balance_layers[0].weight.device+ x_mean = x_mean.to(device)++ group_size = self.group_size if (self.group_size is not None and self.group_size > 0) else -1+ if self.duo_scaling is not False:+ w_mean = self._compute_layer_means(mapping.balance_layers, group_size).to(device)++ parent_kwargs_list = self._parent_args_cache.get(mapping.parent, [])+ use_parent_forward = len(parent_kwargs_list) > 0++ if use_parent_forward:+ fp16_outputs = self._run_parent_samples(mapping.parent, parent_kwargs_list)+ if not fp16_outputs or all(f.numel() == 0 for f in fp16_outputs):+ use_parent_forward = False++ orig_state = {bl: bl.weight.data.clone() for bl in mapping.balance_layers}+ if not use_parent_forward:+ orig_weights = orig_state # same reference is fine++ # Pre-resolve quant function once to avoid repeated dispatch in loop.+ ref_layer = mapping.balance_layers[0]+ ref_name = getattr(ref_layer, "global_name", None) or ""+ ref_cfg = (self.layer_config or {}).get(ref_name, {})+ try:+ cached_quant_func, _ = get_quant_func(+ ref_cfg.get("data_type", self.data_type),+ ref_cfg.get("bits", self.bits),+ ref_cfg.get("sym", self.sym),+ disable_opt_rtn=True,+ group_size=ref_cfg.get("group_size", self.group_size),+ iters=0,+ )+ except Exception as exc:+ logger.debug("AWQ: failed to pre-resolve quant function for '%s': %s", ref_name, exc)+ cached_quant_func = None++ best_error = float("inf")+ best_scales = None+ best_ratio = -1++ for ratio, use_duo in self._get_grid_search_params():+ if use_duo:+ scales = (x_mean.pow(ratio) / (w_mean.pow(1 - ratio) + 1e-4)).clamp(min=1e-4)+ else:+ scales = x_mean.pow(ratio).clamp(min=1e-4).view(-1)+ scales = scales / (scales.max() * scales.min()).sqrt()+ scales[torch.isinf(scales)] = 1+ scales[torch.isnan(scales)] = 1+ scales_view = scales.view(1, -1).to(device)++ if use_parent_forward:+ for bl in mapping.balance_layers:+ bl.weight.data.copy_(orig_state[bl] * scales_view)+ w_qdq = self._quantize_dequantize_weight(bl, bl.weight.data.float(), quant_func=cached_quant_func)+ if w_qdq is not None:+ bl.weight.data = (w_qdq / scales_view).to(bl.weight.dtype)+ else:+ bl.weight.data.copy_(orig_state[bl])++ int_w_outputs = self._run_parent_samples(mapping.parent, parent_kwargs_list)+ total_loss = self._compute_loss(fp16_outputs, int_w_outputs)+ del int_w_outputs+ for bl in mapping.balance_layers:+ bl.weight.data.copy_(orig_state[bl])+ else:+ total_loss = 0.0+ for bl in mapping.balance_layers:+ w_orig = orig_weights[bl].to(device)+ w_qdq = self._quantize_dequantize_weight(bl, w_orig * scales_view, quant_func=cached_quant_func)+ if w_qdq is None:+ total_loss = float("inf")+ break+ total_loss += (w_orig - w_qdq / scales_view).pow(2).sum().item()++ if total_loss < best_error:+ best_error = total_loss+ best_scales = scales.clone()+ best_ratio = ratio++ if best_ratio < 0:+ logger.warning("AWQ: grid search failed for '%s': no finite error.", mapping.smooth_name)+ return None++ logger.debug("AWQ '%s': best_ratio=%.2f, best_error=%.3e", mapping.smooth_name, best_ratio, best_error)+ return best_scales++ @torch.no_grad()+ def _run_parent_samples(+ self,+ parent: torch.nn.Module,+ kwargs_list: list[dict],+ ) -> list[torch.Tensor]:+ outputs = []+ for stored_kwargs in kwargs_list:+ out = parent(**stored_kwargs)+ if isinstance(out, tuple):+ out = out[0]+ outputs.append(out)+ return outputs++ @staticmethod+ @torch.no_grad()+ def _compute_loss(+ fp16_outputs: list[torch.Tensor],+ int_w_outputs: list[torch.Tensor],+ ) -> float:+ device = fp16_outputs[0].device+ loss = torch.tensor(0.0, device=device)+ num_elements = torch.tensor(0, device=device, dtype=torch.long)+ for fp16_out, int_w_out in zip(fp16_outputs, int_w_outputs):+ loss += torch.nn.functional.mse_loss(+ fp16_out.float(),+ int_w_out.to(fp16_out.device).float(),+ reduction="sum",+ )+ num_elements += fp16_out.numel()+ if num_elements == 0:+ return float("inf")+ return (loss / num_elements).item()++ def _quantize_dequantize_weight(+ self,+ layer: torch.nn.Module,+ weight: torch.Tensor,+ quant_func=None,+ ) -> torch.Tensor | None:+ """Quantize-dequantize a weight tensor using the layer's config.++ Used internally for grid search loss calculation only. Does NOT+ modify the layer's stored weights.+ """+ layer_name = getattr(layer, "global_name", None) or ""+ config = (self.layer_config or {}).get(layer_name, {})+ bits = config.get("bits", self.bits)+ group_size = config.get("group_size", self.group_size)+ sym = config.get("sym", self.sym)+ data_type = config.get("data_type", self.data_type)++ if quant_func is None:+ try:+ quant_func, _ = get_quant_func(+ data_type,+ bits,+ sym,+ disable_opt_rtn=True,+ group_size=group_size,+ iters=0,+ )+ except Exception as exc:+ logger.debug("AWQ: failed to resolve quant function for '%s': %s", layer_name, exc)+ return None++ if quant_func is None:+ return None++ try:+ qdq_weight, _, _ = quant_func(weight, bits=bits, group_size=group_size)+ return qdq_weight+ except Exception as exc:+ logger.debug("AWQ: quantize-dequantize failed for '%s': %s", layer_name, exc)+ return None++ @torch.no_grad()+ def _apply_scales(self, mapping: ResolvedMapping, scales: torch.Tensor) -> None:+ """Apply computed AWQ scales to smooth and balance layers in-place."""+ for bl in mapping.balance_layers:+ s = scales.to(bl.weight.device).view(1, -1)+ bl.weight.data.mul_(s)++ smooth = mapping.smooth_layer+ s = scales.to(smooth.weight.device)+ if smooth.weight.ndim == 1:+ smooth.weight.data.div_(s)+ else:+ smooth.weight.data[-s.size(0) :].div_(s.view(-1, 1))++ if hasattr(smooth, "bias") and smooth.bias is not None:+ smooth.bias.data.div_(s)diff --git a/auto_round/algorithms/transforms/base.py b/auto_round/algorithms/transforms/base.pyindex db48c6059..4a36b8080 100644--- a/auto_round/algorithms/transforms/base.py+++ b/auto_round/algorithms/transforms/base.py@@ -27,6 +27,21 @@ from typing import Any, Optionalimport torchimport torch.nn as nn+from auto_round.algorithms.base import BasePipelineMember+++class BaseWeightTransformer(BasePipelineMember):+ """Base class for weight-transformation algorithms in a QuantizationPipeline."""++ def pre_quantize_block(self, ctx) -> None:+ """Called after the reference forward, before block quantization."""+ return++ def post_quantize_block(self, ctx) -> None:+ """Called after the block quantizer completes."""+ return++# ---------------------------------------------------------------------------# Config base# ---------------------------------------------------------------------------@@ -159,7 +174,7 @@ def _ensure_registry_populated() -> None:# Import each sub-package here. Add new entries as more algorithms land.import importlib- for sub in ("rotation", "spinquant"):+ for sub in ("quarot", "spinquant"):try:importlib.import_module(f"auto_round.algorithms.transforms.{sub}")except ImportError:diff --git a/auto_round/algorithms/transforms/rotation/__init__.py b/auto_round/algorithms/transforms/quarot/__init__.pysimilarity index 85%rename from auto_round/algorithms/transforms/rotation/__init__.pyrename to auto_round/algorithms/transforms/quarot/__init__.pyindex 640cf06ab..3faf385e4 100644--- a/auto_round/algorithms/transforms/rotation/__init__.py+++ b/auto_round/algorithms/transforms/quarot/__init__.py@@ -13,15 +13,15 @@# limitations under the License."""Hadamard rotation sub-package for ``algorithms/transforms``."""-from auto_round.algorithms.transforms.rotation.apply import (+from auto_round.algorithms.transforms.quarot.apply import (HadamardRotation,apply_rotation_transform,)-from auto_round.algorithms.transforms.rotation.config import (+from auto_round.algorithms.transforms.quarot.config import (RotationConfig,normalize_rotation_config,)-from auto_round.algorithms.transforms.rotation.transforms import (+from auto_round.algorithms.transforms.quarot.transforms import (HADAMARDS,HadamardTransform,RandomHadamardTransform,diff --git a/auto_round/algorithms/transforms/rotation/apply.py b/auto_round/algorithms/transforms/quarot/apply.pysimilarity index 91%rename from auto_round/algorithms/transforms/rotation/apply.pyrename to auto_round/algorithms/transforms/quarot/apply.pyindex 62ceb02b5..9656002cc 100644--- a/auto_round/algorithms/transforms/rotation/apply.py+++ b/auto_round/algorithms/transforms/quarot/apply.py@@ -27,8 +27,8 @@ import torchimport tqdmfrom auto_round.algorithms.transforms.base import BaseRotation-from auto_round.algorithms.transforms.rotation.config import RotationConfig, normalize_rotation_config-from auto_round.algorithms.transforms.rotation.transforms import build_hadamard_transform+from auto_round.algorithms.transforms.quarot.config import RotationConfig, normalize_rotation_config+from auto_round.algorithms.transforms.quarot.transforms import build_hadamard_transformfrom auto_round.compressors.utils import is_nv_fpfrom auto_round.experimental.qmodules.base import QModuleBase@@ -44,7 +44,7 @@ def _triton_available(data_type: str = "mx_fp") -> bool:if not torch.cuda.is_available():return False- from auto_round.algorithms.transforms.rotation.utils.triton.mxfp4 import ( # noqa: F401+ from auto_round.algorithms.transforms.quarot.utils.triton.mxfp4 import ( # noqa: F401mxfp4_forward_kernel_wrapper,)@@ -67,7 +67,7 @@ class HadamardRotation(BaseRotation):Or directly::- from auto_round.algorithms.transforms.rotation import apply_rotation_transform+ from auto_round.algorithms.transforms.quarot import apply_rotation_transformmodel = apply_rotation_transform(model, config=RotationConfig(), need_calibration=True)"""@@ -109,13 +109,13 @@ class HadamardRotation(BaseRotation):# Dispatch by backend. The transform backend (triton-fused per-Linear)# is implemented below; the inplace (QuaRot) backend is delegated to- # :mod:`auto_round.algorithms.transforms.rotation.inplace`.- from auto_round.algorithms.transforms.rotation.dispatcher import resolve_hadamard_backend+ # :mod:`auto_round.algorithms.transforms.quarot.inplace`.+ from auto_round.algorithms.transforms.quarot.dispatcher import resolve_hadamard_backendbackend = resolve_hadamard_backend(cfg, data_type)if backend == "inplace":import auto_round.envs as envs- from auto_round.algorithms.transforms.rotation.inplace import apply_rotation_transform as _inplace_apply+ from auto_round.algorithms.transforms.quarot.inplace import apply_rotation_transform as _inplace_apply# Resolve fuse flag: explicit > env var > default(False).fuse_online_to_weight = cfg.fuse_online_to_weight@@ -181,7 +181,7 @@ def _apply_to_module(def _apply_input_transform(module: torch.nn.Module, config: RotationConfig, data_type: str = "mx_fp") -> None:"""Register a forward pre-hook that applies the Hadamard to the input activation."""- from auto_round.algorithms.transforms.rotation.utils.matrix import multihead_matmul+ from auto_round.algorithms.transforms.quarot.utils.matrix import multihead_matmulinp_transform = build_hadamard_transform(**config.model_dump(),@@ -197,7 +197,7 @@ def _apply_input_transform(module: torch.nn.Module, config: RotationConfig, datahadamard_weight = Noneif _triton_available(data_type):- from auto_round.algorithms.transforms.rotation.utils.triton.mxfp4 import mxfp4_forward_kernel_wrapper+ from auto_round.algorithms.transforms.quarot.utils.triton.mxfp4 import mxfp4_forward_kernel_wrapperdef _input_hook(self, args):x = args[0]@@ -232,7 +232,7 @@ def _apply_weight_transform(config: RotationConfig,) -> None:"""Fuse or patch the Hadamard rotation into the weight of *module*."""- from auto_round.algorithms.transforms.rotation.patch import (+ from auto_round.algorithms.transforms.quarot.patch import (patch_quantlinear,patch_wrapperlinear_to_apply_transform,patch_wrapperwalayer_forward_to_apply_transform,@@ -248,7 +248,7 @@ def _apply_weight_transform(# For random Hadamard, save the matrix as a submodule for serialisation.if config.hadamard_type == "random_hadamard":- from auto_round.algorithms.transforms.rotation.patch import patch_quantlinear as _patch_ql+ from auto_round.algorithms.transforms.quarot.patch import patch_quantlinear as _patch_ql_patch_ql(w_transform)diff --git a/auto_round/algorithms/transforms/rotation/config.py b/auto_round/algorithms/transforms/quarot/config.pysimilarity index 85%rename from auto_round/algorithms/transforms/rotation/config.pyrename to auto_round/algorithms/transforms/quarot/config.pyindex 8143e95d1..5763054f9 100644--- a/auto_round/algorithms/transforms/rotation/config.py+++ b/auto_round/algorithms/transforms/quarot/config.py@@ -19,13 +19,13 @@ schema.Two implementation backends share this one schema (method B):* ``backend="inplace"`` – QuaRot-style residual-stream rotation, implemented- under :mod:`auto_round.algorithms.transforms.rotation.inplace`. Works for any+ under :mod:`auto_round.algorithms.transforms.quarot.inplace`. Works for anyweight/activation dtype and can optionally fuse the online Hadamard intoweights (``fuse_online_to_weight=True``).* ``backend="transform"`` – Per-Linear weight + activation Hadamard with afused triton kernel, implemented under- :mod:`auto_round.algorithms.transforms.rotation.apply`. Supports only+ :mod:`auto_round.algorithms.transforms.quarot.apply`. Supports onlyMXFP4 / NVFP4 and cannot fuse online to weight.* ``backend="auto"`` – dispatcher picks inplace when a fused online rotation@@ -85,6 +85,27 @@ class RotationConfig(BaseModel, BaseRotationConfig):model_config = {"arbitrary_types_allowed": True}+ def __init__(self, **data: Any) -> None:+ """Initialize a Hadamard rotation configuration.++ Args:+ algorithm: Canonical algorithm name used for registry lookup.+ backend: Rotation backend to use. ``auto`` lets AutoRound pick+ an implementation, ``inplace`` uses QuaRot-style online+ rotation, and ``transform`` uses the transform backend.+ block_size: Grouped Hadamard block size. None keeps the backend+ default behavior.+ hadamard_type: Hadamard transform variant, such as+ ``hadamard``, ``random_hadamard``, or ``quarot_hadamard``.+ fuse_online_to_weight: Whether online Hadamard rotation should+ be fused into the weights when supported.+ allow_online_rotation: Whether online activation rotation is+ allowed.+ random_seed: Internal flag used by random Hadamard paths.+ **data: Additional Pydantic field values forwarded to BaseModel.+ """+ super().__init__(**data)+@field_validator("backend")@classmethoddef _validate_backend(cls, v: str) -> str:diff --git a/auto_round/algorithms/transforms/rotation/dispatcher.py b/auto_round/algorithms/transforms/quarot/dispatcher.pysimilarity index 93%rename from auto_round/algorithms/transforms/rotation/dispatcher.pyrename to auto_round/algorithms/transforms/quarot/dispatcher.pyindex 5aebd9443..ae4d6c2e4 100644--- a/auto_round/algorithms/transforms/rotation/dispatcher.py+++ b/auto_round/algorithms/transforms/quarot/dispatcher.py@@ -9,11 +9,11 @@Two backend implementations exist:-* ``inplace`` – :mod:`auto_round.algorithms.transforms.rotation.inplace`+* ``inplace`` – :mod:`auto_round.algorithms.transforms.quarot.inplace`QuaRot-style residual-stream rotation. Works for any weight/activationdtype. Optionally fuses the online Hadamard into weights(``fuse_online_to_weight=True``).-* ``transform`` – :mod:`auto_round.algorithms.transforms.rotation`+* ``transform`` – :mod:`auto_round.algorithms.transforms.quarot`Per-Linear weight + activation Hadamard with a fused triton kernel.Only supports MXFP4 / NVFP4 and **cannot** fuse online to weight.@@ -33,7 +33,7 @@ from typing import Any, Unionimport torchimport auto_round.envs as envs-from auto_round.algorithms.transforms.rotation.config import RotationConfig, normalize_rotation_config+from auto_round.algorithms.transforms.quarot.config import RotationConfig, normalize_rotation_configfrom auto_round.compressors.utils import is_mx_fp, is_nv_fpfrom auto_round.utils import logger@@ -123,7 +123,7 @@ def apply_hadamard_rotation(if backend == "inplace":logger.warning("this backend does not support real exporting, please export the model to fake format")- from auto_round.algorithms.transforms.rotation.inplace import apply_rotation_transform+ from auto_round.algorithms.transforms.quarot.inplace import apply_rotation_transform# block_size -> group_size (None / -1 / 0 means full-dimension)bs = config.block_size@@ -146,7 +146,7 @@ def apply_hadamard_rotation(supported_hadamard_types = ("hadamard", "random_hadamard")if config.hadamard_type not in supported_hadamard_types:raise ValueError("this backend only supports hadamard or random_hadamard")- from auto_round.algorithms.transforms.rotation.apply import apply_rotation_transform+ from auto_round.algorithms.transforms.quarot.apply import apply_rotation_transformreturn apply_rotation_transform(model, config, data_type=data_type)else:diff --git a/auto_round/algorithms/transforms/rotation/inplace/__init__.py b/auto_round/algorithms/transforms/quarot/inplace/__init__.pysimilarity index 56%rename from auto_round/algorithms/transforms/rotation/inplace/__init__.pyrename to auto_round/algorithms/transforms/quarot/inplace/__init__.pyindex d7b85d9b1..53fa25fc1 100644--- a/auto_round/algorithms/transforms/rotation/inplace/__init__.py+++ b/auto_round/algorithms/transforms/quarot/inplace/__init__.py@@ -5,7 +5,7 @@Canonical home of the residual-stream Hadamard rotation implementation."""-from auto_round.algorithms.transforms.rotation.inplace.apply import apply_rotation_transform # noqa: F401-from auto_round.algorithms.transforms.rotation.inplace.hooks import clear_random_hadamard_cache # noqa: F401+from auto_round.algorithms.transforms.quarot.inplace.apply import apply_rotation_transform # noqa: F401+from auto_round.algorithms.transforms.quarot.inplace.hooks import clear_random_hadamard_cache # noqa: F401__all__ = ["apply_rotation_transform", "clear_random_hadamard_cache"]diff --git a/auto_round/algorithms/transforms/rotation/inplace/apply.py b/auto_round/algorithms/transforms/quarot/inplace/apply.pysimilarity index 99%rename from auto_round/algorithms/transforms/rotation/inplace/apply.pyrename to auto_round/algorithms/transforms/quarot/inplace/apply.pyindex 6590798da..8093ead98 100644--- a/auto_round/algorithms/transforms/rotation/inplace/apply.py+++ b/auto_round/algorithms/transforms/quarot/inplace/apply.py@@ -14,7 +14,7 @@ from typing import Dict, Unionimport torchimport tqdm-from auto_round.algorithms.transforms.rotation.inplace.hooks import (+from auto_round.algorithms.transforms.quarot.inplace.hooks import (CrossHeadOnlineHadamardHook,FullOnlineHadamardHook,GroupOnlineHadamardHook,@@ -29,7 +29,7 @@ from auto_round.algorithms.transforms.rotation.inplace.hooks import (get_hadK,get_or_create_random_hadamard,)-from auto_round.algorithms.transforms.rotation.inplace.model_config import (+from auto_round.algorithms.transforms.quarot.inplace.model_config import (MAPPING_REGISTRY,RotationMapping,_resolve,@@ -210,7 +210,7 @@ def _subtract_embedding_mean(model, mapping: RotationMapping) -> None:class _RMSNorm(torch.nn.Module):"""RMS Normalization (no mean subtraction)."""- def __init__(self, dim: int, eps: float = 1e-5):+ def __init__(self, dim: int, eps: float = 1e-5) -> None:super().__init__()self.eps = epsself.register_buffer("weight", torch.ones(dim))diff --git a/auto_round/algorithms/transforms/rotation/inplace/hooks.py b/auto_round/algorithms/transforms/quarot/inplace/hooks.pysimilarity index 96%rename from auto_round/algorithms/transforms/rotation/inplace/hooks.pyrename to auto_round/algorithms/transforms/quarot/inplace/hooks.pyindex 2b3d26c6e..c2f73e62c 100644--- a/auto_round/algorithms/transforms/rotation/inplace/hooks.py+++ b/auto_round/algorithms/transforms/quarot/inplace/hooks.py@@ -9,6 +9,7 @@ provides the hooks and a helper to register them on the model."""import math+from typing import Optionalimport torchimport torch.nn as nn@@ -153,7 +154,14 @@ def _get_custom_had(had_dict, size):class FullOnlineHadamardHook(nn.Module):"""Pre-forward hook: full Hadamard on the entire last dimension (for ``down_proj``)."""- def __init__(self, had_K, K, fp32_had=False, use_fast_had=True, had_matrix=None):+ def __init__(+ self,+ had_K: Optional[torch.Tensor],+ K: Optional[int],+ fp32_had: bool = False,+ use_fast_had: bool = True,+ had_matrix: Optional[torch.Tensor] = None,+ ) -> None:super().__init__()self.custom_had = had_matrix is not Noneif had_matrix is not None:@@ -209,7 +217,15 @@ class CrossHeadOnlineHadamardHook(nn.Module):* transpose back and reshape"""- def __init__(self, had_K, K, head_dim, fp32_had=False, use_fast_had=True, had_matrix=None):+ def __init__(+ self,+ had_K: Optional[torch.Tensor],+ K: Optional[int],+ head_dim: int,+ fp32_had: bool = False,+ use_fast_had: bool = True,+ had_matrix: Optional[torch.Tensor] = None,+ ) -> None:"""Args:had_K: Hadamard sub-matrix from ``get_hadK(num_heads)``.@@ -295,7 +311,7 @@ def register_online_had_hooks(model, mapping=None, fp32_had=False, use_fast_had=list of hook handles (call ``handle.remove()`` to detach)."""if mapping is None:- from auto_round.algorithms.transforms.rotation.inplace.model_config import infer_mapping_from_model+ from auto_round.algorithms.transforms.quarot.inplace.model_config import infer_mapping_from_modelmapping = infer_mapping_from_model(model)@@ -352,7 +368,7 @@ def get_hadK(n: int, transpose=False) -> (torch.Tensor, int):K = 1return hadK, Kelse:- from auto_round.algorithms.transforms.rotation.utils.math import _fetch_hadamard_divisor+ from auto_round.algorithms.transforms.quarot.utils.math import _fetch_hadamard_divisorhadK = _fetch_hadamard_divisor(n, torch.float, torch.device("cpu"))if transpose:@@ -574,7 +590,7 @@ class OnlineHadamardPostHook(nn.Module):rotation is not fused into weights."""- def __init__(self, pre_hook):+ def __init__(self, pre_hook: nn.Module) -> None:super().__init__()self.pre_hook = pre_hook@@ -592,7 +608,13 @@ class GroupOnlineHadamardHook(nn.Module):per group, then reshapes back. Much cheaper than a full-dimension Hadamard."""- def __init__(self, group_size, fp32_had=False, use_fast_had=True, had_matrix=None):+ def __init__(+ self,+ group_size: int,+ fp32_had: bool = False,+ use_fast_had: bool = True,+ had_matrix: Optional[torch.Tensor] = None,+ ) -> None:super().__init__()self.group_size = group_sizeself.fp32_had = fp32_haddiff --git a/auto_round/algorithms/transforms/rotation/inplace/model_config.py b/auto_round/algorithms/transforms/quarot/inplace/model_config.pysimilarity index 100%rename from auto_round/algorithms/transforms/rotation/inplace/model_config.pyrename to auto_round/algorithms/transforms/quarot/inplace/model_config.pydiff --git a/auto_round/algorithms/transforms/rotation/patch.py b/auto_round/algorithms/transforms/quarot/patch.pysimilarity index 100%rename from auto_round/algorithms/transforms/rotation/patch.pyrename to auto_round/algorithms/transforms/quarot/patch.pydiff --git a/auto_round/algorithms/transforms/rotation/transforms.py b/auto_round/algorithms/transforms/quarot/transforms.pysimilarity index 97%rename from auto_round/algorithms/transforms/rotation/transforms.pyrename to auto_round/algorithms/transforms/quarot/transforms.pyindex 78dcd0d6b..71320e61e 100644--- a/auto_round/algorithms/transforms/rotation/transforms.py+++ b/auto_round/algorithms/transforms/quarot/transforms.py@@ -27,11 +27,11 @@ from typing import Any, Callable, Dictimport torchimport torch.nn as nn-from auto_round.algorithms.transforms.rotation.utils.math import (+from auto_round.algorithms.transforms.quarot.utils.math import (deterministic_hadamard_matrix,random_hadamard_matrix,)-from auto_round.algorithms.transforms.rotation.utils.matrix import apply_transform_weight+from auto_round.algorithms.transforms.quarot.utils.matrix import apply_transform_weight__all__ = ["HadamardTransform",diff --git a/auto_round/algorithms/transforms/rotation/utils/__init__.py b/auto_round/algorithms/transforms/quarot/utils/__init__.pysimilarity index 100%rename from auto_round/algorithms/transforms/rotation/utils/__init__.pyrename to auto_round/algorithms/transforms/quarot/utils/__init__.pydiff --git a/auto_round/algorithms/transforms/rotation/utils/hadamards.safetensors b/auto_round/algorithms/transforms/quarot/utils/hadamards.safetensorssimilarity index 100%rename from auto_round/algorithms/transforms/rotation/utils/hadamards.safetensorsrename to auto_round/algorithms/transforms/quarot/utils/hadamards.safetensorsdiff --git a/auto_round/algorithms/transforms/rotation/utils/math.py b/auto_round/algorithms/transforms/quarot/utils/math.pysimilarity index 100%rename from auto_round/algorithms/transforms/rotation/utils/math.pyrename to auto_round/algorithms/transforms/quarot/utils/math.pydiff --git a/auto_round/algorithms/transforms/rotation/utils/matrix.py b/auto_round/algorithms/transforms/quarot/utils/matrix.pysimilarity index 100%rename from auto_round/algorithms/transforms/rotation/utils/matrix.pyrename to auto_round/algorithms/transforms/quarot/utils/matrix.pydiff --git a/auto_round/algorithms/transforms/rotation/utils/triton/__init__.py b/auto_round/algorithms/transforms/quarot/utils/triton/__init__.pysimilarity index 100%rename from auto_round/algorithms/transforms/rotation/utils/triton/__init__.pyrename to auto_round/algorithms/transforms/quarot/utils/triton/__init__.pydiff --git a/auto_round/algorithms/transforms/rotation/utils/triton/mxfp4.py b/auto_round/algorithms/transforms/quarot/utils/triton/mxfp4.pysimilarity index 100%rename from auto_round/algorithms/transforms/rotation/utils/triton/mxfp4.pyrename to auto_round/algorithms/transforms/quarot/utils/triton/mxfp4.pydiff --git a/auto_round/algorithms/transforms/spinquant/cayley_optimizer.py b/auto_round/algorithms/transforms/spinquant/cayley_optimizer.pyindex 7b0d4f488..f919deaef 100644--- a/auto_round/algorithms/transforms/spinquant/cayley_optimizer.py+++ b/auto_round/algorithms/transforms/spinquant/cayley_optimizer.py@@ -11,7 +11,7 @@ Core algorithm: SGD with Cayley transform to maintain orthogonality constraints.from __future__ import annotations-from typing import Any, Callable+from typing import Any, Callable, Iterableimport torchfrom torch.optim.optimizer import Optimizer@@ -32,12 +32,12 @@ class SGDG(Optimizer):def __init__(self,- params,+ params: Iterable[torch.Tensor],lr: float = 1e-4,stiefel: bool = True,momentum: float = 0.0,weight_decay: float = 0.0,- ):+ ) -> None:defaults = dict(lr=lr, momentum=momentum, weight_decay=weight_decay, stiefel=stiefel)super().__init__(params, defaults)self.stiefel = stiefel@@ -139,7 +139,7 @@ class AdamAndSGDG(torch.optim.Optimizer):smooth_learning_rate: float = 1e-3,adam_betas: tuple[float, float] = (0.9, 0.999),adam_eps: float = 1e-8,- ):+ ) -> None:self._has_adam = len(adam_params) > 0self._has_sgdg = len(sgdg_params) > 0diff --git a/auto_round/algorithms/transforms/spinquant/inplace/__init__.py b/auto_round/algorithms/transforms/spinquant/inplace/__init__.pyindex dc80ac2a5..ac3071a93 100644--- a/auto_round/algorithms/transforms/spinquant/inplace/__init__.py+++ b/auto_round/algorithms/transforms/spinquant/inplace/__init__.py@@ -4,7 +4,7 @@"""SpinQuant in-place application sub-package.-Follows the same structure as ``auto_round.algorithms.transforms.rotation.inplace``.+Follows the same structure as ``auto_round.algorithms.transforms.quarot.inplace``."""from auto_round.algorithms.transforms.spinquant.inplace.apply import (diff --git a/auto_round/algorithms/transforms/spinquant/inplace/apply.py b/auto_round/algorithms/transforms/spinquant/inplace/apply.pyindex c831c193f..f4e7cba9f 100644--- a/auto_round/algorithms/transforms/spinquant/inplace/apply.py+++ b/auto_round/algorithms/transforms/spinquant/inplace/apply.py@@ -6,7 +6,7 @@ SpinQuant in-place application utilities.This module provides ``apply_spinquant_in_place`` and hook registrationthat follow the same patterns used by AutoRound's-``auto_round.algorithms.transforms.rotation.inplace`` package.+``auto_round.algorithms.transforms.quarot.inplace`` package.R3 rotation uses the architecture-generic monkeypatch approach from QuaRot/Quark:we replace ``apply_rotary_pos_emb`` in the attention forward's globals with adiff --git a/auto_round/algorithms/transforms/spinquant/monkeypatch.py b/auto_round/algorithms/transforms/spinquant/monkeypatch.pyindex d7f249c63..7fc3d9b46 100644--- a/auto_round/algorithms/transforms/spinquant/monkeypatch.py+++ b/auto_round/algorithms/transforms/spinquant/monkeypatch.py@@ -113,7 +113,7 @@ class QKRotationWrapper(nn.Module):since R is orthogonal (R @ R.T = I)."""- def __init__(self, original_func: Callable[..., Any]):+ def __init__(self, original_func: Callable[..., Any]) -> None:super().__init__()self.original_func = original_funcself._had_K: torch.Tensor | None = Nonediff --git a/auto_round/algorithms/transforms/spinquant/preprocessor.py b/auto_round/algorithms/transforms/spinquant/preprocessor.pyindex 7c2944ea1..c816c3d98 100644--- a/auto_round/algorithms/transforms/spinquant/preprocessor.py+++ b/auto_round/algorithms/transforms/spinquant/preprocessor.py@@ -161,7 +161,7 @@ class TrainableRMSNorm(nn.Module):with rotation matrices to minimise quantisation error."""- def __init__(self, original_norm: nn.Module, trainable: bool = True):+ def __init__(self, original_norm: nn.Module, trainable: bool = True) -> None:super().__init__()self.original_norm = original_normself.trainable = trainable@@ -190,7 +190,7 @@ class SpinQuantPreprocessor:original but with weight distributions better suited for quantisation."""- def __init__(self, model: nn.Module, config: Optional[SpinQuantConfig] = None):+ def __init__(self, model: nn.Module, config: Optional[SpinQuantConfig] = None) -> None:self.model = modelself.config = config or SpinQuantConfig()diff --git a/auto_round/algorithms/transforms/spinquant/rotation_utils.py b/auto_round/algorithms/transforms/spinquant/rotation_utils.pyindex aff9b2395..857cc45bc 100644--- a/auto_round/algorithms/transforms/spinquant/rotation_utils.py+++ b/auto_round/algorithms/transforms/spinquant/rotation_utils.py@@ -154,7 +154,7 @@ def matmul_hadU(X: torch.Tensor, hadamard_K: Optional[torch.Tensor] = None, K: O# the fallback implementations below are used.# ---------------------------------------------------------------------------try:- from auto_round.algorithms.transforms.rotation.utils.matrix import apply_transform_weight+ from auto_round.algorithms.transforms.quarot.utils.matrix import apply_transform_weightexcept ImportError:# Fallback for standalone usage.def apply_transform_weight(@@ -237,7 +237,7 @@ class InputRotationWrapperHadamard(nn.Module):rotation_size: int,hadamard_K: Optional[torch.Tensor] = None,K: Optional[int] = None,- ):+ ) -> None:super().__init__()if not isinstance(original_module, nn.Linear):@@ -462,7 +462,7 @@ def fuse_rmsnorm_in_model(model: nn.Module) -> None:"""# Attempt to use AutoRound's model_config layer discovery if available.try:- from auto_round.algorithms.transforms.rotation.inplace.model_config import get_scaling_layers+ from auto_round.algorithms.transforms.quarot.inplace.model_config import get_scaling_layerslayer_paths = get_scaling_layers(model.config.model_type if hasattr(model, "config") else "")if layer_paths:@@ -756,7 +756,7 @@ def get_model_arch_info(model: nn.Module) -> dict:def get_attention_layers(model: nn.Module):"""Yield attention modules using model_config if available, else fall back."""try:- from auto_round.algorithms.transforms.rotation.inplace.model_config import get_attention_layers as _get+ from auto_round.algorithms.transforms.quarot.inplace.model_config import get_attention_layers as _getreturn _get(model)except ImportError:diff --git a/auto_round/algorithms/transforms/spinquant/training.py b/auto_round/algorithms/transforms/spinquant/training.pyindex e3e1e2b64..618bfa7b1 100644--- a/auto_round/algorithms/transforms/spinquant/training.py+++ b/auto_round/algorithms/transforms/spinquant/training.py@@ -327,9 +327,9 @@ class SpinQuantTrainingHook:def __init__(self,model: nn.Module,- config=None,+ config: Any = None,enabled: bool = True,- ):+ ) -> None:from auto_round.algorithms.transforms.spinquant.preprocessor import (SpinQuantConfig,SpinQuantPreprocessor,@@ -365,7 +365,7 @@ class OrthogonalTrainingCallback:maximum deviation after each training step."""- def __init__(self, model: nn.Module, log_interval: int = 50):+ def __init__(self, model: nn.Module, log_interval: int = 50) -> None:self.model = modelself.log_interval = log_intervalself.step = 0@@ -407,7 +407,7 @@ class SpinQuantState:SpinQuant-specific metrics."""- def __init__(self):+ def __init__(self) -> None:self.enabled = Falseself.iteration = 0self.max_iterations = 0@@ -529,7 +529,7 @@ class OrthogonalityMonitor(RotationTrainerCallback):Prints a warning when the deviation exceeds a threshold."""- def __init__(self, threshold: float = 1e-3, log_interval: int = 50):+ def __init__(self, threshold: float = 1e-3, log_interval: int = 50) -> None:self.threshold = thresholdself.log_interval = log_interval@@ -561,7 +561,7 @@ class OrthogonalityMonitor(RotationTrainerCallback):class LossLogger(RotationTrainerCallback):"""Callback that prints loss every ``log_interval`` steps."""- def __init__(self, log_interval: int = 50):+ def __init__(self, log_interval: int = 50) -> None:self.log_interval = log_intervaldef on_step_end(self, args, state, control):@@ -607,7 +607,7 @@ class RotationTrainer:config: Optional[RotationTrainerConfig] = None,callbacks: Optional[list[RotationTrainerCallback]] = None,compute_loss_fn: Optional[Callable[[torch.Tensor, torch.Tensor, RotationTrainerConfig], torch.Tensor]] = None,- ):+ ) -> None:from auto_round.algorithms.transforms.spinquant.preprocessor import (SpinQuantConfig,SpinQuantPreprocessor,diff --git a/auto_round/autoround.py b/auto_round/autoround.pyindex bd3f52874..f26d1aa44 100644--- a/auto_round/autoround.py+++ b/auto_round/autoround.py@@ -19,11 +19,82 @@ import torchfrom auto_round.logger import deprecated, loggerfrom auto_round.schemes import QuantizationScheme+from auto_round.utils.device_manager import normalize_default_device_mapif TYPE_CHECKING:from auto_round.auto_scheme.gen_auto_scheme import AutoSchemefrom auto_round.compressors.base import BaseCompressor- from auto_round.compressors.config import ExtraConfig+++_COMPAT_KWARGS = {+ "format",+ "bits",+ "group_size",+ "sym",+ "data_type",+ "act_bits",+ "act_group_size",+ "act_sym",+ "act_data_type",+ "act_dynamic",+ "super_bits",+ "super_group_size",+ "scale_dtype",+ "ignore_layers",+ "quant_lm_head",+ "to_quant_block_names",+ "model_free",+ "disable_model_free",+ "model_dtype",+ "trust_remote_code",+ "amp",+ "nblocks",+ "lr",+ "minmax_lr",+ "enable_minmax_tuning",+ "enable_norm_bias_tuning",+ "enable_quanted_input",+ "enable_opt_rtn",+ "disable_deterministic_algorithms",+ "enable_deterministic_algorithms",+ "static_kv_dtype",+ "static_attention_dtype",+ "rotation_config",+ "processor",+ "image_processor",+ "template",+ "extra_data_dir",+ "quant_nontext_module",+ "guidance_scale",+ "num_inference_steps",+ "generator_seed",+ "duo_scaling",+ "n_grid",+ "mappings",+ "algorithm",+ "optimizer",+ "lr_scheduler",+ "not_use_best_mse",+ "dynamic_max_gap",+ "momentum",+ "device",+}+++def _filter_supported_compat_kwargs(kwargs: dict) -> dict:+ supported = {}+ unknown = []+ for key, value in kwargs.items():+ if key in _COMPAT_KWARGS:+ supported[key] = value+ else:+ unknown.append(key)+ if unknown:+ logger.warning_once(+ "AutoRound compatibility path received unsupported kwargs %s. They will be ignored.",+ ", ".join(sorted(unknown)),+ )+ return supportedclass AutoRound:@@ -45,7 +116,7 @@ class AutoRound:enable_torch_compile (bool): Whether to enable torch.compile for quant blocks/layers."""- SKIP_ARGS = ("local_args", "kwargs", "cls", "model_cls", "dynamic_compressor", "extra_config")+ SKIP_ARGS = ("local_args", "kwargs", "cls", "model_cls", "dynamic_compressor", "alg_configs")bits: int | Nonegroup_size: int | tuple | None@@ -65,7 +136,7 @@ class AutoRound:model: Union[torch.nn.Module, str],tokenizer=None,platform: str = "hf",- scheme: Union[str, dict, QuantizationScheme, AutoScheme] = "W4A16",+ scheme: Union[str, dict, QuantizationScheme, "AutoScheme"] = "W4A16",layer_config: dict[str, Union[str, dict, QuantizationScheme]] = None,dataset: Union[str, list, tuple, torch.utils.data.DataLoader] = "NeelNanda/pile-10k",iters: int = 200,@@ -78,10 +149,10 @@ class AutoRound:enable_torch_compile: bool = False,seed: int = 42,enable_adam: bool = False,- extra_config: "ExtraConfig" = None,enable_alg_ext: bool = False,disable_opt_rtn: bool | None = None,low_cpu_mem_usage: bool = True,+ alg_configs=None,**kwargs,) -> "BaseCompressor":"""Initialize AutoRound with quantization and tuning configuration.@@ -103,7 +174,6 @@ class AutoRound:enable_torch_compile (bool, optional): Enable torch.compile for low cost in quantization. Defaults to False.seed (int, optional): Random seed. Defaults to 42.enable_adam (bool, optional): Enable Adam-based optimizer. Defaults to False.- extra_config(ExtraConfig, optional): Extra configuration for lots of configurations. Defaults to None.enable_alg_ext (bool, optional): Enable algorithm extension (primarily for INT2)for better accuracy. Defaults to False.disable_opt_rtn (bool, optional): Disable RTN-mode optimization (iters=0) for fast quatnziation@@ -149,28 +219,64 @@ class AutoRound:... # ...... }"""- if torch.mps.is_available() and (- device_map == 0 or device_map == "0" or device_map is None or device_map == "auto"- ):- logger.warning(- "MPS detected. Using CPU by default to avoid potential memory issues. "- "Set --device_map=mps to force MPS usage."+ device_map = normalize_default_device_map(device_map)++ # Short-circuit: if alg_configs is provided, bypass AutoRoundCompatible and go directly+ # to the new-arch entry point to avoid duplicate keyword argument errors.+ if alg_configs is not None:+ from auto_round.compressors.entry import AutoRound as _NewAutoRound+ from auto_round.compressors.entry import filter_supported_entry_kwargs++ entry_kwargs = filter_supported_entry_kwargs(kwargs, context="AutoRound")++ return _NewAutoRound(+ model,+ scheme,+ alg_configs,+ tokenizer=tokenizer,+ platform=platform,+ format=entry_kwargs.pop("format", None),+ low_gpu_mem_usage=low_gpu_mem_usage,+ device_map=device_map,+ iters=iters,+ gradient_accumulate_steps=gradient_accumulate_steps,+ enable_torch_compile=enable_torch_compile,+ seed=seed,+ low_cpu_mem_usage=low_cpu_mem_usage,+ layer_config=layer_config,+ nsamples=nsamples,+ seqlen=seqlen,+ **entry_kwargs,)- device_map = "cpu"-- local_args = {k: v for k, v in locals().items() if k not in cls.SKIP_ARGS}- if extra_config is not None:- for key, value in extra_config.to_dict().items():- if value is None:- continue- if key in local_args:- local_args[key] = value- else:- kwargs[key] = value++ compat_kwargs = _filter_supported_compat_kwargs(kwargs)+ compat_kwargs.update(+ enable_adam=enable_adam,+ enable_alg_ext=enable_alg_ext,+ disable_opt_rtn=disable_opt_rtn,+ )from auto_round.compressors.entry import AutoRoundCompatible- return AutoRoundCompatible(**local_args, **kwargs)+ return AutoRoundCompatible(+ model=model,+ tokenizer=tokenizer,+ platform=platform,+ scheme=scheme,+ layer_config=layer_config,+ dataset=dataset,+ iters=iters,+ seqlen=seqlen,+ nsamples=nsamples,+ batch_size=batch_size,+ gradient_accumulate_steps=gradient_accumulate_steps,+ low_gpu_mem_usage=low_gpu_mem_usage,+ device_map=device_map,+ enable_torch_compile=enable_torch_compile,+ seed=seed,+ low_cpu_mem_usage=low_cpu_mem_usage,+ **compat_kwargs,+ )@classmethod@torch.no_grad()diff --git a/auto_round/calibration/state.py b/auto_round/calibration/state.pyindex f4fdfc9f7..49c10e512 100644--- a/auto_round/calibration/state.py+++ b/auto_round/calibration/state.py@@ -15,7 +15,7 @@This dataclass owns every per-run calibration field shared between:class:`~auto_round.compressors.base.BaseCompressor` and-:class:`~auto_round.algorithms.quantization.base.BaseQuantizers`:+:class:`~auto_round.algorithms.quantization.base.BaseQuantizer`:- Cache state ``(inputs, to_cached_layers, last_cache_name, blocks_requiring_input_ids)``- Per-batch shape state ``(attention_mask, batch_dim)``diff --git a/auto_round/cli/__init__.py b/auto_round/cli/__init__.pynew file mode 100644index 000000000..f52f12468--- /dev/null+++ b/auto_round/cli/__init__.py@@ -0,0 +1,12 @@+# # Copyright (C) 2026 Intel Corporation+# # SPDX-License-Identifier: Apache-2.0++"""CLI package for AutoRound.++Modules:+ main.py - command routing, RECIPES, tune(), eval entry points+ parser.py - argparse parser construction (all explicit flags here)+ algorithms.py - per-algorithm flag registration and config building+"""++from auto_round.cli.main import run, run_best, run_light, run_rtn, run_opt_rtn, run_mllm, run_eval # noqa: F401diff --git a/auto_round/cli/algorithms.py b/auto_round/cli/algorithms.pynew file mode 100644index 000000000..d81f21d44--- /dev/null+++ b/auto_round/cli/algorithms.py@@ -0,0 +1,424 @@+# # Copyright (C) 2026 Intel Corporation+# # SPDX-License-Identifier: Apache-2.0++"""Algorithm discovery, registration, listing, and config building for the CLI.++To add a new algorithm:+1. Write a class that extends AlgorithmHandler and set name/aliases/summary.+2. Implement register(group) to declare argparse flags.+3. Implement build(args, common_kwargs) to return a config object.++No manual registry update needed — subclasses are auto-registered on definition.+"""++from __future__ import annotations++import argparse+from abc import ABC, abstractmethod+from typing import Any, ClassVar++from auto_round.algorithms.registry import (+ get_algorithm_entry,+ iter_algorithm_entries,+ register_algorithm,+ resolve_algorithm_alias,+)++# ============================================================================+# Base class + registry+# ============================================================================+++class AlgorithmHandler(ABC):+ """Bundles everything the CLI needs to know about one algorithm.++ Concrete subclasses are auto-registered in the class-level registry+ the moment their class body is processed.+ """++ name: str # canonical name used in --algorithm+ aliases: tuple[str, ...] = () # all accepted names, including canonical+ summary: str = "" # one-liner shown by `auto_round list alg`+ config_factory: ClassVar[type | None] = None++ def __init_subclass__(cls, **kwargs) -> None:+ super().__init_subclass__(**kwargs)+ # A class that overrides both register and build is considered a concrete+ # algorithm implementation and must also declare a `name` class attribute.+ if "register" in cls.__dict__ and "build" in cls.__dict__:+ if "name" not in cls.__dict__:+ raise TypeError(f"{cls.__name__} must define a 'name' class attribute " f"(e.g. name = 'my_algo').")+ register_algorithm(+ cls.name,+ aliases=cls.aliases,+ config_factory=cls.config_factory,+ cli_handler=cls,+ summary=cls.summary,+ )++ # ------------------------------------------------------------------+ # Abstract interface — implement in each subclass+ # ------------------------------------------------------------------++ @abstractmethod+ def register(self, group) -> None:+ """Add argparse arguments to *group*."""++ @abstractmethod+ def build(self, args, common_kwargs: dict[str, Any]) -> Any:+ """Build and return an algorithm config from parsed *args*."""++ # ------------------------------------------------------------------+ # Registry operations — called on the class, not on instances+ # ------------------------------------------------------------------++ @classmethod+ def get(cls, name: str) -> AlgorithmHandler:+ """Return the handler for a canonical algorithm name. Raises KeyError if unknown."""+ entry = get_algorithm_entry(name)+ if entry.cli_handler is None:+ raise KeyError(f"No handler registered for algorithm '{name}'.")+ return entry.cli_handler()++ @classmethod+ def resolve_alias(cls, user_name: str) -> str | None:+ """Resolve a user-supplied algorithm name or alias to the canonical name.++ Returns None instead of raising so callers can silently skip unknowns.+ """+ return resolve_algorithm_alias(user_name)++ @classmethod+ def add_groups(cls, parser) -> None:+ """Add an argparse argument group for every registered algorithm."""+ for entry in iter_algorithm_entries():+ if entry.cli_handler is None:+ continue+ handler = entry.cli_handler()+ group = parser.add_argument_group(f"Algorithm: {handler.name}")+ handler.register(group)++ @classmethod+ def build_configs(cls, args, common_kwargs: dict[str, Any]) -> list:+ """Build the ordered algorithm config list from parsed CLI args."""+ raw = getattr(args, "algorithm", None) or ""+ names = [n.strip().lower() for n in raw.split(",") if n.strip()]++ # Infer hadamard when --rotation_type is given+ if getattr(args, "rotation_hadamard_type", None) and "hadamard" not in names:+ names.append("hadamard")++ # Resolve aliases, drop unknowns, deduplicate preserving order+ seen: set[str] = set()+ canonical: list[str] = []+ for n in names:+ c = cls.resolve_alias(n)+ if c and c not in seen:+ canonical.append(c)+ seen.add(c)++ # Default quantization algorithm if none was specified+ if not ({"awq", "rtn", "auto_round"} & seen):+ canonical.append("rtn" if getattr(args, "iters", 0) == 0 else "auto_round")++ return [cls.get(name).build(args, common_kwargs) for name in canonical]++ @classmethod+ def format_listing(cls) -> str:+ """Render the short `list alg` output."""+ lines = []+ for entry in iter_algorithm_entries():+ if entry.cli_handler is None:+ continue+ handler = entry.cli_handler()+ other = [a for a in handler.aliases if a != handler.name]+ alias_str = f" (aliases: {', '.join(other)})" if other else ""+ lines.append(f"- {handler.name}{alias_str}: {handler.summary}")+ return "\n".join(lines)++ @classmethod+ def format_detail(cls, name: str) -> str:+ """Render detailed help text for one algorithm."""+ canon = cls.resolve_alias(name)+ if canon is None:+ supported = [entry.name for entry in iter_algorithm_entries() if entry.cli_handler is not None]+ raise ValueError(f"Unknown algorithm '{name}'. " f"Supported: {', '.join(supported)}.")+ handler = cls.get(canon)+ lines = [f"{handler.name}: {handler.summary}"]+ other = [a for a in handler.aliases if a != handler.name]+ if other:+ lines.append(f"Aliases: {', '.join(other)}")+ temp = argparse.ArgumentParser(add_help=False)+ group = temp.add_argument_group(f"Flags for {handler.name}")+ handler.register(group)+ for action in group._group_actions:+ flags = ", ".join(action.option_strings)+ default = f" (default: {action.default})" if action.default is not None else ""+ lines.append(f" {flags}: {action.help or ''}{default}")+ return "\n".join(lines)+++# ============================================================================+# Helpers+# ============================================================================+++def _parse_bool_or_mode(value: str) -> bool | str:+ """Parse AWQ duo_scaling's tri-state: true / false / both."""+ lowered = value.strip().lower()+ if lowered == "true":+ return True+ if lowered == "false":+ return False+ if lowered == "both":+ return "both"+ raise argparse.ArgumentTypeError("Expected one of: true, false, both")+++# ============================================================================+# Algorithm implementations (auto-registered via __init_subclass__)+# ============================================================================+++class AWQ(AlgorithmHandler):+ name = "awq"+ aliases = ("awq",)+ summary = "Activation-Aware Weight Quantization (pre-processing)."+ config_factory = None++ def register(self, group) -> None:+ group.add_argument(+ "--duo-scaling",+ dest="duo_scaling",+ default=True,+ type=_parse_bool_or_mode,+ metavar="{true,false,both}",+ help="Use activation+weight duo scaling (true/false/both).",+ )+ group.add_argument(+ "--n-grid",+ dest="n_grid",+ default=20,+ type=int,+ help="Number of grid-search points for AWQ scaling ratio.",+ )++ def build(self, args, common_kwargs: dict[str, Any]):+ from auto_round.algorithms.transforms.awq.config import AWQConfig++ return AWQConfig(+ duo_scaling=getattr(args, "duo_scaling", True),+ n_grid=getattr(args, "n_grid", 20),+ **common_kwargs,+ )+++class RTN(AlgorithmHandler):+ name = "rtn"+ aliases = ("rtn",)+ summary = "Round-To-Nearest quantization."+ config_factory = None++ def register(self, group) -> None:+ mutex = group.add_mutually_exclusive_group()+ mutex.add_argument(+ "--disable_opt_rtn",+ dest="disable_opt_rtn",+ default=None,+ action="store_const",+ const=True,+ help="Force plain RTN (disable optimized path).",+ )+ mutex.add_argument(+ "--enable_opt_rtn",+ dest="disable_opt_rtn",+ action="store_const",+ const=False,+ help="Force optimized RTN path.",+ )++ def build(self, args, common_kwargs: dict[str, Any]):+ from auto_round.algorithms.quantization.rtn.config import RTNConfig++ return RTNConfig(+ disable_opt_rtn=getattr(args, "disable_opt_rtn", None),+ **common_kwargs,+ )+++class AutoRound(AlgorithmHandler):+ name = "auto_round"+ aliases = ("auto_round", "autoround", "sign_round", "signround")+ summary = "SignRound-style iterative block quantization."+ config_factory = None++ def register(self, group) -> None:+ group.add_argument(+ "--iters", "--iter", default=None, type=int, help="Number of optimization iterations per block."+ )+ group.add_argument("--lr", default=None, type=float, help="Learning rate for rounding optimization.")+ group.add_argument("--minmax_lr", default=None, type=float, help="Learning rate for min-max tuning.")+ group.add_argument("--momentum", default=0.0, type=float, help="Momentum factor for the optimizer.")+ group.add_argument("--nblocks", default=1, type=int, help="Number of blocks to optimize together.")+ minmax_mutex = group.add_mutually_exclusive_group()+ minmax_mutex.add_argument(+ "--enable_minmax_tuning",+ default=True,+ dest="enable_minmax_tuning",+ action="store_true",+ help="Tune weight min/max ranges.",+ )+ minmax_mutex.add_argument(+ "--no-enable_minmax_tuning",+ "--disable_minmax_tuning",+ dest="enable_minmax_tuning",+ action="store_false",+ help="Disable weight min/max tuning.",+ )+ group.add_argument(+ "--enable_norm_bias_tuning",+ default=False,+ action=argparse.BooleanOptionalAction,+ help="Tune normalization and bias terms.",+ )+ group.add_argument(+ "--gradient_accumulate_steps", default=1, type=int, help="Gradient accumulation steps per update."+ )+ group.add_argument(+ "--enable_alg_ext",+ default=False,+ action=argparse.BooleanOptionalAction,+ help="Enable experimental SignRound extension.",+ )+ group.add_argument(+ "--not_use_best_mse",+ default=False,+ action=argparse.BooleanOptionalAction,+ help="Skip restoring best-MSE checkpoint.",+ )+ quanted_input_mutex = group.add_mutually_exclusive_group()+ quanted_input_mutex.add_argument(+ "--enable_quanted_input",+ default=True,+ dest="enable_quanted_input",+ action="store_true",+ help="Consume quantized output of previous blocks.",+ )+ quanted_input_mutex.add_argument(+ "--no-enable_quanted_input",+ "--disable_quanted_input",+ dest="enable_quanted_input",+ action="store_false",+ help="Disable quantized-input propagation across blocks.",+ )+ group.add_argument(+ "--enable_adam",+ default=False,+ action=argparse.BooleanOptionalAction,+ help="Use the Adam-based SignRound variant.",+ )++ def build(self, args, common_kwargs: dict[str, Any]):+ from auto_round.algorithms.quantization.sign_round.config import SignRoundConfig++ return SignRoundConfig(+ iters=getattr(args, "iters", 200),+ lr=getattr(args, "lr", None),+ minmax_lr=getattr(args, "minmax_lr", None),+ momentum=getattr(args, "momentum", 0.0),+ nblocks=getattr(args, "nblocks", 1),+ enable_minmax_tuning=getattr(args, "enable_minmax_tuning", True),+ enable_norm_bias_tuning=getattr(args, "enable_norm_bias_tuning", False),+ gradient_accumulate_steps=getattr(args, "gradient_accumulate_steps", 1),+ enable_alg_ext=getattr(args, "enable_alg_ext", False),+ not_use_best_mse=getattr(args, "not_use_best_mse", False),+ enable_quanted_input=getattr(args, "enable_quanted_input", True),+ enable_adam=getattr(args, "enable_adam", False),+ **common_kwargs,+ )+++class Hadamard(AlgorithmHandler):+ name = "hadamard"+ aliases = ("hadamard", "random_hadamard", "quarot_hadamard")+ summary = "Hadamard rotation/transform applied before quantization."+ config_factory = None++ def register(self, group) -> None:+ group.add_argument(+ "--rotation_type",+ "--rotation-hadamard-type",+ dest="rotation_hadamard_type",+ default=None,+ choices=["hadamard", "random_hadamard", "quarot_hadamard"],+ help="Hadamard transform variant.",+ )+ group.add_argument(+ "--rotation_backend",+ dest="rotation_backend",+ default="auto",+ choices=["auto", "inplace", "transform"],+ help="Rotation backend to use.",+ )+ group.add_argument(+ "--rotation_block_size",+ dest="rotation_block_size",+ default=None,+ type=int,+ help="Grouped Hadamard block size.",+ )+ group.add_argument(+ "--fuse_online_to_weight",+ default=None,+ action=argparse.BooleanOptionalAction,+ help="Fuse online Hadamard rotation into weights.",+ )+ group.add_argument(+ "--allow_online_rotation",+ default=True,+ action=argparse.BooleanOptionalAction,+ help="Allow online activation rotation.",+ )++ def build(self, args, common_kwargs: dict[str, Any]):+ from auto_round.algorithms.transforms.quarot.config import RotationConfig++ hadamard_type = getattr(args, "rotation_hadamard_type", None) or "hadamard"+ return RotationConfig(+ hadamard_type=hadamard_type,+ backend=getattr(args, "rotation_backend", "auto"),+ block_size=getattr(args, "rotation_block_size", None),+ fuse_online_to_weight=getattr(args, "fuse_online_to_weight", None),+ allow_online_rotation=getattr(args, "allow_online_rotation", True),+ )+++def _register_builtin_algorithm_factories() -> None:+ from auto_round.algorithms.quantization.rtn.config import RTNConfig+ from auto_round.algorithms.quantization.sign_round.config import SignRoundConfig+ from auto_round.algorithms.transforms.awq.config import AWQConfig+ from auto_round.algorithms.transforms.quarot.config import RotationConfig++ register_algorithm("rtn", aliases=("rtn",), config_factory=RTNConfig, cli_handler=RTN, summary=RTN.summary)+ register_algorithm(+ "auto_round",+ aliases=("auto_round", "autoround", "sign_round", "signround"),+ config_factory=SignRoundConfig,+ cli_handler=AutoRound,+ summary=AutoRound.summary,+ )+ register_algorithm("awq", aliases=("awq",), config_factory=AWQConfig, cli_handler=AWQ, summary=AWQ.summary)+ register_algorithm(+ "hadamard",+ aliases=("hadamard", "random_hadamard", "quarot_hadamard"),+ config_factory=RotationConfig,+ cli_handler=Hadamard,+ summary=Hadamard.summary,+ alias_factories={+ "random_hadamard": lambda: RotationConfig(hadamard_type="random_hadamard"),+ "quarot_hadamard": lambda: RotationConfig(hadamard_type="quarot_hadamard"),+ },+ )+++_register_builtin_algorithm_factories()diff --git a/auto_round/cli/main.py b/auto_round/cli/main.pynew file mode 100644index 000000000..82144efa9--- /dev/null+++ b/auto_round/cli/main.py@@ -0,0 +1,486 @@+# # Copyright (C) 2026 Intel Corporation+# # SPDX-License-Identifier: Apache-2.0++"""CLI entry points: command routing, RECIPES, tune, eval, list.++This module is the single place that wires together:+ parser.py - argparse construction (all flag declarations here)+ algorithms.py - algorithm config building++All console_scripts (auto_round, auto-round-best, etc.) point here.+"""++from __future__ import annotations++import argparse+import difflib+import sys++from auto_round.cli.algorithms import AlgorithmHandler+from auto_round.cli.parser import (+ add_common_quantization_arguments,+ build_eval_parser,+ build_list_parser,+ build_quantize_parser,+ build_root_parser,+)+++def _extract_common_quantization_kwargs(args) -> dict:+ """Map parsed CLI args back to QuantizationConfig constructor kwargs.++ Handles inverted flags: --asym -> sym, --act_asym -> act_sym,+ --disable_act_dynamic -> act_dynamic.+ When the flag was not set (False), the value is None (defer to scheme).+ """+ return {+ "bits": args.bits,+ "group_size": args.group_size,+ "sym": None if not args.asym else False,+ "data_type": args.data_type,+ "act_bits": args.act_bits,+ "act_group_size": args.act_group_size,+ "act_sym": None if not args.act_asym else False,+ "act_data_type": args.act_data_type,+ "act_dynamic": None if not args.disable_act_dynamic else False,+ "super_bits": args.super_bits,+ "super_group_size": args.super_group_size,+ }+++def _build_entry_base_kwargs(args, *, low_cpu_mem_usage, enable_torch_compile, layer_config) -> dict:+ return {+ "platform": args.platform,+ "format": args.format,+ "dataset": args.dataset,+ "seqlen": args.seqlen,+ "nsamples": args.nsamples,+ "batch_size": args.batch_size,+ "low_gpu_mem_usage": args.low_gpu_mem_usage,+ "low_cpu_mem_usage": low_cpu_mem_usage,+ "device_map": args.device_map,+ "enable_torch_compile": enable_torch_compile,+ "seed": args.seed,+ "layer_config": layer_config,+ "model_dtype": args.model_dtype,+ "trust_remote_code": not args.disable_trust_remote_code,+ }+++def _build_entry_route_kwargs(args) -> dict:+ return {+ "model_free": args.model_free,+ "disable_model_free": args.disable_model_free,+ }+++def _build_entry_compressor_kwargs(args) -> dict:+ return {+ "scale_dtype": args.scale_dtype,+ "ignore_layers": args.ignore_layers,+ "quant_lm_head": args.quant_lm_head,+ "to_quant_block_names": args.to_quant_block_names,+ }+++def _build_entry_model_type_kwargs(args) -> dict:+ return {+ "quant_nontext_module": args.quant_nontext_module,+ "extra_data_dir": args.extra_data_dir,+ "template": args.template,+ "guidance_scale": args.guidance_scale,+ "num_inference_steps": args.num_inference_steps,+ "generator_seed": args.generator_seed,+ }+++def _to_autoround_kwargs(args, *, low_cpu_mem_usage, enable_torch_compile, layer_config) -> dict:+ """Collect only the kwargs accepted by the new AutoRound entry API."""+ kwargs = _build_entry_base_kwargs(+ args,+ low_cpu_mem_usage=low_cpu_mem_usage,+ enable_torch_compile=enable_torch_compile,+ layer_config=layer_config,+ )+ kwargs.update(_build_entry_route_kwargs(args))+ kwargs.update(_build_entry_compressor_kwargs(args))+ kwargs.update(_build_entry_model_type_kwargs(args))+ return kwargs+++RECIPES = {+ "default": {+ "batch_size": 8,+ "iters": 200,+ "seqlen": 2048,+ "nsamples": 128,+ "lr": None,+ },+ "best": {+ "batch_size": 8,+ "iters": 1000,+ "seqlen": 2048,+ "nsamples": 512,+ "lr": None,+ },+ "light": {+ "batch_size": 8,+ "iters": 50,+ "seqlen": 2048,+ "nsamples": 128,+ "lr": 5e-3,+ },+ "rtn": {"batch_size": 8, "iters": 0, "seqlen": 2048, "nsamples": 1, "lr": None, "disable_opt_rtn": True},+ "opt_rtn": {"batch_size": 8, "iters": 0, "seqlen": 2048, "nsamples": 128, "lr": None, "disable_opt_rtn": False},+}++# ============================================================================+# list subcommand+# ============================================================================+++def list_item(argv=None):+ args = build_list_parser().parse_args(argv)+ if args.item in {"format", "formats"}:+ from auto_round.formats import OutputFormat++ print("AutoRound supported output formats and quantization scheme:")+ print(OutputFormat.get_support_matrix())+ elif args.item in {"alg", "algs", "algorithm", "algorithms"}:+ if args.name:+ print(AlgorithmHandler.format_detail(args.name))+ else:+ print("AutoRound supported algorithms:")+ print(AlgorithmHandler.format_listing())+ print("\nUse `auto_round list alg <name>` or `auto_round --algorithm <name> --help` for details.")+ else:+ raise ValueError(f"Unsupported list target: {args.item}")+++# ============================================================================+# quantize subcommand+# ============================================================================+++def _print_algorithm_help(argv: list[str]) -> bool:+ """If --algorithm <name> --help is present, print algorithm-focused help and return True."""+ if not any(flag in argv for flag in ("-h", "--help")):+ return False++ # Pre-parse just --algorithm++ pre = argparse.ArgumentParser(add_help=False)+ pre.add_argument("--algorithm", default=None)+ known, _ = pre.parse_known_args(argv)+ names = [n.strip().lower() for n in (known.algorithm or "").split(",") if n.strip()]+ if not names:+ return False++ # Resolve aliases to canonical names, silently ignore unknowns+ canonical_names: list[str] = []+ unknown_names: list[str] = []+ for name in names:+ canon = AlgorithmHandler.resolve_alias(name)+ if canon and canon not in canonical_names:+ canonical_names.append(canon)+ continue+ if canon is None:+ unknown_names.append(name)+ if unknown_names:+ from auto_round.algorithms.registry import list_registered_algorithms++ suggestions = []+ supported = list_registered_algorithms()+ for name in unknown_names:+ match = difflib.get_close_matches(name, supported, n=1)+ if match:+ suggestions.append(f"Unknown algorithm '{name}'. Did you mean '{match[0]}'?")+ else:+ suggestions.append(f"Unknown algorithm '{name}'.")+ raise SystemExit(" ".join(suggestions) + " Use `auto_round list alg` to see supported algorithms.")+ if not canonical_names:+ return False++ mini = argparse.ArgumentParser(+ prog=f"auto_round --algorithm {','.join(canonical_names)}",+ description=f"Flags for algorithm(s): {', '.join(canonical_names)}. "+ "Use `auto_round --help` for the full argument list.",+ formatter_class=argparse.ArgumentDefaultsHelpFormatter,+ )+ quant_group = mini.add_argument_group("Common Quantization Arguments")+ add_common_quantization_arguments(quant_group)+ for name in canonical_names:+ alg_group = mini.add_argument_group(f"Algorithm: {name}")+ AlgorithmHandler.get(name).register(alg_group)+ mini.print_help()+ return True+++def start(recipe="default", argv=None):+ recipe_defaults = RECIPES[recipe]+ argv = list(sys.argv[1:] if argv is None else argv)++ if _print_algorithm_help(argv):+ return++ parser = build_quantize_parser(prog="auto_round quantize")+ args = parser.parse_args(argv)++ # Apply recipe defaults for fields the user didn't set+ for key, value in recipe_defaults.items():+ if getattr(args, key, None) is None:+ setattr(args, key, value)++ tune(args)+++def tune(args):+ assert args.model or args.model_name, "[model] or --model MODEL_NAME should be set."+ if args.model is None:+ args.model = args.model_name+ if args.eval_bs is None:+ args.eval_bs = "auto"++ from transformers.utils.versions import require_version++ if args.tasks is not None:+ require_version(+ "lm_eval>=0.4.2",+ "lm-eval is required for evaluation, please install it with `pip install 'lm-eval>=0.4.2'`",+ )++ from auto_round.utils import logger++ if args.low_cpu_mem_usage:+ logger.warning(+ "`low_cpu_mem_usage` is deprecated and is now enabled by default. "+ "To disable it, use `--disable_low_cpu_mem_usage`."+ )++ if args.format is None:+ args.format = "auto_round"++ formats = args.format.lower().replace(" ", "").split(",")+ from auto_round.utils import SUPPORTED_FORMATS++ for fmt in formats:+ if fmt not in SUPPORTED_FORMATS:+ raise ValueError(f"{fmt} is not supported, we only support {SUPPORTED_FORMATS}")++ if "auto_gptq" in args.format and args.asym is True:+ logger.warning(+ "the auto_gptq kernel has issues with asymmetric quantization. "+ "It is recommended to use sym quantization or --format='auto_round'"+ )++ if "marlin" in args.format and args.asym is True:+ raise RuntimeError("marlin backend only supports sym quantization, please remove --asym")++ from auto_round.utils import get_device_and_parallelism++ device_str, use_auto_mapping = get_device_and_parallelism(args.device_map)++ if args.enable_torch_compile:+ logger.info(+ "`torch.compile` is enabled to reduce tuning costs. "+ "If it causes issues, you can disable it by removing `--enable_torch_compile` argument."+ )++ model_name = args.model+ if model_name[-1] == "/":+ model_name = model_name[:-1]+ logger.info(f"start to quantize {model_name}")++ from auto_round.compressors.base import BaseCompressor+ from auto_round.compressors.entry import AutoRound as PipelineAutoRound++ if "bloom" in model_name:+ args.low_gpu_mem_usage = False++ if args.quant_lm_head:+ for fmt in formats:+ if "auto_round" not in fmt and "fake" not in fmt and "mlx" not in fmt:+ auto_round_formats = [s for s in SUPPORTED_FORMATS if s.startswith("auto_round") or s == "mlx"]+ raise ValueError(+ f"{fmt} is not supported for lm-head quantization, please change to {auto_round_formats}"+ )++ enable_torch_compile = True if "--enable_torch_compile" in sys.argv else False+ scheme = args.scheme.upper()++ from auto_round.schemes import PRESET_SCHEMES++ if scheme not in PRESET_SCHEMES:+ raise ValueError(f"{scheme} is not supported. only {PRESET_SCHEMES.keys()} are supported ")++ if args.disable_deterministic_algorithms:+ logger.warning(+ "default not use deterministic_algorithms. disable_deterministic_algorithms is deprecated,"+ " please use enable_deterministic_algorithms instead. "+ )++ from auto_round.utils import parse_layer_config_arg++ layer_config = {}+ if args.layer_config:+ layer_config = parse_layer_config_arg(args.layer_config)+ args.layer_config = layer_config++ low_cpu_mem_usage = True+ if args.disable_low_cpu_mem_usage:+ low_cpu_mem_usage = False++ from auto_round.auto_scheme import AutoScheme++ if args.avg_bits is not None:+ if args.options is None:+ raise ValueError("please set --options for auto scheme")+ if enable_torch_compile:+ logger.warning(+ "`enable_torch_compile=True` with AutoScheme may cause compile errors "+ "on some models. If so, try removing `--enable_torch_compile`."+ )+ scheme = AutoScheme(+ options=args.options,+ avg_bits=args.avg_bits,+ shared_layers=args.shared_layers,+ ignore_scale_zp_bits=args.ignore_scale_zp_bits,+ low_gpu_mem_usage=True,+ low_cpu_mem_usage=low_cpu_mem_usage,+ )++ common_kwargs = _extract_common_quantization_kwargs(args)+ alg_configs = AlgorithmHandler.build_configs(args, common_kwargs)++ from auto_round.utils import clear_memory++ autoround: BaseCompressor = PipelineAutoRound(+ model_name,+ scheme,+ alg_configs if len(alg_configs) > 1 else alg_configs[0],+ **_to_autoround_kwargs(+ args,+ low_cpu_mem_usage=low_cpu_mem_usage,+ enable_torch_compile=enable_torch_compile,+ layer_config=layer_config,+ ),+ )++ model, folders = autoround.quantize_and_save(args.output_dir, format=args.format) # pylint: disable=no-member+ tokenizer = autoround.tokenizer # pylint: disable=no-member+ clear_memory()++ from auto_round.eval.evaluation import run_model_evaluation++ run_model_evaluation(model, tokenizer, autoround, folders, formats, args)+++# ============================================================================+# eval subcommand+# ============================================================================+++def setup_eval_parser(argv=None):+ parser = build_eval_parser(prog="auto_round eval")+ return parser.parse_args(argv)+++def run_eval(argv=None):+ from auto_round.eval.eval_cli import eval, eval_task_by_task+ from auto_round.logger import logger+ from auto_round.utils import is_gguf_model, is_mllm_model++ args = setup_eval_parser(argv)+ assert args.model or args.model_name, "[model] or --model MODEL_NAME should be set."++ if args.model is None:+ args.model = args.model_name+ if "llama" in args.model.lower() and not args.add_bos_token:+ logger.warning("set add_bos_token=True for llama model.")+ args.add_bos_token = True+ if not is_gguf_model(args.model) and is_mllm_model(args.model):+ args.mllm = True++ if args.eval_task_by_task:+ eval_task_by_task(+ model=args.model,+ device=args.device_map,+ limit=args.limit,+ tasks=args.tasks,+ batch_size=args.eval_bs,+ trust_remote_code=not args.disable_trust_remote_code,+ eval_model_dtype=args.eval_model_dtype,+ add_bos_token=args.add_bos_token,+ )+ else:+ eval(args)+++# ============================================================================+# Command routing+# ============================================================================+++def _normalize_cli_invocation(argv):+ """Normalize legacy invocation styles to (command, rest_argv)."""+ if argv and argv[0] in {"quantize", "list", "eval", "help"}:+ return argv[0], argv[1:]+ if "--list" in argv:+ normalized = list(argv)+ normalized.remove("--list")+ return "list", normalized+ if "--eval" in argv:+ normalized = list(argv)+ normalized.remove("--eval")+ return "eval", normalized+ return "quantize", list(argv)+++def _print_help(topic=None):+ if topic == "quantize":+ build_quantize_parser(prog="auto_round quantize").print_help()+ return+ if topic == "list":+ build_list_parser(prog="auto_round list").print_help()+ return+ if topic == "eval":+ build_eval_parser(prog="auto_round eval").print_help()+ return+ build_root_parser().print_help()+++def run():+ argv = list(sys.argv[1:])+ command, command_argv = _normalize_cli_invocation(argv)++ if command == "help":+ root_args = build_root_parser().parse_args(argv)+ _print_help(root_args.topic)+ return+ if command == "list":+ list_item(command_argv)+ return+ if command == "eval":+ run_eval(command_argv)+ return+ start(argv=command_argv)+++def run_best():+ start("best")+++def run_light():+ start("light")+++def run_rtn():+ start("rtn")+++def run_opt_rtn():+ start("opt_rtn")+++def run_mllm():+ run()diff --git a/auto_round/cli/parser.py b/auto_round/cli/parser.pynew file mode 100644index 000000000..c823022ca--- /dev/null+++ b/auto_round/cli/parser.py@@ -0,0 +1,273 @@+# # Copyright (C) 2026 Intel Corporation+# # SPDX-License-Identifier: Apache-2.0++"""Argparse parser construction for all CLI subcommands.++All static flags are declared directly via add_argument() — no intermediate+dataclass wrappers. To add a new flag, find the right argument group below+and add a line.++Public API:+ build_quantize_parser() — the main quantize parser+ build_list_parser() — the `list` subcommand parser+ build_eval_parser() — the `eval` subcommand parser+ build_root_parser() — top-level router+"""++from __future__ import annotations++import argparse++from auto_round.cli.algorithms import AlgorithmHandler+from auto_round.eval.eval_cli import EvalArgumentParser+++def _parse_group_size(s: str):+ """Parse group_size: a plain int, or comma-separated ints for block-wise fp8."""+ if s.lstrip("-").isdigit():+ return int(s)+ return tuple(int(x.strip()) for x in s.split(","))+++def add_common_quantization_arguments(group) -> None:+ """Add common quantization flags to an argparse group.++ To add a new flag, append an add_argument() call here and mirror it in+ _extract_common_quantization_kwargs() in main.py.+ """+ group.add_argument("--scheme", default="W4A16", type=str, help="Quantization scheme preset, e.g. W4A16, W8A16.")+ group.add_argument("--bits", default=None, type=int, help="Weight quantization bit width.")+ group.add_argument(+ "--group_size",+ default=None,+ type=_parse_group_size,+ help="Weight group size: positive int, -1 per-channel, 0 per-tensor.",+ )+ group.add_argument(+ "--asym", default=False, action="store_true", help="Use asymmetric weight quantization instead of symmetric."+ )+ group.add_argument(+ "--data_type", "--dtype", default=None, type=str, help="Weight quantization data type, e.g. int, fp8."+ )+ group.add_argument("--act_bits", default=None, type=int, help="Activation quantization bit width.")+ group.add_argument("--act_group_size", default=None, type=int, help="Activation quantization group size.")+ group.add_argument(+ "--act_asym",+ default=False,+ action="store_true",+ help="Use asymmetric activation quantization instead of symmetric.",+ )+ group.add_argument(+ "--act_data_type", "--act_dtype", default=None, type=str, help="Activation quantization data type."+ )+ group.add_argument(+ "--disable_act_dynamic",+ default=False,+ action="store_true",+ help="Use static activation quantization instead of dynamic.",+ )+ group.add_argument("--super_bits", default=None, type=int, help="Bit width for double quantization metadata.")+ group.add_argument(+ "--super_group_size", default=None, type=int, help="Group size for double quantization metadata."+ )+ group.add_argument(+ "--scale_dtype",+ default=None,+ choices=["fp16", "float16", "bf16", "bfloat16", "fp32", "float32"],+ help="Data type used to store quantization scales.",+ )+ group.add_argument(+ "--ignore_layers",+ "--fp_layers",+ default="",+ type=str,+ help="Comma-separated layer names to keep in higher precision.",+ )+ group.add_argument(+ "--quant_lm_head", default=False, action=argparse.BooleanOptionalAction, help="Quantize the lm_head module."+ )+ group.add_argument(+ "--to_quant_block_names", default=None, type=str, help="Comma-separated subset of block names to quantize."+ )+++def build_quantize_parser(*, prog: str = "auto_round quantize") -> argparse.ArgumentParser:+ """Build the quantize parser with all argument groups."""+ parser = argparse.ArgumentParser(prog=prog)++ # ---- Model / Runtime ----+ rt = parser.add_argument_group("Runtime Arguments")+ rt.add_argument("model", default=None, nargs="?", help="Path to the pre-trained model or Hugging Face model id.")+ rt.add_argument(+ "--model_name",+ "--model",+ "--model_name_or_path",+ default="facebook/opt-125m",+ help="Path to the pre-trained model or Hugging Face model id.",+ )+ rt.add_argument("--model_dtype", default=None, help="Model dtype used when loading the model.")+ rt.add_argument("--platform", default="hf", help="Model loading platform. Options: hf or model_scope.")+ rt.add_argument(+ "--batch_size", "--train_bs", "--bs", default=None, type=int, help="Batch size for calibration and tuning."+ )+ rt.add_argument("--seqlen", "--seq_len", default=None, type=int, help="Sequence length of the calibration samples.")+ rt.add_argument("--nsamples", "--nsample", default=None, type=int, help="Number of calibration samples to use.")+ rt.add_argument(+ "--device_map", "--device", "--devices", default="0", type=str, help="Device mapping used for quantization."+ )+ rt.add_argument(+ "--dataset", default="NeelNanda/pile-10k", type=str, help="Calibration dataset or local dataset path."+ )+ rt.add_argument("--seed", default=42, type=int, help="Random seed for reproducibility.")+ rt.add_argument(+ "--format", "--formats", default="auto_round", type=str, help="Output format for the quantized model."+ )+ rt.add_argument(+ "--algorithm", default=None, type=str, help="Comma-separated algorithms such as 'awq' or 'awq,auto_round'."+ )+ rt.add_argument("--output_dir", default="./tmp_autoround", type=str, help="Directory to save quantized artifacts.")+ rt.add_argument("--avg_bits", "--target_bits", default=None, type=float, help="Average target bits for AutoScheme.")+ rt.add_argument("--options", default=None, type=str, help="AutoScheme options, for example 'W4A16,W8A16'.")+ rt.add_argument(+ "--low_gpu_mem_usage", action="store_true", help="Enable memory-efficient mode by offloading features to CPU."+ )+ rt.add_argument(+ "--low_cpu_mem_usage",+ action="store_true",+ help="Deprecated compatibility flag. Low CPU memory mode is enabled by default.",+ )+ rt.add_argument("--disable_low_cpu_mem_usage", action="store_true", help="Disable low CPU memory mode.")+ rt.add_argument("--enable_torch_compile", action="store_true", help="Enable torch.compile during quantization.")+ rt.add_argument(+ "--disable_trust_remote_code", action="store_true", help="Disable trust_remote_code when loading models."+ )+ rt.add_argument(+ "--layer_config", default=None, type=str, help="Per-layer quantization overrides encoded as JSON-like text."+ )+ rt.add_argument(+ "--shared_layers",+ type=str,+ nargs="+",+ action="append",+ default=None,+ help="Ensure listed layers share the same quantization data type.",+ )+ rt.add_argument(+ "--static_kv_dtype",+ default=None,+ type=str,+ choices=["fp8", "float8_e4m3fn"],+ help="Static KV-cache quantization data type.",+ )+ rt.add_argument(+ "--static_attention_dtype",+ default=None,+ type=str,+ choices=["fp8", "float8_e4m3fn"],+ help="Static attention quantization data type.",+ )++ # ---- Evaluation ----+ ev = parser.add_argument_group("Evaluation Arguments")+ ev.add_argument(+ "--tasks",+ "--task",+ nargs="?",+ const="lambada_openai,hellaswag,winogrande,piqa,mmlu,wikitext,"+ "truthfulqa_mc1,openbookqa,boolq,arc_easy,arc_challenge",+ default=None,+ help="LM-Eval tasks to run after quantization.",+ )+ ev.add_argument("--eval_bs", default=None, type=int, help="Batch size for evaluation.")+ ev.add_argument(+ "--limit", type=float, default=None, metavar="N|0<N<1", help="Evaluation example limit as a count or fraction."+ )+ ev.add_argument("--eval_task_by_task", action="store_true", help="Evaluate tasks sequentially instead of batching.")+ ev.add_argument(+ "--eval_backend", default="hf", type=str, choices=["hf", "vllm"], help="Backend to use for evaluation."+ )+ ev.add_argument("--vllm_args", default=None, type=str, help="Comma-separated custom vLLM arguments.")+ ev.add_argument(+ "--eval_model_dtype", default=None, type=str, help="Torch dtype used when loading the evaluation model."+ )+ ev.add_argument("--add_bos_token", action="store_true", help="Add a BOS token during evaluation.")++ # ---- Compatibility ----+ compat = parser.add_argument_group("Compatibility Arguments")+ compat.add_argument(+ "--ignore_scale_zp_bits",+ action="store_true",+ help="Ignore scale and zero-point overhead when computing AutoScheme target bits.",+ )+ compat.add_argument("--disable_amp", action="store_true", help="Disable AMP during tuning.")+ compat.add_argument(+ "--disable_deterministic_algorithms",+ action="store_true",+ help="Deprecated flag to disable deterministic algorithms.",+ )+ compat.add_argument(+ "--enable_deterministic_algorithms",+ action="store_true",+ help="Enable deterministic algorithms for reproducible runs.",+ )+ compat.add_argument("--model_free", action="store_true", help="Force model-free quantization mode.")+ compat.add_argument("--disable_model_free", action="store_true", help="Disable automatic model-free routing.")++ # ---- Multimodal ----+ mllm = parser.add_argument_group("Multimodal Arguments")+ mllm.add_argument("--mllm", action="store_true", help="Deprecated compatibility flag for multimodal mode.")+ mllm.add_argument(+ "--quant_nontext_module", action="store_true", help="Quantize non-text modules such as vision or audio towers."+ )+ mllm.add_argument(+ "--extra_data_dir", default=None, type=str, help="Directory containing multimodal calibration assets."+ )+ mllm.add_argument("--template", default=None, type=str, help="Custom template for dataset construction.")++ # ---- Diffusion ----+ diff = parser.add_argument_group("Diffusion Arguments")+ diff.add_argument("--prompt_file", default=None, type=str, help="File containing prompts for diffusion evaluation.")+ diff.add_argument("--prompt", default=None, type=str, help="Single prompt used for quick diffusion testing.")+ diff.add_argument("--metrics", "--metric", default="clip", help="Diffusion evaluation metrics.")+ diff.add_argument(+ "--image_save_dir", default="./tmp_image_save", type=str, help="Directory to save generated images."+ )+ diff.add_argument("--guidance_scale", default=7.5, type=float, help="Classifier-free guidance scale.")+ diff.add_argument(+ "--num_inference_steps", default=50, type=int, help="Number of denoising steps for diffusion evaluation."+ )+ diff.add_argument("--generator_seed", default=None, type=int, help="Random seed used for diffusion generation.")++ # ---- Common Quantization Arguments ----+ quant_group = parser.add_argument_group("Common Quantization Arguments")+ add_common_quantization_arguments(quant_group)++ # ---- Algorithm-specific groups ----+ AlgorithmHandler.add_groups(parser)++ return parser+++def build_list_parser(*, prog: str = "auto_round list") -> argparse.ArgumentParser:+ parser = argparse.ArgumentParser(prog=prog)+ parser.add_argument("item", type=str, help="item to list, e.g., format, alg")+ parser.add_argument("name", nargs="?", default=None, help="optional specific format/algorithm name")+ return parser+++def build_eval_parser(*, prog: str = "auto_round eval") -> argparse.ArgumentParser:+ return EvalArgumentParser(prog=prog)+++def build_root_parser() -> argparse.ArgumentParser:+ parser = argparse.ArgumentParser(+ prog="auto_round",+ description="AutoRound command line interface.",+ )+ subparsers = parser.add_subparsers(dest="command")+ subparsers.add_parser("quantize", help="Quantize a model.", add_help=False)+ subparsers.add_parser("list", help="List supported algorithms or formats.", add_help=False)+ subparsers.add_parser("eval", help="Evaluate a model.", add_help=False)+ help_parser = subparsers.add_parser("help", help="Show help for the CLI or a subcommand.")+ help_parser.add_argument("topic", nargs="?", choices=["quantize", "list", "eval"], default=None)+ return parserdiff --git a/auto_round/compressors/__init__.py b/auto_round/compressors/__init__.pyindex 531fc237f..a9b4467af 100644--- a/auto_round/compressors/__init__.py+++ b/auto_round/compressors/__init__.py@@ -12,20 +12,12 @@# See the License for the specific language governing permissions and# limitations under the License.-# Lazy imports to avoid circular dependencies-# Users should import from specific modules instead of this __init__.py+# Lazy imports to avoid circular dependencies.from typing import TYPE_CHECKINGif TYPE_CHECKING:from auto_round.compressors.base import BaseCompressor- from auto_round.compressors.config import (- DiffusionExtraConfig,- ExtraConfig,- MLLMExtraConfig,- SchemeExtraConfig,- TuningExtraConfig,- )from auto_round.compressors.data_driven import CalibratedRTNCompressor, DataDrivenCompressorfrom auto_round.compressors.entry import AutoRoundCompatible, AutoRoundfrom auto_round.compressors.model_free import ModelFreeCompressor@@ -39,11 +31,6 @@ __all__ = ["ZeroShotCompressor","AutoRoundCompatible","ModelFreeCompressor",- "ExtraConfig",- "TuningExtraConfig",- "SchemeExtraConfig",- "MLLMExtraConfig",- "DiffusionExtraConfig",]@@ -74,20 +61,4 @@ def __getattr__(name):from auto_round.compressors.model_free import ModelFreeCompressorreturn ModelFreeCompressor- elif name in ("ExtraConfig", "TuningExtraConfig", "SchemeExtraConfig", "MLLMExtraConfig", "DiffusionExtraConfig"):- from auto_round.compressors.config import (- DiffusionExtraConfig,- ExtraConfig,- MLLMExtraConfig,- SchemeExtraConfig,- TuningExtraConfig,- )-- return {- "ExtraConfig": ExtraConfig,- "TuningExtraConfig": TuningExtraConfig,- "SchemeExtraConfig": SchemeExtraConfig,- "MLLMExtraConfig": MLLMExtraConfig,- "DiffusionExtraConfig": DiffusionExtraConfig,- }[name]raise AttributeError(f"module '{__name__}' has no attribute '{name}'")diff --git a/auto_round/compressors/base.py b/auto_round/compressors/base.pyindex 4ad5babfd..9dcfc2239 100644--- a/auto_round/compressors/base.py+++ b/auto_round/compressors/base.py@@ -21,12 +21,12 @@ from typing import Any, Optional, Unionimport torchfrom transformers import AutoConfig, set_seed-from auto_round.algorithms.alg_config import AlgConfig-from auto_round.algorithms.quantization import BaseQuantizers, QuantizationConfig+from auto_round.algorithms.quantization import BaseQuantizer, QuantizationConfigfrom auto_round.algorithms.transforms import (BaseRotationConfig,apply_rotation,)+from auto_round.auto_scheme.gen_auto_scheme import AutoSchemefrom auto_round.compressors.shard_writer import ShardWriterfrom auto_round.compressors.utils import _get_save_folder_name, is_mx_fp, is_nv_fp, set_layer_configfrom auto_round.context.compress import CompressContext@@ -107,6 +107,47 @@ class SerializedCompressorConfig:SERIALIZATION_KEYS = tuple(field.name for field in fields(SerializedCompressorConfig))+def collect_user_scheme_overrides(configs: list[Any]) -> dict[str, Any]:+ scheme_fields = {f.name for f in fields(QuantizationScheme)}+ user_scheme_overrides = {}+ user_scheme_sources = {}+ for config in configs:+ for key in getattr(config, "_user_set_scheme_fields", set()):+ if key not in scheme_fields:+ continue+ value = getattr(config, key, None)+ if value is None:+ continue+ if key in user_scheme_overrides and value != user_scheme_overrides[key]:+ prev_config, prev_value = user_scheme_sources[key]+ raise ValueError(+ f"Conflicting shared scheme field {key!r}: "+ f"{type(prev_config).__name__}.{key}={prev_value!r}, "+ f"{type(config).__name__}.{key}={value!r}. "+ "Use the same value for shared fields or pass scheme arguments through Compressor."+ )+ user_scheme_overrides[key] = value+ user_scheme_sources[key] = (config, value)+ return user_scheme_overrides+++def _make_compressor_scheme_property(name):+ def getter(self):+ scheme_context = getattr(self, "scheme_context", None)+ if scheme_context is not None:+ return getattr(scheme_context, name)+ return self.__dict__.get(name, getattr(type(self), name, None))++ def setter(self, value):+ scheme_context = getattr(self, "scheme_context", None)+ if scheme_context is not None:+ setattr(scheme_context, name, value)+ else:+ self.__dict__[name] = value++ return property(getter, setter)++class BaseCompressor(object):need_calib: bool = Truecompress_context: CompressContext = None@@ -129,6 +170,9 @@ class BaseCompressor(object):quant_lm_head: bool = False_scheme_resolved: bool = Falsescheme_generator = None+ _scheme_context_fields = set(QuantizationScheme.get_attributes())+ for _scheme_field in QuantizationScheme.get_attributes():+ locals()[_scheme_field] = _make_compressor_scheme_property(_scheme_field)@staticmethoddef _preload_model_config(model: Union[torch.nn.Module, str], trust_remote_code: bool) -> Optional[AutoConfig]:@@ -148,22 +192,26 @@ class BaseCompressor(object):def __init__(self,- config: Union[AlgConfig, list[AlgConfig]],+ config: Union[object, list[object]],model: Union[torch.nn.Module, str],- tokenizer=None,- platform="hf",- format=None,- scheme="W4A16",+ tokenizer: Any = None,+ platform: str = "hf",+ format: Union[str, list, None] = None,+ scheme: Union[str, dict, QuantizationScheme, AutoScheme] = "W4A16",low_gpu_mem_usage: bool = False,device_map: Union[str, torch.device, int, dict] = 0,enable_torch_compile: bool = False,seed: int = 42,low_cpu_mem_usage: bool = True,- layer_config=None,+ layer_config: Optional[dict] = None,nsamples: int = None,seqlen: int = None,+ scale_dtype: Optional[Union[str, torch.dtype]] = None,+ ignore_layers: str = "",+ quant_lm_head: bool = False,+ to_quant_block_names: Optional[Union[str, list[str]]] = None,**kwargs,- ):+ ) -> None:# ``CalibrationState`` is the single source of truth for calibration# runtime state. Seed every calibration field here in one block so# the rest of ``__init__`` only ever interacts with the state object@@ -188,10 +236,25 @@ class BaseCompressor(object):self.quantize_config = Noneself.rotation_configs: list[BaseRotationConfig] = []_config_list = config if isinstance(config, list) else [config]- for _cfg in _config_list:- if isinstance(_cfg, QuantizationConfig):- self.quantize_config = _cfg- elif isinstance(_cfg, BaseRotationConfig):+ # Keep full list for pipeline construction (includes preprocessor configs).+ self._alg_configs: list = list(_config_list)+ from auto_round.algorithms.config_resolver import split_quantization_configs++ _preprocessor_configs, _block_quantizer_configs = split_quantization_configs(self._alg_configs)+ if len(_block_quantizer_configs) > 1:+ raise ValueError(+ f"Only one block-quantization config is allowed, but got {len(_block_quantizer_configs)}: "+ f"{[type(c).__name__ for c in _block_quantizer_configs]}"+ )+ if _block_quantizer_configs:+ self.quantize_config = _block_quantizer_configs[0]+ elif _preprocessor_configs:+ from auto_round.algorithms.quantization.rtn.config import RTNConfig as _RTNConfig++ self.quantize_config = _RTNConfig()+ self._alg_configs.append(self.quantize_config)+ for _cfg in self._alg_configs:+ if isinstance(_cfg, BaseRotationConfig):if hasattr(_cfg, "block_size") and _cfg.block_size is None:if "group_size" in kwargs:block_size = kwargs["group_size"]@@ -206,6 +269,10 @@ class BaseCompressor(object):# ``self._calibration_state`` (seeded above) and exposed via# ``@property`` forwarders.self.layer_config = layer_config+ self.scale_dtype = scale_dtype+ self.ignore_layers = ignore_layers+ self.quant_lm_head = quant_lm_head+ self.to_quant_block_names = to_quant_block_names# ``post_init()`` may run before ``quantize_and_save()`` in tests and# compatibility paths, so seed the same default used by# ``quantize_and_save(..., inplace=True)`` here.@@ -213,6 +280,7 @@ class BaseCompressor(object):# Scheme is passed directly to the compressor, not stored in QuantizationConfig.self.scheme = scheme+ self.scheme_context = None# Calibrator strategy (auto_round.calibration.base.Calibrator). Constructed# lazily by ``DataDrivenCompressor.post_init`` based on ``_get_calibrator_kind()``;@@ -332,9 +400,6 @@ class BaseCompressor(object):)self.shard_writer = None- # scale_dtype is resolved in quantizer.resolve_scheme() after scheme resolution,- # so it is not initialized here to avoid premature evaluation with an unresolved scheme.-# Flag for post_init idempotency. Set to False here so post_init() can be called# either via quantize_and_save() (preferred, outside inference_mode) or directly# from quantize() as a fallback for non-AutoScheme cases.@@ -354,9 +419,21 @@ class BaseCompressor(object):for key, value in fixed_attr.items():setattr(self, key, value)+ # ── Convenience properties ────────────────────────────────────────────────++ @property+ def tokenizer(self) -> Any:+ """Convenience accessor for the tokenizer stored in ``model_context``."""+ return self.model_context.tokenizer+# ── Scheme resolution ─────────────────────────────────────────────────────- def resolve_scheme(self, model_context=None, compress_context=None, dataset: str = None) -> None:+ def resolve_scheme(+ self,+ model_context: Optional[ModelContext] = None,+ compress_context: Optional[CompressContext] = None,+ dataset: Optional[str] = None,+ ) -> None:"""Phase-1 init: resolve scheme and bind config attrs (no model structure needed).Must be called BEFORE ``get_formats()`` and BEFORE ``_scheme_post_init()``.@@ -372,18 +449,13 @@ class BaseCompressor(object):if dataset is not None:self.dataset = dataset- scheme_fields = {f.name for f in fields(QuantizationScheme)}- user_scheme_overrides = {- k: getattr(self.quantize_config, k)- for k in scheme_fields- if getattr(self.quantize_config, k, None) is not None- }+ user_scheme_overrides = collect_user_scheme_overrides(self._alg_configs)default_scheme, self.is_auto_scheme, final_attrs = parse_scheme(self.scheme, user_scheme_overrides)- for key, value in final_attrs.items():- setattr(self.quantize_config, key, value)- if hasattr(self, key):- setattr(self, key, value)+ self.scheme_context = QuantizationScheme.from_dict(final_attrs)+ for config in self._alg_configs:+ if hasattr(config, "scheme"):+ config.scheme = self.scheme_contextself.quantize_config.check_config()self.orig_scheme = copy.deepcopy(self.scheme)@@ -417,7 +489,6 @@ class BaseCompressor(object):)if self.to_quant_block_names is None and self.quant_block_list:self.to_quant_block_names = extract_block_names_to_str(self.quant_block_list)- self.quantize_config.to_quant_block_names = self.to_quant_block_namesself.configure_layer_config(enable_gguf_official_mixed=enable_gguf_official_mixed)@@ -633,11 +704,11 @@ class BaseCompressor(object):# ─────────────────────────────────────────────────────────────────────────@property- def mllm(self):+ def mllm(self) -> bool:return self.model_context.is_mllm@property- def diffusion(self):+ def diffusion(self) -> bool:return self.model_context.is_diffusiondef _get_torch_compile_guard_state(self) -> tuple[bool, bool, int]:@@ -788,8 +859,6 @@ class BaseCompressor(object):- ``self.quantize_config`` is a valid :class:`QuantizationConfig`.Work performed:- - Seeds scheme-related attrs (``scale_dtype``, ``ignore_layers``,- ``quant_lm_head``, ``to_quant_block_names``) from ``quantize_config``.- Calls :meth:`resolve_scheme` to derive ``data_type``, ``bits``,``sym``, ``scale_dtype`` etc. and write them back to both ``self``and ``self.quantize_config``.@@ -797,16 +866,8 @@ class BaseCompressor(object):Postconditions:- ``self.scheme`` and ``self.quantize_config`` carry resolved scheme attrs."""- cfg = self.quantize_config- self.scale_dtype = cfg.scale_dtype- # self.layer_config is already set from __init__ (direct compressor param).- self.ignore_layers = cfg.ignore_layers- self.quant_lm_head = cfg.quant_lm_head- self.to_quant_block_names = cfg.to_quant_block_namesif self.to_quant_block_names is None:self.to_quant_block_names = getattr(self.model_context.model, "_autoround_to_quant_block_names", None)- if self.to_quant_block_names is not None:- self.quantize_config.to_quant_block_names = self.to_quant_block_names# Resolve the scheme (pure config work: sets data_type / bits / sym /# scale_dtype etc. on both self and self.quantize_config).@@ -826,18 +887,51 @@ class BaseCompressor(object):been synced back to ``self.quantize_config``.Work performed:- - Constructs ``self.quantizer`` from the resolved config.+ - Constructs the block_quantizer from the resolved config.+ - Wraps it in a :class:`~auto_round.algorithms.pipeline.QuantizationPipeline`+ so that the entire compressor operates through the pipeline abstraction.- Calls ``quantizer.bind(self)`` so the quantizer pulls``model_context`` / ``compress_context`` / ``scale_dtype`` /``CalibrationState`` from this compressor. ``quantizer.model``is a property that reads ``model_context.model``.+ - Exposes ``self.quantizer`` as a ``@property`` (see below) that+ transparently delegates to ``self.pipeline.block_quantizer`` so all+ existing call-sites continue to work without modification.Postconditions:- - ``self.quantizer`` is ready and shares ``CalibrationState`` with- the compressor.+ - ``self.pipeline`` is a ``QuantizationPipeline`` wrapping the block quantizer.+ - ``self.quantizer`` (via property) is ready and shares ``CalibrationState``+ with the compressor."""- self.quantizer = BaseQuantizers.from_config(self.quantize_config)- self.quantizer.bind(self)+ from auto_round.algorithms.pipeline import QuantizationPipeline++ self._pipeline = QuantizationPipeline.from_configs(self._alg_configs, compressor=self)++ @property+ def quantizer(self) -> BaseQuantizer:+ """Transparent forwarder to ``self.pipeline.block_quantizer``.++ All existing ``self.quantizer.xxx`` call-sites continue to work+ unchanged. New code should prefer ``self.pipeline`` for pipeline-aware+ operations.+ """+ _pipeline = self.__dict__.get("_pipeline")+ if _pipeline is not None:+ return _pipeline.block_quantizer+ return self.__dict__["_quantizer"]++ @quantizer.setter+ def quantizer(self, value: BaseQuantizer) -> None:+ _pipeline = self.__dict__.get("_pipeline")+ if _pipeline is not None:+ _pipeline.block_quantizer = value+ else:+ self.__dict__["_quantizer"] = value++ @property+ def pipeline(self) -> Any:+ """The active :class:`~auto_round.algorithms.pipeline.QuantizationPipeline`."""+ return self._pipelinedef _resolve_formats(self) -> None:"""Phase 2 – Format resolution and config attr sync.@@ -1044,14 +1138,25 @@ class BaseCompressor(object):self.quantizer.scale_dtype = self.scale_dtypeself.quantizer.ignore_layers = self.ignore_layers+ from auto_round.algorithms.config_resolver import sync_shared_config_from++ sync_shared_config_from(self.quantizer.config, [pre.config for pre in self._pipeline.preprocessors])++ # Also sync runtime-only state to all preprocessors in the pipeline so+ # they have access to per-layer quant config during pre-processing (e.g.+ # AWQ grid search uses layer_config to look up bits/group_size for each layer).+ for pre in self._pipeline.preprocessors:+ pre.layer_config = self.layer_config+ pre.scale_dtype = self.scale_dtype+def _hardware_setup(self) -> None:"""Phase 5 – Hardware and compile configuration.Preconditions:- Phase 4 complete: ``layer_config`` is built and``has_qlayer_outside_block`` is known.- - ``self.quantize_config.data_type`` is the final resolved value- (needed by :meth:`_finalize_torch_compile`).+ - ``self.quantize_config.data_type`` is the final resolved value+ (needed by :meth:`_finalize_torch_compile`).Work performed:- Applies the device map via :func:`~auto_round.utils.device.set_non_auto_device_map`.@@ -1097,10 +1202,31 @@ class BaseCompressor(object):if name in self.__dict__:return self.__dict__[name]- for obj in ["quantizer", "quantize_config", "model_context", "compress_context"]:- if obj not in self.__dict__:+ # Never proxy private/dunder attributes — they should be set explicitly+ # in __init__. Proxying them hides bugs (e.g. missing _post_init_done)+ # and can cause infinite recursion.+ if name.startswith("_"):+ raise AttributeError(f"'{type(self).__name__}' object has no attribute '{name}'")++ # Delegate to block_quantizer: access _pipeline directly from __dict__ to+ # avoid recursion (quantizer is now a @property backed by _pipeline; going+ # through the property inside __getattr__ would re-trigger __getattr__+ # if _pipeline itself isn't ready yet).+ _pipeline = self.__dict__.get("_pipeline")+ if _pipeline is not None:+ try:+ return object.__getattribute__(_pipeline.block_quantizer, name)+ except AttributeError:+ pass++ for attr in ["quantize_config", "model_context", "compress_context"]:+ # These are regular instance attributes; use object.__getattribute__+ # so Python's normal descriptor protocol is used without re-entering+ # __getattr__ on self.+ try:+ obj = object.__getattribute__(self, attr)+ except AttributeError:continue- obj = object.__getattribute__(self, obj)try:return object.__getattribute__(obj, name)except AttributeError:@@ -1114,7 +1240,7 @@ class BaseCompressor(object):return device_manager.device@device.setter- def device(self, value) -> None:+ def device(self, value: Union[str, torch.device]) -> None:device_manager.device = value@property@@ -1122,21 +1248,23 @@ class BaseCompressor(object):return device_manager.device_list@property- def device_map(self):+ def device_map(self) -> Any:return device_manager.device_map# ── Forwarding properties to ``self._calibration_state`` ──────────────────@property- def calibration_state(self):+ def calibration_state(self) -> Any:return self._calibration_state@calibration_state.setter- def calibration_state(self, value) -> None:+ def calibration_state(self, value: Any) -> None:self._calibration_state = value# Re-wire quantizer if it already exists so they keep sharing.- q = self.__dict__.get("quantizer")- if q is not None:- q.calibration_state = value+ # quantizer is now a @property forwarding to _pipeline.block_quantizer;+ # use _pipeline directly to avoid triggering __getattr__ loops.+ _pipeline = self.__dict__.get("_pipeline")+ if _pipeline is not None:+ _pipeline.block_quantizer.calibration_state = value@propertydef inputs(self) -> dict:@@ -1159,11 +1287,11 @@ class BaseCompressor(object):self._calibration_state.to_cached_layers = []@property- def last_cache_name(self):+ def last_cache_name(self) -> Optional[str]:return self._calibration_state.last_cache_name@last_cache_name.setter- def last_cache_name(self, value) -> None:+ def last_cache_name(self, value: Optional[str]) -> None:self._calibration_state.last_cache_name = value@last_cache_name.deleter@@ -1214,19 +1342,19 @@ class BaseCompressor(object):self._calibration_state.seqlen = value@property- def dataset(self):+ def dataset(self) -> Any:return self._calibration_state.dataset@dataset.setter- def dataset(self, value) -> None:+ def dataset(self, value: Any) -> None:self._calibration_state.dataset = value@property- def dataloader(self):+ def dataloader(self) -> Any:return self._calibration_state.dataloader@dataloader.setter- def dataloader(self, value) -> None:+ def dataloader(self, value: Any) -> None:self._calibration_state.dataloader = value@dataloader.deleter@@ -1234,7 +1362,7 @@ class BaseCompressor(object):self._calibration_state.dataloader = None@property- def optimizer(self):+ def optimizer(self) -> Any:"""Return the actual optimizer class, converting string to class for backward compat.Old API stored ``self.optimizer = torch.optim.AdamW`` (the class itself).@@ -1347,7 +1475,7 @@ class BaseCompressor(object):output_dir: str = None,format: Union[str, list[OutputFormat]] = None,inplace: bool = True,- return_folders=False,+ return_folders: bool = False,**kwargs,) -> torch.nn.Module:"""Save the quantized model to the specified output directory in the specified format.diff --git a/auto_round/compressors/config.py b/auto_round/compressors/config.pydeleted file mode 100644index dd028b6cd..000000000--- a/auto_round/compressors/config.py+++ /dev/null@@ -1,296 +0,0 @@-# Copyright (c) 2025 Intel Corporation-#-# Licensed under the Apache License, Version 2.0 (the "License");-# you may not use this file except in compliance with the License.-# You may obtain a copy of the License at-#-# http://www.apache.org/licenses/LICENSE-2.0-#-# Unless required by applicable law or agreed to in writing, software-# distributed under the License is distributed on an "AS IS" BASIS,-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.-# See the License for the specific language governing permissions and-# limitations under the License.-from __future__ import annotations--from dataclasses import dataclass, fields-from typing import Any, Callable, Optional, Union--import torch---class ExtraConfig:- """Class for extra or legacy configs."""-- _model_config = None- _scheme_config = None- _tuning_config = None- _mllm_config = None- _diffusion_config = None-- def __init__(- self,- # tuning- amp: bool = True,- disable_opt_rtn: bool | None = None,- enable_alg_ext: bool = False,- enable_minmax_tuning: bool = True,- enable_norm_bias_tuning: bool = False,- enable_quanted_input: bool = True,- enable_deterministic_algorithms: bool = False,- lr: float = None,- lr_scheduler: Callable = None,- minmax_lr: float = None,- nblocks: int = 1,- to_quant_block_names: Union[str, list, None] = None,- scale_dtype: str = "fp16",- # scheme- bits: int = None,- group_size: int = None,- sym: bool = None,- data_type: str = None,- act_bits: int = None,- act_group_size: int = None,- act_sym: bool = None,- act_data_type: str = None,- act_dynamic: bool = None,- super_bits: int = None,- super_group_size: int = None,- static_kv_dtype: Union[str, torch.dtype] = None,- quant_lm_head: bool = False,- ignore_layers: str = None,- # mllm- processor: Callable = None,- image_processor: Callable = None,- quant_nontext_module: bool = False,- extra_data_dir: str = None,- template: str = None,- # diffusion- guidance_scale: float = 7.5,- num_inference_steps: int = 50,- generator_seed: int = None,- ):- """Initialize-- Args:- amp (bool): Whether to use automatic mixed precision (default is True).- disable_opt_rtn (bool, optional): Disable RTN-mode optimization (iters=0). Defaults to True.- enable_alg_ext (bool, optional): Enable algorithm extension (primarily for INT2). Defaults to False.- enable_minmax_tuning (bool, optional): Enable weight min-max tuning. Defaults to True.- enable_norm_bias_tuning (bool): Whether to enable fast norm/layer_bias tuning.- enable_quanted_input (bool): Whether to use quantized input data (default is True).- enable_deterministic_algorithms (bool): Whether to use deterministic_algorithms.- lr (float): The learning rate (default is 0.005).- lr_scheduler: The learning rate scheduler to be used.- minmax_lr (float): The learning rate for min-max tuning (default is None).- nblocks (int): Number of blocks (default is 1).- quant_lm_head (bool): Whether to quant lm_head.- to_quant_block_names (str|list): Names of quantitative blocks, please use commas to separate them.- scale_dtype (str): The data type of quantization scale to be used (default is "float16"), different kernels- bits (int, optional): Weight quantization bits. Defaults to 4.- group_size (int, optional): Weight quantization group size. Defaults to 128.- sym (bool, optional): Symmetric weight quantization. Defaults to True.- data_type (str, optional): Weight data type string, e.g., "int". Defaults to "int".- act_bits (int, optional): Activation quantization bits. Defaults to 16.- act_group_size (int, optional): Activation group size. Defaults to None.- act_sym (bool, optional): Symmetric activation quantization. Defaults to None.- act_data_type (str, optional): Activation data type; inherits weight dtype if None and act_bits < 16.- act_dynamic (bool, optional): Dynamic activation quantization. Defaults to True.- super_bits (int): number of scale and mins quant bits for double quant.- super_group_size (int): the number of super group size when use double quant.- static_kv_dtype (str): The data type of kv-cache to be used.- processor: Any multi-modal model will require an object to encode or- decode the data that groups several modalities (among text, vision and audio).- image_processor: Image processor for special model like llava.- quant_nontext_module: Whether to quantize nontext module.- extra_data_dir: The path of extra data such as images, audio and videos.- template: The path or name of template used to specify process for different MLLMs.- guidance_scale (float): Control how much the image generation process follows the text prompt.- The more it is, the more closely it follows the prompt (default is 7.5).- num_inference_steps (int): The reference number of denoising steps (default is 50).- generator_seed (int): A seed that controls the initial noise for image generation (default is None).- """- self.tuning_config = TuningExtraConfig(- amp=amp,- disable_opt_rtn=disable_opt_rtn,- enable_alg_ext=enable_alg_ext,- enable_minmax_tuning=enable_minmax_tuning,- enable_norm_bias_tuning=enable_norm_bias_tuning,- enable_quanted_input=enable_quanted_input,- enable_deterministic_algorithms=enable_deterministic_algorithms,- lr=lr,- lr_scheduler=lr_scheduler,- minmax_lr=minmax_lr,- nblocks=nblocks,- to_quant_block_names=to_quant_block_names,- scale_dtype=scale_dtype,- )- self.scheme_config = SchemeExtraConfig(- bits=bits,- group_size=group_size,- sym=sym,- data_type=data_type,- act_bits=act_bits,- act_group_size=act_group_size,- act_sym=act_sym,- act_data_type=act_data_type,- act_dynamic=act_dynamic,- super_bits=super_bits,- super_group_size=super_group_size,- static_kv_dtype=static_kv_dtype,- quant_lm_head=quant_lm_head,- ignore_layers=ignore_layers,- )- self.mllm_config = MLLMExtraConfig(- processor=processor,- image_processor=image_processor,- quant_nontext_module=quant_nontext_module,- extra_data_dir=extra_data_dir,- template=template,- )- self.diffusion_config = DiffusionExtraConfig(- guidance_scale=guidance_scale,- num_inference_steps=num_inference_steps,- generator_seed=generator_seed,- )-- @property- def tuning_config(self):- return self._tuning_config-- @tuning_config.setter- def tuning_config(self, config: TuningExtraConfig):- assert isinstance(- config, TuningExtraConfig- ), f"tuning_config should be ModelExtraConfig, but got {config.__class__.__name__}"- self._tuning_config = config-- @property- def scheme_config(self):- return self._scheme_config-- @scheme_config.setter- def scheme_config(self, config: SchemeExtraConfig):- assert isinstance(- config, SchemeExtraConfig- ), f"scheme_config should be SchemeExtraConfig, but got {config.__class__.__name__}"- self._scheme_config = config-- @property- def mllm_config(self):- return self._mllm_config-- @mllm_config.setter- def mllm_config(self, config: MLLMExtraConfig):- if config is None:- self._mllm_config = None- else:- assert isinstance(- config, MLLMExtraConfig- ), f"mllm_config should be MLLMExtraConfig, but got {config.__class__.__name__}"- self._mllm_config = config-- @property- def diffusion_config(self):- return self._diffusion_config-- @diffusion_config.setter- def diffusion_config(self, config: DiffusionExtraConfig):- if config is None:- self._diffusion_config = None- else:- assert isinstance(- config, DiffusionExtraConfig- ), f"diffusion_config should be DiffusionExtraConfig, but got {config.__class__.__name__}"- self._diffusion_config = config-- def to_dict(self):- output_dict = {}- for config in self.__dict__.values():- if config:- output_dict.update(config.to_dict())- return output_dict---@dataclass-class BaseExtraConfig:-- @classmethod- def get_attributes(cls: "BaseExtraConfig") -> list[str]:- return [field.name for field in fields(cls)]-- def __getitem__(self, key: str):- if key not in self.get_attributes():- raise KeyError(f"{key} is not a valid attribute")- return getattr(self, key)-- def __setitem__(self, key: str, value: None | int | str):- if key not in self.get_attributes():- raise KeyError(f"{key} is not a valid attribute")- setattr(self, key, value)-- def __contains__(self, item):- return item in self.get_attributes()-- def to_dict(self):- return self.__dict__-- def is_default(self):- for field in fields(self):- default_value = field.default- current_value = getattr(self, field.name)- if current_value != default_value:- return False- return True---@dataclass-class TuningExtraConfig(BaseExtraConfig):- amp: bool = True- disable_opt_rtn: bool | None = None- enable_alg_ext: bool = False- enable_minmax_tuning: bool = True- enable_norm_bias_tuning: bool = False- enable_quanted_input: bool = True- enable_deterministic_algorithms: bool = False- lr: float = None- lr_scheduler: Callable = None- minmax_lr: float = None- nblocks: int = 1- to_quant_block_names: Union[str, list, None] = None- scale_dtype: str = "fp16"---@dataclass-class SchemeExtraConfig(BaseExtraConfig):- bits: int = None- group_size: int = None- sym: bool = None- data_type: str = None- act_bits: int = None- act_group_size: int = None- act_sym: bool = None- act_data_type: str = None- act_dynamic: bool = None- super_bits: int = None- super_group_size: int = None- static_kv_dtype: Union[str, torch.dtype] = None- static_attention_dtype: Union[str, torch.dtype] = None- quant_lm_head: bool = False- ignore_layers: str = None---@dataclass-class MLLMExtraConfig(BaseExtraConfig):- processor: Callable = None- image_processor: Callable = None- quant_nontext_module: bool = False- extra_data_dir: str = None- template: str = None---@dataclass-class DiffusionExtraConfig(BaseExtraConfig):- guidance_scale: float = 7.5- num_inference_steps: int = 50- generator_seed: int = Nonediff --git a/auto_round/compressors/data_driven.py b/auto_round/compressors/data_driven.pyindex c0838773e..99c8ac579 100644--- a/auto_round/compressors/data_driven.py+++ b/auto_round/compressors/data_driven.py@@ -15,6 +15,7 @@ import copyimport gcimport timeimport traceback+from contextlib import ExitStackfrom functools import partialfrom typing import Any, Callable, Optional, Union@@ -25,7 +26,6 @@ from accelerate.utils import get_balanced_memory, get_max_memoryfrom tqdm import tqdmfrom auto_round import envs-from auto_round.algorithms.alg_config import AlgConfigfrom auto_round.calibration.utils import (_infer_last_cache_name,_split_inputs_diffusion,@@ -76,11 +76,11 @@ class DataDrivenCompressor(BaseCompressor):def __init__(self,- config: Union[AlgConfig, list[AlgConfig]],+ config: Union[object, list[object]],model: Union[torch.nn.Module, str],- tokenizer=None,- platform="hf",- format=None,+ tokenizer: Any = None,+ platform: str = "hf",+ format: Union[str, list, None] = None,dataset: Union[str, list, tuple, torch.utils.data.DataLoader] = "NeelNanda/pile-10k",iters: int = 200,low_gpu_mem_usage: bool = False,@@ -89,7 +89,9 @@ class DataDrivenCompressor(BaseCompressor):seed: int = 42,low_cpu_mem_usage: bool = True,**kwargs,- ):+ ) -> None:+ if iters is None:+ iters = 200self.iters = iterssuper().__init__(config=config,@@ -135,7 +137,13 @@ class DataDrivenCompressor(BaseCompressor):return "llm"@torch.no_grad()- def try_cache_inter_data_gpucpu(self, block_names, nsamples, layer_names=None, last_cache_name=None):+ def try_cache_inter_data_gpucpu(+ self,+ block_names: list,+ nsamples: int,+ layer_names: Optional[list] = None,+ last_cache_name: Optional[str] = None,+ ) -> Any:"""Thin wrapper around ``self.calibration.collect``.Public API kept for backward compatibility (entry.py and@@ -146,7 +154,13 @@ class DataDrivenCompressor(BaseCompressor):return self.calibration.collect(block_names, nsamples, layer_names=layer_names, last_cache_name=last_cache_name)@torch.no_grad()- def cache_inter_data(self, block_names, nsamples, layer_names=None, last_cache_name=None):+ def cache_inter_data(+ self,+ block_names: list,+ nsamples: int,+ layer_names: Optional[list] = None,+ last_cache_name: Optional[str] = None,+ ) -> Any:"""Thin wrapper around ``self.calibration.cache_inter_data``.Public API kept for backward compatibility.@@ -158,7 +172,7 @@ class DataDrivenCompressor(BaseCompressor):)@torch.no_grad()- def calib(self, nsamples, bs):+ def calib(self, nsamples: int, bs: int) -> Any:"""Thin wrapper around ``self.calibration.calib``.``MLLMMixin`` and ``DiffusionMixin`` override this method directly via@@ -263,11 +277,11 @@ class DataDrivenCompressor(BaseCompressor):def quantize_block(self,block: torch.nn.Module,- inputs,+ inputs: Any,q_input: Union[torch.Tensor, dict, None] = None,device: Union[str, torch.device] = "cpu",auto_offload: bool = True,- ):+ ) -> Any:"""Quantize a single decoded block of the model (public API for LLM-Compressor).This method is the new-arch equivalent of the old ``BaseCompressor.quantize_block``@@ -315,6 +329,20 @@ class DataDrivenCompressor(BaseCompressor):if not self._post_init_done:self.post_init()+ if len(self.quant_block_list) != 1 or len(self.quant_block_list[0]) != 1:+ raise ValueError(+ f"{self.__class__.__name__}.quantize_block supports exactly one target block, "+ f"but quant_block_list is {self.quant_block_list!r}. "+ "Use to_quant_block_names to select a single block."+ )+ expected_block_name = self.quant_block_list[0][0]+ actual_block_name = getattr(block, "global_name", None)+ if actual_block_name is not None and actual_block_name != expected_block_name:+ raise ValueError(+ f"quantize_block received block {actual_block_name!r}, but cached inputs are for "+ f"{expected_block_name!r}. Pass the matching block or update to_quant_block_names."+ )+# When called from LLM-Compressor, `wrapped_model` is a single decoder layer# (not the full VL model), so it must not be treated as an MLLM regardless of# whether the original model had multimodal assets. Force is_mllm=False for@@ -368,50 +396,114 @@ class DataDrivenCompressor(BaseCompressor):continueadd_hook_to_module(m, AlignDevicesHook(m.tuning_device, io_same_device=True), True)- # ── Infrastructure: collect reference output and act_max ──────────────+ blk_name = self.quant_block_list[0][0]bs = self.quantizer.batch_size * self.quantizer.infer_bs_coeff- if q_input is None:- hook_handles = self.quantizer.register_calibration_hooks(block)- reference_output = self.quantizer._get_block_outputs(block, input_ids, input_others, bs)- for h in hook_handles:- h.remove()+ mid_iter_mem_check = self.compress_context.low_gpu_mem_usage and card_0_in_high_risk++ if not hasattr(self.quantizer, "create_block_io"):+ if q_input is None:+ hook_handles = self.quantizer.register_calibration_hooks(block)+ reference_output = self.quantizer._get_block_outputs(block, input_ids, input_others, bs)+ for h in hook_handles:+ h.remove()+ else:+ reference_output = self.quantizer._get_block_outputs(block, input_ids, input_others, bs)+ hook_handles = self.quantizer.register_calibration_hooks(block)+ if hook_handles:+ self.quantizer._get_block_outputs(block, q_input, input_others, bs, save_output=False)+ for h in hook_handles:+ h.remove()+ if input_ids is not q_input:+ clear_memory(input_ids, device_list=device_manager.device_list)+ else:+ clear_memory(device_list=device_manager.device_list)+ input_ids = q_input++ self.quantizer.quantize_block(+ block,+ input_ids,+ input_others,+ reference_output,+ loss_device=loss_device,+ mid_iter_mem_check=mid_iter_mem_check,+ )++ if is_nv_fp(self.quantizer.act_data_type) or is_static_wfp8afp8(self.quantizer):+ set_amax_for_all_moe_layers(block, attr_name="act_max")++ if self.quantizer.enable_quanted_input:+ q_outputs = self.quantizer._get_block_outputs(block, input_ids, input_others, bs)+ else:+ q_outputs = None++ if len(device_manager.device_list) > 1:+ accelerate.hooks.remove_hook_from_submodules(block)+ mv_module_from_gpu(block)+ return q_outputs, reference_output++ from auto_round.algorithms.pipeline import BlockContext, InputSource++ ctx = BlockContext(+ model=self.model_context.model,+ block=block,+ block_names=[blk_name],+ block_name=blk_name,+ block_index=0,+ io=self.quantizer.create_block_io(input_ids, input_others, q_input, block),+ bs=bs,+ loss_device=loss_device,+ device=device,+ mid_iter_mem_check=mid_iter_mem_check,+ is_mllm=False,+ is_diffusion=False,+ )+ policy = self.pipeline.get_merged_policy(ctx)++ if policy.source == InputSource.QUANTIZED_INPUT and q_input is not None:+ with ExitStack() as fwd_stack:+ self.pipeline.enter_preprocessor_hooks(ctx, fwd_stack)+ reference_output = ctx.collect_reference(fwd_stack)+ with ExitStack() as fwd_stack:+ quantizer_hooks = self.pipeline.enter_quantizer_hooks(ctx, fwd_stack)+ if quantizer_hooks:+ ctx.collect_quantized_stats(fwd_stack)else:- reference_output = self.quantizer._get_block_outputs(block, input_ids, input_others, bs)- hook_handles = self.quantizer.register_calibration_hooks(block)- if hook_handles:- self.quantizer._get_block_outputs(block, q_input, input_others, bs, save_output=False)- for h in hook_handles:- h.remove()+ with ExitStack() as fwd_stack:+ self.pipeline.enter_block_forward_hooks(ctx, fwd_stack)+ reference_output = ctx.collect_reference(fwd_stack)++ if q_input is not None:if input_ids is not q_input:clear_memory(input_ids, device_list=device_manager.device_list)else:clear_memory(device_list=device_manager.device_list)input_ids = q_input- # ── Pure algorithm: delegates to quantizer ────────────────────────────- mid_iter_mem_check = self.compress_context.low_gpu_mem_usage and card_0_in_high_risk- self.quantizer.quantize_block(- block,- input_ids,- input_others,- reference_output,- loss_device=loss_device,- mid_iter_mem_check=mid_iter_mem_check,- )+ # pre_quantize_block: consolidate stats and apply weight transforms.+ for pre in self.pipeline.preprocessors:+ pre.pre_quantize_block(ctx)++ # ── Pure algorithm: block_quantizer.quantize_block ────────────────────+ self.pipeline.block_quantizer.quantize_block(ctx)++ # ── Pipeline lifecycle: post_quantize_block ───────────────────────────+ for pre in self.pipeline.preprocessors:+ pre.post_quantize_block(ctx)# ── MoE scale alignment for FP8 dispatch efficiency ────────────────if is_nv_fp(self.quantizer.act_data_type) or is_static_wfp8afp8(self.quantizer):set_amax_for_all_moe_layers(block, attr_name="act_max")# ── Collect quantized-block outputs ───────────────────────────────────- if self.quantizer.enable_quanted_input:- q_outputs = self.quantizer._get_block_outputs(block, input_ids, input_others, bs)+ if self.pipeline.block_quantizer.enable_quanted_input:+ q_outputs = ctx.collect_next_inputs()else:q_outputs = None# ── Cleanup ───────────────────────────────────────────────────────────if len(device_manager.device_list) > 1:accelerate.hooks.remove_hook_from_submodules(block)+ ctx.finish()mv_module_from_gpu(block)return q_outputs, reference_outputfinally:@@ -445,19 +537,6 @@ class DataDrivenCompressor(BaseCompressor):input_ids, input_others = self._preprocess_block_inputs(inputs)- # For diffusion models, the heuristic split ("hidden_state" in key) may- # place keys like encoder_hidden_states in input_ids even though they are- # not block outputs. Move those to input_others so they persist across- # blocks (only output keys get refreshed via reference_output each iteration).- if self.model_context.is_diffusion and isinstance(input_ids, dict):- first_block = get_module(model, block_names[0])- output_config = self.quantizer.DIFFUSION_OUTPUT_CONFIGS.get(- first_block.__class__.__name__, ["hidden_states"]- )- extra_keys = [k for k in list(input_ids.keys()) if k not in output_config]- for k in extra_keys:- input_others[k] = input_ids.pop(k)-if pbar is None:pbar = tqdm(range(0, len(block_names), nblocks))@@ -517,26 +596,52 @@ class DataDrivenCompressor(BaseCompressor):continueadd_hook_to_module(_mod, AlignDevicesHook(_mod.tuning_device, io_same_device=True), True)- # ── Infrastructure: collect reference output and act_max ──────────+ # ── Pipeline lifecycle: per-block setup ───────────────────────────+ from auto_round.algorithms.pipeline import BlockContext, InputSource++ current_block_names = (+ block_name_or_names if isinstance(block_name_or_names, list) else [block_name_or_names]+ )+ current_block_name = current_block_names[0] if len(current_block_names) == 1 else str(block_name_or_names)bs = self.quantizer.batch_size * self.quantizer.infer_bs_coeff- if q_input is None:- hook_handles = self.quantizer.register_calibration_hooks(m)- reference_output = self.quantizer._get_block_outputs(- m, input_ids, input_others, bs, device_override=loss_device- )- for h in hook_handles:- h.remove()+ mid_iter_mem_check = self.compress_context.low_gpu_mem_usage and card_0_in_high_risk++ ctx = BlockContext(+ model=model,+ block=m,+ block_names=current_block_names,+ block_name=current_block_name,+ block_index=i,+ io=self.quantizer.create_block_io(input_ids, input_others, q_input, m),+ bs=bs,+ loss_device=loss_device,+ device=device_manager.device,+ mid_iter_mem_check=mid_iter_mem_check,+ is_mllm=self.model_context.is_mllm,+ is_diffusion=self.model_context.is_diffusion,+ pbar=pbar,+ )++ # ── Infrastructure: collect reference output and act calib ────────+ # All forward hooks (preprocessor stats + act-calib) are active during+ # the reference forward and removed when the ExitStack exits.+ policy = self.pipeline.get_merged_policy(ctx)++ if policy.source == InputSource.QUANTIZED_INPUT and q_input is not None:+ # First: reference forward with FP inputs and preprocessor hooks only.+ with ExitStack() as fwd_stack:+ self.pipeline.enter_preprocessor_hooks(ctx, fwd_stack)+ reference_output = ctx.collect_reference(fwd_stack)+ # Second: quantizer stats forward with q_input.+ with ExitStack() as fwd_stack:+ quantizer_hooks = self.pipeline.enter_quantizer_hooks(ctx, fwd_stack)+ if quantizer_hooks:+ ctx.collect_quantized_stats(fwd_stack)else:- reference_output = self.quantizer._get_block_outputs(- m, input_ids, input_others, bs, device_override=loss_device- )- hook_handles = self.quantizer.register_calibration_hooks(m)- if hook_handles:- self.quantizer._get_block_outputs(- m, q_input, input_others, bs, save_output=False, device_override=loss_device- )- for h in hook_handles:- h.remove()+ # Unified: reference forward with all hooks active (or no hooks).+ with ExitStack() as fwd_stack:+ self.pipeline.enter_block_forward_hooks(ctx, fwd_stack)+ reference_output = ctx.collect_reference(fwd_stack)# ── Infrastructure: swap q_input ──────────────────────────────────if q_input is not None:@@ -546,24 +651,24 @@ class DataDrivenCompressor(BaseCompressor):clear_memory(device_list=device_manager.device_list)input_ids = q_input- # ── Pure algorithm: delegates to quantizer ────────────────────────- mid_iter_mem_check = self.compress_context.low_gpu_mem_usage and card_0_in_high_risk- self.quantizer.quantize_block(- m,- input_ids,- input_others,- reference_output,- loss_device=loss_device,- mid_iter_mem_check=mid_iter_mem_check,- )+ # ── Pipeline lifecycle: pre_quantize_block (stats consolidation + weight transforms) ──+ for pre in self.pipeline.preprocessors:+ pre.pre_quantize_block(ctx)++ # ── Pure algorithm: block_quantizer.quantize_block ────────────────+ self.pipeline.block_quantizer.quantize_block(ctx)++ # ── Pipeline lifecycle: post_quantize_block ───────────────────────+ for pre in self.pipeline.preprocessors:+ pre.post_quantize_block(ctx)# ── MoE scale alignment for FP8 dispatch efficiency ────────────────if is_nv_fp(self.quantizer.act_data_type) or is_static_wfp8afp8(self.quantizer):set_amax_for_all_moe_layers(m, attr_name="act_max")# ── Infrastructure: collect q_outputs if needed ───────────────────- if self.quantizer.enable_quanted_input:- q_input = self.quantizer._get_block_outputs(m, input_ids, input_others, bs)+ if self.pipeline.block_quantizer.enable_quanted_input:+ q_input = ctx.collect_next_inputs()else:q_input = None@@ -579,6 +684,7 @@ class DataDrivenCompressor(BaseCompressor):# enabled) is only used as the quantized-input companion for the# next block.next_input_ids = reference_output+ ctx.finish()clear_memory(input_ids if input_ids is not next_input_ids else None, device_list=device_manager.device_list)memory_monitor.log_summary()@@ -588,7 +694,12 @@ class DataDrivenCompressor(BaseCompressor):if hasattr(_mod, "bits") and check_to_quantized(_mod):from auto_round.compressors.utils import immediate_pack as _immediate_pack- _immediate_pack(_mod.global_name, self.quantizer.layer_config)+ module_name = getattr(_mod, "global_name", None)+ if module_name is None and nblocks == 1 and _n:+ module_name = f"{n}.{_n}"+ if module_name is None:+ continue+ _immediate_pack(module_name, self.quantizer.layer_config)input_ids = next_input_ids@@ -666,7 +777,7 @@ class DataDrivenCompressor(BaseCompressor):last_cache_name=_last_cache_name,)self.inputs = all_inputs- is_quantized_embedding = self._quantize_embedding_layer()+ is_quantized_embedding = self.quantizer.quantize_embedding_layer()clear_memory(device_list=device_manager.device_list)all_q_inputs = Noneif is_quantized_embedding:@@ -700,38 +811,51 @@ class DataDrivenCompressor(BaseCompressor):pbar = tqdm(range(0, len(all_blocks[0]), self.nblocks)) # move the alg warning outside pbarstart_time = time.time()- for block_names in all_blocks:- inputs = all_inputs[block_names[0]]- all_inputs.pop(block_names[0])- q_inputs = None- if all_q_inputs is not None:- q_inputs = all_q_inputs[block_names[0]]- all_q_inputs.pop(block_names[0])- inputs, q_inputs = _update_inputs(inputs, q_inputs)+ for alg in self.pipeline.all():+ alg.prepare_run(self)- clear_memory(self.inputs, device_list=device_manager.device_list)+ try:+ for block_names in all_blocks:+ inputs = all_inputs[block_names[0]]+ all_inputs.pop(block_names[0])+ q_inputs = None+ if all_q_inputs is not None:+ q_inputs = all_q_inputs[block_names[0]]+ all_q_inputs.pop(block_names[0])- if "input_ids" in inputs.keys():- total_samples = len(inputs["input_ids"])- if total_samples < self.quantizer.batch_size:- self.quantizer.batch_size = total_samples- logger.warning(f"force the train batch size to {total_samples}")-- self._quantize_blocks(- self.model_context.model,- inputs,- block_names,- q_input=q_inputs if q_inputs is not None else None,- nblocks=self.nblocks,- pbar=pbar,- input_others_extra_blocks=all_inputs,- )- if self.compress_context.is_immediate_packing and len(self.formats) != 1:- raise ValueError(- f"Expected exactly one packing format when 'immediate_packing' is True, "- f"but got {len(self.formats)} formats."+ inputs, q_inputs = _update_inputs(inputs, q_inputs)++ clear_memory(self.inputs, device_list=device_manager.device_list)++ if "input_ids" in inputs.keys():+ total_samples = len(inputs["input_ids"])+ if total_samples < self.quantizer.batch_size:+ self.quantizer.batch_size = total_samples+ logger.warning(f"force the train batch size to {total_samples}")++ self._quantize_blocks(+ self.model_context.model,+ inputs,+ block_names,+ q_input=q_inputs if q_inputs is not None else None,+ nblocks=self.nblocks,+ pbar=pbar,+ input_others_extra_blocks=all_inputs,)+ if self.compress_context.is_immediate_packing and len(self.formats) != 1:+ raise ValueError(+ f"Expected exactly one packing format when 'immediate_packing' is True, "+ f"but got {len(self.formats)} formats."+ )+ finally:+ # ── Pipeline lifecycle: finalize_quantization (model-level teardown) ─+ for alg in self.pipeline.all():+ try:+ alg.finalize_run(self)+ except Exception as _fe:+ logger.warning("finalize_run error in %s: %s", type(alg).__name__, _fe)+pbar.set_description("Quantizing done")pbar.close()if self.compress_context.low_cpu_mem_usage:@@ -782,6 +906,7 @@ class DataDrivenCompressor(BaseCompressor):"""# TODO currently we take all the layers outside blocks as post block layers which is not optimal# if there is no input for layer, we use rtn+for layer_name in copy.deepcopy(layer_names):if layer_name not in layer_inputs:if self.act_bits < 16 and not self.act_dynamic:@@ -891,10 +1016,10 @@ class CalibratedRTNCompressor(DataDrivenCompressor):def __init__(self,- config: AlgConfig,+ config: object,model: torch.nn.Module,**kwargs,- ):+ ) -> None:kwargs["iters"] = 0super().__init__(config,@@ -1010,7 +1135,7 @@ class CalibratedRTNCompressor(DataDrivenCompressor):self.quantizer.batch_size,device_manager.device,)- if len(device_manager.device_list) > 1 and not self.model_context.is_diffusion:+ if len(device_manager.device_list) > 1:from accelerate.hooks import AlignDevicesHook, add_hook_to_modulefor _, _mod in block.named_modules():@@ -1020,17 +1145,26 @@ class CalibratedRTNCompressor(DataDrivenCompressor):else:block = block.to(device_manager.device)- # ── Infrastructure: register act_max hook and run forward pass ──- hook_handles = self.quantizer.register_calibration_hooks(block, imatrix=False)- block_input_ids = input_ids # keep reference for quantize_block- input_ids = self.quantizer._get_block_outputs(- block,- input_ids,- input_others,- self.quantizer.batch_size * self.quantizer.infer_bs_coeff,+ # ── Infrastructure: collect block outputs and hook stats ──+ from auto_round.algorithms.pipeline import BlockContext++ block_input_ids = input_ids+ bs = self.quantizer.batch_size * self.quantizer.infer_bs_coeff+ ctx = BlockContext(+ model=self.model_context.model,+ block=block,+ block_names=[block_name],+ block_name=block_name,+ block_index=0,+ io=self.quantizer.create_block_io(input_ids, input_others, None, block),+ bs=bs,+ device=device_manager.device,+ is_mllm=self.model_context.is_mllm,+ is_diffusion=self.model_context.is_diffusion,)- for h in hook_handles:- h.remove()+ with ExitStack() as fwd_stack:+ self.pipeline.enter_block_forward_hooks(ctx, fwd_stack)+ input_ids = ctx.collect_reference(fwd_stack)if len(device_manager.device_list) > 1:accelerate.hooks.remove_hook_from_submodules(block)@@ -1040,7 +1174,9 @@ class CalibratedRTNCompressor(DataDrivenCompressor):self.compress_context.clear_memory()# ── Pure algorithm ────────────────────────────────────────────- self.quantizer.quantize_block(block, block_input_ids, input_others, block_name=block_name)+ ctx.io.seed_reference(fp_inputs=block_input_ids, reference_outputs=input_ids)+ self.quantizer.quantize_block(ctx)+ ctx.finish()# ── Infrastructure: cleanup ───────────────────────────────────mv_module_from_gpu(block)@@ -1077,14 +1213,6 @@ class CalibratedRTNCompressor(DataDrivenCompressor):# shard_writer(self, is_finalize=True)def _quant_rtn_with_imatrix(self) -> None:- """Performs RTN quantization using input activation statistics (imatrix).-- OptimizedRTNQuantizer owns imatrix hook registration. This method only- enables the quantizer-side collection path and keeps the OOM fallback.-- Returns:- None- """logger.info("start to compute imatrix")self.quantizer.enable_imatrix = True@@ -1097,7 +1225,6 @@ class CalibratedRTNCompressor(DataDrivenCompressor):if hasattr(model, "hf_device_map") and len(model.hf_device_map) > 1:dispatch_model(model, model.hf_device_map)- hooks = self.quantizer.register_calibration_hooks(model, act_max=False)try:if hasattr(model, "hf_device_map") and len(model.hf_device_map) > 1:import accelerate@@ -1110,7 +1237,6 @@ class CalibratedRTNCompressor(DataDrivenCompressor):cuda_error_msg = traceback.format_exc()try:logger.error(cuda_error_msg)- # Final fallback: warn and use CPU-only quantizationlogger.warning("Fallback to CPU. ""Consider enabling `low_gpu_mem_usage` or using more GPUs via `--device 0,1,2,3`."@@ -1135,8 +1261,6 @@ class CalibratedRTNCompressor(DataDrivenCompressor):except Exception as e:raisefinally:- for hook in hooks:- hook.remove()self.quantizer.enable_imatrix = Falsedef quantize(self):@@ -1160,7 +1284,7 @@ class CalibratedRTNCompressor(DataDrivenCompressor):formats = getattr(self, "formats", None) or []if not (any(fmt.is_gguf() for fmt in formats) or self.super_bits is not None):- self._quantize_embedding_layer() # leave to gguf itself to handle+ self.quantizer.quantize_embedding_layer() # leave to gguf itself to handle# Release memoryclear_memory(device_list=device_manager.device_list)diff --git a/auto_round/compressors/diffusion/dataset.py b/auto_round/compressors/diffusion/dataset.pyindex b3ee20d34..477c088a8 100644--- a/auto_round/compressors/diffusion/dataset.py+++ b/auto_round/compressors/diffusion/dataset.py@@ -13,7 +13,8 @@# limitations under the License.import os-from typing import Dict+from io import StringIO+from typing import Dict, Optionalimport pandas as pdimport torch@@ -58,20 +59,30 @@ class Text2ImgDataset(Dataset):def __init__(self,- dataset_path,- nsamples=128,+ dataset_path: str,+ nsamples: int = 128,+ dataframe: Optional[pd.DataFrame] = None,) -> None:super().__init__()self.captions = []self.caption_ids = []- logger.info(f"use dataset {dataset_path}, loading from disk...")- df = pd.read_csv(dataset_path, sep="\t")+ if dataframe is not None:+ df = dataframe+ else:+ logger.info(f"use dataset {dataset_path}, loading from disk...")+ df = pd.read_csv(dataset_path, sep="\t")++ required_cols = {"id", "caption"}+ if not required_cols.issubset(df.columns):+ raise ValueError(+ f"Invalid diffusion caption data from {dataset_path!r}: "+ f"expected columns {sorted(required_cols)}, got {list(df.columns)}"+ )for index, row in df.iterrows():if nsamples > 0 and index + 1 > nsamples:break- assert "id" in row and "caption" in rowcaption_id = row["id"]caption_text = row["caption"]self.caption_ids.append(caption_id)@@ -93,7 +104,7 @@ class AudioCapsDataset(Dataset):``audiocap_id`` as the sample id and ``caption`` as the text prompt."""- def __init__(self, dataset_path, nsamples=128) -> None:+ def __init__(self, dataset_path: str, nsamples: int = 128) -> None:super().__init__()self.captions = []self.caption_ids = []@@ -140,15 +151,17 @@ def get_diffusion_dataloader(import requestslogger.info(f"use dataset {dataset}, downloading ...")- text_data = requests.get(COCO_URL[dataset]).text- with open("captions_source.tsv", "w") as f:- f.write(text_data)- dataset = "captions_source.tsv"+ response = requests.get(COCO_URL[dataset], timeout=30)+ response.raise_for_status()+ dataframe = pd.read_csv(StringIO(response.text), sep="\t")+ dataset = DIFFUSION_DATASET["local"](dataset, nsamples, dataframe=dataframe)if dataset in ("audiocaps",):dataset = download_audiocaps_csv()- if isinstance(dataset, str) and os.path.exists(dataset):+ if isinstance(dataset, Dataset):+ pass+ elif isinstance(dataset, str) and os.path.exists(dataset):if dataset.endswith(".csv"):dataset = DIFFUSION_DATASET["audiocaps"](dataset, nsamples)else:diff --git a/auto_round/compressors/diffusion_mixin.py b/auto_round/compressors/diffusion_mixin.pyindex fa42db945..a693d9df8 100644--- a/auto_round/compressors/diffusion_mixin.py+++ b/auto_round/compressors/diffusion_mixin.py@@ -13,7 +13,7 @@# limitations under the License.import inspectimport os-from typing import Union+from typing import Any, Optional, Unionimport torchfrom tqdm import tqdm@@ -54,7 +54,14 @@ class DiffusionMixin:saving so that ``model_context`` remains the single source of truth."""- def __init__(self, *args, guidance_scale=7.5, num_inference_steps=50, generator_seed=None, **kwargs):+ def __init__(+ self,+ *args,+ guidance_scale: float = 7.5,+ num_inference_steps: int = 50,+ generator_seed: Optional[int] = None,+ **kwargs,+ ) -> None:# Store diffusion-specific attributesself.guidance_scale = guidance_scaleself.num_inference_steps = num_inference_steps@@ -235,7 +242,7 @@ class DiffusionMixin:pipe.to(target_device)@torch.no_grad()- def calib(self, nsamples, bs):+ def calib(self, nsamples: int, bs: int) -> None:"""Perform diffusion-specific calibration for quantization.Override parent's calib method to use diffusion dataset loading logic.@@ -348,7 +355,7 @@ class DiffusionMixin:# torch.cuda.empty_cache()- def try_cache_inter_data_gpucpu(self, *args, **kwargs):+ def try_cache_inter_data_gpucpu(self, *args, **kwargs) -> Any:"""Skip re-caching when DiffusionMixin.quantize has already populated self.inputs.CalibCompressor.quantize() always calls try_cache_inter_data_gpucpu, but for@@ -361,7 +368,7 @@ class DiffusionMixin:if hasattr(super(), "try_cache_inter_data_gpucpu"):return super().try_cache_inter_data_gpucpu(*args, **kwargs)- def quantize(self):+ def quantize(self) -> tuple[torch.nn.Module, dict]:"""Quantize the diffusion model.Overrides the parent to use diffusion-specific cache_inter_data instead of@@ -524,7 +531,14 @@ class DiffusionMixin:return self.model_context.model, self.quantizer.layer_config- def save_quantized(self, output_dir=None, format="auto_round", inplace=True, return_folders=False, **kwargs):+ def save_quantized(+ self,+ output_dir: Optional[str] = None,+ format: Union[str, list] = "auto_round",+ inplace: bool = True,+ return_folders: bool = False,+ **kwargs,+ ) -> Any:"""Save the quantized model to the specified output directory in the specified format.For multi-transformer pipelines, all quantized transformers are saved.diff --git a/auto_round/compressors/entry.py b/auto_round/compressors/entry.pyindex 94b03d1bf..ca6407420 100644--- a/auto_round/compressors/entry.py+++ b/auto_round/compressors/entry.py@@ -6,19 +6,91 @@ from typing import Any, Callable, Optional, Unionimport torch-from auto_round.algorithms.alg_config import AlgConfig-from auto_round.algorithms.quantization.awq.config import AWQConfig-from auto_round.algorithms.quantization.rtn.config import RTNConfig+from auto_round.algorithms.config_resolver import split_quantization_configs+from auto_round.algorithms.quantization.config import QuantizationConfig+from auto_round.algorithms.quantization.rtn.config import OptimizedRTNConfig, RTNConfigfrom auto_round.algorithms.quantization.sign_round.config import SignRoundConfig-from auto_round.algorithms.transforms import normalize_rotation_config as _normalize_any_rotation_config-from auto_round.algorithms.transforms.base import BaseRotationConfig as _BaseRotationConfig-from auto_round.algorithms.transforms.rotation.config import RotationConfig as _NewArchRotationConfig+from auto_round.algorithms.registry import normalize_algorithm_config, resolve_alg_config+from auto_round.algorithms.transforms import normalize_rotation_config as _normalize_rotation_alg_config+from auto_round.algorithms.transforms.awq.config import AWQConfig+from auto_round.algorithms.transforms.quarot.config import RotationConfig as _NewArchRotationConfigfrom auto_round.auto_scheme.gen_auto_scheme import AutoScheme+from auto_round.compressors.base import BaseCompressorfrom auto_round.compressors.data_driven import CalibratedRTNCompressor, DataDrivenCompressorfrom auto_round.compressors.utils import check_need_act_calibrationfrom auto_round.compressors.zero_shot import ZeroShotCompressorfrom auto_round.logger import loggerfrom auto_round.schemes import QuantizationScheme, parse_scheme+from auto_round.utils.device_manager import normalize_default_device_map++_ENTRY_ROUTE_KWARGS = {"model_free", "disable_model_free", "disable_opt_rtn"}+_ENTRY_COMPRESSOR_KWARGS = {"scale_dtype", "ignore_layers", "quant_lm_head", "to_quant_block_names"}+_ENTRY_BASE_KWARGS = {+ "format",+ "dataset",+ "batch_size",+ "model_dtype",+ "trust_remote_code",+ "amp",+ "nblocks",+ "disable_deterministic_algorithms",+ "enable_deterministic_algorithms",+ "static_kv_dtype",+ "static_attention_dtype",+}+_ENTRY_MLLM_KWARGS = {"processor", "image_processor", "template", "extra_data_dir", "quant_nontext_module"}+_ENTRY_DIFFUSION_KWARGS = {"guidance_scale", "num_inference_steps", "generator_seed"}+_ENTRY_ALLOWED_KWARGS = (+ _ENTRY_ROUTE_KWARGS | _ENTRY_COMPRESSOR_KWARGS | _ENTRY_BASE_KWARGS | _ENTRY_MLLM_KWARGS | _ENTRY_DIFFUSION_KWARGS+)+++def filter_supported_entry_kwargs(kwargs: dict[str, Any], *, context: str) -> dict[str, Any]:+ """Return only kwargs supported by the new entry API.++ Unsupported kwargs are ignored with a warning so callers can cleanly migrate+ without leaking old-API parameters into compressor constructors.+ """++ supported = {}+ unknown = []+ for key, value in kwargs.items():+ if key in _ENTRY_ALLOWED_KWARGS:+ supported[key] = value+ else:+ unknown.append(key)+ if unknown:+ logger.warning_once(+ "%s received unsupported kwargs %s. They will be ignored.",+ context,+ ", ".join(sorted(unknown)),+ )+ return supported+++def _split_entry_kwargs(kwargs: dict[str, Any]) -> dict[str, dict[str, Any]]:+ """Partition new-entry kwargs by ownership."""++ kwargs = filter_supported_entry_kwargs(kwargs, context="AutoRound entry")+ buckets = {+ "route": {},+ "compressor": {},+ "base": {},+ "mllm": {},+ "diffusion": {},+ }+ for key, value in kwargs.items():+ if key in _ENTRY_ROUTE_KWARGS:+ buckets["route"][key] = value+ elif key in _ENTRY_COMPRESSOR_KWARGS:+ buckets["compressor"][key] = value+ elif key in _ENTRY_BASE_KWARGS:+ buckets["base"][key] = value+ elif key in _ENTRY_MLLM_KWARGS:+ buckets["mllm"][key] = value+ elif key in _ENTRY_DIFFUSION_KWARGS:+ buckets["diffusion"][key] = value+ return bucketsdef _preview_resolved_attrs(config, scheme=None) -> dict:@@ -36,7 +108,7 @@ def _preview_resolved_attrs(config, scheme=None) -> dict:if isinstance(scheme, AutoScheme):# AutoScheme needs model info — cannot preview, rely on raw config attrsreturn {}- scheme_attr_names = QuantizationScheme.get_attributes()+ scheme_attr_names = tuple(config._scheme_fields)user_overrides = {k: getattr(config, k) for k in scheme_attr_names if getattr(config, k, None) is not None}try:_, _, final_attrs = parse_scheme(scheme, user_overrides)@@ -59,7 +131,7 @@ def _eager_validate_scheme(config, scheme=None) -> None:if isinstance(scheme, AutoScheme):return- scheme_attr_names = QuantizationScheme.get_attributes()+ scheme_attr_names = tuple(config._scheme_fields)user_overrides = {k: getattr(config, k) for k in scheme_attr_names if getattr(config, k, None) is not None}try:_, _, final_attrs = parse_scheme(scheme, user_overrides)@@ -71,6 +143,9 @@ def _eager_validate_scheme(config, scheme=None) -> None:import copytemp_config = copy.copy(config)+ if hasattr(config, "scheme"):+ temp_config.scheme = config.scheme.copy()+ temp_config._user_set_scheme_fields = set(getattr(config, "_user_set_scheme_fields", set()))for key, value in final_attrs.items():setattr(temp_config, key, value)temp_config.check_config() # raises ValueError / NotImplementedError if invalid@@ -111,7 +186,7 @@ def _get_compressor_class(model_type: str, base_cls: type) -> type:return combined-def is_weight_scheme(scheme):+def is_weight_scheme(scheme: Union[str, dict, AutoScheme, object]) -> bool:if isinstance(scheme, str):return scheme.upper().startswith("W")if isinstance(scheme, dict):@@ -125,7 +200,7 @@ def is_weight_scheme(scheme):return False-def is_gguf_k_target(value) -> bool:+def is_gguf_k_target(value: Union[str, AutoScheme, object]) -> bool:if isinstance(value, str):normalized = value.strip().lower()return normalized.startswith("gguf:") and "_k" in normalized@@ -138,38 +213,118 @@ def is_gguf_k_target(value) -> bool:return False+def _resolve_quant_config_for_routing(alg_configs) -> tuple[list, list, QuantizationConfig]:+ preprocessor_configs, block_quant_configs = split_quantization_configs(alg_configs)+ if len(block_quant_configs) == 0 and preprocessor_configs:+ from auto_round.algorithms.quantization.rtn.config import RTNConfig as _RTNConfig++ return preprocessor_configs, block_quant_configs, _RTNConfig()+ if len(block_quant_configs) > 1:+ raise ValueError(+ f"Only one block-quantization config is allowed, but got {len(block_quant_configs)}: "+ f"{[type(c).__name__ for c in block_quant_configs]}"+ )+ if len(block_quant_configs) == 1:+ return preprocessor_configs, block_quant_configs, block_quant_configs[0]+ raise ValueError(+ "At least one quantization algorithm config is required. "+ "Pass a block quantizer such as RTNConfig or SignRoundConfig, "+ "or a quantization preprocessor such as AWQConfig."+ )+++def _build_model_type_ctor_kwargs(model, base_kwargs, mllm_kwargs, diffusion_kwargs) -> tuple[str, dict[str, Any]]:+ from auto_round.utils.model import detect_model_type++ model_type = detect_model_type(model)+ has_multimodal_assets = mllm_kwargs.get("processor") is not None or mllm_kwargs.get("image_processor") is not None+ if has_multimodal_assets and model_type != "mllm":+ model_type = "mllm"++ ctor_kwargs = dict(base_kwargs)+ if model_type == "mllm":+ ctor_kwargs.update(mllm_kwargs)+ if model_type == "diffusion":+ ctor_kwargs.update(diffusion_kwargs)+ return model_type, ctor_kwargs+++def _select_rtn_compressor_base_cls(quant_config: RTNConfig, scheme, format, base_kwargs) -> type:+ enable_imatrix = False+ resolved_attrs = {}+ disable_opt_rtn = getattr(quant_config, "disable_opt_rtn", False)++ # If disable_opt_rtn was not explicitly set and scheme is W8A16/W8A8,+ # auto-disable optimization to improve efficiency.+ if getattr(quant_config, "orig_disable_opt_rtn", None) is None:+ if isinstance(scheme, str) and scheme.upper() in ["W8A16", "W8A8"]:+ logger.warning("`disable_opt_rtn` is turned on for W8A16/W8A8 quantization to improve efficiency.")+ disable_opt_rtn = True+ quant_config.disable_opt_rtn = True++ if not disable_opt_rtn:+ has_gguf_k = is_gguf_k_target(format) or is_gguf_k_target(scheme)+ if has_gguf_k:+ enable_imatrix = True+ else:+ # Resolve scheme attrs for routing. SchemeMixin will do the authoritative+ # resolution later; this preview only chooses the compressor class.+ resolved_attrs = _preview_resolved_attrs(quant_config, scheme)+ sym = resolved_attrs.get("sym", getattr(quant_config, "sym", None))+ data_type = resolved_attrs.get("data_type", getattr(quant_config, "data_type", "") or "")+ bits = resolved_attrs.get("bits", getattr(quant_config, "bits", None))+ if sym is not None and sym is False:+ enable_imatrix = False+ elif data_type == "int" and (bits is None or bits < 8):+ enable_imatrix = True+ elif is_weight_scheme(scheme):+ enable_imatrix = True++ resolved_attrs = resolved_attrs if not disable_opt_rtn else _preview_resolved_attrs(quant_config, scheme)+ act_bits = resolved_attrs.get("act_bits", getattr(quant_config, "act_bits", None))+ act_data_type = resolved_attrs.get("act_data_type", getattr(quant_config, "act_data_type", None))+ act_dynamic = resolved_attrs.get("act_dynamic", getattr(quant_config, "act_dynamic", None))+ is_act_quantize = act_bits is not None and act_bits <= 8+ needs_act_calib = is_act_quantize and check_need_act_calibration(+ act_dynamic,+ act_data_type,+ act_bits if act_bits is not None else 16,+ static_kv_dtype=base_kwargs.get("static_kv_dtype"),+ static_attention_dtype=base_kwargs.get("static_attention_dtype"),+ )++ # AutoScheme always requires calibration data for delta-loss based scheme+ # selection, regardless of whether imatrix is needed.+ quant_config.enable_imatrix = enable_imatrix+ if enable_imatrix or needs_act_calib or isinstance(scheme, AutoScheme):+ if not isinstance(quant_config, OptimizedRTNConfig):+ quant_config.__class__ = OptimizedRTNConfig+ return CalibratedRTNCompressor++ if isinstance(quant_config, OptimizedRTNConfig):+ quant_config.__class__ = RTNConfig+ return ZeroShotCompressor++class AutoRound(object):- # Mapping from string alias to config class (and optional defaults override).- _CONFIG_ALIASES: dict[str, type] = {- "sign_round": SignRoundConfig,- "signround": SignRoundConfig,- "rtn": RTNConfig,- "hadamard": _NewArchRotationConfig,- }@classmethod- def _resolve_config(cls, config: Union[str, AlgConfig, list]) -> Union[AlgConfig, list[AlgConfig]]:+ def _resolve_config(cls, config: Union[str, object, list]) -> Union[object, list[object]]:"""Convert string alias(es) to the corresponding config instance(s) with default parameters."""if isinstance(config, str):- key = config.strip().lower()- # Handle spinquant/quarot via unified normalizer- if key in ("spinquant", "quarot"):- return _normalize_any_rotation_config(key)- if key not in cls._CONFIG_ALIASES:- raise ValueError(f"Unknown config alias '{config}'. " f"Supported: {list(cls._CONFIG_ALIASES.keys())}")- return cls._CONFIG_ALIASES[key]()+ return resolve_alg_config(config)if isinstance(config, list):return [cls._resolve_config(c) for c in config]return configdef __new__(cls,- alg_configs: Union[str, AlgConfig, list[Union[str, AlgConfig]]],model: Union[torch.nn.Module, str],+ scheme="W4A16",+ alg_configs: Union[str, object, list[Union[str, object]]] = None,tokenizer=None,platform="hf",format=None,- scheme="W4A16",low_gpu_mem_usage: bool = False,device_map: Union[str, torch.device, int, dict] = 0,iters: int = None,@@ -181,34 +336,61 @@ class AutoRound(object):nsamples: int = None,seqlen: int = None,**kwargs,- ):- from auto_round.algorithms.quantization.config import QuantizationConfig+ ) -> "BaseCompressor":+ from auto_round.utils.model import is_model_free_route++ device_map = normalize_default_device_map(device_map)+ split_kwargs = _split_entry_kwargs(kwargs)+ route_kwargs = dict(split_kwargs["route"])+ compressor_kwargs = dict(split_kwargs["compressor"])+ base_kwargs = dict(split_kwargs["base"])+ mllm_kwargs = dict(split_kwargs["mllm"])+ diffusion_kwargs = dict(split_kwargs["diffusion"])++ if alg_configs is None:+ alg_configs = "auto_round"# Resolve string alias(es) to config instance(s) before routing.alg_configs = cls._resolve_config(alg_configs)-- # Extract the single QuantizationConfig from a list; validate at most one exists.if isinstance(alg_configs, list):- quant_configs = [c for c in alg_configs if isinstance(c, QuantizationConfig)]- if len(quant_configs) == 0:- raise ValueError("At least one QuantizationConfig (SignRoundConfig / RTNConfig) is required.")- if len(quant_configs) > 1:- raise ValueError(- f"Only one QuantizationConfig is allowed, but got {len(quant_configs)}: "- f"{[type(c).__name__ for c in quant_configs]}"- )- quant_config = quant_configs[0]+ alg_configs = [normalize_algorithm_config(cfg) for cfg in alg_configs]else:- quant_config = alg_configs+ alg_configs = normalize_algorithm_config(alg_configs)+ configs_for_routing = alg_configs if isinstance(alg_configs, list) else [alg_configs]+ preprocessor_configs, _, quant_config = _resolve_quant_config_for_routing(configs_for_routing)++ # Model-free routing is now supported directly by the new entry path.+ model_free_iters = 0 if isinstance(quant_config, RTNConfig) else getattr(quant_config, "iters", None)+ model_free_disable_opt_rtn = getattr(quant_config, "disable_opt_rtn", None)+ if is_model_free_route(model, scheme, model_free_iters, model_free_disable_opt_rtn, route_kwargs):+ from auto_round.compressors.model_free import ModelFreeCompressor++ if not isinstance(model, str):+ raise ValueError("model_free=True requires `model` to be a HuggingFace ID or local path string.")+ if not bool(route_kwargs.get("model_free", False)):+ logger.info(+ "Auto-routing to model-free quantization "+ "(iters=0, disable_opt_rtn=True, supported scheme). "+ "Pass disable_model_free=True to use the regular flow."+ )+ return ModelFreeCompressor(+ model_name_or_path=model,+ scheme=scheme,+ layer_config=layer_config,+ tokenizer=tokenizer,+ device_map=device_map,+ **compressor_kwargs,+ **base_kwargs,+ **mllm_kwargs,+ **diffusion_kwargs,+ **route_kwargs,+ )# Eagerly validate scheme constraints that do not require model info.# This mirrors old-arch _check_configs() called at __init__ time so that# callers get ValueError/NotImplementedError on construction, not deferred._eager_validate_scheme(quant_config, scheme)- # Explicitly build the dict of constructor args to forward to the- # compressor. This avoids the fragile locals()-based approach that- # required a growing SKIP_ARGS blocklist.local_args = dict(model=model,tokenizer=tokenizer,@@ -225,94 +407,25 @@ class AutoRound(object):layer_config=layer_config,nsamples=nsamples,seqlen=seqlen,+ **compressor_kwargs,)+ model_type, ctor_kwargs = _build_model_type_ctor_kwargs(model, base_kwargs, mllm_kwargs, diffusion_kwargs)- # Detect model type to determine if we need special compressor- from auto_round.utils.model import detect_model_type-- model_type = detect_model_type(model)-- # If the user explicitly passes processor/image_processor, treat as MLLM even if- # auto-detection missed it (mirrors the has_multimodal_assets check in autoround.py).- has_multimodal_assets = kwargs.get("processor") is not None or kwargs.get("image_processor") is not None- if has_multimodal_assets and model_type != "mllm":- model_type = "mllm"-- # Pop kwargs that are only consumed by specific Mixins so they don't- # leak through to BaseCompressor as unrecognized keys.- if model_type != "diffusion":- for _k in ("guidance_scale", "num_inference_steps", "generator_seed"):- kwargs.pop(_k, None)- if model_type != "mllm":- for _k in ("processor", "image_processor", "template", "extra_data_dir", "quant_nontext_module"):- kwargs.pop(_k, None)- kwargs.pop("disable_opt_rtn", None) # consumed by RTN routing above, not a compressor param+ # Preprocessor algorithms (AWQ, …) require a data-driven host so that+ # the per-block preprocessor lifecycle (prepare_block_group ->+ # block_forward_hooks -> pre_quantize_block -> pre_quantize_block ->+ # post_quantize_block) actually runs. CalibratedRTNCompressor's+ # Preprocessor algorithms require DataDrivenCompressor for per-block lifecycle hooks.+ # The pipeline auto-appends RTN when no block_quantizer is supplied.+ if preprocessor_configs:+ return _get_compressor_class(model_type, DataDrivenCompressor)(alg_configs, **local_args, **ctor_kwargs)if isinstance(quant_config, SignRoundConfig):- return _get_compressor_class(model_type, DataDrivenCompressor)(alg_configs, **local_args, **kwargs)-- elif isinstance(quant_config, AWQConfig):- # AWQ requires calibration for activation collection + smoothing- quant_config._alg_cls = "AWQQuantizer"- return _get_compressor_class(model_type, CalibratedRTNCompressor)(alg_configs, **local_args, **kwargs)+ return _get_compressor_class(model_type, DataDrivenCompressor)(alg_configs, **local_args, **ctor_kwargs)elif isinstance(quant_config, RTNConfig):- enable_imatrix = False- _resolved = {}- disable_opt_rtn = getattr(quant_config, "disable_opt_rtn", False)- # If disable_opt_rtn was not explicitly set and scheme is W8A16/W8A8,- # auto-disable optimization to improve efficiency.- if getattr(quant_config, "orig_disable_opt_rtn", None) is None:- if isinstance(scheme, str) and scheme.upper() in ["W8A16", "W8A8"]:- logger.warning("`disable_opt_rtn` is turned on for W8A16/W8A8 quantization to improve efficiency.")- disable_opt_rtn = True- quant_config.disable_opt_rtn = True- if not disable_opt_rtn:- has_gguf_k = is_gguf_k_target(format) or is_gguf_k_target(scheme)- if has_gguf_k:- enable_imatrix = True- else:- # Resolve scheme attrs for routing (config hasn't been through- # SchemeMixin yet; user may have specified only scheme="W4A16").- _resolved = _preview_resolved_attrs(quant_config, scheme)- _sym = _resolved.get("sym", getattr(quant_config, "sym", None))- _data_type = _resolved.get("data_type", getattr(quant_config, "data_type", "") or "")- _bits = _resolved.get("bits", getattr(quant_config, "bits", None))- if _sym is not None and _sym is False:- enable_imatrix = False- elif _data_type == "int" and (_bits is None or _bits < 8):- enable_imatrix = True- elif is_weight_scheme(scheme):- enable_imatrix = True- else:- _resolved = {}-- _resolved = _resolved if not disable_opt_rtn else _preview_resolved_attrs(quant_config, scheme)- _act_bits = _resolved.get("act_bits", getattr(quant_config, "act_bits", None))- _act_data_type = _resolved.get("act_data_type", getattr(quant_config, "act_data_type", None))- _act_dynamic = _resolved.get("act_dynamic", getattr(quant_config, "act_dynamic", None))- _is_act_quantize = _act_bits is not None and _act_bits <= 8- needs_act_calib = _is_act_quantize and check_need_act_calibration(- _act_dynamic,- _act_data_type,- _act_bits if _act_bits is not None else 16,- static_kv_dtype=kwargs.get("static_kv_dtype"),- static_attention_dtype=kwargs.get("static_attention_dtype"),- )-- # AutoScheme always requires calibration data for delta-loss based- # scheme selection, regardless of whether imatrix is needed.- from auto_round.auto_scheme.gen_auto_scheme import AutoScheme as _AutoScheme-- is_auto_scheme = isinstance(scheme, _AutoScheme)- quant_config.enable_imatrix = enable_imatrix-- if enable_imatrix or needs_act_calib or is_auto_scheme:- quant_config._alg_cls = "OptimizedRTNQuantizer"- return _get_compressor_class(model_type, CalibratedRTNCompressor)(alg_configs, **local_args, **kwargs)- else:- quant_config._alg_cls = "RTNQuantizer"- return _get_compressor_class(model_type, ZeroShotCompressor)(alg_configs, **local_args, **kwargs)+ base_cls = _select_rtn_compressor_base_cls(quant_config, scheme, format, base_kwargs)+ return _get_compressor_class(model_type, base_cls)(alg_configs, **local_args, **ctor_kwargs)class AutoRoundCompatible:@@ -369,14 +482,7 @@ class AutoRoundCompatible:@staticmethoddef _pop_config_kwargs(kwargs: dict[str, Any]) -> tuple[dict[str, Any], dict[str, Any]]:"""Extract old-API config kwargs and split them by config type."""- common_keys = (- "ignore_layers",- "quant_lm_head",- "scale_dtype",- "super_bits",- "super_group_size",- "to_quant_block_names",- )+ common_keys = ("super_bits", "super_group_size")auto_round_only_keys = ("nblocks","enable_alg_ext",@@ -397,6 +503,157 @@ class AutoRoundCompatible:auto_round_kwargs[key] = kwargs.pop(key)return common_kwargs, auto_round_kwargs+ @staticmethod+ def _pop_compressor_only_kwargs(kwargs: dict[str, Any]) -> dict[str, Any]:+ return {+ "scale_dtype": kwargs.pop("scale_dtype", None),+ "ignore_layers": kwargs.pop("ignore_layers", ""),+ "quant_lm_head": kwargs.pop("quant_lm_head", False),+ "to_quant_block_names": kwargs.pop("to_quant_block_names", None),+ }++ @staticmethod+ def _resolve_compat_algorithm(algorithm, iters) -> str:+ if algorithm and algorithm.lower() == "awq":+ return "awq"+ if (algorithm and algorithm.lower() == "rtn") or iters == 0:+ return "rtn"+ return "signround"++ @staticmethod+ def _pop_shared_quant_kwargs(kwargs: dict[str, Any]) -> dict[str, Any]:+ return {+ "bits": kwargs.pop("bits", None),+ "group_size": kwargs.pop("group_size", None),+ "sym": kwargs.pop("sym", None),+ "data_type": kwargs.pop("data_type", None),+ "act_bits": kwargs.pop("act_bits", None),+ "act_group_size": kwargs.pop("act_group_size", None),+ "act_sym": kwargs.pop("act_sym", None),+ "act_data_type": kwargs.pop("act_data_type", None),+ "act_dynamic": kwargs.pop("act_dynamic", None),+ }++ @staticmethod+ def _build_awq_config(+ shared_quant_kwargs: dict[str, Any],+ *,+ seqlen,+ nsamples,+ batch_size,+ kwargs,+ common_config_kwargs,+ ):+ return AWQConfig(+ **shared_quant_kwargs,+ duo_scaling=kwargs.pop("duo_scaling", True),+ n_grid=kwargs.pop("n_grid", 20),+ seqlen=seqlen,+ nsamples=nsamples,+ batch_size=batch_size,+ mappings=kwargs.pop("mappings", None),+ **common_config_kwargs,+ )++ @staticmethod+ def _build_rtn_config(shared_quant_kwargs: dict[str, Any], *, kwargs, common_config_kwargs):+ cfg = RTNConfig(+ **shared_quant_kwargs,+ disable_opt_rtn=kwargs.pop("disable_opt_rtn", None),+ enable_opt_rtn=kwargs.pop("enable_opt_rtn", None),+ **common_config_kwargs,+ )+ return normalize_algorithm_config(cfg)++ @staticmethod+ def _build_signround_config(+ shared_quant_kwargs: dict[str, Any],+ *,+ iters,+ gradient_accumulate_steps,+ kwargs,+ common_config_kwargs,+ auto_round_config_kwargs,+ ):+ cfg = SignRoundConfig(+ **shared_quant_kwargs,+ iters=iters,+ gradient_accumulate_steps=gradient_accumulate_steps,+ lr=kwargs.pop("lr", None),+ minmax_lr=kwargs.pop("minmax_lr", None),+ enable_minmax_tuning=kwargs.pop("enable_minmax_tuning", True),+ enable_norm_bias_tuning=kwargs.pop("enable_norm_bias_tuning", False),+ enable_quanted_input=kwargs.pop("enable_quanted_input", True),+ **common_config_kwargs,+ **auto_round_config_kwargs,+ )+ return normalize_algorithm_config(cfg)++ @classmethod+ def _build_alg_config(+ cls,+ *,+ algorithm,+ iters,+ gradient_accumulate_steps,+ seqlen,+ nsamples,+ batch_size,+ kwargs,+ common_config_kwargs,+ auto_round_config_kwargs,+ ):+ alg_name = cls._resolve_compat_algorithm(algorithm, iters)+ shared_quant_kwargs = cls._pop_shared_quant_kwargs(kwargs)++ if alg_name == "awq":+ return cls._build_awq_config(+ shared_quant_kwargs,+ seqlen=seqlen,+ nsamples=nsamples,+ batch_size=batch_size,+ kwargs=kwargs,+ common_config_kwargs=common_config_kwargs,+ )+ if alg_name == "rtn":+ return cls._build_rtn_config(+ shared_quant_kwargs,+ kwargs=kwargs,+ common_config_kwargs=common_config_kwargs,+ )+ return cls._build_signround_config(+ shared_quant_kwargs,+ iters=iters,+ gradient_accumulate_steps=gradient_accumulate_steps,+ kwargs=kwargs,+ common_config_kwargs=common_config_kwargs,+ auto_round_config_kwargs=auto_round_config_kwargs,+ )++ @staticmethod+ def _build_entry_forward_kwargs(kwargs: dict[str, Any]) -> dict[str, Any]:+ format_name = kwargs.pop("format", None)+ rotation_config = kwargs.pop("rotation_config", None)+ mllm_kwargs = {+ "processor": kwargs.pop("processor", None),+ "image_processor": kwargs.pop("image_processor", None),+ "template": kwargs.pop("template", None),+ "extra_data_dir": kwargs.pop("extra_data_dir", None),+ "quant_nontext_module": kwargs.pop("quant_nontext_module", False),+ }+ diffusion_kwargs = {+ "guidance_scale": kwargs.pop("guidance_scale", 7.5),+ "num_inference_steps": kwargs.pop("num_inference_steps", 50),+ "generator_seed": kwargs.pop("generator_seed", None),+ }+ return {+ "format": format_name,+ "rotation_config": rotation_config,+ **mllm_kwargs,+ **diffusion_kwargs,+ **kwargs,+ }+def __new__(cls,model: Union[torch.nn.Module, str],@@ -417,16 +674,10 @@ class AutoRoundCompatible:low_cpu_mem_usage: bool = True,algorithm: str = None,**kwargs,- ):+ ) -> "BaseCompressor":"""Create AutoRoundCompatible instance using new AutoRound architecture.This method translates old AutoRoundCompatible API to new AutoRound API.-- Args:- algorithm: Quantization algorithm to use. Options:- - None or "auto_round": SignSGD-based optimization (default when iters > 0)- - "rtn": Round-to-nearest (default when iters == 0)- - "awq": Activation-Aware Weight Quantization (AWQ smoothing + RTN)"""from auto_round.utils import is_diffusion_model, is_mllm_modelfrom auto_round.utils.model import is_model_free_route@@ -441,6 +692,8 @@ class AutoRoundCompatible:if is_model_free_route(model, scheme, iters, kwargs.get("disable_opt_rtn"), kwargs):from auto_round.compressors.model_free import ModelFreeCompressor+ compressor_only_kwargs = cls._pop_compressor_only_kwargs(kwargs)+if not isinstance(model, str):raise ValueError("model_free=True requires `model` to be a HuggingFace ID or local path string.")if not bool(kwargs.get("model_free", False)):@@ -455,131 +708,34 @@ class AutoRoundCompatible:layer_config=layer_config,tokenizer=tokenizer,device_map=device_map,+ **compressor_only_kwargs,**kwargs,)# --------------------------------------------------------------------+ compressor_only_kwargs = cls._pop_compressor_only_kwargs(kwargs)common_config_kwargs, auto_round_config_kwargs = cls._pop_config_kwargs(kwargs)- # Extract quantization parameters from kwargs or use defaults- bits = kwargs.pop("bits", None)- group_size = kwargs.pop("group_size", None)- sym = kwargs.pop("sym", None)- data_type = kwargs.pop("data_type", None)- act_bits = kwargs.pop("act_bits", None)- act_group_size = kwargs.pop("act_group_size", None)- act_sym = kwargs.pop("act_sym", None)- act_data_type = kwargs.pop("act_data_type", None)- act_dynamic = kwargs.pop("act_dynamic", None)- enable_opt_rtn = kwargs.pop("enable_opt_rtn", None)- lr = kwargs.pop("lr", None)- minmax_lr = kwargs.pop("minmax_lr", None)- enable_minmax_tuning = kwargs.pop("enable_minmax_tuning", True)- enable_norm_bias_tuning = kwargs.pop("enable_norm_bias_tuning", False)- enable_quanted_input = kwargs.pop("enable_quanted_input", True)-- # Pop AWQ-only kwargs early so they don't leak into non-AWQ constructors- duo_scaling = kwargs.pop("duo_scaling", True)- n_grid = kwargs.pop("n_grid", 20)- awq_mappings = kwargs.pop("mappings", None)-- # Decide which algorithm to use- if algorithm and algorithm.lower() == "awq":- # AWQ mode: activation-aware weight quantization- config = AWQConfig(- bits=bits,- group_size=group_size,- sym=sym,- data_type=data_type,- act_bits=act_bits,- act_group_size=act_group_size,- act_sym=act_sym,- act_data_type=act_data_type,- act_dynamic=act_dynamic,- duo_scaling=duo_scaling,- n_grid=n_grid,- seqlen=seqlen,- nsamples=nsamples,- batch_size=batch_size,- mappings=awq_mappings,- **common_config_kwargs,- )- elif (algorithm and algorithm.lower() == "rtn") or iters == 0:- # RTN mode- disable_opt_rtn = kwargs.pop("disable_opt_rtn", None)- config = RTNConfig(- bits=bits,- group_size=group_size,- sym=sym,- data_type=data_type,- act_bits=act_bits,- act_group_size=act_group_size,- act_sym=act_sym,- act_data_type=act_data_type,- act_dynamic=act_dynamic,- disable_opt_rtn=disable_opt_rtn,- enable_opt_rtn=enable_opt_rtn,- **common_config_kwargs,- )- else:- # AutoRoundCompatible mode- config = SignRoundConfig(- iters=iters,- gradient_accumulate_steps=gradient_accumulate_steps,- bits=bits,- group_size=group_size,- sym=sym,- data_type=data_type,- act_bits=act_bits,- act_group_size=act_group_size,- act_sym=act_sym,- act_data_type=act_data_type,- act_dynamic=act_dynamic,- lr=lr,- minmax_lr=minmax_lr,- enable_minmax_tuning=enable_minmax_tuning,- enable_norm_bias_tuning=enable_norm_bias_tuning,- enable_quanted_input=enable_quanted_input,- **common_config_kwargs,- **auto_round_config_kwargs,- )-- # Determine output format if specified- format = kwargs.pop("format", None)+ config = cls._build_alg_config(+ algorithm=algorithm,+ iters=iters,+ gradient_accumulate_steps=gradient_accumulate_steps,+ seqlen=seqlen,+ nsamples=nsamples,+ batch_size=batch_size,+ kwargs=kwargs,+ common_config_kwargs=common_config_kwargs,+ auto_round_config_kwargs=auto_round_config_kwargs,+ )- # Extract rotation_config (old-API kwarg) and thread it into alg_configs.- # In old arch this was a standalone keyword arg; the new arch passes rotation- # transforms as part of the alg_configs list. All backends (auto / inplace /- # transform) are dispatched inside ``HadamardRotation.apply_to_model``.- # Also supports SpinQuantConfig and string shorthands ("quarot", "spinquant").- _rotation_config_raw = kwargs.pop("rotation_config", None)+ forward_kwargs = cls._build_entry_forward_kwargs(kwargs)+ format_name = forward_kwargs.pop("format", None)+ _rotation_config_raw = forward_kwargs.pop("rotation_config", None)if _rotation_config_raw is not None:- if isinstance(_rotation_config_raw, _BaseRotationConfig):- # Already a valid config (RotationConfig, SpinQuantConfig, etc.)- _rc = _rotation_config_raw- elif isinstance(_rotation_config_raw, dict):- # Use unified normalizer which dispatches by "algorithm" key- _rc = _normalize_any_rotation_config(_rotation_config_raw)- elif isinstance(_rotation_config_raw, str):- # String shorthands: "quarot", "spinquant", "hadamard",- # "random_hadamard", "default", etc.- _rc = _normalize_any_rotation_config(_rotation_config_raw)- else:+ _rc = _normalize_rotation_alg_config(_rotation_config_raw)+ if _rc is None:_rc = _NewArchRotationConfig()- if _rc is not None:- config = [config, _rc]-- # Extract MLLM-specific parameters- processor = kwargs.pop("processor", None)- image_processor = kwargs.pop("image_processor", None)- template = kwargs.pop("template", None)- extra_data_dir = kwargs.pop("extra_data_dir", None)- quant_nontext_module = kwargs.pop("quant_nontext_module", False)-- # Extract Diffusion-specific parameters- guidance_scale = kwargs.pop("guidance_scale", 7.5)- num_inference_steps = kwargs.pop("num_inference_steps", 50)- generator_seed = kwargs.pop("generator_seed", None)+ config = [config, _rc]# Check model type for logging (use warning_once to avoid repeating for every block# when called from LLM-Compressor which instantiates AutoRound per block)@@ -592,12 +748,12 @@ class AutoRoundCompatible:# Create AutoRound instance using new architecturecompressor = AutoRound(- alg_configs=config,- model=model,+ model,+ scheme,+ config,tokenizer=tokenizer,platform=platform,- format=format,- scheme=scheme,+ format=format_name,dataset=dataset,iters=iters,gradient_accumulate_steps=gradient_accumulate_steps,@@ -610,18 +766,8 @@ class AutoRoundCompatible:nsamples=nsamples,seqlen=seqlen,batch_size=batch_size,- # MLLM parameters- processor=processor,- image_processor=image_processor,- template=template,- extra_data_dir=extra_data_dir,- quant_nontext_module=quant_nontext_module,- # Diffusion parameters- guidance_scale=guidance_scale,- num_inference_steps=num_inference_steps,- generator_seed=generator_seed,- # Pass remaining kwargs- **kwargs,+ **compressor_only_kwargs,+ **forward_kwargs,)return compressordiff --git a/auto_round/compressors/mllm/dataset.py b/auto_round/compressors/mllm/dataset.pyindex e441d764b..67f2c7ba8 100644--- a/auto_round/compressors/mllm/dataset.py+++ b/auto_round/compressors/mllm/dataset.py@@ -16,7 +16,7 @@ import jsonimport osfrom collections import OrderedDictfrom inspect import signature-from typing import Dict+from typing import Any, Dict, Optionalimport torchfrom torch.utils.data import DataLoader, Dataset@@ -68,16 +68,16 @@ class LlavaDataset(Dataset):def __init__(self,- template,- model,- tokenizer,- dataset_path,- extra_data_dir=None,- seqlen=512,- padding=True,- truncation=True,- nsamples=512,- cache_size=0,+ template: Template,+ model: torch.nn.Module,+ tokenizer: Any,+ dataset_path: str,+ extra_data_dir: Optional[str] = None,+ seqlen: int = 512,+ padding: bool = True,+ truncation: bool = True,+ nsamples: int = 512,+ cache_size: int = 0,) -> None:super().__init__()self.model = modeldiff --git a/auto_round/compressors/mllm/processor.py b/auto_round/compressors/mllm/processor.pyindex 2165e5456..bf3674c02 100644--- a/auto_round/compressors/mllm/processor.py+++ b/auto_round/compressors/mllm/processor.py@@ -50,7 +50,7 @@ def register_processor(name):@register_processor("basic")class BasicProcessor:- def __init__(self):+ def __init__(self) -> None:passdef post_init(self, model, tokenizer, processor=None, image_processor=None, use_rtn=False, **kwargs):@@ -91,7 +91,7 @@ class HFProcessor(BasicProcessor):# evaluation on: Qwen2-VL, mllama, Mistral-SmallIMAGE_TOKEN = "<image>"- def __init__(self):+ def __init__(self) -> None:self.process_func = self._process_v1def post_init(self, model, tokenizer, processor=None, image_processor=None, use_rtn=False, **kwargs):diff --git a/auto_round/compressors/mllm_mixin.py b/auto_round/compressors/mllm_mixin.pyindex 26bf4d06a..7b24c7b6d 100644--- a/auto_round/compressors/mllm_mixin.py+++ b/auto_round/compressors/mllm_mixin.py@@ -12,6 +12,8 @@# See the License for the specific language governing permissions and# limitations under the License.+from typing import Any, Optional, Union+from auto_round.logger import logger@@ -46,13 +48,13 @@ class MLLMMixin:def __init__(self,*args,- processor=None,- image_processor=None,- template=None,- extra_data_dir=None,- quant_nontext_module=False,+ processor: Any = None,+ image_processor: Any = None,+ template: Optional[str] = None,+ extra_data_dir: Optional[str] = None,+ quant_nontext_module: bool = False,**kwargs,- ):+ ) -> None:self.template = templateself.extra_data_dir = extra_data_dirself.quant_nontext_module = quant_nontext_module@@ -68,8 +70,8 @@ class MLLMMixin:new_grad_acc = batch_size * grad_acckwargs["gradient_accumulate_steps"] = new_grad_acckwargs["batch_size"] = 1- # Also patch ``gradient_accumulate_steps`` on AlgConfig (still- # owned there) so behaviour matches the old arch.+ # Also patch ``gradient_accumulate_steps`` on algorithm configs so+ # behaviour matches the old arch._alg_cfg = args[0] if args else Noneif _alg_cfg is not None:cfgs = _alg_cfg if isinstance(_alg_cfg, list) else [_alg_cfg]@@ -100,7 +102,13 @@ class MLLMMixin:"""return "mllm"- def save_quantized(self, output_dir=None, format="auto_round", inplace=True, **kwargs):+ def save_quantized(+ self,+ output_dir: Optional[str] = None,+ format: Union[str, list] = "auto_round",+ inplace: bool = True,+ **kwargs,+ ) -> Any:"""Save the quantized model to the specified output directory in the specified format.Args:diff --git a/auto_round/compressors/model_free.py b/auto_round/compressors/model_free.pyindex 362ca7b51..47a41d030 100644--- a/auto_round/compressors/model_free.py+++ b/auto_round/compressors/model_free.py@@ -97,7 +97,7 @@ import shutilimport timefrom concurrent.futures import ThreadPoolExecutorfrom dataclasses import asdict, fields-from typing import Optional, Union+from typing import Any, Optional, Unionimport torch@@ -388,7 +388,7 @@ class _PatternMatcher:ignore_patterns: list[str],layer_config: dict[str, dict],default_scheme: dict,- ):+ ) -> None:self._default_scheme = default_schemeself._layer_config = layer_config@@ -520,7 +520,7 @@ def _quantize_weight_mxfp(# quant_mx returns (qdq_tensor, shared_exp, None). We only need shared_exp# (the per-block log2 scale). The element-wise rounding to the FP4/FP8 grid# is performed inside QuantLinear.pack via dtype casts / pack_fp4_to_uint8.- weight_dev, shared_exp, _ = quant_mx(weight_dev, bits=bits, group_size=group_size, data_type=data_type)+ _, shared_exp, _ = quant_mx(weight_dev, bits=bits, group_size=group_size, data_type=data_type)# Reshape to (out_features, n_groups) so the on-disk weight_scale matches# the llm-compressor convention (and QuantLinear's registered buffer shape).shared_exp = shared_exp.reshape(out_features, in_features // group_size)@@ -1211,7 +1211,7 @@ class _ModelFreeCompressorCore:device: str = "cpu",quant_lm_head: bool = False,quant_nontext_module: bool = False,- ):+ ) -> None:# --- raw inputs ---self.model_name_or_path = model_name_or_pathself.output_dir = output_dir@@ -1708,10 +1708,10 @@ class ModelFreeCompressor(_ModelFreeCompressorCore):quant_lm_head: bool = False,quant_nontext_module: bool = False,# --- AutoRound compressor-role aliases ---- tokenizer=None,- device_map=None,+ tokenizer: Any = None,+ device_map: Any = None,**kwargs,- ):+ ) -> None:import copyfrom dataclasses import fields as dc_fields@@ -1805,7 +1805,7 @@ class ModelFreeCompressor(_ModelFreeCompressorCore):def quantize(self,- ):+ ) -> Any:"""fallback to base compressor's quantize."""self._fallback_to_base_compressor()return self._fallback_compressor.quantize() # pylint: disable=E1101@@ -1852,7 +1852,7 @@ class ModelFreeCompressor(_ModelFreeCompressorCore):format: str = "auto_round",inplace: bool = True,**kwargs,- ):+ ) -> Any:"""Quantize and save — AutoRound compressor entry point."""# Accept the standard auto_round formats._accepted_formats = {diff --git a/auto_round/compressors/shard_writer.py b/auto_round/compressors/shard_writer.pyindex d4443b080..e95468fbe 100644--- a/auto_round/compressors/shard_writer.py+++ b/auto_round/compressors/shard_writer.py@@ -15,6 +15,7 @@import jsonimport osfrom collections import OrderedDict+from typing import Optional, Unionimport torch@@ -49,11 +50,11 @@ class ShardWriter:def __init__(self,- model,- bits,- max_shard_size=None,- safe_serialization=True,- ):+ model: torch.nn.Module,+ bits: int,+ max_shard_size: Optional[Union[int, str]] = None,+ safe_serialization: bool = True,+ ) -> None:if ShardWriter._initialized:returnself.model = model@@ -114,13 +115,13 @@ class ShardWriter:return os.path.join(base_dir, "")@classmethod- def reset(cls):+ def reset(cls) -> None:"""Reset the singleton state so the next instantiation creates a fresh ShardWriter."""cls._initialized = Falsecls._instance = None@classmethod- def get_shard_writer(cls, *args, **kwargs):+ def get_shard_writer(cls, *args, **kwargs) -> Optional["ShardWriter"]:"""Return the current singleton instance, or None if not yet initialized.Callers that require a valid writer should guard the result with@@ -148,7 +149,7 @@ class ShardWriter:logger.warning("safetensors not installed; falling back to torch.save.")return False- def save_module(self, m: torch.nn.Module, name: str = None):+ def save_module(self, m: torch.nn.Module, name: str = None) -> None:"""Extracts and accumulates tensors from a module."""prefix = name if name is not None else getattr(m, "global_name", "model")sd = m.state_dict()@@ -300,7 +301,7 @@ class ShardWriter:):module.to("meta")- def finalize(self):+ def finalize(self) -> None:"""Saves remaining weights, renames files, and writes the index JSON."""# 1. Capture remaining weights not yet savedfull_sd = self.model.state_dict()@@ -367,7 +368,7 @@ class ShardWriter:logger.info(f"model has been saved to {self.output_dir}")@torch.no_grad()- def write(self, m: torch.nn.Module = None, name: str = None, is_finalize: bool = False):+ def write(self, m: torch.nn.Module = None, name: str = None, is_finalize: bool = False) -> None:if m is None and name is None and not is_finalize and not is_finalize:raise ValueError("Must specify either name or m")if m is None and name is not None:diff --git a/auto_round/compressors/zero_shot.py b/auto_round/compressors/zero_shot.pyindex 3b471c6a2..b0933997d 100644--- a/auto_round/compressors/zero_shot.py+++ b/auto_round/compressors/zero_shot.py@@ -17,7 +17,7 @@ from typing import Any, Unionimport torchfrom tqdm import tqdm-from auto_round.algorithms.alg_config import AlgConfig+from auto_round.algorithms.pipeline import BlockContextfrom auto_round.compressors.base import BaseCompressorfrom auto_round.compressors.utils import is_nv_fp, is_static_wfp8afp8from auto_round.logger import logger@@ -36,6 +36,7 @@ from auto_round.utils import (set_amax_for_all_moe_layers,set_module,)+from auto_round.utils.device_manager import device_managerclass ZeroShotCompressor(BaseCompressor):@@ -43,11 +44,11 @@ class ZeroShotCompressor(BaseCompressor):def __init__(self,- config: Union[AlgConfig, list[AlgConfig]],+ config: Union[object, list[object]],model: Union[torch.nn.Module, str],- tokenizer=None,- platform="hf",- format=None,+ tokenizer: Any = None,+ platform: str = "hf",+ format: Union[str, list, None] = None,low_gpu_mem_usage: bool = False,device_map: Union[str, torch.device, int, dict] = 0,enable_torch_compile: bool = False,@@ -55,7 +56,7 @@ class ZeroShotCompressor(BaseCompressor):seed: int = 42,low_cpu_mem_usage: bool = True,**kwargs,- ):+ ) -> None:super().__init__(config=config,model=model,@@ -78,13 +79,13 @@ class ZeroShotCompressor(BaseCompressor):q_input: Union[torch.Tensor, dict, None] = None,device: Union[str, torch.device] = "cpu",auto_offload: bool = True,- ):+ ) -> Any:"""Quantize a single block via RTN (public API for LLM-Compressor).ZeroShotCompressor does not need calibration data, so ``inputs`` and- ``q_input`` are accepted for interface compatibility but not used for- algorithm purposes. The block is materialized, converted to the target- dtype, moved to ``device``, and quantized in-place via RTN.+ ``q_input`` are accepted for interface compatibility+ but not used for algorithm purposes. The block is materialized, converted+ to the target dtype, moved to ``device``, and quantized in-place via RTN.Returns:tuple: ``(None, None)`` — RTN does not produce reference outputs.@@ -100,12 +101,22 @@ class ZeroShotCompressor(BaseCompressor):convert_module_to_hp_if_necessary(block, self.model_context.amp_dtype, device)block = block.to(device)- self.quantizer.quantize_block(block)+ ctx = BlockContext(+ model=self.model_context.model,+ block=block,+ block_names=[getattr(block, "global_name", "")],+ block_name=getattr(block, "global_name", ""),+ block_index=0,+ io=self.quantizer.create_block_io(None, {}, None, block),+ device=device,+ )+ self.quantizer.quantize_block(ctx)# ── MoE scale alignment for FP8 dispatch efficiency ────────────────if is_nv_fp(self.quantizer.act_data_type) or is_static_wfp8afp8(self.quantizer):set_amax_for_all_moe_layers(block, attr_name="act_max")+ ctx.finish()mv_module_from_gpu(block)return None, None@@ -122,7 +133,7 @@ class ZeroShotCompressor(BaseCompressor):formats = self.formats if isinstance(self.formats, list) else []if not (any(fmt.is_gguf() for fmt in formats) or self.super_bits is not None):- self._quantize_embedding_layer() # leave to gguf itself to handle+ self.quantizer.quantize_embedding_layer() # leave to gguf itself to handle# Release memoryclear_memory(device_list=self.device_list)@@ -164,7 +175,17 @@ class ZeroShotCompressor(BaseCompressor):materialize_model_(block)# ── Pure algorithm ────────────────────────────────────────- self.quantizer.quantize_block(block)+ ctx = BlockContext(+ model=self.model_context.model,+ block=block,+ block_names=[block_name],+ block_name=block_name,+ block_index=0,+ io=self.quantizer.create_block_io(None, {}, None, block),+ device=device_manager.device,+ )+ self.quantizer.quantize_block(ctx)+ ctx.finish()# ── MoE scale alignment for FP8 dispatch efficiency ────────────────if is_nv_fp(self.quantizer.act_data_type) or is_static_wfp8afp8(self.quantizer):diff --git a/auto_round/context/__init__.py b/auto_round/context/__init__.pyindex 14a492441..7548429bf 100644--- a/auto_round/context/__init__.py+++ b/auto_round/context/__init__.py@@ -11,3 +11,7 @@# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.# See the License for the specific language governing permissions and# limitations under the License.++from auto_round.schemes import QuantizationScheme++__all__ = ["QuantizationScheme"]diff --git a/auto_round/context/base.py b/auto_round/context/base.pyindex 82c5d9770..5e7241a50 100644--- a/auto_round/context/base.py+++ b/auto_round/context/base.py@@ -44,7 +44,7 @@ class AutoSkipInitMeta(type):class BaseContext(metaclass=AutoSkipInitMeta):_instances = {}- def __init__(self):+ def __init__(self) -> None:logger.debug(f"{self.__class__.__name__} context initialized.")@classmethoddiff --git a/auto_round/context/compress.py b/auto_round/context/compress.pyindex 0197ab9a6..36de16ec6 100644--- a/auto_round/context/compress.py+++ b/auto_round/context/compress.py@@ -11,7 +11,7 @@# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.# See the License for the specific language governing permissions and# limitations under the License.-from typing import Any, Callable, Optional, Union+from typing import Callable, Optional, Unionimport torch@@ -41,7 +41,7 @@ class CompressContext(BaseContext):static_kv_dtype: Optional[torch.dtype] = None,static_attention_dtype: Optional[torch.dtype] = None,**kwargs,- ):+ ) -> None:super().__init__()self.low_cpu_mem_usage = low_cpu_mem_usageself.low_gpu_mem_usage = low_gpu_mem_usagediff --git a/auto_round/context/model.py b/auto_round/context/model.pyindex 442976ef6..e016d72b9 100644--- a/auto_round/context/model.py+++ b/auto_round/context/model.py@@ -57,18 +57,18 @@ class ModelContext(BaseContext):def __init__(self,- model=None,- tokenizer=None,- platform="hf",- model_dtype=None,- trust_remote_code=True,+ model: Union[torch.nn.Module, str, None] = None,+ tokenizer: Any = None,+ platform: str = "hf",+ model_dtype: Optional[Union[str, torch.dtype]] = None,+ trust_remote_code: bool = True,config: Optional[AutoConfig] = None,- amp=True,- need_calib=True,- is_act_quantize=False,- quant_nontext_module=False,+ amp: bool = True,+ need_calib: bool = True,+ is_act_quantize: bool = False,+ quant_nontext_module: bool = False,**kwargs,- ):+ ) -> None:super().__init__()self.quantized = Falseself.is_mllm = False@@ -263,7 +263,7 @@ class ModelContext(BaseContext):"""Apply format-specific model structure patches.Must be called after formats are resolved (list[OutputFormat]) and before- BaseQuantizers.post_init() so that configure_layer_config() operates on the+ BaseQuantizer.post_init() so that configure_layer_config() operates on thefinal model structure (post update_module). Eliminates the need for asubsequent refresh_quantizer_for_initialized_model() call."""diff --git a/auto_round/experimental/apply_rotation_transform.py b/auto_round/experimental/apply_rotation_transform.pyindex 520f6b2fb..c04da06af 100644--- a/auto_round/experimental/apply_rotation_transform.py+++ b/auto_round/experimental/apply_rotation_transform.py@@ -3,10 +3,10 @@"""Backward-compat re-export shim.The canonical implementation now lives in-:mod:`auto_round.algorithms.transforms.rotation.dispatcher`.+:mod:`auto_round.algorithms.transforms.quarot.dispatcher`."""-from auto_round.algorithms.transforms.rotation.dispatcher import ( # noqa: F401+from auto_round.algorithms.transforms.quarot.dispatcher import ( # noqa: F401apply_hadamard_rotation,resolve_hadamard_backend,)diff --git a/auto_round/experimental/utils.py b/auto_round/experimental/utils.pyindex d27c73abb..672a1e0fb 100644--- a/auto_round/experimental/utils.py+++ b/auto_round/experimental/utils.py@@ -16,8 +16,7 @@ from typing import Anyimport torch-from auto_round.algorithms.transforms.rotation.config import RotationConfig-from auto_round.algorithms.transforms.rotation.transforms import HADAMARDS+from auto_round.algorithms.transforms.quarot import HADAMARDS, RotationConfigfrom auto_round.compressors.utils import is_mx_fp, is_nv_fpfrom auto_round.utils import logger@@ -130,7 +129,7 @@ def is_triton_kernel_available(data_type: str) -> bool:return Falsetry:- from auto_round.algorithms.transforms.rotation.utils.triton.mxfp4 import ( # pylint: disable=E0401+ from auto_round.algorithms.transforms.quarot.utils.triton.mxfp4 import ( # pylint: disable=E0401mxfp4_forward_kernel_wrapper,)except Exception:@@ -140,19 +139,19 @@ def is_triton_kernel_available(data_type: str) -> bool:def dump_group_size_to_rotation_config(rotation_config: str | dict | RotationConfig, group_size: int):- from auto_round.algorithms.transforms.rotation.config import dump_group_size_to_rotation_config as _impl+ from auto_round.algorithms.transforms.quarot.config import dump_group_size_to_rotation_config as _implreturn _impl(rotation_config, group_size)def to_dict_rotation_config(rotation_config: str | dict | RotationConfig):- from auto_round.algorithms.transforms.rotation.config import to_dict_rotation_config as _impl+ from auto_round.algorithms.transforms.quarot.config import to_dict_rotation_config as _implreturn _impl(rotation_config)def normalize_rotation_config(rotation_config: str | dict | RotationConfig | None, data_type: str) -> dict[str, Any]:- from auto_round.algorithms.transforms.rotation.config import normalize_rotation_config as _impl+ from auto_round.algorithms.transforms.quarot.config import normalize_rotation_config as _implreturn _impl(rotation_config, data_type)diff --git a/auto_round/formats.py b/auto_round/formats.pyindex 86af6244f..5dcd3ca6b 100644--- a/auto_round/formats.py+++ b/auto_round/formats.py@@ -769,13 +769,14 @@ class GGUFFormat(OutputFormat):def __init__(self, format: str, ar: BaseCompressor):if format.startswith("gguf:"):self._original_format = format # preserve "gguf:q2_k_mixed" etc. for Phase 2b- self.gguf_args_check(ar, format, model_type=ModelType.TEXT)- if ar.mllm:- self.gguf_args_check(ar, format, model_type=ModelType.MMPROJ)-self.output_format = "gguf"self.backend_cls = GGUFFormatself.backend = GGUFFormat(format.split(":")[-1], ar)++ resolved_format = self.backend.output_format+ self.gguf_args_check(ar, resolved_format, model_type=ModelType.TEXT)+ if ar.mllm:+ self.gguf_args_check(ar, resolved_format, model_type=ModelType.MMPROJ)else:scheme = ar.schemegguf_format = f"gguf:{format.lower()}"@@ -783,12 +784,12 @@ class GGUFFormat(OutputFormat):from auto_round.schemes import _handle_special_schemesfrom auto_round.utils.model import is_moe_model- if format.lower() == "q2_k_mixed" and getattr(ar, "iters", 0) > 0 and not is_moe_model(ar.model):+ if format.lower() == "q2_k_mixed" and (getattr(ar, "iters", 0) or 0) > 0 and not is_moe_model(ar.model):logger.warning("gguf:q2_k_mixed only supports MoE models with iters>0. "- "It is not an MoE model, falling back to gguf:q4_k_m."+ "It is not an MoE model, falling back to gguf:q2_k_s.")- gguf_format = "gguf:q4_k_m"+ gguf_format = "gguf:q2_k_s"else:ar.layer_config = _handle_special_schemes(gguf_format, ar.layer_config, ar.model, quant_nontext_module=ar.quant_nontext_modulediff --git a/auto_round/inference/convert_model.py b/auto_round/inference/convert_model.pyindex 7d19b1e3e..b6de09d66 100644--- a/auto_round/inference/convert_model.py+++ b/auto_round/inference/convert_model.py@@ -875,8 +875,8 @@ def convert_hf_model(model: nn.Module, target_device: str = "cpu") -> tuple[nn.Mrotation_config = getattr(quantization_config, "rotation_config", None)if rotation_config is not None and rotation_config:- from auto_round.algorithms.transforms.rotation.apply import apply_rotation_transform- from auto_round.algorithms.transforms.rotation.config import RotationConfig+ from auto_round.algorithms.transforms.quarot.apply import apply_rotation_transform+ from auto_round.algorithms.transforms.quarot.config import RotationConfig# apply forward hookact_rotation_config = RotationConfig(diff --git a/auto_round/schemes.py b/auto_round/schemes.pyindex 1f202a2ca..208d3d76c 100644--- a/auto_round/schemes.py+++ b/auto_round/schemes.py@@ -42,12 +42,27 @@ class QuantizationScheme:super_group_size: Optional[int] = Nonerotation_config: Optional[dict] = None+ @classmethod+ def empty(cls):+ return cls(**{field.name: None for field in fields(cls)})+@classmethoddef from_dict(cls, config: dict):field_names = {f.name for f in fields(cls)}filtered_config = {k: v for k, v in config.items() if k in field_names}return cls(**filtered_config)+ def to_dict(self) -> dict:+ return asdict(self)++ def copy(self):+ return copy.deepcopy(self)++ def update_from_dict(self, config: dict) -> None:+ for key, value in config.items():+ if key in self.get_attributes():+ setattr(self, key, value)+@classmethoddef get_attributes(cls: "QuantizationScheme") -> list[str]:return [field.name for field in fields(cls)]diff --git a/auto_round/utils/device_manager.py b/auto_round/utils/device_manager.pyindex 5f90e75ae..8ed74643c 100644--- a/auto_round/utils/device_manager.py+++ b/auto_round/utils/device_manager.py@@ -58,6 +58,7 @@ __all__ = ["ARDevice","DeviceManager","device_manager",+ "normalize_default_device_map","get_ar_device","get_current_device_manager","get_current_device_type",@@ -86,6 +87,23 @@ __all__ = [_PREFERRED_ORDER = ("cuda", "xpu", "hpu") # add mps later+def normalize_default_device_map(device_map: Union[None, str, int, torch.device, dict]):+ """Normalize default device selection across entry points.++ On Apple Silicon, the default ``0`` / ``"0"`` / ``None`` / ``"auto"``+ selection would otherwise resolve to MPS. That tends to OOM on larger+ models, so keep the historical behavior of defaulting to CPU unless the+ caller explicitly requests MPS.+ """+ if torch.mps.is_available() and device_map in (0, "0", None, "auto"):+ logger.warning(+ "MPS detected. Using CPU by default to avoid potential memory issues. "+ "Set --device_map=mps to force MPS usage."+ )+ return "cpu"+ return device_map++def _torch_accelerator_type() -> Optional[str]:"""Return the canonical accelerator type reported by ``torch.accelerator``.diff --git a/auto_round/utils/model.py b/auto_round/utils/model.pyindex 94ff1b199..a809ccd60 100644--- a/auto_round/utils/model.py+++ b/auto_round/utils/model.py@@ -2295,8 +2295,8 @@ def is_model_free_route("""from auto_round.compressors.model_free import is_model_free_supported_scheme- explicit = bool(kwargs.pop("model_free", False))- disabled = bool(kwargs.pop("disable_model_free", False))+ explicit = bool(kwargs.get("model_free", False))+ disabled = bool(kwargs.get("disable_model_free", False))if explicit:return True# Only auto-route when format is auto_round (or not specified).diff --git a/auto_round/wrapper.py b/auto_round/wrapper.pyindex bd6a78570..0f6c222ea 100644--- a/auto_round/wrapper.py+++ b/auto_round/wrapper.py@@ -335,6 +335,12 @@ class WrapperLinear(torch.nn.Module):Returns:torch.nn.Module: The unwrapped and restored original layer."""++ def _preserve_global_name(layer):+ if hasattr(self.orig_layer, "global_name") and not hasattr(layer, "global_name"):+ layer.global_name = self.orig_layer.global_name+ return layer+best_params = best_params or {}v = best_params.get("value", torch.tensor(0.0)).to(self.device)min_scale = best_params.get("min_scale", torch.tensor(1.0)).to(self.device)@@ -441,9 +447,9 @@ class WrapperLinear(torch.nn.Module):enable_torch_compile=self.enable_torch_compile,device=self.device,)- return wrapper_layer+ return _preserve_global_name(wrapper_layer)- return self.orig_layer+ return _preserve_global_name(self.orig_layer)def linear_forward(self, x, weight, bias):"""Performs the forward pass for a linear layer.diff --git a/docs/step_by_step.md b/docs/step_by_step.mdindex cea28ea1d..228f778b2 100644--- a/docs/step_by_step.md+++ b/docs/step_by_step.md@@ -485,6 +485,42 @@ We will try to optimize the RAM usage in the future. The RAM usage is about 1.1-Embedding layer is not supported in AutoScheme, it will use the best scheme in options.+### AWQ Quantization Algorithm++AWQ (`algorithm="awq"`) is a pre-processing quantization algorithm that analyzes activation patterns and applies channel-wise scaling to protect salient weights. It runs BEFORE the actual quantization (RTN by default, or auto_round/SignRound).++#### CLI Usage+```bash+# AWQ + default RTN (iters=0 auto-selected)+auto-round --model Qwen/Qwen3-0.6B --algorithm awq --scheme W4A16++# AWQ + AutoRound optimization+auto-round --model Qwen/Qwen3-0.6B --algorithm awq,auto_round --scheme W4A16++# AWQ flags+--duo-scaling true|false|both (default: true)+--n-grid 20 (default: 20)+```++#### API Usage+```python+from auto_round import AutoRound+from auto_round.algorithms.quantization.awq.config import AWQConfig+from auto_round.algorithms.quantization.sign_round.config import SignRoundConfig++# AWQ + default RTN (simplest)+ar = AutoRound(model, tokenizer, algorithm="awq", scheme="W4A16")++# AWQ + AutoRound via alg_configs (explicit pipeline)+ar = AutoRound(model, tokenizer, alg_configs=[AWQConfig(), SignRoundConfig(iters=200)], scheme="W4A16")+ar.quantize_and_save(output_dir="./qmodel")+```++**Important Note**: `algorithm="awq"` (quantization algorithm) and `format="auto_awq"` (export format) are independent. You can use:+- `algorithm="awq"` + `format="auto_round"`: AWQ smoothing + AutoRound packing+- `algorithm="auto_round"` + `format="auto_awq"`: No AWQ smoothing + AutoAWQ packing++### OPT RTN ModeAutoRound also supports Optimized RTN (Round-To-Nearest) mode for fast, calibration-free baseline quantization. Setting `iters=0` tp enable it and we recommend using `group_size=32` for better results. Check [accuracy comparison](./opt_rtn.md) between RTN and OPT RTN modediff --git a/docs/step_by_step_CN.md b/docs/step_by_step_CN.mdindex 7da4e43a7..248a6e9a5 100644--- a/docs/step_by_step_CN.md+++ b/docs/step_by_step_CN.md@@ -482,7 +482,45 @@ ar.quantize_and_save()#### 局限性AutoScheme 目前还**不支持对嵌入层(Embedding layer)进行自动量化**。该层将直接采用候选方案中精度最高的配置。++### AWQ 量化算法++AWQ(`algorithm="awq"`)是一种预处理量化算法,通过分析激活分布并应用通道缩放(channel-wise scaling)来保护重要的权重。它在实际量化(默认为 RTN,或使用 auto_round/SignRound)之前运行。++#### 命令行用法+```bash+# AWQ + 默认 RTN (自动选择 iters=0)+auto-round --model Qwen/Qwen3-0.6B --algorithm awq --scheme W4A16++# AWQ + AutoRound 优化+auto-round --model Qwen/Qwen3-0.6B --algorithm awq,auto_round --scheme W4A16++# AWQ 相关参数+--duo-scaling true|false|both (默认: true)+--n-grid 20 (默认: 20)+```++#### API 用法+```python+from auto_round import AutoRound+from auto_round.algorithms.quantization.awq.config import AWQConfig+from auto_round.algorithms.quantization.sign_round.config import SignRoundConfig++# AWQ + 默认 RTN (最简用法)+ar = AutoRound(model, tokenizer, algorithm="awq", scheme="W4A16")++# 通过 alg_configs 指定 AWQ + AutoRound (显式流水线)+ar = AutoRound(model, tokenizer, alg_configs=[AWQConfig(), SignRoundConfig(iters=200)], scheme="W4A16")+ar.quantize_and_save(output_dir="./qmodel")+```++**重要提示**:`algorithm="awq"`(量化算法)与 `format="auto_awq"`(导出格式)是相互独立的。你可以使用:+- `algorithm="awq"` + `format="auto_round"`:AWQ 平滑 + AutoRound 打包+- `algorithm="auto_round"` + `format="auto_awq"`:不使用 AWQ 平滑 + AutoAWQ 打包++### OPT-RTN 模式+AutoRound 还提供优化版 RTN(Round-To-Nearest,就近舍入)模式,无需标定数据即可实现快速基线量化。**启用方式为 `iters=0`**。同时为获得更好的效果,推荐搭配 `group_size=32` 。RTN 与 OPT RTN 模式的精度对比详见[《精度对比报告》](./opt_rtn.md)。对于 GGUF 格式,我们参考 llamacpp 的思路,优化了 RTN 算法。若需使用原始(非优化)RTN 算法,开启 `--disable_opt_rtn` 即可。diff --git a/setup.py b/setup.pyindex 6bb184bcb..bac4362f7 100644--- a/setup.py+++ b/setup.py@@ -187,7 +187,7 @@ if __name__ == "__main__":package_data={"": ["mllm/templates/*.json",- "algorithms/transforms/rotation/utils/hadamards.safetensors",+ "algorithms/transforms/quarot/utils/hadamards.safetensors",]},)diff --git a/test/test_cpu/algorithms/test_awq.py b/test/test_cpu/algorithms/test_awq.pyindex 85a3e815f..e6b55e3a1 100644--- a/test/test_cpu/algorithms/test_awq.py+++ b/test/test_cpu/algorithms/test_awq.py@@ -176,7 +176,7 @@ class TestAWQMoE:def test_awq_moe_dynamic_smoothing(self, tiny_qwen_moe_model_path):"""AWQ dynamic smoothing should resolve mappings on a MoE model without error."""- from auto_round.algorithms.quantization.awq.mappings import resolve_mappings+ from auto_round.algorithms.transforms.awq.mappings import resolve_mappingsmodel = AutoModelForCausalLM.from_pretrained(tiny_qwen_moe_model_path,diff --git a/test/test_cpu/core/test_awq_autoround_smoke.py b/test/test_cpu/core/test_awq_autoround_smoke.pynew file mode 100644index 000000000..d529c9c6e--- /dev/null+++ b/test/test_cpu/core/test_awq_autoround_smoke.py@@ -0,0 +1,21 @@+"""Minimal runtime smoke for AWQ + AutoRound fusion."""++from auto_round.algorithms.quantization.sign_round.config import SignRoundConfig+from auto_round.algorithms.transforms.awq.config import AWQConfig+from auto_round.compressors.entry import AutoRound+++def test_awq_plus_autoround_quantize_smoke(tiny_opt_model_path):+ ar = AutoRound(+ tiny_opt_model_path,+ scheme="W4A16",+ alg_configs=[AWQConfig(n_grid=2), SignRoundConfig(iters=1)],+ nsamples=1,+ seqlen=8,+ low_cpu_mem_usage=False,+ )++ model, layer_config = ar.quantize()++ assert model is not None+ assert layer_configdiff --git a/test/test_cpu/core/test_pipeline_fail_fast.py b/test/test_cpu/core/test_pipeline_fail_fast.pynew file mode 100644index 000000000..09c8d733e--- /dev/null+++ b/test/test_cpu/core/test_pipeline_fail_fast.py@@ -0,0 +1,208 @@+"""Fast unit tests for algorithm registry and pipeline construction."""++import pytest++from auto_round import AWQConfig, OptimizedRTNConfig, RotationConfig, RTNConfig, SignRoundConfig, SpinQuantConfig+from auto_round.algorithms.config_resolver import (+ get_algorithm_class,+ resolve_shared_config_values,+ split_quantization_configs,+ sync_shared_config_from,+)+from auto_round.algorithms.pipeline import QuantizationPipeline+from auto_round.algorithms.quantization import registry as _r+from auto_round.algorithms.quantization.rtn.quantizer import RTNQuantizer+from auto_round.compressors.base import collect_user_scheme_overrides+from auto_round.compressors.data_driven import DataDrivenCompressor+from auto_round.compressors.entry import AutoRound as NewAutoRound+from auto_round.logger import logger+++class PartialSharedConfig(RTNConfig):+ def __init__(self, *, weight_clip_ratio=None, **kwargs):+ super().__init__(**kwargs)+ self.weight_clip_ratio = weight_clip_ratio+++class NoWeightClipConfig(RTNConfig):+ pass+++def test_split_awq_plus_rtn():+ pre, block = split_quantization_configs([AWQConfig(), RTNConfig()])+ assert len(pre) == 1 and type(pre[0]).__name__ == "AWQConfig"+ assert len(block) == 1 and type(block[0]).__name__ == "RTNConfig"+++def test_pipeline_preprocessor_only_auto_appends_rtn():+ pipeline = QuantizationPipeline.from_configs([AWQConfig()])+ assert type(pipeline.preprocessors[0]).__name__ == "AWQQuantizer"+ assert isinstance(pipeline.block_quantizer, RTNQuantizer)+++def test_pipeline_duplicate_preprocessor_rejected():+ with pytest.raises(ValueError, match="Duplicate preprocessor"):+ QuantizationPipeline.from_configs([AWQConfig(), AWQConfig()])+++def test_pipeline_multiple_block_quantizers_rejected():+ with pytest.raises(ValueError, match="exactly one block-quantization config"):+ QuantizationPipeline.from_configs([RTNConfig(), SignRoundConfig()])+++def test_registry_builtin_aliases_and_unknown():+ assert isinstance(_r.resolve_alg_config("RTN"), RTNConfig)+ assert isinstance(_r.resolve_alg_config("awq"), AWQConfig)+ assert isinstance(_r.resolve_alg_config("autoround"), SignRoundConfig)+ with pytest.raises(ValueError, match="Unknown algorithm alias"):+ _r.resolve_alg_config("definitely_not_registered_abc123")+++def test_registry_resolves_variant_configs_to_registered_members():+ assert get_algorithm_class(OptimizedRTNConfig()) is not None+ assert get_algorithm_class(SignRoundConfig(enable_adam=True)).__name__ == "AdamRoundQuantizer"+++def test_top_level_config_exports():+ from auto_round import AWQConfig as TopAWQConfig+ from auto_round import OptimizedRTNConfig as TopOptimizedRTNConfig+ from auto_round import RotationConfig as TopRotationConfig+ from auto_round import RTNConfig as TopRTNConfig+ from auto_round import SignRoundConfig as TopSignRoundConfig+ from auto_round import SpinQuantConfig as TopSpinQuantConfig++ assert TopAWQConfig is AWQConfig+ assert TopOptimizedRTNConfig is OptimizedRTNConfig+ assert TopRTNConfig is RTNConfig+ assert TopSignRoundConfig is SignRoundConfig+ assert TopRotationConfig is RotationConfig+ assert TopSpinQuantConfig is SpinQuantConfig+++def test_new_entry_defaults_to_autoround_config(monkeypatch):+ captured = {}++ def _fake_init(self, config, **kwargs):+ captured["config"] = config++ monkeypatch.setattr(DataDrivenCompressor, "__init__", _fake_init)+ monkeypatch.setattr("auto_round.utils.model.detect_model_type", lambda *args, **kwargs: "llm")++ NewAutoRound("dummy-model", "W4A16", iters=1, seqlen=8, nsamples=1)++ assert isinstance(captured["config"], SignRoundConfig)+++def test_entry_rejects_configs_without_quantization_members():+ with pytest.raises(ValueError, match="At least one quantization algorithm config"):+ NewAutoRound("dummy-model", "W4A16", [RotationConfig()])+++def test_compat_entry_preserves_spinquant_dict_config(monkeypatch):+ captured = {}+ rotation_config = {+ "algorithm": "spinquant",+ "r1": True,+ "r2": True,+ "r3": False,+ "r4": False,+ "rotation_size": 128,+ "trainable_rotation": False,+ "trainable_smooth": False,+ }++ def _fake_init(self, config, **kwargs):+ captured["config"] = config++ monkeypatch.setattr(DataDrivenCompressor, "__init__", _fake_init)+ monkeypatch.setattr("auto_round.utils.is_mllm_model", lambda *args, **kwargs: False)+ monkeypatch.setattr("auto_round.utils.is_diffusion_model", lambda *args, **kwargs: False)+ monkeypatch.setattr("auto_round.utils.model.detect_model_type", lambda *args, **kwargs: "llm")++ from auto_round.autoround import AutoRound as CompatAutoRound++ CompatAutoRound(+ "dummy-model",+ scheme="W4A16",+ iters=1,+ seqlen=8,+ nsamples=1,+ rotation_config=rotation_config,+ )++ configs = captured["config"] if isinstance(captured["config"], list) else [captured["config"]]+ spinquant_cfg = next(cfg for cfg in configs if isinstance(cfg, SpinQuantConfig))+ assert spinquant_cfg.rotation_size == rotation_config["rotation_size"]+ assert spinquant_cfg.r1 is rotation_config["r1"]+ assert spinquant_cfg.r2 is rotation_config["r2"]+ assert spinquant_cfg.r3 is rotation_config["r3"]+ assert spinquant_cfg.r4 is rotation_config["r4"]+ assert spinquant_cfg.trainable_rotation is rotation_config["trainable_rotation"]+ assert spinquant_cfg.trainable_smooth is rotation_config["trainable_smooth"]+++def test_entry_warns_and_drops_unsupported_kwargs(monkeypatch, tiny_opt_model_path):+ calls = []++ def _record_warning(message, *args):+ calls.append(message % args)++ monkeypatch.setattr(logger, "warning_once", _record_warning)++ NewAutoRound(+ tiny_opt_model_path,+ "W4A16",+ RTNConfig(disable_opt_rtn=True),+ nsamples=1,+ seqlen=8,+ low_cpu_mem_usage=False,+ nonsense_kwarg=123,+ )++ assert any("unsupported kwargs nonsense_kwarg" in msg for msg in calls)+++def test_shared_config_values_inherit_across_matching_attrs_only():+ awq = PartialSharedConfig(weight_clip_ratio=0.9)+ smoothquant_like = NoWeightClipConfig()+ signround = PartialSharedConfig(weight_clip_ratio=None)++ resolve_shared_config_values([awq, smoothquant_like, signround])++ assert signround.weight_clip_ratio == 0.9+ assert not hasattr(smoothquant_like, "weight_clip_ratio")+++def test_shared_config_values_reject_conflicts():+ with pytest.raises(ValueError, match="Conflicting shared config field 'weight_clip_ratio'"):+ resolve_shared_config_values(+ [PartialSharedConfig(weight_clip_ratio=0.8), PartialSharedConfig(weight_clip_ratio=0.9)]+ )+++def test_shared_config_sync_from_source_skips_missing_attrs():+ source = PartialSharedConfig(weight_clip_ratio=0.75)+ target = PartialSharedConfig()+ no_clip_target = NoWeightClipConfig()++ sync_shared_config_from(source, [target, no_clip_target, RotationConfig()])++ assert target.weight_clip_ratio == 0.75+ assert not hasattr(no_clip_target, "weight_clip_ratio")+++def test_user_scheme_overrides_merge_across_all_configs():+ awq = AWQConfig(bits=8)+ rtn = RTNConfig()+ assert collect_user_scheme_overrides([awq, rtn])["bits"] == 8++ resolve_shared_config_values([awq, rtn])++ assert rtn.bits == 8+++def test_user_scheme_overrides_reject_explicit_conflicts():+ with pytest.raises(ValueError, match="Conflicting shared scheme field 'bits'"):+ collect_user_scheme_overrides([AWQConfig(bits=8), RTNConfig(bits=4)])+ with pytest.raises(ValueError, match="Conflicting shared scheme field 'bits'"):+ resolve_shared_config_values([AWQConfig(bits=8), RTNConfig(bits=4)])diff --git a/test/test_cpu/export/test_gguf_format.py b/test/test_cpu/export/test_gguf_format.pyindex bef7f7712..1e5f54937 100644--- a/test/test_cpu/export/test_gguf_format.py+++ b/test/test_cpu/export/test_gguf_format.py@@ -8,6 +8,7 @@ from packaging import versionfrom transformers import AutoModelForCausalLM, AutoTokenizerfrom auto_round import AutoRound+from auto_round.algorithms.quantization.rtn.config import OptimizedRTNConfigfrom ...helpers import eval_generated_prompt, get_model_path, get_tiny_model, save_tiny_model@@ -63,7 +64,7 @@ class TestGGUF:)assert type(autoround).__name__ == "CalibratedRTNCompressor"- assert autoround.quantize_config._alg_cls == "OptimizedRTNQuantizer"+ assert isinstance(autoround.quantize_config, OptimizedRTNConfig)def test_func(self):bits, group_size, sym = 4, 128, Truediff --git a/test/test_cpu/models/test_audio_model.py b/test/test_cpu/models/test_audio_model.pyindex aaa29bc26..59bbec20e 100644--- a/test/test_cpu/models/test_audio_model.py+++ b/test/test_cpu/models/test_audio_model.py@@ -196,10 +196,10 @@ class TestStableAudioRegistration:"""Verify StableAudio-specific registrations."""def test_config_and_special_registered(self):- from auto_round.algorithms.quantization.base import BaseQuantizers+ from auto_round.algorithms.quantization.base import DiffusionMixin- assert "StableAudioDiTBlock" in BaseQuantizers.DIFFUSION_OUTPUT_CONFIGS- assert BaseQuantizers.DIFFUSION_OUTPUT_CONFIGS["StableAudioDiTBlock"] == ["hidden_states"]+ assert "StableAudioDiTBlock" in DiffusionMixin.DIFFUSION_OUTPUT_CONFIGS+ assert DiffusionMixin.DIFFUSION_OUTPUT_CONFIGS["StableAudioDiTBlock"] == ["hidden_states"]assert "StableAudioDiTModel" in SPECIAL_SHARED_CACHE_KEYSassert "encoder_hidden_states" in SPECIAL_SHARED_CACHE_KEYS["StableAudioDiTModel"]diff --git a/test/test_cpu/models/test_diffusion_dataset.py b/test/test_cpu/models/test_diffusion_dataset.pynew file mode 100644index 000000000..6bec3f1d8--- /dev/null+++ b/test/test_cpu/models/test_diffusion_dataset.py@@ -0,0 +1,36 @@+import pytest++from auto_round.compressors.diffusion.dataset import DIFFUSION_DATASET, get_diffusion_dataloader+++def test_text2img_dataset_rejects_missing_required_columns(tmp_path):+ dataset_path = tmp_path / "bad.tsv"+ dataset_path.write_text("image_id\ttext\n1\thello\n", encoding="utf-8")++ with pytest.raises(ValueError, match="expected columns"):+ DIFFUSION_DATASET["local"](str(dataset_path), nsamples=1)+++def test_get_diffusion_dataloader_parses_coco2014_response_without_temp_file(monkeypatch):+ sample_tsv = "id\tcaption\n1\thello\n2\tworld\n"++ class _Response:+ text = sample_tsv++ def raise_for_status(self):+ return None++ def _fake_get(url, timeout):+ assert "captions_source.tsv" in url+ assert timeout == 30+ return _Response()++ monkeypatch.setattr("requests.get", _fake_get)++ dataloader, bs, grad_steps = get_diffusion_dataloader(dataset="coco2014", bs=1, nsamples=2)++ assert bs == 1+ assert grad_steps == 1+ dataset = dataloader.dataset+ assert dataset.caption_ids == [1, 2]+ assert dataset.captions == ["hello", "world"]diff --git a/test/test_cpu/quantization/test_model_free.py b/test/test_cpu/quantization/test_model_free.pyindex d8e45336c..f40812108 100644--- a/test/test_cpu/quantization/test_model_free.py+++ b/test/test_cpu/quantization/test_model_free.py@@ -351,23 +351,6 @@ class TestModelFreeMXFP:assert out["layer.weight_scale"].shape == (64, 4)assert out["layer.weight_scale"].dtype == torch.uint8- def test_mxfp8_max_value_scaling(self):- """Input bfloat16 weight with max value 240; after MXFP8 quantization- the stored FP8 values should be clamped to 448 (fp8_e4m3fn max) and- must NOT be NaN.-- quant_mx computes shared_exp = floor(log2(240)) - emax = 7 - 8 = -1,- so scale = 2^-1 = 0.5. The normalised value is 240/0.5 = 480 which- exceeds the FP8 max of 448 and must be clamped before the fp8 cast.- """- w_bf16 = torch.full((64, 128), 240, dtype=torch.bfloat16)- out = _quantize_weight_mxfp(w_bf16, "layer", bits=8, group_size=32, data_type="mx_fp")- assert out["layer.weight"].dtype == torch.float8_e4m3fn- weight_fp32 = out["layer.weight"].to(torch.float32)- assert not torch.isnan(weight_fp32).any(), "FP8 weight must not contain NaN"- max_val = torch.max(weight_fp32).item()- assert max_val == 448, f"Expected 448 (fp8_e4m3fn max after clamp), got {max_val}"-@require_compressed_tensors@pytest.mark.parametrize("scheme,fmt", [("MXFP4", "mxfp4-pack-quantized"), ("MXFP8", "mxfp8-quantized")])def test_e2e_mxfp(self, tmp_path, scheme, fmt):@@ -476,9 +459,10 @@ class TestCliAutoRouting:def test_auto_routes(self, tmp_path):model_dir = _make_model_dir(tmp_path, _LLAMA_CFG, {"layer.weight": torch.randn(64, 128)})out_dir = str(tmp_path / "out")- from auto_round.__main__ import BasicArgumentParser, tune+ from auto_round.cli.main import tune+ from auto_round.cli.parser import build_quantize_parser- args = BasicArgumentParser().parse_args(+ args = build_quantize_parser().parse_args(["--model",model_dir,@@ -497,9 +481,9 @@ class TestCliAutoRouting:assert _read_qconfig(out_dir).get("model_free") is Truedef test_disable_model_free_flag(self):- from auto_round.__main__ import BasicArgumentParser+ from auto_round.cli.parser import build_quantize_parser- args = BasicArgumentParser().parse_args(+ args = build_quantize_parser().parse_args(["--model","dummy",diff --git a/test/test_cpu/utils/test_alg_ext.py b/test/test_cpu/utils/test_alg_ext.pyindex 2daf2aada..d5f64fb72 100644--- a/test/test_cpu/utils/test_alg_ext.py+++ b/test/test_cpu/utils/test_alg_ext.py@@ -21,7 +21,7 @@ class TestAlgExt:ar.quantize()def test_alg_ext_import(self):- from auto_round.alg_ext import wrapper_autoround+ from auto_round.algorithms.quantization.sign_roundv2 import SignRoundV2Quantizerdef test_all_support_dtype(self, tiny_opt_model_path):model_name = tiny_opt_model_pathdiff --git a/test/test_cpu/utils/test_cli_usage.py b/test/test_cpu/utils/test_cli_usage.pyindex 64308cf43..81c490ad7 100644--- a/test/test_cpu/utils/test_cli_usage.py+++ b/test/test_cpu/utils/test_cli_usage.py@@ -131,7 +131,7 @@ def test_parse_layer_config_with_single_escaped_regex_keys():def test_run_rtn_uses_zero_shot_recipe(monkeypatch):- from auto_round import __main__ as cli_main+ from auto_round.cli import main as cli_maincaptured = {}@@ -160,7 +160,7 @@ def test_run_rtn_uses_zero_shot_recipe(monkeypatch):def test_run_rtn_preserves_eval_args(monkeypatch, tmp_path):- from auto_round import __main__ as cli_main+ from auto_round.cli import main as cli_maincaptured = {}@@ -198,7 +198,7 @@ def test_run_rtn_preserves_eval_args(monkeypatch, tmp_path):def test_run_opt_rtn_uses_recipe(monkeypatch):- from auto_round import __main__ as cli_main+ from auto_round.cli import main as cli_maincaptured = {}@@ -224,3 +224,28 @@ def test_run_opt_rtn_uses_recipe(monkeypatch):assert args.disable_opt_rtn is Falseassert args.batch_size == 8assert args.nsamples == 128+++def test_unknown_algorithm_help_exits_with_suggestion(monkeypatch):+ from auto_round.cli import main as cli_main++ monkeypatch.setattr(sys, "argv", ["auto_round", "--algorithm", "hadarmard", "--help"])++ with pytest.raises(SystemExit, match="Unknown algorithm 'hadarmard'. Did you mean 'hadamard'\\?"):+ cli_main.run()+++def test_legacy_disable_flags_map_to_enable_bools():+ from auto_round.cli.parser import build_quantize_parser++ args = build_quantize_parser().parse_args(+ [+ "--model",+ "dummy-model",+ "--disable_minmax_tuning",+ "--disable_quanted_input",+ ]+ )++ assert args.enable_minmax_tuning is False+ assert args.enable_quanted_input is Falsediff --git a/test/test_cuda/algorithms/test_alg_ext.py b/test/test_cuda/algorithms/test_alg_ext.pyindex e661e22f9..9d23aa86c 100644--- a/test/test_cuda/algorithms/test_alg_ext.py+++ b/test/test_cuda/algorithms/test_alg_ext.py@@ -31,11 +31,11 @@ class TestAlgExt:gguf:q2_k_s overrides data_type to "int_asym_dq" at format-resolutiontime. The quantizer must be created *after* that override so that- wrapper_autoround() sees the final data_type and sets dq_wrapper_block+ SignRoundV2 sees the final data_type and sets dq_wrapper_block(which wraps layers with DQWrapperLinear) instead of falling back tothe plain wrapper_block (which produces WrapperLinear)."""- from auto_round.alg_ext import dq_wrapper_block+ from auto_round.algorithms.quantization.sign_roundv2.quantizer import SignRoundDQWrapperLinearar = AutoRound(tiny_qwen_model_path,@@ -50,8 +50,8 @@ class TestAlgExt:# create_quantizer → ...). quantizer only exists afterwards.ar.post_init()- assert ar.quantizer.wrapper_block.__name__ == dq_wrapper_block.__name__, (- f"Expected wrapper_block to be '{dq_wrapper_block.__name__}', "+ assert ar.quantizer.wrapper_block.keywords["wrapper_cls"] is SignRoundDQWrapperLinear, (+ f"Expected wrapper_block to use '{SignRoundDQWrapperLinear.__name__}', "f"got '{ar.quantizer.wrapper_block.__name__}'. ""This likely means the quantizer was created before GGUF format ""overrides were applied (data_type was not yet 'int_asym_dq')."diff --git a/test/test_cuda/algorithms/test_awq.py b/test/test_cuda/algorithms/test_awq.pyindex 11f28b783..eaca31c79 100644--- a/test/test_cuda/algorithms/test_awq.py+++ b/test/test_cuda/algorithms/test_awq.py@@ -121,7 +121,7 @@ class TestAWQMoE:def test_awq_moe_dynamic_smoothing(self, tiny_qwen_moe_model_path):"""AWQ mapping resolution works on MoE model."""- from auto_round.algorithms.quantization.awq.mappings import resolve_mappings+ from auto_round.algorithms.transforms.awq.mappings import resolve_mappingsmodel = AutoModelForCausalLM.from_pretrained(tiny_qwen_moe_model_path,diff --git a/test/test_cuda/models/test_audio_model.py b/test/test_cuda/models/test_audio_model.pyindex 2d32053e3..e9996a054 100644--- a/test/test_cuda/models/test_audio_model.py+++ b/test/test_cuda/models/test_audio_model.py@@ -261,10 +261,10 @@ class TestStableAudioRegistration:"""Verify StableAudio-specific registrations."""def test_config_and_special_registered(self):- from auto_round.algorithms.quantization.base import BaseQuantizers+ from auto_round.algorithms.quantization.base import DiffusionMixin- assert "StableAudioDiTBlock" in BaseQuantizers.DIFFUSION_OUTPUT_CONFIGS- assert BaseQuantizers.DIFFUSION_OUTPUT_CONFIGS["StableAudioDiTBlock"] == ["hidden_states"]+ assert "StableAudioDiTBlock" in DiffusionMixin.DIFFUSION_OUTPUT_CONFIGS+ assert DiffusionMixin.DIFFUSION_OUTPUT_CONFIGS["StableAudioDiTBlock"] == ["hidden_states"]assert "StableAudioDiTModel" in SPECIAL_SHARED_CACHE_KEYSassert "encoder_hidden_states" in SPECIAL_SHARED_CACHE_KEYS["StableAudioDiTModel"]diff --git a/test/test_cuda/quantization/test_torch_compile.py b/test/test_cuda/quantization/test_torch_compile.pyindex a578e8821..f451c6b5b 100644--- a/test/test_cuda/quantization/test_torch_compile.py+++ b/test/test_cuda/quantization/test_torch_compile.py@@ -6,7 +6,7 @@ import pytestfrom transformers import AutoTokenizerfrom auto_round import AutoRound-from auto_round.algorithms.quantization.base import BaseQuantizers+from auto_round.algorithms.quantization.base import BaseQuantizerfrom auto_round.algorithms.quantization.rtn.config import RTNConfigfrom auto_round.compressors.utils import block_forward@@ -76,7 +76,7 @@ class TestTorchCompile:def test_opt_rtn_uses_plain_block_forward(self):config = RTNConfig(bits=4, data_type="int", act_bits=16, disable_opt_rtn=False)- quantizer = BaseQuantizers(config)+ quantizer = BaseQuantizer(config)quantizer.compress_context = SimpleNamespace(enable_torch_compile=True, device="cpu")assert quantizer._resolve_block_forward() is block_forward