PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
[STEP 1] merge all api(MLLM, Adam) into AutoRound (#791)
Heng Guobd4bc2c92128
diff --git a/auto_round/__init__.py b/auto_round/__init__.pyindex 93cce6299..15bbc373d 100644--- a/auto_round/__init__.py+++ b/auto_round/__init__.py@@ -11,8 +11,10 @@# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.# See the License for the specific language governing permissions and# limitations under the License.-from auto_round.autoround import AutoRound, AutoRoundAdam-from auto_round.mllm import AutoRoundMLLM+from auto_round.autoround import AutoRound++# support for old api+from auto_round.autoround import AutoRoundLLM, AutoRoundMLLM, AutoRoundAdamfrom auto_round.utils import LazyImportdiff --git a/auto_round/__main__.py b/auto_round/__main__.pyindex 23749e21d..1671d157d 100644--- a/auto_round/__main__.py+++ b/auto_round/__main__.py@@ -11,41 +11,658 @@# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.# See the License for the specific language governing permissions and# limitations under the License.+import argparse+import logging+import os+import reimport sys+from auto_round.compressors import BaseCompressor+from auto_round.eval.eval_cli import EvalArgumentParser, _eval_init, eval, eval_task_by_task, eval_with_vllm+from auto_round.schemes import PRESET_SCHEMES+from auto_round.utils import (+ clear_memory,+ get_device_and_parallelism,+ get_model_dtype,+ set_cuda_visible_devices,+)-def run_eval():- if "--vlmeval" in sys.argv:- sys.argv.remove("--vlmeval")- run_vlmeavl()- elif "--lmms" in sys.argv:- sys.argv.remove("--lmms")- run_lmms()- elif "--vllm" in sys.argv:- sys.argv.remove("--vllm")- from auto_round.script.llm import eval_with_vllm, setup_eval_parser-- args = setup_eval_parser()- eval_with_vllm(args)++class BasicArgumentParser(argparse.ArgumentParser):++ def __init__(self, *args, **kwargs):+ super().__init__(*args, **kwargs)+ self.add_argument(+ "--model", "--model_name", "--model_name_or_path", default="facebook/opt-125m", help="model name or path"+ )++ self.add_argument("--mllm", action="store_true", help="whether to quant multi-modal model.")++ self.add_argument("--eval", action="store_true", help="whether to use eval only mode")++ self.add_argument(+ "--scheme",+ default="W4A16",+ type=str,+ # choices=["W4A16", "W2A16", "W3A16", "W8A16", "MXFP4", "MXFP8", "NVFP4", "FPW8A16", "FP8_STATIC"],+ help="quantization scheme",+ )++ self.add_argument("--bits", default=None, type=int, help="number of weight bits")+ self.add_argument("--group_size", default=None, type=int, help="group size")+ self.add_argument("--asym", action="store_true", help="whether to use asym quantization")+ self.add_argument("--data_type", "--dtype", default=None, help="data type for tuning, 'int', 'mx_fp' and etc")+ self.add_argument("--act_bits", default=None, type=int, help="activation bits")+ self.add_argument("--act_group_size", default=None, type=int, help="activation group size")+ self.add_argument(+ "--super_group_size", default=None, type=int, help="the number of super group size when use double quant."+ )++ self.add_argument(+ "--super_bits", default=None, type=int, help="number of scale and mins quant bits for double quant."+ )+ self.add_argument("--act_data_type", "--act_dtype", default=None, type=str, help="activation data type")++ self.add_argument("--disable_act_dynamic", action="store_true", help="activation static quantization")++ self.add_argument(+ "--device_map",+ "--device",+ "--devices",+ default="0",+ type=str,+ help="the device to be used for tuning. "+ "Currently, device settings support CPU, GPU, and HPU."+ "The default is set to cuda:0,"+ "allowing for automatic detection and switch to HPU or CPU."+ "set --device 0,1,2 to use multiple cards.",+ )++ self.add_argument(+ "--dataset", default="NeelNanda/pile-10k", type=str, help="the dataset for quantization training"+ )++ self.add_argument(+ "--minmax_lr",+ default=None,+ type=float,+ help="minmax learning rate, if None, it will beset to be the same with lr",+ )++ self.add_argument(+ "--mem_per_param_scale",+ default=13,+ type=float,+ help="Scale factor for memory per parameter, used to adjust memory usage estimation for tuning",+ )++ self.add_argument("--seed", default=42, type=int, help="random seed")++ self.add_argument("--adam", action="store_true", help="whether to use adam optimizer instead of SignSGD")++ self.add_argument("--gradient_accumulate_steps", default=1, type=int, help="gradient accumulate steps")++ self.add_argument("--nblocks", default=1, type=int, help="how many blocks to tune together")++ self.add_argument("--low_gpu_mem_usage", action="store_true", help="offload intermediate features to cpu")++ self.add_argument("--format", default="auto_round", type=str, help="the format to save the model")++ self.add_argument(+ "--scale_dtype",+ default="fp16",+ choices=["fp16", "float16", "bf16", "bfloat16", "fp32", "float32"],+ help="scale data type to use for quantization",+ )++ self.add_argument(+ "--output_dir", default="./tmp_autoround", type=str, help="the directory to save quantized model"+ )++ self.add_argument("--disable_amp", action="store_true", help="disable amp")++ self.add_argument(+ "--disable_minmax_tuning", action="store_true", help="whether to disable enable weight minmax tuning"+ )++ self.add_argument("--enable_norm_bias_tuning", action="store_true", help="whether to enable norm bias tuning")++ self.add_argument(+ "--disable_trust_remote_code", action="store_true", help="whether to disable trust_remote_code"+ )++ self.add_argument(+ "--disable_quanted_input",+ action="store_true",+ help="whether to disuse the output of quantized block to tune the next block",+ )++ self.add_argument("--quant_lm_head", action="store_true", help="whether to quant lm_head")++ self.add_argument(+ "--low_cpu_mem_mode",+ default=0,+ type=int,+ choices=[0, 1, 2],+ help="choose which low cpu memory mode to use. "+ "Can significantly reduce cpu memory footprint but cost more time."+ "1 means choose block-wise mode, load the weights of each block"+ " from disk when tuning and release the memory of the block after tuning."+ "2 means choose layer-wise mode, load the weights of each layer from disk when tuning,"+ " minimum memory consumption and also slowest running speed."+ "others means not use low cpu memory. Default to 0, not use low cpu memory.",+ )++ self.add_argument(+ "--low_cpu_mem_tmp_dir",+ default=None,+ type=str,+ help="temporary work space to store the temporary files "+ "when using low cpu memory mode. Will remove after tuning.",+ )++ self.add_argument(+ "--model_dtype",+ default=None,+ type=str,+ choices=["fp16", "float16", "bf16", "bfloat16", "fp32", "float32"],+ help="force to convert the dtype, some backends supports fp16 dtype better",+ )++ self.add_argument(+ "--fp_layers", default="", type=str, help="list of Layer names to maintain original data type"+ )++ self.add_argument(+ "--not_use_best_mse",+ action="store_true",+ help="whether to use the iter of best mes loss in the tuning phase",+ )++ self.add_argument(+ "--to_quant_block_names",+ default=None,+ type=str,+ help="Names of quantitative blocks, please use commas to separate them.",+ )++ self.add_argument("--enable_torch_compile", action="store_true", help="whether to enable torch compile")++ self.add_argument("--enable_alg_ext", action="store_true", help="whether to enable probably better algorithm")++ self.add_argument(+ "--disable_deterministic_algorithms",+ action="store_true",+ help="deprecated, disable torch deterministic algorithms.",+ )+ self.add_argument(+ "--enable_deterministic_algorithms", action="store_true", help="enable torch deterministic algorithms."+ )++ self.add_argument(+ "--disable_opt_rtn",+ action="store_true",+ help="whether to disable optimization of the RTN mode(iters=0) (default is False).",+ )++ ## ======================= MLLM =======================+ self.add_argument(+ "--quant_nontext_module",+ action="store_true",+ help="whether to quantize non-text module, e.g. vision component",+ )++ self.add_argument(+ "--extra_data_dir",+ default=None,+ type=str,+ help="dataset dir for storing images/audio/videos. "+ "Can be a dir path or multiple dir path with format as "+ "'image=path_to_image,video=path_to_video,audio=path_to_audio'"+ "By default, it will search in the relative path, "+ "and if not find, will automatic download.",+ )++ self.add_argument(+ "--template",+ default=None,+ type=str,+ help="the template for building training dataset. It can be a custom one.",+ )++ ## ======================= eval =======================+ self.add_argument(+ "--tasks",+ "--task",+ nargs="?",+ const="lambada_openai,hellaswag,winogrande,piqa,mmlu,wikitext,truthfulqa_mc1,"+ "openbookqa,boolq,arc_easy,arc_challenge",+ default=None,+ help="lm-eval tasks",+ )++ self.add_argument("--eval_bs", default=None, type=int, help="batch size in evaluation")++ self.add_argument(+ "--limit",+ type=float,+ default=None,+ metavar="N|0<N<1",+ help="Limit the number of examples per task. "+ "If <1, limit is a percentage of the total number of examples.",+ )++ self.add_argument("--eval_task_by_task", action="store_true", help="whether to eval task by task.")++ self.add_argument(+ "--eval_model_dtype", default=None, type=str, help="the torch_dytpe to load the model for evaluation."+ )+++def setup_parser():+ parser = BasicArgumentParser()++ parser.add_argument("--batch_size", "--train_bs", "--bs", default=8, type=int, help="train batch size")++ parser.add_argument("--iters", "--iter", default=200, type=int, help="iteration to tune each block")++ parser.add_argument(+ "--seqlen", "--seq_len", default=2048, type=int, help="sequence length of the calibration samples"+ )++ parser.add_argument("--nsamples", "--nsample", default=128, type=int, help="number of samples")++ parser.add_argument(+ "--lr", default=None, type=float, help="learning rate, if None, it will be set to 1.0/iters automatically"+ )++ args = parser.parse_args()+ return args+++def setup_best_parser():+ parser = BasicArgumentParser()++ parser.add_argument("--batch_size", "--train_bs", "--bs", default=8, type=int, help="train batch size")++ parser.add_argument("--iters", "--iter", default=1000, type=int, help="iterations to tune each block")++ parser.add_argument(+ "--seqlen", "--seq_len", default=2048, type=int, help="sequence length of the calibration samples"+ )++ parser.add_argument("--nsamples", "--nsample", default=512, type=int, help="number of samples")++ parser.add_argument(+ "--lr", default=None, type=float, help="learning rate, if None, it will be set to 1.0/iters automatically"+ )++ args = parser.parse_args()+ args.low_gpu_mem_usage = True++ return args+++def setup_light_parser():+ parser = BasicArgumentParser()++ parser.add_argument("--batch_size", "--train_bs", "--bs", default=8, type=int, help="train batch size")++ parser.add_argument("--iters", "--iter", default=50, type=int, help="iterations to tune each block")++ parser.add_argument(+ "--seqlen", "--seq_len", default=2048, type=int, help="sequence length of the calibration samples"+ )++ parser.add_argument("--nsamples", "--nsample", default=128, type=int, help="number of samples")++ parser.add_argument(+ "--lr", default=5e-3, type=float, help="learning rate, if None, it will be set to 1.0/iters automatically"+ )++ args = parser.parse_args()++ return args+++def setup_fast_parser():+ parser = BasicArgumentParser()++ parser.add_argument("--batch_size", "--train_bs", "--bs", default=4, type=int, help="train batch size")++ parser.add_argument("--iters", default=200, type=int, help="iterations to tune each block")++ parser.add_argument(+ "--seqlen", "--seq_len", default=512, type=int, help="sequence length of the calibration samples"+ )++ parser.add_argument("--nsamples", "--nsample", default=128, type=int, help="number of samples")++ parser.add_argument(+ "--lr", default=None, type=float, help="learning rate, if None, it will be set to 1.0/iters automatically"+ )++ args = parser.parse_args()++ return args+++def setup_eval_parser():++ parser = EvalArgumentParser()+ args = parser.parse_args()+ return args+++def tune(args):+ if args.eval_bs is None:+ args.eval_bs = "auto"+ from transformers import AutoConfig, AutoModelForCausalLM, AutoTokenizer++ from auto_round.utils import detect_device, get_library_version, logger++ if args.format is None:+ args.format = "auto_round"++ formats = args.format.lower().replace(" ", "").split(",")+ from auto_round.utils import SUPPORTED_FORMATS++ for format in formats:+ if format not in SUPPORTED_FORMATS:+ raise ValueError(f"{format} is not supported, we only support {SUPPORTED_FORMATS}")++ if "auto_gptq" in args.format and args.asym is True:+ logger.warning(+ "the auto_gptq kernel has issues with asymmetric quantization. "+ "It is recommended to use sym quantization or --format='auto_round'"+ )++ if "marlin" in args.format and args.asym is True:+ raise RuntimeError("marlin backend only supports sym quantization, please remove --asym")++ # Must set this before import torch+ # set_cuda_visible_devices(args.device_map)+ device_str, use_auto_mapping = get_device_and_parallelism(args.device_map)++ import torch++ if args.enable_torch_compile:+ logger.info(+ "`torch.compile` is enabled to reduce tuning costs. "+ "If it causes issues, you can disable it by removing `--enable_torch_compile` argument."+ )++ model_name = args.model+ if model_name[-1] == "/":+ model_name = model_name[:-1]+ logger.info(f"start to quantize {model_name}")++ from auto_round import AutoRound++ if "bloom" in model_name:+ args.low_gpu_mem_usage = False++ if args.quant_lm_head:+ for format in formats:+ if "auto_round" not in format and "fake" not in format:+ auto_round_formats = [s for s in SUPPORTED_FORMATS if s.startswith("auto_round")]+ raise ValueError(+ f"{format} is not supported for lm-head quantization, please change to {auto_round_formats}"+ )++ enable_torch_compile = True if "--enable_torch_compile" in sys.argv else False+ sym = None # the default value should be None now+ if args.asym: # if the scheme is asym, how to set it to sym is an issue+ sym = False+ act_dynamic = None+ if args.disable_act_dynamic:+ act_dynamic = False+ scheme = args.scheme.upper()+ if scheme not in PRESET_SCHEMES:+ raise ValueError(f"{scheme} is not supported. only {PRESET_SCHEMES.keys()} are supported ")+ if args.disable_deterministic_algorithms:+ logger.warning(+ "default not use deterministic_algorithms. disable_deterministic_algorithms is deprecated,"+ " please use enable_deterministic_algorithms instead. "+ )++ from auto_round.compressors import (+ ExtraConfig,+ MLLMExtraConfig,+ SchemeExtraConfig,+ TuningExtraConfig,+ )++ extra_config = ExtraConfig()+ tuning_config = TuningExtraConfig(+ amp=not args.disable_amp,+ lr=args.lr,+ minmax_lr=args.minmax_lr,+ enable_quanted_input=not args.disable_quanted_input,+ nblocks=args.nblocks,+ enable_minmax_tuning=not args.disable_minmax_tuning,+ enable_norm_bias_tuning=args.enable_norm_bias_tuning,+ enable_deterministic_algorithms=args.enable_deterministic_algorithms,+ to_quant_block_names=args.to_quant_block_names,+ disable_opt_rtn=args.disable_opt_rtn,+ enable_alg_ext=args.enable_alg_ext,+ scale_dtype=args.scale_dtype,+ )+ scheme_config = SchemeExtraConfig(+ bits=args.bits,+ group_size=args.group_size,+ sym=sym,+ data_type=args.data_type,+ act_bits=args.act_bits,+ act_group_size=args.act_group_size,+ act_data_type=args.act_data_type,+ act_dynamic=act_dynamic,+ super_bits=args.super_bits,+ super_group_size=args.super_group_size,+ )+ mllm_config = MLLMExtraConfig(+ quant_nontext_module=args.quant_nontext_module, extra_data_dir=args.extra_data_dir, template=args.template+ )+ extra_config.tuning_config = tuning_config+ extra_config.scheme_config = scheme_config+ extra_config.mllm_config = mllm_config++ autoround: BaseCompressor = AutoRound(+ model=model_name,+ scheme=scheme,+ dataset=args.dataset,+ iters=args.iters,+ seqlen=args.seqlen,+ nsamples=args.nsamples,+ batch_size=args.batch_size,+ gradient_accumulate_steps=args.gradient_accumulate_steps,+ low_gpu_mem_usage=args.low_gpu_mem_usage,+ device_map=args.device_map,+ enable_torch_compile=enable_torch_compile,+ seed=args.seed,+ fp_layers=args.fp_layers,+ not_use_best_mse=args.not_use_best_mse,+ enable_adam=args.adam,+ extra_config=extra_config,+ )++ model_name = args.model.rstrip("/")++ if model_name.split("/")[-1].strip(".") == "" and "gguf" not in args.format:+ if autoround.group_size <= 0:+ if "fp" in autoround.act_data_type:+ suffix = f"afp{autoround.act_bits}"+ else:+ suffix = f"a{autoround.act_bits}"+ else:+ suffix = f"g{autoround.group_size}"+ export_dir = os.path.join(args.output_dir, f"w{autoround.bits}{suffix}")+ elif model_name.split("/")[-1].strip(".") == "" and "gguf" in args.format:+ export_dir = args.output_dir+ elif model_name.split("./")[-1].strip("./") != "" and "gguf" in args.format:+ export_dir = os.path.join(args.output_dir, model_name.split("/")[-1] + "-gguf")else:- from auto_round.script.llm import setup_eval_parser+ if autoround.group_size <= 0:+ if "fp" in autoround.act_data_type:+ suffix = f"afp{autoround.act_bits}"+ else:+ suffix = f"a{autoround.act_bits}"+ else:+ suffix = f"g{autoround.group_size}"+ export_dir = os.path.join(args.output_dir, model_name.split("/")[-1] + f"-w{autoround.bits}{suffix}")++ model, folders = autoround.quantize_and_save(export_dir, format=args.format) # pylint: disable=E1101+ tokenizer = autoround.tokenizer # pylint: disable=E1101++ if args.low_cpu_mem_mode == 1 or args.low_cpu_mem_mode == 2:+ import shutil++ shutil.rmtree(args.low_cpu_mem_tmp_dir, ignore_errors=True)++ model.eval()+ clear_memory()++ lm_eval_version = get_library_version("lm-eval")++ eval_folder = folders[-1]+ if args.tasks is None or args.tasks == "" or eval_folder is None:+ return++ tasks = args.tasks+ if isinstance(tasks, str):+ tasks = tasks.split(",")++ from lm_eval.utils import make_table # pylint: disable=E0401++ logger.info(f"Using lm-eval version {lm_eval_version}")+ eval_gguf_model = False+ for file in os.listdir(eval_folder):+ if file.endswith("gguf"):+ eval_gguf_model = True+ break++ import time++ eval_model_dtype = get_model_dtype(args.eval_model_dtype, "auto")++ if autoround.act_bits <= 8 or eval_gguf_model:+ if eval_gguf_model:+ # for file in os.listdir(eval_folder):+ # gguf_file = file+ gguf_file = None+ gguf_format = None # Initialize gguf_format to None+ # gguf folder only contains one file+ for format in formats:+ if format.startswith("gguf"):+ gguf_format = format.split(":")[-1].upper()+ if gguf_format is None: # Validate gguf_format after the loop+ logger.error("No valid gguf format found in formats. Please check the input.")+ sys.exit(-1)+ for file in os.listdir(eval_folder):+ if gguf_format in file:+ gguf_file = file++ logger.warning("evaluating gguf model is an experimental feature, the accuracy may be not correct.")+ if eval_model_dtype == "float32" or eval_model_dtype == "auto":+ logger.warning(+ "set '--eval_model_dtype bf16' can significantly speed up evaluation for gguf model,"+ " but may affect accuracy."+ )+ if gguf_file is None:+ logger.error("Cannot find correct gguf file for evaluation, please check.")+ sys.exit(-1)+ model = AutoModelForCausalLM.from_pretrained(+ eval_folder, gguf_file=gguf_file, device_map="auto", torch_dtype=eval_model_dtype+ )+ model.eval()+ tokenizer = AutoTokenizer.from_pretrained(eval_folder, gguf_file=gguf_file)+ else:+ if hasattr(model, "hf_device_map") and len(model.hf_device_map) > 1:+ from accelerate.big_modeling import dispatch_model++ dispatch_model(model, model.hf_device_map)+ else:+ device_str = detect_device(device_str)+ model = model.to(device_str)+ if model.dtype != eval_model_dtype and eval_model_dtype != "auto":+ model.to(getattr(torch, eval_model_dtype))- args = setup_eval_parser()if args.eval_task_by_task:- from auto_round.script.llm import eval_task_by_task+ eval_task_by_task(+ model,+ tokenizer=tokenizer,+ device=device_str,+ tasks=args.tasks,+ limit=args.limit,+ batch_size=args.eval_bs,+ eval_model_dtype=eval_model_dtype,+ )+ else:+ if args.eval_bs is None or args.eval_bs == "auto":+ logger.warning("This API does not support auto currently, reset eval_bs to 16")+ args.eval_bs = 16+ from auto_round.eval.evaluation import simple_evaluate_user_model+ st = time.time()+ add_bos_token = False+ if "llama" in args.model.lower():+ add_bos_token = True+ res = simple_evaluate_user_model(+ model,+ tokenizer,+ tasks=tasks,+ batch_size=args.eval_bs,+ limit=args.limit,+ device=device_str,+ eval_model_dtype=eval_model_dtype,+ add_bos_token=add_bos_token,+ )+ print(make_table(res))+ print("evaluation running time=%ds" % (time.time() - st))+ else:+ if args.eval_task_by_task:eval_task_by_task(- model=args.model,- device=args.device,+ eval_folder,+ device=device_str,tasks=args.tasks,batch_size=args.eval_bs,- trust_remote_code=not args.disable_trust_remote_code,- eval_model_dtype=args.eval_model_dtype,+ limit=args.limit,+ eval_model_dtype=eval_model_dtype,)else:- from auto_round.script.llm import eval+ from auto_round.eval.evaluation import simple_evaluate- eval(args)+ tasks, model_args, device_str = _eval_init(+ args.tasks, eval_folder, args.device_map, args.disable_trust_remote_code, dtype=eval_model_dtype+ )+ st = time.time()+ if "llama" in args.model.lower():+ model_args += ",add_bos_token=True"+ res = simple_evaluate(+ model="hf",+ model_args=model_args,+ tasks=tasks,+ device=device_str,+ batch_size=args.eval_bs,+ limit=args.limit,+ )+ print(make_table(res))+ print("evaluation running time=%ds" % (time.time() - st))+++def run_eval():+ args = setup_eval_parser()+ if args.eval_task_by_task:+ eval_task_by_task(+ model=args.model,+ device=args.device,+ tasks=args.tasks,+ batch_size=args.eval_bs,+ trust_remote_code=not args.disable_trust_remote_code,+ eval_model_dtype=args.eval_model_dtype,+ )+ else:+ eval(args)def run():@@ -53,8 +670,6 @@ def run():sys.argv.remove("--eval")run_eval()else:- from auto_round.script.llm import setup_parser, tune-args = setup_parser()tune(args)@@ -65,40 +680,19 @@ def run_mllm():def run_best():- from auto_round.script.llm import setup_best_parser, tune-args = setup_best_parser()tune(args)def run_light():- from auto_round.script.llm import setup_light_parser, tune-args = setup_light_parser()tune(args)def run_fast():- from auto_round.script.llm import setup_fast_parser, tune-args = setup_fast_parser()tune(args)-def run_lmms():- # from auto_round.script.lmms_eval import setup_lmms_args, eval- from auto_round.script.mllm import lmms_eval, setup_lmms_parser-- args = setup_lmms_parser()- lmms_eval(args)---def run_vlmeavl():- from auto_round.script.mllm import setup_lmeval_parser, vlmeval-- args = setup_lmeval_parser()- vlmeval(args)--if __name__ == "__main__":run()diff --git a/auto_round/autoround.py b/auto_round/autoround.pyindex 9d56cb8b4..4074213a9 100644--- a/auto_round/autoround.py+++ b/auto_round/autoround.py@@ -1,4 +1,4 @@-# Copyright (c) 2023 Intel Corporation+# Copyright (c) 2025 Intel Corporation## Licensed under the Apache License, Version 2.0 (the "License");# you may not use this file except in compliance with the License.@@ -11,94 +11,25 @@# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.# See the License for the specific language governing permissions and# limitations under the License.+from __future__ import annotations-import copy-import os-import re-import sys-import time-import traceback-from dataclasses import asdict, fields-from enum import Enumfrom typing import Any, Callable, Union-import accelerateimport torch-from accelerate.big_modeling import dispatch_model, infer_auto_device_map-from torch import autocast-from tqdm import tqdm-from transformers import set_seed-from auto_round.data_type import QUANT_FUNC_WITH_DTYPE-from auto_round.data_type.utils import reshape_pad_tensor_by_group_size-from auto_round.export.export_to_autoround import AutoRoundFormat-from auto_round.export.export_to_gguf.config import GGUF_CONFIG, GGUF_INNER_CONFIG, ModelType-from auto_round.logger import logger-from auto_round.low_cpu_mem.utils import get_layers_before_block-from auto_round.schemes import QuantizationScheme, preset_name_to_scheme-from auto_round.sign_sgd import SignSGD-from auto_round.special_model_handler import _handle_moe_model-from auto_round.utils import (- INNER_SUPPORTED_LAYER_TYPES,- SUPPORTED_DTYPES,- SUPPORTED_FORMATS,- SUPPORTED_LAYER_TYPES,- TORCH_VERSION_AT_LEAST_2_6,- CpuInfo,- _gguf_args_check,- _is_fp8_linear,- _is_fp8_model,- block_forward,- check_and_mark_fp8_model,- check_is_cpu,- check_need_act_calibration,- check_seqlen_compatible,- check_skippable_keywords,- check_to_quantized,- clear_memory,- collect_best_params,- compile_func,- convert_dtype_str2torch,- convert_fp8_layer_to_linear,- convert_fp8_model_to_16b_model,- copy_python_files_from_model_cache,- detect_device,- estimate_tuning_block_mem,- find_matching_blocks,- flatten_list,- get_block_names,- get_device_memory,- get_layer_config_by_gguf_format,- get_layer_features,- get_layer_names_in_block,- get_lm_head_name,- get_max_vram,- get_module,- get_quant_keys,- get_shared_keys,- htcore,- infer_bits_by_data_type,- init_cache,- is_debug_mode,- is_mx_fp,- is_nv_fp,- is_optimum_habana_available,- is_standard_fp,- is_static_wfp8afp8,- is_wfp8afp8,- llm_load_model,- mv_module_from_gpu,- reset_params,- set_amax_for_all_moe_layers,- set_module,- to_device,- to_dtype,- unsupport_meta_device,+from auto_round.compressors import (+ AdamCompressor,+ BaseCompressor,+ ExtraConfig,+ LLMCompressor,+ MLLMCompressor,)-from auto_round.wrapper import WrapperLinear, WrapperMultiblock, unwrapper_block, unwrapper_layer, wrapper_block+from auto_round.logger import deprecated, logger+from auto_round.schemes import QuantizationScheme+from auto_round.utils import is_mllm_model-class AutoRound(object):+class AutoRound:"""Automatic weight rounding (Signed Gradient Descent) for LLM quantizationReference:@@ -128,3137 +59,120 @@ class AutoRound(object):super_bits: int | Nonesuper_group_size: int | None- def __init__(- self,+ def __new__(+ cls,model: Union[torch.nn.Module, str],- tokenizer=None,- scheme: Union[str, dict, QuantizationScheme] = "W4A16",- layer_config: dict[str, Union[str, dict, QuantizationScheme]] = None,- dataset: Union[str, list, tuple, torch.utils.data.DataLoader] = "NeelNanda/pile-10k",- iters: int = 200,- seqlen: int = 2048,- nsamples: int = 128,- batch_size: int = 8,- gradient_accumulate_steps: int = 1,- low_gpu_mem_usage: bool = False,- device_map: Union[str, torch.device, int, dict] = 0,- enable_torch_compile: bool = False,- seed: int = 42,- **kwargs,- ):- """Initialize AutoRound with quantization and tuning configuration.-- Args:- model (torch.nn.Module | str): Model object or model name to load.- tokenizer: Tokenizer for text processing. Required if `model` is not a string and `iters > 0`.- scheme (str| dict | QuantizationScheme ): A preset scheme that defines the quantization configurations- bits (int, optional): Weight quantization bits. Defaults to 4.- group_size (int, optional): Weight quantization group size. Defaults to 128.- sym (bool, optional): Symmetric weight quantization. Defaults to True.- layer_config (dict, optional): Layer-wise quantization config. Defaults to None.- batch_size (int, optional): Calibration batch size. Defaults to 8.- amp (bool, optional): Use AMP for tuning. Defaults to True.- device (str | torch.device | int, optional): Compute device. Defaults to 0.- dataset (str | list | tuple | DataLoader, optional): Calibration data. Defaults to "NeelNanda/pile-10k".- enable_minmax_tuning (bool, optional): Enable weight min-max tuning. Defaults to True.- lr (float, optional): Learning rate; if None, set to 1.0 / iters except when iters==0.- minmax_lr (float, optional): Learning rate for min-max tuning; defaults to `lr`.- low_gpu_mem_usage (bool, optional): Lower GPU memory mode. Defaults to False.- iters (int, optional): Optimization iterations. Defaults to 200.- seqlen (int, optional): Calibration sequence length. Defaults to 2048.- nsamples (int, optional): Number of calibration samples. Defaults to 128.- seed (int, optional): Random seed. Defaults to 42.- gradient_accumulate_steps (int, optional): Gradient accumulation steps. Defaults to 1.- data_type (str, optional): Weight data type string, e.g., "int". Defaults to "int".- act_bits (int, optional): Activation quantization bits. Defaults to 16.- act_group_size (int, optional): Activation group size. Defaults to None.- act_sym (bool, optional): Symmetric activation quantization. Defaults to None.- act_data_type (str, optional): Activation data type; inherits weight dtype if None and act_bits < 16.- act_dynamic (bool, optional): Dynamic activation quantization. Defaults to True.- enable_torch_compile (bool, optional): Enable torch.compile for quant blocks/layers. Defaults to False.- device_map (str | dict, optional): Device placement map. Defaults to None.- disable_opt_rtn (bool, optional): Disable RTN-mode optimization (iters=0). Defaults to False.- enable_alg_ext (bool, optional): Enable algorithm extension (primarily for INT2). Defaults to False.- **kwargs: Backward compatible options:- - enable_alg_ext, quant_lm_head, lr, lr_scheduler, sampler, not_use_best_mse, dynamic_max_gap,- super_group_size, super_bits, scale_dtype ("fp16" etc.),- nblocks, low_cpu_mem_usage, to_quant_block_names,- enable_norm_bias_tuning, enable_quanted_input,- disable_deterministic_algorithms, vlm, static_kv_dtype- Raises:- ValueError: If invalid device is provided or tokenizer is missing for non-str model with iters > 0.- RuntimeError: If model parameters are on meta device.- Example:- Layer-wise configuration structure:-- >>> layer_config = {- ... "layer1": {- ... "data_type": "int",- ... "bits": 4,- ... "group_size": 128,- ... "sym": True,- ... "act_data_type": None,- ... "act_bits": 16,- ... "act_group_size": None,- ... "act_sym": None,- ... },- ... # ...- ... }- """- self.scheme = None- self._parse_and_set_scheme(scheme, kwargs)-- # Extra/legacy kwargs for backward compatibility- # Major version releases may pack them with extra configuration options- amp = kwargs.pop("amp", True)- lr = kwargs.pop("lr", None)- enable_alg_ext = kwargs.pop("enable_alg_ext", False)- enable_minmax_tuning = kwargs.pop("enable_minmax_tuning", True)- minmax_lr = kwargs.pop("minmax_lr", None)- disable_opt_rtn = kwargs.pop("disable_opt_rtn", False)- lr_scheduler = kwargs.pop("lr_scheduler", None)- sampler = kwargs.pop("sampler", "rand")- not_use_best_mse = kwargs.pop("not_use_best_mse", False)- dynamic_max_gap = kwargs.pop("dynamic_max_gap", -1)- scale_dtype = kwargs.pop("scale_dtype", "fp16")- nblocks = kwargs.pop("nblocks", 1)- low_cpu_mem_usage = kwargs.pop("low_cpu_mem_usage", False)- to_quant_block_names: Union[str, list, None] = kwargs.pop("to_quant_block_names", None)- enable_norm_bias_tuning: bool = kwargs.pop("enable_norm_bias_tuning", False)- enable_quanted_input: bool = kwargs.pop("enable_quanted_input", True)- disable_deterministic_algorithms = kwargs.pop("disable_deterministic_algorithms", True)- enable_deterministic_algorithms = kwargs.pop("enable_deterministic_algorithms", False)- static_kv_dtype = kwargs.pop("static_kv_dtype", None)- device = kwargs.pop("device", None)- self.quant_lm_head = kwargs.pop("quant_lm_head", False)- self.vlm = kwargs.pop("vlm") if "vlm" in kwargs else False- # Scale factor for RAM usage per parameter.- self.mem_per_param_scale = kwargs.pop("mem_per_param_scale", None)-- if kwargs:- logger.warning(f"unrecognized keys {list(kwargs.keys())} were passed. Please check them.")- if "CUBLAS_WORKSPACE_CONFIG" not in os.environ:- os.environ["CUBLAS_WORKSPACE_CONFIG"] = ":4096:8"- # deprecated, default not to use torch.use_deterministic_algorithms- if not disable_deterministic_algorithms or enable_deterministic_algorithms:- if not disable_deterministic_algorithms:- logger.warning(- "default not use deterministic_algorithms. disable_deterministic_algorithms is deprecated,"- " please use enable_deterministic_algorithms instead. "- )-- torch.use_deterministic_algorithms(True, warn_only=False)- else:- torch.use_deterministic_algorithms(True, warn_only=True)-- if device is not None:- logger.warning("`device` is deprecated, please use `device_map` instead")-- if device_map is None:- device_map = 0-- # Set device, must place after model loading- self._set_device(device_map)-- if (isinstance(device_map, dict) and device_map) or device_map == "auto":- self.device_map = device_map- elif isinstance(device_map, str) and "," in device_map:- device_map = device_map.replace(" ", "") # Remove any spaces- self.device_list = [int(dev) for dev in device_map.split(",") if dev.isdigit()]- self.device_map = "auto"- else:- self.device_map = None- self._set_device_map_in_blocks(self.device_map)-- # Model related- self.quantized = False- if isinstance(model, str):- model, tokenizer, low_cpu_mem_usage = llm_load_model(- model,- device="cpu",- low_cpu_mem_mode=low_cpu_mem_usage, # always load cpu first- )- elif tokenizer is None and iters > 0:- raise ValueError("A tokenizer must be set for non-str model input")- self.low_cpu_mem_usage = bool(low_cpu_mem_usage)- if unsupport_meta_device(model):- raise RuntimeError(- "AutoRound does not support parameters on meta device. "- "Please use more GPUs by setting `--device 0,1,2,3` or just place the model on CPU."- )- check_and_mark_fp8_model(model)- self.model = model.eval()- self.tokenizer = tokenizer- self.shared_cache_keys = get_shared_keys(self.model)-- self._parse_layer_config(layer_config) # must place after model init-- self.to_quant_block_names = to_quant_block_names-- # Tuning hyperparameters- self.seed = seed- set_seed(self.seed)- self.amp = amp- self.enable_quanted_input = enable_quanted_input- self.enable_minmax_tuning = enable_minmax_tuning- self.nsamples = nsamples- self.enable_norm_bias_tuning = enable_norm_bias_tuning- self.low_gpu_mem_usage = low_gpu_mem_usage- self.seqlen = seqlen- self.batch_size, self.gradient_accumulate_steps = batch_size, gradient_accumulate_steps- self.nblocks = nblocks- self.dataset = dataset- self.iters = iters- if self.iters < 0:- logger.warning("`iters` must be non-negative, reset it to 200")- self.iters = 200- if self.iters == 0:- self.lr = 5e-3- else:- self.lr = lr or (1.0 / self.iters) # must place after iter setting- self.minmax_lr = minmax_lr or self.lr- self.enable_alg_ext = enable_alg_ext- self.sampler = sampler- self.not_use_best_mse = not_use_best_mse- self.dynamic_max_gap = dynamic_max_gap- self.lr_scheduler = lr_scheduler- self.optimizer = self._get_optimizer(None)- self.disable_opt_rtn = disable_opt_rtn- self.is_packing_immediate = False # whether to pack the layer immediately after tuning-- # KV cache, this one does not affect tuning but will collect some infos during tuning- self.static_kv_dtype = static_kv_dtype- if self.static_kv_dtype is not None:- logger.warning("The static kv is experimental and currently has limited support.")-- # Model related- self.quantized = False- if isinstance(model, str):- model, tokenizer, low_cpu_mem_usage = llm_load_model(- model, device=device, low_cpu_mem_mode=low_cpu_mem_usage- )- elif tokenizer is None and iters > 0:- raise ValueError("A tokenizer must be set for non-str model input")- self.low_cpu_mem_usage = bool(low_cpu_mem_usage)- if unsupport_meta_device(model):- raise RuntimeError(- "AutoRound does not support parameters on meta device. "- "Please use more GPUs by setting `--device_map 0,1,2,3` or just place the model on CPU."- )- self.model = model.eval()- self.tokenizer = tokenizer- self.shared_cache_keys = get_shared_keys(self.model)- if not hasattr(self, "quant_block_list"):- all_blocks = get_block_names(model)- self.quant_block_list = find_matching_blocks(model, all_blocks, self.to_quant_block_names)-- self.scale_dtype = convert_dtype_str2torch(scale_dtype)- self._set_amp_dtype()- self.cache_device = torch.device("cpu") if self.low_gpu_mem_usage else self.device- if self.act_bits <= 8 and self.amp_dtype == torch.float16:- logger.warning("force to use bf16 to for quantization tuning when enabling activation quantization")- self.amp_dtype = torch.bfloat16- if self.model.dtype != torch.bfloat16: # keep the model's buffer dtype unchanged- self.model = self.model.to(torch.bfloat16)- else:- logger.info(f"using {self.model.dtype} for quantization tuning")-- # Some helpers- self.supported_types = SUPPORTED_LAYER_TYPES- self.inner_supported_types = INNER_SUPPORTED_LAYER_TYPES- if "hpu" in str(self.device):- self.inner_supported_types = tuple(x for x in INNER_SUPPORTED_LAYER_TYPES if x != "FP8Linear")- self.batch_dim = None- self.infer_bs_coeff = 1- self.enable_torch_compile = enable_torch_compile- self._adjust_torch_compile(enable_torch_compile)- self._check_configs()- torch.set_printoptions(precision=3, sci_mode=True)-- if is_optimum_habana_available():- logger.info("optimum Habana is available, import htcore explicitly.")- import habana_frameworks.torch.core as htcore # pylint: disable=E0401- import habana_frameworks.torch.hpu as hthpu # pylint: disable=E0401]-- def _set_device(self, device_map):- if hasattr(self, "device") and self.device is not None:- return- if isinstance(device_map, (str, torch.device, int)):- self.device = detect_device(device_map)-- elif isinstance(device_map, dict) and device_map:- tmp_devices = []- for val in device_map.values():- if isinstance(val, (str, torch.device, int)): # could optimize- tmp_device = detect_device(val)- tmp_device = tmp_device.split(":")[0]- tmp_devices.append(tmp_device)- tmp_devices = list(set(tmp_devices))- if len(tmp_devices) > 1:- logger.warning(- f"there are multiple device types in the device_map, "- f"please make sure they are correct,use the first device {tmp_devices[0]} as the core device "- )-- self.device = tmp_devices[0]- else:- raise TypeError(f"device_map should be [str, torch.device, int, dict], but got {type(device_map)}")-- def _parse_layer_config(self, layer_config: dict[str, Union[str, dict, QuantizationScheme]]) -> None:- """Parse and set the layer-wise quantization configuration."""- # Some other quantization configs- self.layer_config = {} if layer_config is None else layer_config- scheme_keys = [f.name for f in fields(QuantizationScheme)]- for key, item in self.layer_config.items():- if isinstance(item, str):- item = asdict(preset_name_to_scheme(item.upper()))- self.layer_config[key] = item-- if isinstance(item, QuantizationScheme):- config = asdict(item)- tmp_keys = copy.deepcopy(list(config.keys()))- for tmp_key in tmp_keys: ## Pop None value to be overridden- if config[tmp_key] is None:- config.pop(tmp_key)- self.layer_config[key] = config- elif isinstance(item, dict):- item_keys = item.keys()- if item_keys not in scheme_keys:- for item_key in item_keys:- if item_key not in scheme_keys:- raise ValueError(- f"the key {item_key} in layer_config for layer {key} is invalid,"- f" only {scheme_keys} are supported"- )-- if not self.quant_lm_head or (isinstance(self.scheme, str) and self.scheme.lower().startswith("gguf")):- return- for n, _ in self.model.named_modules():- lm_head_layer_name = n-- if (- hasattr(self.model, "config")- and self.model.config.tie_word_embeddings- and hasattr(self.model, "_tied_weights_keys")- ):- tied_keys = self.model._tied_weights_keys- for item in tied_keys:- if lm_head_layer_name in item: # TODO extend to encoder-decoder layer, seq classification model- self.quant_lm_head = False- logger.warning(- "reset `quant_lm_head` to `False` as quantizing lm_head with tied weights has not been "- "supported currently"- )- break-- lm_head_layer_config = self.layer_config[lm_head_layer_name] if lm_head_layer_name in self.layer_config else {}-- for key in scheme_keys:- if key not in lm_head_layer_config:- lm_head_layer_config[key] = getattr(self, key)-- def _parse_and_set_scheme(self, scheme: Union[str, dict, QuantizationScheme], kwargs) -> None:- """Parse and set the quantization scheme."""- if isinstance(scheme, QuantizationScheme):- scheme = asdict(scheme)- elif isinstance(scheme, dict):- scheme = scheme- elif isinstance(scheme, str):- scheme = scheme.upper()- self.scheme = scheme- scheme = asdict(preset_name_to_scheme(scheme))- scheme_keys = [f.name for f in fields(QuantizationScheme)]- for key in scheme_keys:- if key in kwargs and kwargs[key] is not None:- setattr(self, key, kwargs[key])- else:- setattr(self, key, scheme.get(key, None))- kwargs.pop(key, None)- if self.act_dynamic is None:- self.act_dynamic = True-- tmp_bits = infer_bits_by_data_type(self.data_type)- if tmp_bits is not None and tmp_bits < 16 and tmp_bits != self.bits:- logger.warning(f"'data_type' do not match the specified 'bits' setting. Resetting 'bits' to {tmp_bits}.")- self.bits = tmp_bits- if tmp_bits is not None and tmp_bits < 16:- for supported_dtype in SUPPORTED_DTYPES: # to easily handle dtype mx_fp4 and layer_config={xxx:{bits:8}}- if self.data_type.startswith(supported_dtype):- if supported_dtype + str(tmp_bits) == self.data_type: # could not replace FP8_e4m3- self.data_type = supported_dtype- break-- self.act_group_size = self.act_group_size if self.act_group_size is not None else self.group_size- self.act_bits = self.act_bits if self.act_bits is not None else 16- self.act_sym = self.act_sym if self.act_sym is not None else self.sym-- if self.act_data_type is None:- if self.data_type in SUPPORTED_DTYPES and self.act_bits < 16:- self.act_data_type = self.data_type- logger.info(f"activation adopts {self.data_type}")- else:- self.act_data_type = "float"- tmp_act_bits = infer_bits_by_data_type(self.act_data_type)- if tmp_act_bits is not None and tmp_act_bits < 16 and tmp_act_bits != self.act_bits:- self.act_bits = tmp_act_bits- logger.warning(- f"`act_data_type` do not"- f" match the specified 'act_bits' setting. Resetting 'act_bits' to {tmp_act_bits}."- )- if tmp_act_bits is not None and tmp_act_bits < 16:- for supported_dtype in SUPPORTED_DTYPES: # to easily handle dtype mx_fp4 and layer_config={xxx:{bits:8}}- if self.act_data_type.startswith(supported_dtype):- if supported_dtype + str(tmp_act_bits) == self.act_data_type: # could not replace FP8_e4m3- self.act_data_type = supported_dtype- break-- def _adjust_torch_compile(self, enable_torch_compile: bool) -> None:- """Sets the torch compile configuration for the tuning."""- self.enable_torch_compile = enable_torch_compile- if (- not self.enable_torch_compile- and TORCH_VERSION_AT_LEAST_2_6- and self.act_bits > 8- and not is_debug_mode()- and not self.low_cpu_mem_usage- and "fp8" not in self.data_type- and "fp8" not in self.act_data_type- ):- logger.info(- "'enable_torch_compile' is set to `False` by default. "- "Enabling it can reduce tuning cost by 20%, but it might throw an exception."- )-- if self.low_cpu_mem_usage and self.enable_torch_compile:- self.enable_torch_compile = False- logger.warning("reset enable_torch_compile to `False` as low_cpu_mem_usage is enabled")-- if is_debug_mode() and self.enable_torch_compile:- self.enable_torch_compile = False- logger.warning("reset enable_torch_compile to `False` as debug mode is enabled")-- if ("fp8" in self.data_type or "fp8" in self.act_data_type) and self.enable_torch_compile:- self.enable_torch_compile = False- logger.warning("reset enable_torch_compile to `False` as fp8 is enabled")-- def _set_device_map_in_blocks(self, device_map: Union[str, dict, None]) -> None:- """Sets the device map for specific blocks in the model.-- Args:- device_map (Union[str, dict, None]): A mapping of module names to devices.- If provided as a string, it should be in the format- "module_name:device,module_name:device". Devices can be integers- (GPU IDs) or strings (e.g., 'cpu', 'cuda:0').- """- if self.device_map is None or len(self.device_map) == 0:- self.device_map = None- if not device_map:- return- if self.device_map == "auto" and device_map == "auto":- return- if isinstance(device_map, str):- device_map = device_map.replace(" ", "")- infos = device_map.split(",")- device_map_dict = {}- for info in infos:- index = info.find(":")- key = info[:index]- value = info[index + 1 :]- device_map_dict[key] = value- device_map = device_map_dict-- names = [n for n, m in self.model.named_modules() if len(list(m.children())) == 0]-- for key, device in device_map.items():- if isinstance(device, str) and device.isdigit():- device = int(device)- device = detect_device(device)- try:- module = get_module(self.model, key)- module.tuning_device = device- except:- matching_names = [name for name in names if re.match(key, name)]- if len(matching_names) > 0:- for name in matching_names:- self._set_device_for_matching_module(name, device)- else:- for name in names:- if key in name:- self._set_device_for_matching_module(name, device)-- def _set_device_for_matching_module(self, name: str, device: str) -> None:- """Sets the device for a module if it matches the given name."""- module = get_module(self.model, name)- if hasattr(module, "tuning_device") and module.tuning_device != device:- logger.warning(- f"multiple devices have been set for layer {name}, keeping original device {module.tuning_device}"- )- else:- module.tuning_device = device-- def _set_auto_device_map_in_block(self, block: torch.nn.Module, input_ids: list[torch.Tensor]) -> None:- """Automatically sets the device map for the block based on available GPUs and memory constraints."""- if torch.cuda.is_available():- num_gpus = torch.cuda.device_count()- elif torch.xpu.is_available():- logger.warning_once("XPU does not support auto device map yet, using device 0 for tuning.")- return- else:- raise RuntimeError("No CUDA or XPU devices found.")- if num_gpus <= 1:- self.device_map = None- return-- if hasattr(self, "device_list") and self.device_list:- cuda_devices = [f"cuda:{i}" for i in self.device_list]- device_0 = cuda_devices[0]- else:- cuda_devices = [f"cuda:{i}" for i in range(num_gpus)]- device_0 = "cuda:0"-- device_0_memory = get_device_memory(- self.device_list[0] if hasattr(self, "device_list") and self.device_list else 0- )- block_memory, input_ouput_memory = estimate_tuning_block_mem(block, input_ids)- if self.low_gpu_mem_usage:- input_ouput_memory = 0-- mem_per_param_scale = 13 if self.mem_per_param_scale is None else self.mem_per_param_scale- if self.iters == 0:- mem_per_param_scale = 1 # for rtn-- if (block_memory * mem_per_param_scale + input_ouput_memory) < device_0_memory:- return # fit in one GPU-- device_map = {}- device_memory = {device: get_device_memory(int(device.split(":")[1])) for device in cuda_devices}- device_memory[device_0] = device_0_memory - input_ouput_memory-- device_idx = 0- # First, fill device 0 to its maximum capacity, then distribute the remaining layers evenly across other devices- for n, m in block.named_modules():- if check_to_quantized(m):- layer_name = block.tmp_name + "." + n- layer_memory = m.weight.nbytes / 1024**3- if device_idx == 0 and layer_memory * mem_per_param_scale < device_memory[cuda_devices[device_idx]]:- device_map[layer_name] = cuda_devices[device_idx]- device_memory[cuda_devices[device_idx]] -= layer_memory * mem_per_param_scale- elif device_idx == 0:- device_idx += 1 # Move to the next device once device 0 is full- device_map[layer_name] = cuda_devices[device_idx]- device_memory[cuda_devices[device_idx]] -= layer_memory * mem_per_param_scale- else:- # Calculate the target device index based on even distribution- sorted_devices = sorted(cuda_devices, key=lambda d: device_memory[d], reverse=True)- device_idx = sorted_devices[0]- if layer_memory * mem_per_param_scale < device_memory[device_idx]:- device_map[layer_name] = device_idx- device_memory[device_idx] -= layer_memory * mem_per_param_scale- else:- logger.warning_once(- f"Block {block.tmp_name} not fit in available GPU memory. "- "Consider using more GPUs or reducing mem_per_param_scale if OOM occurs."- )- self._set_device_map_in_blocks(device_map)-- def _dq_check(self) -> None:- """Reset the default value of super_bits and super_group_size"""- if self.data_type.endswith("_dq"):- gguf_config = GGUF_INNER_CONFIG[f"gguf:q{self.bits}_k"]- self.super_bits = gguf_config["super_bits"] if self.super_bits is None else self.super_bits- self.super_group_size = (- gguf_config["super_group_size"] if self.super_group_size is None else self.super_group_size- )-- def _check_configs(self) -> None:- """Checks if the configurations are valid.-- Raises:- ValueError, TypeError: If any of the configurations are invalid.- """- if not isinstance(self.model, torch.nn.Module):- raise TypeError("model must be an instance of torch.nn.Module")- if self.bits <= 0:- raise ValueError("`bits` must be positive")- if self.act_bits <= 0:- raise ValueError("`act_bits` must be positive")- if not (self.group_size == -1 or self.group_size >= 0):- raise ValueError("`group_size` must be -1 (per channel) or 0 (per-tensor) or a positive integer")- if not (self.act_group_size == -1 or self.act_group_size >= 0):- raise ValueError("`act_group_size` must be -1 (per channel) or 0 (per-tensor) or a positive integer")- if self.batch_size <= 0:- raise ValueError("`batch_size` must be positive")- if self.iters < 0:- raise ValueError("`iters` must be non-negative")- if self.seqlen <= 0:- raise ValueError("`seqlen` must be positive")- if self.nblocks <= 0:- raise ValueError("`nblocks` must be positive")- if self.gradient_accumulate_steps <= 0:- raise ValueError("`gradient_accumulate_steps` must be positive")-- if (- self.act_bits <= 8- and (not is_nv_fp(self.act_data_type) or "static_gs" not in self.act_data_type)- and not is_mx_fp(self.act_data_type)- ):- logger.warning(- "activation quantization is an experimental feature with limited support and a complex API. "- "And please save the quantized model to fake format as real deployment is not supported currently"- )-- if is_mx_fp(self.data_type) and self.group_size != 32:- logger.warning("dtype mx_fp should only support group_size of 32 in real deployment")-- if is_nv_fp(self.data_type) and (self.group_size != 16):- logger.warning("dtype nv_fp should only support group_size of 16 in real deployment")-- if self.nsamples < self.gradient_accumulate_steps * self.batch_size:- if self.batch_size > self.nsamples:- if self.iters > 0: # GGUF should log this warning, but we don't know the format here- logger.warning(- f"reset `batch_size` to {self.nsamples} as `nsamples`({self.nsamples})"- f" is smaller than batch_size({self.batch_size})"- )- self.batch_size = self.nsamples- if self.gradient_accumulate_steps > self.nsamples // self.batch_size:- self.gradient_accumulate_steps = self.nsamples // self.batch_size- logger.warning(- f"reset `gradient_accumulate_steps` to {self.gradient_accumulate_steps}"- f" as nsamples must equal or greater"- f" than gradient_accumulate_steps * batch_size"- )-- if self.enable_norm_bias_tuning:- logger.warning("the `enable_norm_bias_tuning` feature is experimental and currently has limited support.")-- self._dq_check()-- def _check_compatibility(self) -> None:- """Checks compatibility of the configurations and model."""- # Check gguf and others- has_gguf = False- if hasattr(self, "formats"):- has_besides_gguf = False- for format_ in self.formats:- if "gguf" in format_:- has_gguf = True- elif format_ != "fake":- has_besides_gguf = True- if has_gguf and has_besides_gguf:- raise ValueError("Gguf format is not compatible with other formats, please choose only one of them")- if has_gguf and self.iters != 0 and self.bits != 3:- logger.warning(- "`iters=0` is recommended when exporting to GGUF format except for bits 3,"- " as we have optimized the RTN method for this case."- " We are likely to release new algorithm for certain configurations in the future."- )-- # Check group_size 32 for auto_round- if (- self.data_type == "int"- and hasattr(self, "formats")- and any(key in fmt for fmt in self.formats for key in ("auto_round", "auto_gptq", "auto_awq"))- ):- for n, m in self.model.named_modules():- if isinstance(m, self.supported_types):- if m.weight.shape[0] % 32 != 0 or m.weight.shape[1] % 32 != 0:- self.layer_config[n] = {"bits": 16}- logger.info(- f"{n} will not be quantized due to its shape not being divisible by 32,"- " resulting in an exporting issue to autogptq"- )-- if (- self.seqlen is not None- and hasattr(self.model, "config")- and hasattr(self.model.config, "max_position_embeddings")- ):- if self.model.config.max_position_embeddings < self.seqlen:- logger.warning(- f"Change sequence length to {self.model.config.max_position_embeddings} "- "due to the limitation of max_position_embeddings"- )- self.seqlen = min(self.seqlen, self.model.config.max_position_embeddings)-- if self.seqlen is not None and hasattr(self.tokenizer, "model_max_length"):- if self.tokenizer.model_max_length < self.seqlen:- logger.warning(- f"Change sequence length to {self.tokenizer.model_max_length} "- "due to the limitation of model_max_length. "- "You can also try to increase the model_max_length to avoid this issue."- )- self.seqlen = min(self.seqlen, self.tokenizer.model_max_length)-- if self.group_size == 0 and "fp8" not in self.data_type:- logger.warning("`group_size==0` is not supported for data_type other than fp8 ")-- def _parse_format_to_list(self, format: str) -> list:- """Parses the format string into a list of formats.-- This method checks the requested format(s) against the model's- quantization settings and adjusts them if necessary. It ensures that- the formats are compatible with the model's data type, bit width,- and activation quantization settings.-- Args:- format (str): The requested format(s) for quantization, separated by commas.-- Returns:- list: A list of validated and updated formats.- """-- # Remove duplicates from formats list- def remove_duplicates(lst):- seen = set()- return [x for x in lst if not (x in seen or seen.add(x))]-- formats = format.replace("q*_", f"q{self.bits}_").replace(" ", "").split(",")- formats = remove_duplicates(formats) # need the keep origin order-- if isinstance(self.scheme, str) and self.scheme.lower().startswith("gguf"):- for i in range(len(formats)):- if formats[i] != "fake" and formats[i] != self.scheme.lower().startswith("gguf"):- logger.warning(- f"reset format {formats[i]} to {self.scheme.lower()} "- f"since scheme {self.scheme} can only be exported to format {self.scheme.lower()}"- )- formats[i] = self.scheme.lower()-- _gguf_args_check(self, formats, model_type=ModelType.TEXT)- if self.vlm:- _gguf_args_check(self, formats, model_type=ModelType.MMPROJ)-- for format_ in formats:- if format_ not in SUPPORTED_FORMATS:- logger.error(f"Unsupported format {format_}, please choose from {SUPPORTED_FORMATS}")- exit(-1)- if self.scale_dtype != torch.float32:- only_gguf = True- for format_ in formats:- if not ("gguf" in format_ or "fake" in format_):- only_gguf = False- break- if len(formats) == 1 and "fake" == formats[0]:- only_gguf = False- if only_gguf:- self.scale_dtype = torch.float32- logger.info("change `scale_dtype` to `torch.float32`")-- # Adjust format settings based on compatibility- for index in range(len(formats)):- format = formats[index]- if format == "auto_round":- if self.sym and "int" in self.data_type:- format = "auto_round:auto_gptq"- elif self.bits == 4 and not self.sym and "int" in self.data_type:- enable_awq = all(- config["bits"] == self.bits or config["bits"] >= 16 for config in self.layer_config.values()- )- if enable_awq:- format = "auto_round:auto_awq"- elif is_nv_fp(self.data_type) or is_mx_fp(self.data_type):- format = f"auto_round:{self.data_type}"- elif is_static_wfp8afp8(self): # staic wfp8afp8- format = f"auto_round:{AutoRoundFormat.TORCH_FP8_STATIC.value}"- elif self.data_type == "fp" and self.bits == 8 and self.act_bits >= 16: # woq fp8- format = "auto_round:fp8"- elif self.act_bits < 16:- raise ValueError(- "AutoRound format does not support exporting "- "for the current quantization configuration, "- "please change to `fake` format for research purpose"- )- formats[index] = format- elif format == "llm_compressor":- from auto_round.export.export_to_llmcompressor import check_compressed_tensors_supported-- if is_nv_fp(self.data_type) or is_mx_fp(self.data_type):- check_compressed_tensors_supported()- format = format.replace("llm_compressor", f"llm_compressor:{self.data_type}")- formats[index] = format- elif not is_wfp8afp8(self):- logger.error(- "Currently, the llm_compressor format only supports MXFP/NVFP/FP8. "- "Please change format to fake or auto_round etc."- )- else:- if (is_nv_fp(self.data_type) or is_mx_fp(self.data_type)) and format != "fake":- logger.warning(f"nv_fp and mx_fp dtypes are not supported for export format: {format}")-- formats = remove_duplicates(formats)- for i in range(len(formats)):- formats[i] = self._check_supported_format(formats[i])- formats = remove_duplicates(formats)- return formats-- def _check_supported_format(self, format: str) -> bool:- """Checks if the specified format is supported.-- This method validates the requested format against the model's bit width,- group size, symmetry, and activation quantization settings. It raises an- error if the format is incompatible with the current model configuration.-- Args:- format (str): The requested format for quantization.-- Returns:- bool: True if the format is supported, False otherwise.- """- if format == "fake":- return format- format = format.replace("q*_", f"q{self.bits}_")-- # format check for fp8- w_fp8 = self.data_type == "fp" and self.bits == 8- act_fp8 = self.act_data_type == "fp" and self.act_bits == 8- if (w_fp8 or act_fp8) and re.search("^auto_round|^llm_compressor", format) is None:- error_msg = (- f"is only supported to export auto_round or llm_compressor format," f" but got {format}, please check."- )- error_msg = ("act_data_type<fp8> " + error_msg) if act_fp8 else error_msg- error_msg = ("data_type<fp8> " + error_msg) if w_fp8 else error_msg- logger.error(error_msg)- sys.exit(-1)-- # Only support to export afp8/nv_fp/mx_fp- if self.act_bits <= 8:- if not is_standard_fp(self.act_data_type) or self.act_dynamic:- if "llm_compressor" in format:- if (is_nv_fp(self.act_data_type) and "static_gs" in self.act_data_type) or (- is_mx_fp(self.act_data_type)- ):- return format- bits, group_size, sym, act_bits = 8, -1, True, 8- assert (- self.bits == bits- and self.group_size == group_size- and self.sym == sym- and self.act_bits == act_bits- and self.act_dynamic- ), (- f"Currently only support to export llm_compressor format for sym dynamic quantized"- f" W{self.bits}A{self.act_bits} model with group_size={group_size},"- f" but got bits={self.bits}, group_size={self.group_size}, sym={self.sym},"- f" act_bits={self.act_bits}"- )- elif "auto_round" in format and (- is_mx_fp(self.act_data_type) or (is_nv_fp(format) and "static_gs" in self.act_data_type)- ):- logger.warning(- f"AutoRound supports exporting to format '{format}', "- "but loading quantized models in this format is not yet supported. "- "It is currently recommended to export to the 'llm_compressor' format."- )- elif format != "fake":- logger.warning(- "Currently only support to export auto_round format quantized model"- " with fp8, mx_fp and nv_fp4 dtype activation for activation quantization."- " Change format to fake and save."- )- format = "fake"- else:- if not (format == "auto_round" or format == f"auto_round:{AutoRoundFormat.TORCH_FP8_STATIC.value}"):- logger.warning(- f"Currently only support to export auto_round or fake format for static W{self.bits}AFP8 model,"- f" change format {format} to auto_round"- )- format = "auto_round"- if self.act_group_size != 0 and not self.act_dynamic and format == "auto_round:fp8":- logger.warning(- f"Please note that quantize activation with act_group_size={self.act_group_size}"- " may result in failure to export or import normally."- )- if re.search(r"q\d_k", format) and not self.data_type.endswith("_dq"):- logger.error(- f"datatype<{self.data_type}> not support to export {format} format."- " Please change export format or `data_type`."- )- sys.exit(-1)-- return format-- def quantize_and_save(- self, output_dir: str = "tmp_autoround", format: str = "auto_round", inplace: bool = True, **kwargs- ) -> tuple[torch.nn.Module, dict[str, Any]]:- """Quantizes the model and saves it in the specified format(s).-- This function checks the validity of the requested format(s), quantizes- the model accordingly, and saves it to the specified output directory.- If multiple formats are provided, the model is saved separately for each format.-- Args:- output_dir (str, optional): The directory where the quantized model- will be saved. Defaults to "tmp_autoround".- format (str, optional): The quantization format(s) to use, separated- by commas if multiple. Defaults to "auto_round".- inplace (bool, optional): Whether to modify the model in place if only- one format is used. Defaults to True.- **kwargs: Additional arguments for the quantization and saving process.-- Returns:- model: A qdq model or packed model based on the configurations- folders: The folder paths where the quantized models are saved.-- Raises:- ValueError: If an unsupported format is specified.- """- # Validate and process the specified formats- self.orig_output_dir = output_dir-- # check and update the format based on the current configuration- format_list = self._parse_format_to_list(format)- self.formats = format_list-- # If multiple formats are specified, enforce inplace=False- if len(format_list) > 1:- inplace = False- self.inplace = kwargs.get("inplace", inplace)- kwargs.pop("inplace", None)-- # Perform model quantization- if self.static_kv_dtype is not None:- from auto_round.experimental.kv_cache import kvcache_quant_context-- with kvcache_quant_context(self.model, static_kv_dtype=self.static_kv_dtype):- model, _ = self.quantize()- else:- model, _ = self.quantize()- # Save the quantized model in the specified format_list- folders = []- for format in format_list:- if "gptq" in format and not self.sym:- logger.warning(- "The asymmetrical kernel of the GPTQ format may result in a noticeable accuracy drop,"- " particularly for 2-bit quantization and smaller models."- " We recommend exporting to either the AutoAWQ format ( only 4 bits) or "- "the AutoRound format(2/3/4/8 bits)."- )- save_folder = self._get_save_folder_name(format)- self.save_quantized(save_folder, format=format, inplace=inplace, **kwargs)-- folders.append(save_folder)-- return model, folders-- def _get_save_folder_name(self, format_str: str) -> str:- """Generates the save folder name based on the provided format string.-- If there are multiple formats to handle, the function creates a subfolder- named after the format string with special characters replaced. If there's- only one format, it returns the original output directory directly.-- Args:- format_str (str): The format identifier (e.g., 'gguf:q2_k_s').-- Returns:- str: The path to the folder where results should be saved.- """- # Replace special characters to make the folder name filesystem-safe- sanitized_format = format_str.replace(":", "-").replace("_", "-")-- # Use a subfolder only if there are multiple formats- if len(self.formats) > 1:- return os.path.join(self.orig_output_dir, sanitized_format)-- return self.orig_output_dir-- @torch.inference_mode()- def _quantize_embedding_layer(self):- """Quantizes embedding layers in the model according to the configuration.-- This method iterates through all modules in the model, identifies embedding- layers specified in `self.layer_config`, and applies the appropriate quantization- function based on bit precision, grouping strategy, and dtype.-- Returns:- bool: True if the quantization process completes without critical errors.- """- is_quantized = False- for name, module in self.model.named_modules():- # Skip non-Embedding modules or layers not in config- if not isinstance(module, torch.nn.Embedding) or name not in self.layer_config:- continue-- config = self.layer_config[name]-- # Skip layers that are not marked for quantization- if not check_to_quantized(config):- continue- is_quantized = True- config["scale_dtype"] = self.scale_dtype- dtype = config["data_type"]-- # Determine quantization function key with symmetry/asymmetry- if dtype not in QUANT_FUNC_WITH_DTYPE:- dtype = f"{dtype}_{'sym' if config['sym'] else 'asym'}"-- # Optionally use optimized rounding (RTN) variant- if not self.disable_opt_rtn and f"rtn_{dtype}" in QUANT_FUNC_WITH_DTYPE:- dtype = f"rtn_{dtype}"-- quant_func = QUANT_FUNC_WITH_DTYPE[dtype]-- # Attempt quantization on GPU, fall back to CPU if OOM- try:- weight, scale, zp = quant_func(- module.weight.to(self.device),- **{k: config[k] for k in ["bits", "group_size", "super_bits", "super_group_size", "scale_dtype"]},- )- except RuntimeError as e:- cuda_error_msg = traceback.format_exc()- try:- logger.error(cuda_error_msg)- logger.warning("falling back to CPU")- weight, scale, zp = quant_func(- module.weight.to("cpu"),- **{- k: config[k]- for k in ["bits", "group_size", "super_bits", "super_group_size", "scale_dtype"]- },- )- except Exception as e:- raise-- # Overwrite the module's weights with the quantized version- module.weight.data.copy_(weight.cpu())-- # Attach scale and zero point (zp) to the module- for param_name, value in zip(["scale", "zp"], [scale, zp]):- if isinstance(value, dict):- for k, v in value.items():- setattr(module, k if k == "scale" else f"w_{k}", v.cpu())- elif isinstance(value, torch.Tensor):- setattr(module, param_name, value.cpu())- else:- setattr(module, param_name, value)-- # Update config- self.layer_config.setdefault(name, {}).update(config)-- # Release memory- clear_memory()-- return is_quantized-- def _quant_rtn_with_imatrix(self, all_to_quantized_module_names: list[str]) -> None:- """Performs RTN quantization using input activation statistics (imatrix).-- This method accumulates per-channel second-moment activation statistics (imatrix)- via forward hooks and uses them to perform RTN quantization. If CUDA memory runs out,- it falls back to CPU-based blockwise quantization.-- Args:- all_to_quantized_module_names (list[str]):- A list of module names (e.g., 'model.layers.0.self_attn.q_proj') to be quantized.-- Returns:- None- """- logger.info("start to compute imatrix for GGUF quantization")-- # Load dataset- from auto_round.calib_dataset import get_dataloader-- if _is_fp8_model(self.model):- convert_fp8_model_to_16b_model(self.model, self.amp_dtype)-- if isinstance(self.dataset, str):- if self.tokenizer is None:- raise ValueError("A tokenizer must be set for the model when using a dataset string.")- dataset_name = self.dataset.replace(" ", "")- self.dataloader = get_dataloader(- self.tokenizer, self.seqlen, dataset_name, self.seed, self.batch_size, self.nsamples- )- else:- self.dataloader = self.dataset-- model = self.model-- # Dispatch multi-GPU model if necessary- if hasattr(model, "hf_device_map") and len(model.hf_device_map) > 1:- dispatch_model(model, model.hf_device_map)-- def register_act_hook(model):- """Registers hooks to accumulate activation squared norms into `imatrix`."""-- def get_imatrix_hook(module, input, output):- input = input[0] if isinstance(input, (tuple, list)) else input- flattened = input.reshape(-1, input.shape[-1]).to(torch.float32)- squared = torch.sum(flattened**2, dim=0).to(torch.float32)-- if not hasattr(module, "imatrix"):- module.imatrix = squared- module.imatrix_cnt = input.shape[0]- else:- module.imatrix += squared.to(module.imatrix.device)- module.imatrix_cnt += input.shape[0]-- hook_handles = []- for name, module in model.named_modules():- if isinstance(module, self.supported_types) and check_to_quantized(module):- hook = module.register_forward_hook(get_imatrix_hook)- hook_handles.append(hook)- return hook_handles-- hooks = register_act_hook(model)-- try:- # Move model to target device- if hasattr(self.model, "hf_device_map") and len(self.model.hf_device_map) > 1:- dispatch_model(self.model, self.model.hf_device_map)- else:- model = model.to(self.device)- cnt = 0-- # Run forward pass to accumulate imatrix- for data in self.dataloader:- cnt += data["input_ids"].shape[0]- data = to_device(data, self.device)- model(**data)- if cnt >= self.nsamples:- break-- # Remove hooks after data collection- for hook in hooks:- hook.remove()-- # Normalize imatrix by count- for _, module in model.named_modules():- if hasattr(module, "imatrix"):- module.imatrix /= module.imatrix_cnt- if hasattr(model, "hf_device_map") and len(model.hf_device_map) > 1:- import accelerate-- accelerate.hooks.remove_hook_from_submodules(model)- # Perform quantization using RTN- pbar = tqdm(all_to_quantized_module_names)- block_names_cnt = len(flatten_list(get_block_names(self.model, True)))- clear_mem_freq = len(all_to_quantized_module_names) // block_names_cnt- if clear_mem_freq == 0:- clear_mem_freq = 1- cnt = 1- for name in pbar:- pbar.set_description(f"Quantizing {name}")- self._quantize_layer_via_rtn(name)- if cnt % clear_mem_freq == 0:- clear_memory()- cnt = 1- cnt += 1- except RuntimeError as e:- try:- if hasattr(model, "hf_device_map") and len(model.hf_device_map) > 1:- import accelerate-- accelerate.hooks.remove_hook_from_submodules(model)- # Fallback: out-of-memory → try CPU blockwise quantization- logger.warning("Out of VRAM, falling back to blockwise quantization. Accuracy may degrade.")- model = model.to("cpu")- clear_memory()- self._quantize_via_rtn_blockwise(all_to_quantized_module_names)- except RuntimeError as e:- cuda_error_msg = traceback.format_exc()- try:- logger.error(cuda_error_msg)- # Final fallback: warn and use CPU-only quantization- logger.warning(- "Fallback to CPU. "- "Consider enabling `low_gpu_mem_usage` or using more GPUs via `--device 0,1,2,3`."- )- model = model.to("cpu")- clear_memory()- if hasattr(model, "hf_device_map") and len(model.hf_device_map) > 1:- import accelerate-- accelerate.hooks.remove_hook_from_submodules(model)-- orig_device = self.device- self.device = "cpu"- self._quantize_via_rtn_blockwise(all_to_quantized_module_names)- self.device = orig_device- except Exception as e:- raise- finally:- # Always remove hooks- for hook in hooks:- hook.remove()-- def _check_need_to_quantize_lm_head_embedding(self) -> bool:- """Checks if LM head and embedding layers need quantization for GGUF format.-- This function inspects the current model's formats and determines whether- it needs to apply quantization settings to the embedding and LM head layers.- The function modifies `self.layer_config` in-place and updates the model modules.-- Returns:- bool: True if the LM head needs quantization, otherwise False.-- Raises:- NotImplementedError: If multiple non-fake GGUF formats are specified.- """- gguf_scheme = False- if isinstance(self.scheme, str) and "gguf" in self.scheme.lower():- gguf_scheme = True-- if not hasattr(self, "formats") and not gguf_scheme:- return False-- has_gguf: bool = gguf_scheme or any("gguf" in fmt for fmt in self.formats)- if not has_gguf:- return False- if hasattr(self, "formats"):- formats: list[str] = [fmt for fmt in self.formats if "fake" not in fmt]- if not (len(formats) == 1 and "gguf" in formats[0]):- raise NotImplementedError("Only one GGUF format can be set at a time.")- target_format: str = formats[0]-- else:- target_format = self.scheme.lower()-- tie_word_embeddings: bool = getattr(getattr(self.model, "config", None), "tie_word_embeddings", True)- for name, module in self.model.named_modules():- if isinstance(module, torch.nn.Embedding):- key: str = "lm_head" if tie_word_embeddings else "embedding"- config: dict[str, Any] = GGUF_INNER_CONFIG[GGUF_CONFIG[target_format][key]]- self._apply_config_to_layer(name, config, True)-- if not tie_word_embeddings:- lm_head_name: str = get_lm_head_name(self.model)- config: dict[str, Any] = GGUF_CONFIG[GGUF_CONFIG[target_format]["lm_head"]]- check_fixed_by_user = (- self.layer_config[lm_head_name].get("fixed_by_user", False)- if lm_head_name in self.layer_config- else None- )- self._apply_config_to_layer(lm_head_name, config, check_fixed_by_user=check_fixed_by_user)- return True-- return False-- def _apply_config_to_layer(- self,- layer_name: str,- config: dict[str, Any],- check_fixed_by_user: bool = False,- ) -> None:- """Applies GGUF quantization configuration to a given layer.-- Args:- layer_name (str): Name of the layer to configure.- config (dict[str, Any]): GGUF layer configuration.- check_fixed_by_user (bool): If True, preserve user-defined settings.- """- act_bits: int = 16- scale_dtype: Any = self.scale_dtype- keys: list[str] = ["bits", "group_size", "super_bits", "super_group_size", "data_type", "sym"]-- self.layer_config[layer_name] = self.layer_config.get(layer_name, {})-- for key in keys:- if (- key in self.layer_config[layer_name]- and check_fixed_by_user- # and self.layer_config[layer_name].get("fixed_by_user", False)- ):- continue- self.layer_config[layer_name][key] = config.get(key)- setattr(get_module(self.model, layer_name), key, config.get(key))-- self.layer_config[layer_name]["act_bits"] = act_bits- self.layer_config[layer_name]["scale_dtype"] = scale_dtype- setattr(get_module(self.model, layer_name), "act_bits", act_bits)- setattr(get_module(self.model, layer_name), "scale_dtype", scale_dtype)-- def _quantize_layer_via_rtn(self, name: str) -> None:- """Quantizes a layer using RTN (Round-To-Nearest) if available.-- This function attempts to quantize a layer by switching its data type to a- `rtn_*` version if supported, then wraps and unwraps the module to apply- quantization. If GPU memory is insufficient, it falls back to CPU.-- If packing is enabled (`is_packing_immediate`), the function will also export- the quantized layer to the appropriate backend format.-- Args:- name (str): Name of the layer to quantize.-- Raises:- RuntimeError: If quantization fails for reasons unrelated to memory.- """- m = get_module(self.model, name)-- # if m.__class__.__name__ == "FP8Linear":- if _is_fp8_linear(m):- m = convert_fp8_layer_to_linear(m, self.amp_dtype)- set_module(self.model, name, m)-- # Step 1: Use optimized RTN data type if available- if not self.disable_opt_rtn and not m.data_type.startswith("rtn_"):- from auto_round.data_type import QUANT_FUNC_WITH_DTYPE-- rtn_dtype = "rtn_" + m.data_type- if rtn_dtype in QUANT_FUNC_WITH_DTYPE:- m.data_type = rtn_dtype- self.layer_config[name]["data_type"] = m.data_type-- # Step 2: Try quantization on GPU first, fall back to CPU if OOM- # if only export gguf, using gguf-packing instead of rtn- if self.is_packing_immediate and self.iters == 0 and "gguf" in self.formats[0] and not self.disable_opt_rtn:- m.scale = None- m.zp = None- else:- try:- m.to(self.device)- m = WrapperLinear(- m,- enable_minmax_tuning=False,- enable_norm_bias_tuning=False,- enable_round_tuning=False,- )- m = m.unwrapper({})- m.to("cpu")- except RuntimeError as e:- cuda_error_msg = traceback.format_exc()- m = m.orig_layer if hasattr(m, "orig_layer") else m- try:- logger.error(cuda_error_msg)- logger.warning("falling back to CPU.")- m.to("cpu")- m = WrapperLinear(- m,- enable_minmax_tuning=False,- enable_norm_bias_tuning=False,- enable_round_tuning=False,- )- m = m.unwrapper({})- except Exception as e:- raise-- # Step 3: Optional immediate packing/export- if self.is_packing_immediate:- from auto_round.export import PACKING_LAYER_WITH_FORMAT-- if check_to_quantized(m):- target_backend = self.formats[0].split(":")[0] if ":" in self.formats[0] else self.formats[0]- has_gguf = any("gguf" in fmt for fmt in self.formats)-- if has_gguf:- from auto_round.export.export_to_gguf.export import pack_gguf_layer-- output_dir = self._get_save_folder_name(self.formats[0])- model_type = ModelType.MMPROJ if self.vlm else ModelType.TEXT- pack_gguf_layer(- name,- self.model,- self.formats[0],- output_dir,- self.layer_config,- self.tokenizer,- processor=self.processor if hasattr(self, "processor") else None,- image_processor=self.image_processor if hasattr(self, "image_processor") else None,- model_type=model_type,- )- else:- PACKING_LAYER_WITH_FORMAT[target_backend](name, self.model, self.formats[0], device=self.device)-- # if self.low_gpu_mem_usage:- # clear_memory()- else:- set_module(self.model, name, m)-- @torch.inference_mode()- def _quantize_rtn(self) -> tuple[torch.nn.Module, dict[str, Any]]:- """Quantize all modules in the model using RTN (Round-To-Nearest) strategy.-- If the target format includes GGUF with `k`, and optimized RTN is enabled,- blockwise quantization with input caching and imatrix is used.-- Returns:- tuple[nn.Module, Dict[str, Any]]: The quantized model and the layer configuration.- """- if self.amp and self.model.dtype != self.amp_dtype:- self.model.to(self.amp_dtype)-- all_to_quantized_module_names: list[str] = [n for n, m in self.model.named_modules() if check_to_quantized(m)]-- if is_nv_fp(self.data_type):- from auto_round.data_type.nvfp import calculate_gparam- from auto_round.data_type.utils import update_fused_layer_global_scales-- pbar = tqdm(all_to_quantized_module_names)- for name in pbar:- pbar.set_description(f"Calculate weight global scale: {name}")- m = get_module(self.model, name)- weight_global_scale = calculate_gparam(m.weight, self.group_size)- setattr(m, "weight_global_scale", weight_global_scale)-- modules = list(self.model.modules())- for module in tqdm(modules, desc="Update weight global scale for fuse module"):- update_fused_layer_global_scales(module)-- has_gguf_k = any("gguf" in fmt and "k" in fmt for fmt in getattr(self, "formats", []))-- self._quantize_embedding_layer()-- self.model.to("cpu")- if has_gguf_k and not self.disable_opt_rtn:- self._quant_rtn_with_imatrix(all_to_quantized_module_names)- elif self.act_bits <= 8 and check_need_act_calibration(- self.act_dynamic, self.act_data_type, self.act_bits- ): # TODO, mixed datatype has bug- hook_handles = self._register_act_max_hook(self.model)- try:- self._quantize_via_rtn_blockwise(all_to_quantized_module_names)- except RuntimeError as e:- logger.warning("Fallback to CPU. Consider using more GPUs via `--device 0,1,2,3`.")- self.model = self.model.to("cpu")- clear_memory()- if hasattr(self.model, "hf_device_map") and len(self.model.hf_device_map) > 1:- import accelerate-- accelerate.hooks.remove_hook_from_submodules(self.model)- orig_device = self.device- self.device = "cpu"- self._quantize_via_rtn_blockwise(all_to_quantized_module_names)- self.device = orig_device- for handle in hook_handles:- handle.remove()- else:- block_names_cnt = len(flatten_list(get_block_names(self.model, True)))- clear_mem_freq = len(all_to_quantized_module_names) // block_names_cnt- if clear_mem_freq == 0:- clear_mem_freq = 1- pbar = tqdm(all_to_quantized_module_names)- cnt = 1- for name in pbar:- pbar.set_description(f"Quantizing {name}")- self._quantize_layer_via_rtn(name)- if cnt % clear_mem_freq == 0:- clear_memory()- cnt = 1- cnt += 1- # Convert remaining fp8- if _is_fp8_model(self.model):- convert_fp8_model_to_16b_model(self.model, self.amp_dtype)- self.quantized = True- return self.model, self.layer_config-- def _quantize_via_rtn_blockwise(self, all_to_quantized_module_names: list[str]) -> None:- """Quantize model layers block by block using cached inputs and imatrix.-- Args:- all_to_quantized_module_names (list[str]): Names of layers to be quantized.- """- all_to_quantized_module_names = list(set(all_to_quantized_module_names))-- all_blocks = self.quant_block_list if self.quant_block_list else get_block_names(self.model)- if not all_blocks:- raise ValueError("Could not find any blocks. Check the model or quant_block_list.")-- all_first_block_names = [block[0] for block in all_blocks]- if self.act_bits < 16 and not self.act_dynamic:- layer_names = self._get_quantized_layer_names_outside_blocks()- if len(layer_names) > 0:- logger.warning(- "quantize layers outside blocks for static activation quantizaiton"- " will significantly increase calibration time"- )- all_inputs = self.try_cache_inter_data_gpucpu(all_first_block_names, self.nsamples, layer_names)- else:- all_inputs = self.cache_inter_data(all_first_block_names, self.nsamples)-- # Clear hooks for multi-GPU setups- if hasattr(self.model, "hf_device_map") and len(self.model.hf_device_map) > 1:- accelerate.hooks.remove_hook_from_submodules(self.model)-- pbar = tqdm(range(sum(len(block) for block in all_blocks)))-- for block_names in all_blocks:- first_block = block_names[0]- inputs = all_inputs.pop(first_block)- input_keys = [k for k in inputs if k.startswith("hidden_state")]- if len(input_keys) != 1:- raise RuntimeError(- "hidden_states arg mismatch. Please file an issue at https://github.com/intel/auto-round/issues"- )- inputs["input_ids"] = inputs.pop(input_keys[0])-- clear_memory(self.inputs)-- total_samples = len(inputs["input_ids"])- if total_samples < self.batch_size:- self.batch_size = total_samples- logger.warning(f"Forcing batch size to {total_samples}")-- input_ids = to_device(inputs.pop("input_ids"), self.cache_device)- input_others = to_device(inputs, self.cache_device)-- tmp_dtype = self.amp_dtype if self.amp else torch.float32- input_ids = [id_.to(tmp_dtype) for id_ in input_ids]-- for key, val in input_others.items():- if isinstance(val, torch.Tensor) and val.dtype in (torch.float16, torch.bfloat16):- input_others[key] = val.to(tmp_dtype)- elif isinstance(val, list):- input_others[key] = [to_dtype(v, tmp_dtype) for v in val]-- for block_name in block_names:- pbar.set_description(f"Quantizing {block_name}")- block = get_module(self.model, block_name)- block = block.to(self.device)- if _is_fp8_model(self.model):- convert_fp8_model_to_16b_model(block, dtype=self.amp_dtype)-- if self.device_map == "auto":- self._set_auto_device_map_in_block(block, input_ids)-- # Dispatch model if needed- if self.device_map is not None:- from accelerate.hooks import AlignDevicesHook, add_hook_to_module-- for _, m in block.named_modules():- if len(list(m.children())) != 0 or not hasattr(m, "tuning_device"):- continue- hook = AlignDevicesHook(m.tuning_device, io_same_device=True)- add_hook_to_module(m, hook, True)- else:- block = block.to(self.device)- input_ids = self._get_block_outputs(- block,- input_ids,- input_others,- self.batch_size * self.infer_bs_coeff,- self.device,- self.cache_device,- )- if self.device_map is not None:- accelerate.hooks.remove_hook_from_submodules(block)-- if (- is_nv_fp(self.act_data_type) and any("nv_fp" in format_ for format_ in self.formats)- ) or is_static_wfp8afp8(self):- # enable moe experts act_max automatic generation for Linear- set_amax_for_all_moe_layers(block, attr_name="act_max")- # Normalize imatrix and quantize layers- for _, m in block.named_modules():- if hasattr(m, "imatrix"):- m.imatrix /= m.imatrix_cnt- if hasattr(m, "tmp_name") and m.tmp_name in all_to_quantized_module_names:- self._quantize_layer_via_rtn(m.tmp_name)- all_to_quantized_module_names.remove(m.tmp_name)-- mv_module_from_gpu(block, self.low_cpu_mem_usage)- pbar.update(1)-- pbar.close()- cnt = 1- block_names_cnt = len(flatten_list(get_block_names(self.model, True)))- clear_mem_freq = len(all_to_quantized_module_names) // block_names_cnt- if clear_mem_freq == 0:- clear_mem_freq = 1- # Process remaining layers not in blocks- for name in all_to_quantized_module_names:- self._quantize_layer_via_rtn(name)- if cnt % clear_mem_freq == 0:- clear_memory()- cnt = 1- cnt += 1-- def quantize(self) -> tuple[torch.nn.Module, dict[str, Any]]:- """Quantize the model and return the quantized model along with layer configurations.The entry of AutoRound.- Returns:- The quantized model and layer configurations.- """- for n, m in self.model.named_modules():- m.tmp_name = n- self._check_compatibility()- formats = self.formats if hasattr(self, "formats") else None- # It is best to modify the model structure in the quantize function and check the format,- # because it may cause the gguf format to not be exported normally.- self.model = _handle_moe_model(self.model, formats=formats)- self.has_qlayer_outside_block = self._set_layerwise_config(self.layer_config)- if not hasattr(self, "formats"):- logger.warning("this API is deprecated, please use `quantize_and_save` instead")- else:- only_gguf = True- for format_ in self.formats:- if not ("gguf" in format_ or "fake" in format_):- only_gguf = False- break- if len(self.formats) == 1 and self.formats[0] == "fake":- only_gguf = False- if only_gguf:- self.layer_config, gguf_format_config = get_layer_config_by_gguf_format(- self.layer_config, self.formats, self.model, model_type=ModelType.TEXT- )- if self.vlm:- self.layer_config, gguf_format_config = get_layer_config_by_gguf_format(- self.layer_config, self.formats, self.model, model_type=ModelType.MMPROJ- )- # Determine if immediate packing is required- formats = self.formats- if (- len(formats) == 1- and (- "awq" in formats[0]- or "gptq" in formats[0]- or "auto_round" in formats[0]- or "gguf" in formats[0]- or "llm_compressor" in formats[0]- )- and self.inplace- ):- self.is_packing_immediate = True- if self.iters == 0:- return self._quantize_rtn()-- if bool(self.quant_block_list):- all_blocks = self.quant_block_list- else:- all_blocks = get_block_names(self.model)-- if len(all_blocks) == 0:- logger.warning("could not find blocks, exit with original model")- return self.model, self.layer_config-- if self.amp and self.model.dtype != self.amp_dtype:- self.model = self.model.to(self.amp_dtype)-- layer_names = self._get_quantized_layer_names_outside_blocks()- self.start_time = time.time()- all_first_block_names = [block[0] for block in all_blocks]- if len(layer_names) > 0:- logger.info(- "Starting to cache block inputs. This may be slow due to external block layers: %s", layer_names- )- else:- logger.info("start to cache block inputs")- all_inputs = self.try_cache_inter_data_gpucpu(all_first_block_names, self.nsamples, layer_names=layer_names)- is_quantized_embedding = self._quantize_embedding_layer()- all_q_inputs = None- if is_quantized_embedding:- all_inputs = copy.deepcopy(self.inputs)- clear_memory(self.inputs)- all_q_inputs = self.try_cache_inter_data_gpucpu(- all_first_block_names, self.nsamples, layer_names=layer_names- )- self.model = mv_module_from_gpu(self.model, self.low_cpu_mem_usage)- clear_memory()- if hasattr(self.model, "hf_device_map") and len(self.model.hf_device_map) > 1:- accelerate.hooks.remove_hook_from_submodules(self.model) # self.model.hf_device_map has not been changed- self.model = mv_module_from_gpu(self.model, self.low_cpu_mem_usage)- logger.info("caching done")- if len(all_blocks) > 1:- pbar = tqdm(range(0, sum([len(i) for i in all_blocks]), self.nblocks))- else:- pbar = None # move the alg warning outside pbar-- for block_names in all_blocks:- inputs = all_inputs[block_names[0]]- all_inputs.pop(block_names[0])- q_inputs = None- if all_q_inputs is not None:- q_inputs = all_q_inputs[block_names[0]]- all_q_inputs.pop(block_names[0])- keys = inputs.keys()- input_id_str = [key for key in keys if key.startswith("hidden_state")]- if len(input_id_str) != 1:- raise RuntimeError(- "hidden_states arg mismatch error,"- "please raise an issue in https://github.com/intel/auto-round/issues"- )- inputs["input_ids"] = inputs.pop(input_id_str[0], None)- if q_inputs is not None:- q_inputs["input_ids"] = q_inputs.pop(input_id_str[0], None)-- clear_memory(self.inputs)-- if "input_ids" in inputs.keys():- total_samples = len(inputs["input_ids"])- if total_samples < self.batch_size:- self.batch_size = total_samples- logger.warning(f"force the train batch size to {total_samples}")-- self._quantize_blocks(- self.model,- inputs,- block_names,- q_input=q_inputs["input_ids"] if q_inputs is not None else None,- nblocks=self.nblocks,- device=self.device,- pbar=pbar,- )- if self.is_packing_immediate and len(self.formats) != 1:- raise ValueError(- f"Expected exactly one packing format when 'is_packing_immediate' is True, "- f"but got {len(self.formats)} formats."- )-- self._quantize_layers(layer_names, all_inputs) ##TODO pack layer immediately-- if _is_fp8_model(self.model):- for n, m in self.model.named_modules():- if _is_fp8_linear(m):- new_layer = convert_fp8_layer_to_linear(m, self.amp_dtype).to("cpu")- set_module(self.model, n, new_layer)-- end_time = time.time()- cost_time = end_time - self.start_time- logger.info(f"quantization tuning time {cost_time}")-- ## dump a summary- quantized_layers = []- unquantized_layers = []- for n, m in self.model.named_modules():- if isinstance(m, tuple(self.supported_types)):- if check_to_quantized(m):- quantized_layers.append(n)- else:- unquantized_layers.append(n)- elif hasattr(m, "scales") or hasattr(m, "scale"): ##packing_immediately- quantized_layers.append(n)- summary_info = (- f"Summary: quantized {len(quantized_layers)}/{len(quantized_layers) + len(unquantized_layers)} in the model"- )- if len(unquantized_layers) > 0:- summary_info += f", {unquantized_layers} have not been quantized"- logger.info(summary_info)-- self.quantized = True- return self.model, self.layer_config-- def _quantize_layers(self, layer_names: list, layer_inputs: dict) -> None:- """Quantizes specified layers based on inputs and configuration.-- Args:- layer_names (list): list of layer names to quantize.- layer_inputs (dict): Dictionary mapping layer names to input data.-- Returns:- None- """- ##TODO currently we take all the layers outside blocks as post block layers which is not optimal- ## if there is no input for layer, we use rtn-- for layer_name in copy.deepcopy(layer_names):- if layer_name not in layer_inputs:- logger.info(f"using rtn to quantize {layer_name}")- from auto_round.data_type import QUANT_FUNC_WITH_DTYPE-- layer = get_module(self.model, layer_name)- if _is_fp8_model(self.model):- new_layer = convert_fp8_layer_to_linear(layer, self.amp_dtype).to(self.device)- set_module(self.model, layer_name, new_layer)- layer = new_layer-- if not self.disable_opt_rtn and "rtn_" + layer.data_type in QUANT_FUNC_WITH_DTYPE:- layer.data_type = "rtn_" + layer.data_type- logger.info("using optimized rtn method for quantizing %s", layer_name)- self.layer_config[layer_name]["data_type"] = layer.data_type- wrapper_layer = WrapperLinear(- layer,- enable_round_tuning=False,- enable_minmax_tuning=False,- enable_norm_bias_tuning=False,- device=self.device,- )- new_layer = wrapper_layer.unwrapper({})- set_module(self.model, layer_name, new_layer)- layer.cpu()- layer_names.remove(layer_name)- if len(layer_names) == 0:- return- q_layer_inputs = None- enable_quanted_input = self.enable_quanted_input- has_gguf = False- if hasattr(self, "formats"):- has_gguf = any("gguf" in format_ for format_ in self.formats)- if has_gguf and self.is_packing_immediate:- enable_quanted_input = False-- if hasattr(self.model, "hf_device_map") and len(self.model.hf_device_map) > 1 and enable_quanted_input:- dispatch_model(self.model, self.model.hf_device_map)-- if enable_quanted_input:- logger.info("starting to cache layer inputs for %s, this may be quite slow ", layer_names)- q_layer_inputs = self.try_cache_inter_data_gpucpu([], self.nsamples, layer_names=layer_names)- if hasattr(self.model, "hf_device_map") and len(self.model.hf_device_map) > 1:- accelerate.hooks.remove_hook_from_submodules(- self.model- ) ##self.model.hf_device_map has not been changed-- self.model = mv_module_from_gpu(self.model, self.low_cpu_mem_usage)- clear_memory()- if self.enable_torch_compile:- quant_layer = compile_func(self._quantize_layer, self.device)- else:- quant_layer = self._quantize_layer- for layer_name in layer_names:- layer_input = layer_inputs[layer_name]- layer_input = to_device(layer_input, self.cache_device)- q_layer_input = q_layer_inputs.get(layer_name, None) if q_layer_inputs is not None else None- q_layer_input = to_device(q_layer_input, self.cache_device)- quant_layer(layer_name, layer_input, q_layer_input, device=self.device)- del layer_input- clear_memory(q_layer_input)-- def _set_layerwise_config(self, layer_config: dict) -> bool:- """- Sets the layer-wise configuration based on the provided `layer_config`.- By default, only quantize layers in blocks.-- Args:- layer_config (dict): The configuration dictionary for each layer containing various configuration options.-- Returns:- bool: Returns True if there are quantized layers outside the blocks (e.g., lm-head),- otherwise returns False.- """- # Get the names of layers in quantization blocks- supported_types = self.supported_types- layers_in_blocks = get_layer_names_in_block(- self.model, supported_types, self.quant_block_list, self.inner_supported_types- )- ##process regex in layer_config- all_supported_layer_names = []- # List of configuration keys- keys = get_quant_keys()-- for n, m in self.model.named_modules():- # Delete previous configuration to avoid conflicts with prior tuning- for key in keys:- if hasattr(m, key):- delattr(m, key)-- if not isinstance(m, supported_types) and m.__class__.__name__ not in self.inner_supported_types:- continue- all_supported_layer_names.append(n)-- names_in_layer_config = list(layer_config.keys())- for name in names_in_layer_config:- if name in all_supported_layer_names:- continue- matched_names = []- for layer_name in all_supported_layer_names:- if re.search(re.compile(name), layer_name) is not None:- matched_names.append(layer_name)- if len(matched_names) > 0:- val = layer_config[name]- layer_config.pop(name)- for match_name in matched_names:- layer_config[match_name] = val- else:- tmp_m = get_module(self.model, name)- if not isinstance(tmp_m, torch.nn.Embedding): # TODO not good code style- raise ValueError(f"key {name} in layer_config is invalid, please have a double check")-- has_qlayer_outside_block = False # Flag to track if there are quantized layers outside blocks (e.g., lm-head)-- # Iterate through all modules in the model- is_gguf = hasattr(self, "formats") and any("gguf" in format_ for format_ in self.formats)- for n, m in self.model.named_modules():- # Skip unsupported types- if not isinstance(m, supported_types) and m.__class__.__name__ not in self.inner_supported_types:- if n in self.layer_config:- if not isinstance(m, torch.nn.Embedding):- logger.warning(f"{n} is not supported, layer_config {n}: {layer_config[n]} will be ignored.")- self.layer_config.pop(n)- continue- if not is_gguf:- if not check_to_quantized(layer_config[n]):- self.layer_config.pop(n)- continue- else:- continue-- # If the layer is not in the config and is part of a quantization block, use default configuration- if n not in layer_config.keys() and n in layers_in_blocks:- layer_config[n] = {}- for key in keys:- layer_config[n][key] = getattr(self, key)-- # If the layer is partially configured, fill in missing values- elif n in layer_config.keys():- if "data_type" in layer_config[n] and "bits" not in layer_config[n]:- tmp_bits = infer_bits_by_data_type(layer_config[n]["data_type"])- if tmp_bits is not None and tmp_bits != self.bits:- logger.warning(- f"'data_type' do not match the specified 'bits' setting for {n}."- f" Resetting 'bits' to {tmp_bits}."- )- layer_config[n]["bits"] = tmp_bits- if "act_data_type" in layer_config[n] and "act_bits" not in layer_config[n]:- tmp_bits = infer_bits_by_data_type(layer_config[n]["act_data_type"])- if tmp_bits is not None and tmp_bits != self.act_bits:- logger.warning(- f"'act_data_type' do not match the specified 'act_bits' setting for {n}."- f" Resetting 'act_bits' to {tmp_bits}."- )- layer_config[n]["act_bits"] = tmp_bits-- for key in keys:- if key not in layer_config[n].keys():- layer_config[n][key] = getattr(self, key)- layer_config[n]["fixed_by_user"] = True-- # If the layer is not in the config and not part of a quantization block,- # use default configuration and set specific values- else:- layer_config[n] = {}- for key in keys:- layer_config[n][key] = getattr(self, key)- layer_config[n]["bits"] = 16- layer_config[n]["act_bits"] = 16-- if n in layers_in_blocks:- layer_config[n]["in_blocks"] = True- else:- layer_config[n]["in_blocks"] = False-- # If the layer is outside a block and requires quantization, mark it as a quantized layer outside the block- if (- n not in layers_in_blocks- and check_to_quantized(layer_config[n])- and not isinstance(m, torch.nn.Embedding)- ):- has_qlayer_outside_block = True-- in_features, out_features = get_layer_features(m)- if in_features <= layer_config[n]["group_size"]:- layer_config[n]["group_size"] = -1-- # Apply the configuration to the corresponding layer in the model- for key in keys:- setattr(m, key, layer_config[n][key])- need_to_quantize_lm_head = self._check_need_to_quantize_lm_head_embedding()- if need_to_quantize_lm_head:- has_qlayer_outside_block = True-- # Return whether there are quantized layers outside the blocks- return has_qlayer_outside_block-- @torch.no_grad()- def _get_block_outputs(- self,- block: torch.nn.Module,- input_ids: torch.Tensor,- input_others: torch.Tensor,- bs: int,- device: Union[str, torch.device],- cache_device: Union[str, torch.device],- save_output: bool = True,- ):- """Compute the output of a given block of the model for a given input.-- Args:- block: The block of the model.- input_ids: The input tensor containing tokenized input ids.- input_others: A dictionary containing additional input data.- bs: The batch size for computing the output.- device: The device for computation.- cache_device: The device for storing the output.- batch_dim: The batch dimension of the output tensor.-- Returns:- The output tensor of the block.- """-- output = []- nsamples = len(input_ids)- for i in range(0, nsamples, bs):- end_index = min(nsamples, i + bs)- indices = torch.arange(i, end_index).to(torch.long)- tmp_input_ids, tmp_input_others = AutoRound._sampling_inputs(- input_ids, input_others, indices, self.seqlen, self.batch_dim, share_cache_keys=self.shared_cache_keys- )- tmp_output = block_forward(block, tmp_input_ids, tmp_input_others, self.amp, self.amp_dtype, device).to(- cache_device- )- if save_output:- if self.batch_size == 1:- output.append(tmp_output)- else:- output.extend(list(torch.split(tmp_output, 1, dim=self.batch_dim)))- if self.low_gpu_mem_usage:- clear_memory()-- return output-- @torch.no_grad()- def calib(self, nsamples, bs):- """Perform calibration for quantization.-- This method calibrates the model for quantization by processing a specified- number of samples from the calibration dataset. It ensures that the data is- properly formatted and feeds it to the model. If the number of samples processed- is less than the specified number, it logs a warning. If no samples are processed,- it logs an error and exits.- Args:- nsamples (int): The number of samples to use for calibration.- bs (int): The number of samples to use for calibration- """- from .calib_dataset import get_dataloader-- if isinstance(self.dataset, str):- dataset = self.dataset.replace(" ", "") ##remove all whitespaces-- # slow here- self.dataloader = get_dataloader(- self.tokenizer,- self.seqlen,- dataset,- self.seed,- bs,- self.nsamples,- )- else:- self.dataloader = self.dataset- total_cnt = 0-- # load embed weight if use low_cpu_mem_usage- if self.low_cpu_mem_usage:- embed_layers = get_layers_before_block(self.model)- for n, m in embed_layers:- m = m.to(self.device)-- for data in self.dataloader:- if data is None:- continue- if isinstance(data, torch.Tensor):- input_ids = data.to(self.model.device)- data_new = input_ids- elif isinstance(data, str):- if self.tokenizer is None:- logger.error("please provide tokenizer for string input")- exit(-1)- data = self.tokenizer(data, truncation=True, max_length=self.seqlen, return_tensors="pt").data- data_new = {}- for key in data.keys():- data_new[key] = data[key].to(self.model.device)- input_ids = data_new["input_ids"]- elif isinstance(data, tuple) or isinstance(data, list):- data_new = to_device(data)- input_ids = data_new[0]- else:- data_new = {}- for key in data.keys():- data_new[key] = to_device(data[key], self.model.device)- if key == "images":- data_new[key] = to_dtype(data_new[key], self.model.dtype)- input_ids = data_new["input_ids"]- if input_ids.shape[-1] < self.seqlen:- continue- try:- if isinstance(data_new, torch.Tensor):- self.model(data_new, use_cache=False)- elif isinstance(data_new, tuple) or isinstance(data_new, list):- self.model(*data_new, use_cache=False)- else:- self.model(**data_new, use_cache=False)- except NotImplementedError:- pass- except RuntimeError as error:- error_msg = str(error)- if "The expanded size of the tensor" in str(error_msg) and "must match the existing size" in error_msg:- check_seqlen_compatible(self.seqlen, self.tokenizer, self.model)- logger.warning(- "When quantization encounters tensor shape mismatch error, "- "you can try to avoid it with batch_size=1"- )- raise error- except Exception as error:- raise error- total_cnt += input_ids.shape[0] if len(input_ids.shape) > 1 else 1- if total_cnt >= nsamples:- break- if total_cnt == 0:- logger.error(- f"no data has been cached, please provide more data with sequence length >={self.seqlen} in the "- f"dataset or decease the sequence length"- )- exit(-1)- elif total_cnt < nsamples:- logger.warning(- f"An insufficient number of samples likely reduces the accuracy of the quantized model. "- f"Target samples count is {nsamples}, while valid samples count is {total_cnt}"- )-- # clean embed weight to save memory- if self.low_cpu_mem_usage:- for n, m in embed_layers:- m = m.to("meta")-- @torch.no_grad()- def try_cache_inter_data_gpucpu(self, block_names, nsamples, layer_names=None, last_cache_name=None):- """Attempts to cache intermediate data on GPU, if failed, then using CPU.-- Args:- block_names (list): List of block names to cache data for.- nsamples (int): Number of samples to use for caching.- layer_names (list, optional): List of layer names to cache data for. Defaults to [].- last_cache_name (str, optional): Name of the last cache. Defaults to None.-- Returns:- all_inputs: Cached intermediate data.-- Raises:- Exception: If caching on GPU fails, switches to CPU and caches there.- """- if _is_fp8_model(self.model):- layer_names = []- if layer_names is None:- layer_names = []-- if self.low_gpu_mem_usage or (- len(block_names) == 1- and len(layer_names) == 0- and not self.has_qlayer_outside_block- and (last_cache_name is None or last_cache_name in block_names)- ):- # low_gpu_mem_usage or calibrate only the embedding layer, which is also very fast on CPU- all_inputs = self.cache_inter_data(block_names, nsamples, layer_names=[], last_cache_name=last_cache_name)- else:- try:- if not self.model.device.type == "meta":- if hasattr(self.model, "hf_device_map") and len(self.model.hf_device_map) > 1:- self.model = dispatch_model(self.model, device_map=self.model.hf_device_map)- else:- # Change this if new device is supported- if str(self.model.device) == "cpu" and (- self.device.startswith("xpu") or self.device.startswith("cuda")- ):- max_memory = get_max_vram() # TODO model is not evenly split- no_split_modules = getattr(self.model, "_no_split_modules", [])- device_map = infer_auto_device_map(- self.model, max_memory=max_memory, no_split_module_classes=no_split_modules- )-- self.model = dispatch_model(self.model, device_map=device_map)- else:- self.model = self.model.to(self.device)-- all_inputs = self.cache_inter_data(- block_names, nsamples, layer_names=layer_names, last_cache_name=last_cache_name- )- if hasattr(self.model, "hf_device_map") and len(self.model.hf_device_map) > 1:- accelerate.hooks.remove_hook_from_submodules(self.model)-- except RuntimeError as e:- cuda_error_msg = traceback.format_exc()- try:- logger.info("switch to cpu to cache block inputs")- if self.has_qlayer_outside_block or self.__class__.__name__ == "AutoRoundMLLM":- logger.warning(- "we recommend using more GPUs in calibration."- " Otherwise, some layers may fall back to `rtn` mode, which can affect accuracy."- )- if hasattr(self.model, "hf_device_map") and len(self.model.hf_device_map) > 1:- accelerate.hooks.remove_hook_from_submodules(- self.model- ) ##self.model.hf_device_map has not been changed- self.model = mv_module_from_gpu(self.model, self.low_cpu_mem_usage)- clear_memory()- ## Important change after v0.51, on cpu, we use rtn mode for layers in layer_names- all_inputs = self.cache_inter_data(- block_names, nsamples, layer_names=[], last_cache_name=last_cache_name- )- except Exception as e:- logger.error(cuda_error_msg)- raise- return all_inputs-- @torch.no_grad()- def cache_inter_data(self, block_names, nsamples, layer_names=None, last_cache_name=None):- """Save the inputs of block_name for calibration.-- This method temporarily replaces the forward method of the model to capture- the inputs passing through the specified block. It then calibrates the model- using a specified number of samples. Finally, it restores the original forward- method and returns the inputs for the specified block.- Args:- block_names (list): The names of the blocks for which inputs are to be saved.- layer_names (list):The names of the layers for which inputs are to be saved.- nsamples (int): The number of samples to use for calibration.- last_cache_name (str, optional): The name of the last layer to be cached,- we could break the forward in this layer to save time-- Returns:- dict: A dictionary containing the inputs for the specified block.- """- if layer_names is None:- layer_names = []- self.inputs = {}- self.to_cached_layers = block_names + layer_names-- tmp_dtype = None # TODO delete this as most model is not fp32 now- ## have bug if block name is not the first block- if (len(block_names) > 1 or len(layer_names) > 0) and self.low_gpu_mem_usage:- tmp_dtype = self.model.dtype- if self.amp:- if self.model.dtype != self.model.dtype:- self.model = self.model.to(torch.bfloat16)- else:- self.model = self.model.to(torch.float32) ##model on cpu-- self.last_cache_name = last_cache_name- if last_cache_name is None and len(block_names) + len(layer_names) == 1:- self.last_cache_name = block_names[0] if len(block_names) == 1 else layer_names[0]- # do not set last_cache_name for multimodal models- calib_bs = self.batch_size- self.hook_handles = []- self._replace_forward()- self.calib(nsamples, calib_bs)- self._recover_forward()- res = self.inputs- del self.last_cache_name- del self.to_cached_layers- if tmp_dtype is not None:- self.model = self.model.to(tmp_dtype)-- return res-- @torch.no_grad()- def _get_block_forward_func(self, name: str) -> Callable:- """Gets the forward function.-- Args:- name (str): The name of the function.- Returns:- function: The forward function.- """-- def post_process_cache_data(batch_size, data, data_name):- """- Processes store data for batch handling, reshaping if necessary.-- Args:- batch_size (int): The size of the batch.- data: The data value to store, potentially for caching.- data_name (str): Name of the data.-- Returns:- Processed data or None- """- new_data = data- if batch_size <= 1:- return new_data- if data_name in self.shared_cache_keys:- return None- if "alibi" in data_name:- if isinstance(data, torch.Tensor):- alibi = data- alibi = alibi.reshape(batch_size, -1, alibi.shape[1], alibi.shape[2])- new_data = alibi- return new_data-- def forward(m, hidden_states=None, *positional_inputs, **kwargs):- """Rewrite forward function, process and collect input data.-- Args:- hidden_states (torch.Tensor): The hidden states tensor.- *positional_inputs: Variable number of positional arguments.- **kwargs: Variable number of keyword arguments.-- Returns:- NotImplementedError: Getting the first layer inputs and then raise the error to save runtime.- """- if name not in self.inputs:- self.inputs[name] = {}- init_cache(positional_inputs, self.inputs[name])-- if self.batch_dim is None:- self.batch_dim = 0- if hidden_states is not None and self.batch_size > 1:- if hidden_states.shape[0] > self.batch_size:- self.batch_dim = 1- if len(hidden_states.shape) > 1 and hidden_states.shape[1] > self.batch_size:- logger.error(- "this model has not been supported, "- "please raise an issue in https://github.com/intel/auto-round/issues"- " or try to set the `batch_size` to 1 and "- "`gradient_accumulate_steps` to your current batch size."- )- exit(-1)-- if hidden_states is not None:- kwargs["hidden_states"] = hidden_states-- for key in kwargs.keys():- if (- isinstance(kwargs[key], torch.Tensor)- or isinstance(kwargs[key], list)- or isinstance(kwargs[key], tuple)- ):- if key not in self.inputs[name].keys(): # initialization- data = to_device(kwargs[key], device=torch.device("cpu"))- if data is None or (self.batch_size > 1 and key in self.shared_cache_keys):- self.inputs[name][key] = data- continue- if self.batch_size <= 1:- self.inputs[name][key] = [data]- else:- data = post_process_cache_data(self.batch_size, data, key)- self.inputs[name][key] = list(torch.split(data, 1, dim=self.batch_dim))- else: # append cache inputs- new_data = post_process_cache_data(self.batch_size, kwargs[key], key)- if new_data is None: # shareable args or NoneType- continue- new_data = to_device(new_data, device=torch.device("cpu"))- if self.batch_size <= 1:- self.inputs[name][key].append(new_data)- else:- self.inputs[name][key].extend(list(torch.split(new_data, 1, dim=self.batch_dim)))- elif isinstance(kwargs[key], (str, bool, type(None))):- if key not in self.inputs[name].keys():- self.inputs[name][key] = kwargs[key]- else:- # Parameters not to be cached- if check_skippable_keywords(key):- logger.warning_once(- f"Please note that '{key}' key" " is not currently used in quantization fine-tuning."- )- reset_params(self.inputs[name])- if name == self.last_cache_name:- raise NotImplementedError- else:- if hidden_states is not None:- kwargs.pop("hidden_states")- return m.orig_forward(hidden_states, *positional_inputs, **kwargs)- else:- # Currently only for Llama-3.2-Vision-Instruct Series- return m.orig_forward(*positional_inputs, **kwargs)-- return forward-- @torch.no_grad()- def _get_cache_data_hook_for_layer(self, name):- """A forward hook to save input max of a module- :param name: the module name- :return: A hook function."""-- def cache_input_hook(module, inputs, outputs):- input = inputs- if isinstance(inputs, tuple) or isinstance(input, list):- input = inputs[0]- if name in self.inputs:- self.inputs[name].extend(list(torch.split(input.to("cpu"), 1, dim=0)))- else:- self.inputs[name] = list(torch.split(input.to("cpu"), 1, dim=0))-- return cache_input_hook-- def _recover_forward(self):- """Recovers the forward function."""- for n, m in self.model.named_modules():- if hasattr(m, "orig_forward"):- m.forward = m.orig_forward- delattr(m, "orig_forward")- for hook_handle in self.hook_handles:- hook_handle.remove()- self.hook_handles = []-- def _replace_forward(self):- """Replaces the forward function."""- from functools import partial-- for n, m in self.model.named_modules():- if n in self.to_cached_layers and not isinstance(m, tuple(self.supported_types)): ##block- m.orig_forward = m.forward- m.forward = partial(self._get_block_forward_func(n), m)- elif n in self.to_cached_layers: ##linear layer or conv1d layer- hook_func = self._get_cache_data_hook_for_layer(n)- hook_handle = m.register_forward_hook(hook_func)- self.hook_handles.append(hook_handle)-- def _quantize_layer(- self, layer_name: str, inputs: torch.Tensor, q_inputs: torch.Tensor = None, device: str = "cpu"- ):- """Quantize a specific layer of the model using the provided inputs.-- Args:- layer_name (str): The name of the layer to quantize.- inputs (torch.Tensor): Input data for quantization.- q_inputs (torch.Tensor, optional): Quantized input data. Defaults to None.- device (torch.device, optional): The device to use for quantization. Defaults to torch.device("cpu").-- Returns:- None- """- logger.info(f"quantizing layer {layer_name}")- layer = get_module(self.model, layer_name)- if hasattr(layer, "tuning_device"):- device = layer.tuning_device-- layer = layer.to(device)- for i in range(len(inputs)):- inputs[i] = inputs[i].to(layer.weight.dtype)- if q_inputs is not None:- q_inputs[i] = q_inputs[i].to(layer.weight.dtype)-- wrapper_linear = WrapperLinear(layer, enable_minmax_tuning=self.enable_minmax_tuning, device=device).to(device)- round_params = []- minmax_params = []- for key in wrapper_linear.params.keys():- if "min" in key or "max" in key:- minmax_params.append(wrapper_linear.params[key])- else:- round_params.append(wrapper_linear.value)- if len(round_params) + len(minmax_params) <= 0:- dump_info = f"quantized {layer_name}"- logger.info(dump_info)- with torch.no_grad():- unwrapper_layer(self.model, wrapper_linear, layer_name, {})- mv_module_from_gpu(layer, self.low_cpu_mem_usage)-- lr = torch.tensor(self.lr)- minmax_lr = torch.tensor(self.minmax_lr)- if self.enable_minmax_tuning:- optimizer = self.optimizer(- [{"params": round_params}, {"params": minmax_params, "lr": minmax_lr}], lr=lr, weight_decay=0- )- else:- optimizer = self.optimizer(round_params, lr=lr, weight_decay=0)-- if self.lr_scheduler is None:- lr_schedule = torch.optim.lr_scheduler.LinearLR(- optimizer, start_factor=1.0, end_factor=0.0, total_iters=self.iters- )- else:- lr_schedule = copy.deepcopy(self.lr_scheduler)- nsamples = len(inputs)- last_best_iter = 0- best_loss = torch.finfo(torch.float).max- scaler = self._get_scaler() # pylint: disable=assignment-from-none- init_loss = None- gradient_accumulate_steps = self.batch_size ##Force to low gpu- batch_size = 1 ##Force to low gpu- pick_samples = batch_size * gradient_accumulate_steps- pick_samples = min(nsamples, pick_samples)- if self.sampler != "rand":- whole_indices = torch.randperm(nsamples)[:pick_samples]- total_loss = 0- num_elm = 1- mse_reduction = "mean"- if gradient_accumulate_steps != 1:- mse_reduction = "sum"- mse_loss = torch.nn.MSELoss(reduction=mse_reduction).to(device)-- for i in range(self.iters):- total_loss = 0- if self.sampler == "rand":- whole_indices = torch.randperm(nsamples)[:pick_samples]- if gradient_accumulate_steps != 1:- if q_inputs is not None:- current_input = [q_inputs[i] for i in whole_indices]- else:- current_input = [inputs[i] for i in whole_indices]- num_elm = sum(id.numel() for id in current_input)- for tmp_step in range(gradient_accumulate_steps):- indices = whole_indices[tmp_step * batch_size : (tmp_step + 1) * batch_size]- if q_inputs is not None:- current_input = [q_inputs[i] for i in indices]- current_input = torch.cat(current_input, dim=0).to(device)- org_input = [inputs[i] for i in indices]- org_input = torch.cat(org_input, dim=0).to(device)- else:- current_input = [inputs[i] for i in indices]- current_input = torch.cat(current_input, dim=0).to(device)- org_input = current_input- with torch.no_grad():- current_output = layer(org_input)-- if self.amp:- with autocast(device_type=device.split(":")[0], dtype=self.amp_dtype):- output_q = wrapper_linear(current_input) # pylint: disable=not-callable- loss = mse_loss(output_q, current_output) # pylint: disable=not-callable- else:- output_q = wrapper_linear(current_input) # pylint: disable=not-callable- loss = mse_loss( # pylint: disable=not-callable- output_q.to(torch.float32), current_output.to(torch.float32)- )- total_loss += loss.item() / num_elm-- self._scale_loss_and_backward(scaler, loss)- if i == 0:- init_loss = total_loss-- if total_loss < best_loss:- best_loss = total_loss- if not self.not_use_best_mse:- best_params = collect_best_params(wrapper_linear)- last_best_iter = i- if self.not_use_best_mse and i == self.iters - 1:- best_params = collect_best_params(wrapper_linear)-- if not self.not_use_best_mse:- if 0 < self.dynamic_max_gap <= i - last_best_iter:- break- self._step(scaler, optimizer, lr_schedule)-- last_loss = total_loss- best_iter = self.iters- if not self.not_use_best_mse:- last_loss = best_loss- best_iter = last_best_iter- with torch.no_grad():- unwrapper_layer(self.model, wrapper_linear, layer_name, best_params)- mv_module_from_gpu(layer, self.low_cpu_mem_usage)- dump_info = f"quantized {layer_name}, loss iter 0: {init_loss:.6f} -> iter {best_iter}: {last_loss:.6f}"- logger.info(dump_info)-- def _register_act_max_hook(self, model):- def get_act_max_hook(module, input, output):- if isinstance(input, (tuple, list)):- input = input[0]- if input.numel() == 0:- return # as no needs for act_max update- input, _, _ = reshape_pad_tensor_by_group_size(input, self.act_group_size)- act_max = torch.max(torch.abs(input), dim=-1).values- if not hasattr(module, "act_max") or module.act_max.numel() == 0:- module.act_max = act_max- else:- act_max = act_max.to(module.act_max.device)- if is_nv_fp(self.act_data_type): ## for nvfp per-tensor input_global_scale calculation usage- module.act_max = torch.max(- torch.tensor([act_max.max(), module.act_max.max()], device=act_max.device)- )- else:- module.act_max = torch.max(act_max, module.act_max)-- hook_handles = []-- for n, m in model.named_modules():- if (- hasattr(m, "act_dynamic")- and check_need_act_calibration(m.act_dynamic, m.act_data_type, m.act_bits)- and check_to_quantized(m)- ):- hook = m.register_forward_hook(get_act_max_hook)- hook_handles.append(hook)- continue-- # for whole model, RTN- if n in self.layer_config:- config = self.layer_config[n]- act_dynamic = config.get("act_dynamic", True)- act_data_type = config.get("act_data_type", None)- act_bits = config.get("act_data_type", 16)- if (- config["bits"] <= 8- and check_need_act_calibration(act_dynamic, act_data_type, act_bits)- and check_to_quantized(config)- ):- hook = m.register_forward_hook(get_act_max_hook)- hook_handles.append(hook)- continue- return hook_handles-- def _quantize_block(- self,- block: torch.nn.Module,- input_ids: list[torch.Tensor],- input_others: dict,- q_input: Union[None, torch.Tensor] = None,- device: Union[str, torch.device] = "cpu",- ):- """Quantize the weights of a given block of the model.-- Args:- block: The block of the model to be quantized.- input_ids: The input tensor containing tokenized input ids.- input_others: A dictionary containing additional input data.- q_input: The quantized input tensor.- device: The device for quantization.-- Returns:- Tuple: (q_outputs, output) if self.enable_quanted_input is True, else (None, output)- """- if _is_fp8_model(self.model):- for n, m in block.named_modules():- if _is_fp8_linear(m):- new_layer = convert_fp8_layer_to_linear(m, self.amp_dtype).to(device)- set_module(block, n, new_layer)-- if self.device_map == "auto":- self._set_auto_device_map_in_block(block, input_ids)-- if self.device_map is not None:- for n, m in block.named_modules():- if len(list(m.children())) != 0 or not hasattr(m, "tuning_device"):- continue- from accelerate.hooks import AlignDevicesHook, add_hook_to_module-- hook = AlignDevicesHook(m.tuning_device, io_same_device=True)- add_hook_to_module(m, hook, True)-- if q_input is None:- hook_handles = self._register_act_max_hook(block)-- output = self._get_block_outputs(- block, input_ids, input_others, self.batch_size * self.infer_bs_coeff, device, self.cache_device- )-- for handle in hook_handles:- handle.remove()- else:- output = self._get_block_outputs(- block, input_ids, input_others, self.batch_size * self.infer_bs_coeff, device, self.cache_device- )- hook_handles = self._register_act_max_hook(block)- if hook_handles:- self._get_block_outputs(- block,- q_input,- input_others,- self.batch_size * self.infer_bs_coeff,- device,- self.cache_device,- save_output=False,- )-- for handle in hook_handles:- handle.remove()-- if q_input is not None:- if input_ids is not q_input:- clear_memory(input_ids)- else:- clear_memory()- input_ids = q_input-- quantized_layer_names, unquantized_layer_names = wrapper_block(- block, self.enable_minmax_tuning, self.enable_norm_bias_tuning, device=self.device- )- if is_nv_fp(self.data_type): # enable qkv and moe structure global_scale fuse- from auto_round.data_type.utils import update_fused_layer_global_scales-- modules = block.modules()- for module in modules:- update_fused_layer_global_scales(module)- round_params = []- minmax_params = []- for n, m in block.named_modules():- if hasattr(m, "orig_layer"):- for key in m.params.keys():- if "min" in key or "max" in key:- minmax_params.append(m.params[key])- else:- round_params.append(m.params[key])-- lr = torch.tensor(self.lr)- minmax_lr = torch.tensor(self.minmax_lr)- if self.enable_minmax_tuning:- optimizer = self.optimizer(- [{"params": round_params}, {"params": minmax_params, "lr": minmax_lr}], lr=lr, weight_decay=0- )- else:- optimizer = self.optimizer(round_params, lr=lr, weight_decay=0)-- if len(round_params) + len(minmax_params) <= 0:- dump_info = (- f"quantized {len(quantized_layer_names)}/{(len(quantized_layer_names) + len(unquantized_layer_names))} "- f"layers in the block"- )- logger.info(dump_info)- unwrapper_block(block, {}) # TODO Quant layer should change- mv_module_from_gpu(block, self.low_cpu_mem_usage)- return output, output-- if self.lr_scheduler is None:- lr_schedule = torch.optim.lr_scheduler.LinearLR(- optimizer, start_factor=1.0, end_factor=0.0, total_iters=self.iters- )- else:- lr_schedule = copy.deepcopy(self.lr_scheduler)-- nsamples = len(input_ids)- pick_samples = self.batch_size * self.gradient_accumulate_steps- pick_samples = min(nsamples, pick_samples)- if self.sampler != "rand":- whole_indices = torch.randperm(nsamples)[:pick_samples]- last_best_iter = 0- best_loss = torch.finfo(torch.float).max- num_elm = 1- mse_reduction = "mean"- if self.gradient_accumulate_steps != 1:- mse_reduction = "sum"- mse_loss = torch.nn.MSELoss(reduction=mse_reduction).to(device)- scaler = self._get_scaler() # pylint: disable=assignment-from-none- init_loss = None- best_params = {}- total_loss = 0- for i in range(self.iters):- total_loss = 0- if self.sampler == "rand":- whole_indices = torch.randperm(nsamples)[:pick_samples]- # We assume the block input and output shape is same- if self.gradient_accumulate_steps != 1:- current_input_ids = [input_ids[i] for i in whole_indices]- num_elm = sum(id.numel() for id in current_input_ids)-- for tmp_step in range(self.gradient_accumulate_steps):- indices = whole_indices[tmp_step * self.batch_size : (tmp_step + 1) * self.batch_size]- current_input_ids, current_input_others = AutoRound._sampling_inputs(- input_ids,- input_others,- indices,- seqlen=self.seqlen,- batch_dim=self.batch_dim,- share_cache_keys=self.shared_cache_keys,- )-- current_output = [output[x] for x in indices]- current_output = torch.cat(current_output, dim=self.batch_dim)- current_output = to_device(current_output, device)-- output_q = block_forward(- block, current_input_ids, current_input_others, self.amp, self.amp_dtype, device- )- if self.amp:- with autocast(device_type=device.split(":")[0], dtype=self.amp_dtype):- loss = mse_loss(output_q, current_output) # pylint: disable=not-callable- else:- loss = mse_loss( # pylint: disable=not-callable- output_q.to(torch.float32), current_output.to(torch.float32)- )-- total_loss += loss.item() / num_elm- self._scale_loss_and_backward(scaler, loss)-- if i == 0:- init_loss = total_loss-- if total_loss < best_loss:- best_loss = total_loss- if not self.not_use_best_mse:- best_params = collect_best_params(block)- # print(f"get better result at iter {i}, the loss is {total_loss}", flush=True)-- last_best_iter = i- if self.not_use_best_mse and i == self.iters - 1:- best_params = collect_best_params(block)-- if not self.not_use_best_mse:- if 0 < self.dynamic_max_gap <= i - last_best_iter:- break- self._step(scaler, optimizer, lr_schedule)-- last_loss = total_loss- best_iter = self.iters- if not self.not_use_best_mse:- last_loss = best_loss- best_iter = last_best_iter- dump_info = (- f"quantized {len(quantized_layer_names)}/{(len(quantized_layer_names) + len(unquantized_layer_names))} "- f"layers in the block, loss iter 0: {init_loss:.6f} -> iter {best_iter}: {last_loss:.6f}"- )- logger.info(dump_info)- if len(unquantized_layer_names) != 0:- logger.info(f"{unquantized_layer_names} have not been quantized")- with torch.no_grad():- unwrapper_block(block, best_params)-- if (- is_nv_fp(self.act_data_type)- and hasattr(self, "formats")- and any("nv_fp" in format_ for format_ in self.formats)- ):- # enable moe experts act_max automatic generation for WrapperWALayer- set_amax_for_all_moe_layers(block, attr_name="orig_layer.act_max")-- if self.enable_quanted_input:- if self.low_cpu_mem_usage:- block = block.to(device)- clear_memory()- q_outputs = self._get_block_outputs(- block,- input_ids,- input_others,- self.batch_size * self.infer_bs_coeff,- device,- cache_device=self.cache_device,- )- if self.device_map is not None:- accelerate.hooks.remove_hook_from_submodules(block)- mv_module_from_gpu(block, self.low_cpu_mem_usage)- clear_memory(input_ids)-- return q_outputs, output-- else:- if self.device_map is not None:- accelerate.hooks.remove_hook_from_submodules(block)- mv_module_from_gpu(block, self.low_cpu_mem_usage)- clear_memory(input_ids)- return None, output-- def _quantize_blocks(- self,- model: torch.nn.Module,- inputs: dict,- block_names: list,- q_input: torch.Tensor = None,- nblocks: int = 1,- device: str = "cpu",- pbar: tqdm = None,- ):- """Quantize and dequantize the weights of the specified blocks in the model.-- Args:- model: The PyTorch model to be quantized.- inputs: The input data for quantization.- block_names: The names of the blocks to be quantized and dequantized.- nblocks: The number of blocks to quantize and dequantize.- device: The device for quantization and dequantization.-- Returns:- None- """- clear_memory()- for n, m in model.named_parameters():- m.requires_grad_(False)- input_ids = inputs["input_ids"]- inputs.pop("input_ids", None)- input_others = inputs- clear_memory()- input_ids = to_device(input_ids, self.cache_device)- input_others = to_device(input_others, self.cache_device)- # As in calibration phase, we may use bf16 for calibration due to low_gpu_memory usage- tmp_dtype = self.amp_dtype if self.amp else torch.float32- for i in range(len(input_ids)):- input_ids[i] = input_ids[i].to(tmp_dtype)-- for key in input_others.keys():- if isinstance(input_others[key], torch.Tensor) and (- input_others[key].dtype == torch.float16 or input_others[key].dtype == torch.bfloat16- ):- input_others[key] = input_others[key].to(tmp_dtype)- elif isinstance(input_others[key], list):- for i in range(len(input_others[key])):- to_dtype(input_others[key][i], tmp_dtype)-- if (- self.sym- and self.enable_alg_ext- and self.super_group_size is None- and (- (self.data_type.startswith("int") and self.act_bits >= 8)- or self.data_type.startswith("mx")- or self.data_type.startswith("nv")- )- ):- try:- from auto_round.alg_ext import quantize_block_ext-- AutoRound.quantize_block_ext = quantize_block_ext- quantize_block = self.quantize_block_ext # must use self.quantize_block_ext- if self.bits > 2 and (not self.data_type.startswith("mx") or not self.data_type.startswith("nv")):- logger.warning(- "algorithm extension has only undergone limited validation on "- "INT2,mxfp4 and nvfp4; use with caution."- )- else:- logger.info("using algorithm extension for quantization.")- except (ImportError, ModuleNotFoundError):- quantize_block = self._quantize_block- if self.enable_torch_compile:- quantize_block = compile_func(quantize_block, device)- else:- quantize_block = quantize_block- else:- quantize_block = self._quantize_block- if self.enable_torch_compile:- quantize_block = compile_func(quantize_block, device)-- if pbar is None:- pbar = tqdm(range(0, len(block_names), nblocks))-- for i in range(0, len(block_names), nblocks):- if i != 0:- pbar.update(1)- if nblocks == 1:- n = block_names[i]- pbar.set_description(f"Quantizing {n}")- m = get_module(model, n)- else:- names = block_names[i : min(i + nblocks, len(block_names))]- pbar.set_description(f"Quantizing [{i + 1}-{min(i + nblocks, len(block_names))}]/{len(block_names)}")- modules = [get_module(model, n) for n in names]- m = WrapperMultiblock(modules)-- if not self.model.device.type == "meta" or self.low_cpu_mem_usage:- m = m.to(device)-- q_input, input_ids = quantize_block(- m,- input_ids,- input_others,- q_input=q_input,- device=device,- )- if self.is_packing_immediate:- from auto_round.export import PACKING_LAYER_WITH_FORMAT-- for _, tmp_m in m.named_modules():- if not (hasattr(tmp_m, "bits") and check_to_quantized(tmp_m)):- continue- target_backend = self.formats[0].split(":")[0] if ":" in self.formats[0] else self.formats[0]- has_gguf = any("gguf" in format_ for format_ in self.formats)- if has_gguf:- from auto_round.export.export_to_gguf.export import pack_gguf_layer-- output_dir = self._get_save_folder_name(self.formats[0])- model_type = ModelType.MMPROJ if self.vlm else ModelType.TEXT- pack_gguf_layer(- tmp_m.tmp_name,- self.model,- self.formats[0],- output_dir,- self.layer_config,- self.tokenizer,- processor=self.processor if hasattr(self, "processor") else None,- image_processor=self.image_processor if hasattr(self, "image_processor") else None,- model_type=model_type,- )- else:- PACKING_LAYER_WITH_FORMAT[target_backend](- tmp_m.tmp_name, self.model, self.formats[0], device=self.device- )- pbar.set_description("Quantizing done")- pbar.update(1)- pbar.close()- self.model = mv_module_from_gpu(self.model, self.low_cpu_mem_usage)- for n, m in self.model.named_modules():- if hasattr(m, "name"):- delattr(m, "name")-- del q_input- del input_ids- del input_others- del inputs-- clear_memory()-- def save_quantized(- self, output_dir: str = None, format: str = "auto_round", inplace: bool = True, **kwargs- ) -> torch.nn.Module:- """Save the quantized model to the specified output directory in the specified format.+ tokenizer=None,+ scheme: Union[str, dict, QuantizationScheme] = "W4A16",+ layer_config: dict[str, Union[str, dict, QuantizationScheme]] = None,+ dataset: Union[str, list, tuple, torch.utils.data.DataLoader] = "NeelNanda/pile-10k",+ iters: int = 200,+ seqlen: int = 2048,+ nsamples: int = 128,+ batch_size: int = 8,+ gradient_accumulate_steps: int = 1,+ low_gpu_mem_usage: bool = False,+ device_map: Union[str, torch.device, int, dict] = 0,+ enable_torch_compile: bool = False,+ seed: int = 42,+ # for adam+ enable_adam: bool = False,+ # for MLLM+ extra_config: ExtraConfig = None,+ **kwargs,+ ) -> BaseCompressor:+ """Initialize AutoRound with quantization and tuning configuration.Args:- output_dir (str, optional): The directory to save the quantized model. Defaults to None.- format (str, optional): The format in which to save the model. Defaults to "auto_round".- inplace (bool, optional): Whether to modify the model in place. Defaults to True.- **kwargs: Additional keyword arguments specific to the export format.+ model (torch.nn.Module | str): Model object or model name to load.+ tokenizer: Tokenizer for text processing. Required if `model` is not a string and `iters > 0`.+ scheme (str| dict | QuantizationScheme ): A preset scheme that defines the quantization configurations+ bits (int, optional): Weight quantization bits. Defaults to 4.+ group_size (int, optional): Weight quantization group size. Defaults to 128.+ sym (bool, optional): Symmetric weight quantization. Defaults to True.+ layer_config (dict, optional): Layer-wise quantization config. Defaults to None.+ batch_size (int, optional): Calibration batch size. Defaults to 8.+ amp (bool, optional): Use AMP for tuning. Defaults to True.+ device (str | torch.device | int, optional): Compute device. Defaults to 0.+ dataset (str | list | tuple | DataLoader, optional): Calibration data. Defaults to "NeelNanda/pile-10k".+ enable_minmax_tuning (bool, optional): Enable weight min-max tuning. Defaults to True.+ lr (float, optional): Learning rate; if None, set to 1.0 / iters except when iters==0.+ minmax_lr (float, optional): Learning rate for min-max tuning; defaults to `lr`.+ low_gpu_mem_usage (bool, optional): Lower GPU memory mode. Defaults to False.+ iters (int, optional): Optimization iterations. Defaults to 200.+ seqlen (int, optional): Calibration sequence length. Defaults to 2048.+ nsamples (int, optional): Number of calibration samples. Defaults to 128.+ seed (int, optional): Random seed. Defaults to 42.+ gradient_accumulate_steps (int, optional): Gradient accumulation steps. Defaults to 1.+ data_type (str, optional): Weight data type string, e.g., "int". Defaults to "int".+ act_bits (int, optional): Activation quantization bits. Defaults to 16.+ act_group_size (int, optional): Activation group size. Defaults to None.+ act_sym (bool, optional): Symmetric activation quantization. Defaults to None.+ act_data_type (str, optional): Activation data type; inherits weight dtype if None and act_bits < 16.+ act_dynamic (bool, optional): Dynamic activation quantization. Defaults to True.+ enable_torch_compile (bool, optional): Enable torch.compile for quant blocks/layers. Defaults to False.+ device_map (str | dict, optional): Device placement map. Defaults to None.+ disable_opt_rtn (bool, optional): Disable RTN-mode optimization (iters=0). Defaults to False.+ enable_alg_ext (bool, optional): Enable algorithm extension (primarily for INT2). Defaults to False.+ **kwargs: Backward compatible options:+ - enable_alg_ext, quant_lm_head, lr, lr_scheduler, sampler, not_use_best_mse, dynamic_max_gap,+ super_group_size, super_bits, scale_dtype ("fp16" etc.),+ nblocks, low_cpu_mem_usage, to_quant_block_names,+ enable_norm_bias_tuning, enable_quanted_input,+ disable_deterministic_algorithms, vlm, static_kv_dtype+ Raises:+ ValueError: If invalid device is provided or tokenizer is missing for non-str model with iters > 0.+ RuntimeError: If model parameters are on meta device.+ Example:+ Layer-wise configuration structure:- Returns:- object: The compressed model object.+ >>> layer_config = {+ ... "layer1": {+ ... "data_type": "int",+ ... "bits": 4,+ ... "group_size": 128,+ ... "sym": True,+ ... "act_data_type": None,+ ... "act_bits": 16,+ ... "act_group_size": None,+ ... "act_sym": None,+ ... },+ ... # ...+ ... }"""- format = self._check_supported_format(format)-- if self.low_cpu_mem_usage:- self.model = self.model.to("cpu")-- if not self.quantized:- logger.warning("please run autoround.quantize first")- return- if format == "fake" or format == "qdq": # TODO fix act quantization later- self.model = self.model.to("cpu")- self.model.save_pretrained(output_dir)- if self.tokenizer is not None:- self.tokenizer.save_pretrained(output_dir)- processor = kwargs.get("processor", None)- if processor is not None:- processor.save_pretrained(output_dir)- try:- copy_python_files_from_model_cache(self.model, output_dir)- except Exception as e:- logger.warning("Skipping source model Python file copy due to error: %s", e)- return- if self.act_bits <= 8 and format == "qdq":- logger.warning(- "Support for exporting activation quantization is limited. "- "Please ensure that your configuration is supported."- )- if format == "llm_compressor" and (is_nv_fp(self.data_type) or is_mx_fp(self.data_type)):- format = format.replace("llm_compressor", f"llm_compressor:{self.data_type}")+ model_cls = []- from auto_round.export import EXPORT_FORMAT-- backend = format- format = format.split(":")[0]- if format not in EXPORT_FORMAT:- logger.error(f"export format only supports {EXPORT_FORMAT.keys()}")- raise ValueError(f"export format only supports {EXPORT_FORMAT.keys()}, but got {format}")- save_quantized_as_format = EXPORT_FORMAT.get(format)- if "gptq" in format and not self.sym:- logger.warning(- "the asymmetrical kernel of the GPTQ format may result in a noticeable accuracy drop,"- " particularly for 2-bit quantization and smaller models."- " We recommend exporting to either the AutoAWQ format ( only 4 bits) or "- "the AutoRound format(2/3/4/8 bits)."- )- if "awq" in format and not self.bits == 4:- raise ValueError("The AWQ format only supports W4 quantization ")- serialization_keys = [- "bits",- "group_size",- "sym",- "data_type",- "enable_quanted_input",- "enable_minmax_tuning",- "seqlen",- "batch_size",- "scale_dtype",- "lr",- "minmax_lr",- "gradient_accumulate_steps",- "iters",- "amp",- "nsamples",- "low_gpu_mem_usage",- "to_quant_block_names",- "enable_norm_bias_tuning",- "act_bits",- "act_group_size",- "act_sym",- "act_dynamic",- "act_data_type",- "super_bits",- "super_group_size",- ]- if isinstance(self.dataset, str):- serialization_keys.append("dataset")- serialization_dict = {}- for key in serialization_keys:- serialization_dict[key] = getattr(self, key)- from .version import __version__-- serialization_dict["autoround_version"] = __version__- if "scale_dtype" in serialization_dict.keys():- serialization_dict["scale_dtype"] = str(serialization_dict["scale_dtype"])+ if (extra_config and not extra_config.mllm_config.is_default()) or is_mllm_model(model):+ logger.info("using MLLM mode for multimodal model.")+ model_cls.append(MLLMCompressor)+ else:+ if extra_config:+ extra_config.mllm_config = None+ model_cls.append(LLMCompressor)- compressed_model = save_quantized_as_format( # TODO refine the code- output_dir,- model=self.model,- layer_config=self.layer_config,- inplace=inplace,- bits=self.bits,- act_bits=self.act_bits,- group_size=self.group_size,- sym=self.sym,- iters=self.iters,- lr=self.lr,- minmax_lr=self.minmax_lr,- enable_minmax_tuning=self.enable_minmax_tuning,- enable_quanted_input=self.enable_quanted_input,- scale_dtype=self.scale_dtype,- tokenizer=self.tokenizer,- supported_types=self.supported_types,- data_type=self.data_type,- act_data_type=self.act_data_type,- serialization_dict=serialization_dict,- backend=backend,- to_quant_block_names=self.to_quant_block_names,- quant_block_list=self.quant_block_list,- device=self.device,+ if enable_adam:+ model_cls.append(AdamCompressor)+ dynamic_compressor = type("AutoRound", tuple(model_cls), {})+ if extra_config:+ kwargs.update(extra_config.to_dict())+ ar = dynamic_compressor(+ model=model,+ tokenizer=tokenizer,+ scheme=scheme,+ layer_config=layer_config,+ dataset=dataset,+ iters=iters,+ seqlen=seqlen,+ nsamples=nsamples,+ batch_size=batch_size,+ gradient_accumulate_steps=gradient_accumulate_steps,+ low_gpu_mem_usage=low_gpu_mem_usage,+ device_map=device_map,+ enable_torch_compile=enable_torch_compile,+ seed=seed,**kwargs,)- return compressed_model-- def _get_quantized_layer_names_outside_blocks(self) -> list:- """Gets the names of quantized layers outside blocks in the model.-- Returns:- list: List of layer names outside blocks.- """- if self.layer_config is None or len(self.layer_config) == 0:- return []-- layer_names = []- all_layers_in_block = get_layer_names_in_block(self.model, self.supported_types, self.quant_block_list)-- for key in self.layer_config.keys():- if key in all_layers_in_block:- continue- layer = get_module(self.model, key)- if layer is None:- logger.error(f"could not find layer {key} in the model, exit...")- exit(-1)- if isinstance(layer, tuple(self.supported_types)) and check_to_quantized(self.layer_config[key]):- layer_names.append(key)-- return layer_names-- def _set_amp_dtype(self) -> None:- """Sets the automatic mixed precision (AMP) data type for the model based on the device and configuration."""- self.amp_dtype = torch.bfloat16- if self.model.dtype != torch.float32:- self.amp_dtype = self.model.dtype- if self.device == "cpu" or "hpu" in self.device:- self.amp_dtype = torch.bfloat16- if self.amp:- if self.device == "cpu" and not CpuInfo().bf16:- self.amp = False- self.amp_dtype = torch.float32- self.model = self.model.to(torch.float32)- logger.warning(- f"amp is set to FALSE as the current {self.device} device does not support the 'bf16' data type."- )- else:- if self.model.dtype != self.amp_dtype:- self.model = self.model.to(self.amp_dtype)- else:- self.amp_dtype = torch.float32- self.model = self.model.to(torch.float32)-- def _get_optimizer(self, optimizer: Any):- """Returns the specified optimizer. In SignRound, we fix the optimizer.-- Args:- optimizer: The optimizer to be used.-- Returns:- The specified optimizer.- """- return SignSGD-- def _get_scaler(self):- """Returns scaler, in SignRound, no need to use scaler."""- return None-- def _scale_loss_and_backward(self, scaler: Any, loss: torch.Tensor) -> torch.Tensor:- """Scales the loss and performs backward pass.-- Args:- scaler: The scaler to be used.- loss: The loss to be scaled.-- Returns:- The scaled loss.- """- scale_loss = loss * 1000- scale_loss.backward()- if is_optimum_habana_available():- htcore.mark_step()- return scale_loss-- def _step(self, scaler: Any, optimizer: Any, lr_schedule: Any):- """Performs a step in the optimization process.-- Args:- scaler: The scaler to be used.- optimizer: The optimizer for the step.- lr_schedule: The learning rate schedule.-- Returns:- None- """- optimizer.step()- # for hpu- if is_optimum_habana_available():- htcore.mark_step()- optimizer.zero_grad()- lr_schedule.step()+ return ar@classmethod@torch.no_grad()@@ -3310,8 +224,120 @@ class AutoRound(object):return current_input_ids, current_input_others-class AutoRoundAdam(AutoRound):- """Class for automatic rounding-based quantization with optimizers like adamw of a PyTorch model.+@deprecated("AutoRound")+class AutoRoundLLM(LLMCompressor):+ """Class for LLM quantization++ Args:+ model (torch.nn.Module | str): Model object or model name to load.+ tokenizer: Tokenizer for text processing. Required if `model` is not a string and `iters > 0`.+ scheme (str| dict | QuantizationScheme ): A preset scheme that defines the quantization configurations+ bits (int, optional): Weight quantization bits. Defaults to 4.+ group_size (int, optional): Weight quantization group size. Defaults to 128.+ sym (bool, optional): Symmetric weight quantization. Defaults to True.+ layer_config (dict, optional): Layer-wise quantization config. Defaults to None.+ batch_size (int, optional): Calibration batch size. Defaults to 8.+ amp (bool, optional): Use AMP for tuning. Defaults to True.+ device (str | torch.device | int, optional): Compute device. Defaults to 0.+ dataset (str | list | tuple | DataLoader, optional): Calibration data. Defaults to "NeelNanda/pile-10k".+ enable_minmax_tuning (bool, optional): Enable weight min-max tuning. Defaults to True.+ lr (float, optional): Learning rate; if None, set to 1.0 / iters except when iters==0.+ minmax_lr (float, optional): Learning rate for min-max tuning; defaults to `lr`.+ low_gpu_mem_usage (bool, optional): Lower GPU memory mode. Defaults to False.+ iters (int, optional): Optimization iterations. Defaults to 200.+ seqlen (int, optional): Calibration sequence length. Defaults to 2048.+ nsamples (int, optional): Number of calibration samples. Defaults to 128.+ seed (int, optional): Random seed. Defaults to 42.+ gradient_accumulate_steps (int, optional): Gradient accumulation steps. Defaults to 1.+ data_type (str, optional): Weight data type string, e.g., "int". Defaults to "int".+ act_bits (int, optional): Activation quantization bits. Defaults to 16.+ act_group_size (int, optional): Activation group size. Defaults to None.+ act_sym (bool, optional): Symmetric activation quantization. Defaults to None.+ act_data_type (str, optional): Activation data type; inherits weight dtype if None and act_bits < 16.+ act_dynamic (bool, optional): Dynamic activation quantization. Defaults to True.+ enable_torch_compile (bool, optional): Enable torch.compile for quant blocks/layers. Defaults to False.+ device_map (str | dict, optional): Device placement map. Defaults to None.+ disable_opt_rtn (bool, optional): Disable RTN-mode optimization (iters=0). Defaults to False.+ enable_alg_ext (bool, optional): Enable algorithm extension (primarily for INT2). Defaults to False.+ **kwargs: Backward compatible options:+ - enable_alg_ext, quant_lm_head, lr, lr_scheduler, sampler, not_use_best_mse, dynamic_max_gap,+ super_group_size, super_bits, scale_dtype ("fp16" etc.),+ nblocks, low_cpu_mem_usage, to_quant_block_names,+ enable_norm_bias_tuning, enable_quanted_input,+ disable_deterministic_algorithms, mllm, static_kv_dtype+ Raises:+ ValueError: If invalid device is provided or tokenizer is missing for non-str model with iters > 0.+ RuntimeError: If model parameters are on meta device.+ Example:+ Layer-wise configuration structure:++ >>> layer_config = {+ ... "layer1": {+ ... "data_type": "int",+ ... "bits": 4,+ ... "group_size": 128,+ ... "sym": True,+ ... "act_data_type": None,+ ... "act_bits": 16,+ ... "act_group_size": None,+ ... "act_sym": None,+ ... },+ ... # ...+ ... }+ """++ bits: int | None+ group_size: int | None+ sym: bool | None+ data_type: str | None+ act_bits: int | None+ act_group_size: int | None+ act_sym: bool | None+ act_data_type: str | None+ act_dynamic: bool | None+ super_bits: int | None+ super_group_size: int | None++ def __init__(+ self,+ model: Union[torch.nn.Module, str],+ tokenizer=None,+ scheme: Union[str, dict, QuantizationScheme] = "W4A16",+ layer_config: dict[str, Union[str, dict, QuantizationScheme]] = None,+ dataset: Union[str, list, tuple, torch.utils.data.DataLoader] = "NeelNanda/pile-10k",+ iters: int = 200,+ seqlen: int = 2048,+ nsamples: int = 128,+ batch_size: int = 8,+ gradient_accumulate_steps: int = 1,+ low_gpu_mem_usage: bool = False,+ device_map: Union[str, torch.device, int, dict] = 0,+ enable_torch_compile: bool = False,+ seed: int = 42,+ **kwargs,+ ):+ super().__init__(+ model=model,+ tokenizer=tokenizer,+ scheme=scheme,+ layer_config=layer_config,+ dataset=dataset,+ iters=iters,+ seqlen=seqlen,+ nsamples=nsamples,+ batch_size=batch_size,+ gradient_accumulate_steps=gradient_accumulate_steps,+ low_gpu_mem_usage=low_gpu_mem_usage,+ device_map=device_map,+ enable_torch_compile=enable_torch_compile,+ seed=seed,+ **kwargs,+ )+++@deprecated("AutoRound")+class AutoRoundAdam(AdamCompressor):+ """Class for quantization with optimizers like adamw of a PyTorch model.Args:model: The PyTorch model to be quantized.@@ -3391,7 +417,7 @@ class AutoRoundAdam(AutoRound):optimizer="AdamW",**kwargs,):- super(AutoRoundAdam, self).__init__(+ super().__init__(model=model,tokenizer=tokenizer,scheme=scheme,@@ -3406,46 +432,111 @@ class AutoRoundAdam(AutoRound):gradient_accumulate_steps=gradient_accumulate_steps,enable_torch_compile=enable_torch_compile,device_map=device_map,+ optimizer=optimizer,**kwargs,)- self.optimizer = self._get_optimizer(optimizer)- def _get_optimizer(self, optimizer):- if optimizer is None:- optimizer = torch.optim.AdamW- elif isinstance(optimizer, str):- optimizer = getattr(torch.optim, optimizer)- else:- optimizer = optimizer- return optimizer-- def _get_scaler(self):- scaler = None- if self.amp and not check_is_cpu(self.device):- from torch.cuda.amp import GradScaler+@deprecated("AutoRound")+class AutoRoundMLLM(MLLMCompressor):+ """Class for automatic rounding-based quantization with MLLMs.- scaler = GradScaler(init_scale=1024, growth_interval=100000)- return scaler-- def _scale_loss_and_backward(self, scaler, loss):- if scaler is not None:- loss = scaler.scale(loss)+ Args:+ model: The PyTorch model to be quantized.+ tokenizer: An optional tokenizer for processing input data.+ processor: Any multi-modal model will require an object to encode or+ decode the data that groups several modalities (among text, vision and audio).+ image_processor: Image processor for special model like llava.+ bits (int): Number of bits for quantization (default is 4).+ group_size (int): Size of the quantization group (default is 128).+ sym (bool): Whether sym to be used (default is True).+ layer_config (dict): Configuration for weight quantization (default is None).+ batch_size (int): Batch size for training (default is 8).+ amp (bool): Whether to use automatic mixed precision (default is True).+ device: The device to be used for training (default is "auto").+ lr_scheduler: The learning rate scheduler to be used.+ dataset: The path or name of the calib dataset.+ extra_data_dir: The path of extra data such as images, audio and videos.+ template: The path or name of template used to specify process for different MLLMs.+ quant_nontext_module: Whether to quantize nontext module.+ enable_quanted_input (bool): Whether to use quantized input data (default is True).+ enable_minmax_tuning (bool): Whether to enable min-max tuning (default is True).+ lr (float): The learning rate (default is 0.005).+ minmax_lr (float): The learning rate for min-max tuning (default is None).+ low_gpu_mem_usage (bool): Whether to use low GPU memory (default is False).+ low_cpu_mem_usage (bool): Whether to use low CPU memory (default is False).+ iters (int): Number of iterations (default is 200).+ seqlen (int): Length of the sequence.+ nsamples (int): Number of samples (default is 128).+ sampler (str): The sampling method (default is "rand").+ seed (int): The random seed (default is 42).s+ nblocks (int): Number of blocks (default is 1).+ gradient_accumulate_steps (int): Number of gradient accumulation steps (default is 1).+ not_use_best_mse (bool): Whether to use mean squared error (default is False).+ dynamic_max_gap (int): The dynamic maximum gap (default is -1).+ data_type (str): The data type to be used (default is "int").+ scale_dtype (str): The data type of quantization scale to be used (default is "float16"), different kernels+ have different choices.+ act_bits (int): Number of bits for activation quantization. Default is 32.+ act_group_size (int): Group size for activation quantization. Default is None.+ act_sym (bool): Whether to use symmetric activation quantization. Default is None.+ act_dynamic (bool): Whether to use dynamic activation quantization. Default is True.+ to_quant_block_names (str|list): A string or list whose elements are list of+ block's layer names to be quantized.+ enable_torch_compile (bool): Whether to enable torch compile to optimize quant_block/layer+ **kwargs: Additional keyword arguments.+ """- loss.backward()- if is_optimum_habana_available():- htcore.mark_step()- return loss+ bits: int | None+ group_size: int | None+ sym: bool | None+ data_type: str | None+ act_bits: int | None+ act_group_size: int | None+ act_sym: bool | None+ act_data_type: str | None+ act_dynamic: bool | None+ super_bits: int | None+ super_group_size: int | None- def _step(self, scaler, optimizer, lr_schedule):- if scaler is not None:- scaler.step(optimizer)- optimizer.zero_grad()- lr_schedule.step()- scaler.update()- else:- optimizer.step()- optimizer.zero_grad()- lr_schedule.step()- if is_optimum_habana_available():- htcore.mark_step()+ def __init__(+ self,+ model: Union[torch.nn.Module, str],+ tokenizer=None,+ processor=None,+ image_processor=None,+ scheme: Union[str, dict, QuantizationScheme] = "W4A16",+ layer_config: dict[str, Union[str, dict, QuantizationScheme]] = None,+ dataset: Union[str, list, tuple, torch.utils.data.DataLoader] = "NeelNanda/pile-10k",+ quant_nontext_module: bool = False,+ iters: int = 200,+ seqlen: int = 2048,+ nsamples: int = 128,+ batch_size: int = 8,+ gradient_accumulate_steps: int = 1,+ low_gpu_mem_usage: bool = False,+ device_map: Union[str, torch.device, int, dict] = 0,+ enable_torch_compile: bool = False,+ seed: int = 42,+ **kwargs,+ ):+ super().__init__(+ model=model,+ tokenizer=tokenizer,+ processor=processor,+ image_processor=image_processor,+ scheme=scheme,+ layer_config=layer_config,+ dataset=dataset,+ quant_nontext_module=quant_nontext_module,+ iters=iters,+ seqlen=seqlen,+ nsamples=nsamples,+ batch_size=batch_size,+ gradient_accumulate_steps=gradient_accumulate_steps,+ low_gpu_mem_usage=low_gpu_mem_usage,+ device_map=device_map,+ enable_torch_compile=enable_torch_compile,+ seed=seed,+ **kwargs,+ )diff --git a/auto_round/script/__init__.py b/auto_round/compressors/__init__.pysimilarity index 66%rename from auto_round/script/__init__.pyrename to auto_round/compressors/__init__.pyindex 28f108cb6..35bbc5666 100644--- a/auto_round/script/__init__.py+++ b/auto_round/compressors/__init__.py@@ -1,4 +1,4 @@-# Copyright (c) 2024 Intel Corporation+# Copyright (c) 2025 Intel Corporation## Licensed under the Apache License, Version 2.0 (the "License");# you may not use this file except in compliance with the License.@@ -11,3 +11,12 @@# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.# See the License for the specific language governing permissions and# limitations under the License.++from auto_round.compressors.base import *+from auto_round.compressors.mllm.compressor import MLLMCompressor+from auto_round.compressors.config import (+ ExtraConfig,+ MLLMExtraConfig,+ SchemeExtraConfig,+ TuningExtraConfig,+)diff --git a/auto_round/compressors/base.py b/auto_round/compressors/base.pynew file mode 100644index 000000000..9ed346bb8--- /dev/null+++ b/auto_round/compressors/base.py@@ -0,0 +1,3468 @@+# Copyright (c) 2023 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.++import copy+import os+import re+import sys+import time+import traceback+from dataclasses import asdict, fields+from enum import Enum+from typing import Any, Callable, Union++import accelerate+import torch+from accelerate.big_modeling import dispatch_model, infer_auto_device_map+from torch import autocast+from tqdm import tqdm+from transformers import set_seed++from auto_round.data_type import QUANT_FUNC_WITH_DTYPE+from auto_round.data_type.utils import reshape_pad_tensor_by_group_size+from auto_round.export.export_to_autoround import AutoRoundFormat+from auto_round.export.export_to_gguf.config import GGUF_CONFIG, GGUF_INNER_CONFIG, ModelType+from auto_round.logger import logger+from auto_round.low_cpu_mem.utils import get_layers_before_block+from auto_round.schemes import QuantizationScheme, preset_name_to_scheme+from auto_round.sign_sgd import SignSGD+from auto_round.special_model_handler import _handle_moe_model+from auto_round.utils import (+ INNER_SUPPORTED_LAYER_TYPES,+ SUPPORTED_DTYPES,+ SUPPORTED_FORMATS,+ SUPPORTED_LAYER_TYPES,+ TORCH_VERSION_AT_LEAST_2_6,+ CpuInfo,+ _gguf_args_check,+ _is_fp8_linear,+ _is_fp8_model,+ block_forward,+ check_and_mark_fp8_model,+ check_is_cpu,+ check_need_act_calibration,+ check_seqlen_compatible,+ check_skippable_keywords,+ check_to_quantized,+ clear_memory,+ collect_best_params,+ compile_func,+ convert_dtype_str2torch,+ convert_fp8_layer_to_linear,+ convert_fp8_model_to_16b_model,+ copy_python_files_from_model_cache,+ detect_device,+ estimate_tuning_block_mem,+ find_matching_blocks,+ flatten_list,+ get_block_names,+ get_device_memory,+ get_fp_layer_names,+ get_layer_config_by_gguf_format,+ get_layer_features,+ get_layer_names_in_block,+ get_lm_head_name,+ get_max_vram,+ get_module,+ get_quant_keys,+ get_shared_keys,+ htcore,+ infer_bits_by_data_type,+ init_cache,+ is_debug_mode,+ is_mx_fp,+ is_nv_fp,+ is_optimum_habana_available,+ is_standard_fp,+ is_static_wfp8afp8,+ is_wfp8afp8,+ llm_load_model,+ mv_module_from_gpu,+ reset_params,+ set_amax_for_all_moe_layers,+ set_module,+ to_device,+ to_dtype,+ unsupport_meta_device,+)+from auto_round.wrapper import WrapperLinear, WrapperMultiblock, unwrapper_block, unwrapper_layer, wrapper_block+++class BaseCompressor(object):+ """Base compressor for LLM quantization++ Attributes:+ model (torch.nn.Module): The loaded PyTorch model in eval mode.+ tokenizer: Tokenizer used to prepare input text for calibration/tuning.+ bits (int): Weight quantization bits.+ group_size (int): Per-group size for weight quantization.+ sym (bool): Whether to use symmetric weight quantization.+ layer_config (dict): Per-layer quantization configuration.+ nsamples (int): Number of calibration samples.+ enable_torch_compile (bool): Whether to enable torch.compile for quant blocks/layers.+ """++ bits: int | None+ group_size: int | None+ sym: bool | None+ data_type: str | None+ act_bits: int | None+ act_group_size: int | None+ act_sym: bool | None+ act_data_type: str | None+ act_dynamic: bool | None+ super_bits: int | None+ super_group_size: int | None++ def __init__(+ self,+ model: Union[torch.nn.Module, str],+ tokenizer=None,+ scheme: Union[str, dict, QuantizationScheme] = "W4A16",+ layer_config: dict[str, Union[str, dict, QuantizationScheme]] = None,+ dataset: Union[str, list, tuple, torch.utils.data.DataLoader] = "NeelNanda/pile-10k",+ iters: int = 200,+ seqlen: int = 2048,+ nsamples: int = 128,+ batch_size: int = 8,+ gradient_accumulate_steps: int = 1,+ low_gpu_mem_usage: bool = False,+ device_map: Union[str, torch.device, int, dict] = 0,+ enable_torch_compile: bool = False,+ seed: int = 42,+ **kwargs,+ ):+ """Initialize AutoRound with quantization and tuning configuration.++ Args:+ model (torch.nn.Module | str): Model object or model name to load.+ tokenizer: Tokenizer for text processing. Required if `model` is not a string and `iters > 0`.+ scheme (str| dict | QuantizationScheme ): A preset scheme that defines the quantization configurations+ bits (int, optional): Weight quantization bits. Defaults to 4.+ group_size (int, optional): Weight quantization group size. Defaults to 128.+ sym (bool, optional): Symmetric weight quantization. Defaults to True.+ layer_config (dict, optional): Layer-wise quantization config. Defaults to None.+ batch_size (int, optional): Calibration batch size. Defaults to 8.+ amp (bool, optional): Use AMP for tuning. Defaults to True.+ device (str | torch.device | int, optional): Compute device. Defaults to 0.+ dataset (str | list | tuple | DataLoader, optional): Calibration data. Defaults to "NeelNanda/pile-10k".+ enable_minmax_tuning (bool, optional): Enable weight min-max tuning. Defaults to True.+ lr (float, optional): Learning rate; if None, set to 1.0 / iters except when iters==0.+ minmax_lr (float, optional): Learning rate for min-max tuning; defaults to `lr`.+ low_gpu_mem_usage (bool, optional): Lower GPU memory mode. Defaults to False.+ iters (int, optional): Optimization iterations. Defaults to 200.+ seqlen (int, optional): Calibration sequence length. Defaults to 2048.+ nsamples (int, optional): Number of calibration samples. Defaults to 128.+ seed (int, optional): Random seed. Defaults to 42.+ gradient_accumulate_steps (int, optional): Gradient accumulation steps. Defaults to 1.+ data_type (str, optional): Weight data type string, e.g., "int". Defaults to "int".+ act_bits (int, optional): Activation quantization bits. Defaults to 16.+ act_group_size (int, optional): Activation group size. Defaults to None.+ act_sym (bool, optional): Symmetric activation quantization. Defaults to None.+ act_data_type (str, optional): Activation data type; inherits weight dtype if None and act_bits < 16.+ act_dynamic (bool, optional): Dynamic activation quantization. Defaults to True.+ enable_torch_compile (bool, optional): Enable torch.compile for quant blocks/layers. Defaults to False.+ device_map (str | dict, optional): Device placement map. Defaults to None.+ disable_opt_rtn (bool, optional): Disable RTN-mode optimization (iters=0). Defaults to False.+ enable_alg_ext (bool, optional): Enable algorithm extension (primarily for INT2). Defaults to False.+ **kwargs: Backward compatible options:+ - enable_alg_ext, quant_lm_head, lr, lr_scheduler, sampler, not_use_best_mse, dynamic_max_gap,+ super_group_size, super_bits, scale_dtype ("fp16" etc.),+ nblocks, low_cpu_mem_usage, to_quant_block_names,+ enable_norm_bias_tuning, enable_quanted_input,+ disable_deterministic_algorithms, mllm, static_kv_dtype+ Raises:+ ValueError: If invalid device is provided or tokenizer is missing for non-str model with iters > 0.+ RuntimeError: If model parameters are on meta device.+ Example:+ Layer-wise configuration structure:++ >>> layer_config = {+ ... "layer1": {+ ... "data_type": "int",+ ... "bits": 4,+ ... "group_size": 128,+ ... "sym": True,+ ... "act_data_type": None,+ ... "act_bits": 16,+ ... "act_group_size": None,+ ... "act_sym": None,+ ... },+ ... # ...+ ... }+ """+ self.scheme = None+ self._parse_and_set_scheme(scheme, kwargs)++ # Extra/legacy kwargs for backward compatibility+ # Major version releases may pack them with extra configuration options+ amp = kwargs.pop("amp", True)+ lr = kwargs.pop("lr", None)+ enable_alg_ext = kwargs.pop("enable_alg_ext", False)+ enable_minmax_tuning = kwargs.pop("enable_minmax_tuning", True)+ minmax_lr = kwargs.pop("minmax_lr", None)+ disable_opt_rtn = kwargs.pop("disable_opt_rtn", False)+ lr_scheduler = kwargs.pop("lr_scheduler", None)+ sampler = kwargs.pop("sampler", "rand")+ not_use_best_mse = kwargs.pop("not_use_best_mse", False)+ dynamic_max_gap = kwargs.pop("dynamic_max_gap", -1)+ scale_dtype = kwargs.pop("scale_dtype", "fp16")+ nblocks = kwargs.pop("nblocks", 1)+ low_cpu_mem_usage = kwargs.pop("low_cpu_mem_usage", False)+ to_quant_block_names: Union[str, list, None] = kwargs.pop("to_quant_block_names", None)+ enable_norm_bias_tuning: bool = kwargs.pop("enable_norm_bias_tuning", False)+ enable_quanted_input: bool = kwargs.pop("enable_quanted_input", True)+ disable_deterministic_algorithms = kwargs.pop("disable_deterministic_algorithms", True)+ enable_deterministic_algorithms = kwargs.pop("enable_deterministic_algorithms", False)+ static_kv_dtype = kwargs.pop("static_kv_dtype", None)+ device = kwargs.pop("device", None)+ self.quant_lm_head = kwargs.pop("quant_lm_head", False)+ self.mllm = kwargs.pop("mllm") if "mllm" in kwargs else False+ # Scale factor for RAM usage per parameter.+ self.mem_per_param_scale = kwargs.pop("mem_per_param_scale", None)+ fp_layers = kwargs.pop("fp_layers", None)++ if kwargs:+ logger.warning(f"unrecognized keys {list(kwargs.keys())} were passed. Please check them.")+ if "CUBLAS_WORKSPACE_CONFIG" not in os.environ:+ os.environ["CUBLAS_WORKSPACE_CONFIG"] = ":4096:8"+ # deprecated, default not to use torch.use_deterministic_algorithms+ if not disable_deterministic_algorithms or enable_deterministic_algorithms:+ if not disable_deterministic_algorithms:+ logger.warning(+ "default not use deterministic_algorithms. disable_deterministic_algorithms is deprecated,"+ " please use enable_deterministic_algorithms instead. "+ )++ torch.use_deterministic_algorithms(True, warn_only=False)+ else:+ torch.use_deterministic_algorithms(True, warn_only=True)++ if device is not None:+ logger.warning("`device` is deprecated, please use `device_map` instead")++ if device_map is None:+ device_map = 0++ # Set device, must place after model loading+ self._set_device(device_map)++ if (isinstance(device_map, dict) and device_map) or device_map == "auto":+ self.device_map = device_map+ elif isinstance(device_map, str) and "," in device_map:+ device_map = device_map.replace(" ", "") # Remove any spaces+ self.device_list = [int(dev) for dev in device_map.split(",") if dev.isdigit()]+ self.device_map = "auto"+ else:+ self.device_map = None+ self._set_device_map_in_blocks(self.device_map)++ # Model related+ self.quantized = False+ if isinstance(model, str):+ model, tokenizer, low_cpu_mem_usage = llm_load_model(+ model,+ device="cpu",+ low_cpu_mem_mode=low_cpu_mem_usage, # always load cpu first+ )+ elif tokenizer is None and iters > 0:+ raise ValueError("A tokenizer must be set for non-str model input")+ self.low_cpu_mem_usage = bool(low_cpu_mem_usage)+ if unsupport_meta_device(model):+ raise RuntimeError(+ "AutoRound does not support parameters on meta device. "+ "Please use more GPUs by setting `--device 0,1,2,3` or just place the model on CPU."+ )+ check_and_mark_fp8_model(model)+ self.model = model.eval()+ self.tokenizer = tokenizer+ self.shared_cache_keys = get_shared_keys(self.model)++ not_quantize_layer_names = get_fp_layer_names(self.model, fp_layers)+ if len(not_quantize_layer_names) > 0:+ logger.info(f"{not_quantize_layer_names} will not be quantized.")+ if layer_config is None:+ layer_config = {}+ for name in not_quantize_layer_names:+ layer_config[name] = {"bits": 16, "act_bits": 16, "data_type": "float", "act_data_type": "float"}+ self._parse_layer_config(layer_config) # must place after model init++ self.to_quant_block_names = to_quant_block_names++ # Tuning hyperparameters+ self.seed = seed+ set_seed(self.seed)+ self.amp = amp+ self.enable_quanted_input = enable_quanted_input+ self.enable_minmax_tuning = enable_minmax_tuning+ self.nsamples = nsamples+ self.enable_norm_bias_tuning = enable_norm_bias_tuning+ self.low_gpu_mem_usage = low_gpu_mem_usage+ self.seqlen = seqlen+ self.batch_size, self.gradient_accumulate_steps = batch_size, gradient_accumulate_steps+ self.nblocks = nblocks+ self.dataset = dataset+ self.iters = iters+ if self.iters < 0:+ logger.warning("`iters` must be non-negative, reset it to 200")+ self.iters = 200+ if self.iters == 0:+ self.lr = 5e-3+ else:+ self.lr = lr or (1.0 / self.iters) # must place after iter setting+ self.minmax_lr = minmax_lr or self.lr+ self.enable_alg_ext = enable_alg_ext+ self.sampler = sampler+ self.not_use_best_mse = not_use_best_mse+ self.dynamic_max_gap = dynamic_max_gap+ self.lr_scheduler = lr_scheduler+ self.optimizer = self._get_optimizer(None)+ self.disable_opt_rtn = disable_opt_rtn+ self.is_packing_immediate = False # whether to pack the layer immediately after tuning++ # KV cache, this one does not affect tuning but will collect some infos during tuning+ self.static_kv_dtype = static_kv_dtype+ if self.static_kv_dtype is not None:+ logger.warning("The static kv is experimental and currently has limited support.")++ # Model related+ self.quantized = False+ if isinstance(model, str):+ model, tokenizer, low_cpu_mem_usage = llm_load_model(+ model, device=device, low_cpu_mem_mode=low_cpu_mem_usage+ )+ elif tokenizer is None and iters > 0:+ raise ValueError("A tokenizer must be set for non-str model input")+ self.low_cpu_mem_usage = bool(low_cpu_mem_usage)+ if unsupport_meta_device(model):+ raise RuntimeError(+ "AutoRound does not support parameters on meta device. "+ "Please use more GPUs by setting `--device_map 0,1,2,3` or just place the model on CPU."+ )+ self.model = model.eval()+ self.tokenizer = tokenizer+ self.shared_cache_keys = get_shared_keys(self.model)+ if not hasattr(self, "quant_block_list"):+ all_blocks = get_block_names(model)+ self.quant_block_list = find_matching_blocks(model, all_blocks, self.to_quant_block_names)++ self.scale_dtype = convert_dtype_str2torch(scale_dtype)+ self._set_amp_dtype()+ self.cache_device = torch.device("cpu") if self.low_gpu_mem_usage else self.device+ if self.act_bits <= 8 and self.amp_dtype == torch.float16:+ logger.warning("force to use bf16 to for quantization tuning when enabling activation quantization")+ self.amp_dtype = torch.bfloat16+ if self.model.dtype != torch.bfloat16: # keep the model's buffer dtype unchanged+ self.model = self.model.to(torch.bfloat16)+ else:+ logger.info(f"using {self.model.dtype} for quantization tuning")++ # Some helpers+ self.supported_types = SUPPORTED_LAYER_TYPES+ self.inner_supported_types = INNER_SUPPORTED_LAYER_TYPES+ if "hpu" in str(self.device):+ self.inner_supported_types = tuple(x for x in INNER_SUPPORTED_LAYER_TYPES if x != "FP8Linear")+ self.batch_dim = None+ self.infer_bs_coeff = 1+ self.enable_torch_compile = enable_torch_compile+ self._adjust_torch_compile(enable_torch_compile)+ self._check_configs()+ torch.set_printoptions(precision=3, sci_mode=True)++ if is_optimum_habana_available():+ logger.info("optimum Habana is available, import htcore explicitly.")+ import habana_frameworks.torch.core as htcore # pylint: disable=E0401+ import habana_frameworks.torch.hpu as hthpu # pylint: disable=E0401]++ def _set_device(self, device_map):+ if hasattr(self, "device") and self.device is not None:+ return+ if isinstance(device_map, (str, torch.device, int)):+ self.device = detect_device(device_map)++ elif isinstance(device_map, dict) and device_map:+ tmp_devices = []+ for val in device_map.values():+ if isinstance(val, (str, torch.device, int)): # could optimize+ tmp_device = detect_device(val)+ tmp_device = tmp_device.split(":")[0]+ tmp_devices.append(tmp_device)+ tmp_devices = list(set(tmp_devices))+ if len(tmp_devices) > 1:+ logger.warning(+ f"there are multiple device types in the device_map, "+ f"please make sure they are correct,use the first device {tmp_devices[0]} as the core device "+ )++ self.device = tmp_devices[0]+ else:+ raise TypeError(f"device_map should be [str, torch.device, int, dict], but got {type(device_map)}")++ def _parse_layer_config(self, layer_config: dict[str, Union[str, dict, QuantizationScheme]]) -> None:+ """Parse and set the layer-wise quantization configuration."""+ # Some other quantization configs+ self.layer_config = {} if layer_config is None else layer_config+ scheme_keys = [f.name for f in fields(QuantizationScheme)]+ for key, item in self.layer_config.items():+ if isinstance(item, str):+ item = asdict(preset_name_to_scheme(item.upper()))+ self.layer_config[key] = item++ if isinstance(item, QuantizationScheme):+ config = asdict(item)+ tmp_keys = copy.deepcopy(list(config.keys()))+ for tmp_key in tmp_keys: ## Pop None value to be overridden+ if config[tmp_key] is None:+ config.pop(tmp_key)+ self.layer_config[key] = config+ elif isinstance(item, dict):+ item_keys = item.keys()+ if item_keys not in scheme_keys:+ for item_key in item_keys:+ if item_key not in scheme_keys:+ raise ValueError(+ f"the key {item_key} in layer_config for layer {key} is invalid,"+ f" only {scheme_keys} are supported"+ )++ if not self.quant_lm_head or (isinstance(self.scheme, str) and self.scheme.lower().startswith("gguf")):+ return+ for n, _ in self.model.named_modules():+ lm_head_layer_name = n++ if (+ hasattr(self.model, "config")+ and self.model.config.tie_word_embeddings+ and hasattr(self.model, "_tied_weights_keys")+ ):+ tied_keys = self.model._tied_weights_keys+ for item in tied_keys:+ if lm_head_layer_name in item: # TODO extend to encoder-decoder layer, seq classification model+ self.quant_lm_head = False+ logger.warning(+ "reset `quant_lm_head` to `False` as quantizing lm_head with tied weights has not been "+ "supported currently"+ )+ break++ lm_head_layer_config = self.layer_config[lm_head_layer_name] if lm_head_layer_name in self.layer_config else {}++ for key in scheme_keys:+ if key not in lm_head_layer_config:+ lm_head_layer_config[key] = getattr(self, key)++ def _parse_and_set_scheme(self, scheme: Union[str, dict, QuantizationScheme], kwargs) -> None:+ """Parse and set the quantization scheme."""+ if isinstance(scheme, QuantizationScheme):+ scheme = asdict(scheme)+ elif isinstance(scheme, dict):+ scheme = scheme+ elif isinstance(scheme, str):+ scheme = scheme.upper()+ self.scheme = scheme+ scheme = asdict(preset_name_to_scheme(scheme))+ scheme_keys = [f.name for f in fields(QuantizationScheme)]+ for key in scheme_keys:+ if key in kwargs and kwargs[key] is not None:+ setattr(self, key, kwargs[key])+ else:+ setattr(self, key, scheme.get(key, None))+ kwargs.pop(key, None)+ if self.act_dynamic is None:+ self.act_dynamic = True++ tmp_bits = infer_bits_by_data_type(self.data_type)+ if tmp_bits is not None and tmp_bits < 16 and tmp_bits != self.bits:+ logger.warning(f"'data_type' do not match the specified 'bits' setting. Resetting 'bits' to {tmp_bits}.")+ self.bits = tmp_bits+ if tmp_bits is not None and tmp_bits < 16:+ for supported_dtype in SUPPORTED_DTYPES: # to easily handle dtype mx_fp4 and layer_config={xxx:{bits:8}}+ if self.data_type.startswith(supported_dtype):+ if supported_dtype + str(tmp_bits) == self.data_type: # could not replace FP8_e4m3+ self.data_type = supported_dtype+ break++ self.act_group_size = self.act_group_size if self.act_group_size is not None else self.group_size+ self.act_bits = self.act_bits if self.act_bits is not None else 16+ self.act_sym = self.act_sym if self.act_sym is not None else self.sym++ if self.act_data_type is None:+ if self.data_type in SUPPORTED_DTYPES and self.act_bits < 16:+ self.act_data_type = self.data_type+ logger.info(f"activation adopts {self.data_type}")+ else:+ self.act_data_type = "float"+ tmp_act_bits = infer_bits_by_data_type(self.act_data_type)+ if tmp_act_bits is not None and tmp_act_bits < 16 and tmp_act_bits != self.act_bits:+ self.act_bits = tmp_act_bits+ logger.warning(+ f"`act_data_type` do not"+ f" match the specified 'act_bits' setting. Resetting 'act_bits' to {tmp_act_bits}."+ )+ if tmp_act_bits is not None and tmp_act_bits < 16:+ for supported_dtype in SUPPORTED_DTYPES: # to easily handle dtype mx_fp4 and layer_config={xxx:{bits:8}}+ if self.act_data_type.startswith(supported_dtype):+ if supported_dtype + str(tmp_act_bits) == self.act_data_type: # could not replace FP8_e4m3+ self.act_data_type = supported_dtype+ break++ def _adjust_torch_compile(self, enable_torch_compile: bool) -> None:+ """Sets the torch compile configuration for the tuning."""+ self.enable_torch_compile = enable_torch_compile+ if (+ not self.enable_torch_compile+ and TORCH_VERSION_AT_LEAST_2_6+ and self.act_bits > 8+ and not is_debug_mode()+ and not self.low_cpu_mem_usage+ and "fp8" not in self.data_type+ and "fp8" not in self.act_data_type+ ):+ logger.info(+ "'enable_torch_compile' is set to `False` by default. "+ "Enabling it can reduce tuning cost by 20%, but it might throw an exception."+ )++ if self.low_cpu_mem_usage and self.enable_torch_compile:+ self.enable_torch_compile = False+ logger.warning("reset enable_torch_compile to `False` as low_cpu_mem_usage is enabled")++ if is_debug_mode() and self.enable_torch_compile:+ self.enable_torch_compile = False+ logger.warning("reset enable_torch_compile to `False` as debug mode is enabled")++ if ("fp8" in self.data_type or "fp8" in self.act_data_type) and self.enable_torch_compile:+ self.enable_torch_compile = False+ logger.warning("reset enable_torch_compile to `False` as fp8 is enabled")++ def _set_device_map_in_blocks(self, device_map: Union[str, dict, None]) -> None:+ """Sets the device map for specific blocks in the model.++ Args:+ device_map (Union[str, dict, None]): A mapping of module names to devices.+ If provided as a string, it should be in the format+ "module_name:device,module_name:device". Devices can be integers+ (GPU IDs) or strings (e.g., 'cpu', 'cuda:0').+ """+ if self.device_map is None or len(self.device_map) == 0:+ self.device_map = None+ if not device_map:+ return+ if self.device_map == "auto" and device_map == "auto":+ return+ if isinstance(device_map, str):+ device_map = device_map.replace(" ", "")+ infos = device_map.split(",")+ device_map_dict = {}+ for info in infos:+ index = info.find(":")+ key = info[:index]+ value = info[index + 1 :]+ device_map_dict[key] = value+ device_map = device_map_dict++ names = [n for n, m in self.model.named_modules() if len(list(m.children())) == 0]++ for key, device in device_map.items():+ if isinstance(device, str) and device.isdigit():+ device = int(device)+ device = detect_device(device)+ try:+ module = get_module(self.model, key)+ module.tuning_device = device+ except:+ matching_names = [name for name in names if re.match(key, name)]+ if len(matching_names) > 0:+ for name in matching_names:+ self._set_device_for_matching_module(name, device)+ else:+ for name in names:+ if key in name:+ self._set_device_for_matching_module(name, device)++ def _set_device_for_matching_module(self, name: str, device: str) -> None:+ """Sets the device for a module if it matches the given name."""+ module = get_module(self.model, name)+ if hasattr(module, "tuning_device") and module.tuning_device != device:+ logger.warning(+ f"multiple devices have been set for layer {name}, keeping original device {module.tuning_device}"+ )+ else:+ module.tuning_device = device++ def _set_auto_device_map_in_block(self, block: torch.nn.Module, input_ids: list[torch.Tensor]) -> None:+ """Automatically sets the device map for the block based on available GPUs and memory constraints."""+ if torch.cuda.is_available():+ num_gpus = torch.cuda.device_count()+ elif torch.xpu.is_available():+ logger.warning_once("XPU does not support auto device map yet, using device 0 for tuning.")+ return+ else:+ raise RuntimeError("No CUDA or XPU devices found.")+ if num_gpus <= 1:+ self.device_map = None+ return++ if hasattr(self, "device_list") and self.device_list:+ cuda_devices = [f"cuda:{i}" for i in self.device_list]+ device_0 = cuda_devices[0]+ else:+ cuda_devices = [f"cuda:{i}" for i in range(num_gpus)]+ device_0 = "cuda:0"++ device_0_memory = get_device_memory(+ self.device_list[0] if hasattr(self, "device_list") and self.device_list else 0+ )+ block_memory, input_ouput_memory = estimate_tuning_block_mem(block, input_ids)+ if self.low_gpu_mem_usage:+ input_ouput_memory = 0++ mem_per_param_scale = 13 if self.mem_per_param_scale is None else self.mem_per_param_scale+ if self.iters == 0:+ mem_per_param_scale = 1 # for rtn++ if (block_memory * mem_per_param_scale + input_ouput_memory) < device_0_memory:+ return # fit in one GPU++ device_map = {}+ device_memory = {device: get_device_memory(int(device.split(":")[1])) for device in cuda_devices}+ device_memory[device_0] = device_0_memory - input_ouput_memory++ device_idx = 0+ # First, fill device 0 to its maximum capacity, then distribute the remaining layers evenly across other devices+ for n, m in block.named_modules():+ if check_to_quantized(m):+ layer_name = block.tmp_name + "." + n+ layer_memory = m.weight.nbytes / 1024**3+ if device_idx == 0 and layer_memory * mem_per_param_scale < device_memory[cuda_devices[device_idx]]:+ device_map[layer_name] = cuda_devices[device_idx]+ device_memory[cuda_devices[device_idx]] -= layer_memory * mem_per_param_scale+ elif device_idx == 0:+ device_idx += 1 # Move to the next device once device 0 is full+ device_map[layer_name] = cuda_devices[device_idx]+ device_memory[cuda_devices[device_idx]] -= layer_memory * mem_per_param_scale+ else:+ # Calculate the target device index based on even distribution+ sorted_devices = sorted(cuda_devices, key=lambda d: device_memory[d], reverse=True)+ device_idx = sorted_devices[0]+ if layer_memory * mem_per_param_scale < device_memory[device_idx]:+ device_map[layer_name] = device_idx+ device_memory[device_idx] -= layer_memory * mem_per_param_scale+ else:+ logger.warning_once(+ f"Block {block.tmp_name} not fit in available GPU memory. "+ "Consider using more GPUs or reducing mem_per_param_scale if OOM occurs."+ )+ self._set_device_map_in_blocks(device_map)++ def _dq_check(self) -> None:+ """Reset the default value of super_bits and super_group_size"""+ if self.data_type.endswith("_dq"):+ gguf_config = GGUF_INNER_CONFIG[f"gguf:q{self.bits}_k"]+ self.super_bits = gguf_config["super_bits"] if self.super_bits is None else self.super_bits+ self.super_group_size = (+ gguf_config["super_group_size"] if self.super_group_size is None else self.super_group_size+ )++ def _check_configs(self) -> None:+ """Checks if the configurations are valid.++ Raises:+ ValueError, TypeError: If any of the configurations are invalid.+ """+ if not isinstance(self.model, torch.nn.Module):+ raise TypeError("model must be an instance of torch.nn.Module")+ if self.bits <= 0:+ raise ValueError("`bits` must be positive")+ if self.act_bits <= 0:+ raise ValueError("`act_bits` must be positive")+ if not (self.group_size == -1 or self.group_size >= 0):+ raise ValueError("`group_size` must be -1 (per channel) or 0 (per-tensor) or a positive integer")+ if not (self.act_group_size == -1 or self.act_group_size >= 0):+ raise ValueError("`act_group_size` must be -1 (per channel) or 0 (per-tensor) or a positive integer")+ if self.batch_size <= 0:+ raise ValueError("`batch_size` must be positive")+ if self.iters < 0:+ raise ValueError("`iters` must be non-negative")+ if self.seqlen <= 0:+ raise ValueError("`seqlen` must be positive")+ if self.nblocks <= 0:+ raise ValueError("`nblocks` must be positive")+ if self.gradient_accumulate_steps <= 0:+ raise ValueError("`gradient_accumulate_steps` must be positive")++ if (+ self.act_bits <= 8+ and (not is_nv_fp(self.act_data_type) or "static_gs" not in self.act_data_type)+ and not is_mx_fp(self.act_data_type)+ ):+ logger.warning(+ "activation quantization is an experimental feature with limited support and a complex API. "+ "And please save the quantized model to fake format as real deployment is not supported currently"+ )++ if is_mx_fp(self.data_type) and self.group_size != 32:+ logger.warning("dtype mx_fp should only support group_size of 32 in real deployment")++ if is_nv_fp(self.data_type) and (self.group_size != 16):+ logger.warning("dtype nv_fp should only support group_size of 16 in real deployment")++ if self.nsamples < self.gradient_accumulate_steps * self.batch_size:+ if self.batch_size > self.nsamples:+ if self.iters > 0: # GGUF should log this warning, but we don't know the format here+ logger.warning(+ f"reset `batch_size` to {self.nsamples} as `nsamples`({self.nsamples})"+ f" is smaller than batch_size({self.batch_size})"+ )+ self.batch_size = self.nsamples+ if self.gradient_accumulate_steps > self.nsamples // self.batch_size:+ self.gradient_accumulate_steps = self.nsamples // self.batch_size+ logger.warning(+ f"reset `gradient_accumulate_steps` to {self.gradient_accumulate_steps}"+ f" as nsamples must equal or greater"+ f" than gradient_accumulate_steps * batch_size"+ )++ if self.enable_norm_bias_tuning:+ logger.warning("the `enable_norm_bias_tuning` feature is experimental and currently has limited support.")++ self._dq_check()++ def _check_compatibility(self) -> None:+ """Checks compatibility of the configurations and model."""+ # Check gguf and others+ has_gguf = False+ if hasattr(self, "formats"):+ has_besides_gguf = False+ for format_ in self.formats:+ if "gguf" in format_:+ has_gguf = True+ elif format_ != "fake":+ has_besides_gguf = True+ if has_gguf and has_besides_gguf:+ raise ValueError("Gguf format is not compatible with other formats, please choose only one of them")+ if has_gguf and self.iters != 0 and self.bits != 3:+ logger.warning(+ "`iters=0` is recommended when exporting to GGUF format except for bits 3,"+ " as we have optimized the RTN method for this case."+ " We are likely to release new algorithm for certain configurations in the future."+ )++ # Check group_size 32 for auto_round+ if (+ self.data_type == "int"+ and hasattr(self, "formats")+ and any(key in fmt for fmt in self.formats for key in ("auto_round", "auto_gptq", "auto_awq"))+ ):+ for n, m in self.model.named_modules():+ if isinstance(m, self.supported_types):+ if m.weight.shape[0] % 32 != 0 or m.weight.shape[1] % 32 != 0:+ self.layer_config[n] = {"bits": 16}+ logger.info(+ f"{n} will not be quantized due to its shape not being divisible by 32,"+ " resulting in an exporting issue to autogptq"+ )++ if (+ self.seqlen is not None+ and hasattr(self.model, "config")+ and hasattr(self.model.config, "max_position_embeddings")+ ):+ if self.model.config.max_position_embeddings < self.seqlen:+ logger.warning(+ f"Change sequence length to {self.model.config.max_position_embeddings} "+ "due to the limitation of max_position_embeddings"+ )+ self.seqlen = min(self.seqlen, self.model.config.max_position_embeddings)++ if self.seqlen is not None and hasattr(self.tokenizer, "model_max_length"):+ if self.tokenizer.model_max_length < self.seqlen:+ logger.warning(+ f"Change sequence length to {self.tokenizer.model_max_length} "+ "due to the limitation of model_max_length. "+ "You can also try to increase the model_max_length to avoid this issue."+ )+ self.seqlen = min(self.seqlen, self.tokenizer.model_max_length)++ if self.group_size == 0 and "fp8" not in self.data_type:+ logger.warning("`group_size==0` is not supported for data_type other than fp8 ")++ def _parse_format_to_list(self, format: str) -> list:+ """Parses the format string into a list of formats.++ This method checks the requested format(s) against the model's+ quantization settings and adjusts them if necessary. It ensures that+ the formats are compatible with the model's data type, bit width,+ and activation quantization settings.++ Args:+ format (str): The requested format(s) for quantization, separated by commas.++ Returns:+ list: A list of validated and updated formats.+ """++ # Remove duplicates from formats list+ def remove_duplicates(lst):+ seen = set()+ return [x for x in lst if not (x in seen or seen.add(x))]++ formats = format.replace("q*_", f"q{self.bits}_").replace(" ", "").split(",")+ formats = remove_duplicates(formats) # need the keep origin order++ if isinstance(self.scheme, str) and self.scheme.lower().startswith("gguf"):+ for i in range(len(formats)):+ if formats[i] != "fake" and formats[i] != self.scheme.lower().startswith("gguf"):+ logger.warning(+ f"reset format {formats[i]} to {self.scheme.lower()} "+ f"since scheme {self.scheme} can only be exported to format {self.scheme.lower()}"+ )+ formats[i] = self.scheme.lower()++ _gguf_args_check(self, formats, model_type=ModelType.TEXT)+ if self.mllm:+ _gguf_args_check(self, formats, model_type=ModelType.MMPROJ)++ for format_ in formats:+ if format_ not in SUPPORTED_FORMATS:+ logger.error(f"Unsupported format {format_}, please choose from {SUPPORTED_FORMATS}")+ exit(-1)+ if self.scale_dtype != torch.float32:+ only_gguf = True+ for format_ in formats:+ if not ("gguf" in format_ or "fake" in format_):+ only_gguf = False+ break+ if len(formats) == 1 and "fake" == formats[0]:+ only_gguf = False+ if only_gguf:+ self.scale_dtype = torch.float32+ logger.info("change `scale_dtype` to `torch.float32`")++ # Adjust format settings based on compatibility+ for index in range(len(formats)):+ format = formats[index]+ if format == "auto_round":+ if self.sym and "int" in self.data_type:+ format = "auto_round:auto_gptq"+ elif self.bits == 4 and not self.sym and "int" in self.data_type:+ enable_awq = all(+ config["bits"] == self.bits or config["bits"] >= 16 for config in self.layer_config.values()+ )+ if enable_awq:+ format = "auto_round:auto_awq"+ elif is_nv_fp(self.data_type) or is_mx_fp(self.data_type):+ format = f"auto_round:{self.data_type}"+ elif is_static_wfp8afp8(self): # staic wfp8afp8+ format = f"auto_round:{AutoRoundFormat.TORCH_FP8_STATIC.value}"+ elif self.data_type == "fp" and self.bits == 8 and self.act_bits >= 16: # woq fp8+ format = "auto_round:fp8"+ elif self.act_bits < 16:+ raise ValueError(+ "AutoRound format does not support exporting "+ "for the current quantization configuration, "+ "please change to `fake` format for research purpose"+ )+ formats[index] = format+ elif format == "llm_compressor":+ from auto_round.export.export_to_llmcompressor import check_compressed_tensors_supported++ if is_nv_fp(self.data_type) or is_mx_fp(self.data_type):+ check_compressed_tensors_supported()+ format = format.replace("llm_compressor", f"llm_compressor:{self.data_type}")+ formats[index] = format+ elif not is_wfp8afp8(self):+ logger.error(+ "Currently, the llm_compressor format only supports MXFP/NVFP/FP8. "+ "Please change format to fake or auto_round etc."+ )+ elif "auto_awq" in format:+ from auto_round.utils import check_awq_gemm_compatibility++ awq_supported, info = check_awq_gemm_compatibility(+ self.model, self.bits, self.group_size, self.sym, self.layer_config+ )+ if not awq_supported:+ logger.warning(f"The AutoAWQ format may not be supported due to {info}")+ else:+ if (is_nv_fp(self.data_type) or is_mx_fp(self.data_type)) and format != "fake":+ logger.warning(f"nv_fp and mx_fp dtypes are not supported for export format: {format}")++ formats = remove_duplicates(formats)+ for i in range(len(formats)):+ formats[i] = self._check_supported_format(formats[i])+ formats = remove_duplicates(formats)+ return formats++ def _check_supported_format(self, format: str) -> bool:+ """Checks if the specified format is supported.++ This method validates the requested format against the model's bit width,+ group size, symmetry, and activation quantization settings. It raises an+ error if the format is incompatible with the current model configuration.++ Args:+ format (str): The requested format for quantization.++ Returns:+ bool: True if the format is supported, False otherwise.+ """+ if format == "fake":+ return format+ format = format.replace("q*_", f"q{self.bits}_")++ # format check for fp8+ w_fp8 = self.data_type == "fp" and self.bits == 8+ act_fp8 = self.act_data_type == "fp" and self.act_bits == 8+ if (w_fp8 or act_fp8) and re.search("^auto_round|^llm_compressor", format) is None:+ error_msg = (+ f"is only supported to export auto_round or llm_compressor format," f" but got {format}, please check."+ )+ error_msg = ("act_data_type<fp8> " + error_msg) if act_fp8 else error_msg+ error_msg = ("data_type<fp8> " + error_msg) if w_fp8 else error_msg+ logger.error(error_msg)+ sys.exit(-1)++ # Only support to export afp8/nv_fp/mx_fp+ if self.act_bits <= 8:+ if not is_standard_fp(self.act_data_type) or self.act_dynamic:+ if "llm_compressor" in format:+ if (is_nv_fp(self.act_data_type) and "static_gs" in self.act_data_type) or (+ is_mx_fp(self.act_data_type)+ ):+ return format+ bits, group_size, sym, act_bits = 8, -1, True, 8+ assert (+ self.bits == bits+ and self.group_size == group_size+ and self.sym == sym+ and self.act_bits == act_bits+ and self.act_dynamic+ ), (+ f"Currently only support to export llm_compressor format for sym dynamic quantized"+ f" W{self.bits}A{self.act_bits} model with group_size={group_size},"+ f" but got bits={self.bits}, group_size={self.group_size}, sym={self.sym},"+ f" act_bits={self.act_bits}"+ )+ elif "auto_round" in format and (+ is_mx_fp(self.act_data_type) or (is_nv_fp(format) and "static_gs" in self.act_data_type)+ ):+ logger.warning(+ f"AutoRound supports exporting to format '{format}', "+ "but loading quantized models in this format is not yet supported. "+ "It is currently recommended to export to the 'llm_compressor' format."+ )+ elif format != "fake":+ logger.warning(+ "Currently only support to export auto_round format quantized model"+ " with fp8, mx_fp and nv_fp4 dtype activation for activation quantization."+ " Change format to fake and save."+ )+ format = "fake"+ else:+ if not (format == "auto_round" or format == f"auto_round:{AutoRoundFormat.TORCH_FP8_STATIC.value}"):+ logger.warning(+ f"Currently only support to export auto_round or fake format for static W{self.bits}AFP8 model,"+ f" change format {format} to auto_round"+ )+ format = "auto_round"+ if self.act_group_size != 0 and not self.act_dynamic and format == "auto_round:fp8":+ logger.warning(+ f"Please note that quantize activation with act_group_size={self.act_group_size}"+ " may result in failure to export or import normally."+ )+ if re.search(r"q\d_k", format) and not self.data_type.endswith("_dq"):+ logger.error(+ f"datatype<{self.data_type}> not support to export {format} format."+ " Please change export format or `data_type`."+ )+ sys.exit(-1)++ return format++ def quantize_and_save(+ self, output_dir: str = "tmp_autoround", format: str = "auto_round", inplace: bool = True, **kwargs+ ) -> tuple[torch.nn.Module, dict[str, Any]]:+ """Quantizes the model and saves it in the specified format(s).++ This function checks the validity of the requested format(s), quantizes+ the model accordingly, and saves it to the specified output directory.+ If multiple formats are provided, the model is saved separately for each format.++ Args:+ output_dir (str, optional): The directory where the quantized model+ will be saved. Defaults to "tmp_autoround".+ format (str, optional): The quantization format(s) to use, separated+ by commas if multiple. Defaults to "auto_round".+ inplace (bool, optional): Whether to modify the model in place if only+ one format is used. Defaults to True.+ **kwargs: Additional arguments for the quantization and saving process.++ Returns:+ model: A qdq model or packed model based on the configurations+ folders: The folder paths where the quantized models are saved.++ Raises:+ ValueError: If an unsupported format is specified.+ """+ # Validate and process the specified formats+ self.orig_output_dir = output_dir++ # check and update the format based on the current configuration+ format_list = self._parse_format_to_list(format)+ self.formats = format_list++ # If multiple formats are specified, enforce inplace=False+ if len(format_list) > 1:+ inplace = False+ self.inplace = kwargs.get("inplace", inplace)+ kwargs.pop("inplace", None)++ # Perform model quantization+ if self.static_kv_dtype is not None:+ from auto_round.experimental.kv_cache import kvcache_quant_context++ with kvcache_quant_context(self.model, static_kv_dtype=self.static_kv_dtype):+ model, _ = self.quantize()+ else:+ model, _ = self.quantize()+ # Save the quantized model in the specified format_list+ folders = []+ for format in format_list:+ if "gptq" in format and not self.sym:+ logger.warning(+ "The asymmetrical kernel of the GPTQ format may result in a noticeable accuracy drop,"+ " particularly for 2-bit quantization and smaller models."+ " We recommend exporting to either the AutoAWQ format ( only 4 bits) or "+ "the AutoRound format(2/3/4/8 bits)."+ )+ save_folder = self._get_save_folder_name(format)+ self.save_quantized(save_folder, format=format, inplace=inplace, **kwargs)++ folders.append(save_folder)++ return model, folders++ def _get_save_folder_name(self, format_str: str) -> str:+ """Generates the save folder name based on the provided format string.++ If there are multiple formats to handle, the function creates a subfolder+ named after the format string with special characters replaced. If there's+ only one format, it returns the original output directory directly.++ Args:+ format_str (str): The format identifier (e.g., 'gguf:q2_k_s').++ Returns:+ str: The path to the folder where results should be saved.+ """+ # Replace special characters to make the folder name filesystem-safe+ sanitized_format = format_str.replace(":", "-").replace("_", "-")++ # Use a subfolder only if there are multiple formats+ if len(self.formats) > 1:+ return os.path.join(self.orig_output_dir, sanitized_format)++ return self.orig_output_dir++ @torch.inference_mode()+ def _quantize_embedding_layer(self):+ """Quantizes embedding layers in the model according to the configuration.++ This method iterates through all modules in the model, identifies embedding+ layers specified in `self.layer_config`, and applies the appropriate quantization+ function based on bit precision, grouping strategy, and dtype.++ Returns:+ bool: True if the quantization process completes without critical errors.+ """+ is_quantized = False+ for name, module in self.model.named_modules():+ # Skip non-Embedding modules or layers not in config+ if not isinstance(module, torch.nn.Embedding) or name not in self.layer_config:+ continue++ config = self.layer_config[name]++ # Skip layers that are not marked for quantization+ if not check_to_quantized(config):+ continue+ is_quantized = True+ config["scale_dtype"] = self.scale_dtype+ dtype = config["data_type"]++ # Determine quantization function key with symmetry/asymmetry+ if dtype not in QUANT_FUNC_WITH_DTYPE:+ dtype = f"{dtype}_{'sym' if config['sym'] else 'asym'}"++ # Optionally use optimized rounding (RTN) variant+ if not self.disable_opt_rtn and f"rtn_{dtype}" in QUANT_FUNC_WITH_DTYPE:+ dtype = f"rtn_{dtype}"++ quant_func = QUANT_FUNC_WITH_DTYPE[dtype]++ # Attempt quantization on GPU, fall back to CPU if OOM+ try:+ weight, scale, zp = quant_func(+ module.weight.to(self.device),+ **{k: config[k] for k in ["bits", "group_size", "super_bits", "super_group_size", "scale_dtype"]},+ )+ except RuntimeError as e:+ cuda_error_msg = traceback.format_exc()+ try:+ logger.error(cuda_error_msg)+ logger.warning("falling back to CPU")+ weight, scale, zp = quant_func(+ module.weight.to("cpu"),+ **{+ k: config[k]+ for k in ["bits", "group_size", "super_bits", "super_group_size", "scale_dtype"]+ },+ )+ except Exception as e:+ raise++ # Overwrite the module's weights with the quantized version+ module.weight.data.copy_(weight.cpu())++ # Attach scale and zero point (zp) to the module+ for param_name, value in zip(["scale", "zp"], [scale, zp]):+ if isinstance(value, dict):+ for k, v in value.items():+ setattr(module, k if k == "scale" else f"w_{k}", v.cpu())+ elif isinstance(value, torch.Tensor):+ setattr(module, param_name, value.cpu())+ else:+ setattr(module, param_name, value)++ # Update config+ self.layer_config.setdefault(name, {}).update(config)++ # Release memory+ clear_memory()++ return is_quantized++ def _quant_rtn_with_imatrix(self, all_to_quantized_module_names: list[str]) -> None:+ """Performs RTN quantization using input activation statistics (imatrix).++ This method accumulates per-channel second-moment activation statistics (imatrix)+ via forward hooks and uses them to perform RTN quantization. If CUDA memory runs out,+ it falls back to CPU-based blockwise quantization.++ Args:+ all_to_quantized_module_names (list[str]):+ A list of module names (e.g., 'model.layers.0.self_attn.q_proj') to be quantized.++ Returns:+ None+ """+ logger.info("start to compute imatrix for GGUF quantization")++ # Load dataset+ from auto_round.calib_dataset import get_dataloader++ if _is_fp8_model(self.model):+ convert_fp8_model_to_16b_model(self.model, self.amp_dtype)++ if isinstance(self.dataset, str):+ if self.tokenizer is None:+ raise ValueError("A tokenizer must be set for the model when using a dataset string.")+ dataset_name = self.dataset.replace(" ", "")+ self.dataloader = get_dataloader(+ self.tokenizer, self.seqlen, dataset_name, self.seed, self.batch_size, self.nsamples+ )+ else:+ self.dataloader = self.dataset++ model = self.model++ # Dispatch multi-GPU model if necessary+ if hasattr(model, "hf_device_map") and len(model.hf_device_map) > 1:+ dispatch_model(model, model.hf_device_map)++ def register_act_hook(model):+ """Registers hooks to accumulate activation squared norms into `imatrix`."""++ def get_imatrix_hook(module, input, output):+ input = input[0] if isinstance(input, (tuple, list)) else input+ flattened = input.reshape(-1, input.shape[-1]).to(torch.float32)+ squared = torch.sum(flattened**2, dim=0).to(torch.float32)++ if not hasattr(module, "imatrix"):+ module.imatrix = squared+ module.imatrix_cnt = input.shape[0]+ else:+ module.imatrix += squared.to(module.imatrix.device)+ module.imatrix_cnt += input.shape[0]++ hook_handles = []+ for name, module in model.named_modules():+ if isinstance(module, self.supported_types) and check_to_quantized(module):+ hook = module.register_forward_hook(get_imatrix_hook)+ hook_handles.append(hook)+ return hook_handles++ hooks = register_act_hook(model)++ try:+ # Move model to target device+ if hasattr(self.model, "hf_device_map") and len(self.model.hf_device_map) > 1:+ dispatch_model(self.model, self.model.hf_device_map)+ else:+ model = model.to(self.device)+ cnt = 0++ # Run forward pass to accumulate imatrix+ for data in self.dataloader:+ cnt += data["input_ids"].shape[0]+ data = to_device(data, self.device)+ model(**data)+ if cnt >= self.nsamples:+ break++ # Remove hooks after data collection+ for hook in hooks:+ hook.remove()++ # Normalize imatrix by count+ for _, module in model.named_modules():+ if hasattr(module, "imatrix"):+ module.imatrix /= module.imatrix_cnt+ if hasattr(model, "hf_device_map") and len(model.hf_device_map) > 1:+ import accelerate++ accelerate.hooks.remove_hook_from_submodules(model)+ # Perform quantization using RTN+ pbar = tqdm(all_to_quantized_module_names)+ block_names_cnt = len(flatten_list(get_block_names(self.model, True)))+ clear_mem_freq = len(all_to_quantized_module_names) // block_names_cnt+ if clear_mem_freq == 0:+ clear_mem_freq = 1+ cnt = 1+ for name in pbar:+ pbar.set_description(f"Quantizing {name}")+ self._quantize_layer_via_rtn(name)+ if cnt % clear_mem_freq == 0:+ clear_memory()+ cnt = 1+ cnt += 1+ except RuntimeError as e:+ try:+ if hasattr(model, "hf_device_map") and len(model.hf_device_map) > 1:+ import accelerate++ accelerate.hooks.remove_hook_from_submodules(model)+ # Fallback: out-of-memory → try CPU blockwise quantization+ logger.warning("Out of VRAM, falling back to blockwise quantization. Accuracy may degrade.")+ model = model.to("cpu")+ clear_memory()+ self._quantize_via_rtn_blockwise(all_to_quantized_module_names)+ except RuntimeError as e:+ cuda_error_msg = traceback.format_exc()+ try:+ logger.error(cuda_error_msg)+ # Final fallback: warn and use CPU-only quantization+ logger.warning(+ "Fallback to CPU. "+ "Consider enabling `low_gpu_mem_usage` or using more GPUs via `--device 0,1,2,3`."+ )+ model = model.to("cpu")+ clear_memory()+ if hasattr(model, "hf_device_map") and len(model.hf_device_map) > 1:+ import accelerate++ accelerate.hooks.remove_hook_from_submodules(model)++ orig_device = self.device+ self.device = "cpu"+ self._quantize_via_rtn_blockwise(all_to_quantized_module_names)+ self.device = orig_device+ except Exception as e:+ raise+ finally:+ # Always remove hooks+ for hook in hooks:+ hook.remove()++ def _check_need_to_quantize_lm_head_embedding(self) -> bool:+ """Checks if LM head and embedding layers need quantization for GGUF format.++ This function inspects the current model's formats and determines whether+ it needs to apply quantization settings to the embedding and LM head layers.+ The function modifies `self.layer_config` in-place and updates the model modules.++ Returns:+ bool: True if the LM head needs quantization, otherwise False.++ Raises:+ NotImplementedError: If multiple non-fake GGUF formats are specified.+ """+ gguf_scheme = False+ if isinstance(self.scheme, str) and "gguf" in self.scheme.lower():+ gguf_scheme = True++ if not hasattr(self, "formats") and not gguf_scheme:+ return False++ has_gguf: bool = gguf_scheme or any("gguf" in fmt for fmt in self.formats)+ if not has_gguf:+ return False+ if hasattr(self, "formats"):+ formats: list[str] = [fmt for fmt in self.formats if "fake" not in fmt]+ if not (len(formats) == 1 and "gguf" in formats[0]):+ raise NotImplementedError("Only one GGUF format can be set at a time.")+ target_format: str = formats[0]++ else:+ target_format = self.scheme.lower()++ tie_word_embeddings: bool = getattr(getattr(self.model, "config", None), "tie_word_embeddings", True)+ for name, module in self.model.named_modules():+ if isinstance(module, torch.nn.Embedding):+ key: str = "lm_head" if tie_word_embeddings else "embedding"+ config: dict[str, Any] = GGUF_INNER_CONFIG[GGUF_CONFIG[target_format][key]]+ self._apply_config_to_layer(name, config, True)++ if not tie_word_embeddings:+ lm_head_name: str = get_lm_head_name(self.model)+ config: dict[str, Any] = GGUF_CONFIG[GGUF_CONFIG[target_format]["lm_head"]]+ check_fixed_by_user = (+ self.layer_config[lm_head_name].get("fixed_by_user", False)+ if lm_head_name in self.layer_config+ else None+ )+ self._apply_config_to_layer(lm_head_name, config, check_fixed_by_user=check_fixed_by_user)+ return True++ return False++ def _apply_config_to_layer(+ self,+ layer_name: str,+ config: dict[str, Any],+ check_fixed_by_user: bool = False,+ ) -> None:+ """Applies GGUF quantization configuration to a given layer.++ Args:+ layer_name (str): Name of the layer to configure.+ config (dict[str, Any]): GGUF layer configuration.+ check_fixed_by_user (bool): If True, preserve user-defined settings.+ """+ act_bits: int = 16+ scale_dtype: Any = self.scale_dtype+ keys: list[str] = ["bits", "group_size", "super_bits", "super_group_size", "data_type", "sym"]++ self.layer_config[layer_name] = self.layer_config.get(layer_name, {})++ for key in keys:+ if (+ key in self.layer_config[layer_name]+ and check_fixed_by_user+ # and self.layer_config[layer_name].get("fixed_by_user", False)+ ):+ continue+ self.layer_config[layer_name][key] = config.get(key)+ setattr(get_module(self.model, layer_name), key, config.get(key))++ self.layer_config[layer_name]["act_bits"] = act_bits+ self.layer_config[layer_name]["scale_dtype"] = scale_dtype+ setattr(get_module(self.model, layer_name), "act_bits", act_bits)+ setattr(get_module(self.model, layer_name), "scale_dtype", scale_dtype)++ def _quantize_layer_via_rtn(self, name: str) -> None:+ """Quantizes a layer using RTN (Round-To-Nearest) if available.++ This function attempts to quantize a layer by switching its data type to a+ `rtn_*` version if supported, then wraps and unwraps the module to apply+ quantization. If GPU memory is insufficient, it falls back to CPU.++ If packing is enabled (`is_packing_immediate`), the function will also export+ the quantized layer to the appropriate backend format.++ Args:+ name (str): Name of the layer to quantize.++ Raises:+ RuntimeError: If quantization fails for reasons unrelated to memory.+ """+ m = get_module(self.model, name)++ # if m.__class__.__name__ == "FP8Linear":+ if _is_fp8_linear(m):+ m = convert_fp8_layer_to_linear(m, self.amp_dtype)+ set_module(self.model, name, m)++ # Step 1: Use optimized RTN data type if available+ if not self.disable_opt_rtn and not m.data_type.startswith("rtn_"):+ from auto_round.data_type import QUANT_FUNC_WITH_DTYPE++ rtn_dtype = "rtn_" + m.data_type+ if rtn_dtype in QUANT_FUNC_WITH_DTYPE:+ m.data_type = rtn_dtype+ self.layer_config[name]["data_type"] = m.data_type++ # Step 2: Try quantization on GPU first, fall back to CPU if OOM+ # if only export gguf, using gguf-packing instead of rtn+ if self.is_packing_immediate and self.iters == 0 and "gguf" in self.formats[0] and not self.disable_opt_rtn:+ m.scale = None+ m.zp = None+ else:+ try:+ m.to(self.device)+ m = WrapperLinear(+ m,+ enable_minmax_tuning=False,+ enable_norm_bias_tuning=False,+ enable_round_tuning=False,+ )+ m = m.unwrapper({})+ m.to("cpu")+ except RuntimeError as e:+ cuda_error_msg = traceback.format_exc()+ m = m.orig_layer if hasattr(m, "orig_layer") else m+ try:+ logger.error(cuda_error_msg)+ logger.warning("falling back to CPU.")+ m.to("cpu")+ m = WrapperLinear(+ m,+ enable_minmax_tuning=False,+ enable_norm_bias_tuning=False,+ enable_round_tuning=False,+ )+ m = m.unwrapper({})+ except Exception as e:+ raise++ # Step 3: Optional immediate packing/export+ if self.is_packing_immediate:+ from auto_round.export import PACKING_LAYER_WITH_FORMAT++ if check_to_quantized(m):+ target_backend = self.formats[0].split(":")[0] if ":" in self.formats[0] else self.formats[0]+ has_gguf = any("gguf" in fmt for fmt in self.formats)++ if has_gguf:+ from auto_round.export.export_to_gguf.export import pack_gguf_layer++ output_dir = self._get_save_folder_name(self.formats[0])+ model_type = ModelType.MMPROJ if self.mllm else ModelType.TEXT+ pack_gguf_layer(+ name,+ self.model,+ self.formats[0],+ output_dir,+ self.layer_config,+ self.tokenizer,+ processor=self.processor if hasattr(self, "processor") else None,+ image_processor=self.image_processor if hasattr(self, "image_processor") else None,+ model_type=model_type,+ )+ else:+ PACKING_LAYER_WITH_FORMAT[target_backend](name, self.model, self.formats[0], device=self.device)++ # if self.low_gpu_mem_usage:+ # clear_memory()+ else:+ set_module(self.model, name, m)++ @torch.inference_mode()+ def _quantize_rtn(self) -> tuple[torch.nn.Module, dict[str, Any]]:+ """Quantize all modules in the model using RTN (Round-To-Nearest) strategy.++ If the target format includes GGUF with `k`, and optimized RTN is enabled,+ blockwise quantization with input caching and imatrix is used.++ Returns:+ tuple[nn.Module, Dict[str, Any]]: The quantized model and the layer configuration.+ """+ if self.amp and self.model.dtype != self.amp_dtype:+ self.model.to(self.amp_dtype)++ all_to_quantized_module_names: list[str] = [n for n, m in self.model.named_modules() if check_to_quantized(m)]++ if is_nv_fp(self.data_type):+ from auto_round.data_type.nvfp import calculate_gparam+ from auto_round.data_type.utils import update_fused_layer_global_scales++ pbar = tqdm(all_to_quantized_module_names)+ for name in pbar:+ pbar.set_description(f"Calculate weight global scale: {name}")+ m = get_module(self.model, name)+ weight_global_scale = calculate_gparam(m.weight, self.group_size)+ setattr(m, "weight_global_scale", weight_global_scale)++ modules = list(self.model.modules())+ for module in tqdm(modules, desc="Update weight global scale for fuse module"):+ update_fused_layer_global_scales(module)++ has_gguf_k = any("gguf" in fmt and "k" in fmt for fmt in getattr(self, "formats", []))++ self._quantize_embedding_layer()++ self.model.to("cpu")+ if has_gguf_k and not self.disable_opt_rtn:+ self._quant_rtn_with_imatrix(all_to_quantized_module_names)+ elif self.act_bits <= 8 and check_need_act_calibration(+ self.act_dynamic, self.act_data_type, self.act_bits+ ): # TODO, mixed datatype has bug+ hook_handles = self._register_act_max_hook(self.model)+ try:+ self._quantize_via_rtn_blockwise(all_to_quantized_module_names)+ except RuntimeError as e:+ logger.warning("Fallback to CPU. Consider using more GPUs via `--device 0,1,2,3`.")+ self.model = self.model.to("cpu")+ clear_memory()+ if hasattr(self.model, "hf_device_map") and len(self.model.hf_device_map) > 1:+ import accelerate++ accelerate.hooks.remove_hook_from_submodules(self.model)+ orig_device = self.device+ self.device = "cpu"+ self._quantize_via_rtn_blockwise(all_to_quantized_module_names)+ self.device = orig_device+ for handle in hook_handles:+ handle.remove()+ else:+ block_names_cnt = len(flatten_list(get_block_names(self.model, True)))+ clear_mem_freq = len(all_to_quantized_module_names) // block_names_cnt+ if clear_mem_freq == 0:+ clear_mem_freq = 1+ pbar = tqdm(all_to_quantized_module_names)+ cnt = 1+ for name in pbar:+ pbar.set_description(f"Quantizing {name}")+ self._quantize_layer_via_rtn(name)+ if cnt % clear_mem_freq == 0:+ clear_memory()+ cnt = 1+ cnt += 1+ # Convert remaining fp8+ if _is_fp8_model(self.model):+ convert_fp8_model_to_16b_model(self.model, self.amp_dtype)+ self.quantized = True+ return self.model, self.layer_config++ def _quantize_via_rtn_blockwise(self, all_to_quantized_module_names: list[str]) -> None:+ """Quantize model layers block by block using cached inputs and imatrix.++ Args:+ all_to_quantized_module_names (list[str]): Names of layers to be quantized.+ """+ all_to_quantized_module_names = list(set(all_to_quantized_module_names))++ all_blocks = self.quant_block_list if self.quant_block_list else get_block_names(self.model)+ if not all_blocks:+ raise ValueError("Could not find any blocks. Check the model or quant_block_list.")++ all_first_block_names = [block[0] for block in all_blocks]+ if self.act_bits < 16 and not self.act_dynamic:+ layer_names = self._get_quantized_layer_names_outside_blocks()+ if len(layer_names) > 0:+ logger.warning(+ "quantize layers outside blocks for static activation quantizaiton"+ " will significantly increase calibration time"+ )+ all_inputs = self.try_cache_inter_data_gpucpu(all_first_block_names, self.nsamples, layer_names)+ else:+ all_inputs = self.cache_inter_data(all_first_block_names, self.nsamples)++ # Clear hooks for multi-GPU setups+ if hasattr(self.model, "hf_device_map") and len(self.model.hf_device_map) > 1:+ accelerate.hooks.remove_hook_from_submodules(self.model)++ pbar = tqdm(range(sum(len(block) for block in all_blocks)))++ for block_names in all_blocks:+ first_block = block_names[0]+ inputs = all_inputs.pop(first_block)+ input_keys = [k for k in inputs if k.startswith("hidden_state")]+ if len(input_keys) != 1:+ raise RuntimeError(+ "hidden_states arg mismatch. Please file an issue at https://github.com/intel/auto-round/issues"+ )+ inputs["input_ids"] = inputs.pop(input_keys[0])++ clear_memory(self.inputs)++ total_samples = len(inputs["input_ids"])+ if total_samples < self.batch_size:+ self.batch_size = total_samples+ logger.warning(f"Forcing batch size to {total_samples}")++ input_ids = to_device(inputs.pop("input_ids"), self.cache_device)+ input_others = to_device(inputs, self.cache_device)++ tmp_dtype = self.amp_dtype if self.amp else torch.float32+ input_ids = [id_.to(tmp_dtype) for id_ in input_ids]++ for key, val in input_others.items():+ if isinstance(val, torch.Tensor) and val.dtype in (torch.float16, torch.bfloat16):+ input_others[key] = val.to(tmp_dtype)+ elif isinstance(val, list):+ input_others[key] = [to_dtype(v, tmp_dtype) for v in val]++ for block_name in block_names:+ pbar.set_description(f"Quantizing {block_name}")+ block = get_module(self.model, block_name)+ block = block.to(self.device)+ if _is_fp8_model(self.model):+ convert_fp8_model_to_16b_model(block, dtype=self.amp_dtype)++ if self.device_map == "auto":+ self._set_auto_device_map_in_block(block, input_ids)++ # Dispatch model if needed+ if self.device_map is not None:+ from accelerate.hooks import AlignDevicesHook, add_hook_to_module++ for _, m in block.named_modules():+ if len(list(m.children())) != 0 or not hasattr(m, "tuning_device"):+ continue+ hook = AlignDevicesHook(m.tuning_device, io_same_device=True)+ add_hook_to_module(m, hook, True)+ else:+ block = block.to(self.device)+ input_ids = self._get_block_outputs(+ block,+ input_ids,+ input_others,+ self.batch_size * self.infer_bs_coeff,+ self.device,+ self.cache_device,+ )+ if self.device_map is not None:+ accelerate.hooks.remove_hook_from_submodules(block)++ if (+ is_nv_fp(self.act_data_type) and any("nv_fp" in format_ for format_ in self.formats)+ ) or is_static_wfp8afp8(self):+ # enable moe experts act_max automatic generation for Linear+ set_amax_for_all_moe_layers(block, attr_name="act_max")+ # Normalize imatrix and quantize layers+ for _, m in block.named_modules():+ if hasattr(m, "imatrix"):+ m.imatrix /= m.imatrix_cnt+ if hasattr(m, "tmp_name") and m.tmp_name in all_to_quantized_module_names:+ self._quantize_layer_via_rtn(m.tmp_name)+ all_to_quantized_module_names.remove(m.tmp_name)++ mv_module_from_gpu(block, self.low_cpu_mem_usage)+ pbar.update(1)++ pbar.close()+ cnt = 1+ block_names_cnt = len(flatten_list(get_block_names(self.model, True)))+ clear_mem_freq = len(all_to_quantized_module_names) // block_names_cnt+ if clear_mem_freq == 0:+ clear_mem_freq = 1+ # Process remaining layers not in blocks+ for name in all_to_quantized_module_names:+ self._quantize_layer_via_rtn(name)+ if cnt % clear_mem_freq == 0:+ clear_memory()+ cnt = 1+ cnt += 1++ def quantize(self) -> tuple[torch.nn.Module, dict[str, Any]]:+ """Quantize the model and return the quantized model along with layer configurations.The entry of AutoRound.+ Returns:+ The quantized model and layer configurations.+ """+ for n, m in self.model.named_modules():+ m.tmp_name = n+ self._check_compatibility()+ formats = self.formats if hasattr(self, "formats") else None+ # It is best to modify the model structure in the quantize function and check the format,+ # because it may cause the gguf format to not be exported normally.+ self.model = _handle_moe_model(self.model, formats=formats)+ self.has_qlayer_outside_block = self._set_layerwise_config(self.layer_config)+ if not hasattr(self, "formats"):+ logger.warning("this API is deprecated, please use `quantize_and_save` instead")+ else:+ only_gguf = True+ for format_ in self.formats:+ if not ("gguf" in format_ or "fake" in format_):+ only_gguf = False+ break+ if len(self.formats) == 1 and self.formats[0] == "fake":+ only_gguf = False+ if only_gguf:+ self.layer_config, gguf_format_config = get_layer_config_by_gguf_format(+ self.layer_config, self.formats, self.model, model_type=ModelType.TEXT+ )+ if self.mllm:+ self.layer_config, gguf_format_config = get_layer_config_by_gguf_format(+ self.layer_config, self.formats, self.model, model_type=ModelType.MMPROJ+ )+ # Determine if immediate packing is required+ formats = self.formats+ if (+ len(formats) == 1+ and (+ "awq" in formats[0]+ or "gptq" in formats[0]+ or "auto_round" in formats[0]+ or "gguf" in formats[0]+ or "llm_compressor" in formats[0]+ )+ and self.inplace+ ):+ self.is_packing_immediate = True+ if self.iters == 0:+ return self._quantize_rtn()++ if bool(self.quant_block_list):+ all_blocks = self.quant_block_list+ else:+ all_blocks = get_block_names(self.model)++ if len(all_blocks) == 0:+ logger.warning("could not find blocks, exit with original model")+ return self.model, self.layer_config++ if self.amp and self.model.dtype != self.amp_dtype:+ self.model = self.model.to(self.amp_dtype)++ layer_names = self._get_quantized_layer_names_outside_blocks()+ self.start_time = time.time()+ all_first_block_names = [block[0] for block in all_blocks]+ if len(layer_names) > 0:+ logger.info(+ "Starting to cache block inputs. This may be slow due to external block layers: %s", layer_names+ )+ else:+ logger.info("start to cache block inputs")+ all_inputs = self.try_cache_inter_data_gpucpu(all_first_block_names, self.nsamples, layer_names=layer_names)+ is_quantized_embedding = self._quantize_embedding_layer()+ all_q_inputs = None+ if is_quantized_embedding:+ all_inputs = copy.deepcopy(self.inputs)+ clear_memory(self.inputs)+ all_q_inputs = self.try_cache_inter_data_gpucpu(+ all_first_block_names, self.nsamples, layer_names=layer_names+ )+ self.model = mv_module_from_gpu(self.model, self.low_cpu_mem_usage)+ clear_memory()+ if hasattr(self.model, "hf_device_map") and len(self.model.hf_device_map) > 1:+ accelerate.hooks.remove_hook_from_submodules(self.model) # self.model.hf_device_map has not been changed+ self.model = mv_module_from_gpu(self.model, self.low_cpu_mem_usage)+ logger.info("caching done")+ if len(all_blocks) > 1:+ pbar = tqdm(range(0, sum([len(i) for i in all_blocks]), self.nblocks))+ else:+ pbar = None # move the alg warning outside pbar++ for block_names in all_blocks:+ inputs = all_inputs[block_names[0]]+ all_inputs.pop(block_names[0])+ q_inputs = None+ if all_q_inputs is not None:+ q_inputs = all_q_inputs[block_names[0]]+ all_q_inputs.pop(block_names[0])+ keys = inputs.keys()+ input_id_str = [key for key in keys if key.startswith("hidden_state")]+ if len(input_id_str) != 1:+ raise RuntimeError(+ "hidden_states arg mismatch error,"+ "please raise an issue in https://github.com/intel/auto-round/issues"+ )+ inputs["input_ids"] = inputs.pop(input_id_str[0], None)+ if q_inputs is not None:+ q_inputs["input_ids"] = q_inputs.pop(input_id_str[0], None)++ clear_memory(self.inputs)++ if "input_ids" in inputs.keys():+ total_samples = len(inputs["input_ids"])+ if total_samples < self.batch_size:+ self.batch_size = total_samples+ logger.warning(f"force the train batch size to {total_samples}")++ self._quantize_blocks(+ self.model,+ inputs,+ block_names,+ q_input=q_inputs["input_ids"] if q_inputs is not None else None,+ nblocks=self.nblocks,+ device=self.device,+ pbar=pbar,+ )+ if self.is_packing_immediate and len(self.formats) != 1:+ raise ValueError(+ f"Expected exactly one packing format when 'is_packing_immediate' is True, "+ f"but got {len(self.formats)} formats."+ )++ self._quantize_layers(layer_names, all_inputs) ##TODO pack layer immediately++ if _is_fp8_model(self.model):+ for n, m in self.model.named_modules():+ if _is_fp8_linear(m):+ new_layer = convert_fp8_layer_to_linear(m, self.amp_dtype).to("cpu")+ set_module(self.model, n, new_layer)++ end_time = time.time()+ cost_time = end_time - self.start_time+ logger.info(f"quantization tuning time {cost_time}")++ ## dump a summary+ quantized_layers = []+ unquantized_layers = []+ for n, m in self.model.named_modules():+ if isinstance(m, tuple(self.supported_types)):+ if check_to_quantized(m):+ quantized_layers.append(n)+ else:+ unquantized_layers.append(n)+ elif hasattr(m, "scales") or hasattr(m, "scale"): ##packing_immediately+ quantized_layers.append(n)+ summary_info = (+ f"Summary: quantized {len(quantized_layers)}/{len(quantized_layers) + len(unquantized_layers)} in the model"+ )+ if len(unquantized_layers) > 0:+ summary_info += f", {unquantized_layers} have not been quantized"+ logger.info(summary_info)++ self.quantized = True+ return self.model, self.layer_config++ def _quantize_layers(self, layer_names: list, layer_inputs: dict) -> None:+ """Quantizes specified layers based on inputs and configuration.++ Args:+ layer_names (list): list of layer names to quantize.+ layer_inputs (dict): Dictionary mapping layer names to input data.++ Returns:+ None+ """+ ##TODO currently we take all the layers outside blocks as post block layers which is not optimal+ ## if there is no input for layer, we use rtn++ for layer_name in copy.deepcopy(layer_names):+ if layer_name not in layer_inputs:+ logger.info(f"using rtn to quantize {layer_name}")+ from auto_round.data_type import QUANT_FUNC_WITH_DTYPE++ layer = get_module(self.model, layer_name)+ if _is_fp8_model(self.model):+ new_layer = convert_fp8_layer_to_linear(layer, self.amp_dtype).to(self.device)+ set_module(self.model, layer_name, new_layer)+ layer = new_layer++ if not self.disable_opt_rtn and "rtn_" + layer.data_type in QUANT_FUNC_WITH_DTYPE:+ layer.data_type = "rtn_" + layer.data_type+ logger.info("using optimized rtn method for quantizing %s", layer_name)+ self.layer_config[layer_name]["data_type"] = layer.data_type+ wrapper_layer = WrapperLinear(+ layer,+ enable_round_tuning=False,+ enable_minmax_tuning=False,+ enable_norm_bias_tuning=False,+ device=self.device,+ )+ new_layer = wrapper_layer.unwrapper({})+ set_module(self.model, layer_name, new_layer)+ layer.cpu()+ layer_names.remove(layer_name)+ if len(layer_names) == 0:+ return+ q_layer_inputs = None+ enable_quanted_input = self.enable_quanted_input+ has_gguf = False+ if hasattr(self, "formats"):+ has_gguf = any("gguf" in format_ for format_ in self.formats)+ if has_gguf and self.is_packing_immediate:+ enable_quanted_input = False++ if hasattr(self.model, "hf_device_map") and len(self.model.hf_device_map) > 1 and enable_quanted_input:+ dispatch_model(self.model, self.model.hf_device_map)++ if enable_quanted_input:+ logger.info("starting to cache layer inputs for %s, this may be quite slow ", layer_names)+ q_layer_inputs = self.try_cache_inter_data_gpucpu([], self.nsamples, layer_names=layer_names)+ if hasattr(self.model, "hf_device_map") and len(self.model.hf_device_map) > 1:+ accelerate.hooks.remove_hook_from_submodules(+ self.model+ ) ##self.model.hf_device_map has not been changed++ self.model = mv_module_from_gpu(self.model, self.low_cpu_mem_usage)+ clear_memory()+ if self.enable_torch_compile:+ quant_layer = compile_func(self._quantize_layer, self.device)+ else:+ quant_layer = self._quantize_layer+ for layer_name in layer_names:+ layer_input = layer_inputs[layer_name]+ layer_input = to_device(layer_input, self.cache_device)+ q_layer_input = q_layer_inputs.get(layer_name, None) if q_layer_inputs is not None else None+ q_layer_input = to_device(q_layer_input, self.cache_device)+ quant_layer(layer_name, layer_input, q_layer_input, device=self.device)+ del layer_input+ clear_memory(q_layer_input)++ def _set_layerwise_config(self, layer_config: dict) -> bool:+ """+ Sets the layer-wise configuration based on the provided `layer_config`.+ By default, only quantize layers in blocks.++ Args:+ layer_config (dict): The configuration dictionary for each layer containing various configuration options.++ Returns:+ bool: Returns True if there are quantized layers outside the blocks (e.g., lm-head),+ otherwise returns False.+ """+ # Get the names of layers in quantization blocks+ supported_types = self.supported_types+ layers_in_blocks = get_layer_names_in_block(+ self.model, supported_types, self.quant_block_list, self.inner_supported_types+ )+ ##process regex in layer_config+ all_supported_layer_names = []+ # List of configuration keys+ keys = get_quant_keys()++ for n, m in self.model.named_modules():+ # Delete previous configuration to avoid conflicts with prior tuning+ for key in keys:+ if hasattr(m, key):+ delattr(m, key)++ if not isinstance(m, supported_types) and m.__class__.__name__ not in self.inner_supported_types:+ continue+ all_supported_layer_names.append(n)++ names_in_layer_config = list(layer_config.keys())+ for name in names_in_layer_config:+ if name in all_supported_layer_names:+ continue+ matched_names = []+ for layer_name in all_supported_layer_names:+ if re.search(re.compile(name), layer_name) is not None:+ matched_names.append(layer_name)+ if len(matched_names) > 0:+ val = layer_config[name]+ layer_config.pop(name)+ for match_name in matched_names:+ layer_config[match_name] = val+ else:+ tmp_m = get_module(self.model, name)+ if not isinstance(tmp_m, torch.nn.Embedding): # TODO not good code style+ raise ValueError(f"key {name} in layer_config is invalid, please have a double check")++ has_qlayer_outside_block = False # Flag to track if there are quantized layers outside blocks (e.g., lm-head)++ # Iterate through all modules in the model+ is_gguf = hasattr(self, "formats") and any("gguf" in format_ for format_ in self.formats)+ for n, m in self.model.named_modules():+ # Skip unsupported types+ if not isinstance(m, supported_types) and m.__class__.__name__ not in self.inner_supported_types:+ if n in self.layer_config:+ if not isinstance(m, torch.nn.Embedding):+ logger.warning(f"{n} is not supported, layer_config {n}: {layer_config[n]} will be ignored.")+ self.layer_config.pop(n)+ continue+ if not is_gguf:+ if not check_to_quantized(layer_config[n]):+ self.layer_config.pop(n)+ continue+ else:+ continue++ # If the layer is not in the config and is part of a quantization block, use default configuration+ if n not in layer_config.keys() and n in layers_in_blocks:+ layer_config[n] = {}+ for key in keys:+ layer_config[n][key] = getattr(self, key)++ # If the layer is partially configured, fill in missing values+ elif n in layer_config.keys():+ if "data_type" in layer_config[n] and "bits" not in layer_config[n]:+ tmp_bits = infer_bits_by_data_type(layer_config[n]["data_type"])+ if tmp_bits is not None and tmp_bits != self.bits:+ logger.warning(+ f"'data_type' do not match the specified 'bits' setting for {n}."+ f" Resetting 'bits' to {tmp_bits}."+ )+ layer_config[n]["bits"] = tmp_bits+ if "act_data_type" in layer_config[n] and "act_bits" not in layer_config[n]:+ tmp_bits = infer_bits_by_data_type(layer_config[n]["act_data_type"])+ if tmp_bits is not None and tmp_bits != self.act_bits:+ logger.warning(+ f"'act_data_type' do not match the specified 'act_bits' setting for {n}."+ f" Resetting 'act_bits' to {tmp_bits}."+ )+ layer_config[n]["act_bits"] = tmp_bits++ for key in keys:+ if key not in layer_config[n].keys():+ layer_config[n][key] = getattr(self, key)+ layer_config[n]["fixed_by_user"] = True++ # If the layer is not in the config and not part of a quantization block,+ # use default configuration and set specific values+ else:+ layer_config[n] = {}+ for key in keys:+ layer_config[n][key] = getattr(self, key)+ layer_config[n]["bits"] = 16+ layer_config[n]["act_bits"] = 16++ if n in layers_in_blocks:+ layer_config[n]["in_blocks"] = True+ else:+ layer_config[n]["in_blocks"] = False++ # If the layer is outside a block and requires quantization, mark it as a quantized layer outside the block+ if (+ n not in layers_in_blocks+ and check_to_quantized(layer_config[n])+ and not isinstance(m, torch.nn.Embedding)+ ):+ has_qlayer_outside_block = True++ in_features, out_features = get_layer_features(m)+ if in_features <= layer_config[n]["group_size"]:+ layer_config[n]["group_size"] = -1++ # Apply the configuration to the corresponding layer in the model+ for key in keys:+ setattr(m, key, layer_config[n][key])+ need_to_quantize_lm_head = self._check_need_to_quantize_lm_head_embedding()+ if need_to_quantize_lm_head:+ has_qlayer_outside_block = True++ # Return whether there are quantized layers outside the blocks+ return has_qlayer_outside_block++ @torch.no_grad()+ def _get_block_outputs(+ self,+ block: torch.nn.Module,+ input_ids: torch.Tensor,+ input_others: torch.Tensor,+ bs: int,+ device: Union[str, torch.device],+ cache_device: Union[str, torch.device],+ save_output: bool = True,+ ):+ """Compute the output of a given block of the model for a given input.++ Args:+ block: The block of the model.+ input_ids: The input tensor containing tokenized input ids.+ input_others: A dictionary containing additional input data.+ bs: The batch size for computing the output.+ device: The device for computation.+ cache_device: The device for storing the output.+ batch_dim: The batch dimension of the output tensor.++ Returns:+ The output tensor of the block.+ """++ output = []+ nsamples = len(input_ids)+ for i in range(0, nsamples, bs):+ end_index = min(nsamples, i + bs)+ indices = torch.arange(i, end_index).to(torch.long)+ tmp_input_ids, tmp_input_others = self._sampling_inputs(+ input_ids, input_others, indices, self.seqlen, self.batch_dim, share_cache_keys=self.shared_cache_keys+ )+ tmp_output = block_forward(block, tmp_input_ids, tmp_input_others, self.amp, self.amp_dtype, device).to(+ cache_device+ )+ if save_output:+ if self.batch_size == 1:+ output.append(tmp_output)+ else:+ output.extend(list(torch.split(tmp_output, 1, dim=self.batch_dim)))+ if self.low_gpu_mem_usage:+ clear_memory()++ return output++ @torch.no_grad()+ def calib(self, nsamples, bs):+ """Perform calibration for quantization.++ This method calibrates the model for quantization by processing a specified+ number of samples from the calibration dataset. It ensures that the data is+ properly formatted and feeds it to the model. If the number of samples processed+ is less than the specified number, it logs a warning. If no samples are processed,+ it logs an error and exits.+ Args:+ nsamples (int): The number of samples to use for calibration.+ bs (int): The number of samples to use for calibration+ """+ from auto_round.calib_dataset import get_dataloader++ if isinstance(self.dataset, str):+ dataset = self.dataset.replace(" ", "") ##remove all whitespaces++ # slow here+ self.dataloader = get_dataloader(+ self.tokenizer,+ self.seqlen,+ dataset,+ self.seed,+ bs,+ self.nsamples,+ )+ else:+ self.dataloader = self.dataset+ total_cnt = 0++ # load embed weight if use low_cpu_mem_usage+ if self.low_cpu_mem_usage:+ embed_layers = get_layers_before_block(self.model)+ for n, m in embed_layers:+ m = m.to(self.device)++ for data in self.dataloader:+ if data is None:+ continue+ if isinstance(data, torch.Tensor):+ input_ids = data.to(self.model.device)+ data_new = input_ids+ elif isinstance(data, str):+ if self.tokenizer is None:+ logger.error("please provide tokenizer for string input")+ exit(-1)+ data = self.tokenizer(data, truncation=True, max_length=self.seqlen, return_tensors="pt").data+ data_new = {}+ for key in data.keys():+ data_new[key] = data[key].to(self.model.device)+ input_ids = data_new["input_ids"]+ elif isinstance(data, tuple) or isinstance(data, list):+ data_new = to_device(data)+ input_ids = data_new[0]+ else:+ data_new = {}+ for key in data.keys():+ data_new[key] = to_device(data[key], self.model.device)+ if key == "images":+ data_new[key] = to_dtype(data_new[key], self.model.dtype)+ input_ids = data_new["input_ids"]+ if input_ids.shape[-1] < self.seqlen:+ continue+ try:+ if isinstance(data_new, torch.Tensor):+ self.model(data_new, use_cache=False)+ elif isinstance(data_new, tuple) or isinstance(data_new, list):+ self.model(*data_new, use_cache=False)+ else:+ self.model(**data_new, use_cache=False)+ except NotImplementedError:+ pass+ except RuntimeError as error:+ error_msg = str(error)+ if "The expanded size of the tensor" in str(error_msg) and "must match the existing size" in error_msg:+ check_seqlen_compatible(self.seqlen, self.tokenizer, self.model)+ logger.warning(+ "When quantization encounters tensor shape mismatch error, "+ "you can try to avoid it with batch_size=1"+ )+ raise error+ except Exception as error:+ raise error+ total_cnt += input_ids.shape[0] if len(input_ids.shape) > 1 else 1+ if total_cnt >= nsamples:+ break+ if total_cnt == 0:+ logger.error(+ f"no data has been cached, please provide more data with sequence length >={self.seqlen} in the "+ f"dataset or decease the sequence length"+ )+ exit(-1)+ elif total_cnt < nsamples:+ logger.warning(+ f"An insufficient number of samples likely reduces the accuracy of the quantized model. "+ f"Target samples count is {nsamples}, while valid samples count is {total_cnt}"+ )++ # clean embed weight to save memory+ if self.low_cpu_mem_usage:+ for n, m in embed_layers:+ m = m.to("meta")++ @torch.no_grad()+ def try_cache_inter_data_gpucpu(self, block_names, nsamples, layer_names=None, last_cache_name=None):+ """Attempts to cache intermediate data on GPU, if failed, then using CPU.++ Args:+ block_names (list): List of block names to cache data for.+ nsamples (int): Number of samples to use for caching.+ layer_names (list, optional): List of layer names to cache data for. Defaults to [].+ last_cache_name (str, optional): Name of the last cache. Defaults to None.++ Returns:+ all_inputs: Cached intermediate data.++ Raises:+ Exception: If caching on GPU fails, switches to CPU and caches there.+ """+ if _is_fp8_model(self.model):+ layer_names = []+ if layer_names is None:+ layer_names = []++ if self.low_gpu_mem_usage or (+ len(block_names) == 1+ and len(layer_names) == 0+ and not self.has_qlayer_outside_block+ and (last_cache_name is None or last_cache_name in block_names)+ ):+ # low_gpu_mem_usage or calibrate only the embedding layer, which is also very fast on CPU+ all_inputs = self.cache_inter_data(block_names, nsamples, layer_names=[], last_cache_name=last_cache_name)+ else:+ try:+ if not self.model.device.type == "meta":+ if hasattr(self.model, "hf_device_map") and len(self.model.hf_device_map) > 1:+ self.model = dispatch_model(self.model, device_map=self.model.hf_device_map)+ else:+ # Change this if new device is supported+ if str(self.model.device) == "cpu" and (+ self.device.startswith("xpu") or self.device.startswith("cuda")+ ):+ max_memory = get_max_vram() # TODO model is not evenly split+ no_split_modules = getattr(self.model, "_no_split_modules", [])+ device_map = infer_auto_device_map(+ self.model, max_memory=max_memory, no_split_module_classes=no_split_modules+ )++ self.model = dispatch_model(self.model, device_map=device_map)+ else:+ self.model = self.model.to(self.device)++ all_inputs = self.cache_inter_data(+ block_names, nsamples, layer_names=layer_names, last_cache_name=last_cache_name+ )+ if hasattr(self.model, "hf_device_map") and len(self.model.hf_device_map) > 1:+ accelerate.hooks.remove_hook_from_submodules(self.model)++ except RuntimeError as e:+ cuda_error_msg = traceback.format_exc()+ try:+ logger.info("switch to cpu to cache block inputs")+ if self.has_qlayer_outside_block or self.__class__.__name__ == "AutoRoundMLLM":+ logger.warning(+ "we recommend using more GPUs in calibration."+ " Otherwise, some layers may fall back to `rtn` mode, which can affect accuracy."+ )+ if hasattr(self.model, "hf_device_map") and len(self.model.hf_device_map) > 1:+ accelerate.hooks.remove_hook_from_submodules(+ self.model+ ) ##self.model.hf_device_map has not been changed+ self.model = mv_module_from_gpu(self.model, self.low_cpu_mem_usage)+ clear_memory()+ ## Important change after v0.51, on cpu, we use rtn mode for layers in layer_names+ all_inputs = self.cache_inter_data(+ block_names, nsamples, layer_names=[], last_cache_name=last_cache_name+ )+ except Exception as e:+ logger.error(cuda_error_msg)+ raise+ return all_inputs++ @torch.no_grad()+ def cache_inter_data(self, block_names, nsamples, layer_names=None, last_cache_name=None):+ """Save the inputs of block_name for calibration.++ This method temporarily replaces the forward method of the model to capture+ the inputs passing through the specified block. It then calibrates the model+ using a specified number of samples. Finally, it restores the original forward+ method and returns the inputs for the specified block.+ Args:+ block_names (list): The names of the blocks for which inputs are to be saved.+ layer_names (list):The names of the layers for which inputs are to be saved.+ nsamples (int): The number of samples to use for calibration.+ last_cache_name (str, optional): The name of the last layer to be cached,+ we could break the forward in this layer to save time++ Returns:+ dict: A dictionary containing the inputs for the specified block.+ """+ if layer_names is None:+ layer_names = []+ self.inputs = {}+ self.to_cached_layers = block_names + layer_names++ tmp_dtype = None # TODO delete this as most model is not fp32 now+ ## have bug if block name is not the first block+ if (len(block_names) > 1 or len(layer_names) > 0) and self.low_gpu_mem_usage:+ tmp_dtype = self.model.dtype+ if self.amp:+ if self.model.dtype != self.model.dtype:+ self.model = self.model.to(torch.bfloat16)+ else:+ self.model = self.model.to(torch.float32) ##model on cpu++ self.last_cache_name = last_cache_name+ if last_cache_name is None and len(block_names) + len(layer_names) == 1:+ self.last_cache_name = block_names[0] if len(block_names) == 1 else layer_names[0]+ # do not set last_cache_name for multimodal models+ calib_bs = self.batch_size+ self.hook_handles = []+ self._replace_forward()+ self.calib(nsamples, calib_bs)+ self._recover_forward()+ res = self.inputs+ del self.last_cache_name+ del self.to_cached_layers+ if tmp_dtype is not None:+ self.model = self.model.to(tmp_dtype)++ return res++ @torch.no_grad()+ def _get_block_forward_func(self, name: str) -> Callable:+ """Gets the forward function.++ Args:+ name (str): The name of the function.+ Returns:+ function: The forward function.+ """++ def post_process_cache_data(batch_size, data, data_name):+ """+ Processes store data for batch handling, reshaping if necessary.++ Args:+ batch_size (int): The size of the batch.+ data: The data value to store, potentially for caching.+ data_name (str): Name of the data.++ Returns:+ Processed data or None+ """+ new_data = data+ if batch_size <= 1:+ return new_data+ if data_name in self.shared_cache_keys:+ return None+ if "alibi" in data_name:+ if isinstance(data, torch.Tensor):+ alibi = data+ alibi = alibi.reshape(batch_size, -1, alibi.shape[1], alibi.shape[2])+ new_data = alibi+ return new_data++ def forward(m, hidden_states=None, *positional_inputs, **kwargs):+ """Rewrite forward function, process and collect input data.++ Args:+ hidden_states (torch.Tensor): The hidden states tensor.+ *positional_inputs: Variable number of positional arguments.+ **kwargs: Variable number of keyword arguments.++ Returns:+ NotImplementedError: Getting the first layer inputs and then raise the error to save runtime.+ """+ if name not in self.inputs:+ self.inputs[name] = {}+ init_cache(positional_inputs, self.inputs[name])++ if self.batch_dim is None:+ self.batch_dim = 0+ if hidden_states is not None and self.batch_size > 1:+ if hidden_states.shape[0] > self.batch_size:+ self.batch_dim = 1+ if len(hidden_states.shape) > 1 and hidden_states.shape[1] > self.batch_size:+ logger.error(+ "this model has not been supported, "+ "please raise an issue in https://github.com/intel/auto-round/issues"+ " or try to set the `batch_size` to 1 and "+ "`gradient_accumulate_steps` to your current batch size."+ )+ exit(-1)++ if hidden_states is not None:+ kwargs["hidden_states"] = hidden_states++ for key in kwargs.keys():+ if (+ isinstance(kwargs[key], torch.Tensor)+ or isinstance(kwargs[key], list)+ or isinstance(kwargs[key], tuple)+ ):+ if key not in self.inputs[name].keys(): # initialization+ data = to_device(kwargs[key], device=torch.device("cpu"))+ if data is None or (self.batch_size > 1 and key in self.shared_cache_keys):+ self.inputs[name][key] = data+ continue+ if self.batch_size <= 1:+ self.inputs[name][key] = [data]+ else:+ data = post_process_cache_data(self.batch_size, data, key)+ self.inputs[name][key] = list(torch.split(data, 1, dim=self.batch_dim))+ else: # append cache inputs+ new_data = post_process_cache_data(self.batch_size, kwargs[key], key)+ if new_data is None: # shareable args or NoneType+ continue+ new_data = to_device(new_data, device=torch.device("cpu"))+ if self.batch_size <= 1:+ self.inputs[name][key].append(new_data)+ else:+ self.inputs[name][key].extend(list(torch.split(new_data, 1, dim=self.batch_dim)))+ elif isinstance(kwargs[key], (str, bool, type(None))):+ if key not in self.inputs[name].keys():+ self.inputs[name][key] = kwargs[key]+ else:+ # Parameters not to be cached+ if check_skippable_keywords(key):+ logger.warning_once(+ f"Please note that '{key}' key" " is not currently used in quantization fine-tuning."+ )+ reset_params(self.inputs[name])+ if name == self.last_cache_name:+ raise NotImplementedError+ else:+ if hidden_states is not None:+ kwargs.pop("hidden_states")+ return m.orig_forward(hidden_states, *positional_inputs, **kwargs)+ else:+ # Currently only for Llama-3.2-Vision-Instruct Series+ return m.orig_forward(*positional_inputs, **kwargs)++ return forward++ @torch.no_grad()+ def _get_cache_data_hook_for_layer(self, name):+ """A forward hook to save input max of a module+ :param name: the module name+ :return: A hook function."""++ def cache_input_hook(module, inputs, outputs):+ input = inputs+ if isinstance(inputs, tuple) or isinstance(input, list):+ input = inputs[0]+ if name in self.inputs:+ self.inputs[name].extend(list(torch.split(input.to("cpu"), 1, dim=0)))+ else:+ self.inputs[name] = list(torch.split(input.to("cpu"), 1, dim=0))++ return cache_input_hook++ def _recover_forward(self):+ """Recovers the forward function."""+ for n, m in self.model.named_modules():+ if hasattr(m, "orig_forward"):+ m.forward = m.orig_forward+ delattr(m, "orig_forward")+ for hook_handle in self.hook_handles:+ hook_handle.remove()+ self.hook_handles = []++ def _replace_forward(self):+ """Replaces the forward function."""+ from functools import partial++ for n, m in self.model.named_modules():+ if n in self.to_cached_layers and not isinstance(m, tuple(self.supported_types)): ##block+ m.orig_forward = m.forward+ m.forward = partial(self._get_block_forward_func(n), m)+ elif n in self.to_cached_layers: ##linear layer or conv1d layer+ hook_func = self._get_cache_data_hook_for_layer(n)+ hook_handle = m.register_forward_hook(hook_func)+ self.hook_handles.append(hook_handle)++ def _quantize_layer(+ self, layer_name: str, inputs: torch.Tensor, q_inputs: torch.Tensor = None, device: str = "cpu"+ ):+ """Quantize a specific layer of the model using the provided inputs.++ Args:+ layer_name (str): The name of the layer to quantize.+ inputs (torch.Tensor): Input data for quantization.+ q_inputs (torch.Tensor, optional): Quantized input data. Defaults to None.+ device (torch.device, optional): The device to use for quantization. Defaults to torch.device("cpu").++ Returns:+ None+ """+ logger.info(f"quantizing layer {layer_name}")+ layer = get_module(self.model, layer_name)+ if hasattr(layer, "tuning_device"):+ device = layer.tuning_device++ layer = layer.to(device)+ for i in range(len(inputs)):+ inputs[i] = inputs[i].to(layer.weight.dtype)+ if q_inputs is not None:+ q_inputs[i] = q_inputs[i].to(layer.weight.dtype)++ wrapper_linear = WrapperLinear(layer, enable_minmax_tuning=self.enable_minmax_tuning, device=device).to(device)+ round_params = []+ minmax_params = []+ for key in wrapper_linear.params.keys():+ if "min" in key or "max" in key:+ minmax_params.append(wrapper_linear.params[key])+ else:+ round_params.append(wrapper_linear.value)+ if len(round_params) + len(minmax_params) <= 0:+ dump_info = f"quantized {layer_name}"+ logger.info(dump_info)+ with torch.no_grad():+ unwrapper_layer(self.model, wrapper_linear, layer_name, {})+ mv_module_from_gpu(layer, self.low_cpu_mem_usage)++ lr = torch.tensor(self.lr)+ minmax_lr = torch.tensor(self.minmax_lr)+ if self.enable_minmax_tuning:+ optimizer = self.optimizer(+ [{"params": round_params}, {"params": minmax_params, "lr": minmax_lr}], lr=lr, weight_decay=0+ )+ else:+ optimizer = self.optimizer(round_params, lr=lr, weight_decay=0)++ if self.lr_scheduler is None:+ lr_schedule = torch.optim.lr_scheduler.LinearLR(+ optimizer, start_factor=1.0, end_factor=0.0, total_iters=self.iters+ )+ else:+ lr_schedule = copy.deepcopy(self.lr_scheduler)+ nsamples = len(inputs)+ last_best_iter = 0+ best_loss = torch.finfo(torch.float).max+ scaler = self._get_scaler() # pylint: disable=assignment-from-none+ init_loss = None+ gradient_accumulate_steps = self.batch_size ##Force to low gpu+ batch_size = 1 ##Force to low gpu+ pick_samples = batch_size * gradient_accumulate_steps+ pick_samples = min(nsamples, pick_samples)+ if self.sampler != "rand":+ whole_indices = torch.randperm(nsamples)[:pick_samples]+ total_loss = 0+ num_elm = 1+ mse_reduction = "mean"+ if gradient_accumulate_steps != 1:+ mse_reduction = "sum"+ mse_loss = torch.nn.MSELoss(reduction=mse_reduction).to(device)++ for i in range(self.iters):+ total_loss = 0+ if self.sampler == "rand":+ whole_indices = torch.randperm(nsamples)[:pick_samples]+ if gradient_accumulate_steps != 1:+ if q_inputs is not None:+ current_input = [q_inputs[i] for i in whole_indices]+ else:+ current_input = [inputs[i] for i in whole_indices]+ num_elm = sum(id.numel() for id in current_input)+ for tmp_step in range(gradient_accumulate_steps):+ indices = whole_indices[tmp_step * batch_size : (tmp_step + 1) * batch_size]+ if q_inputs is not None:+ current_input = [q_inputs[i] for i in indices]+ current_input = torch.cat(current_input, dim=0).to(device)+ org_input = [inputs[i] for i in indices]+ org_input = torch.cat(org_input, dim=0).to(device)+ else:+ current_input = [inputs[i] for i in indices]+ current_input = torch.cat(current_input, dim=0).to(device)+ org_input = current_input+ with torch.no_grad():+ current_output = layer(org_input)++ if self.amp:+ with autocast(device_type=device.split(":")[0], dtype=self.amp_dtype):+ output_q = wrapper_linear(current_input) # pylint: disable=not-callable+ loss = mse_loss(output_q, current_output) # pylint: disable=not-callable+ else:+ output_q = wrapper_linear(current_input) # pylint: disable=not-callable+ loss = mse_loss( # pylint: disable=not-callable+ output_q.to(torch.float32), current_output.to(torch.float32)+ )+ total_loss += loss.item() / num_elm++ self._scale_loss_and_backward(scaler, loss)+ if i == 0:+ init_loss = total_loss++ if total_loss < best_loss:+ best_loss = total_loss+ if not self.not_use_best_mse:+ best_params = collect_best_params(wrapper_linear)+ last_best_iter = i+ if self.not_use_best_mse and i == self.iters - 1:+ best_params = collect_best_params(wrapper_linear)++ if not self.not_use_best_mse:+ if 0 < self.dynamic_max_gap <= i - last_best_iter:+ break+ self._step(scaler, optimizer, lr_schedule)++ last_loss = total_loss+ best_iter = self.iters+ if not self.not_use_best_mse:+ last_loss = best_loss+ best_iter = last_best_iter+ with torch.no_grad():+ unwrapper_layer(self.model, wrapper_linear, layer_name, best_params)+ mv_module_from_gpu(layer, self.low_cpu_mem_usage)+ dump_info = f"quantized {layer_name}, loss iter 0: {init_loss:.6f} -> iter {best_iter}: {last_loss:.6f}"+ logger.info(dump_info)++ def _register_act_max_hook(self, model):+ def get_act_max_hook(module, input, output):+ if isinstance(input, (tuple, list)):+ input = input[0]+ if input.numel() == 0:+ return # as no needs for act_max update+ input, _, _ = reshape_pad_tensor_by_group_size(input, self.act_group_size)+ act_max = torch.max(torch.abs(input), dim=-1).values+ if not hasattr(module, "act_max") or module.act_max.numel() == 0:+ module.act_max = act_max+ else:+ act_max = act_max.to(module.act_max.device)+ if is_nv_fp(self.act_data_type): ## for nvfp per-tensor input_global_scale calculation usage+ module.act_max = torch.max(+ torch.tensor([act_max.max(), module.act_max.max()], device=act_max.device)+ )+ else:+ module.act_max = torch.max(act_max, module.act_max)++ hook_handles = []++ for n, m in model.named_modules():+ if (+ hasattr(m, "act_dynamic")+ and check_need_act_calibration(m.act_dynamic, m.act_data_type, m.act_bits)+ and check_to_quantized(m)+ ):+ hook = m.register_forward_hook(get_act_max_hook)+ hook_handles.append(hook)+ continue++ # for whole model, RTN+ if n in self.layer_config:+ config = self.layer_config[n]+ act_dynamic = config.get("act_dynamic", True)+ act_data_type = config.get("act_data_type", None)+ act_bits = config.get("act_data_type", 16)+ if (+ config["bits"] <= 8+ and check_need_act_calibration(act_dynamic, act_data_type, act_bits)+ and check_to_quantized(config)+ ):+ hook = m.register_forward_hook(get_act_max_hook)+ hook_handles.append(hook)+ continue+ return hook_handles++ def _quantize_block(+ self,+ block: torch.nn.Module,+ input_ids: list[torch.Tensor],+ input_others: dict,+ q_input: Union[None, torch.Tensor] = None,+ device: Union[str, torch.device] = "cpu",+ ):+ """Quantize the weights of a given block of the model.++ Args:+ block: The block of the model to be quantized.+ input_ids: The input tensor containing tokenized input ids.+ input_others: A dictionary containing additional input data.+ q_input: The quantized input tensor.+ device: The device for quantization.++ Returns:+ Tuple: (q_outputs, output) if self.enable_quanted_input is True, else (None, output)+ """+ if _is_fp8_model(self.model):+ for n, m in block.named_modules():+ if _is_fp8_linear(m):+ new_layer = convert_fp8_layer_to_linear(m, self.amp_dtype).to(device)+ set_module(block, n, new_layer)++ if self.device_map == "auto":+ self._set_auto_device_map_in_block(block, input_ids)++ if self.device_map is not None:+ for n, m in block.named_modules():+ if len(list(m.children())) != 0 or not hasattr(m, "tuning_device"):+ continue+ from accelerate.hooks import AlignDevicesHook, add_hook_to_module++ hook = AlignDevicesHook(m.tuning_device, io_same_device=True)+ add_hook_to_module(m, hook, True)++ if q_input is None:+ hook_handles = self._register_act_max_hook(block)++ output = self._get_block_outputs(+ block, input_ids, input_others, self.batch_size * self.infer_bs_coeff, device, self.cache_device+ )++ for handle in hook_handles:+ handle.remove()+ else:+ output = self._get_block_outputs(+ block, input_ids, input_others, self.batch_size * self.infer_bs_coeff, device, self.cache_device+ )+ hook_handles = self._register_act_max_hook(block)+ if hook_handles:+ self._get_block_outputs(+ block,+ q_input,+ input_others,+ self.batch_size * self.infer_bs_coeff,+ device,+ self.cache_device,+ save_output=False,+ )++ for handle in hook_handles:+ handle.remove()++ if q_input is not None:+ if input_ids is not q_input:+ clear_memory(input_ids)+ else:+ clear_memory()+ input_ids = q_input++ quantized_layer_names, unquantized_layer_names = wrapper_block(+ block, self.enable_minmax_tuning, self.enable_norm_bias_tuning, device=self.device+ )+ if is_nv_fp(self.data_type): # enable qkv and moe structure global_scale fuse+ from auto_round.data_type.utils import update_fused_layer_global_scales++ modules = block.modules()+ for module in modules:+ update_fused_layer_global_scales(module)+ round_params = []+ minmax_params = []+ for n, m in block.named_modules():+ if hasattr(m, "orig_layer"):+ for key in m.params.keys():+ if "min" in key or "max" in key:+ minmax_params.append(m.params[key])+ else:+ round_params.append(m.params[key])++ lr = torch.tensor(self.lr)+ minmax_lr = torch.tensor(self.minmax_lr)+ if self.enable_minmax_tuning:+ optimizer = self.optimizer(+ [{"params": round_params}, {"params": minmax_params, "lr": minmax_lr}], lr=lr, weight_decay=0+ )+ else:+ optimizer = self.optimizer(round_params, lr=lr, weight_decay=0)++ if len(round_params) + len(minmax_params) <= 0:+ dump_info = (+ f"quantized {len(quantized_layer_names)}/{(len(quantized_layer_names) + len(unquantized_layer_names))} "+ f"layers in the block"+ )+ logger.info(dump_info)+ unwrapper_block(block, {}) # TODO Quant layer should change+ mv_module_from_gpu(block, self.low_cpu_mem_usage)+ return output, output++ if self.lr_scheduler is None:+ lr_schedule = torch.optim.lr_scheduler.LinearLR(+ optimizer, start_factor=1.0, end_factor=0.0, total_iters=self.iters+ )+ else:+ lr_schedule = copy.deepcopy(self.lr_scheduler)++ nsamples = len(input_ids)+ pick_samples = self.batch_size * self.gradient_accumulate_steps+ pick_samples = min(nsamples, pick_samples)+ if self.sampler != "rand":+ whole_indices = torch.randperm(nsamples)[:pick_samples]+ last_best_iter = 0+ best_loss = torch.finfo(torch.float).max+ num_elm = 1+ mse_reduction = "mean"+ if self.gradient_accumulate_steps != 1:+ mse_reduction = "sum"+ mse_loss = torch.nn.MSELoss(reduction=mse_reduction).to(device)+ scaler = self._get_scaler() # pylint: disable=assignment-from-none+ init_loss = None+ best_params = {}+ total_loss = 0+ for i in range(self.iters):+ total_loss = 0+ if self.sampler == "rand":+ whole_indices = torch.randperm(nsamples)[:pick_samples]+ # We assume the block input and output shape is same+ if self.gradient_accumulate_steps != 1:+ current_input_ids = [input_ids[i] for i in whole_indices]+ num_elm = sum(id.numel() for id in current_input_ids)++ for tmp_step in range(self.gradient_accumulate_steps):+ indices = whole_indices[tmp_step * self.batch_size : (tmp_step + 1) * self.batch_size]+ current_input_ids, current_input_others = self._sampling_inputs(+ input_ids,+ input_others,+ indices,+ seqlen=self.seqlen,+ batch_dim=self.batch_dim,+ share_cache_keys=self.shared_cache_keys,+ )++ current_output = [output[x] for x in indices]+ current_output = torch.cat(current_output, dim=self.batch_dim)+ current_output = to_device(current_output, device)++ output_q = block_forward(+ block, current_input_ids, current_input_others, self.amp, self.amp_dtype, device+ )+ if self.amp:+ with autocast(device_type=device.split(":")[0], dtype=self.amp_dtype):+ loss = mse_loss(output_q, current_output) # pylint: disable=not-callable+ else:+ loss = mse_loss( # pylint: disable=not-callable+ output_q.to(torch.float32), current_output.to(torch.float32)+ )++ total_loss += loss.item() / num_elm+ self._scale_loss_and_backward(scaler, loss)++ if i == 0:+ init_loss = total_loss++ if total_loss < best_loss:+ best_loss = total_loss+ if not self.not_use_best_mse:+ best_params = collect_best_params(block)+ # print(f"get better result at iter {i}, the loss is {total_loss}", flush=True)++ last_best_iter = i+ if self.not_use_best_mse and i == self.iters - 1:+ best_params = collect_best_params(block)++ if not self.not_use_best_mse:+ if 0 < self.dynamic_max_gap <= i - last_best_iter:+ break+ self._step(scaler, optimizer, lr_schedule)++ last_loss = total_loss+ best_iter = self.iters+ if not self.not_use_best_mse:+ last_loss = best_loss+ best_iter = last_best_iter+ dump_info = (+ f"quantized {len(quantized_layer_names)}/{(len(quantized_layer_names) + len(unquantized_layer_names))} "+ f"layers in the block, loss iter 0: {init_loss:.6f} -> iter {best_iter}: {last_loss:.6f}"+ )+ logger.info(dump_info)+ if len(unquantized_layer_names) != 0:+ logger.info(f"{unquantized_layer_names} have not been quantized")+ with torch.no_grad():+ unwrapper_block(block, best_params)++ if (+ is_nv_fp(self.act_data_type)+ and hasattr(self, "formats")+ and any("nv_fp" in format_ for format_ in self.formats)+ ):+ # enable moe experts act_max automatic generation for WrapperWALayer+ set_amax_for_all_moe_layers(block, attr_name="orig_layer.act_max")++ if self.enable_quanted_input:+ if self.low_cpu_mem_usage:+ block = block.to(device)+ clear_memory()+ q_outputs = self._get_block_outputs(+ block,+ input_ids,+ input_others,+ self.batch_size * self.infer_bs_coeff,+ device,+ cache_device=self.cache_device,+ )+ if self.device_map is not None:+ accelerate.hooks.remove_hook_from_submodules(block)+ mv_module_from_gpu(block, self.low_cpu_mem_usage)+ clear_memory(input_ids)++ return q_outputs, output++ else:+ if self.device_map is not None:+ accelerate.hooks.remove_hook_from_submodules(block)+ mv_module_from_gpu(block, self.low_cpu_mem_usage)+ clear_memory(input_ids)+ return None, output++ def _quantize_blocks(+ self,+ model: torch.nn.Module,+ inputs: dict,+ block_names: list,+ q_input: torch.Tensor = None,+ nblocks: int = 1,+ device: str = "cpu",+ pbar: tqdm = None,+ ):+ """Quantize and dequantize the weights of the specified blocks in the model.++ Args:+ model: The PyTorch model to be quantized.+ inputs: The input data for quantization.+ block_names: The names of the blocks to be quantized and dequantized.+ nblocks: The number of blocks to quantize and dequantize.+ device: The device for quantization and dequantization.++ Returns:+ None+ """+ clear_memory()+ for n, m in model.named_parameters():+ m.requires_grad_(False)+ input_ids = inputs["input_ids"]+ inputs.pop("input_ids", None)+ input_others = inputs+ clear_memory()+ input_ids = to_device(input_ids, self.cache_device)+ input_others = to_device(input_others, self.cache_device)+ # As in calibration phase, we may use bf16 for calibration due to low_gpu_memory usage+ tmp_dtype = self.amp_dtype if self.amp else torch.float32+ for i in range(len(input_ids)):+ input_ids[i] = input_ids[i].to(tmp_dtype)++ for key in input_others.keys():+ if isinstance(input_others[key], torch.Tensor) and (+ input_others[key].dtype == torch.float16 or input_others[key].dtype == torch.bfloat16+ ):+ input_others[key] = input_others[key].to(tmp_dtype)+ elif isinstance(input_others[key], list):+ for i in range(len(input_others[key])):+ to_dtype(input_others[key][i], tmp_dtype)++ if (+ self.sym+ and self.enable_alg_ext+ and self.super_group_size is None+ and (+ (self.data_type.startswith("int") and self.act_bits >= 8)+ or self.data_type.startswith("mx")+ or self.data_type.startswith("nv")+ )+ ):+ try:+ from auto_round.alg_ext import quantize_block_ext++ BaseCompressor.quantize_block_ext = quantize_block_ext+ quantize_block = self.quantize_block_ext # must use self.quantize_block_ext+ if self.bits > 2 and (not self.data_type.startswith("mx") or not self.data_type.startswith("nv")):+ logger.warning(+ "algorithm extension has only undergone limited validation on "+ "INT2,mxfp4 and nvfp4; use with caution."+ )+ else:+ logger.info("using algorithm extension for quantization.")+ except (ImportError, ModuleNotFoundError):+ quantize_block = self._quantize_block+ if self.enable_torch_compile:+ quantize_block = compile_func(quantize_block, device)+ else:+ quantize_block = quantize_block+ else:+ quantize_block = self._quantize_block+ if self.enable_torch_compile:+ quantize_block = compile_func(quantize_block, device)++ if pbar is None:+ pbar = tqdm(range(0, len(block_names), nblocks))++ for i in range(0, len(block_names), nblocks):+ if i != 0:+ pbar.update(1)+ if nblocks == 1:+ n = block_names[i]+ pbar.set_description(f"Quantizing {n}")+ m = get_module(model, n)+ else:+ names = block_names[i : min(i + nblocks, len(block_names))]+ pbar.set_description(f"Quantizing [{i + 1}-{min(i + nblocks, len(block_names))}]/{len(block_names)}")+ modules = [get_module(model, n) for n in names]+ m = WrapperMultiblock(modules)++ if not self.model.device.type == "meta" or self.low_cpu_mem_usage:+ m = m.to(device)++ q_input, input_ids = quantize_block(+ m,+ input_ids,+ input_others,+ q_input=q_input,+ device=device,+ )+ if self.is_packing_immediate:+ from auto_round.export import PACKING_LAYER_WITH_FORMAT++ for _, tmp_m in m.named_modules():+ if not (hasattr(tmp_m, "bits") and check_to_quantized(tmp_m)):+ continue+ target_backend = self.formats[0].split(":")[0] if ":" in self.formats[0] else self.formats[0]+ has_gguf = any("gguf" in format_ for format_ in self.formats)+ if has_gguf:+ from auto_round.export.export_to_gguf.export import pack_gguf_layer++ output_dir = self._get_save_folder_name(self.formats[0])+ model_type = ModelType.MMPROJ if self.mllm else ModelType.TEXT+ pack_gguf_layer(+ tmp_m.tmp_name,+ self.model,+ self.formats[0],+ output_dir,+ self.layer_config,+ self.tokenizer,+ processor=self.processor if hasattr(self, "processor") else None,+ image_processor=self.image_processor if hasattr(self, "image_processor") else None,+ model_type=model_type,+ )+ else:+ PACKING_LAYER_WITH_FORMAT[target_backend](+ tmp_m.tmp_name, self.model, self.formats[0], device=self.device+ )+ pbar.set_description("Quantizing done")+ pbar.update(1)+ pbar.close()+ self.model = mv_module_from_gpu(self.model, self.low_cpu_mem_usage)+ for n, m in self.model.named_modules():+ if hasattr(m, "name"):+ delattr(m, "name")++ del q_input+ del input_ids+ del input_others+ del inputs++ clear_memory()++ def save_quantized(+ self, output_dir: str = None, format: str = "auto_round", inplace: bool = True, **kwargs+ ) -> torch.nn.Module:+ """Save the quantized model to the specified output directory in the specified format.++ Args:+ output_dir (str, optional): The directory to save the quantized model. Defaults to None.+ format (str, optional): The format in which to save the model. Defaults to "auto_round".+ inplace (bool, optional): Whether to modify the model in place. Defaults to True.+ **kwargs: Additional keyword arguments specific to the export format.++ Returns:+ object: The compressed model object.+ """+ format = self._check_supported_format(format)++ if self.low_cpu_mem_usage:+ self.model = self.model.to("cpu")++ if not self.quantized:+ logger.warning("please run autoround.quantize first")+ return+ if format == "fake" or format == "qdq": # TODO fix act quantization later+ self.model = self.model.to("cpu")+ self.model.save_pretrained(output_dir)+ if self.tokenizer is not None:+ self.tokenizer.save_pretrained(output_dir)+ processor = kwargs.get("processor", None)+ if processor is not None:+ processor.save_pretrained(output_dir)+ try:+ copy_python_files_from_model_cache(self.model, output_dir)+ except Exception as e:+ logger.warning("Skipping source model Python file copy due to error: %s", e)+ return+ if self.act_bits <= 8 and format == "qdq":+ logger.warning(+ "Support for exporting activation quantization is limited. "+ "Please ensure that your configuration is supported."+ )+ if format == "llm_compressor" and (is_nv_fp(self.data_type) or is_mx_fp(self.data_type)):+ format = format.replace("llm_compressor", f"llm_compressor:{self.data_type}")++ from auto_round.export import EXPORT_FORMAT++ backend = format+ format = format.split(":")[0]+ if format not in EXPORT_FORMAT:+ logger.error(f"export format only supports {EXPORT_FORMAT.keys()}")+ raise ValueError(f"export format only supports {EXPORT_FORMAT.keys()}, but got {format}")+ save_quantized_as_format = EXPORT_FORMAT.get(format)+ if "gptq" in format and not self.sym:+ logger.warning(+ "the asymmetrical kernel of the GPTQ format may result in a noticeable accuracy drop,"+ " particularly for 2-bit quantization and smaller models."+ " We recommend exporting to either the AutoAWQ format ( only 4 bits) or "+ "the AutoRound format(2/3/4/8 bits)."+ )+ if "awq" in format and not self.bits == 4:+ raise ValueError("The AWQ format only supports W4 quantization ")+ serialization_keys = [+ "bits",+ "group_size",+ "sym",+ "data_type",+ "enable_quanted_input",+ "enable_minmax_tuning",+ "seqlen",+ "batch_size",+ "scale_dtype",+ "lr",+ "minmax_lr",+ "gradient_accumulate_steps",+ "iters",+ "amp",+ "nsamples",+ "low_gpu_mem_usage",+ "to_quant_block_names",+ "enable_norm_bias_tuning",+ "act_bits",+ "act_group_size",+ "act_sym",+ "act_dynamic",+ "act_data_type",+ "super_bits",+ "super_group_size",+ ]+ if isinstance(self.dataset, str):+ serialization_keys.append("dataset")+ serialization_dict = {}+ for key in serialization_keys:+ serialization_dict[key] = getattr(self, key)+ from auto_round.version import __version__++ serialization_dict["autoround_version"] = __version__+ if "scale_dtype" in serialization_dict.keys():+ serialization_dict["scale_dtype"] = str(serialization_dict["scale_dtype"])++ compressed_model = save_quantized_as_format( # TODO refine the code+ output_dir,+ model=self.model,+ layer_config=self.layer_config,+ inplace=inplace,+ bits=self.bits,+ act_bits=self.act_bits,+ group_size=self.group_size,+ sym=self.sym,+ iters=self.iters,+ lr=self.lr,+ minmax_lr=self.minmax_lr,+ enable_minmax_tuning=self.enable_minmax_tuning,+ enable_quanted_input=self.enable_quanted_input,+ scale_dtype=self.scale_dtype,+ tokenizer=self.tokenizer,+ supported_types=self.supported_types,+ data_type=self.data_type,+ act_data_type=self.act_data_type,+ serialization_dict=serialization_dict,+ backend=backend,+ to_quant_block_names=self.to_quant_block_names,+ quant_block_list=self.quant_block_list,+ device=self.device,+ **kwargs,+ )+ return compressed_model++ def _get_quantized_layer_names_outside_blocks(self) -> list:+ """Gets the names of quantized layers outside blocks in the model.++ Returns:+ list: List of layer names outside blocks.+ """+ if self.layer_config is None or len(self.layer_config) == 0:+ return []++ layer_names = []+ all_layers_in_block = get_layer_names_in_block(self.model, self.supported_types, self.quant_block_list)++ for key in self.layer_config.keys():+ if key in all_layers_in_block:+ continue+ layer = get_module(self.model, key)+ if layer is None:+ logger.error(f"could not find layer {key} in the model, exit...")+ exit(-1)+ if isinstance(layer, tuple(self.supported_types)) and check_to_quantized(self.layer_config[key]):+ layer_names.append(key)++ return layer_names++ def _set_amp_dtype(self) -> None:+ """Sets the automatic mixed precision (AMP) data type for the model based on the device and configuration."""+ self.amp_dtype = torch.bfloat16+ if self.model.dtype != torch.float32:+ self.amp_dtype = self.model.dtype+ if self.device == "cpu" or "hpu" in self.device:+ self.amp_dtype = torch.bfloat16+ if self.amp:+ if self.device == "cpu" and not CpuInfo().bf16:+ self.amp = False+ self.amp_dtype = torch.float32+ self.model = self.model.to(torch.float32)+ logger.warning(+ f"amp is set to FALSE as the current {self.device} device does not support the 'bf16' data type."+ )+ else:+ if self.model.dtype != self.amp_dtype:+ self.model = self.model.to(self.amp_dtype)+ else:+ self.amp_dtype = torch.float32+ self.model = self.model.to(torch.float32)++ def _get_optimizer(self, optimizer: Any):+ """Returns the specified optimizer. In SignRound, we fix the optimizer.++ Args:+ optimizer: The optimizer to be used.++ Returns:+ The specified optimizer.+ """+ return SignSGD++ def _get_scaler(self):+ """Returns scaler, in SignRound, no need to use scaler."""+ return None++ def _scale_loss_and_backward(self, scaler: Any, loss: torch.Tensor) -> torch.Tensor:+ """Scales the loss and performs backward pass.++ Args:+ scaler: The scaler to be used.+ loss: The loss to be scaled.++ Returns:+ The scaled loss.+ """+ scale_loss = loss * 1000+ scale_loss.backward()+ if is_optimum_habana_available():+ htcore.mark_step()+ return scale_loss++ def _step(self, scaler: Any, optimizer: Any, lr_schedule: Any):+ """Performs a step in the optimization process.++ Args:+ scaler: The scaler to be used.+ optimizer: The optimizer for the step.+ lr_schedule: The learning rate schedule.++ Returns:+ None+ """+ optimizer.step()+ # for hpu+ if is_optimum_habana_available():+ htcore.mark_step()+ optimizer.zero_grad()+ lr_schedule.step()++ @classmethod+ @torch.no_grad()+ def _sampling_inputs(+ cls,+ input_ids: list[torch.Tensor],+ input_others: dict,+ indices: list[int],+ seqlen: int,+ batch_dim: int = 0,+ share_cache_keys: tuple = (),+ ):+ """Samples inputs based on the given indices and sequence length.++ Args:+ input_ids: The list of input tensor containing input_ids.+ input_others: A dictionary containing other input data.+ indices: The indices to sample from the input.+ seqlen: The sequence length.++ Returns:+ current_input_ids: The sampled input IDs.+ current_input_others: The sampled other input data.+ """+ current_input_ids = [input_ids[i] for i in indices]++ current_input_ids = torch.cat(current_input_ids, dim=batch_dim)++ current_input_others = {"positional_inputs": input_others["positional_inputs"]}+ for key in input_others.keys():+ if "positional_inputs" in key:+ continue+ if (key not in share_cache_keys or len(indices) == 1) and not isinstance(+ input_others[key], (str, bool, type(None))+ ):+ current_input_others[key] = None+ if input_others[key] is not None:+ current_input_others[key] = [input_others[key][i] for i in indices]+ if len(indices) == 1:+ current_input_others[key] = current_input_others[key][0]+ else:+ try:+ current_input_others[key] = torch.cat(current_input_others[key], dim=0)+ except TypeError as err:+ logger.warning_once("Please check the model cache inputs or try setting batch_size to 1.")+ else:+ current_input_others[key] = input_others[key]++ return current_input_ids, current_input_others+++class LLMCompressor(BaseCompressor):+ pass+++class AdamCompressor(BaseCompressor):+ """Class for quantization with optimizers like adamw of a PyTorch model.++ Args:+ model: The PyTorch model to be quantized.+ tokenizer: An optional tokenizer for processing input data.+ scheme (str| dict | QuantizationScheme ): A preset scheme that defines the quantization configurations+ bits (int): Number of bits for quantization (default is 4).+ group_size (int): Size of the quantization group (default is 128).+ sym (bool): Whether sym to be used (default is True).+ layer_config (dict): Configuration for weight quantization (default is None).+ batch_size (int): Batch size for training (default is 8).+ amp (bool): Whether to use automatic mixed precision (default is True).+ device: The device to be used for training (default is "auto").+ lr_scheduler: The learning rate scheduler to be used.+ dataset: The default dataset name (default is "NeelNanda/pile-10k").+ enable_quanted_input (bool): Whether to use quantized input data (default is True).+ enable_minmax_tuning (bool): Whether to enable min-max tuning (default is True).+ lr (float): The learning rate (default is 0.005).+ minmax_lr (float): The learning rate for min-max tuning (default is None).+ low_gpu_mem_usage (bool): Whether to use low GPU memory (default is False).+ low_cpu_mem_usage (bool): Whether to use low CPU memory (default is False).+ iters (int): Number of iterations (default is 200).+ seqlen (int): Length of the sequence.+ nsamples (int): Number of samples (default is 128).+ sampler (str): The sampling method (default is "rand").+ seed (int): The random seed (default is 42).+ nblocks (int): Number of blocks (default is 1).+ gradient_accumulate_steps (int): Number of gradient accumulation steps (default is 1).+ not_use_best_mse (bool): Whether to use mean squared error (default is False).+ dynamic_max_gap (int): The dynamic maximum gap (default is -1).+ data_type (str): The data type to be used (default is "int").+ scale_dtype (str): The data type of quantization scale to be used (default is "float16"), different kernels+ have different choices.+ act_bits (int): Number of bits for activation quantization. Default is 16.+ act_group_size (int): Group size for activation quantization. Default is None.+ act_sym (bool): Whether to use symmetric activation quantization. Default is None.+ act_data_type (str): Specifies the data type for activations.+ Defaults to None, in which case it inherits the weight data type.+ act_dynamic (bool): Whether to use dynamic activation quantization. Default is True.+ to_quant_block_names (str|list): A string or list whose elements are list of+ block's layer names to be quantized.+ enable_norm_bias_tuning (bool): Whether to enable fast norm/layer_bias tuning+ enable_torch_compile (bool): Whether to enable torch compile to optimize quant_block/layer function+ **kwargs: Additional keyword arguments.++ Returns:+ The quantized model.+ """++ bits: int | None+ group_size: int | None+ sym: bool | None+ data_type: str | None+ act_bits: int | None+ act_group_size: int | None+ act_sym: bool | None+ act_data_type: str | None+ act_dynamic: bool | None+ super_bits: int | None+ super_group_size: int | None++ def __init__(+ self,+ model: Union[torch.nn.Module, str],+ tokenizer=None,+ scheme: Union[str, dict, QuantizationScheme] = "W4A16",+ layer_config: dict[str, Union[str, dict, QuantizationScheme]] = None,+ dataset: Union[str, list, tuple, torch.utils.data.DataLoader] = "NeelNanda/pile-10k",+ iters: int = 200,+ seqlen: int = 2048,+ nsamples: int = 128,+ batch_size: int = 8,+ gradient_accumulate_steps: int = 1,+ low_gpu_mem_usage: bool = False,+ device_map: Union[str, int, torch.device, dict] = 0,+ enable_torch_compile: bool = False,+ seed: int = 42,+ optimizer="AdamW",+ **kwargs,+ ):+ super(AdamCompressor, self).__init__(+ model=model,+ tokenizer=tokenizer,+ scheme=scheme,+ layer_config=layer_config,+ batch_size=batch_size,+ dataset=dataset,+ low_gpu_mem_usage=low_gpu_mem_usage,+ iters=iters,+ seqlen=seqlen,+ nsamples=nsamples,+ seed=seed,+ gradient_accumulate_steps=gradient_accumulate_steps,+ enable_torch_compile=enable_torch_compile,+ device_map=device_map,+ **kwargs,+ )++ self.optimizer = self._get_optimizer(optimizer)++ def _get_optimizer(self, optimizer):+ if optimizer is None:+ optimizer = torch.optim.AdamW+ elif isinstance(optimizer, str):+ optimizer = getattr(torch.optim, optimizer)+ else:+ optimizer = optimizer+ return optimizer++ def _get_scaler(self):+ scaler = None+ if self.amp and not check_is_cpu(self.device):+ from torch.cuda.amp import GradScaler++ scaler = GradScaler(init_scale=1024, growth_interval=100000)+ return scaler++ def _scale_loss_and_backward(self, scaler, loss):+ if scaler is not None:+ loss = scaler.scale(loss)++ loss.backward()+ if is_optimum_habana_available():+ htcore.mark_step()+ return loss++ def _step(self, scaler, optimizer, lr_schedule):+ if scaler is not None:+ scaler.step(optimizer)+ optimizer.zero_grad()+ lr_schedule.step()+ scaler.update()+ else:+ optimizer.step()+ optimizer.zero_grad()+ lr_schedule.step()+ if is_optimum_habana_available():+ htcore.mark_step()diff --git a/auto_round/compressors/config.py b/auto_round/compressors/config.pynew file mode 100644index 000000000..cc9081ce9--- /dev/null+++ b/auto_round/compressors/config.py@@ -0,0 +1,262 @@+# Copyright (c) 2025 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+from __future__ import annotations++from dataclasses import dataclass, fields+from typing import Any, Callable, Union++import torch+++class ExtraConfig:+ """Class for extra or legacy configs."""++ _model_config = None+ _scheme_config = None+ _tuning_config = None+ _mllm_config = None++ def __init__(+ self,+ # tuning+ amp: bool = True,+ disable_opt_rtn: bool = True,+ enable_alg_ext: bool = False,+ enable_minmax_tuning: bool = True,+ enable_norm_bias_tuning: bool = False,+ enable_quanted_input: bool = True,+ enable_deterministic_algorithms: bool = False,+ lr: float = None,+ lr_scheduler: Callable = None,+ minmax_lr: float = None,+ mem_per_param_scale: int = None,+ nblocks: int = 1,+ quant_lm_head: bool = False,+ to_quant_block_names: Union[str, list, None] = None,+ scale_dtype: str = "fp16",+ # scheme+ bits: int = None,+ group_size: int = None,+ sym: bool = None,+ data_type: str = None,+ act_bits: int = None,+ act_group_size: int = None,+ act_sym: bool = None,+ act_data_type: str = None,+ act_dynamic: bool = None,+ super_bits: int = None,+ super_group_size: int = None,+ static_kv_dtype: Union[str, torch.dtype] = None,+ # mllm+ processor: Callable = None,+ image_processor: Callable = None,+ quant_nontext_module: bool = False,+ extra_data_dir: str = None,+ template: str = None,+ ):+ """Initialize++ Args:+ amp (bool): Whether to use automatic mixed precision (default is True).+ disable_opt_rtn (bool, optional): Disable RTN-mode optimization (iters=0). Defaults to False.+ enable_alg_ext (bool, optional): Enable algorithm extension (primarily for INT2). Defaults to False.+ enable_minmax_tuning (bool, optional): Enable weight min-max tuning. Defaults to True.+ enable_norm_bias_tuning (bool): Whether to enable fast norm/layer_bias tuning.+ enable_quanted_input (bool): Whether to use quantized input data (default is True).+ enable_deterministic_algorithms (bool): Whether to use deterministic_algorithms.+ lr (float): The learning rate (default is 0.005).+ lr_scheduler: The learning rate scheduler to be used.+ minmax_lr (float): The learning rate for min-max tuning (default is None).+ mem_per_param_scale (int): Scale factor for memory per parameter,+ used to adjust memory usage estimation for tuning.+ nblocks (int): Number of blocks (default is 1).+ quant_lm_head (bool): Whether to quant lm_head.+ to_quant_block_names (str|list): Names of quantitative blocks, please use commas to separate them.+ scale_dtype (str): The data type of quantization scale to be used (default is "float16"), different kernels+ bits (int, optional): Weight quantization bits. Defaults to 4.+ group_size (int, optional): Weight quantization group size. Defaults to 128.+ sym (bool, optional): Symmetric weight quantization. Defaults to True.+ data_type (str, optional): Weight data type string, e.g., "int". Defaults to "int".+ act_bits (int, optional): Activation quantization bits. Defaults to 16.+ act_group_size (int, optional): Activation group size. Defaults to None.+ act_sym (bool, optional): Symmetric activation quantization. Defaults to None.+ act_data_type (str, optional): Activation data type; inherits weight dtype if None and act_bits < 16.+ act_dynamic (bool, optional): Dynamic activation quantization. Defaults to True.+ super_bits (int): number of scale and mins quant bits for double quant.+ super_group_size (int): the number of super group size when use double quant.+ static_kv_dtype (str): The data type of kv-cache to be used.+ processor: Any multi-modal model will require an object to encode or+ decode the data that groups several modalities (among text, vision and audio).+ image_processor: Image processor for special model like llava.+ quant_nontext_module: Whether to quantize nontext module.+ extra_data_dir: The path of extra data such as images, audio and videos.+ template: The path or name of template used to specify process for different MLLMs.+ """+ self.tuning_config = TuningExtraConfig(+ amp=amp,+ disable_opt_rtn=disable_opt_rtn,+ enable_alg_ext=enable_alg_ext,+ enable_minmax_tuning=enable_minmax_tuning,+ enable_norm_bias_tuning=enable_norm_bias_tuning,+ enable_quanted_input=enable_quanted_input,+ enable_deterministic_algorithms=enable_deterministic_algorithms,+ lr=lr,+ lr_scheduler=lr_scheduler,+ minmax_lr=minmax_lr,+ mem_per_param_scale=mem_per_param_scale,+ nblocks=nblocks,+ quant_lm_head=quant_lm_head,+ to_quant_block_names=to_quant_block_names,+ scale_dtype=scale_dtype,+ )+ self.scheme_config = SchemeExtraConfig(+ bits=bits,+ group_size=group_size,+ sym=sym,+ data_type=data_type,+ act_bits=act_bits,+ act_group_size=act_group_size,+ act_sym=act_sym,+ act_data_type=act_data_type,+ act_dynamic=act_dynamic,+ super_bits=super_bits,+ super_group_size=super_group_size,+ static_kv_dtype=static_kv_dtype,+ )+ self.mllm_config = MLLMExtraConfig(+ processor=processor,+ image_processor=image_processor,+ quant_nontext_module=quant_nontext_module,+ extra_data_dir=extra_data_dir,+ template=template,+ )++ @property+ def tuning_config(self):+ return self._tuning_config++ @tuning_config.setter+ def tuning_config(self, config: TuningExtraConfig):+ assert isinstance(+ config, TuningExtraConfig+ ), f"tuning_config should be ModelExtraConfig, but got {config.__class__.__name__}"+ self._tuning_config = config++ @property+ def scheme_config(self):+ return self._scheme_config++ @scheme_config.setter+ def scheme_config(self, config: SchemeExtraConfig):+ assert isinstance(+ config, SchemeExtraConfig+ ), f"scheme_config should be SchemeExtraConfig, but got {config.__class__.__name__}"+ self._scheme_config = config++ @property+ def mllm_config(self):+ return self._mllm_config++ @mllm_config.setter+ def mllm_config(self, config: MLLMExtraConfig):+ if config is None:+ self._mllm_config = None+ else:+ assert isinstance(+ config, MLLMExtraConfig+ ), f"mllm_config should be MLLMExtraConfig, but got {config.__class__.__name__}"+ self._mllm_config = config++ def to_dict(self):+ output_dict = {}+ for config in self.__dict__.values():+ if config:+ output_dict.update(config.to_dict())+ return output_dict+++@dataclass+class BaseExtraConfig:++ @classmethod+ def get_attributes(cls: "BaseExtraConfig") -> list[str]:+ return [field.name for field in fields(cls)]++ def __getitem__(self, key: str):+ if key not in self.get_attributes():+ raise KeyError(f"{key} is not a valid attribute")+ return getattr(self, key)++ def __setitem__(self, key: str, value: None | int | str):+ if key not in self.get_attributes():+ raise KeyError(f"{key} is not a valid attribute")+ setattr(self, key, value)++ def __contains__(self, item):+ return item in self.get_attributes()++ def to_dict(self):+ return self.__dict__++ def is_default(self):+ for field in fields(self):+ default_value = field.default+ current_value = getattr(self, field.name)+ if current_value != default_value:+ return False+ return True+++@dataclass+class TuningExtraConfig(BaseExtraConfig):+ amp: bool = True+ disable_opt_rtn: bool = True+ enable_alg_ext: bool = False+ enable_minmax_tuning: bool = True+ enable_norm_bias_tuning: bool = False+ enable_quanted_input: bool = True+ enable_deterministic_algorithms: bool = False+ lr: float = None+ lr_scheduler: Callable = None+ minmax_lr: float = None+ mem_per_param_scale: int = None+ nblocks: int = 1+ quant_lm_head: bool = False+ to_quant_block_names: Union[str, list, None] = None+ scale_dtype: str = "fp16"+++@dataclass+class SchemeExtraConfig(BaseExtraConfig):+ bits: int = None+ group_size: int = None+ sym: bool = None+ data_type: str = None+ act_bits: int = None+ act_group_size: int = None+ act_sym: bool = None+ act_data_type: str = None+ act_dynamic: bool = None+ super_bits: int = None+ super_group_size: int = None+ static_kv_dtype: Union[str, torch.dtype] = None+++@dataclass+class MLLMExtraConfig(BaseExtraConfig):+ processor: Callable = None+ image_processor: Callable = None+ quant_nontext_module: bool = False+ extra_data_dir: str = None+ template: str = Nonediff --git a/auto_round/mllm/README.md b/auto_round/compressors/mllm/README.mdsimilarity index 100%rename from auto_round/mllm/README.mdrename to auto_round/compressors/mllm/README.mddiff --git a/auto_round/mllm/__init__.py b/auto_round/compressors/mllm/__init__.pysimilarity index 67%rename from auto_round/mllm/__init__.pyrename to auto_round/compressors/mllm/__init__.pyindex d77883b7e..e92f378d4 100644--- a/auto_round/mllm/__init__.py+++ b/auto_round/compressors/mllm/__init__.py@@ -12,8 +12,7 @@# See the License for the specific language governing permissions and# limitations under the License.-from .mllm_dataset import get_mllm_dataloader-from .template import Template, get_template, TEMPLATES-from .autoround_mllm import AutoRoundMLLM-from ..utils import LazyImport-from .eval import mllm_eval, lmms_eval+from auto_round.compressors.mllm.dataset import get_mllm_dataloader+from auto_round.compressors.mllm.template import Template, get_template, TEMPLATES+from auto_round.compressors.mllm.compressor import MLLMCompressor+from auto_round.compressors.mllm.eval import mllm_eval, lmms_evaldiff --git a/auto_round/mllm/autoround_mllm.py b/auto_round/compressors/mllm/compressor.pysimilarity index 97%rename from auto_round/mllm/autoround_mllm.pyrename to auto_round/compressors/mllm/compressor.pyindex f6be55851..63199b081 100644--- a/auto_round/mllm/autoround_mllm.py+++ b/auto_round/compressors/mllm/compressor.py@@ -18,11 +18,11 @@ from typing import Unionimport torchfrom tqdm import tqdm-from auto_round.autoround import AutoRound+from auto_round.compressors.base import BaseCompressor+from auto_round.compressors.mllm.dataset import get_mllm_dataloader+from auto_round.compressors.mllm.template import Template, get_templatefrom auto_round.logger import loggerfrom auto_round.low_cpu_mem.utils import get_layers_before_block-from auto_round.mllm.mllm_dataset import get_mllm_dataloader-from auto_round.mllm.template import Template, get_templatefrom auto_round.schemes import QuantizationSchemefrom auto_round.special_model_handler import (NOT_SUPPORT_ONLY_TEXT_MODELS,@@ -76,7 +76,7 @@ def _only_text_test(model, tokenizer, device, model_type):return False-class AutoRoundMLLM(AutoRound):+class MLLMCompressor(BaseCompressor):"""Class for automatic rounding-based quantization with MLLMs.Args:@@ -123,8 +123,6 @@ class AutoRoundMLLM(AutoRound):block's layer names to be quantized.enable_torch_compile (bool): Whether to enable torch compile to optimize quant_block/layer**kwargs: Additional keyword arguments.--"""bits: int | None@@ -205,8 +203,9 @@ class AutoRoundMLLM(AutoRound):model = _handle_special_model(model)- from ..calib_dataset import CALIB_DATASETS- from .mllm_dataset import MLLM_DATASET+ from auto_round.calib_dataset import CALIB_DATASETS++ from .dataset import MLLM_DATASETif iters > 0 and isinstance(dataset, str) and dataset in CALIB_DATASETS.keys():if quant_nontext_module:@@ -251,7 +250,8 @@ class AutoRoundMLLM(AutoRound):nsamples = (nsamples // batch_size + 1) * batch_sizelogger.warning(f"'nsamples' is not divisible by 'batch_size', will adjusted to {nsamples}")- super(AutoRoundMLLM, self).__init__(+ kwargs["mllm"] = True+ super(MLLMCompressor, self).__init__(model=model,tokenizer=tokenizer,scheme=scheme,@@ -266,7 +266,6 @@ class AutoRoundMLLM(AutoRound):device_map=device_map,enable_torch_compile=enable_torch_compile,seed=seed,- vlm=True,to_quant_block_names=to_quant_block_names,**kwargs,)@@ -435,7 +434,7 @@ class AutoRoundMLLM(AutoRound):def _check_quant_nontext(self, layer_config, quant_nontext_module):if not layer_config:return quant_nontext_module- from auto_round.mllm.utils import VISUAL_KEYS+ from auto_round.compressors.mllm.utils import VISUAL_KEYSfor layer_name in layer_config.keys():for vlm_key in VISUAL_KEYS:diff --git a/auto_round/mllm/mllm_dataset.py b/auto_round/compressors/mllm/dataset.pysimilarity index 98%rename from auto_round/mllm/mllm_dataset.pyrename to auto_round/compressors/mllm/dataset.pyindex 267b8eb4a..21da6ba25 100644--- a/auto_round/mllm/mllm_dataset.py+++ b/auto_round/compressors/mllm/dataset.py@@ -20,8 +20,9 @@ import torchfrom torch.utils.data import DataLoader, Datasetfrom transformers import set_seed-from ..special_model_handler import check_mllm_model_batch-from ..utils import logger+from auto_round.logger import logger+from auto_round.special_model_handler import check_mllm_model_batch+from .template import Templatefrom .utils import _extract_data_dir@@ -246,7 +247,7 @@ def get_mllm_dataloader(return DataLoader(dataset, **dataloader_params), bs, gradient_accumulate_stepselse:# try to load text calibration dataset- from ..calib_dataset import get_dataloader+ from auto_round.calib_dataset import get_dataloaderdataloader = get_dataloader(tokenizer, seqlen, dataset, seed, bs, nsamples)if quant_nontext_module:diff --git a/auto_round/mllm/eval.py b/auto_round/compressors/mllm/eval.pysimilarity index 99%rename from auto_round/mllm/eval.pyrename to auto_round/compressors/mllm/eval.pyindex e08c88ebd..d9e7bdeaf 100644--- a/auto_round/mllm/eval.py+++ b/auto_round/compressors/mllm/eval.py@@ -53,7 +53,8 @@ from functools import partialimport numpy as np-from auto_round.utils import LazyImport, logger+from auto_round.logger import logger+from auto_round.utils import LazyImportvlmeval = LazyImport("vlmeval")diff --git a/auto_round/mllm/processor.py b/auto_round/compressors/mllm/processor.pysimilarity index 99%rename from auto_round/mllm/processor.pyrename to auto_round/compressors/mllm/processor.pyindex 62e76e488..9d98be1e9 100644--- a/auto_round/mllm/processor.py+++ b/auto_round/compressors/mllm/processor.py@@ -262,7 +262,7 @@ class CogVLM2Processor(BasicProcessor):return fetch_image(image_path_or_url).convert("RGB")-from ..utils import LazyImport+from auto_round.utils import LazyImportllava_train = LazyImport("llava.train.train")diff --git a/auto_round/mllm/template.py b/auto_round/compressors/mllm/template.pysimilarity index 99%rename from auto_round/mllm/template.pyrename to auto_round/compressors/mllm/template.pyindex 7f4531e07..b3f9802ba 100644--- a/auto_round/mllm/template.py+++ b/auto_round/compressors/mllm/template.py@@ -18,7 +18,8 @@ from dataclasses import dataclassfrom enum import Enum, uniquefrom typing import Dict, List, Optional-from ..utils import logger+from auto_round.logger import logger+from .processor import PROCESSORS, BasicProcessorTEMPLATES: Dict[str, "Template"] = {}diff --git a/auto_round/mllm/templates/cogvlm2.json b/auto_round/compressors/mllm/templates/cogvlm2.jsonsimilarity index 100%rename from auto_round/mllm/templates/cogvlm2.jsonrename to auto_round/compressors/mllm/templates/cogvlm2.jsondiff --git a/auto_round/mllm/templates/default.json b/auto_round/compressors/mllm/templates/default.jsonsimilarity index 100%rename from auto_round/mllm/templates/default.jsonrename to auto_round/compressors/mllm/templates/default.jsondiff --git a/auto_round/mllm/templates/llava.json b/auto_round/compressors/mllm/templates/llava.jsonsimilarity index 100%rename from auto_round/mllm/templates/llava.jsonrename to auto_round/compressors/mllm/templates/llava.jsondiff --git a/auto_round/mllm/templates/phi3_v.json b/auto_round/compressors/mllm/templates/phi3_v.jsonsimilarity index 100%rename from auto_round/mllm/templates/phi3_v.jsonrename to auto_round/compressors/mllm/templates/phi3_v.jsondiff --git a/auto_round/mllm/utils.py b/auto_round/compressors/mllm/utils.pysimilarity index 97%rename from auto_round/mllm/utils.pyrename to auto_round/compressors/mllm/utils.pyindex f9a012627..39256f704 100644--- a/auto_round/mllm/utils.py+++ b/auto_round/compressors/mllm/utils.py@@ -16,7 +16,7 @@ import osimport requests-from ..utils import LazyImport+from auto_round.utils import LazyImportPIL = LazyImport("PIL")from PIL import Image # pylint: disable=E0401diff --git a/auto_round/eval/eval_cli.py b/auto_round/eval/eval_cli.pynew file mode 100644index 000000000..82164b5a6--- /dev/null+++ b/auto_round/eval/eval_cli.py@@ -0,0 +1,331 @@+# Copyright (c) 2025 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+import argparse+import os++from auto_round.utils import (+ clear_memory,+ get_device_and_parallelism,+ get_model_dtype,+ set_cuda_visible_devices,+)+++class EvalArgumentParser(argparse.ArgumentParser):++ def __init__(self, *args, **kwargs):+ super().__init__(*args, **kwargs)+ self.add_argument(+ "--model", "--model_name", "--model_name_or_path", default="facebook/opt-125m", help="model name or path"+ )+ self.add_argument("--mllm", action="store_true", help="whether to eval multi-modal model.")+ self.add_argument(+ "--device_map",+ "--device",+ "--devices",+ default="0",+ type=str,+ help="the device to be used for tuning. "+ "Currently, device settings support CPU, GPU, and HPU."+ "The default is set to cuda:0,"+ "allowing for automatic detection and switch to HPU or CPU."+ "set --device 0,1,2 to use multiple cards.",+ )++ self.add_argument(+ "--tasks",+ "--task",+ default="lambada_openai,hellaswag,winogrande,piqa,mmlu,wikitext,truthfulqa_mc1,"+ "truthfulqa_mc2,openbookqa,boolq,rte,arc_easy,arc_challenge",+ help="lm-eval tasks",+ )+ self.add_argument(+ "--disable_trust_remote_code", action="store_true", help="whether to disable trust_remote_code"+ )+ self.add_argument("--eval_bs", "--bs", "--batch_size", default=None, type=int, help="batch size in evaluation")+ self.add_argument("--eval_task_by_task", action="store_true", help="whether to eval task by task.")+ self.add_argument(+ "--eval_model_dtype", default=None, type=str, help="the torch_dytpe to load the model for evaluation."+ )+ self.add_argument(+ "--limit",+ type=float,+ default=None,+ metavar="N|0<N<1",+ help="Limit the number of examples per task. "+ "If <1, limit is a percentage of the total number of examples.",+ )+ # vllm related arguments+ self.add_argument("--revision", default=None, type=str, help="model revision for vllm")+ self.add_argument("--tokenizer", default=None, type=str, help="tokenizer to use with vllm")+ self.add_argument(+ "--tokenizer_mode", default="auto", type=str, help="tokenizer mode for vllm (e.g. auto/fast/slow)"+ )+ self.add_argument("--tokenizer_revision", default=None, type=str, help="tokenizer revision for vllm")+ self.add_argument("--add_bos_token", action="store_true", help="add BOS token when using vllm")+ self.add_argument("--prefix_token_id", default=None, type=int, help="prefix token id for vllm")+ self.add_argument("--tensor_parallel_size", default=1, type=int, help="tensor parallel size for vllm")+ self.add_argument("--data_parallel_size", default=1, type=int, help="data parallel size for vllm")+ self.add_argument("--quantization", default=None, type=str, help="quantization setting for vllm")+ self.add_argument("--max_gen_toks", default=256, type=int, help="max generation tokens for vllm")+ self.add_argument("--swap_space", default=4, type=float, help="swap space (GB) for vllm")+ self.add_argument("--max_batch_size", default=None, type=int, help="max batch size for vllm")+ self.add_argument("--max_length", default=None, type=int, help="max generation length for vllm")+ self.add_argument("--max_model_len", default=None, type=int, help="maximum model sequence length for vllm")+ self.add_argument(+ "--gpu_memory_utilization", default=0.9, type=float, help="target GPU memory utilization for vllm"+ )+ self.add_argument("--lora_local_path", default=None, type=str, help="local LoRA path for vllm")+++def _eval_init(tasks, model_path, device, disable_trust_remote_code=False, dtype="auto"):+ set_cuda_visible_devices(device)+ device_str, parallelism = get_device_and_parallelism(device)+ if dtype != "auto":+ dtype = get_model_dtype(model_dtype=dtype)+ # ,add_bos_token={True}+ model_args = f"pretrained={model_path},trust_remote_code={not disable_trust_remote_code},dtype={dtype}"+ if parallelism:+ model_args += ",parallelize=True"+ if isinstance(tasks, str):+ tasks = tasks.split(",")+ return tasks, model_args, device_str+++def eval(args):+ import time++ tasks, model_args, device_str = _eval_init(+ args.tasks, args.model, args.device_map, args.disable_trust_remote_code, args.eval_model_dtype+ )++ # load after _eval_int in order to make sure import torch after set CUDA_VISIBLE_DEVICES+ from auto_round.eval.evaluation import simple_evaluate, simple_evaluate_user_model+ from auto_round.utils import logger++ if (batch_size := args.eval_bs) is None:+ batch_size = "auto:8"+ is_gguf_file = False+ if os.path.exists(args.model):+ if os.path.isfile(args.model) and args.model.endswith(".gguf"):+ is_gguf_file = True+ gguf_file = os.path.basename(args.model)+ model = os.path.dirname(args.model)+ else:+ for file in os.listdir(args.model):+ if file.endswith(".gguf"):+ is_gguf_file = True+ gguf_file = file+ eval_model_dtype = get_model_dtype(args.eval_model_dtype)+ if is_gguf_file:+ import torch+ from lm_eval.utils import make_table # pylint: disable=E0401+ from transformers import AutoModelForCausalLM, AutoTokenizer++ tokenizer = AutoTokenizer.from_pretrained(model, gguf_file=gguf_file)++ logger.warning("evaluating gguf model is an experimental feature, the accuracy may be not correct.")+ if eval_model_dtype == "float32" or eval_model_dtype == "auto":+ logger.warning(+ "set '--eval_model_dtype bf16' can significantly speed up evaluation for gguf model,"+ " but may affect accuracy."+ )+ model = AutoModelForCausalLM.from_pretrained(+ model, gguf_file=gguf_file, device_map="auto", torch_dtype=eval_model_dtype+ )+ model.eval()+ st = time.time()+ res = simple_evaluate_user_model(+ model, tokenizer, tasks=tasks, batch_size=batch_size, device=device_str, limit=args.limit+ )+ print(make_table(res))+ print("evaluation running time=%ds" % (time.time() - st))+ else:+ st = time.time()+ if "auto" in str(batch_size) and args.mllm:+ logger.warning("Batch size 'auto' is not yet supported for hf-multimodal models, reset to 16")+ batch_size = 16+ res = simple_evaluate(+ model="hf" if not args.mllm else "hf-multimodal",+ model_args=model_args,+ tasks=tasks,+ device=device_str,+ batch_size=batch_size,+ limit=args.limit,+ )+ from lm_eval.utils import make_table # pylint: disable=E0401++ print(make_table(res))+ print("evaluation running time=%ds" % (time.time() - st))+++def eval_task_by_task(+ model,+ device=None,+ tasks=None,+ tokenizer=None,+ batch_size=None,+ limit=None,+ max_batch_size=64,+ trust_remote_code=True,+ eval_model_dtype=None,+ retry_times=3,+):+ set_cuda_visible_devices(device)+ device_str, parallelism = get_device_and_parallelism(device)++ # load after _eval_int in order to make sure import torch after set CUDA_VISIBLE_DEVICES+ import traceback++ from lm_eval import simple_evaluate as lm_simple_evaluate # pylint: disable=E0611+ from lm_eval.models.huggingface import HFLM+ from transformers import AutoModelForCausalLM, AutoTokenizer++ from auto_round.utils import logger++ if batch_size is None:+ batch_size = "auto:8"+ is_gguf_file = False+ if not isinstance(model, str):+ parallelism = False+ else:+ if os.path.isfile(model) and model.endswith(".gguf"):+ is_gguf_file = True+ gguf_file = os.path.basename(model)+ model = os.path.dirname(model)+ else:+ for file in os.listdir(model):+ if file.endswith(".gguf"):+ is_gguf_file = True+ gguf_file = file+ eval_model_dtype = get_model_dtype(eval_model_dtype)+ if is_gguf_file:+ tokenizer = AutoTokenizer.from_pretrained(model, gguf_file=gguf_file)+ logger.warning("evaluating gguf model is an experimental feature, the accuracy may be not correct.")+ if eval_model_dtype == "float32" or eval_model_dtype == "auto":+ logger.warning(+ "set '--eval_model_dtype bf16' can significantly speed up evaluation for gguf model,"+ " but may affect accuracy."+ )++ model = AutoModelForCausalLM.from_pretrained(+ model, gguf_file=gguf_file, device_map="auto", torch_dtype=eval_model_dtype+ )+ model.eval()+ parallelism = False+ hflm = HFLM(+ pretrained=model,+ tokenizer=tokenizer,+ device=device_str,+ batch_size=batch_size,+ max_batch_size=max_batch_size,+ parallelize=parallelism,+ trust_remote_code=trust_remote_code,+ dtype=eval_model_dtype,+ )++ if isinstance(tasks, str):+ tasks = tasks.replace(" ", "").split(",")++ from lm_eval.utils import make_table # pylint: disable=E0611++ res_all = {}+ res_keys = ["results", "versions", "n-shot", "higher_is_better"]+ import time++ st = time.time()+ for task in tasks:+ while retry_times:+ try:+ res = lm_simple_evaluate(+ model=hflm, model_args=None, device=device_str, tasks=task, batch_size=batch_size, limit=limit+ )+ break+ except Exception as e:+ cuda_error_msg = traceback.format_exc()+ try:+ ori_batch_sizes = hflm.batch_sizes if hflm.batch_sizes else {"0": 64}+ try:+ for k, v in hflm.batch_sizes.items():+ hflm.batch_sizes[k] = max(v // 2, 1)+ logger.warning(f"Out of memory, reset batch_size to {hflm.batch_sizes} and re-try.")+ res = lm_simple_evaluate(+ model=hflm, model_args=None, device=device_str, tasks=task, batch_size=1, limit=limit+ )+ hflm.batch_sizes = ori_batch_sizes+ except Exception as e:+ traceback.print_exc()+ pass+ except Exception as e:+ logger.error(cuda_error_msg)+ traceback.print_exc()+ break+ retry_times -= 1++ if not res_all:+ res_all = res+ else:+ for key in res_keys:+ res_all[key].update(res[key])+ print(make_table(res_all))++ print("total eval time:", time.time() - st)+++def eval_with_vllm(args):+ import time++ from lm_eval import evaluator # pylint: disable=E0401+ from lm_eval.models.vllm_causallms import VLLM # pylint: disable=E0401+ from lm_eval.utils import make_table # pylint: disable=E0401++ st = time.time()+ os.environ["TOKENIZERS_PARALLELISM"] = "false"+ device_str, _ = get_device_and_parallelism(args.device)+ eval_model_dtype = get_model_dtype(args.eval_model_dtype, "auto")+ if (batch_size := args.eval_bs) is None:+ batch_size = "auto:8"++ vllm_lm = VLLM(+ pretrained=args.model,+ dtype=eval_model_dtype,+ revision=args.revision,+ trust_remote_code=not args.disable_trust_remote_code,+ tokenizer=args.tokenizer,+ tokenizer_mode=args.tokenizer_mode,+ tokenizer_revision=args.tokenizer_revision,+ add_bos_token=args.add_bos_token,+ prefix_token_id=args.prefix_token_id,+ tensor_parallel_size=args.tensor_parallel_size,+ quantization=args.quantization,+ max_gen_toks=args.max_gen_toks,+ swap_space=args.swap_space,+ batch_size=batch_size,+ max_batch_size=args.max_batch_size,+ max_length=args.max_length,+ max_model_len=args.max_model_len,+ seed=args.seed,+ gpu_memory_utilization=args.gpu_memory_utilization,+ device=device_str,+ data_parallel_size=args.data_parallel_size,+ lora_local_path=args.lora_local_path,+ )+ res = evaluator.simple_evaluate(+ model=vllm_lm,+ tasks=args.tasks,+ limit=args.limit,+ )++ print(make_table(res))+ print("evaluation running time=%ds" % (time.time() - st))diff --git a/auto_round/export/export_to_autoround/export.py b/auto_round/export/export_to_autoround/export.pyindex 8877f4d4b..26d721ec0 100644--- a/auto_round/export/export_to_autoround/export.py+++ b/auto_round/export/export_to_autoround/export.py@@ -275,7 +275,6 @@ def save_quantized_as_autoround(output_dir, inplace=True, backend="auto_round:exfrom auto_round.export.export_to_autoround.export_to_fp8 import save_quantized_as_autoroundreturn save_quantized_as_autoround(output_dir, inplace=inplace, backend="auto_round", **kwargs)- from auto_round.autoround import AutoRoundFormat##if using sym, we change to gptq sym kernel to avoid compiling from auto_round sourceif (diff --git a/auto_round/logger.py b/auto_round/logger.pyindex 73f891315..a43be3167 100644--- a/auto_round/logger.py+++ b/auto_round/logger.py@@ -13,10 +13,14 @@# limitations under the License.import logging-from functools import lru_cache+import warnings+from functools import lru_cache, wraps+from typing import TYPE_CHECKING, Any, Callable, Dict, List, Mapping, Optional, TypeVarimport auto_round.envs as envs+T = TypeVar("T", bound="Callable") # used by `deprecated`+@lru_cache(maxsize=None)def warning_once(self, msg, *args):@@ -84,3 +88,29 @@ logger.propagate = Falsefh = logging.StreamHandler()fh.setFormatter(AutoRoundFormatter())logger.addHandler(fh)+++def deprecated(future_name: Optional[str] = None, message: Optional[str] = None) -> Callable[[T], T]:+ """+ Decorator to mark functions as deprecated++ :param new_function: Function called in place of deprecated function+ :param message: Deprecation message, replaces default deprecation message+ """++ def decorator(func: T) -> T:+ nonlocal message++ if message is None:+ message = f"{func.__name__} is deprecated and will be removed in a future release"+ if future_name is not None:+ message += f". Please use {future_name} instead."++ @wraps(func)+ def wrapped(*args, **kwargs):+ logger.warning_once(message)+ return func(*args, **kwargs)++ return wrapped++ return decoratordiff --git a/auto_round/script/llm.py b/auto_round/script/llm.pydeleted file mode 100644index f489bd12e..000000000--- a/auto_round/script/llm.py+++ /dev/null@@ -1,1021 +0,0 @@-# Copyright (c) 2024 Intel Corporation-#-# Licensed under the Apache License, Version 2.0 (the "License");-# you may not use this file except in compliance with the License.-# You may obtain a copy of the License at-#-# http://www.apache.org/licenses/LICENSE-2.0-#-# Unless required by applicable law or agreed to in writing, software-# distributed under the License is distributed on an "AS IS" BASIS,-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.-# See the License for the specific language governing permissions and-# limitations under the License.--import argparse-import logging--# Copyright (c) 2024 Intel Corporation-#-# Licensed under the Apache License, Version 2.0 (the "License");-# you may not use this file except in compliance with the License.-# You may obtain a copy of the License at-#-# http://www.apache.org/licenses/LICENSE-2.0-#-# Unless required by applicable law or agreed to in writing, software-# distributed under the License is distributed on an "AS IS" BASIS,-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.-# See the License for the specific language governing permissions and-# limitations under the License.-import os-import re-import sys--from auto_round.schemes import PRESET_SCHEMES-from auto_round.utils import (- clear_memory,- get_device_and_parallelism,- get_fp_layer_names,- get_model_dtype,- infer_bits_by_data_type,- set_cuda_visible_devices,-)---class BasicArgumentParser(argparse.ArgumentParser):-- def __init__(self, *args, **kwargs):- super().__init__(*args, **kwargs)- self.add_argument(- "--model", "--model_name", "--model_name_or_path", default="facebook/opt-125m", help="model name or path"- )-- self.add_argument("--mllm", action="store_true", help="whether to quant multi-modal model.")-- self.add_argument("--eval", action="store_true", help="whether to use eval only mode")-- self.add_argument(- "--scheme",- default="W4A16",- type=str,- # choices=["W4A16", "W2A16", "W3A16", "W8A16", "MXFP4", "MXFP8", "NVFP4", "FPW8A16", "FP8_STATIC"],- help="quantization scheme",- )-- self.add_argument("--bits", default=None, type=int, help="number of weight bits")- self.add_argument("--group_size", default=None, type=int, help="group size")- self.add_argument("--asym", action="store_true", help="whether to use asym quantization")- self.add_argument("--data_type", "--dtype", default=None, help="data type for tuning, 'int', 'mx_fp' and etc")- self.add_argument("--act_bits", default=None, type=int, help="activation bits")- self.add_argument("--act_group_size", default=None, type=int, help="activation group size")- self.add_argument(- "--super_group_size", default=None, type=int, help="the number of super group size when use double quant."- )-- self.add_argument(- "--super_bits", default=None, type=int, help="number of scale and mins quant bits for double quant."- )- self.add_argument("--act_data_type", "--act_dtype", default=None, type=str, help="activation data type")-- self.add_argument("--disable_act_dynamic", action="store_true", help="activation static quantization")-- self.add_argument(- "--device_map",- "--device",- "--devices",- default="0",- type=str,- help="the device to be used for tuning. "- "Currently, device settings support CPU, GPU, and HPU."- "The default is set to cuda:0,"- "allowing for automatic detection and switch to HPU or CPU."- "set --device 0,1,2 to use multiple cards.",- )-- self.add_argument(- "--dataset", default="NeelNanda/pile-10k", type=str, help="the dataset for quantization training"- )-- self.add_argument(- "--minmax_lr",- default=None,- type=float,- help="minmax learning rate, if None, it will beset to be the same with lr",- )-- self.add_argument(- "--mem_per_param_scale",- default=13,- type=float,- help="Scale factor for memory per parameter, used to adjust memory usage estimation for tuning",- )-- self.add_argument("--seed", default=42, type=int, help="random seed")-- self.add_argument("--adam", action="store_true", help="whether to use adam optimizer instead of SignSGD")-- self.add_argument("--gradient_accumulate_steps", default=1, type=int, help="gradient accumulate steps")-- self.add_argument("--nblocks", default=1, type=int, help="how many blocks to tune together")-- self.add_argument("--low_gpu_mem_usage", action="store_true", help="offload intermediate features to cpu")-- self.add_argument("--format", default="auto_round", type=str, help="the format to save the model")-- self.add_argument(- "--scale_dtype",- default="fp16",- choices=["fp16", "float16", "bf16", "bfloat16", "fp32", "float32"],- help="scale data type to use for quantization",- )-- self.add_argument(- "--output_dir", default="./tmp_autoround", type=str, help="the directory to save quantized model"- )-- self.add_argument("--disable_amp", action="store_true", help="disable amp")-- self.add_argument(- "--disable_minmax_tuning", action="store_true", help="whether to disable enable weight minmax tuning"- )-- self.add_argument("--enable_norm_bias_tuning", action="store_true", help="whether to enable norm bias tuning")-- self.add_argument(- "--disable_trust_remote_code", action="store_true", help="whether to disable trust_remote_code"- )-- self.add_argument(- "--disable_quanted_input",- action="store_true",- help="whether to disuse the output of quantized block to tune the next block",- )-- self.add_argument("--quant_lm_head", action="store_true", help="whether to quant lm_head")-- self.add_argument(- "--low_cpu_mem_mode",- default=0,- type=int,- choices=[0, 1, 2],- help="choose which low cpu memory mode to use. "- "Can significantly reduce cpu memory footprint but cost more time."- "1 means choose block-wise mode, load the weights of each block"- " from disk when tuning and release the memory of the block after tuning."- "2 means choose layer-wise mode, load the weights of each layer from disk when tuning,"- " minimum memory consumption and also slowest running speed."- "others means not use low cpu memory. Default to 0, not use low cpu memory.",- )-- self.add_argument(- "--low_cpu_mem_tmp_dir",- default=None,- type=str,- help="temporary work space to store the temporary files "- "when using low cpu memory mode. Will remove after tuning.",- )-- self.add_argument(- "--model_dtype",- default=None,- type=str,- choices=["fp16", "float16", "bf16", "bfloat16", "fp32", "float32"],- help="force to convert the dtype, some backends supports fp16 dtype better",- )-- self.add_argument(- "--fp_layers", default="", type=str, help="list of Layer names to maintain original data type"- )-- self.add_argument(- "--not_use_best_mse",- action="store_true",- help="whether to use the iter of best mes loss in the tuning phase",- )-- self.add_argument(- "--to_quant_block_names",- default=None,- type=str,- help="Names of quantitative blocks, please use commas to separate them.",- )-- self.add_argument("--enable_torch_compile", action="store_true", help="whether to enable torch compile")-- self.add_argument("--enable_alg_ext", action="store_true", help="whether to enable probably better algorithm")-- self.add_argument(- "--disable_deterministic_algorithms",- action="store_true",- help="deprecated, disable torch deterministic algorithms.",- )- self.add_argument(- "--enable_deterministic_algorithms", action="store_true", help="enable torch deterministic algorithms."- )-- self.add_argument(- "--disable_opt_rtn",- action="store_true",- help="whether to disable optimization of the RTN mode(iters=0) (default is False).",- )-- ## ======================= MLLM =======================- self.add_argument(- "--quant_nontext_module",- action="store_true",- help="whether to quantize non-text module, e.g. vision component",- )-- self.add_argument(- "--extra_data_dir",- default=None,- type=str,- help="dataset dir for storing images/audio/videos. "- "Can be a dir path or multiple dir path with format as "- "'image=path_to_image,video=path_to_video,audio=path_to_audio'"- "By default, it will search in the relative path, "- "and if not find, will automatic download.",- )-- self.add_argument(- "--template",- default=None,- type=str,- help="the template for building training dataset. It can be a custom one.",- )-- ## ======================= eval =======================- self.add_argument(- "--disable_eval", action="store_true", help="whether to disable lm-eval evaluation after tuning"- )-- self.add_argument(- "--tasks",- "--task",- nargs="?",- const="lambada_openai,hellaswag,winogrande,piqa,mmlu,wikitext,truthfulqa_mc1,"- "openbookqa,boolq,arc_easy,arc_challenge",- default=None,- help="lm-eval tasks",- )-- self.add_argument("--eval_bs", default=None, type=int, help="batch size in evaluation")-- self.add_argument(- "--limit",- type=float,- default=None,- metavar="N|0<N<1",- help="Limit the number of examples per task. "- "If <1, limit is a percentage of the total number of examples.",- )-- self.add_argument("--eval_task_by_task", action="store_true", help="whether to eval task by task.")-- self.add_argument(- "--eval_model_dtype", default=None, type=str, help="the torch_dytpe to load the model for evaluation."- )---class EvalArgumentParser(argparse.ArgumentParser):-- def __init__(self, *args, **kwargs):- super().__init__(*args, **kwargs)- self.add_argument(- "--model", "--model_name", "--model_name_or_path", default="facebook/opt-125m", help="model name or path"- )- self.add_argument("--mllm", action="store_true", help="whether to eval multi-modal model.")- self.add_argument(- "--device_map",- "--device",- "--devices",- default="0",- type=str,- help="the device to be used for tuning. "- "Currently, device settings support CPU, GPU, and HPU."- "The default is set to cuda:0,"- "allowing for automatic detection and switch to HPU or CPU."- "set --device 0,1,2 to use multiple cards.",- )-- self.add_argument(- "--tasks",- "--task",- default="lambada_openai,hellaswag,winogrande,piqa,mmlu,wikitext,truthfulqa_mc1,"- "truthfulqa_mc2,openbookqa,boolq,rte,arc_easy,arc_challenge",- help="lm-eval tasks",- )- self.add_argument(- "--disable_trust_remote_code", action="store_true", help="whether to disable trust_remote_code"- )- self.add_argument("--eval_bs", "--bs", "--batch_size", default=None, type=int, help="batch size in evaluation")- self.add_argument("--eval_task_by_task", action="store_true", help="whether to eval task by task.")- self.add_argument(- "--eval_model_dtype", default=None, type=str, help="the torch_dytpe to load the model for evaluation."- )- self.add_argument(- "--limit",- type=float,- default=None,- metavar="N|0<N<1",- help="Limit the number of examples per task. "- "If <1, limit is a percentage of the total number of examples.",- )- # vllm related arguments- self.add_argument("--revision", default=None, type=str, help="model revision for vllm")- self.add_argument("--tokenizer", default=None, type=str, help="tokenizer to use with vllm")- self.add_argument(- "--tokenizer_mode", default="auto", type=str, help="tokenizer mode for vllm (e.g. auto/fast/slow)"- )- self.add_argument("--tokenizer_revision", default=None, type=str, help="tokenizer revision for vllm")- self.add_argument("--add_bos_token", action="store_true", help="add BOS token when using vllm")- self.add_argument("--prefix_token_id", default=None, type=int, help="prefix token id for vllm")- self.add_argument("--tensor_parallel_size", default=1, type=int, help="tensor parallel size for vllm")- self.add_argument("--data_parallel_size", default=1, type=int, help="data parallel size for vllm")- self.add_argument("--quantization", default=None, type=str, help="quantization setting for vllm")- self.add_argument("--max_gen_toks", default=256, type=int, help="max generation tokens for vllm")- self.add_argument("--swap_space", default=4, type=float, help="swap space (GB) for vllm")- self.add_argument("--max_batch_size", default=None, type=int, help="max batch size for vllm")- self.add_argument("--max_length", default=None, type=int, help="max generation length for vllm")- self.add_argument("--max_model_len", default=None, type=int, help="maximum model sequence length for vllm")- self.add_argument(- "--gpu_memory_utilization", default=0.9, type=float, help="target GPU memory utilization for vllm"- )- self.add_argument("--lora_local_path", default=None, type=str, help="local LoRA path for vllm")---def setup_parser():- parser = BasicArgumentParser()-- parser.add_argument("--batch_size", "--train_bs", "--bs", default=8, type=int, help="train batch size")-- parser.add_argument("--iters", "--iter", default=200, type=int, help="iteration to tune each block")-- parser.add_argument(- "--seqlen", "--seq_len", default=2048, type=int, help="sequence length of the calibration samples"- )-- parser.add_argument("--nsamples", "--nsample", default=128, type=int, help="number of samples")-- parser.add_argument(- "--lr", default=None, type=float, help="learning rate, if None, it will be set to 1.0/iters automatically"- )-- args = parser.parse_args()- return args---def setup_best_parser():- parser = BasicArgumentParser()-- parser.add_argument("--batch_size", "--train_bs", "--bs", default=8, type=int, help="train batch size")-- parser.add_argument("--iters", "--iter", default=1000, type=int, help="iterations to tune each block")-- parser.add_argument(- "--seqlen", "--seq_len", default=2048, type=int, help="sequence length of the calibration samples"- )-- parser.add_argument("--nsamples", "--nsample", default=512, type=int, help="number of samples")-- parser.add_argument(- "--lr", default=None, type=float, help="learning rate, if None, it will be set to 1.0/iters automatically"- )-- args = parser.parse_args()- args.low_gpu_mem_usage = True-- return args---def setup_light_parser():- parser = BasicArgumentParser()-- parser.add_argument("--batch_size", "--train_bs", "--bs", default=8, type=int, help="train batch size")-- parser.add_argument("--iters", "--iter", default=50, type=int, help="iterations to tune each block")-- parser.add_argument(- "--seqlen", "--seq_len", default=2048, type=int, help="sequence length of the calibration samples"- )-- parser.add_argument("--nsamples", "--nsample", default=128, type=int, help="number of samples")-- parser.add_argument(- "--lr", default=5e-3, type=float, help="learning rate, if None, it will be set to 1.0/iters automatically"- )-- args = parser.parse_args()-- return args---def setup_fast_parser():- parser = BasicArgumentParser()-- parser.add_argument("--batch_size", "--train_bs", "--bs", default=4, type=int, help="train batch size")-- parser.add_argument("--iters", default=200, type=int, help="iterations to tune each block")-- parser.add_argument(- "--seqlen", "--seq_len", default=512, type=int, help="sequence length of the calibration samples"- )-- parser.add_argument("--nsamples", "--nsample", default=128, type=int, help="number of samples")-- parser.add_argument(- "--lr", default=None, type=float, help="learning rate, if None, it will be set to 1.0/iters automatically"- )-- args = parser.parse_args()-- return args---def setup_eval_parser():- parser = EvalArgumentParser()- args = parser.parse_args()- return args---def tune(args):- if args.disable_eval:- logging.warning("`disable_eval` is deprecated and is now set by default.")-- if args.eval_bs is None:- args.eval_bs = "auto"- from transformers import AutoConfig, AutoModelForCausalLM, AutoTokenizer-- from auto_round.utils import detect_device, get_library_version, logger-- if args.format is None:- args.format = "auto_round"-- formats = args.format.lower().replace(" ", "").split(",")- from auto_round.utils import SUPPORTED_FORMATS-- for format in formats:- if format not in SUPPORTED_FORMATS:- raise ValueError(f"{format} is not supported, we only support {SUPPORTED_FORMATS}")-- if "auto_gptq" in args.format and args.asym is True:- logger.warning(- "the auto_gptq kernel has issues with asymmetric quantization. "- "It is recommended to use sym quantization or --format='auto_round'"- )-- if "marlin" in args.format and args.asym is True:- raise RuntimeError("marlin backend only supports sym quantization, please remove --asym")-- # Must set this before import torch- # set_cuda_visible_devices(args.device_map)- device_str, use_auto_mapping = get_device_and_parallelism(args.device_map)-- import torch-- if args.enable_torch_compile:- logger.info(- "`torch.compile` is enabled to reduce tuning costs. "- "If it causes issues, you can disable it by removing `--enable_torch_compile` argument."- )-- model_name = args.model- if model_name[-1] == "/":- model_name = model_name[:-1]- logger.info(f"start to quantize {model_name}")-- from auto_round.utils import llm_load_model, mllm_load_model-- if args.mllm:- model, processor, tokenizer, image_processor = mllm_load_model(- model_name,- device="cpu",- use_auto_mapping=False,- trust_remote_code=not args.disable_trust_remote_code,- model_dtype=args.model_dtype,- )- low_cpu_mem_usage = False- else:- model, tokenizer, low_cpu_mem_usage = llm_load_model(- model_name,- use_auto_mapping=False,- trust_remote_code=not args.disable_trust_remote_code,- device="cpu",- low_cpu_mem_mode=args.low_cpu_mem_mode,- low_cpu_mem_tmp_dir=args.low_cpu_mem_tmp_dir,- model_dtype=args.model_dtype,- )-- from auto_round import AutoRound, AutoRoundAdam, AutoRoundMLLM-- if "bloom" in model_name:- args.low_gpu_mem_usage = False-- round = AutoRound- mllm_kwargs = {}- if args.adam:- round = AutoRoundAdam- if args.mllm:- round = AutoRoundMLLM- mllm_kwargs = {- "processor": processor,- "image_processor": image_processor,- "extra_data_dir": args.extra_data_dir,- "quant_nontext_module": args.quant_nontext_module,- "template": args.template,- }-- layer_config = {}- not_quantize_layer_names = get_fp_layer_names(model, args.fp_layers)- for name in not_quantize_layer_names:- layer_config[name] = {"bits": 16, "act_bits": 16, "data_type": "float", "act_data_type": "float"}- if len(not_quantize_layer_names) > 0:- logger.info(f"{not_quantize_layer_names} will not be quantized.")- for format in formats:- if (- "auto_round" not in format- and "fake" not in format- and "awq" not in format- and "llm_compressor" not in format- ):- # TODO gptq could support some mixed precision config- logger.warning(f"mixed precision exporting does not support {format} currently")-- if args.quant_lm_head:- for format in formats:- if "auto_round" not in format and "fake" not in format:- auto_round_formats = [s for s in SUPPORTED_FORMATS if s.startswith("auto_round")]- raise ValueError(- f"{format} is not supported for lm-head quantization, please change to {auto_round_formats}"- )-- if "auto_awq" in args.format:- from auto_round.utils import check_awq_gemm_compatibility-- awq_supported, info = check_awq_gemm_compatibility(- model, args.bits, args.group_size, not args.asym, layer_config- )- if not awq_supported:- logger.warning(f"The AutoAWQ format may not be supported due to {info}")-- enable_torch_compile = True if "--enable_torch_compile" in sys.argv else False- sym = None # the default value should be None now- if args.asym: # if the scheme is asym, how to set it to sym is an issue- sym = False- act_dynamic = None- if args.disable_act_dynamic:- act_dynamic = False- scheme = args.scheme.upper()- if scheme not in PRESET_SCHEMES:- raise ValueError(f"{scheme} is not supported. only {PRESET_SCHEMES.keys()} are supported ")- if args.disable_deterministic_algorithms:- logger.warning(- "default not use deterministic_algorithms. disable_deterministic_algorithms is deprecated,"- " please use enable_deterministic_algorithms instead. "- )- autoround = round(- model=model,- tokenizer=tokenizer,- scheme=scheme,- bits=args.bits,- group_size=args.group_size,- sym=sym,- batch_size=args.batch_size,- dataset=args.dataset,- seqlen=args.seqlen,- nblocks=args.nblocks,- iters=args.iters,- lr=args.lr,- minmax_lr=args.minmax_lr,- enable_quanted_input=not args.disable_quanted_input,- amp=not args.disable_amp,- nsamples=args.nsamples,- seed=args.seed,- low_gpu_mem_usage=args.low_gpu_mem_usage,- scale_dtype=args.scale_dtype,- gradient_accumulate_steps=args.gradient_accumulate_steps,- layer_config=layer_config,- enable_minmax_tuning=not args.disable_minmax_tuning,- act_bits=args.act_bits,- act_group_size=args.act_group_size,- low_cpu_mem_usage=args.low_cpu_mem_mode,- data_type=args.data_type,- enable_norm_bias_tuning=args.enable_norm_bias_tuning,- not_use_best_mse=args.not_use_best_mse,- to_quant_block_names=args.to_quant_block_names,- enable_torch_compile=enable_torch_compile,- act_data_type=args.act_data_type,- act_dynamic=act_dynamic,- device_map=args.device_map,- super_group_size=args.super_group_size,- super_bits=args.super_bits,- disable_opt_rtn=args.disable_opt_rtn,- enable_deterministic_algorithms=args.enable_deterministic_algorithms,- enable_alg_ext=args.enable_alg_ext,- **mllm_kwargs,- )-- model_name = args.model.rstrip("/")-- if model_name.split("/")[-1].strip(".") == "" and "gguf" not in args.format:- if autoround.group_size <= 0:- if "fp" in autoround.act_data_type:- suffix = f"afp{autoround.act_bits}"- else:- suffix = f"a{autoround.act_bits}"- else:- suffix = f"g{autoround.group_size}"- export_dir = os.path.join(args.output_dir, f"w{autoround.bits}{suffix}")- elif model_name.split("/")[-1].strip(".") == "" and "gguf" in args.format:- export_dir = args.output_dir- elif model_name.split("./")[-1].strip("./") != "" and "gguf" in args.format:- export_dir = os.path.join(args.output_dir, model_name.split("/")[-1] + "-gguf")- else:- if autoround.group_size <= 0:- if "fp" in autoround.act_data_type:- suffix = f"afp{autoround.act_bits}"- else:- suffix = f"a{autoround.act_bits}"- else:- suffix = f"g{autoround.group_size}"- export_dir = os.path.join(args.output_dir, model_name.split("/")[-1] + f"-w{autoround.bits}{suffix}")-- model, folders = autoround.quantize_and_save(export_dir, format=args.format)-- if args.low_cpu_mem_mode == 1 or args.low_cpu_mem_mode == 2:- import shutil-- shutil.rmtree(args.low_cpu_mem_tmp_dir, ignore_errors=True)-- model.eval()- clear_memory()-- lm_eval_version = get_library_version("lm-eval")-- eval_folder = folders[-1]- if args.tasks is None or args.tasks == "" or eval_folder is None:- return-- tasks = args.tasks- if isinstance(tasks, str):- tasks = tasks.split(",")-- from lm_eval.utils import make_table # pylint: disable=E0401-- logger.info(f"Using lm-eval version {lm_eval_version}")- eval_gguf_model = False- for file in os.listdir(eval_folder):- if file.endswith("gguf"):- eval_gguf_model = True- break-- import time-- eval_model_dtype = get_model_dtype(args.eval_model_dtype, "auto")-- if autoround.act_bits <= 8 or eval_gguf_model:- if eval_gguf_model:- # for file in os.listdir(eval_folder):- # gguf_file = file- gguf_file = None- gguf_format = None # Initialize gguf_format to None- # gguf folder only contains one file- for format in formats:- if format.startswith("gguf"):- gguf_format = format.split(":")[-1].upper()- if gguf_format is None: # Validate gguf_format after the loop- logger.error("No valid gguf format found in formats. Please check the input.")- sys.exit(-1)- for file in os.listdir(eval_folder):- if gguf_format in file:- gguf_file = file-- logger.warning("evaluating gguf model is an experimental feature, the accuracy may be not correct.")- if eval_model_dtype == "float32" or eval_model_dtype == "auto":- logger.warning(- "set '--eval_model_dtype bf16' can significantly speed up evaluation for gguf model,"- " but may affect accuracy."- )- if gguf_file is None:- logger.error("Cannot find correct gguf file for evaluation, please check.")- sys.exit(-1)- model = AutoModelForCausalLM.from_pretrained(- eval_folder, gguf_file=gguf_file, device_map="auto", torch_dtype=eval_model_dtype- )- model.eval()- tokenizer = AutoTokenizer.from_pretrained(eval_folder, gguf_file=gguf_file)- else:- if hasattr(model, "hf_device_map") and len(model.hf_device_map) > 1:- from accelerate.big_modeling import dispatch_model-- dispatch_model(model, model.hf_device_map)- else:- device_str = detect_device(device_str)- model = model.to(device_str)- if model.dtype != eval_model_dtype and eval_model_dtype != "auto":- model.to(getattr(torch, eval_model_dtype))-- if args.eval_task_by_task:- eval_task_by_task(- model,- tokenizer=tokenizer,- device=device_str,- tasks=args.tasks,- limit=args.limit,- batch_size=args.eval_bs,- eval_model_dtype=eval_model_dtype,- )- else:- if args.eval_bs is None or args.eval_bs == "auto":- logger.warning("This API does not support auto currently, reset eval_bs to 16")- args.eval_bs = 16- from auto_round.eval.evaluation import simple_evaluate_user_model-- st = time.time()- add_bos_token = False- if "llama" in args.model.lower():- add_bos_token = True- res = simple_evaluate_user_model(- model,- tokenizer,- tasks=tasks,- batch_size=args.eval_bs,- limit=args.limit,- device=device_str,- eval_model_dtype=eval_model_dtype,- add_bos_token=add_bos_token,- )- print(make_table(res))- print("evaluation running time=%ds" % (time.time() - st))- else:- if args.eval_task_by_task:- eval_task_by_task(- eval_folder,- device=device_str,- tasks=args.tasks,- batch_size=args.eval_bs,- limit=args.limit,- eval_model_dtype=eval_model_dtype,- )- else:- from auto_round.eval.evaluation import simple_evaluate-- tasks, model_args, device_str = _eval_init(- args.tasks, eval_folder, args.device_map, args.disable_trust_remote_code, dtype=eval_model_dtype- )- st = time.time()- if "llama" in args.model.lower():- model_args += ",add_bos_token=True"- res = simple_evaluate(- model="hf",- model_args=model_args,- tasks=tasks,- device=device_str,- batch_size=args.eval_bs,- limit=args.limit,- )- print(make_table(res))- print("evaluation running time=%ds" % (time.time() - st))---def _eval_init(tasks, model_path, device, disable_trust_remote_code=False, dtype="auto"):- set_cuda_visible_devices(device)- device_str, parallelism = get_device_and_parallelism(device)- if dtype != "auto":- dtype = get_model_dtype(model_dtype=dtype)- # ,add_bos_token={True}- model_args = f"pretrained={model_path},trust_remote_code={not disable_trust_remote_code},dtype={dtype}"- if parallelism:- model_args += ",parallelize=True"- if isinstance(tasks, str):- tasks = tasks.split(",")- return tasks, model_args, device_str---def eval(args):- import time-- tasks, model_args, device_str = _eval_init(- args.tasks, args.model, args.device_map, args.disable_trust_remote_code, args.eval_model_dtype- )-- # load after _eval_int in order to make sure import torch after set CUDA_VISIBLE_DEVICES- from auto_round.eval.evaluation import simple_evaluate, simple_evaluate_user_model- from auto_round.utils import logger-- if (batch_size := args.eval_bs) is None:- batch_size = "auto:8"- is_gguf_file = False- if os.path.exists(args.model):- if os.path.isfile(args.model) and args.model.endswith(".gguf"):- is_gguf_file = True- gguf_file = os.path.basename(args.model)- model = os.path.dirname(args.model)- else:- for file in os.listdir(args.model):- if file.endswith(".gguf"):- is_gguf_file = True- gguf_file = file- eval_model_dtype = get_model_dtype(args.eval_model_dtype)- if is_gguf_file:- import torch- from lm_eval.utils import make_table # pylint: disable=E0401- from transformers import AutoModelForCausalLM, AutoTokenizer-- tokenizer = AutoTokenizer.from_pretrained(model, gguf_file=gguf_file)-- logger.warning("evaluating gguf model is an experimental feature, the accuracy may be not correct.")- if eval_model_dtype == "float32" or eval_model_dtype == "auto":- logger.warning(- "set '--eval_model_dtype bf16' can significantly speed up evaluation for gguf model,"- " but may affect accuracy."- )- model = AutoModelForCausalLM.from_pretrained(- model, gguf_file=gguf_file, device_map="auto", torch_dtype=eval_model_dtype- )- model.eval()- st = time.time()- res = simple_evaluate_user_model(- model, tokenizer, tasks=tasks, batch_size=batch_size, device=device_str, limit=args.limit- )- print(make_table(res))- print("evaluation running time=%ds" % (time.time() - st))- else:- st = time.time()- if "auto" in str(batch_size) and args.mllm:- logger.warning("Batch size 'auto' is not yet supported for hf-multimodal models, reset to 16")- batch_size = 16- res = simple_evaluate(- model="hf" if not args.mllm else "hf-multimodal",- model_args=model_args,- tasks=tasks,- device=device_str,- batch_size=batch_size,- limit=args.limit,- )- from lm_eval.utils import make_table # pylint: disable=E0401-- print(make_table(res))- print("evaluation running time=%ds" % (time.time() - st))---def eval_task_by_task(- model,- device=None,- tasks=None,- tokenizer=None,- batch_size=None,- limit=None,- max_batch_size=64,- trust_remote_code=True,- eval_model_dtype=None,- retry_times=3,-):- set_cuda_visible_devices(device)- device_str, parallelism = get_device_and_parallelism(device)-- # load after _eval_int in order to make sure import torch after set CUDA_VISIBLE_DEVICES- import traceback-- from lm_eval import simple_evaluate as lm_simple_evaluate # pylint: disable=E0611- from lm_eval.models.huggingface import HFLM- from transformers import AutoModelForCausalLM, AutoTokenizer-- from auto_round.utils import logger-- if batch_size is None:- batch_size = "auto:8"- is_gguf_file = False- if not isinstance(model, str):- parallelism = False- else:- if os.path.isfile(model) and model.endswith(".gguf"):- is_gguf_file = True- gguf_file = os.path.basename(model)- model = os.path.dirname(model)- else:- for file in os.listdir(model):- if file.endswith(".gguf"):- is_gguf_file = True- gguf_file = file- eval_model_dtype = get_model_dtype(eval_model_dtype)- if is_gguf_file:- tokenizer = AutoTokenizer.from_pretrained(model, gguf_file=gguf_file)- logger.warning("evaluating gguf model is an experimental feature, the accuracy may be not correct.")- if eval_model_dtype == "float32" or eval_model_dtype == "auto":- logger.warning(- "set '--eval_model_dtype bf16' can significantly speed up evaluation for gguf model,"- " but may affect accuracy."- )-- model = AutoModelForCausalLM.from_pretrained(- model, gguf_file=gguf_file, device_map="auto", torch_dtype=eval_model_dtype- )- model.eval()- parallelism = False- hflm = HFLM(- pretrained=model,- tokenizer=tokenizer,- device=device_str,- batch_size=batch_size,- max_batch_size=max_batch_size,- parallelize=parallelism,- trust_remote_code=trust_remote_code,- dtype=eval_model_dtype,- )-- if isinstance(tasks, str):- tasks = tasks.replace(" ", "").split(",")-- from lm_eval.utils import make_table # pylint: disable=E0611-- res_all = {}- res_keys = ["results", "versions", "n-shot", "higher_is_better"]- import time-- st = time.time()- for task in tasks:- while retry_times:- try:- res = lm_simple_evaluate(- model=hflm, model_args=None, device=device_str, tasks=task, batch_size=batch_size, limit=limit- )- break- except Exception as e:- cuda_error_msg = traceback.format_exc()- try:- ori_batch_sizes = hflm.batch_sizes if hflm.batch_sizes else {"0": 64}- try:- for k, v in hflm.batch_sizes.items():- hflm.batch_sizes[k] = max(v // 2, 1)- logger.warning(f"Out of memory, reset batch_size to {hflm.batch_sizes} and re-try.")- res = lm_simple_evaluate(- model=hflm, model_args=None, device=device_str, tasks=task, batch_size=1, limit=limit- )- hflm.batch_sizes = ori_batch_sizes- except Exception as e:- traceback.print_exc()- pass- except Exception as e:- logger.error(cuda_error_msg)- traceback.print_exc()- break- retry_times -= 1-- if not res_all:- res_all = res- else:- for key in res_keys:- res_all[key].update(res[key])- print(make_table(res_all))-- print("total eval time:", time.time() - st)---def eval_with_vllm(args):- import time-- from lm_eval import evaluator # pylint: disable=E0401- from lm_eval.models.vllm_causallms import VLLM # pylint: disable=E0401- from lm_eval.utils import make_table # pylint: disable=E0401-- st = time.time()- os.environ["TOKENIZERS_PARALLELISM"] = "false"- device_str, _ = get_device_and_parallelism(args.device)- eval_model_dtype = get_model_dtype(args.eval_model_dtype, "auto")- if (batch_size := args.eval_bs) is None:- batch_size = "auto:8"-- vllm_lm = VLLM(- pretrained=args.model,- dtype=eval_model_dtype,- revision=args.revision,- trust_remote_code=not args.disable_trust_remote_code,- tokenizer=args.tokenizer,- tokenizer_mode=args.tokenizer_mode,- tokenizer_revision=args.tokenizer_revision,- add_bos_token=args.add_bos_token,- prefix_token_id=args.prefix_token_id,- tensor_parallel_size=args.tensor_parallel_size,- quantization=args.quantization,- max_gen_toks=args.max_gen_toks,- swap_space=args.swap_space,- batch_size=batch_size,- max_batch_size=args.max_batch_size,- max_length=args.max_length,- max_model_len=args.max_model_len,- seed=args.seed,- gpu_memory_utilization=args.gpu_memory_utilization,- device=device_str,- data_parallel_size=args.data_parallel_size,- lora_local_path=args.lora_local_path,- )- res = evaluator.simple_evaluate(- model=vllm_lm,- tasks=args.tasks,- limit=args.limit,- )-- print(make_table(res))- print("evaluation running time=%ds" % (time.time() - st))diff --git a/auto_round/script/mllm.py b/auto_round/script/mllm.pydeleted file mode 100644index cb3edd93b..000000000--- a/auto_round/script/mllm.py+++ /dev/null@@ -1,610 +0,0 @@-# Copyright (c) 2024 Intel Corporation-#-# Licensed under the Apache License, Version 2.0 (the "License");-# you may not use this file except in compliance with the License.-# You may obtain a copy of the License at-#-# http://www.apache.org/licenses/LICENSE-2.0-#-# Unless required by applicable law or agreed to in writing, software-# distributed under the License is distributed on an "AS IS" BASIS,-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.-# See the License for the specific language governing permissions and-# limitations under the License.--import argparse-import os-import sys--from auto_round.logger import logger-from auto_round.schemes import PRESET_SCHEMES-from auto_round.utils import (- clear_memory,- get_device_and_parallelism,- get_fp_layer_names,- is_debug_mode,- set_cuda_visible_devices,-)---class BasicArgumentParser(argparse.ArgumentParser):-- def __init__(self, *args, **kwargs):- super().__init__(*args, **kwargs)- self.add_argument(- "--model",- "--model_name",- "--model_name_or_path",- default="Qwen/Qwen2-VL-2B-Instruct",- help="model name or path",- )-- self.add_argument("--eval", action="store_true", help="whether to use eval only mode.")-- self.add_argument(- "--scheme",- default="W4A16",- type=str,- help="quantization scheme",- )-- self.add_argument("--bits", default=None, type=int, help="number of weight bits")- self.add_argument("--group_size", default=None, type=int, help="group size")- self.add_argument("--asym", action="store_true", help="whether to use asym quantization")- self.add_argument("--data_type", "--dtype", default=None, help="data type for tuning, 'int', 'mx_fp' and etc")- self.add_argument("--act_bits", default=None, type=int, help="activation bits")-- self.add_argument("--eval_bs", default=None, type=int, help="batch size in evaluation")-- self.add_argument(- "--device",- "--devices",- default="0",- type=str,- help="the device to be used for tuning. "- "Currently, device settings support CPU, GPU, and HPU."- "The default is set to cuda:0,"- "allowing for automatic detection and switch to HPU or CPU."- "set --device 0,1,2 to use multiple cards.",- )-- self.add_argument(- "--dataset",- type=str,- default=None,- help="the dataset for quantization training."- " current support NeelNanda/pile-10k,liuhaotian/llava_conv_58k,"- "liuhaotian/llava_instruct_80k,liuhaotian/llava_instruct_150k"- "It can be a custom one. Default is NeelNanda/pile-10k",- )-- self.add_argument(- "--lr", default=None, type=float, help="learning rate, if None, it will be set to 1.0/iters automatically"- )-- self.add_argument(- "--minmax_lr",- default=None,- type=float,- help="minmax learning rate, if None,it will beset to be the same with lr",- )-- self.add_argument("--seed", default=42, type=int, help="random seed")-- self.add_argument("--adam", action="store_true", help="whether to use adam optimizer instead of SignSGD")-- self.add_argument("--gradient_accumulate_steps", default=1, type=int, help="gradient accumulate steps")-- self.add_argument("--nblocks", default=1, type=int, help="how many blocks to tune together")-- self.add_argument("--low_gpu_mem_usage", action="store_true", help="offload intermediate features to cpu")-- self.add_argument("--format", default="auto_round", type=str, help="the format to save the model")-- self.add_argument(- "--scale_dtype",- default="fp16",- choices=["fp16", "float16", "bf16", "bfloat16", "fp32", "float32"],- help="scale data type to use for quantization",- )-- self.add_argument(- "--output_dir", default="./tmp_autoround", type=str, help="the directory to save quantized model"- )-- self.add_argument("--disable_amp", action="store_true", help="disable amp")-- self.add_argument(- "--disable_minmax_tuning", action="store_true", help="whether disable enable weight minmax tuning"- )-- self.add_argument("--enable_norm_bias_tuning", action="store_true", help="whether enable norm bias tuning")-- self.add_argument(- "--disable_trust_remote_code", action="store_true", help="whether to disable trust_remote_code"- )-- self.add_argument(- "--disable_quanted_input",- action="store_true",- help="whether to disuse the output of quantized block to tune the next block",- )-- self.add_argument("--quant_lm_head", action="store_true", help="whether to quant lm_head")-- self.add_argument(- "--low_cpu_mem_mode",- default=0,- type=int,- choices=[0, 1, 2],- help="choose which low cpu memory mode to use. "- "Can significantly reduce cpu memory footprint but cost more time."- "1 means choose block-wise mode, load the weights of each block"- " from disk when tuning and release the memory of the block after tuning."- "2 means choose layer-wise mode, load the weights of each layer from disk when tuning,"- " minimum memory consumption and also slowest running speed."- "others means not use low cpu memory. Default to 0, not use low cpu memory.",- )-- self.add_argument(- "--low_cpu_mem_tmp_dir",- default=None,- type=str,- help="temporary work space to store the temporary files "- "when using low cpu memory mode. Will remove after tuning.",- )-- self.add_argument(- "--model_dtype",- default=None,- type=str,- choices=["fp16", "float16", "bf16", "bfloat16", "fp32", "float32"],- help="force to convert the dtype, some backends supports fp16 dtype better",- )-- self.add_argument("--fp_layers", default="", type=str, help="layers to maintain original data type")-- self.add_argument(- "--not_use_best_mse",- action="store_true",- help="whether to use the iter of best mes loss in the tuning phase",- )-- self.add_argument("--enable_torch_compile", action="store_true", help="whether to enable torch compile")-- self.add_argument(- "--disable_deterministic_algorithms",- action="store_true",- help="deprecated, disable torch deterministic algorithms.",- )-- self.add_argument(- "--enable_deterministic_algorithms", action="store_true", help="enable torch deterministic algorithms."- )-- ## ======================= VLM =======================- self.add_argument(- "--quant_nontext_module",- action="store_true",- help="whether to quantize non-text module, e.g. vision component",- )-- self.add_argument(- "--extra_data_dir",- default=None,- type=str,- help="dataset dir for storing images/audio/videos. "- "Can be a dir path or multiple dir path with format as "- "'image=path_to_image,video=path_to_video,audio=path_to_audio'"- "By default, it will search in the relative path, "- "and if not find, will automatic download.",- )-- self.add_argument(- "--template",- default=None,- type=str,- help="the template for building training dataset. It can be a custom one.",- )-- self.add_argument(- "--truncation",- action="store_true",- help="whether to truncate sequences at the maximum length."- " Default True for pile and False for llava dataset.",- )-- self.add_argument(- "--to_quant_block_names",- default=None,- type=str,- help="Names of quantitative blocks, please use commas to separate them.",- )-- self.add_argument("--device_map", default=None, type=str, help="device_map for block in tuning phase")-- self.add_argument(- "--disable_opt_rtn",- action="store_true",- help="whether to disable optimization of the RTN mode(iters=0) (default is False).",- )---def setup_parser():- parser = BasicArgumentParser()-- parser.add_argument("--batch_size", "--train_bs", "--bs", default=8, type=int, help="train batch size")-- parser.add_argument("--iters", "--iter", default=200, type=int, help=" iters")-- parser.add_argument(- "--seqlen",- "--seq_len",- default=None,- type=int,- help="sequence length, default 2048 for text-only, 512 for liuhaotian/llava",- )-- parser.add_argument("--nsamples", "--nsample", default=128, type=int, help="number of samples")-- args = parser.parse_args()- return args---def setup_lmeval_parser():- parser = argparse.ArgumentParser()- parser.add_argument("--model", "--model_name", "--model_name_or_path", help="model name or path")- parser.add_argument(- "--device",- "--devices",- default="0",- type=str,- help="the device to be used for tuning. "- "Currently, device settings support CPU, GPU, and HPU."- "The default is set to cuda:0,"- "allowing for automatic detection and switch to HPU or CPU."- "set --device 0,1,2 to use multiple cards.",- )- parser.add_argument(- "--tasks",- type=str,- default="MMBench_DEV_EN_V11,ScienceQA_VAL,TextVQA_VAL,POPE",- help="eval tasks for VLMEvalKit.",- )- # Args that only apply to Video Dataset- parser.add_argument(- "--nframe",- type=int,- default=8,- help="the number of frames to sample from a video," " only applicable to the evaluation of video benchmarks.",- )- parser.add_argument(- "--pack",- action="store_true",- help="a video may associate with multiple questions, if pack==True,"- " will ask all questions for a video in a single",- )- parser.add_argument("--fps", type=float, default=-1, help="set the fps for a video.")- # Work Dir- # Infer + Eval or Infer Only- parser.add_argument(- "--mode",- type=str,- default="all",- choices=["all", "infer"],- help="when mode set to 'all', will perform both inference and evaluation;"- " when set to 'infer' will only perform the inference.",- )- parser.add_argument(- "--eval_data_dir",- type=str,- default=None,- help="path for VLMEvalKit to store the eval data. Default will store in ~/LMUData",- )- # API Kwargs, Apply to API VLMs and Judge API LLMs- parser.add_argument("--retry", type=int, default=None, help="retry numbers for API VLMs")- # Explicitly Set the Judge Model- parser.add_argument("--judge", type=str, default=None, help="whether is a judge model.")- # Logging Utils- parser.add_argument("--verbose", action="store_true", help="whether to display verbose information.")- # Configuration for Resume- # Ignore: will not rerun failed VLM inference- parser.add_argument("--ignore", action="store_true", help="ignore failed indices. ")- # Rerun: will remove all evaluation temp files- parser.add_argument(- "--rerun", action="store_true", help="if true, will remove all evaluation temp files and rerun."- )- parser.add_argument("--output_dir", default="./eval_result", type=str, help="the directory to save quantized model")- args = parser.parse_args()- return args---def tune(args):- import transformers- from transformers import AutoConfig, AutoModelForCausalLM, AutoProcessor, AutoTokenizer-- if args.format is None:- args.format = "auto_round"-- formats = args.format.replace(" ", "").split(",")- from auto_round.utils import SUPPORTED_FORMATS-- for format in formats:- if format not in SUPPORTED_FORMATS:- raise ValueError(f"{format} is not supported, we only support {SUPPORTED_FORMATS}")-- # Must set this before import torch- # set_cuda_visible_devices(args.device_map)- device_str, use_auto_mapping = get_device_and_parallelism(args.device_map)-- import torch-- model_name = args.model- if model_name[-1] == "/":- model_name = model_name[:-1]- logger.info(f"start to quantize {model_name}")- torch_dtype = "auto"- if device_str is not None and "hpu" in device_str:- torch_dtype = torch.bfloat16-- # load_model- from auto_round.utils import mllm_load_model-- model, processor, tokenizer, image_processor = mllm_load_model(- model_name,- device="cpu",- torch_dtype=torch_dtype,- use_auto_mapping=False,- trust_remote_code=not args.disable_trust_remote_code,- model_dtype=args.model_dtype,- )-- from auto_round import AutoRoundMLLM-- seqlen = args.seqlen- if seqlen is not None and hasattr(model, "config") and hasattr(model.config, "max_position_embeddings"):- seqlen = min(seqlen, model.config.max_position_embeddings)-- if seqlen is not None and hasattr(tokenizer, "model_max_length"):- if tokenizer.model_max_length < seqlen:- logger.info(- f"change sequence length to {tokenizer.model_max_length} due to the limitation of model_max_length"- )- seqlen = min(seqlen, tokenizer.model_max_length)- args.seqlen = seqlen-- model = model.eval()-- round = AutoRoundMLLM-- layer_config = {}- not_quantize_layer_names = get_fp_layer_names(model, args.fp_layers)- for name in not_quantize_layer_names:- layer_config[name] = {"bits": 16, "act_bits": 16}- if len(not_quantize_layer_names) > 0:- logger.info(f"{not_quantize_layer_names} will not be quantized.")- for format in formats:- if "auto_round" not in format and "fake" not in format and "awq" not in format:- ##TODO gptq could support some mixed precision config- logger.warning(f"mixed precision exporting does not support {format} currently")-- if args.fp_layers != "":- fp_layers = args.fp_layers.replace(" ", "").split(",")- for n, m in model.named_modules():- if not isinstance(m, (torch.nn.Linear, transformers.pytorch_utils.Conv1D)):- continue- for fp_layer in fp_layers:- if fp_layer in n:- layer_config[n] = {"bits": 16}- logger.info(f"{n} will not be quantized.")- if len(layer_config) > 0:- for format in formats:- if "auto_round" not in format and "fake" not in format:- ##TODO gptq, awq could support some mixed precision config- logger.warning(f"mixed precision exporting does not support {format} currently")-- if args.quant_lm_head:- for format in formats:- if "auto_round" not in format and "fake" not in format:- auto_round_formats = [s for s in SUPPORTED_FORMATS if s.startswith("auto_round")]- raise ValueError(- f"{format} is not supported for lm-head quantization, please change to {auto_round_formats}"- )-- if args.quant_lm_head and args.low_gpu_mem_usage:- print("warning, low_gpu_mem_usage=False is strongly recommended if the whole model could be loaded to " "gpu")-- if "--truncation" not in sys.argv:- args.truncation = None-- if "auto_awq" in args.format:- from auto_round.utils import check_awq_gemm_compatibility-- awq_supported, info = check_awq_gemm_compatibility(- model, args.bits, args.group_size, not args.asym, layer_config- )- if not awq_supported:- logger.warning(f"The AutoAWQ format may not be supported due to {info}")-- enable_torch_compile = True if "--enable_torch_compile" in sys.argv else False-- model_kwargs = {- "use_auto_mapping": use_auto_mapping,- "trust_remote_code": not args.disable_trust_remote_code,- "model_dtype": args.model_dtype,- }-- sym = None # the default value should be None now- if args.asym: # if the scheme is asym, how to set it to sym is an issue- sym = False-- scheme = args.scheme.upper()- if scheme not in PRESET_SCHEMES:- raise ValueError(f"{scheme} is not supported. only {PRESET_SCHEMES.keys()} are supported ")- if args.disable_deterministic_algorithms:- logger.warning(- "default not use deterministic_algorithms. disable_deterministic_algorithms is deprecated,"- " please use enable_deterministic_algorithms instead. "- )- autoround = round(- model,- tokenizer,- scheme=args.scheme,- processor=processor,- image_processor=image_processor,- dataset=args.dataset,- extra_data_dir=args.extra_data_dir,- bits=args.bits,- group_size=args.group_size,- sym=sym,- batch_size=args.batch_size,- seqlen=seqlen,- nblocks=args.nblocks,- iters=args.iters,- lr=args.lr,- minmax_lr=args.minmax_lr,- amp=not args.disable_amp,- enable_quanted_input=not args.disable_quanted_input,- truncation=args.truncation,- nsamples=args.nsamples,- low_gpu_mem_usage=args.low_gpu_mem_usage,- seed=args.seed,- gradient_accumulate_steps=args.gradient_accumulate_steps,- scale_dtype=args.scale_dtype,- layer_config=layer_config,- template=args.template,- enable_minmax_tuning=not args.disable_minmax_tuning,- act_bits=args.act_bits,- quant_nontext_module=args.quant_nontext_module,- not_use_best_mse=args.not_use_best_mse,- to_quant_block_names=args.to_quant_block_names,- enable_torch_compile=enable_torch_compile,- device_map=args.device_map,- model_kwargs=model_kwargs,- data_type=args.data_type,- disable_opt_rtn=args.disable_opt_rtn,- enable_deterministic_algorithms=args.enable_deterministic_algorithms,- )-- model_name = args.model.rstrip("/")-- if model_name.split("/")[-1].strip(".") == "" and "gguf" not in args.format:- export_dir = os.path.join(args.output_dir, f"w{autoround.bits}g{autoround.group_size}")- elif model_name.split("/")[-1].strip(".") == "" and "gguf" in args.format:- export_dir = args.output_dir- elif model_name.split("./")[-1].strip("./") != "" and "gguf" in args.format:- export_dir = os.path.join(args.output_dir, model_name.split("/")[-1] + "-gguf")- else:- export_dir = os.path.join(- args.output_dir, model_name.split("/")[-1] + f"-w{autoround.bits}g{autoround.group_size}"- )-- model, folders = autoround.quantize_and_save(export_dir, format=args.format)-- if args.low_cpu_mem_mode == 1 or args.low_cpu_mem_mode == 2:- import shutil-- shutil.rmtree(args.low_cpu_mem_tmp_dir, ignore_errors=True)-- model.eval()- clear_memory()---def vlmeval(args):- set_cuda_visible_devices(args.device_map)- device_str, parallelism = get_device_and_parallelism(args.device_map)- if parallelism:- os.environ["AUTO_SPLIT"] = "1"- if isinstance(args.tasks, str):- args.tasks = args.tasks.replace(" ", "").split(",")- from auto_round.mllm import mllm_eval-- mllm_eval(- args.model,- work_dir=args.output_dir,- data_store_dir=args.eval_data_dir,- dataset=args.tasks,- pack=args.pack,- fps=args.fps,- nframe=args.nframe,- rerun=args.rerun,- judge=args.judge,- verbose=args.verbose,- mode=args.mode,- ignore=args.ignore,- )---def setup_lmms_parser():- parser = argparse.ArgumentParser()- parser.add_argument("--model", "--model_name", "--model_name_or_path", help="model name or path")- parser.add_argument(- "--tasks",- default="pope,textvqa_val,scienceqa,mmbench_en",- help="To get full list of tasks, use the command lmms-eval --tasks list",- )- parser.add_argument("--output_dir", default="./eval_result", type=str, help="the directory to save quantized model")- parser.add_argument(- "--num_fewshot",- type=int,- default=None,- help="Number of examples in few-shot context",- )- parser.add_argument(- "--batch_size",- "--bs",- "-b",- type=str,- default=1,- metavar="auto|auto:N|N",- help="Acceptable values are 'auto', 'auto:N' or N, where N is an integer. Default 1.",- )- parser.add_argument(- "--max_batch_size",- type=int,- default=None,- metavar="N",- help="Maximal batch size to try with --batch_size auto.",- )- parser.add_argument(- "--device",- "--devices",- default="0",- type=str,- help="the device to be used for tuning. "- "Currently, device settings support CPU, GPU, and HPU."- "The default is set to cuda:0,"- "allowing for automatic detection and switch to HPU or CPU."- "set --device 0,1,2 to use multiple cards.",- )- parser.add_argument(- "--limit",- type=float,- default=None,- help="Limit the number of examples per task. "- "If <1, limit is a percentage of the total"- " number of examples.",- )- args = parser.parse_args()- return args---def lmms_eval(args):- set_cuda_visible_devices(args.device_map)- device_str, parallelism = get_device_and_parallelism(args.device_map)-- from auto_round.mllm import lmms_eval-- results = lmms_eval(- model=args.model,- tasks=args.tasks,- output_dir=args.output_dir,- num_fewshot=args.num_fewshot,- limit=args.limit,- batch_size=args.batch_size,- max_batch_size=args.max_batch_size,- device=device_str,- use_cache=None,- apply_chat_template=False,- )- return resultsdiff --git a/auto_round/utils.py b/auto_round/utils.pyindex dcb794521..2d98ae121 100644--- a/auto_round/utils.py+++ b/auto_round/utils.py@@ -1042,6 +1042,8 @@ def get_fp_layer_names(model, fp_layers):list: A list of layer names that match the specified FP layers or aresubcomponents of those layers."""+ if not fp_layers:+ return []fp_layers = fp_layers.replace(" ", "").split(",")all_layer_names = []for n, m in model.named_modules():@@ -2686,3 +2688,45 @@ def copy_python_files_from_model_cache(model, save_path: str):if file.endswith(".py") and os.path.isfile(full_file_name):logger.debug(f"Transferring {full_file_name} to {save_path}")shutil.copy(full_file_name, save_path)+++def is_mllm_model(model_or_path: Union[str, torch.nn.Module]):+ MM_KEYS = [+ "multi_modal_projector",+ "vision_tower",+ "multimodal_projector",+ "thinker",+ "visual",+ "audio",+ "talker",+ "token2wav",+ "vision_model",+ "audio_tower",+ "vision_encoder",+ "vision_language_adapter",+ "patch_merger",+ "pre_mm_projector_norm",+ "vision",+ ]++ model_path = model_or_path if isinstance(model_or_path, str) else model_or_path.name_or_path+ if not os.path.isdir(model_path):+ model_path = download_hf_model(model_path)++ if isinstance(model_path, str):+ if os.path.exists(os.path.join(model_path, "preprocessor_config.json")):+ return True+ if os.path.exists(os.path.join(model_path, "processor_config.json")):+ return True+ with open(os.path.join(model_path, "config.json")) as f:+ config = json.load(f)+ for key in config.keys():+ if any([k in key for k in MM_KEYS]):+ return True++ if isinstance(model_or_path, torch.nn.Module):+ for name, module in model_or_path.named_modules():+ if any([k in name for k in MM_KEYS]):+ return True++ return Falsediff --git a/test/test_cpu/test_autoround.py b/test/test_cpu/test_autoround.pyindex ea53aaf94..827688812 100644--- a/test/test_cpu/test_autoround.py+++ b/test/test_cpu/test_autoround.py@@ -677,6 +677,18 @@ class TestAutoRound(unittest.TestCase):ar = AutoRound(model_name, quant_lm_head=True, iters=1, nsamples=1, seqlen=32)ar.quantize()+ def test_compressor(self):+ model_name = "Qwen/Qwen2-VL-2B-Instruct"+ ar = AutoRound(model_name, enable_adam=True)+ self.assertEqual(ar.optimizer, torch.optim.AdamW)+ self.assertTrue(ar.mllm)++ # test old api+ from auto_round import AutoRoundMLLM++ ar = AutoRoundMLLM(model_name)+ self.assertTrue(ar.mllm)+if __name__ == "__main__":unittest.main()diff --git a/test/test_cpu/test_gguf_format.py b/test/test_cpu/test_gguf_format.pyindex eb146d574..2a25b0f5f 100644--- a/test/test_cpu/test_gguf_format.py+++ b/test/test_cpu/test_gguf_format.py@@ -312,7 +312,7 @@ class TestGGUF(unittest.TestCase):def test_vlm_gguf(self):model_name = "Qwen/Qwen2-VL-2B-Instruct"- from auto_round.mllm.autoround_mllm import AutoRoundMLLM+ from auto_round import AutoRoundMLLMfrom auto_round.utils import mllm_load_modelmodel, processor, tokenizer, image_processor = mllm_load_model(model_name)diff --git a/test/test_cpu/test_mllm.py b/test/test_cpu/test_mllm.pyindex 0fd102407..10ccd9d37 100644--- a/test/test_cpu/test_mllm.py+++ b/test/test_cpu/test_mllm.py@@ -97,7 +97,7 @@ class TestAutoRoundMLLM(unittest.TestCase):assert target_blocks == blocksdef test_dataset_check(self):- from auto_round.mllm.mllm_dataset import MLLM_DATASET+ from auto_round.compressors.mllm.dataset import MLLM_DATASETclass Myclass:model_type = Nonediff --git a/test/test_cuda/test_gguf.py b/test/test_cuda/test_gguf.pyindex 33473a116..50bb23aa6 100644--- a/test/test_cuda/test_gguf.py+++ b/test/test_cuda/test_gguf.py@@ -183,7 +183,7 @@ class TestAutoRound(unittest.TestCase):@require_ggufdef test_vlm_gguf(self):model_name = "/models/Qwen2.5-VL-7B-Instruct"- from auto_round.mllm.autoround_mllm import AutoRoundMLLM+ from auto_round import AutoRoundMLLMfrom auto_round.utils import mllm_load_modelmodel, processor, tokenizer, image_processor = mllm_load_model(model_name)@@ -205,7 +205,7 @@ class TestAutoRound(unittest.TestCase):shutil.rmtree("./saved", ignore_errors=True)model_name = "/models/gemma-3-12b-it"- from auto_round.mllm.autoround_mllm import AutoRoundMLLM+ from auto_round import AutoRoundMLLMfrom auto_round.utils import mllm_load_modelmodel, processor, tokenizer, image_processor = mllm_load_model(model_name)@@ -230,7 +230,7 @@ class TestAutoRound(unittest.TestCase):@require_ggufdef test_llama_4(self):model_name = "/dataset/Llama-4-Scout-17B-16E-Instruct/"- from auto_round.mllm.autoround_mllm import AutoRoundMLLM+ from auto_round import AutoRoundMLLMfrom auto_round.utils import mllm_load_modelmodel, processor, tokenizer, image_processor = mllm_load_model(model_name, use_auto_mapping=False)diff --git a/test/test_cuda/test_vlms.py b/test/test_cuda/test_vlms.pyindex d18c53a66..b3bf0f0fd 100644--- a/test/test_cuda/test_vlms.py+++ b/test/test_cuda/test_vlms.py@@ -130,6 +130,33 @@ class TestAutoRound(unittest.TestCase):self.assertTrue(len(block_name) == 1)self.assertTrue(get_block_names(model) == block_name)+ def test_mllm_detect(self):+ from auto_round.utils import is_mllm_model, llm_load_model, mllm_load_model++ for model_name in [+ "/models/Llama-3.2-11B-Vision-Instruct/",+ "/models/deepseek-vl2-tiny/",+ "/models/gemma-3-12b-it/",+ "/models/Phi-3.5-vision-instruct",+ "/models/Qwen2-VL-2B-Instruct",+ "/models/SmolVLM-256M-Instruct",+ "/models/Llama-4-Maverick-17B-128E-Instruct",+ "/models/Mistral-Small-3.1-24B-Instruct-2503",+ "/models/InternVL3-1B",+ "/models/pixtral-12b",+ ]:+ self.assertTrue(is_mllm_model(model_name))+ try:+ model, _, _, _ = mllm_load_model(model_name)+ except:+ continue+ self.assertTrue(is_mllm_model(model))++ for model_name in ["/models/glm-4-9b-chat", "/models/Qwen2.5-1.5B-Instruct/"]:+ self.assertFalse(is_mllm_model(model_name))+ model, _, _ = llm_load_model(model_name)+ self.assertFalse(is_mllm_model(model))+if __name__ == "__main__":unittest.main()