Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

[STEP 1] merge all api(MLLM, Adam) into AutoRound (#791)

Heng Guobd4bc2c92128
diff --git a/auto_round/__init__.py b/auto_round/__init__.py
index 93cce6299..15bbc373d 100644
--- a/auto_round/__init__.py
+++ b/auto_round/__init__.py
@@ -11,8 +11,10 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-from auto_round.autoround import AutoRound, AutoRoundAdam
-from auto_round.mllm import AutoRoundMLLM
+from auto_round.autoround import AutoRound
+
+# support for old api
+from auto_round.autoround import AutoRoundLLM, AutoRoundMLLM, AutoRoundAdam
from auto_round.utils import LazyImport
diff --git a/auto_round/__main__.py b/auto_round/__main__.py
index 23749e21d..1671d157d 100644
--- a/auto_round/__main__.py
+++ b/auto_round/__main__.py
@@ -11,41 +11,658 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
+import argparse
+import logging
+import os
+import re
import sys
+from auto_round.compressors import BaseCompressor
+from auto_round.eval.eval_cli import EvalArgumentParser, _eval_init, eval, eval_task_by_task, eval_with_vllm
+from auto_round.schemes import PRESET_SCHEMES
+from auto_round.utils import (
+ clear_memory,
+ get_device_and_parallelism,
+ get_model_dtype,
+ set_cuda_visible_devices,
+)
-def run_eval():
- if "--vlmeval" in sys.argv:
- sys.argv.remove("--vlmeval")
- run_vlmeavl()
- elif "--lmms" in sys.argv:
- sys.argv.remove("--lmms")
- run_lmms()
- elif "--vllm" in sys.argv:
- sys.argv.remove("--vllm")
- from auto_round.script.llm import eval_with_vllm, setup_eval_parser
-
- args = setup_eval_parser()
- eval_with_vllm(args)
+
+class BasicArgumentParser(argparse.ArgumentParser):
+
+ def __init__(self, *args, **kwargs):
+ super().__init__(*args, **kwargs)
+ self.add_argument(
+ "--model", "--model_name", "--model_name_or_path", default="facebook/opt-125m", help="model name or path"
+ )
+
+ self.add_argument("--mllm", action="store_true", help="whether to quant multi-modal model.")
+
+ self.add_argument("--eval", action="store_true", help="whether to use eval only mode")
+
+ self.add_argument(
+ "--scheme",
+ default="W4A16",
+ type=str,
+ # choices=["W4A16", "W2A16", "W3A16", "W8A16", "MXFP4", "MXFP8", "NVFP4", "FPW8A16", "FP8_STATIC"],
+ help="quantization scheme",
+ )
+
+ self.add_argument("--bits", default=None, type=int, help="number of weight bits")
+ self.add_argument("--group_size", default=None, type=int, help="group size")
+ self.add_argument("--asym", action="store_true", help="whether to use asym quantization")
+ self.add_argument("--data_type", "--dtype", default=None, help="data type for tuning, 'int', 'mx_fp' and etc")
+ self.add_argument("--act_bits", default=None, type=int, help="activation bits")
+ self.add_argument("--act_group_size", default=None, type=int, help="activation group size")
+ self.add_argument(
+ "--super_group_size", default=None, type=int, help="the number of super group size when use double quant."
+ )
+
+ self.add_argument(
+ "--super_bits", default=None, type=int, help="number of scale and mins quant bits for double quant."
+ )
+ self.add_argument("--act_data_type", "--act_dtype", default=None, type=str, help="activation data type")
+
+ self.add_argument("--disable_act_dynamic", action="store_true", help="activation static quantization")
+
+ self.add_argument(
+ "--device_map",
+ "--device",
+ "--devices",
+ default="0",
+ type=str,
+ help="the device to be used for tuning. "
+ "Currently, device settings support CPU, GPU, and HPU."
+ "The default is set to cuda:0,"
+ "allowing for automatic detection and switch to HPU or CPU."
+ "set --device 0,1,2 to use multiple cards.",
+ )
+
+ self.add_argument(
+ "--dataset", default="NeelNanda/pile-10k", type=str, help="the dataset for quantization training"
+ )
+
+ self.add_argument(
+ "--minmax_lr",
+ default=None,
+ type=float,
+ help="minmax learning rate, if None, it will beset to be the same with lr",
+ )
+
+ self.add_argument(
+ "--mem_per_param_scale",
+ default=13,
+ type=float,
+ help="Scale factor for memory per parameter, used to adjust memory usage estimation for tuning",
+ )
+
+ self.add_argument("--seed", default=42, type=int, help="random seed")
+
+ self.add_argument("--adam", action="store_true", help="whether to use adam optimizer instead of SignSGD")
+
+ self.add_argument("--gradient_accumulate_steps", default=1, type=int, help="gradient accumulate steps")
+
+ self.add_argument("--nblocks", default=1, type=int, help="how many blocks to tune together")
+
+ self.add_argument("--low_gpu_mem_usage", action="store_true", help="offload intermediate features to cpu")
+
+ self.add_argument("--format", default="auto_round", type=str, help="the format to save the model")
+
+ self.add_argument(
+ "--scale_dtype",
+ default="fp16",
+ choices=["fp16", "float16", "bf16", "bfloat16", "fp32", "float32"],
+ help="scale data type to use for quantization",
+ )
+
+ self.add_argument(
+ "--output_dir", default="./tmp_autoround", type=str, help="the directory to save quantized model"
+ )
+
+ self.add_argument("--disable_amp", action="store_true", help="disable amp")
+
+ self.add_argument(
+ "--disable_minmax_tuning", action="store_true", help="whether to disable enable weight minmax tuning"
+ )
+
+ self.add_argument("--enable_norm_bias_tuning", action="store_true", help="whether to enable norm bias tuning")
+
+ self.add_argument(
+ "--disable_trust_remote_code", action="store_true", help="whether to disable trust_remote_code"
+ )
+
+ self.add_argument(
+ "--disable_quanted_input",
+ action="store_true",
+ help="whether to disuse the output of quantized block to tune the next block",
+ )
+
+ self.add_argument("--quant_lm_head", action="store_true", help="whether to quant lm_head")
+
+ self.add_argument(
+ "--low_cpu_mem_mode",
+ default=0,
+ type=int,
+ choices=[0, 1, 2],
+ help="choose which low cpu memory mode to use. "
+ "Can significantly reduce cpu memory footprint but cost more time."
+ "1 means choose block-wise mode, load the weights of each block"
+ " from disk when tuning and release the memory of the block after tuning."
+ "2 means choose layer-wise mode, load the weights of each layer from disk when tuning,"
+ " minimum memory consumption and also slowest running speed."
+ "others means not use low cpu memory. Default to 0, not use low cpu memory.",
+ )
+
+ self.add_argument(
+ "--low_cpu_mem_tmp_dir",
+ default=None,
+ type=str,
+ help="temporary work space to store the temporary files "
+ "when using low cpu memory mode. Will remove after tuning.",
+ )
+
+ self.add_argument(
+ "--model_dtype",
+ default=None,
+ type=str,
+ choices=["fp16", "float16", "bf16", "bfloat16", "fp32", "float32"],
+ help="force to convert the dtype, some backends supports fp16 dtype better",
+ )
+
+ self.add_argument(
+ "--fp_layers", default="", type=str, help="list of Layer names to maintain original data type"
+ )
+
+ self.add_argument(
+ "--not_use_best_mse",
+ action="store_true",
+ help="whether to use the iter of best mes loss in the tuning phase",
+ )
+
+ self.add_argument(
+ "--to_quant_block_names",
+ default=None,
+ type=str,
+ help="Names of quantitative blocks, please use commas to separate them.",
+ )
+
+ self.add_argument("--enable_torch_compile", action="store_true", help="whether to enable torch compile")
+
+ self.add_argument("--enable_alg_ext", action="store_true", help="whether to enable probably better algorithm")
+
+ self.add_argument(
+ "--disable_deterministic_algorithms",
+ action="store_true",
+ help="deprecated, disable torch deterministic algorithms.",
+ )
+ self.add_argument(
+ "--enable_deterministic_algorithms", action="store_true", help="enable torch deterministic algorithms."
+ )
+
+ self.add_argument(
+ "--disable_opt_rtn",
+ action="store_true",
+ help="whether to disable optimization of the RTN mode(iters=0) (default is False).",
+ )
+
+ ## ======================= MLLM =======================
+ self.add_argument(
+ "--quant_nontext_module",
+ action="store_true",
+ help="whether to quantize non-text module, e.g. vision component",
+ )
+
+ self.add_argument(
+ "--extra_data_dir",
+ default=None,
+ type=str,
+ help="dataset dir for storing images/audio/videos. "
+ "Can be a dir path or multiple dir path with format as "
+ "'image=path_to_image,video=path_to_video,audio=path_to_audio'"
+ "By default, it will search in the relative path, "
+ "and if not find, will automatic download.",
+ )
+
+ self.add_argument(
+ "--template",
+ default=None,
+ type=str,
+ help="the template for building training dataset. It can be a custom one.",
+ )
+
+ ## ======================= eval =======================
+ self.add_argument(
+ "--tasks",
+ "--task",
+ nargs="?",
+ const="lambada_openai,hellaswag,winogrande,piqa,mmlu,wikitext,truthfulqa_mc1,"
+ "openbookqa,boolq,arc_easy,arc_challenge",
+ default=None,
+ help="lm-eval tasks",
+ )
+
+ self.add_argument("--eval_bs", default=None, type=int, help="batch size in evaluation")
+
+ self.add_argument(
+ "--limit",
+ type=float,
+ default=None,
+ metavar="N|0<N<1",
+ help="Limit the number of examples per task. "
+ "If <1, limit is a percentage of the total number of examples.",
+ )
+
+ self.add_argument("--eval_task_by_task", action="store_true", help="whether to eval task by task.")
+
+ self.add_argument(
+ "--eval_model_dtype", default=None, type=str, help="the torch_dytpe to load the model for evaluation."
+ )
+
+
+def setup_parser():
+ parser = BasicArgumentParser()
+
+ parser.add_argument("--batch_size", "--train_bs", "--bs", default=8, type=int, help="train batch size")
+
+ parser.add_argument("--iters", "--iter", default=200, type=int, help="iteration to tune each block")
+
+ parser.add_argument(
+ "--seqlen", "--seq_len", default=2048, type=int, help="sequence length of the calibration samples"
+ )
+
+ parser.add_argument("--nsamples", "--nsample", default=128, type=int, help="number of samples")
+
+ parser.add_argument(
+ "--lr", default=None, type=float, help="learning rate, if None, it will be set to 1.0/iters automatically"
+ )
+
+ args = parser.parse_args()
+ return args
+
+
+def setup_best_parser():
+ parser = BasicArgumentParser()
+
+ parser.add_argument("--batch_size", "--train_bs", "--bs", default=8, type=int, help="train batch size")
+
+ parser.add_argument("--iters", "--iter", default=1000, type=int, help="iterations to tune each block")
+
+ parser.add_argument(
+ "--seqlen", "--seq_len", default=2048, type=int, help="sequence length of the calibration samples"
+ )
+
+ parser.add_argument("--nsamples", "--nsample", default=512, type=int, help="number of samples")
+
+ parser.add_argument(
+ "--lr", default=None, type=float, help="learning rate, if None, it will be set to 1.0/iters automatically"
+ )
+
+ args = parser.parse_args()
+ args.low_gpu_mem_usage = True
+
+ return args
+
+
+def setup_light_parser():
+ parser = BasicArgumentParser()
+
+ parser.add_argument("--batch_size", "--train_bs", "--bs", default=8, type=int, help="train batch size")
+
+ parser.add_argument("--iters", "--iter", default=50, type=int, help="iterations to tune each block")
+
+ parser.add_argument(
+ "--seqlen", "--seq_len", default=2048, type=int, help="sequence length of the calibration samples"
+ )
+
+ parser.add_argument("--nsamples", "--nsample", default=128, type=int, help="number of samples")
+
+ parser.add_argument(
+ "--lr", default=5e-3, type=float, help="learning rate, if None, it will be set to 1.0/iters automatically"
+ )
+
+ args = parser.parse_args()
+
+ return args
+
+
+def setup_fast_parser():
+ parser = BasicArgumentParser()
+
+ parser.add_argument("--batch_size", "--train_bs", "--bs", default=4, type=int, help="train batch size")
+
+ parser.add_argument("--iters", default=200, type=int, help="iterations to tune each block")
+
+ parser.add_argument(
+ "--seqlen", "--seq_len", default=512, type=int, help="sequence length of the calibration samples"
+ )
+
+ parser.add_argument("--nsamples", "--nsample", default=128, type=int, help="number of samples")
+
+ parser.add_argument(
+ "--lr", default=None, type=float, help="learning rate, if None, it will be set to 1.0/iters automatically"
+ )
+
+ args = parser.parse_args()
+
+ return args
+
+
+def setup_eval_parser():
+
+ parser = EvalArgumentParser()
+ args = parser.parse_args()
+ return args
+
+
+def tune(args):
+ if args.eval_bs is None:
+ args.eval_bs = "auto"
+ from transformers import AutoConfig, AutoModelForCausalLM, AutoTokenizer
+
+ from auto_round.utils import detect_device, get_library_version, logger
+
+ if args.format is None:
+ args.format = "auto_round"
+
+ formats = args.format.lower().replace(" ", "").split(",")
+ from auto_round.utils import SUPPORTED_FORMATS
+
+ for format in formats:
+ if format not in SUPPORTED_FORMATS:
+ raise ValueError(f"{format} is not supported, we only support {SUPPORTED_FORMATS}")
+
+ if "auto_gptq" in args.format and args.asym is True:
+ logger.warning(
+ "the auto_gptq kernel has issues with asymmetric quantization. "
+ "It is recommended to use sym quantization or --format='auto_round'"
+ )
+
+ if "marlin" in args.format and args.asym is True:
+ raise RuntimeError("marlin backend only supports sym quantization, please remove --asym")
+
+ # Must set this before import torch
+ # set_cuda_visible_devices(args.device_map)
+ device_str, use_auto_mapping = get_device_and_parallelism(args.device_map)
+
+ import torch
+
+ if args.enable_torch_compile:
+ logger.info(
+ "`torch.compile` is enabled to reduce tuning costs. "
+ "If it causes issues, you can disable it by removing `--enable_torch_compile` argument."
+ )
+
+ model_name = args.model
+ if model_name[-1] == "/":
+ model_name = model_name[:-1]
+ logger.info(f"start to quantize {model_name}")
+
+ from auto_round import AutoRound
+
+ if "bloom" in model_name:
+ args.low_gpu_mem_usage = False
+
+ if args.quant_lm_head:
+ for format in formats:
+ if "auto_round" not in format and "fake" not in format:
+ auto_round_formats = [s for s in SUPPORTED_FORMATS if s.startswith("auto_round")]
+ raise ValueError(
+ f"{format} is not supported for lm-head quantization, please change to {auto_round_formats}"
+ )
+
+ enable_torch_compile = True if "--enable_torch_compile" in sys.argv else False
+ sym = None # the default value should be None now
+ if args.asym: # if the scheme is asym, how to set it to sym is an issue
+ sym = False
+ act_dynamic = None
+ if args.disable_act_dynamic:
+ act_dynamic = False
+ scheme = args.scheme.upper()
+ if scheme not in PRESET_SCHEMES:
+ raise ValueError(f"{scheme} is not supported. only {PRESET_SCHEMES.keys()} are supported ")
+ if args.disable_deterministic_algorithms:
+ logger.warning(
+ "default not use deterministic_algorithms. disable_deterministic_algorithms is deprecated,"
+ " please use enable_deterministic_algorithms instead. "
+ )
+
+ from auto_round.compressors import (
+ ExtraConfig,
+ MLLMExtraConfig,
+ SchemeExtraConfig,
+ TuningExtraConfig,
+ )
+
+ extra_config = ExtraConfig()
+ tuning_config = TuningExtraConfig(
+ amp=not args.disable_amp,
+ lr=args.lr,
+ minmax_lr=args.minmax_lr,
+ enable_quanted_input=not args.disable_quanted_input,
+ nblocks=args.nblocks,
+ enable_minmax_tuning=not args.disable_minmax_tuning,
+ enable_norm_bias_tuning=args.enable_norm_bias_tuning,
+ enable_deterministic_algorithms=args.enable_deterministic_algorithms,
+ to_quant_block_names=args.to_quant_block_names,
+ disable_opt_rtn=args.disable_opt_rtn,
+ enable_alg_ext=args.enable_alg_ext,
+ scale_dtype=args.scale_dtype,
+ )
+ scheme_config = SchemeExtraConfig(
+ bits=args.bits,
+ group_size=args.group_size,
+ sym=sym,
+ data_type=args.data_type,
+ act_bits=args.act_bits,
+ act_group_size=args.act_group_size,
+ act_data_type=args.act_data_type,
+ act_dynamic=act_dynamic,
+ super_bits=args.super_bits,
+ super_group_size=args.super_group_size,
+ )
+ mllm_config = MLLMExtraConfig(
+ quant_nontext_module=args.quant_nontext_module, extra_data_dir=args.extra_data_dir, template=args.template
+ )
+ extra_config.tuning_config = tuning_config
+ extra_config.scheme_config = scheme_config
+ extra_config.mllm_config = mllm_config
+
+ autoround: BaseCompressor = AutoRound(
+ model=model_name,
+ scheme=scheme,
+ dataset=args.dataset,
+ iters=args.iters,
+ seqlen=args.seqlen,
+ nsamples=args.nsamples,
+ batch_size=args.batch_size,
+ gradient_accumulate_steps=args.gradient_accumulate_steps,
+ low_gpu_mem_usage=args.low_gpu_mem_usage,
+ device_map=args.device_map,
+ enable_torch_compile=enable_torch_compile,
+ seed=args.seed,
+ fp_layers=args.fp_layers,
+ not_use_best_mse=args.not_use_best_mse,
+ enable_adam=args.adam,
+ extra_config=extra_config,
+ )
+
+ model_name = args.model.rstrip("/")
+
+ if model_name.split("/")[-1].strip(".") == "" and "gguf" not in args.format:
+ if autoround.group_size <= 0:
+ if "fp" in autoround.act_data_type:
+ suffix = f"afp{autoround.act_bits}"
+ else:
+ suffix = f"a{autoround.act_bits}"
+ else:
+ suffix = f"g{autoround.group_size}"
+ export_dir = os.path.join(args.output_dir, f"w{autoround.bits}{suffix}")
+ elif model_name.split("/")[-1].strip(".") == "" and "gguf" in args.format:
+ export_dir = args.output_dir
+ elif model_name.split("./")[-1].strip("./") != "" and "gguf" in args.format:
+ export_dir = os.path.join(args.output_dir, model_name.split("/")[-1] + "-gguf")
else:
- from auto_round.script.llm import setup_eval_parser
+ if autoround.group_size <= 0:
+ if "fp" in autoround.act_data_type:
+ suffix = f"afp{autoround.act_bits}"
+ else:
+ suffix = f"a{autoround.act_bits}"
+ else:
+ suffix = f"g{autoround.group_size}"
+ export_dir = os.path.join(args.output_dir, model_name.split("/")[-1] + f"-w{autoround.bits}{suffix}")
+
+ model, folders = autoround.quantize_and_save(export_dir, format=args.format) # pylint: disable=E1101
+ tokenizer = autoround.tokenizer # pylint: disable=E1101
+
+ if args.low_cpu_mem_mode == 1 or args.low_cpu_mem_mode == 2:
+ import shutil
+
+ shutil.rmtree(args.low_cpu_mem_tmp_dir, ignore_errors=True)
+
+ model.eval()
+ clear_memory()
+
+ lm_eval_version = get_library_version("lm-eval")
+
+ eval_folder = folders[-1]
+ if args.tasks is None or args.tasks == "" or eval_folder is None:
+ return
+
+ tasks = args.tasks
+ if isinstance(tasks, str):
+ tasks = tasks.split(",")
+
+ from lm_eval.utils import make_table # pylint: disable=E0401
+
+ logger.info(f"Using lm-eval version {lm_eval_version}")
+ eval_gguf_model = False
+ for file in os.listdir(eval_folder):
+ if file.endswith("gguf"):
+ eval_gguf_model = True
+ break
+
+ import time
+
+ eval_model_dtype = get_model_dtype(args.eval_model_dtype, "auto")
+
+ if autoround.act_bits <= 8 or eval_gguf_model:
+ if eval_gguf_model:
+ # for file in os.listdir(eval_folder):
+ # gguf_file = file
+ gguf_file = None
+ gguf_format = None # Initialize gguf_format to None
+ # gguf folder only contains one file
+ for format in formats:
+ if format.startswith("gguf"):
+ gguf_format = format.split(":")[-1].upper()
+ if gguf_format is None: # Validate gguf_format after the loop
+ logger.error("No valid gguf format found in formats. Please check the input.")
+ sys.exit(-1)
+ for file in os.listdir(eval_folder):
+ if gguf_format in file:
+ gguf_file = file
+
+ logger.warning("evaluating gguf model is an experimental feature, the accuracy may be not correct.")
+ if eval_model_dtype == "float32" or eval_model_dtype == "auto":
+ logger.warning(
+ "set '--eval_model_dtype bf16' can significantly speed up evaluation for gguf model,"
+ " but may affect accuracy."
+ )
+ if gguf_file is None:
+ logger.error("Cannot find correct gguf file for evaluation, please check.")
+ sys.exit(-1)
+ model = AutoModelForCausalLM.from_pretrained(
+ eval_folder, gguf_file=gguf_file, device_map="auto", torch_dtype=eval_model_dtype
+ )
+ model.eval()
+ tokenizer = AutoTokenizer.from_pretrained(eval_folder, gguf_file=gguf_file)
+ else:
+ if hasattr(model, "hf_device_map") and len(model.hf_device_map) > 1:
+ from accelerate.big_modeling import dispatch_model
+
+ dispatch_model(model, model.hf_device_map)
+ else:
+ device_str = detect_device(device_str)
+ model = model.to(device_str)
+ if model.dtype != eval_model_dtype and eval_model_dtype != "auto":
+ model.to(getattr(torch, eval_model_dtype))
- args = setup_eval_parser()
if args.eval_task_by_task:
- from auto_round.script.llm import eval_task_by_task
+ eval_task_by_task(
+ model,
+ tokenizer=tokenizer,
+ device=device_str,
+ tasks=args.tasks,
+ limit=args.limit,
+ batch_size=args.eval_bs,
+ eval_model_dtype=eval_model_dtype,
+ )
+ else:
+ if args.eval_bs is None or args.eval_bs == "auto":
+ logger.warning("This API does not support auto currently, reset eval_bs to 16")
+ args.eval_bs = 16
+ from auto_round.eval.evaluation import simple_evaluate_user_model
+ st = time.time()
+ add_bos_token = False
+ if "llama" in args.model.lower():
+ add_bos_token = True
+ res = simple_evaluate_user_model(
+ model,
+ tokenizer,
+ tasks=tasks,
+ batch_size=args.eval_bs,
+ limit=args.limit,
+ device=device_str,
+ eval_model_dtype=eval_model_dtype,
+ add_bos_token=add_bos_token,
+ )
+ print(make_table(res))
+ print("evaluation running time=%ds" % (time.time() - st))
+ else:
+ if args.eval_task_by_task:
eval_task_by_task(
- model=args.model,
- device=args.device,
+ eval_folder,
+ device=device_str,
tasks=args.tasks,
batch_size=args.eval_bs,
- trust_remote_code=not args.disable_trust_remote_code,
- eval_model_dtype=args.eval_model_dtype,
+ limit=args.limit,
+ eval_model_dtype=eval_model_dtype,
)
else:
- from auto_round.script.llm import eval
+ from auto_round.eval.evaluation import simple_evaluate
- eval(args)
+ tasks, model_args, device_str = _eval_init(
+ args.tasks, eval_folder, args.device_map, args.disable_trust_remote_code, dtype=eval_model_dtype
+ )
+ st = time.time()
+ if "llama" in args.model.lower():
+ model_args += ",add_bos_token=True"
+ res = simple_evaluate(
+ model="hf",
+ model_args=model_args,
+ tasks=tasks,
+ device=device_str,
+ batch_size=args.eval_bs,
+ limit=args.limit,
+ )
+ print(make_table(res))
+ print("evaluation running time=%ds" % (time.time() - st))
+
+
+def run_eval():
+ args = setup_eval_parser()
+ if args.eval_task_by_task:
+ eval_task_by_task(
+ model=args.model,
+ device=args.device,
+ tasks=args.tasks,
+ batch_size=args.eval_bs,
+ trust_remote_code=not args.disable_trust_remote_code,
+ eval_model_dtype=args.eval_model_dtype,
+ )
+ else:
+ eval(args)
def run():
@@ -53,8 +670,6 @@ def run():
sys.argv.remove("--eval")
run_eval()
else:
- from auto_round.script.llm import setup_parser, tune
-
args = setup_parser()
tune(args)
@@ -65,40 +680,19 @@ def run_mllm():
def run_best():
- from auto_round.script.llm import setup_best_parser, tune
-
args = setup_best_parser()
tune(args)
def run_light():
- from auto_round.script.llm import setup_light_parser, tune
-
args = setup_light_parser()
tune(args)
def run_fast():
- from auto_round.script.llm import setup_fast_parser, tune
-
args = setup_fast_parser()
tune(args)
-def run_lmms():
- # from auto_round.script.lmms_eval import setup_lmms_args, eval
- from auto_round.script.mllm import lmms_eval, setup_lmms_parser
-
- args = setup_lmms_parser()
- lmms_eval(args)
-
-
-def run_vlmeavl():
- from auto_round.script.mllm import setup_lmeval_parser, vlmeval
-
- args = setup_lmeval_parser()
- vlmeval(args)
-
-
if __name__ == "__main__":
run()
diff --git a/auto_round/autoround.py b/auto_round/autoround.py
index 9d56cb8b4..4074213a9 100644
--- a/auto_round/autoround.py
+++ b/auto_round/autoround.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2023 Intel Corporation
+# Copyright (c) 2025 Intel Corporation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -11,94 +11,25 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
+from __future__ import annotations
-import copy
-import os
-import re
-import sys
-import time
-import traceback
-from dataclasses import asdict, fields
-from enum import Enum
from typing import Any, Callable, Union
-import accelerate
import torch
-from accelerate.big_modeling import dispatch_model, infer_auto_device_map
-from torch import autocast
-from tqdm import tqdm
-from transformers import set_seed
-from auto_round.data_type import QUANT_FUNC_WITH_DTYPE
-from auto_round.data_type.utils import reshape_pad_tensor_by_group_size
-from auto_round.export.export_to_autoround import AutoRoundFormat
-from auto_round.export.export_to_gguf.config import GGUF_CONFIG, GGUF_INNER_CONFIG, ModelType
-from auto_round.logger import logger
-from auto_round.low_cpu_mem.utils import get_layers_before_block
-from auto_round.schemes import QuantizationScheme, preset_name_to_scheme
-from auto_round.sign_sgd import SignSGD
-from auto_round.special_model_handler import _handle_moe_model
-from auto_round.utils import (
- INNER_SUPPORTED_LAYER_TYPES,
- SUPPORTED_DTYPES,
- SUPPORTED_FORMATS,
- SUPPORTED_LAYER_TYPES,
- TORCH_VERSION_AT_LEAST_2_6,
- CpuInfo,
- _gguf_args_check,
- _is_fp8_linear,
- _is_fp8_model,
- block_forward,
- check_and_mark_fp8_model,
- check_is_cpu,
- check_need_act_calibration,
- check_seqlen_compatible,
- check_skippable_keywords,
- check_to_quantized,
- clear_memory,
- collect_best_params,
- compile_func,
- convert_dtype_str2torch,
- convert_fp8_layer_to_linear,
- convert_fp8_model_to_16b_model,
- copy_python_files_from_model_cache,
- detect_device,
- estimate_tuning_block_mem,
- find_matching_blocks,
- flatten_list,
- get_block_names,
- get_device_memory,
- get_layer_config_by_gguf_format,
- get_layer_features,
- get_layer_names_in_block,
- get_lm_head_name,
- get_max_vram,
- get_module,
- get_quant_keys,
- get_shared_keys,
- htcore,
- infer_bits_by_data_type,
- init_cache,
- is_debug_mode,
- is_mx_fp,
- is_nv_fp,
- is_optimum_habana_available,
- is_standard_fp,
- is_static_wfp8afp8,
- is_wfp8afp8,
- llm_load_model,
- mv_module_from_gpu,
- reset_params,
- set_amax_for_all_moe_layers,
- set_module,
- to_device,
- to_dtype,
- unsupport_meta_device,
+from auto_round.compressors import (
+ AdamCompressor,
+ BaseCompressor,
+ ExtraConfig,
+ LLMCompressor,
+ MLLMCompressor,
)
-from auto_round.wrapper import WrapperLinear, WrapperMultiblock, unwrapper_block, unwrapper_layer, wrapper_block
+from auto_round.logger import deprecated, logger
+from auto_round.schemes import QuantizationScheme
+from auto_round.utils import is_mllm_model
-class AutoRound(object):
+class AutoRound:
"""Automatic weight rounding (Signed Gradient Descent) for LLM quantization
Reference:
@@ -128,3137 +59,120 @@ class AutoRound(object):
super_bits: int | None
super_group_size: int | None
- def __init__(
- self,
+ def __new__(
+ cls,
model: Union[torch.nn.Module, str],
- tokenizer=None,
- scheme: Union[str, dict, QuantizationScheme] = "W4A16",
- layer_config: dict[str, Union[str, dict, QuantizationScheme]] = None,
- dataset: Union[str, list, tuple, torch.utils.data.DataLoader] = "NeelNanda/pile-10k",
- iters: int = 200,
- seqlen: int = 2048,
- nsamples: int = 128,
- batch_size: int = 8,
- gradient_accumulate_steps: int = 1,
- low_gpu_mem_usage: bool = False,
- device_map: Union[str, torch.device, int, dict] = 0,
- enable_torch_compile: bool = False,
- seed: int = 42,
- **kwargs,
- ):
- """Initialize AutoRound with quantization and tuning configuration.
-
- Args:
- model (torch.nn.Module | str): Model object or model name to load.
- tokenizer: Tokenizer for text processing. Required if `model` is not a string and `iters > 0`.
- scheme (str| dict | QuantizationScheme ): A preset scheme that defines the quantization configurations
- bits (int, optional): Weight quantization bits. Defaults to 4.
- group_size (int, optional): Weight quantization group size. Defaults to 128.
- sym (bool, optional): Symmetric weight quantization. Defaults to True.
- layer_config (dict, optional): Layer-wise quantization config. Defaults to None.
- batch_size (int, optional): Calibration batch size. Defaults to 8.
- amp (bool, optional): Use AMP for tuning. Defaults to True.
- device (str | torch.device | int, optional): Compute device. Defaults to 0.
- dataset (str | list | tuple | DataLoader, optional): Calibration data. Defaults to "NeelNanda/pile-10k".
- enable_minmax_tuning (bool, optional): Enable weight min-max tuning. Defaults to True.
- lr (float, optional): Learning rate; if None, set to 1.0 / iters except when iters==0.
- minmax_lr (float, optional): Learning rate for min-max tuning; defaults to `lr`.
- low_gpu_mem_usage (bool, optional): Lower GPU memory mode. Defaults to False.
- iters (int, optional): Optimization iterations. Defaults to 200.
- seqlen (int, optional): Calibration sequence length. Defaults to 2048.
- nsamples (int, optional): Number of calibration samples. Defaults to 128.
- seed (int, optional): Random seed. Defaults to 42.
- gradient_accumulate_steps (int, optional): Gradient accumulation steps. Defaults to 1.
- data_type (str, optional): Weight data type string, e.g., "int". Defaults to "int".
- act_bits (int, optional): Activation quantization bits. Defaults to 16.
- act_group_size (int, optional): Activation group size. Defaults to None.
- act_sym (bool, optional): Symmetric activation quantization. Defaults to None.
- act_data_type (str, optional): Activation data type; inherits weight dtype if None and act_bits < 16.
- act_dynamic (bool, optional): Dynamic activation quantization. Defaults to True.
- enable_torch_compile (bool, optional): Enable torch.compile for quant blocks/layers. Defaults to False.
- device_map (str | dict, optional): Device placement map. Defaults to None.
- disable_opt_rtn (bool, optional): Disable RTN-mode optimization (iters=0). Defaults to False.
- enable_alg_ext (bool, optional): Enable algorithm extension (primarily for INT2). Defaults to False.
- **kwargs: Backward compatible options:
- - enable_alg_ext, quant_lm_head, lr, lr_scheduler, sampler, not_use_best_mse, dynamic_max_gap,
- super_group_size, super_bits, scale_dtype ("fp16" etc.),
- nblocks, low_cpu_mem_usage, to_quant_block_names,
- enable_norm_bias_tuning, enable_quanted_input,
- disable_deterministic_algorithms, vlm, static_kv_dtype
- Raises:
- ValueError: If invalid device is provided or tokenizer is missing for non-str model with iters > 0.
- RuntimeError: If model parameters are on meta device.
- Example:
- Layer-wise configuration structure:
-
- >>> layer_config = {
- ... "layer1": {
- ... "data_type": "int",
- ... "bits": 4,
- ... "group_size": 128,
- ... "sym": True,
- ... "act_data_type": None,
- ... "act_bits": 16,
- ... "act_group_size": None,
- ... "act_sym": None,
- ... },
- ... # ...
- ... }
- """
- self.scheme = None
- self._parse_and_set_scheme(scheme, kwargs)
-
- # Extra/legacy kwargs for backward compatibility
- # Major version releases may pack them with extra configuration options
- amp = kwargs.pop("amp", True)
- lr = kwargs.pop("lr", None)
- enable_alg_ext = kwargs.pop("enable_alg_ext", False)
- enable_minmax_tuning = kwargs.pop("enable_minmax_tuning", True)
- minmax_lr = kwargs.pop("minmax_lr", None)
- disable_opt_rtn = kwargs.pop("disable_opt_rtn", False)
- lr_scheduler = kwargs.pop("lr_scheduler", None)
- sampler = kwargs.pop("sampler", "rand")
- not_use_best_mse = kwargs.pop("not_use_best_mse", False)
- dynamic_max_gap = kwargs.pop("dynamic_max_gap", -1)
- scale_dtype = kwargs.pop("scale_dtype", "fp16")
- nblocks = kwargs.pop("nblocks", 1)
- low_cpu_mem_usage = kwargs.pop("low_cpu_mem_usage", False)
- to_quant_block_names: Union[str, list, None] = kwargs.pop("to_quant_block_names", None)
- enable_norm_bias_tuning: bool = kwargs.pop("enable_norm_bias_tuning", False)
- enable_quanted_input: bool = kwargs.pop("enable_quanted_input", True)
- disable_deterministic_algorithms = kwargs.pop("disable_deterministic_algorithms", True)
- enable_deterministic_algorithms = kwargs.pop("enable_deterministic_algorithms", False)
- static_kv_dtype = kwargs.pop("static_kv_dtype", None)
- device = kwargs.pop("device", None)
- self.quant_lm_head = kwargs.pop("quant_lm_head", False)
- self.vlm = kwargs.pop("vlm") if "vlm" in kwargs else False
- # Scale factor for RAM usage per parameter.
- self.mem_per_param_scale = kwargs.pop("mem_per_param_scale", None)
-
- if kwargs:
- logger.warning(f"unrecognized keys {list(kwargs.keys())} were passed. Please check them.")
- if "CUBLAS_WORKSPACE_CONFIG" not in os.environ:
- os.environ["CUBLAS_WORKSPACE_CONFIG"] = ":4096:8"
- # deprecated, default not to use torch.use_deterministic_algorithms
- if not disable_deterministic_algorithms or enable_deterministic_algorithms:
- if not disable_deterministic_algorithms:
- logger.warning(
- "default not use deterministic_algorithms. disable_deterministic_algorithms is deprecated,"
- " please use enable_deterministic_algorithms instead. "
- )
-
- torch.use_deterministic_algorithms(True, warn_only=False)
- else:
- torch.use_deterministic_algorithms(True, warn_only=True)
-
- if device is not None:
- logger.warning("`device` is deprecated, please use `device_map` instead")
-
- if device_map is None:
- device_map = 0
-
- # Set device, must place after model loading
- self._set_device(device_map)
-
- if (isinstance(device_map, dict) and device_map) or device_map == "auto":
- self.device_map = device_map
- elif isinstance(device_map, str) and "," in device_map:
- device_map = device_map.replace(" ", "") # Remove any spaces
- self.device_list = [int(dev) for dev in device_map.split(",") if dev.isdigit()]
- self.device_map = "auto"
- else:
- self.device_map = None
- self._set_device_map_in_blocks(self.device_map)
-
- # Model related
- self.quantized = False
- if isinstance(model, str):
- model, tokenizer, low_cpu_mem_usage = llm_load_model(
- model,
- device="cpu",
- low_cpu_mem_mode=low_cpu_mem_usage, # always load cpu first
- )
- elif tokenizer is None and iters > 0:
- raise ValueError("A tokenizer must be set for non-str model input")
- self.low_cpu_mem_usage = bool(low_cpu_mem_usage)
- if unsupport_meta_device(model):
- raise RuntimeError(
- "AutoRound does not support parameters on meta device. "
- "Please use more GPUs by setting `--device 0,1,2,3` or just place the model on CPU."
- )
- check_and_mark_fp8_model(model)
- self.model = model.eval()
- self.tokenizer = tokenizer
- self.shared_cache_keys = get_shared_keys(self.model)
-
- self._parse_layer_config(layer_config) # must place after model init
-
- self.to_quant_block_names = to_quant_block_names
-
- # Tuning hyperparameters
- self.seed = seed
- set_seed(self.seed)
- self.amp = amp
- self.enable_quanted_input = enable_quanted_input
- self.enable_minmax_tuning = enable_minmax_tuning
- self.nsamples = nsamples
- self.enable_norm_bias_tuning = enable_norm_bias_tuning
- self.low_gpu_mem_usage = low_gpu_mem_usage
- self.seqlen = seqlen
- self.batch_size, self.gradient_accumulate_steps = batch_size, gradient_accumulate_steps
- self.nblocks = nblocks
- self.dataset = dataset
- self.iters = iters
- if self.iters < 0:
- logger.warning("`iters` must be non-negative, reset it to 200")
- self.iters = 200
- if self.iters == 0:
- self.lr = 5e-3
- else:
- self.lr = lr or (1.0 / self.iters) # must place after iter setting
- self.minmax_lr = minmax_lr or self.lr
- self.enable_alg_ext = enable_alg_ext
- self.sampler = sampler
- self.not_use_best_mse = not_use_best_mse
- self.dynamic_max_gap = dynamic_max_gap
- self.lr_scheduler = lr_scheduler
- self.optimizer = self._get_optimizer(None)
- self.disable_opt_rtn = disable_opt_rtn
- self.is_packing_immediate = False # whether to pack the layer immediately after tuning
-
- # KV cache, this one does not affect tuning but will collect some infos during tuning
- self.static_kv_dtype = static_kv_dtype
- if self.static_kv_dtype is not None:
- logger.warning("The static kv is experimental and currently has limited support.")
-
- # Model related
- self.quantized = False
- if isinstance(model, str):
- model, tokenizer, low_cpu_mem_usage = llm_load_model(
- model, device=device, low_cpu_mem_mode=low_cpu_mem_usage
- )
- elif tokenizer is None and iters > 0:
- raise ValueError("A tokenizer must be set for non-str model input")
- self.low_cpu_mem_usage = bool(low_cpu_mem_usage)
- if unsupport_meta_device(model):
- raise RuntimeError(
- "AutoRound does not support parameters on meta device. "
- "Please use more GPUs by setting `--device_map 0,1,2,3` or just place the model on CPU."
- )
- self.model = model.eval()
- self.tokenizer = tokenizer
- self.shared_cache_keys = get_shared_keys(self.model)
- if not hasattr(self, "quant_block_list"):
- all_blocks = get_block_names(model)
- self.quant_block_list = find_matching_blocks(model, all_blocks, self.to_quant_block_names)
-
- self.scale_dtype = convert_dtype_str2torch(scale_dtype)
- self._set_amp_dtype()
- self.cache_device = torch.device("cpu") if self.low_gpu_mem_usage else self.device
- if self.act_bits <= 8 and self.amp_dtype == torch.float16:
- logger.warning("force to use bf16 to for quantization tuning when enabling activation quantization")
- self.amp_dtype = torch.bfloat16
- if self.model.dtype != torch.bfloat16: # keep the model's buffer dtype unchanged
- self.model = self.model.to(torch.bfloat16)
- else:
- logger.info(f"using {self.model.dtype} for quantization tuning")
-
- # Some helpers
- self.supported_types = SUPPORTED_LAYER_TYPES
- self.inner_supported_types = INNER_SUPPORTED_LAYER_TYPES
- if "hpu" in str(self.device):
- self.inner_supported_types = tuple(x for x in INNER_SUPPORTED_LAYER_TYPES if x != "FP8Linear")
- self.batch_dim = None
- self.infer_bs_coeff = 1
- self.enable_torch_compile = enable_torch_compile
- self._adjust_torch_compile(enable_torch_compile)
- self._check_configs()
- torch.set_printoptions(precision=3, sci_mode=True)
-
- if is_optimum_habana_available():
- logger.info("optimum Habana is available, import htcore explicitly.")
- import habana_frameworks.torch.core as htcore # pylint: disable=E0401
- import habana_frameworks.torch.hpu as hthpu # pylint: disable=E0401]
-
- def _set_device(self, device_map):
- if hasattr(self, "device") and self.device is not None:
- return
- if isinstance(device_map, (str, torch.device, int)):
- self.device = detect_device(device_map)
-
- elif isinstance(device_map, dict) and device_map:
- tmp_devices = []
- for val in device_map.values():
- if isinstance(val, (str, torch.device, int)): # could optimize
- tmp_device = detect_device(val)
- tmp_device = tmp_device.split(":")[0]
- tmp_devices.append(tmp_device)
- tmp_devices = list(set(tmp_devices))
- if len(tmp_devices) > 1:
- logger.warning(
- f"there are multiple device types in the device_map, "
- f"please make sure they are correct,use the first device {tmp_devices[0]} as the core device "
- )
-
- self.device = tmp_devices[0]
- else:
- raise TypeError(f"device_map should be [str, torch.device, int, dict], but got {type(device_map)}")
-
- def _parse_layer_config(self, layer_config: dict[str, Union[str, dict, QuantizationScheme]]) -> None:
- """Parse and set the layer-wise quantization configuration."""
- # Some other quantization configs
- self.layer_config = {} if layer_config is None else layer_config
- scheme_keys = [f.name for f in fields(QuantizationScheme)]
- for key, item in self.layer_config.items():
- if isinstance(item, str):
- item = asdict(preset_name_to_scheme(item.upper()))
- self.layer_config[key] = item
-
- if isinstance(item, QuantizationScheme):
- config = asdict(item)
- tmp_keys = copy.deepcopy(list(config.keys()))
- for tmp_key in tmp_keys: ## Pop None value to be overridden
- if config[tmp_key] is None:
- config.pop(tmp_key)
- self.layer_config[key] = config
- elif isinstance(item, dict):
- item_keys = item.keys()
- if item_keys not in scheme_keys:
- for item_key in item_keys:
- if item_key not in scheme_keys:
- raise ValueError(
- f"the key {item_key} in layer_config for layer {key} is invalid,"
- f" only {scheme_keys} are supported"
- )
-
- if not self.quant_lm_head or (isinstance(self.scheme, str) and self.scheme.lower().startswith("gguf")):
- return
- for n, _ in self.model.named_modules():
- lm_head_layer_name = n
-
- if (
- hasattr(self.model, "config")
- and self.model.config.tie_word_embeddings
- and hasattr(self.model, "_tied_weights_keys")
- ):
- tied_keys = self.model._tied_weights_keys
- for item in tied_keys:
- if lm_head_layer_name in item: # TODO extend to encoder-decoder layer, seq classification model
- self.quant_lm_head = False
- logger.warning(
- "reset `quant_lm_head` to `False` as quantizing lm_head with tied weights has not been "
- "supported currently"
- )
- break
-
- lm_head_layer_config = self.layer_config[lm_head_layer_name] if lm_head_layer_name in self.layer_config else {}
-
- for key in scheme_keys:
- if key not in lm_head_layer_config:
- lm_head_layer_config[key] = getattr(self, key)
-
- def _parse_and_set_scheme(self, scheme: Union[str, dict, QuantizationScheme], kwargs) -> None:
- """Parse and set the quantization scheme."""
- if isinstance(scheme, QuantizationScheme):
- scheme = asdict(scheme)
- elif isinstance(scheme, dict):
- scheme = scheme
- elif isinstance(scheme, str):
- scheme = scheme.upper()
- self.scheme = scheme
- scheme = asdict(preset_name_to_scheme(scheme))
- scheme_keys = [f.name for f in fields(QuantizationScheme)]
- for key in scheme_keys:
- if key in kwargs and kwargs[key] is not None:
- setattr(self, key, kwargs[key])
- else:
- setattr(self, key, scheme.get(key, None))
- kwargs.pop(key, None)
- if self.act_dynamic is None:
- self.act_dynamic = True
-
- tmp_bits = infer_bits_by_data_type(self.data_type)
- if tmp_bits is not None and tmp_bits < 16 and tmp_bits != self.bits:
- logger.warning(f"'data_type' do not match the specified 'bits' setting. Resetting 'bits' to {tmp_bits}.")
- self.bits = tmp_bits
- if tmp_bits is not None and tmp_bits < 16:
- for supported_dtype in SUPPORTED_DTYPES: # to easily handle dtype mx_fp4 and layer_config={xxx:{bits:8}}
- if self.data_type.startswith(supported_dtype):
- if supported_dtype + str(tmp_bits) == self.data_type: # could not replace FP8_e4m3
- self.data_type = supported_dtype
- break
-
- self.act_group_size = self.act_group_size if self.act_group_size is not None else self.group_size
- self.act_bits = self.act_bits if self.act_bits is not None else 16
- self.act_sym = self.act_sym if self.act_sym is not None else self.sym
-
- if self.act_data_type is None:
- if self.data_type in SUPPORTED_DTYPES and self.act_bits < 16:
- self.act_data_type = self.data_type
- logger.info(f"activation adopts {self.data_type}")
- else:
- self.act_data_type = "float"
- tmp_act_bits = infer_bits_by_data_type(self.act_data_type)
- if tmp_act_bits is not None and tmp_act_bits < 16 and tmp_act_bits != self.act_bits:
- self.act_bits = tmp_act_bits
- logger.warning(
- f"`act_data_type` do not"
- f" match the specified 'act_bits' setting. Resetting 'act_bits' to {tmp_act_bits}."
- )
- if tmp_act_bits is not None and tmp_act_bits < 16:
- for supported_dtype in SUPPORTED_DTYPES: # to easily handle dtype mx_fp4 and layer_config={xxx:{bits:8}}
- if self.act_data_type.startswith(supported_dtype):
- if supported_dtype + str(tmp_act_bits) == self.act_data_type: # could not replace FP8_e4m3
- self.act_data_type = supported_dtype
- break
-
- def _adjust_torch_compile(self, enable_torch_compile: bool) -> None:
- """Sets the torch compile configuration for the tuning."""
- self.enable_torch_compile = enable_torch_compile
- if (
- not self.enable_torch_compile
- and TORCH_VERSION_AT_LEAST_2_6
- and self.act_bits > 8
- and not is_debug_mode()
- and not self.low_cpu_mem_usage
- and "fp8" not in self.data_type
- and "fp8" not in self.act_data_type
- ):
- logger.info(
- "'enable_torch_compile' is set to `False` by default. "
- "Enabling it can reduce tuning cost by 20%, but it might throw an exception."
- )
-
- if self.low_cpu_mem_usage and self.enable_torch_compile:
- self.enable_torch_compile = False
- logger.warning("reset enable_torch_compile to `False` as low_cpu_mem_usage is enabled")
-
- if is_debug_mode() and self.enable_torch_compile:
- self.enable_torch_compile = False
- logger.warning("reset enable_torch_compile to `False` as debug mode is enabled")
-
- if ("fp8" in self.data_type or "fp8" in self.act_data_type) and self.enable_torch_compile:
- self.enable_torch_compile = False
- logger.warning("reset enable_torch_compile to `False` as fp8 is enabled")
-
- def _set_device_map_in_blocks(self, device_map: Union[str, dict, None]) -> None:
- """Sets the device map for specific blocks in the model.
-
- Args:
- device_map (Union[str, dict, None]): A mapping of module names to devices.
- If provided as a string, it should be in the format
- "module_name:device,module_name:device". Devices can be integers
- (GPU IDs) or strings (e.g., 'cpu', 'cuda:0').
- """
- if self.device_map is None or len(self.device_map) == 0:
- self.device_map = None
- if not device_map:
- return
- if self.device_map == "auto" and device_map == "auto":
- return
- if isinstance(device_map, str):
- device_map = device_map.replace(" ", "")
- infos = device_map.split(",")
- device_map_dict = {}
- for info in infos:
- index = info.find(":")
- key = info[:index]
- value = info[index + 1 :]
- device_map_dict[key] = value
- device_map = device_map_dict
-
- names = [n for n, m in self.model.named_modules() if len(list(m.children())) == 0]
-
- for key, device in device_map.items():
- if isinstance(device, str) and device.isdigit():
- device = int(device)
- device = detect_device(device)
- try:
- module = get_module(self.model, key)
- module.tuning_device = device
- except:
- matching_names = [name for name in names if re.match(key, name)]
- if len(matching_names) > 0:
- for name in matching_names:
- self._set_device_for_matching_module(name, device)
- else:
- for name in names:
- if key in name:
- self._set_device_for_matching_module(name, device)
-
- def _set_device_for_matching_module(self, name: str, device: str) -> None:
- """Sets the device for a module if it matches the given name."""
- module = get_module(self.model, name)
- if hasattr(module, "tuning_device") and module.tuning_device != device:
- logger.warning(
- f"multiple devices have been set for layer {name}, keeping original device {module.tuning_device}"
- )
- else:
- module.tuning_device = device
-
- def _set_auto_device_map_in_block(self, block: torch.nn.Module, input_ids: list[torch.Tensor]) -> None:
- """Automatically sets the device map for the block based on available GPUs and memory constraints."""
- if torch.cuda.is_available():
- num_gpus = torch.cuda.device_count()
- elif torch.xpu.is_available():
- logger.warning_once("XPU does not support auto device map yet, using device 0 for tuning.")
- return
- else:
- raise RuntimeError("No CUDA or XPU devices found.")
- if num_gpus <= 1:
- self.device_map = None
- return
-
- if hasattr(self, "device_list") and self.device_list:
- cuda_devices = [f"cuda:{i}" for i in self.device_list]
- device_0 = cuda_devices[0]
- else:
- cuda_devices = [f"cuda:{i}" for i in range(num_gpus)]
- device_0 = "cuda:0"
-
- device_0_memory = get_device_memory(
- self.device_list[0] if hasattr(self, "device_list") and self.device_list else 0
- )
- block_memory, input_ouput_memory = estimate_tuning_block_mem(block, input_ids)
- if self.low_gpu_mem_usage:
- input_ouput_memory = 0
-
- mem_per_param_scale = 13 if self.mem_per_param_scale is None else self.mem_per_param_scale
- if self.iters == 0:
- mem_per_param_scale = 1 # for rtn
-
- if (block_memory * mem_per_param_scale + input_ouput_memory) < device_0_memory:
- return # fit in one GPU
-
- device_map = {}
- device_memory = {device: get_device_memory(int(device.split(":")[1])) for device in cuda_devices}
- device_memory[device_0] = device_0_memory - input_ouput_memory
-
- device_idx = 0
- # First, fill device 0 to its maximum capacity, then distribute the remaining layers evenly across other devices
- for n, m in block.named_modules():
- if check_to_quantized(m):
- layer_name = block.tmp_name + "." + n
- layer_memory = m.weight.nbytes / 1024**3
- if device_idx == 0 and layer_memory * mem_per_param_scale < device_memory[cuda_devices[device_idx]]:
- device_map[layer_name] = cuda_devices[device_idx]
- device_memory[cuda_devices[device_idx]] -= layer_memory * mem_per_param_scale
- elif device_idx == 0:
- device_idx += 1 # Move to the next device once device 0 is full
- device_map[layer_name] = cuda_devices[device_idx]
- device_memory[cuda_devices[device_idx]] -= layer_memory * mem_per_param_scale
- else:
- # Calculate the target device index based on even distribution
- sorted_devices = sorted(cuda_devices, key=lambda d: device_memory[d], reverse=True)
- device_idx = sorted_devices[0]
- if layer_memory * mem_per_param_scale < device_memory[device_idx]:
- device_map[layer_name] = device_idx
- device_memory[device_idx] -= layer_memory * mem_per_param_scale
- else:
- logger.warning_once(
- f"Block {block.tmp_name} not fit in available GPU memory. "
- "Consider using more GPUs or reducing mem_per_param_scale if OOM occurs."
- )
- self._set_device_map_in_blocks(device_map)
-
- def _dq_check(self) -> None:
- """Reset the default value of super_bits and super_group_size"""
- if self.data_type.endswith("_dq"):
- gguf_config = GGUF_INNER_CONFIG[f"gguf:q{self.bits}_k"]
- self.super_bits = gguf_config["super_bits"] if self.super_bits is None else self.super_bits
- self.super_group_size = (
- gguf_config["super_group_size"] if self.super_group_size is None else self.super_group_size
- )
-
- def _check_configs(self) -> None:
- """Checks if the configurations are valid.
-
- Raises:
- ValueError, TypeError: If any of the configurations are invalid.
- """
- if not isinstance(self.model, torch.nn.Module):
- raise TypeError("model must be an instance of torch.nn.Module")
- if self.bits <= 0:
- raise ValueError("`bits` must be positive")
- if self.act_bits <= 0:
- raise ValueError("`act_bits` must be positive")
- if not (self.group_size == -1 or self.group_size >= 0):
- raise ValueError("`group_size` must be -1 (per channel) or 0 (per-tensor) or a positive integer")
- if not (self.act_group_size == -1 or self.act_group_size >= 0):
- raise ValueError("`act_group_size` must be -1 (per channel) or 0 (per-tensor) or a positive integer")
- if self.batch_size <= 0:
- raise ValueError("`batch_size` must be positive")
- if self.iters < 0:
- raise ValueError("`iters` must be non-negative")
- if self.seqlen <= 0:
- raise ValueError("`seqlen` must be positive")
- if self.nblocks <= 0:
- raise ValueError("`nblocks` must be positive")
- if self.gradient_accumulate_steps <= 0:
- raise ValueError("`gradient_accumulate_steps` must be positive")
-
- if (
- self.act_bits <= 8
- and (not is_nv_fp(self.act_data_type) or "static_gs" not in self.act_data_type)
- and not is_mx_fp(self.act_data_type)
- ):
- logger.warning(
- "activation quantization is an experimental feature with limited support and a complex API. "
- "And please save the quantized model to fake format as real deployment is not supported currently"
- )
-
- if is_mx_fp(self.data_type) and self.group_size != 32:
- logger.warning("dtype mx_fp should only support group_size of 32 in real deployment")
-
- if is_nv_fp(self.data_type) and (self.group_size != 16):
- logger.warning("dtype nv_fp should only support group_size of 16 in real deployment")
-
- if self.nsamples < self.gradient_accumulate_steps * self.batch_size:
- if self.batch_size > self.nsamples:
- if self.iters > 0: # GGUF should log this warning, but we don't know the format here
- logger.warning(
- f"reset `batch_size` to {self.nsamples} as `nsamples`({self.nsamples})"
- f" is smaller than batch_size({self.batch_size})"
- )
- self.batch_size = self.nsamples
- if self.gradient_accumulate_steps > self.nsamples // self.batch_size:
- self.gradient_accumulate_steps = self.nsamples // self.batch_size
- logger.warning(
- f"reset `gradient_accumulate_steps` to {self.gradient_accumulate_steps}"
- f" as nsamples must equal or greater"
- f" than gradient_accumulate_steps * batch_size"
- )
-
- if self.enable_norm_bias_tuning:
- logger.warning("the `enable_norm_bias_tuning` feature is experimental and currently has limited support.")
-
- self._dq_check()
-
- def _check_compatibility(self) -> None:
- """Checks compatibility of the configurations and model."""
- # Check gguf and others
- has_gguf = False
- if hasattr(self, "formats"):
- has_besides_gguf = False
- for format_ in self.formats:
- if "gguf" in format_:
- has_gguf = True
- elif format_ != "fake":
- has_besides_gguf = True
- if has_gguf and has_besides_gguf:
- raise ValueError("Gguf format is not compatible with other formats, please choose only one of them")
- if has_gguf and self.iters != 0 and self.bits != 3:
- logger.warning(
- "`iters=0` is recommended when exporting to GGUF format except for bits 3,"
- " as we have optimized the RTN method for this case."
- " We are likely to release new algorithm for certain configurations in the future."
- )
-
- # Check group_size 32 for auto_round
- if (
- self.data_type == "int"
- and hasattr(self, "formats")
- and any(key in fmt for fmt in self.formats for key in ("auto_round", "auto_gptq", "auto_awq"))
- ):
- for n, m in self.model.named_modules():
- if isinstance(m, self.supported_types):
- if m.weight.shape[0] % 32 != 0 or m.weight.shape[1] % 32 != 0:
- self.layer_config[n] = {"bits": 16}
- logger.info(
- f"{n} will not be quantized due to its shape not being divisible by 32,"
- " resulting in an exporting issue to autogptq"
- )
-
- if (
- self.seqlen is not None
- and hasattr(self.model, "config")
- and hasattr(self.model.config, "max_position_embeddings")
- ):
- if self.model.config.max_position_embeddings < self.seqlen:
- logger.warning(
- f"Change sequence length to {self.model.config.max_position_embeddings} "
- "due to the limitation of max_position_embeddings"
- )
- self.seqlen = min(self.seqlen, self.model.config.max_position_embeddings)
-
- if self.seqlen is not None and hasattr(self.tokenizer, "model_max_length"):
- if self.tokenizer.model_max_length < self.seqlen:
- logger.warning(
- f"Change sequence length to {self.tokenizer.model_max_length} "
- "due to the limitation of model_max_length. "
- "You can also try to increase the model_max_length to avoid this issue."
- )
- self.seqlen = min(self.seqlen, self.tokenizer.model_max_length)
-
- if self.group_size == 0 and "fp8" not in self.data_type:
- logger.warning("`group_size==0` is not supported for data_type other than fp8 ")
-
- def _parse_format_to_list(self, format: str) -> list:
- """Parses the format string into a list of formats.
-
- This method checks the requested format(s) against the model's
- quantization settings and adjusts them if necessary. It ensures that
- the formats are compatible with the model's data type, bit width,
- and activation quantization settings.
-
- Args:
- format (str): The requested format(s) for quantization, separated by commas.
-
- Returns:
- list: A list of validated and updated formats.
- """
-
- # Remove duplicates from formats list
- def remove_duplicates(lst):
- seen = set()
- return [x for x in lst if not (x in seen or seen.add(x))]
-
- formats = format.replace("q*_", f"q{self.bits}_").replace(" ", "").split(",")
- formats = remove_duplicates(formats) # need the keep origin order
-
- if isinstance(self.scheme, str) and self.scheme.lower().startswith("gguf"):
- for i in range(len(formats)):
- if formats[i] != "fake" and formats[i] != self.scheme.lower().startswith("gguf"):
- logger.warning(
- f"reset format {formats[i]} to {self.scheme.lower()} "
- f"since scheme {self.scheme} can only be exported to format {self.scheme.lower()}"
- )
- formats[i] = self.scheme.lower()
-
- _gguf_args_check(self, formats, model_type=ModelType.TEXT)
- if self.vlm:
- _gguf_args_check(self, formats, model_type=ModelType.MMPROJ)
-
- for format_ in formats:
- if format_ not in SUPPORTED_FORMATS:
- logger.error(f"Unsupported format {format_}, please choose from {SUPPORTED_FORMATS}")
- exit(-1)
- if self.scale_dtype != torch.float32:
- only_gguf = True
- for format_ in formats:
- if not ("gguf" in format_ or "fake" in format_):
- only_gguf = False
- break
- if len(formats) == 1 and "fake" == formats[0]:
- only_gguf = False
- if only_gguf:
- self.scale_dtype = torch.float32
- logger.info("change `scale_dtype` to `torch.float32`")
-
- # Adjust format settings based on compatibility
- for index in range(len(formats)):
- format = formats[index]
- if format == "auto_round":
- if self.sym and "int" in self.data_type:
- format = "auto_round:auto_gptq"
- elif self.bits == 4 and not self.sym and "int" in self.data_type:
- enable_awq = all(
- config["bits"] == self.bits or config["bits"] >= 16 for config in self.layer_config.values()
- )
- if enable_awq:
- format = "auto_round:auto_awq"
- elif is_nv_fp(self.data_type) or is_mx_fp(self.data_type):
- format = f"auto_round:{self.data_type}"
- elif is_static_wfp8afp8(self): # staic wfp8afp8
- format = f"auto_round:{AutoRoundFormat.TORCH_FP8_STATIC.value}"
- elif self.data_type == "fp" and self.bits == 8 and self.act_bits >= 16: # woq fp8
- format = "auto_round:fp8"
- elif self.act_bits < 16:
- raise ValueError(
- "AutoRound format does not support exporting "
- "for the current quantization configuration, "
- "please change to `fake` format for research purpose"
- )
- formats[index] = format
- elif format == "llm_compressor":
- from auto_round.export.export_to_llmcompressor import check_compressed_tensors_supported
-
- if is_nv_fp(self.data_type) or is_mx_fp(self.data_type):
- check_compressed_tensors_supported()
- format = format.replace("llm_compressor", f"llm_compressor:{self.data_type}")
- formats[index] = format
- elif not is_wfp8afp8(self):
- logger.error(
- "Currently, the llm_compressor format only supports MXFP/NVFP/FP8. "
- "Please change format to fake or auto_round etc."
- )
- else:
- if (is_nv_fp(self.data_type) or is_mx_fp(self.data_type)) and format != "fake":
- logger.warning(f"nv_fp and mx_fp dtypes are not supported for export format: {format}")
-
- formats = remove_duplicates(formats)
- for i in range(len(formats)):
- formats[i] = self._check_supported_format(formats[i])
- formats = remove_duplicates(formats)
- return formats
-
- def _check_supported_format(self, format: str) -> bool:
- """Checks if the specified format is supported.
-
- This method validates the requested format against the model's bit width,
- group size, symmetry, and activation quantization settings. It raises an
- error if the format is incompatible with the current model configuration.
-
- Args:
- format (str): The requested format for quantization.
-
- Returns:
- bool: True if the format is supported, False otherwise.
- """
- if format == "fake":
- return format
- format = format.replace("q*_", f"q{self.bits}_")
-
- # format check for fp8
- w_fp8 = self.data_type == "fp" and self.bits == 8
- act_fp8 = self.act_data_type == "fp" and self.act_bits == 8
- if (w_fp8 or act_fp8) and re.search("^auto_round|^llm_compressor", format) is None:
- error_msg = (
- f"is only supported to export auto_round or llm_compressor format," f" but got {format}, please check."
- )
- error_msg = ("act_data_type<fp8> " + error_msg) if act_fp8 else error_msg
- error_msg = ("data_type<fp8> " + error_msg) if w_fp8 else error_msg
- logger.error(error_msg)
- sys.exit(-1)
-
- # Only support to export afp8/nv_fp/mx_fp
- if self.act_bits <= 8:
- if not is_standard_fp(self.act_data_type) or self.act_dynamic:
- if "llm_compressor" in format:
- if (is_nv_fp(self.act_data_type) and "static_gs" in self.act_data_type) or (
- is_mx_fp(self.act_data_type)
- ):
- return format
- bits, group_size, sym, act_bits = 8, -1, True, 8
- assert (
- self.bits == bits
- and self.group_size == group_size
- and self.sym == sym
- and self.act_bits == act_bits
- and self.act_dynamic
- ), (
- f"Currently only support to export llm_compressor format for sym dynamic quantized"
- f" W{self.bits}A{self.act_bits} model with group_size={group_size},"
- f" but got bits={self.bits}, group_size={self.group_size}, sym={self.sym},"
- f" act_bits={self.act_bits}"
- )
- elif "auto_round" in format and (
- is_mx_fp(self.act_data_type) or (is_nv_fp(format) and "static_gs" in self.act_data_type)
- ):
- logger.warning(
- f"AutoRound supports exporting to format '{format}', "
- "but loading quantized models in this format is not yet supported. "
- "It is currently recommended to export to the 'llm_compressor' format."
- )
- elif format != "fake":
- logger.warning(
- "Currently only support to export auto_round format quantized model"
- " with fp8, mx_fp and nv_fp4 dtype activation for activation quantization."
- " Change format to fake and save."
- )
- format = "fake"
- else:
- if not (format == "auto_round" or format == f"auto_round:{AutoRoundFormat.TORCH_FP8_STATIC.value}"):
- logger.warning(
- f"Currently only support to export auto_round or fake format for static W{self.bits}AFP8 model,"
- f" change format {format} to auto_round"
- )
- format = "auto_round"
- if self.act_group_size != 0 and not self.act_dynamic and format == "auto_round:fp8":
- logger.warning(
- f"Please note that quantize activation with act_group_size={self.act_group_size}"
- " may result in failure to export or import normally."
- )
- if re.search(r"q\d_k", format) and not self.data_type.endswith("_dq"):
- logger.error(
- f"datatype<{self.data_type}> not support to export {format} format."
- " Please change export format or `data_type`."
- )
- sys.exit(-1)
-
- return format
-
- def quantize_and_save(
- self, output_dir: str = "tmp_autoround", format: str = "auto_round", inplace: bool = True, **kwargs
- ) -> tuple[torch.nn.Module, dict[str, Any]]:
- """Quantizes the model and saves it in the specified format(s).
-
- This function checks the validity of the requested format(s), quantizes
- the model accordingly, and saves it to the specified output directory.
- If multiple formats are provided, the model is saved separately for each format.
-
- Args:
- output_dir (str, optional): The directory where the quantized model
- will be saved. Defaults to "tmp_autoround".
- format (str, optional): The quantization format(s) to use, separated
- by commas if multiple. Defaults to "auto_round".
- inplace (bool, optional): Whether to modify the model in place if only
- one format is used. Defaults to True.
- **kwargs: Additional arguments for the quantization and saving process.
-
- Returns:
- model: A qdq model or packed model based on the configurations
- folders: The folder paths where the quantized models are saved.
-
- Raises:
- ValueError: If an unsupported format is specified.
- """
- # Validate and process the specified formats
- self.orig_output_dir = output_dir
-
- # check and update the format based on the current configuration
- format_list = self._parse_format_to_list(format)
- self.formats = format_list
-
- # If multiple formats are specified, enforce inplace=False
- if len(format_list) > 1:
- inplace = False
- self.inplace = kwargs.get("inplace", inplace)
- kwargs.pop("inplace", None)
-
- # Perform model quantization
- if self.static_kv_dtype is not None:
- from auto_round.experimental.kv_cache import kvcache_quant_context
-
- with kvcache_quant_context(self.model, static_kv_dtype=self.static_kv_dtype):
- model, _ = self.quantize()
- else:
- model, _ = self.quantize()
- # Save the quantized model in the specified format_list
- folders = []
- for format in format_list:
- if "gptq" in format and not self.sym:
- logger.warning(
- "The asymmetrical kernel of the GPTQ format may result in a noticeable accuracy drop,"
- " particularly for 2-bit quantization and smaller models."
- " We recommend exporting to either the AutoAWQ format ( only 4 bits) or "
- "the AutoRound format(2/3/4/8 bits)."
- )
- save_folder = self._get_save_folder_name(format)
- self.save_quantized(save_folder, format=format, inplace=inplace, **kwargs)
-
- folders.append(save_folder)
-
- return model, folders
-
- def _get_save_folder_name(self, format_str: str) -> str:
- """Generates the save folder name based on the provided format string.
-
- If there are multiple formats to handle, the function creates a subfolder
- named after the format string with special characters replaced. If there's
- only one format, it returns the original output directory directly.
-
- Args:
- format_str (str): The format identifier (e.g., 'gguf:q2_k_s').
-
- Returns:
- str: The path to the folder where results should be saved.
- """
- # Replace special characters to make the folder name filesystem-safe
- sanitized_format = format_str.replace(":", "-").replace("_", "-")
-
- # Use a subfolder only if there are multiple formats
- if len(self.formats) > 1:
- return os.path.join(self.orig_output_dir, sanitized_format)
-
- return self.orig_output_dir
-
- @torch.inference_mode()
- def _quantize_embedding_layer(self):
- """Quantizes embedding layers in the model according to the configuration.
-
- This method iterates through all modules in the model, identifies embedding
- layers specified in `self.layer_config`, and applies the appropriate quantization
- function based on bit precision, grouping strategy, and dtype.
-
- Returns:
- bool: True if the quantization process completes without critical errors.
- """
- is_quantized = False
- for name, module in self.model.named_modules():
- # Skip non-Embedding modules or layers not in config
- if not isinstance(module, torch.nn.Embedding) or name not in self.layer_config:
- continue
-
- config = self.layer_config[name]
-
- # Skip layers that are not marked for quantization
- if not check_to_quantized(config):
- continue
- is_quantized = True
- config["scale_dtype"] = self.scale_dtype
- dtype = config["data_type"]
-
- # Determine quantization function key with symmetry/asymmetry
- if dtype not in QUANT_FUNC_WITH_DTYPE:
- dtype = f"{dtype}_{'sym' if config['sym'] else 'asym'}"
-
- # Optionally use optimized rounding (RTN) variant
- if not self.disable_opt_rtn and f"rtn_{dtype}" in QUANT_FUNC_WITH_DTYPE:
- dtype = f"rtn_{dtype}"
-
- quant_func = QUANT_FUNC_WITH_DTYPE[dtype]
-
- # Attempt quantization on GPU, fall back to CPU if OOM
- try:
- weight, scale, zp = quant_func(
- module.weight.to(self.device),
- **{k: config[k] for k in ["bits", "group_size", "super_bits", "super_group_size", "scale_dtype"]},
- )
- except RuntimeError as e:
- cuda_error_msg = traceback.format_exc()
- try:
- logger.error(cuda_error_msg)
- logger.warning("falling back to CPU")
- weight, scale, zp = quant_func(
- module.weight.to("cpu"),
- **{
- k: config[k]
- for k in ["bits", "group_size", "super_bits", "super_group_size", "scale_dtype"]
- },
- )
- except Exception as e:
- raise
-
- # Overwrite the module's weights with the quantized version
- module.weight.data.copy_(weight.cpu())
-
- # Attach scale and zero point (zp) to the module
- for param_name, value in zip(["scale", "zp"], [scale, zp]):
- if isinstance(value, dict):
- for k, v in value.items():
- setattr(module, k if k == "scale" else f"w_{k}", v.cpu())
- elif isinstance(value, torch.Tensor):
- setattr(module, param_name, value.cpu())
- else:
- setattr(module, param_name, value)
-
- # Update config
- self.layer_config.setdefault(name, {}).update(config)
-
- # Release memory
- clear_memory()
-
- return is_quantized
-
- def _quant_rtn_with_imatrix(self, all_to_quantized_module_names: list[str]) -> None:
- """Performs RTN quantization using input activation statistics (imatrix).
-
- This method accumulates per-channel second-moment activation statistics (imatrix)
- via forward hooks and uses them to perform RTN quantization. If CUDA memory runs out,
- it falls back to CPU-based blockwise quantization.
-
- Args:
- all_to_quantized_module_names (list[str]):
- A list of module names (e.g., 'model.layers.0.self_attn.q_proj') to be quantized.
-
- Returns:
- None
- """
- logger.info("start to compute imatrix for GGUF quantization")
-
- # Load dataset
- from auto_round.calib_dataset import get_dataloader
-
- if _is_fp8_model(self.model):
- convert_fp8_model_to_16b_model(self.model, self.amp_dtype)
-
- if isinstance(self.dataset, str):
- if self.tokenizer is None:
- raise ValueError("A tokenizer must be set for the model when using a dataset string.")
- dataset_name = self.dataset.replace(" ", "")
- self.dataloader = get_dataloader(
- self.tokenizer, self.seqlen, dataset_name, self.seed, self.batch_size, self.nsamples
- )
- else:
- self.dataloader = self.dataset
-
- model = self.model
-
- # Dispatch multi-GPU model if necessary
- if hasattr(model, "hf_device_map") and len(model.hf_device_map) > 1:
- dispatch_model(model, model.hf_device_map)
-
- def register_act_hook(model):
- """Registers hooks to accumulate activation squared norms into `imatrix`."""
-
- def get_imatrix_hook(module, input, output):
- input = input[0] if isinstance(input, (tuple, list)) else input
- flattened = input.reshape(-1, input.shape[-1]).to(torch.float32)
- squared = torch.sum(flattened**2, dim=0).to(torch.float32)
-
- if not hasattr(module, "imatrix"):
- module.imatrix = squared
- module.imatrix_cnt = input.shape[0]
- else:
- module.imatrix += squared.to(module.imatrix.device)
- module.imatrix_cnt += input.shape[0]
-
- hook_handles = []
- for name, module in model.named_modules():
- if isinstance(module, self.supported_types) and check_to_quantized(module):
- hook = module.register_forward_hook(get_imatrix_hook)
- hook_handles.append(hook)
- return hook_handles
-
- hooks = register_act_hook(model)
-
- try:
- # Move model to target device
- if hasattr(self.model, "hf_device_map") and len(self.model.hf_device_map) > 1:
- dispatch_model(self.model, self.model.hf_device_map)
- else:
- model = model.to(self.device)
- cnt = 0
-
- # Run forward pass to accumulate imatrix
- for data in self.dataloader:
- cnt += data["input_ids"].shape[0]
- data = to_device(data, self.device)
- model(**data)
- if cnt >= self.nsamples:
- break
-
- # Remove hooks after data collection
- for hook in hooks:
- hook.remove()
-
- # Normalize imatrix by count
- for _, module in model.named_modules():
- if hasattr(module, "imatrix"):
- module.imatrix /= module.imatrix_cnt
- if hasattr(model, "hf_device_map") and len(model.hf_device_map) > 1:
- import accelerate
-
- accelerate.hooks.remove_hook_from_submodules(model)
- # Perform quantization using RTN
- pbar = tqdm(all_to_quantized_module_names)
- block_names_cnt = len(flatten_list(get_block_names(self.model, True)))
- clear_mem_freq = len(all_to_quantized_module_names) // block_names_cnt
- if clear_mem_freq == 0:
- clear_mem_freq = 1
- cnt = 1
- for name in pbar:
- pbar.set_description(f"Quantizing {name}")
- self._quantize_layer_via_rtn(name)
- if cnt % clear_mem_freq == 0:
- clear_memory()
- cnt = 1
- cnt += 1
- except RuntimeError as e:
- try:
- if hasattr(model, "hf_device_map") and len(model.hf_device_map) > 1:
- import accelerate
-
- accelerate.hooks.remove_hook_from_submodules(model)
- # Fallback: out-of-memory → try CPU blockwise quantization
- logger.warning("Out of VRAM, falling back to blockwise quantization. Accuracy may degrade.")
- model = model.to("cpu")
- clear_memory()
- self._quantize_via_rtn_blockwise(all_to_quantized_module_names)
- except RuntimeError as e:
- cuda_error_msg = traceback.format_exc()
- try:
- logger.error(cuda_error_msg)
- # Final fallback: warn and use CPU-only quantization
- logger.warning(
- "Fallback to CPU. "
- "Consider enabling `low_gpu_mem_usage` or using more GPUs via `--device 0,1,2,3`."
- )
- model = model.to("cpu")
- clear_memory()
- if hasattr(model, "hf_device_map") and len(model.hf_device_map) > 1:
- import accelerate
-
- accelerate.hooks.remove_hook_from_submodules(model)
-
- orig_device = self.device
- self.device = "cpu"
- self._quantize_via_rtn_blockwise(all_to_quantized_module_names)
- self.device = orig_device
- except Exception as e:
- raise
- finally:
- # Always remove hooks
- for hook in hooks:
- hook.remove()
-
- def _check_need_to_quantize_lm_head_embedding(self) -> bool:
- """Checks if LM head and embedding layers need quantization for GGUF format.
-
- This function inspects the current model's formats and determines whether
- it needs to apply quantization settings to the embedding and LM head layers.
- The function modifies `self.layer_config` in-place and updates the model modules.
-
- Returns:
- bool: True if the LM head needs quantization, otherwise False.
-
- Raises:
- NotImplementedError: If multiple non-fake GGUF formats are specified.
- """
- gguf_scheme = False
- if isinstance(self.scheme, str) and "gguf" in self.scheme.lower():
- gguf_scheme = True
-
- if not hasattr(self, "formats") and not gguf_scheme:
- return False
-
- has_gguf: bool = gguf_scheme or any("gguf" in fmt for fmt in self.formats)
- if not has_gguf:
- return False
- if hasattr(self, "formats"):
- formats: list[str] = [fmt for fmt in self.formats if "fake" not in fmt]
- if not (len(formats) == 1 and "gguf" in formats[0]):
- raise NotImplementedError("Only one GGUF format can be set at a time.")
- target_format: str = formats[0]
-
- else:
- target_format = self.scheme.lower()
-
- tie_word_embeddings: bool = getattr(getattr(self.model, "config", None), "tie_word_embeddings", True)
- for name, module in self.model.named_modules():
- if isinstance(module, torch.nn.Embedding):
- key: str = "lm_head" if tie_word_embeddings else "embedding"
- config: dict[str, Any] = GGUF_INNER_CONFIG[GGUF_CONFIG[target_format][key]]
- self._apply_config_to_layer(name, config, True)
-
- if not tie_word_embeddings:
- lm_head_name: str = get_lm_head_name(self.model)
- config: dict[str, Any] = GGUF_CONFIG[GGUF_CONFIG[target_format]["lm_head"]]
- check_fixed_by_user = (
- self.layer_config[lm_head_name].get("fixed_by_user", False)
- if lm_head_name in self.layer_config
- else None
- )
- self._apply_config_to_layer(lm_head_name, config, check_fixed_by_user=check_fixed_by_user)
- return True
-
- return False
-
- def _apply_config_to_layer(
- self,
- layer_name: str,
- config: dict[str, Any],
- check_fixed_by_user: bool = False,
- ) -> None:
- """Applies GGUF quantization configuration to a given layer.
-
- Args:
- layer_name (str): Name of the layer to configure.
- config (dict[str, Any]): GGUF layer configuration.
- check_fixed_by_user (bool): If True, preserve user-defined settings.
- """
- act_bits: int = 16
- scale_dtype: Any = self.scale_dtype
- keys: list[str] = ["bits", "group_size", "super_bits", "super_group_size", "data_type", "sym"]
-
- self.layer_config[layer_name] = self.layer_config.get(layer_name, {})
-
- for key in keys:
- if (
- key in self.layer_config[layer_name]
- and check_fixed_by_user
- # and self.layer_config[layer_name].get("fixed_by_user", False)
- ):
- continue
- self.layer_config[layer_name][key] = config.get(key)
- setattr(get_module(self.model, layer_name), key, config.get(key))
-
- self.layer_config[layer_name]["act_bits"] = act_bits
- self.layer_config[layer_name]["scale_dtype"] = scale_dtype
- setattr(get_module(self.model, layer_name), "act_bits", act_bits)
- setattr(get_module(self.model, layer_name), "scale_dtype", scale_dtype)
-
- def _quantize_layer_via_rtn(self, name: str) -> None:
- """Quantizes a layer using RTN (Round-To-Nearest) if available.
-
- This function attempts to quantize a layer by switching its data type to a
- `rtn_*` version if supported, then wraps and unwraps the module to apply
- quantization. If GPU memory is insufficient, it falls back to CPU.
-
- If packing is enabled (`is_packing_immediate`), the function will also export
- the quantized layer to the appropriate backend format.
-
- Args:
- name (str): Name of the layer to quantize.
-
- Raises:
- RuntimeError: If quantization fails for reasons unrelated to memory.
- """
- m = get_module(self.model, name)
-
- # if m.__class__.__name__ == "FP8Linear":
- if _is_fp8_linear(m):
- m = convert_fp8_layer_to_linear(m, self.amp_dtype)
- set_module(self.model, name, m)
-
- # Step 1: Use optimized RTN data type if available
- if not self.disable_opt_rtn and not m.data_type.startswith("rtn_"):
- from auto_round.data_type import QUANT_FUNC_WITH_DTYPE
-
- rtn_dtype = "rtn_" + m.data_type
- if rtn_dtype in QUANT_FUNC_WITH_DTYPE:
- m.data_type = rtn_dtype
- self.layer_config[name]["data_type"] = m.data_type
-
- # Step 2: Try quantization on GPU first, fall back to CPU if OOM
- # if only export gguf, using gguf-packing instead of rtn
- if self.is_packing_immediate and self.iters == 0 and "gguf" in self.formats[0] and not self.disable_opt_rtn:
- m.scale = None
- m.zp = None
- else:
- try:
- m.to(self.device)
- m = WrapperLinear(
- m,
- enable_minmax_tuning=False,
- enable_norm_bias_tuning=False,
- enable_round_tuning=False,
- )
- m = m.unwrapper({})
- m.to("cpu")
- except RuntimeError as e:
- cuda_error_msg = traceback.format_exc()
- m = m.orig_layer if hasattr(m, "orig_layer") else m
- try:
- logger.error(cuda_error_msg)
- logger.warning("falling back to CPU.")
- m.to("cpu")
- m = WrapperLinear(
- m,
- enable_minmax_tuning=False,
- enable_norm_bias_tuning=False,
- enable_round_tuning=False,
- )
- m = m.unwrapper({})
- except Exception as e:
- raise
-
- # Step 3: Optional immediate packing/export
- if self.is_packing_immediate:
- from auto_round.export import PACKING_LAYER_WITH_FORMAT
-
- if check_to_quantized(m):
- target_backend = self.formats[0].split(":")[0] if ":" in self.formats[0] else self.formats[0]
- has_gguf = any("gguf" in fmt for fmt in self.formats)
-
- if has_gguf:
- from auto_round.export.export_to_gguf.export import pack_gguf_layer
-
- output_dir = self._get_save_folder_name(self.formats[0])
- model_type = ModelType.MMPROJ if self.vlm else ModelType.TEXT
- pack_gguf_layer(
- name,
- self.model,
- self.formats[0],
- output_dir,
- self.layer_config,
- self.tokenizer,
- processor=self.processor if hasattr(self, "processor") else None,
- image_processor=self.image_processor if hasattr(self, "image_processor") else None,
- model_type=model_type,
- )
- else:
- PACKING_LAYER_WITH_FORMAT[target_backend](name, self.model, self.formats[0], device=self.device)
-
- # if self.low_gpu_mem_usage:
- # clear_memory()
- else:
- set_module(self.model, name, m)
-
- @torch.inference_mode()
- def _quantize_rtn(self) -> tuple[torch.nn.Module, dict[str, Any]]:
- """Quantize all modules in the model using RTN (Round-To-Nearest) strategy.
-
- If the target format includes GGUF with `k`, and optimized RTN is enabled,
- blockwise quantization with input caching and imatrix is used.
-
- Returns:
- tuple[nn.Module, Dict[str, Any]]: The quantized model and the layer configuration.
- """
- if self.amp and self.model.dtype != self.amp_dtype:
- self.model.to(self.amp_dtype)
-
- all_to_quantized_module_names: list[str] = [n for n, m in self.model.named_modules() if check_to_quantized(m)]
-
- if is_nv_fp(self.data_type):
- from auto_round.data_type.nvfp import calculate_gparam
- from auto_round.data_type.utils import update_fused_layer_global_scales
-
- pbar = tqdm(all_to_quantized_module_names)
- for name in pbar:
- pbar.set_description(f"Calculate weight global scale: {name}")
- m = get_module(self.model, name)
- weight_global_scale = calculate_gparam(m.weight, self.group_size)
- setattr(m, "weight_global_scale", weight_global_scale)
-
- modules = list(self.model.modules())
- for module in tqdm(modules, desc="Update weight global scale for fuse module"):
- update_fused_layer_global_scales(module)
-
- has_gguf_k = any("gguf" in fmt and "k" in fmt for fmt in getattr(self, "formats", []))
-
- self._quantize_embedding_layer()
-
- self.model.to("cpu")
- if has_gguf_k and not self.disable_opt_rtn:
- self._quant_rtn_with_imatrix(all_to_quantized_module_names)
- elif self.act_bits <= 8 and check_need_act_calibration(
- self.act_dynamic, self.act_data_type, self.act_bits
- ): # TODO, mixed datatype has bug
- hook_handles = self._register_act_max_hook(self.model)
- try:
- self._quantize_via_rtn_blockwise(all_to_quantized_module_names)
- except RuntimeError as e:
- logger.warning("Fallback to CPU. Consider using more GPUs via `--device 0,1,2,3`.")
- self.model = self.model.to("cpu")
- clear_memory()
- if hasattr(self.model, "hf_device_map") and len(self.model.hf_device_map) > 1:
- import accelerate
-
- accelerate.hooks.remove_hook_from_submodules(self.model)
- orig_device = self.device
- self.device = "cpu"
- self._quantize_via_rtn_blockwise(all_to_quantized_module_names)
- self.device = orig_device
- for handle in hook_handles:
- handle.remove()
- else:
- block_names_cnt = len(flatten_list(get_block_names(self.model, True)))
- clear_mem_freq = len(all_to_quantized_module_names) // block_names_cnt
- if clear_mem_freq == 0:
- clear_mem_freq = 1
- pbar = tqdm(all_to_quantized_module_names)
- cnt = 1
- for name in pbar:
- pbar.set_description(f"Quantizing {name}")
- self._quantize_layer_via_rtn(name)
- if cnt % clear_mem_freq == 0:
- clear_memory()
- cnt = 1
- cnt += 1
- # Convert remaining fp8
- if _is_fp8_model(self.model):
- convert_fp8_model_to_16b_model(self.model, self.amp_dtype)
- self.quantized = True
- return self.model, self.layer_config
-
- def _quantize_via_rtn_blockwise(self, all_to_quantized_module_names: list[str]) -> None:
- """Quantize model layers block by block using cached inputs and imatrix.
-
- Args:
- all_to_quantized_module_names (list[str]): Names of layers to be quantized.
- """
- all_to_quantized_module_names = list(set(all_to_quantized_module_names))
-
- all_blocks = self.quant_block_list if self.quant_block_list else get_block_names(self.model)
- if not all_blocks:
- raise ValueError("Could not find any blocks. Check the model or quant_block_list.")
-
- all_first_block_names = [block[0] for block in all_blocks]
- if self.act_bits < 16 and not self.act_dynamic:
- layer_names = self._get_quantized_layer_names_outside_blocks()
- if len(layer_names) > 0:
- logger.warning(
- "quantize layers outside blocks for static activation quantizaiton"
- " will significantly increase calibration time"
- )
- all_inputs = self.try_cache_inter_data_gpucpu(all_first_block_names, self.nsamples, layer_names)
- else:
- all_inputs = self.cache_inter_data(all_first_block_names, self.nsamples)
-
- # Clear hooks for multi-GPU setups
- if hasattr(self.model, "hf_device_map") and len(self.model.hf_device_map) > 1:
- accelerate.hooks.remove_hook_from_submodules(self.model)
-
- pbar = tqdm(range(sum(len(block) for block in all_blocks)))
-
- for block_names in all_blocks:
- first_block = block_names[0]
- inputs = all_inputs.pop(first_block)
- input_keys = [k for k in inputs if k.startswith("hidden_state")]
- if len(input_keys) != 1:
- raise RuntimeError(
- "hidden_states arg mismatch. Please file an issue at https://github.com/intel/auto-round/issues"
- )
- inputs["input_ids"] = inputs.pop(input_keys[0])
-
- clear_memory(self.inputs)
-
- total_samples = len(inputs["input_ids"])
- if total_samples < self.batch_size:
- self.batch_size = total_samples
- logger.warning(f"Forcing batch size to {total_samples}")
-
- input_ids = to_device(inputs.pop("input_ids"), self.cache_device)
- input_others = to_device(inputs, self.cache_device)
-
- tmp_dtype = self.amp_dtype if self.amp else torch.float32
- input_ids = [id_.to(tmp_dtype) for id_ in input_ids]
-
- for key, val in input_others.items():
- if isinstance(val, torch.Tensor) and val.dtype in (torch.float16, torch.bfloat16):
- input_others[key] = val.to(tmp_dtype)
- elif isinstance(val, list):
- input_others[key] = [to_dtype(v, tmp_dtype) for v in val]
-
- for block_name in block_names:
- pbar.set_description(f"Quantizing {block_name}")
- block = get_module(self.model, block_name)
- block = block.to(self.device)
- if _is_fp8_model(self.model):
- convert_fp8_model_to_16b_model(block, dtype=self.amp_dtype)
-
- if self.device_map == "auto":
- self._set_auto_device_map_in_block(block, input_ids)
-
- # Dispatch model if needed
- if self.device_map is not None:
- from accelerate.hooks import AlignDevicesHook, add_hook_to_module
-
- for _, m in block.named_modules():
- if len(list(m.children())) != 0 or not hasattr(m, "tuning_device"):
- continue
- hook = AlignDevicesHook(m.tuning_device, io_same_device=True)
- add_hook_to_module(m, hook, True)
- else:
- block = block.to(self.device)
- input_ids = self._get_block_outputs(
- block,
- input_ids,
- input_others,
- self.batch_size * self.infer_bs_coeff,
- self.device,
- self.cache_device,
- )
- if self.device_map is not None:
- accelerate.hooks.remove_hook_from_submodules(block)
-
- if (
- is_nv_fp(self.act_data_type) and any("nv_fp" in format_ for format_ in self.formats)
- ) or is_static_wfp8afp8(self):
- # enable moe experts act_max automatic generation for Linear
- set_amax_for_all_moe_layers(block, attr_name="act_max")
- # Normalize imatrix and quantize layers
- for _, m in block.named_modules():
- if hasattr(m, "imatrix"):
- m.imatrix /= m.imatrix_cnt
- if hasattr(m, "tmp_name") and m.tmp_name in all_to_quantized_module_names:
- self._quantize_layer_via_rtn(m.tmp_name)
- all_to_quantized_module_names.remove(m.tmp_name)
-
- mv_module_from_gpu(block, self.low_cpu_mem_usage)
- pbar.update(1)
-
- pbar.close()
- cnt = 1
- block_names_cnt = len(flatten_list(get_block_names(self.model, True)))
- clear_mem_freq = len(all_to_quantized_module_names) // block_names_cnt
- if clear_mem_freq == 0:
- clear_mem_freq = 1
- # Process remaining layers not in blocks
- for name in all_to_quantized_module_names:
- self._quantize_layer_via_rtn(name)
- if cnt % clear_mem_freq == 0:
- clear_memory()
- cnt = 1
- cnt += 1
-
- def quantize(self) -> tuple[torch.nn.Module, dict[str, Any]]:
- """Quantize the model and return the quantized model along with layer configurations.The entry of AutoRound.
- Returns:
- The quantized model and layer configurations.
- """
- for n, m in self.model.named_modules():
- m.tmp_name = n
- self._check_compatibility()
- formats = self.formats if hasattr(self, "formats") else None
- # It is best to modify the model structure in the quantize function and check the format,
- # because it may cause the gguf format to not be exported normally.
- self.model = _handle_moe_model(self.model, formats=formats)
- self.has_qlayer_outside_block = self._set_layerwise_config(self.layer_config)
- if not hasattr(self, "formats"):
- logger.warning("this API is deprecated, please use `quantize_and_save` instead")
- else:
- only_gguf = True
- for format_ in self.formats:
- if not ("gguf" in format_ or "fake" in format_):
- only_gguf = False
- break
- if len(self.formats) == 1 and self.formats[0] == "fake":
- only_gguf = False
- if only_gguf:
- self.layer_config, gguf_format_config = get_layer_config_by_gguf_format(
- self.layer_config, self.formats, self.model, model_type=ModelType.TEXT
- )
- if self.vlm:
- self.layer_config, gguf_format_config = get_layer_config_by_gguf_format(
- self.layer_config, self.formats, self.model, model_type=ModelType.MMPROJ
- )
- # Determine if immediate packing is required
- formats = self.formats
- if (
- len(formats) == 1
- and (
- "awq" in formats[0]
- or "gptq" in formats[0]
- or "auto_round" in formats[0]
- or "gguf" in formats[0]
- or "llm_compressor" in formats[0]
- )
- and self.inplace
- ):
- self.is_packing_immediate = True
- if self.iters == 0:
- return self._quantize_rtn()
-
- if bool(self.quant_block_list):
- all_blocks = self.quant_block_list
- else:
- all_blocks = get_block_names(self.model)
-
- if len(all_blocks) == 0:
- logger.warning("could not find blocks, exit with original model")
- return self.model, self.layer_config
-
- if self.amp and self.model.dtype != self.amp_dtype:
- self.model = self.model.to(self.amp_dtype)
-
- layer_names = self._get_quantized_layer_names_outside_blocks()
- self.start_time = time.time()
- all_first_block_names = [block[0] for block in all_blocks]
- if len(layer_names) > 0:
- logger.info(
- "Starting to cache block inputs. This may be slow due to external block layers: %s", layer_names
- )
- else:
- logger.info("start to cache block inputs")
- all_inputs = self.try_cache_inter_data_gpucpu(all_first_block_names, self.nsamples, layer_names=layer_names)
- is_quantized_embedding = self._quantize_embedding_layer()
- all_q_inputs = None
- if is_quantized_embedding:
- all_inputs = copy.deepcopy(self.inputs)
- clear_memory(self.inputs)
- all_q_inputs = self.try_cache_inter_data_gpucpu(
- all_first_block_names, self.nsamples, layer_names=layer_names
- )
- self.model = mv_module_from_gpu(self.model, self.low_cpu_mem_usage)
- clear_memory()
- if hasattr(self.model, "hf_device_map") and len(self.model.hf_device_map) > 1:
- accelerate.hooks.remove_hook_from_submodules(self.model) # self.model.hf_device_map has not been changed
- self.model = mv_module_from_gpu(self.model, self.low_cpu_mem_usage)
- logger.info("caching done")
- if len(all_blocks) > 1:
- pbar = tqdm(range(0, sum([len(i) for i in all_blocks]), self.nblocks))
- else:
- pbar = None # move the alg warning outside pbar
-
- for block_names in all_blocks:
- inputs = all_inputs[block_names[0]]
- all_inputs.pop(block_names[0])
- q_inputs = None
- if all_q_inputs is not None:
- q_inputs = all_q_inputs[block_names[0]]
- all_q_inputs.pop(block_names[0])
- keys = inputs.keys()
- input_id_str = [key for key in keys if key.startswith("hidden_state")]
- if len(input_id_str) != 1:
- raise RuntimeError(
- "hidden_states arg mismatch error,"
- "please raise an issue in https://github.com/intel/auto-round/issues"
- )
- inputs["input_ids"] = inputs.pop(input_id_str[0], None)
- if q_inputs is not None:
- q_inputs["input_ids"] = q_inputs.pop(input_id_str[0], None)
-
- clear_memory(self.inputs)
-
- if "input_ids" in inputs.keys():
- total_samples = len(inputs["input_ids"])
- if total_samples < self.batch_size:
- self.batch_size = total_samples
- logger.warning(f"force the train batch size to {total_samples}")
-
- self._quantize_blocks(
- self.model,
- inputs,
- block_names,
- q_input=q_inputs["input_ids"] if q_inputs is not None else None,
- nblocks=self.nblocks,
- device=self.device,
- pbar=pbar,
- )
- if self.is_packing_immediate and len(self.formats) != 1:
- raise ValueError(
- f"Expected exactly one packing format when 'is_packing_immediate' is True, "
- f"but got {len(self.formats)} formats."
- )
-
- self._quantize_layers(layer_names, all_inputs) ##TODO pack layer immediately
-
- if _is_fp8_model(self.model):
- for n, m in self.model.named_modules():
- if _is_fp8_linear(m):
- new_layer = convert_fp8_layer_to_linear(m, self.amp_dtype).to("cpu")
- set_module(self.model, n, new_layer)
-
- end_time = time.time()
- cost_time = end_time - self.start_time
- logger.info(f"quantization tuning time {cost_time}")
-
- ## dump a summary
- quantized_layers = []
- unquantized_layers = []
- for n, m in self.model.named_modules():
- if isinstance(m, tuple(self.supported_types)):
- if check_to_quantized(m):
- quantized_layers.append(n)
- else:
- unquantized_layers.append(n)
- elif hasattr(m, "scales") or hasattr(m, "scale"): ##packing_immediately
- quantized_layers.append(n)
- summary_info = (
- f"Summary: quantized {len(quantized_layers)}/{len(quantized_layers) + len(unquantized_layers)} in the model"
- )
- if len(unquantized_layers) > 0:
- summary_info += f", {unquantized_layers} have not been quantized"
- logger.info(summary_info)
-
- self.quantized = True
- return self.model, self.layer_config
-
- def _quantize_layers(self, layer_names: list, layer_inputs: dict) -> None:
- """Quantizes specified layers based on inputs and configuration.
-
- Args:
- layer_names (list): list of layer names to quantize.
- layer_inputs (dict): Dictionary mapping layer names to input data.
-
- Returns:
- None
- """
- ##TODO currently we take all the layers outside blocks as post block layers which is not optimal
- ## if there is no input for layer, we use rtn
-
- for layer_name in copy.deepcopy(layer_names):
- if layer_name not in layer_inputs:
- logger.info(f"using rtn to quantize {layer_name}")
- from auto_round.data_type import QUANT_FUNC_WITH_DTYPE
-
- layer = get_module(self.model, layer_name)
- if _is_fp8_model(self.model):
- new_layer = convert_fp8_layer_to_linear(layer, self.amp_dtype).to(self.device)
- set_module(self.model, layer_name, new_layer)
- layer = new_layer
-
- if not self.disable_opt_rtn and "rtn_" + layer.data_type in QUANT_FUNC_WITH_DTYPE:
- layer.data_type = "rtn_" + layer.data_type
- logger.info("using optimized rtn method for quantizing %s", layer_name)
- self.layer_config[layer_name]["data_type"] = layer.data_type
- wrapper_layer = WrapperLinear(
- layer,
- enable_round_tuning=False,
- enable_minmax_tuning=False,
- enable_norm_bias_tuning=False,
- device=self.device,
- )
- new_layer = wrapper_layer.unwrapper({})
- set_module(self.model, layer_name, new_layer)
- layer.cpu()
- layer_names.remove(layer_name)
- if len(layer_names) == 0:
- return
- q_layer_inputs = None
- enable_quanted_input = self.enable_quanted_input
- has_gguf = False
- if hasattr(self, "formats"):
- has_gguf = any("gguf" in format_ for format_ in self.formats)
- if has_gguf and self.is_packing_immediate:
- enable_quanted_input = False
-
- if hasattr(self.model, "hf_device_map") and len(self.model.hf_device_map) > 1 and enable_quanted_input:
- dispatch_model(self.model, self.model.hf_device_map)
-
- if enable_quanted_input:
- logger.info("starting to cache layer inputs for %s, this may be quite slow ", layer_names)
- q_layer_inputs = self.try_cache_inter_data_gpucpu([], self.nsamples, layer_names=layer_names)
- if hasattr(self.model, "hf_device_map") and len(self.model.hf_device_map) > 1:
- accelerate.hooks.remove_hook_from_submodules(
- self.model
- ) ##self.model.hf_device_map has not been changed
-
- self.model = mv_module_from_gpu(self.model, self.low_cpu_mem_usage)
- clear_memory()
- if self.enable_torch_compile:
- quant_layer = compile_func(self._quantize_layer, self.device)
- else:
- quant_layer = self._quantize_layer
- for layer_name in layer_names:
- layer_input = layer_inputs[layer_name]
- layer_input = to_device(layer_input, self.cache_device)
- q_layer_input = q_layer_inputs.get(layer_name, None) if q_layer_inputs is not None else None
- q_layer_input = to_device(q_layer_input, self.cache_device)
- quant_layer(layer_name, layer_input, q_layer_input, device=self.device)
- del layer_input
- clear_memory(q_layer_input)
-
- def _set_layerwise_config(self, layer_config: dict) -> bool:
- """
- Sets the layer-wise configuration based on the provided `layer_config`.
- By default, only quantize layers in blocks.
-
- Args:
- layer_config (dict): The configuration dictionary for each layer containing various configuration options.
-
- Returns:
- bool: Returns True if there are quantized layers outside the blocks (e.g., lm-head),
- otherwise returns False.
- """
- # Get the names of layers in quantization blocks
- supported_types = self.supported_types
- layers_in_blocks = get_layer_names_in_block(
- self.model, supported_types, self.quant_block_list, self.inner_supported_types
- )
- ##process regex in layer_config
- all_supported_layer_names = []
- # List of configuration keys
- keys = get_quant_keys()
-
- for n, m in self.model.named_modules():
- # Delete previous configuration to avoid conflicts with prior tuning
- for key in keys:
- if hasattr(m, key):
- delattr(m, key)
-
- if not isinstance(m, supported_types) and m.__class__.__name__ not in self.inner_supported_types:
- continue
- all_supported_layer_names.append(n)
-
- names_in_layer_config = list(layer_config.keys())
- for name in names_in_layer_config:
- if name in all_supported_layer_names:
- continue
- matched_names = []
- for layer_name in all_supported_layer_names:
- if re.search(re.compile(name), layer_name) is not None:
- matched_names.append(layer_name)
- if len(matched_names) > 0:
- val = layer_config[name]
- layer_config.pop(name)
- for match_name in matched_names:
- layer_config[match_name] = val
- else:
- tmp_m = get_module(self.model, name)
- if not isinstance(tmp_m, torch.nn.Embedding): # TODO not good code style
- raise ValueError(f"key {name} in layer_config is invalid, please have a double check")
-
- has_qlayer_outside_block = False # Flag to track if there are quantized layers outside blocks (e.g., lm-head)
-
- # Iterate through all modules in the model
- is_gguf = hasattr(self, "formats") and any("gguf" in format_ for format_ in self.formats)
- for n, m in self.model.named_modules():
- # Skip unsupported types
- if not isinstance(m, supported_types) and m.__class__.__name__ not in self.inner_supported_types:
- if n in self.layer_config:
- if not isinstance(m, torch.nn.Embedding):
- logger.warning(f"{n} is not supported, layer_config {n}: {layer_config[n]} will be ignored.")
- self.layer_config.pop(n)
- continue
- if not is_gguf:
- if not check_to_quantized(layer_config[n]):
- self.layer_config.pop(n)
- continue
- else:
- continue
-
- # If the layer is not in the config and is part of a quantization block, use default configuration
- if n not in layer_config.keys() and n in layers_in_blocks:
- layer_config[n] = {}
- for key in keys:
- layer_config[n][key] = getattr(self, key)
-
- # If the layer is partially configured, fill in missing values
- elif n in layer_config.keys():
- if "data_type" in layer_config[n] and "bits" not in layer_config[n]:
- tmp_bits = infer_bits_by_data_type(layer_config[n]["data_type"])
- if tmp_bits is not None and tmp_bits != self.bits:
- logger.warning(
- f"'data_type' do not match the specified 'bits' setting for {n}."
- f" Resetting 'bits' to {tmp_bits}."
- )
- layer_config[n]["bits"] = tmp_bits
- if "act_data_type" in layer_config[n] and "act_bits" not in layer_config[n]:
- tmp_bits = infer_bits_by_data_type(layer_config[n]["act_data_type"])
- if tmp_bits is not None and tmp_bits != self.act_bits:
- logger.warning(
- f"'act_data_type' do not match the specified 'act_bits' setting for {n}."
- f" Resetting 'act_bits' to {tmp_bits}."
- )
- layer_config[n]["act_bits"] = tmp_bits
-
- for key in keys:
- if key not in layer_config[n].keys():
- layer_config[n][key] = getattr(self, key)
- layer_config[n]["fixed_by_user"] = True
-
- # If the layer is not in the config and not part of a quantization block,
- # use default configuration and set specific values
- else:
- layer_config[n] = {}
- for key in keys:
- layer_config[n][key] = getattr(self, key)
- layer_config[n]["bits"] = 16
- layer_config[n]["act_bits"] = 16
-
- if n in layers_in_blocks:
- layer_config[n]["in_blocks"] = True
- else:
- layer_config[n]["in_blocks"] = False
-
- # If the layer is outside a block and requires quantization, mark it as a quantized layer outside the block
- if (
- n not in layers_in_blocks
- and check_to_quantized(layer_config[n])
- and not isinstance(m, torch.nn.Embedding)
- ):
- has_qlayer_outside_block = True
-
- in_features, out_features = get_layer_features(m)
- if in_features <= layer_config[n]["group_size"]:
- layer_config[n]["group_size"] = -1
-
- # Apply the configuration to the corresponding layer in the model
- for key in keys:
- setattr(m, key, layer_config[n][key])
- need_to_quantize_lm_head = self._check_need_to_quantize_lm_head_embedding()
- if need_to_quantize_lm_head:
- has_qlayer_outside_block = True
-
- # Return whether there are quantized layers outside the blocks
- return has_qlayer_outside_block
-
- @torch.no_grad()
- def _get_block_outputs(
- self,
- block: torch.nn.Module,
- input_ids: torch.Tensor,
- input_others: torch.Tensor,
- bs: int,
- device: Union[str, torch.device],
- cache_device: Union[str, torch.device],
- save_output: bool = True,
- ):
- """Compute the output of a given block of the model for a given input.
-
- Args:
- block: The block of the model.
- input_ids: The input tensor containing tokenized input ids.
- input_others: A dictionary containing additional input data.
- bs: The batch size for computing the output.
- device: The device for computation.
- cache_device: The device for storing the output.
- batch_dim: The batch dimension of the output tensor.
-
- Returns:
- The output tensor of the block.
- """
-
- output = []
- nsamples = len(input_ids)
- for i in range(0, nsamples, bs):
- end_index = min(nsamples, i + bs)
- indices = torch.arange(i, end_index).to(torch.long)
- tmp_input_ids, tmp_input_others = AutoRound._sampling_inputs(
- input_ids, input_others, indices, self.seqlen, self.batch_dim, share_cache_keys=self.shared_cache_keys
- )
- tmp_output = block_forward(block, tmp_input_ids, tmp_input_others, self.amp, self.amp_dtype, device).to(
- cache_device
- )
- if save_output:
- if self.batch_size == 1:
- output.append(tmp_output)
- else:
- output.extend(list(torch.split(tmp_output, 1, dim=self.batch_dim)))
- if self.low_gpu_mem_usage:
- clear_memory()
-
- return output
-
- @torch.no_grad()
- def calib(self, nsamples, bs):
- """Perform calibration for quantization.
-
- This method calibrates the model for quantization by processing a specified
- number of samples from the calibration dataset. It ensures that the data is
- properly formatted and feeds it to the model. If the number of samples processed
- is less than the specified number, it logs a warning. If no samples are processed,
- it logs an error and exits.
- Args:
- nsamples (int): The number of samples to use for calibration.
- bs (int): The number of samples to use for calibration
- """
- from .calib_dataset import get_dataloader
-
- if isinstance(self.dataset, str):
- dataset = self.dataset.replace(" ", "") ##remove all whitespaces
-
- # slow here
- self.dataloader = get_dataloader(
- self.tokenizer,
- self.seqlen,
- dataset,
- self.seed,
- bs,
- self.nsamples,
- )
- else:
- self.dataloader = self.dataset
- total_cnt = 0
-
- # load embed weight if use low_cpu_mem_usage
- if self.low_cpu_mem_usage:
- embed_layers = get_layers_before_block(self.model)
- for n, m in embed_layers:
- m = m.to(self.device)
-
- for data in self.dataloader:
- if data is None:
- continue
- if isinstance(data, torch.Tensor):
- input_ids = data.to(self.model.device)
- data_new = input_ids
- elif isinstance(data, str):
- if self.tokenizer is None:
- logger.error("please provide tokenizer for string input")
- exit(-1)
- data = self.tokenizer(data, truncation=True, max_length=self.seqlen, return_tensors="pt").data
- data_new = {}
- for key in data.keys():
- data_new[key] = data[key].to(self.model.device)
- input_ids = data_new["input_ids"]
- elif isinstance(data, tuple) or isinstance(data, list):
- data_new = to_device(data)
- input_ids = data_new[0]
- else:
- data_new = {}
- for key in data.keys():
- data_new[key] = to_device(data[key], self.model.device)
- if key == "images":
- data_new[key] = to_dtype(data_new[key], self.model.dtype)
- input_ids = data_new["input_ids"]
- if input_ids.shape[-1] < self.seqlen:
- continue
- try:
- if isinstance(data_new, torch.Tensor):
- self.model(data_new, use_cache=False)
- elif isinstance(data_new, tuple) or isinstance(data_new, list):
- self.model(*data_new, use_cache=False)
- else:
- self.model(**data_new, use_cache=False)
- except NotImplementedError:
- pass
- except RuntimeError as error:
- error_msg = str(error)
- if "The expanded size of the tensor" in str(error_msg) and "must match the existing size" in error_msg:
- check_seqlen_compatible(self.seqlen, self.tokenizer, self.model)
- logger.warning(
- "When quantization encounters tensor shape mismatch error, "
- "you can try to avoid it with batch_size=1"
- )
- raise error
- except Exception as error:
- raise error
- total_cnt += input_ids.shape[0] if len(input_ids.shape) > 1 else 1
- if total_cnt >= nsamples:
- break
- if total_cnt == 0:
- logger.error(
- f"no data has been cached, please provide more data with sequence length >={self.seqlen} in the "
- f"dataset or decease the sequence length"
- )
- exit(-1)
- elif total_cnt < nsamples:
- logger.warning(
- f"An insufficient number of samples likely reduces the accuracy of the quantized model. "
- f"Target samples count is {nsamples}, while valid samples count is {total_cnt}"
- )
-
- # clean embed weight to save memory
- if self.low_cpu_mem_usage:
- for n, m in embed_layers:
- m = m.to("meta")
-
- @torch.no_grad()
- def try_cache_inter_data_gpucpu(self, block_names, nsamples, layer_names=None, last_cache_name=None):
- """Attempts to cache intermediate data on GPU, if failed, then using CPU.
-
- Args:
- block_names (list): List of block names to cache data for.
- nsamples (int): Number of samples to use for caching.
- layer_names (list, optional): List of layer names to cache data for. Defaults to [].
- last_cache_name (str, optional): Name of the last cache. Defaults to None.
-
- Returns:
- all_inputs: Cached intermediate data.
-
- Raises:
- Exception: If caching on GPU fails, switches to CPU and caches there.
- """
- if _is_fp8_model(self.model):
- layer_names = []
- if layer_names is None:
- layer_names = []
-
- if self.low_gpu_mem_usage or (
- len(block_names) == 1
- and len(layer_names) == 0
- and not self.has_qlayer_outside_block
- and (last_cache_name is None or last_cache_name in block_names)
- ):
- # low_gpu_mem_usage or calibrate only the embedding layer, which is also very fast on CPU
- all_inputs = self.cache_inter_data(block_names, nsamples, layer_names=[], last_cache_name=last_cache_name)
- else:
- try:
- if not self.model.device.type == "meta":
- if hasattr(self.model, "hf_device_map") and len(self.model.hf_device_map) > 1:
- self.model = dispatch_model(self.model, device_map=self.model.hf_device_map)
- else:
- # Change this if new device is supported
- if str(self.model.device) == "cpu" and (
- self.device.startswith("xpu") or self.device.startswith("cuda")
- ):
- max_memory = get_max_vram() # TODO model is not evenly split
- no_split_modules = getattr(self.model, "_no_split_modules", [])
- device_map = infer_auto_device_map(
- self.model, max_memory=max_memory, no_split_module_classes=no_split_modules
- )
-
- self.model = dispatch_model(self.model, device_map=device_map)
- else:
- self.model = self.model.to(self.device)
-
- all_inputs = self.cache_inter_data(
- block_names, nsamples, layer_names=layer_names, last_cache_name=last_cache_name
- )
- if hasattr(self.model, "hf_device_map") and len(self.model.hf_device_map) > 1:
- accelerate.hooks.remove_hook_from_submodules(self.model)
-
- except RuntimeError as e:
- cuda_error_msg = traceback.format_exc()
- try:
- logger.info("switch to cpu to cache block inputs")
- if self.has_qlayer_outside_block or self.__class__.__name__ == "AutoRoundMLLM":
- logger.warning(
- "we recommend using more GPUs in calibration."
- " Otherwise, some layers may fall back to `rtn` mode, which can affect accuracy."
- )
- if hasattr(self.model, "hf_device_map") and len(self.model.hf_device_map) > 1:
- accelerate.hooks.remove_hook_from_submodules(
- self.model
- ) ##self.model.hf_device_map has not been changed
- self.model = mv_module_from_gpu(self.model, self.low_cpu_mem_usage)
- clear_memory()
- ## Important change after v0.51, on cpu, we use rtn mode for layers in layer_names
- all_inputs = self.cache_inter_data(
- block_names, nsamples, layer_names=[], last_cache_name=last_cache_name
- )
- except Exception as e:
- logger.error(cuda_error_msg)
- raise
- return all_inputs
-
- @torch.no_grad()
- def cache_inter_data(self, block_names, nsamples, layer_names=None, last_cache_name=None):
- """Save the inputs of block_name for calibration.
-
- This method temporarily replaces the forward method of the model to capture
- the inputs passing through the specified block. It then calibrates the model
- using a specified number of samples. Finally, it restores the original forward
- method and returns the inputs for the specified block.
- Args:
- block_names (list): The names of the blocks for which inputs are to be saved.
- layer_names (list):The names of the layers for which inputs are to be saved.
- nsamples (int): The number of samples to use for calibration.
- last_cache_name (str, optional): The name of the last layer to be cached,
- we could break the forward in this layer to save time
-
- Returns:
- dict: A dictionary containing the inputs for the specified block.
- """
- if layer_names is None:
- layer_names = []
- self.inputs = {}
- self.to_cached_layers = block_names + layer_names
-
- tmp_dtype = None # TODO delete this as most model is not fp32 now
- ## have bug if block name is not the first block
- if (len(block_names) > 1 or len(layer_names) > 0) and self.low_gpu_mem_usage:
- tmp_dtype = self.model.dtype
- if self.amp:
- if self.model.dtype != self.model.dtype:
- self.model = self.model.to(torch.bfloat16)
- else:
- self.model = self.model.to(torch.float32) ##model on cpu
-
- self.last_cache_name = last_cache_name
- if last_cache_name is None and len(block_names) + len(layer_names) == 1:
- self.last_cache_name = block_names[0] if len(block_names) == 1 else layer_names[0]
- # do not set last_cache_name for multimodal models
- calib_bs = self.batch_size
- self.hook_handles = []
- self._replace_forward()
- self.calib(nsamples, calib_bs)
- self._recover_forward()
- res = self.inputs
- del self.last_cache_name
- del self.to_cached_layers
- if tmp_dtype is not None:
- self.model = self.model.to(tmp_dtype)
-
- return res
-
- @torch.no_grad()
- def _get_block_forward_func(self, name: str) -> Callable:
- """Gets the forward function.
-
- Args:
- name (str): The name of the function.
- Returns:
- function: The forward function.
- """
-
- def post_process_cache_data(batch_size, data, data_name):
- """
- Processes store data for batch handling, reshaping if necessary.
-
- Args:
- batch_size (int): The size of the batch.
- data: The data value to store, potentially for caching.
- data_name (str): Name of the data.
-
- Returns:
- Processed data or None
- """
- new_data = data
- if batch_size <= 1:
- return new_data
- if data_name in self.shared_cache_keys:
- return None
- if "alibi" in data_name:
- if isinstance(data, torch.Tensor):
- alibi = data
- alibi = alibi.reshape(batch_size, -1, alibi.shape[1], alibi.shape[2])
- new_data = alibi
- return new_data
-
- def forward(m, hidden_states=None, *positional_inputs, **kwargs):
- """Rewrite forward function, process and collect input data.
-
- Args:
- hidden_states (torch.Tensor): The hidden states tensor.
- *positional_inputs: Variable number of positional arguments.
- **kwargs: Variable number of keyword arguments.
-
- Returns:
- NotImplementedError: Getting the first layer inputs and then raise the error to save runtime.
- """
- if name not in self.inputs:
- self.inputs[name] = {}
- init_cache(positional_inputs, self.inputs[name])
-
- if self.batch_dim is None:
- self.batch_dim = 0
- if hidden_states is not None and self.batch_size > 1:
- if hidden_states.shape[0] > self.batch_size:
- self.batch_dim = 1
- if len(hidden_states.shape) > 1 and hidden_states.shape[1] > self.batch_size:
- logger.error(
- "this model has not been supported, "
- "please raise an issue in https://github.com/intel/auto-round/issues"
- " or try to set the `batch_size` to 1 and "
- "`gradient_accumulate_steps` to your current batch size."
- )
- exit(-1)
-
- if hidden_states is not None:
- kwargs["hidden_states"] = hidden_states
-
- for key in kwargs.keys():
- if (
- isinstance(kwargs[key], torch.Tensor)
- or isinstance(kwargs[key], list)
- or isinstance(kwargs[key], tuple)
- ):
- if key not in self.inputs[name].keys(): # initialization
- data = to_device(kwargs[key], device=torch.device("cpu"))
- if data is None or (self.batch_size > 1 and key in self.shared_cache_keys):
- self.inputs[name][key] = data
- continue
- if self.batch_size <= 1:
- self.inputs[name][key] = [data]
- else:
- data = post_process_cache_data(self.batch_size, data, key)
- self.inputs[name][key] = list(torch.split(data, 1, dim=self.batch_dim))
- else: # append cache inputs
- new_data = post_process_cache_data(self.batch_size, kwargs[key], key)
- if new_data is None: # shareable args or NoneType
- continue
- new_data = to_device(new_data, device=torch.device("cpu"))
- if self.batch_size <= 1:
- self.inputs[name][key].append(new_data)
- else:
- self.inputs[name][key].extend(list(torch.split(new_data, 1, dim=self.batch_dim)))
- elif isinstance(kwargs[key], (str, bool, type(None))):
- if key not in self.inputs[name].keys():
- self.inputs[name][key] = kwargs[key]
- else:
- # Parameters not to be cached
- if check_skippable_keywords(key):
- logger.warning_once(
- f"Please note that '{key}' key" " is not currently used in quantization fine-tuning."
- )
- reset_params(self.inputs[name])
- if name == self.last_cache_name:
- raise NotImplementedError
- else:
- if hidden_states is not None:
- kwargs.pop("hidden_states")
- return m.orig_forward(hidden_states, *positional_inputs, **kwargs)
- else:
- # Currently only for Llama-3.2-Vision-Instruct Series
- return m.orig_forward(*positional_inputs, **kwargs)
-
- return forward
-
- @torch.no_grad()
- def _get_cache_data_hook_for_layer(self, name):
- """A forward hook to save input max of a module
- :param name: the module name
- :return: A hook function."""
-
- def cache_input_hook(module, inputs, outputs):
- input = inputs
- if isinstance(inputs, tuple) or isinstance(input, list):
- input = inputs[0]
- if name in self.inputs:
- self.inputs[name].extend(list(torch.split(input.to("cpu"), 1, dim=0)))
- else:
- self.inputs[name] = list(torch.split(input.to("cpu"), 1, dim=0))
-
- return cache_input_hook
-
- def _recover_forward(self):
- """Recovers the forward function."""
- for n, m in self.model.named_modules():
- if hasattr(m, "orig_forward"):
- m.forward = m.orig_forward
- delattr(m, "orig_forward")
- for hook_handle in self.hook_handles:
- hook_handle.remove()
- self.hook_handles = []
-
- def _replace_forward(self):
- """Replaces the forward function."""
- from functools import partial
-
- for n, m in self.model.named_modules():
- if n in self.to_cached_layers and not isinstance(m, tuple(self.supported_types)): ##block
- m.orig_forward = m.forward
- m.forward = partial(self._get_block_forward_func(n), m)
- elif n in self.to_cached_layers: ##linear layer or conv1d layer
- hook_func = self._get_cache_data_hook_for_layer(n)
- hook_handle = m.register_forward_hook(hook_func)
- self.hook_handles.append(hook_handle)
-
- def _quantize_layer(
- self, layer_name: str, inputs: torch.Tensor, q_inputs: torch.Tensor = None, device: str = "cpu"
- ):
- """Quantize a specific layer of the model using the provided inputs.
-
- Args:
- layer_name (str): The name of the layer to quantize.
- inputs (torch.Tensor): Input data for quantization.
- q_inputs (torch.Tensor, optional): Quantized input data. Defaults to None.
- device (torch.device, optional): The device to use for quantization. Defaults to torch.device("cpu").
-
- Returns:
- None
- """
- logger.info(f"quantizing layer {layer_name}")
- layer = get_module(self.model, layer_name)
- if hasattr(layer, "tuning_device"):
- device = layer.tuning_device
-
- layer = layer.to(device)
- for i in range(len(inputs)):
- inputs[i] = inputs[i].to(layer.weight.dtype)
- if q_inputs is not None:
- q_inputs[i] = q_inputs[i].to(layer.weight.dtype)
-
- wrapper_linear = WrapperLinear(layer, enable_minmax_tuning=self.enable_minmax_tuning, device=device).to(device)
- round_params = []
- minmax_params = []
- for key in wrapper_linear.params.keys():
- if "min" in key or "max" in key:
- minmax_params.append(wrapper_linear.params[key])
- else:
- round_params.append(wrapper_linear.value)
- if len(round_params) + len(minmax_params) <= 0:
- dump_info = f"quantized {layer_name}"
- logger.info(dump_info)
- with torch.no_grad():
- unwrapper_layer(self.model, wrapper_linear, layer_name, {})
- mv_module_from_gpu(layer, self.low_cpu_mem_usage)
-
- lr = torch.tensor(self.lr)
- minmax_lr = torch.tensor(self.minmax_lr)
- if self.enable_minmax_tuning:
- optimizer = self.optimizer(
- [{"params": round_params}, {"params": minmax_params, "lr": minmax_lr}], lr=lr, weight_decay=0
- )
- else:
- optimizer = self.optimizer(round_params, lr=lr, weight_decay=0)
-
- if self.lr_scheduler is None:
- lr_schedule = torch.optim.lr_scheduler.LinearLR(
- optimizer, start_factor=1.0, end_factor=0.0, total_iters=self.iters
- )
- else:
- lr_schedule = copy.deepcopy(self.lr_scheduler)
- nsamples = len(inputs)
- last_best_iter = 0
- best_loss = torch.finfo(torch.float).max
- scaler = self._get_scaler() # pylint: disable=assignment-from-none
- init_loss = None
- gradient_accumulate_steps = self.batch_size ##Force to low gpu
- batch_size = 1 ##Force to low gpu
- pick_samples = batch_size * gradient_accumulate_steps
- pick_samples = min(nsamples, pick_samples)
- if self.sampler != "rand":
- whole_indices = torch.randperm(nsamples)[:pick_samples]
- total_loss = 0
- num_elm = 1
- mse_reduction = "mean"
- if gradient_accumulate_steps != 1:
- mse_reduction = "sum"
- mse_loss = torch.nn.MSELoss(reduction=mse_reduction).to(device)
-
- for i in range(self.iters):
- total_loss = 0
- if self.sampler == "rand":
- whole_indices = torch.randperm(nsamples)[:pick_samples]
- if gradient_accumulate_steps != 1:
- if q_inputs is not None:
- current_input = [q_inputs[i] for i in whole_indices]
- else:
- current_input = [inputs[i] for i in whole_indices]
- num_elm = sum(id.numel() for id in current_input)
- for tmp_step in range(gradient_accumulate_steps):
- indices = whole_indices[tmp_step * batch_size : (tmp_step + 1) * batch_size]
- if q_inputs is not None:
- current_input = [q_inputs[i] for i in indices]
- current_input = torch.cat(current_input, dim=0).to(device)
- org_input = [inputs[i] for i in indices]
- org_input = torch.cat(org_input, dim=0).to(device)
- else:
- current_input = [inputs[i] for i in indices]
- current_input = torch.cat(current_input, dim=0).to(device)
- org_input = current_input
- with torch.no_grad():
- current_output = layer(org_input)
-
- if self.amp:
- with autocast(device_type=device.split(":")[0], dtype=self.amp_dtype):
- output_q = wrapper_linear(current_input) # pylint: disable=not-callable
- loss = mse_loss(output_q, current_output) # pylint: disable=not-callable
- else:
- output_q = wrapper_linear(current_input) # pylint: disable=not-callable
- loss = mse_loss( # pylint: disable=not-callable
- output_q.to(torch.float32), current_output.to(torch.float32)
- )
- total_loss += loss.item() / num_elm
-
- self._scale_loss_and_backward(scaler, loss)
- if i == 0:
- init_loss = total_loss
-
- if total_loss < best_loss:
- best_loss = total_loss
- if not self.not_use_best_mse:
- best_params = collect_best_params(wrapper_linear)
- last_best_iter = i
- if self.not_use_best_mse and i == self.iters - 1:
- best_params = collect_best_params(wrapper_linear)
-
- if not self.not_use_best_mse:
- if 0 < self.dynamic_max_gap <= i - last_best_iter:
- break
- self._step(scaler, optimizer, lr_schedule)
-
- last_loss = total_loss
- best_iter = self.iters
- if not self.not_use_best_mse:
- last_loss = best_loss
- best_iter = last_best_iter
- with torch.no_grad():
- unwrapper_layer(self.model, wrapper_linear, layer_name, best_params)
- mv_module_from_gpu(layer, self.low_cpu_mem_usage)
- dump_info = f"quantized {layer_name}, loss iter 0: {init_loss:.6f} -> iter {best_iter}: {last_loss:.6f}"
- logger.info(dump_info)
-
- def _register_act_max_hook(self, model):
- def get_act_max_hook(module, input, output):
- if isinstance(input, (tuple, list)):
- input = input[0]
- if input.numel() == 0:
- return # as no needs for act_max update
- input, _, _ = reshape_pad_tensor_by_group_size(input, self.act_group_size)
- act_max = torch.max(torch.abs(input), dim=-1).values
- if not hasattr(module, "act_max") or module.act_max.numel() == 0:
- module.act_max = act_max
- else:
- act_max = act_max.to(module.act_max.device)
- if is_nv_fp(self.act_data_type): ## for nvfp per-tensor input_global_scale calculation usage
- module.act_max = torch.max(
- torch.tensor([act_max.max(), module.act_max.max()], device=act_max.device)
- )
- else:
- module.act_max = torch.max(act_max, module.act_max)
-
- hook_handles = []
-
- for n, m in model.named_modules():
- if (
- hasattr(m, "act_dynamic")
- and check_need_act_calibration(m.act_dynamic, m.act_data_type, m.act_bits)
- and check_to_quantized(m)
- ):
- hook = m.register_forward_hook(get_act_max_hook)
- hook_handles.append(hook)
- continue
-
- # for whole model, RTN
- if n in self.layer_config:
- config = self.layer_config[n]
- act_dynamic = config.get("act_dynamic", True)
- act_data_type = config.get("act_data_type", None)
- act_bits = config.get("act_data_type", 16)
- if (
- config["bits"] <= 8
- and check_need_act_calibration(act_dynamic, act_data_type, act_bits)
- and check_to_quantized(config)
- ):
- hook = m.register_forward_hook(get_act_max_hook)
- hook_handles.append(hook)
- continue
- return hook_handles
-
- def _quantize_block(
- self,
- block: torch.nn.Module,
- input_ids: list[torch.Tensor],
- input_others: dict,
- q_input: Union[None, torch.Tensor] = None,
- device: Union[str, torch.device] = "cpu",
- ):
- """Quantize the weights of a given block of the model.
-
- Args:
- block: The block of the model to be quantized.
- input_ids: The input tensor containing tokenized input ids.
- input_others: A dictionary containing additional input data.
- q_input: The quantized input tensor.
- device: The device for quantization.
-
- Returns:
- Tuple: (q_outputs, output) if self.enable_quanted_input is True, else (None, output)
- """
- if _is_fp8_model(self.model):
- for n, m in block.named_modules():
- if _is_fp8_linear(m):
- new_layer = convert_fp8_layer_to_linear(m, self.amp_dtype).to(device)
- set_module(block, n, new_layer)
-
- if self.device_map == "auto":
- self._set_auto_device_map_in_block(block, input_ids)
-
- if self.device_map is not None:
- for n, m in block.named_modules():
- if len(list(m.children())) != 0 or not hasattr(m, "tuning_device"):
- continue
- from accelerate.hooks import AlignDevicesHook, add_hook_to_module
-
- hook = AlignDevicesHook(m.tuning_device, io_same_device=True)
- add_hook_to_module(m, hook, True)
-
- if q_input is None:
- hook_handles = self._register_act_max_hook(block)
-
- output = self._get_block_outputs(
- block, input_ids, input_others, self.batch_size * self.infer_bs_coeff, device, self.cache_device
- )
-
- for handle in hook_handles:
- handle.remove()
- else:
- output = self._get_block_outputs(
- block, input_ids, input_others, self.batch_size * self.infer_bs_coeff, device, self.cache_device
- )
- hook_handles = self._register_act_max_hook(block)
- if hook_handles:
- self._get_block_outputs(
- block,
- q_input,
- input_others,
- self.batch_size * self.infer_bs_coeff,
- device,
- self.cache_device,
- save_output=False,
- )
-
- for handle in hook_handles:
- handle.remove()
-
- if q_input is not None:
- if input_ids is not q_input:
- clear_memory(input_ids)
- else:
- clear_memory()
- input_ids = q_input
-
- quantized_layer_names, unquantized_layer_names = wrapper_block(
- block, self.enable_minmax_tuning, self.enable_norm_bias_tuning, device=self.device
- )
- if is_nv_fp(self.data_type): # enable qkv and moe structure global_scale fuse
- from auto_round.data_type.utils import update_fused_layer_global_scales
-
- modules = block.modules()
- for module in modules:
- update_fused_layer_global_scales(module)
- round_params = []
- minmax_params = []
- for n, m in block.named_modules():
- if hasattr(m, "orig_layer"):
- for key in m.params.keys():
- if "min" in key or "max" in key:
- minmax_params.append(m.params[key])
- else:
- round_params.append(m.params[key])
-
- lr = torch.tensor(self.lr)
- minmax_lr = torch.tensor(self.minmax_lr)
- if self.enable_minmax_tuning:
- optimizer = self.optimizer(
- [{"params": round_params}, {"params": minmax_params, "lr": minmax_lr}], lr=lr, weight_decay=0
- )
- else:
- optimizer = self.optimizer(round_params, lr=lr, weight_decay=0)
-
- if len(round_params) + len(minmax_params) <= 0:
- dump_info = (
- f"quantized {len(quantized_layer_names)}/{(len(quantized_layer_names) + len(unquantized_layer_names))} "
- f"layers in the block"
- )
- logger.info(dump_info)
- unwrapper_block(block, {}) # TODO Quant layer should change
- mv_module_from_gpu(block, self.low_cpu_mem_usage)
- return output, output
-
- if self.lr_scheduler is None:
- lr_schedule = torch.optim.lr_scheduler.LinearLR(
- optimizer, start_factor=1.0, end_factor=0.0, total_iters=self.iters
- )
- else:
- lr_schedule = copy.deepcopy(self.lr_scheduler)
-
- nsamples = len(input_ids)
- pick_samples = self.batch_size * self.gradient_accumulate_steps
- pick_samples = min(nsamples, pick_samples)
- if self.sampler != "rand":
- whole_indices = torch.randperm(nsamples)[:pick_samples]
- last_best_iter = 0
- best_loss = torch.finfo(torch.float).max
- num_elm = 1
- mse_reduction = "mean"
- if self.gradient_accumulate_steps != 1:
- mse_reduction = "sum"
- mse_loss = torch.nn.MSELoss(reduction=mse_reduction).to(device)
- scaler = self._get_scaler() # pylint: disable=assignment-from-none
- init_loss = None
- best_params = {}
- total_loss = 0
- for i in range(self.iters):
- total_loss = 0
- if self.sampler == "rand":
- whole_indices = torch.randperm(nsamples)[:pick_samples]
- # We assume the block input and output shape is same
- if self.gradient_accumulate_steps != 1:
- current_input_ids = [input_ids[i] for i in whole_indices]
- num_elm = sum(id.numel() for id in current_input_ids)
-
- for tmp_step in range(self.gradient_accumulate_steps):
- indices = whole_indices[tmp_step * self.batch_size : (tmp_step + 1) * self.batch_size]
- current_input_ids, current_input_others = AutoRound._sampling_inputs(
- input_ids,
- input_others,
- indices,
- seqlen=self.seqlen,
- batch_dim=self.batch_dim,
- share_cache_keys=self.shared_cache_keys,
- )
-
- current_output = [output[x] for x in indices]
- current_output = torch.cat(current_output, dim=self.batch_dim)
- current_output = to_device(current_output, device)
-
- output_q = block_forward(
- block, current_input_ids, current_input_others, self.amp, self.amp_dtype, device
- )
- if self.amp:
- with autocast(device_type=device.split(":")[0], dtype=self.amp_dtype):
- loss = mse_loss(output_q, current_output) # pylint: disable=not-callable
- else:
- loss = mse_loss( # pylint: disable=not-callable
- output_q.to(torch.float32), current_output.to(torch.float32)
- )
-
- total_loss += loss.item() / num_elm
- self._scale_loss_and_backward(scaler, loss)
-
- if i == 0:
- init_loss = total_loss
-
- if total_loss < best_loss:
- best_loss = total_loss
- if not self.not_use_best_mse:
- best_params = collect_best_params(block)
- # print(f"get better result at iter {i}, the loss is {total_loss}", flush=True)
-
- last_best_iter = i
- if self.not_use_best_mse and i == self.iters - 1:
- best_params = collect_best_params(block)
-
- if not self.not_use_best_mse:
- if 0 < self.dynamic_max_gap <= i - last_best_iter:
- break
- self._step(scaler, optimizer, lr_schedule)
-
- last_loss = total_loss
- best_iter = self.iters
- if not self.not_use_best_mse:
- last_loss = best_loss
- best_iter = last_best_iter
- dump_info = (
- f"quantized {len(quantized_layer_names)}/{(len(quantized_layer_names) + len(unquantized_layer_names))} "
- f"layers in the block, loss iter 0: {init_loss:.6f} -> iter {best_iter}: {last_loss:.6f}"
- )
- logger.info(dump_info)
- if len(unquantized_layer_names) != 0:
- logger.info(f"{unquantized_layer_names} have not been quantized")
- with torch.no_grad():
- unwrapper_block(block, best_params)
-
- if (
- is_nv_fp(self.act_data_type)
- and hasattr(self, "formats")
- and any("nv_fp" in format_ for format_ in self.formats)
- ):
- # enable moe experts act_max automatic generation for WrapperWALayer
- set_amax_for_all_moe_layers(block, attr_name="orig_layer.act_max")
-
- if self.enable_quanted_input:
- if self.low_cpu_mem_usage:
- block = block.to(device)
- clear_memory()
- q_outputs = self._get_block_outputs(
- block,
- input_ids,
- input_others,
- self.batch_size * self.infer_bs_coeff,
- device,
- cache_device=self.cache_device,
- )
- if self.device_map is not None:
- accelerate.hooks.remove_hook_from_submodules(block)
- mv_module_from_gpu(block, self.low_cpu_mem_usage)
- clear_memory(input_ids)
-
- return q_outputs, output
-
- else:
- if self.device_map is not None:
- accelerate.hooks.remove_hook_from_submodules(block)
- mv_module_from_gpu(block, self.low_cpu_mem_usage)
- clear_memory(input_ids)
- return None, output
-
- def _quantize_blocks(
- self,
- model: torch.nn.Module,
- inputs: dict,
- block_names: list,
- q_input: torch.Tensor = None,
- nblocks: int = 1,
- device: str = "cpu",
- pbar: tqdm = None,
- ):
- """Quantize and dequantize the weights of the specified blocks in the model.
-
- Args:
- model: The PyTorch model to be quantized.
- inputs: The input data for quantization.
- block_names: The names of the blocks to be quantized and dequantized.
- nblocks: The number of blocks to quantize and dequantize.
- device: The device for quantization and dequantization.
-
- Returns:
- None
- """
- clear_memory()
- for n, m in model.named_parameters():
- m.requires_grad_(False)
- input_ids = inputs["input_ids"]
- inputs.pop("input_ids", None)
- input_others = inputs
- clear_memory()
- input_ids = to_device(input_ids, self.cache_device)
- input_others = to_device(input_others, self.cache_device)
- # As in calibration phase, we may use bf16 for calibration due to low_gpu_memory usage
- tmp_dtype = self.amp_dtype if self.amp else torch.float32
- for i in range(len(input_ids)):
- input_ids[i] = input_ids[i].to(tmp_dtype)
-
- for key in input_others.keys():
- if isinstance(input_others[key], torch.Tensor) and (
- input_others[key].dtype == torch.float16 or input_others[key].dtype == torch.bfloat16
- ):
- input_others[key] = input_others[key].to(tmp_dtype)
- elif isinstance(input_others[key], list):
- for i in range(len(input_others[key])):
- to_dtype(input_others[key][i], tmp_dtype)
-
- if (
- self.sym
- and self.enable_alg_ext
- and self.super_group_size is None
- and (
- (self.data_type.startswith("int") and self.act_bits >= 8)
- or self.data_type.startswith("mx")
- or self.data_type.startswith("nv")
- )
- ):
- try:
- from auto_round.alg_ext import quantize_block_ext
-
- AutoRound.quantize_block_ext = quantize_block_ext
- quantize_block = self.quantize_block_ext # must use self.quantize_block_ext
- if self.bits > 2 and (not self.data_type.startswith("mx") or not self.data_type.startswith("nv")):
- logger.warning(
- "algorithm extension has only undergone limited validation on "
- "INT2,mxfp4 and nvfp4; use with caution."
- )
- else:
- logger.info("using algorithm extension for quantization.")
- except (ImportError, ModuleNotFoundError):
- quantize_block = self._quantize_block
- if self.enable_torch_compile:
- quantize_block = compile_func(quantize_block, device)
- else:
- quantize_block = quantize_block
- else:
- quantize_block = self._quantize_block
- if self.enable_torch_compile:
- quantize_block = compile_func(quantize_block, device)
-
- if pbar is None:
- pbar = tqdm(range(0, len(block_names), nblocks))
-
- for i in range(0, len(block_names), nblocks):
- if i != 0:
- pbar.update(1)
- if nblocks == 1:
- n = block_names[i]
- pbar.set_description(f"Quantizing {n}")
- m = get_module(model, n)
- else:
- names = block_names[i : min(i + nblocks, len(block_names))]
- pbar.set_description(f"Quantizing [{i + 1}-{min(i + nblocks, len(block_names))}]/{len(block_names)}")
- modules = [get_module(model, n) for n in names]
- m = WrapperMultiblock(modules)
-
- if not self.model.device.type == "meta" or self.low_cpu_mem_usage:
- m = m.to(device)
-
- q_input, input_ids = quantize_block(
- m,
- input_ids,
- input_others,
- q_input=q_input,
- device=device,
- )
- if self.is_packing_immediate:
- from auto_round.export import PACKING_LAYER_WITH_FORMAT
-
- for _, tmp_m in m.named_modules():
- if not (hasattr(tmp_m, "bits") and check_to_quantized(tmp_m)):
- continue
- target_backend = self.formats[0].split(":")[0] if ":" in self.formats[0] else self.formats[0]
- has_gguf = any("gguf" in format_ for format_ in self.formats)
- if has_gguf:
- from auto_round.export.export_to_gguf.export import pack_gguf_layer
-
- output_dir = self._get_save_folder_name(self.formats[0])
- model_type = ModelType.MMPROJ if self.vlm else ModelType.TEXT
- pack_gguf_layer(
- tmp_m.tmp_name,
- self.model,
- self.formats[0],
- output_dir,
- self.layer_config,
- self.tokenizer,
- processor=self.processor if hasattr(self, "processor") else None,
- image_processor=self.image_processor if hasattr(self, "image_processor") else None,
- model_type=model_type,
- )
- else:
- PACKING_LAYER_WITH_FORMAT[target_backend](
- tmp_m.tmp_name, self.model, self.formats[0], device=self.device
- )
- pbar.set_description("Quantizing done")
- pbar.update(1)
- pbar.close()
- self.model = mv_module_from_gpu(self.model, self.low_cpu_mem_usage)
- for n, m in self.model.named_modules():
- if hasattr(m, "name"):
- delattr(m, "name")
-
- del q_input
- del input_ids
- del input_others
- del inputs
-
- clear_memory()
-
- def save_quantized(
- self, output_dir: str = None, format: str = "auto_round", inplace: bool = True, **kwargs
- ) -> torch.nn.Module:
- """Save the quantized model to the specified output directory in the specified format.
+ tokenizer=None,
+ scheme: Union[str, dict, QuantizationScheme] = "W4A16",
+ layer_config: dict[str, Union[str, dict, QuantizationScheme]] = None,
+ dataset: Union[str, list, tuple, torch.utils.data.DataLoader] = "NeelNanda/pile-10k",
+ iters: int = 200,
+ seqlen: int = 2048,
+ nsamples: int = 128,
+ batch_size: int = 8,
+ gradient_accumulate_steps: int = 1,
+ low_gpu_mem_usage: bool = False,
+ device_map: Union[str, torch.device, int, dict] = 0,
+ enable_torch_compile: bool = False,
+ seed: int = 42,
+ # for adam
+ enable_adam: bool = False,
+ # for MLLM
+ extra_config: ExtraConfig = None,
+ **kwargs,
+ ) -> BaseCompressor:
+ """Initialize AutoRound with quantization and tuning configuration.
Args:
- output_dir (str, optional): The directory to save the quantized model. Defaults to None.
- format (str, optional): The format in which to save the model. Defaults to "auto_round".
- inplace (bool, optional): Whether to modify the model in place. Defaults to True.
- **kwargs: Additional keyword arguments specific to the export format.
+ model (torch.nn.Module | str): Model object or model name to load.
+ tokenizer: Tokenizer for text processing. Required if `model` is not a string and `iters > 0`.
+ scheme (str| dict | QuantizationScheme ): A preset scheme that defines the quantization configurations
+ bits (int, optional): Weight quantization bits. Defaults to 4.
+ group_size (int, optional): Weight quantization group size. Defaults to 128.
+ sym (bool, optional): Symmetric weight quantization. Defaults to True.
+ layer_config (dict, optional): Layer-wise quantization config. Defaults to None.
+ batch_size (int, optional): Calibration batch size. Defaults to 8.
+ amp (bool, optional): Use AMP for tuning. Defaults to True.
+ device (str | torch.device | int, optional): Compute device. Defaults to 0.
+ dataset (str | list | tuple | DataLoader, optional): Calibration data. Defaults to "NeelNanda/pile-10k".
+ enable_minmax_tuning (bool, optional): Enable weight min-max tuning. Defaults to True.
+ lr (float, optional): Learning rate; if None, set to 1.0 / iters except when iters==0.
+ minmax_lr (float, optional): Learning rate for min-max tuning; defaults to `lr`.
+ low_gpu_mem_usage (bool, optional): Lower GPU memory mode. Defaults to False.
+ iters (int, optional): Optimization iterations. Defaults to 200.
+ seqlen (int, optional): Calibration sequence length. Defaults to 2048.
+ nsamples (int, optional): Number of calibration samples. Defaults to 128.
+ seed (int, optional): Random seed. Defaults to 42.
+ gradient_accumulate_steps (int, optional): Gradient accumulation steps. Defaults to 1.
+ data_type (str, optional): Weight data type string, e.g., "int". Defaults to "int".
+ act_bits (int, optional): Activation quantization bits. Defaults to 16.
+ act_group_size (int, optional): Activation group size. Defaults to None.
+ act_sym (bool, optional): Symmetric activation quantization. Defaults to None.
+ act_data_type (str, optional): Activation data type; inherits weight dtype if None and act_bits < 16.
+ act_dynamic (bool, optional): Dynamic activation quantization. Defaults to True.
+ enable_torch_compile (bool, optional): Enable torch.compile for quant blocks/layers. Defaults to False.
+ device_map (str | dict, optional): Device placement map. Defaults to None.
+ disable_opt_rtn (bool, optional): Disable RTN-mode optimization (iters=0). Defaults to False.
+ enable_alg_ext (bool, optional): Enable algorithm extension (primarily for INT2). Defaults to False.
+ **kwargs: Backward compatible options:
+ - enable_alg_ext, quant_lm_head, lr, lr_scheduler, sampler, not_use_best_mse, dynamic_max_gap,
+ super_group_size, super_bits, scale_dtype ("fp16" etc.),
+ nblocks, low_cpu_mem_usage, to_quant_block_names,
+ enable_norm_bias_tuning, enable_quanted_input,
+ disable_deterministic_algorithms, vlm, static_kv_dtype
+ Raises:
+ ValueError: If invalid device is provided or tokenizer is missing for non-str model with iters > 0.
+ RuntimeError: If model parameters are on meta device.
+ Example:
+ Layer-wise configuration structure:
- Returns:
- object: The compressed model object.
+ >>> layer_config = {
+ ... "layer1": {
+ ... "data_type": "int",
+ ... "bits": 4,
+ ... "group_size": 128,
+ ... "sym": True,
+ ... "act_data_type": None,
+ ... "act_bits": 16,
+ ... "act_group_size": None,
+ ... "act_sym": None,
+ ... },
+ ... # ...
+ ... }
"""
- format = self._check_supported_format(format)
-
- if self.low_cpu_mem_usage:
- self.model = self.model.to("cpu")
-
- if not self.quantized:
- logger.warning("please run autoround.quantize first")
- return
- if format == "fake" or format == "qdq": # TODO fix act quantization later
- self.model = self.model.to("cpu")
- self.model.save_pretrained(output_dir)
- if self.tokenizer is not None:
- self.tokenizer.save_pretrained(output_dir)
- processor = kwargs.get("processor", None)
- if processor is not None:
- processor.save_pretrained(output_dir)
- try:
- copy_python_files_from_model_cache(self.model, output_dir)
- except Exception as e:
- logger.warning("Skipping source model Python file copy due to error: %s", e)
- return
- if self.act_bits <= 8 and format == "qdq":
- logger.warning(
- "Support for exporting activation quantization is limited. "
- "Please ensure that your configuration is supported."
- )
- if format == "llm_compressor" and (is_nv_fp(self.data_type) or is_mx_fp(self.data_type)):
- format = format.replace("llm_compressor", f"llm_compressor:{self.data_type}")
+ model_cls = []
- from auto_round.export import EXPORT_FORMAT
-
- backend = format
- format = format.split(":")[0]
- if format not in EXPORT_FORMAT:
- logger.error(f"export format only supports {EXPORT_FORMAT.keys()}")
- raise ValueError(f"export format only supports {EXPORT_FORMAT.keys()}, but got {format}")
- save_quantized_as_format = EXPORT_FORMAT.get(format)
- if "gptq" in format and not self.sym:
- logger.warning(
- "the asymmetrical kernel of the GPTQ format may result in a noticeable accuracy drop,"
- " particularly for 2-bit quantization and smaller models."
- " We recommend exporting to either the AutoAWQ format ( only 4 bits) or "
- "the AutoRound format(2/3/4/8 bits)."
- )
- if "awq" in format and not self.bits == 4:
- raise ValueError("The AWQ format only supports W4 quantization ")
- serialization_keys = [
- "bits",
- "group_size",
- "sym",
- "data_type",
- "enable_quanted_input",
- "enable_minmax_tuning",
- "seqlen",
- "batch_size",
- "scale_dtype",
- "lr",
- "minmax_lr",
- "gradient_accumulate_steps",
- "iters",
- "amp",
- "nsamples",
- "low_gpu_mem_usage",
- "to_quant_block_names",
- "enable_norm_bias_tuning",
- "act_bits",
- "act_group_size",
- "act_sym",
- "act_dynamic",
- "act_data_type",
- "super_bits",
- "super_group_size",
- ]
- if isinstance(self.dataset, str):
- serialization_keys.append("dataset")
- serialization_dict = {}
- for key in serialization_keys:
- serialization_dict[key] = getattr(self, key)
- from .version import __version__
-
- serialization_dict["autoround_version"] = __version__
- if "scale_dtype" in serialization_dict.keys():
- serialization_dict["scale_dtype"] = str(serialization_dict["scale_dtype"])
+ if (extra_config and not extra_config.mllm_config.is_default()) or is_mllm_model(model):
+ logger.info("using MLLM mode for multimodal model.")
+ model_cls.append(MLLMCompressor)
+ else:
+ if extra_config:
+ extra_config.mllm_config = None
+ model_cls.append(LLMCompressor)
- compressed_model = save_quantized_as_format( # TODO refine the code
- output_dir,
- model=self.model,
- layer_config=self.layer_config,
- inplace=inplace,
- bits=self.bits,
- act_bits=self.act_bits,
- group_size=self.group_size,
- sym=self.sym,
- iters=self.iters,
- lr=self.lr,
- minmax_lr=self.minmax_lr,
- enable_minmax_tuning=self.enable_minmax_tuning,
- enable_quanted_input=self.enable_quanted_input,
- scale_dtype=self.scale_dtype,
- tokenizer=self.tokenizer,
- supported_types=self.supported_types,
- data_type=self.data_type,
- act_data_type=self.act_data_type,
- serialization_dict=serialization_dict,
- backend=backend,
- to_quant_block_names=self.to_quant_block_names,
- quant_block_list=self.quant_block_list,
- device=self.device,
+ if enable_adam:
+ model_cls.append(AdamCompressor)
+ dynamic_compressor = type("AutoRound", tuple(model_cls), {})
+ if extra_config:
+ kwargs.update(extra_config.to_dict())
+ ar = dynamic_compressor(
+ model=model,
+ tokenizer=tokenizer,
+ scheme=scheme,
+ layer_config=layer_config,
+ dataset=dataset,
+ iters=iters,
+ seqlen=seqlen,
+ nsamples=nsamples,
+ batch_size=batch_size,
+ gradient_accumulate_steps=gradient_accumulate_steps,
+ low_gpu_mem_usage=low_gpu_mem_usage,
+ device_map=device_map,
+ enable_torch_compile=enable_torch_compile,
+ seed=seed,
**kwargs,
)
- return compressed_model
-
- def _get_quantized_layer_names_outside_blocks(self) -> list:
- """Gets the names of quantized layers outside blocks in the model.
-
- Returns:
- list: List of layer names outside blocks.
- """
- if self.layer_config is None or len(self.layer_config) == 0:
- return []
-
- layer_names = []
- all_layers_in_block = get_layer_names_in_block(self.model, self.supported_types, self.quant_block_list)
-
- for key in self.layer_config.keys():
- if key in all_layers_in_block:
- continue
- layer = get_module(self.model, key)
- if layer is None:
- logger.error(f"could not find layer {key} in the model, exit...")
- exit(-1)
- if isinstance(layer, tuple(self.supported_types)) and check_to_quantized(self.layer_config[key]):
- layer_names.append(key)
-
- return layer_names
-
- def _set_amp_dtype(self) -> None:
- """Sets the automatic mixed precision (AMP) data type for the model based on the device and configuration."""
- self.amp_dtype = torch.bfloat16
- if self.model.dtype != torch.float32:
- self.amp_dtype = self.model.dtype
- if self.device == "cpu" or "hpu" in self.device:
- self.amp_dtype = torch.bfloat16
- if self.amp:
- if self.device == "cpu" and not CpuInfo().bf16:
- self.amp = False
- self.amp_dtype = torch.float32
- self.model = self.model.to(torch.float32)
- logger.warning(
- f"amp is set to FALSE as the current {self.device} device does not support the 'bf16' data type."
- )
- else:
- if self.model.dtype != self.amp_dtype:
- self.model = self.model.to(self.amp_dtype)
- else:
- self.amp_dtype = torch.float32
- self.model = self.model.to(torch.float32)
-
- def _get_optimizer(self, optimizer: Any):
- """Returns the specified optimizer. In SignRound, we fix the optimizer.
-
- Args:
- optimizer: The optimizer to be used.
-
- Returns:
- The specified optimizer.
- """
- return SignSGD
-
- def _get_scaler(self):
- """Returns scaler, in SignRound, no need to use scaler."""
- return None
-
- def _scale_loss_and_backward(self, scaler: Any, loss: torch.Tensor) -> torch.Tensor:
- """Scales the loss and performs backward pass.
-
- Args:
- scaler: The scaler to be used.
- loss: The loss to be scaled.
-
- Returns:
- The scaled loss.
- """
- scale_loss = loss * 1000
- scale_loss.backward()
- if is_optimum_habana_available():
- htcore.mark_step()
- return scale_loss
-
- def _step(self, scaler: Any, optimizer: Any, lr_schedule: Any):
- """Performs a step in the optimization process.
-
- Args:
- scaler: The scaler to be used.
- optimizer: The optimizer for the step.
- lr_schedule: The learning rate schedule.
-
- Returns:
- None
- """
- optimizer.step()
- # for hpu
- if is_optimum_habana_available():
- htcore.mark_step()
- optimizer.zero_grad()
- lr_schedule.step()
+ return ar
@classmethod
@torch.no_grad()
@@ -3310,8 +224,120 @@ class AutoRound(object):
return current_input_ids, current_input_others
-class AutoRoundAdam(AutoRound):
- """Class for automatic rounding-based quantization with optimizers like adamw of a PyTorch model.
+@deprecated("AutoRound")
+class AutoRoundLLM(LLMCompressor):
+ """Class for LLM quantization
+
+ Args:
+ model (torch.nn.Module | str): Model object or model name to load.
+ tokenizer: Tokenizer for text processing. Required if `model` is not a string and `iters > 0`.
+ scheme (str| dict | QuantizationScheme ): A preset scheme that defines the quantization configurations
+ bits (int, optional): Weight quantization bits. Defaults to 4.
+ group_size (int, optional): Weight quantization group size. Defaults to 128.
+ sym (bool, optional): Symmetric weight quantization. Defaults to True.
+ layer_config (dict, optional): Layer-wise quantization config. Defaults to None.
+ batch_size (int, optional): Calibration batch size. Defaults to 8.
+ amp (bool, optional): Use AMP for tuning. Defaults to True.
+ device (str | torch.device | int, optional): Compute device. Defaults to 0.
+ dataset (str | list | tuple | DataLoader, optional): Calibration data. Defaults to "NeelNanda/pile-10k".
+ enable_minmax_tuning (bool, optional): Enable weight min-max tuning. Defaults to True.
+ lr (float, optional): Learning rate; if None, set to 1.0 / iters except when iters==0.
+ minmax_lr (float, optional): Learning rate for min-max tuning; defaults to `lr`.
+ low_gpu_mem_usage (bool, optional): Lower GPU memory mode. Defaults to False.
+ iters (int, optional): Optimization iterations. Defaults to 200.
+ seqlen (int, optional): Calibration sequence length. Defaults to 2048.
+ nsamples (int, optional): Number of calibration samples. Defaults to 128.
+ seed (int, optional): Random seed. Defaults to 42.
+ gradient_accumulate_steps (int, optional): Gradient accumulation steps. Defaults to 1.
+ data_type (str, optional): Weight data type string, e.g., "int". Defaults to "int".
+ act_bits (int, optional): Activation quantization bits. Defaults to 16.
+ act_group_size (int, optional): Activation group size. Defaults to None.
+ act_sym (bool, optional): Symmetric activation quantization. Defaults to None.
+ act_data_type (str, optional): Activation data type; inherits weight dtype if None and act_bits < 16.
+ act_dynamic (bool, optional): Dynamic activation quantization. Defaults to True.
+ enable_torch_compile (bool, optional): Enable torch.compile for quant blocks/layers. Defaults to False.
+ device_map (str | dict, optional): Device placement map. Defaults to None.
+ disable_opt_rtn (bool, optional): Disable RTN-mode optimization (iters=0). Defaults to False.
+ enable_alg_ext (bool, optional): Enable algorithm extension (primarily for INT2). Defaults to False.
+ **kwargs: Backward compatible options:
+ - enable_alg_ext, quant_lm_head, lr, lr_scheduler, sampler, not_use_best_mse, dynamic_max_gap,
+ super_group_size, super_bits, scale_dtype ("fp16" etc.),
+ nblocks, low_cpu_mem_usage, to_quant_block_names,
+ enable_norm_bias_tuning, enable_quanted_input,
+ disable_deterministic_algorithms, mllm, static_kv_dtype
+ Raises:
+ ValueError: If invalid device is provided or tokenizer is missing for non-str model with iters > 0.
+ RuntimeError: If model parameters are on meta device.
+ Example:
+ Layer-wise configuration structure:
+
+ >>> layer_config = {
+ ... "layer1": {
+ ... "data_type": "int",
+ ... "bits": 4,
+ ... "group_size": 128,
+ ... "sym": True,
+ ... "act_data_type": None,
+ ... "act_bits": 16,
+ ... "act_group_size": None,
+ ... "act_sym": None,
+ ... },
+ ... # ...
+ ... }
+ """
+
+ bits: int | None
+ group_size: int | None
+ sym: bool | None
+ data_type: str | None
+ act_bits: int | None
+ act_group_size: int | None
+ act_sym: bool | None
+ act_data_type: str | None
+ act_dynamic: bool | None
+ super_bits: int | None
+ super_group_size: int | None
+
+ def __init__(
+ self,
+ model: Union[torch.nn.Module, str],
+ tokenizer=None,
+ scheme: Union[str, dict, QuantizationScheme] = "W4A16",
+ layer_config: dict[str, Union[str, dict, QuantizationScheme]] = None,
+ dataset: Union[str, list, tuple, torch.utils.data.DataLoader] = "NeelNanda/pile-10k",
+ iters: int = 200,
+ seqlen: int = 2048,
+ nsamples: int = 128,
+ batch_size: int = 8,
+ gradient_accumulate_steps: int = 1,
+ low_gpu_mem_usage: bool = False,
+ device_map: Union[str, torch.device, int, dict] = 0,
+ enable_torch_compile: bool = False,
+ seed: int = 42,
+ **kwargs,
+ ):
+ super().__init__(
+ model=model,
+ tokenizer=tokenizer,
+ scheme=scheme,
+ layer_config=layer_config,
+ dataset=dataset,
+ iters=iters,
+ seqlen=seqlen,
+ nsamples=nsamples,
+ batch_size=batch_size,
+ gradient_accumulate_steps=gradient_accumulate_steps,
+ low_gpu_mem_usage=low_gpu_mem_usage,
+ device_map=device_map,
+ enable_torch_compile=enable_torch_compile,
+ seed=seed,
+ **kwargs,
+ )
+
+
+@deprecated("AutoRound")
+class AutoRoundAdam(AdamCompressor):
+ """Class for quantization with optimizers like adamw of a PyTorch model.
Args:
model: The PyTorch model to be quantized.
@@ -3391,7 +417,7 @@ class AutoRoundAdam(AutoRound):
optimizer="AdamW",
**kwargs,
):
- super(AutoRoundAdam, self).__init__(
+ super().__init__(
model=model,
tokenizer=tokenizer,
scheme=scheme,
@@ -3406,46 +432,111 @@ class AutoRoundAdam(AutoRound):
gradient_accumulate_steps=gradient_accumulate_steps,
enable_torch_compile=enable_torch_compile,
device_map=device_map,
+ optimizer=optimizer,
**kwargs,
)
- self.optimizer = self._get_optimizer(optimizer)
- def _get_optimizer(self, optimizer):
- if optimizer is None:
- optimizer = torch.optim.AdamW
- elif isinstance(optimizer, str):
- optimizer = getattr(torch.optim, optimizer)
- else:
- optimizer = optimizer
- return optimizer
-
- def _get_scaler(self):
- scaler = None
- if self.amp and not check_is_cpu(self.device):
- from torch.cuda.amp import GradScaler
+@deprecated("AutoRound")
+class AutoRoundMLLM(MLLMCompressor):
+ """Class for automatic rounding-based quantization with MLLMs.
- scaler = GradScaler(init_scale=1024, growth_interval=100000)
- return scaler
-
- def _scale_loss_and_backward(self, scaler, loss):
- if scaler is not None:
- loss = scaler.scale(loss)
+ Args:
+ model: The PyTorch model to be quantized.
+ tokenizer: An optional tokenizer for processing input data.
+ processor: Any multi-modal model will require an object to encode or
+ decode the data that groups several modalities (among text, vision and audio).
+ image_processor: Image processor for special model like llava.
+ bits (int): Number of bits for quantization (default is 4).
+ group_size (int): Size of the quantization group (default is 128).
+ sym (bool): Whether sym to be used (default is True).
+ layer_config (dict): Configuration for weight quantization (default is None).
+ batch_size (int): Batch size for training (default is 8).
+ amp (bool): Whether to use automatic mixed precision (default is True).
+ device: The device to be used for training (default is "auto").
+ lr_scheduler: The learning rate scheduler to be used.
+ dataset: The path or name of the calib dataset.
+ extra_data_dir: The path of extra data such as images, audio and videos.
+ template: The path or name of template used to specify process for different MLLMs.
+ quant_nontext_module: Whether to quantize nontext module.
+ enable_quanted_input (bool): Whether to use quantized input data (default is True).
+ enable_minmax_tuning (bool): Whether to enable min-max tuning (default is True).
+ lr (float): The learning rate (default is 0.005).
+ minmax_lr (float): The learning rate for min-max tuning (default is None).
+ low_gpu_mem_usage (bool): Whether to use low GPU memory (default is False).
+ low_cpu_mem_usage (bool): Whether to use low CPU memory (default is False).
+ iters (int): Number of iterations (default is 200).
+ seqlen (int): Length of the sequence.
+ nsamples (int): Number of samples (default is 128).
+ sampler (str): The sampling method (default is "rand").
+ seed (int): The random seed (default is 42).s
+ nblocks (int): Number of blocks (default is 1).
+ gradient_accumulate_steps (int): Number of gradient accumulation steps (default is 1).
+ not_use_best_mse (bool): Whether to use mean squared error (default is False).
+ dynamic_max_gap (int): The dynamic maximum gap (default is -1).
+ data_type (str): The data type to be used (default is "int").
+ scale_dtype (str): The data type of quantization scale to be used (default is "float16"), different kernels
+ have different choices.
+ act_bits (int): Number of bits for activation quantization. Default is 32.
+ act_group_size (int): Group size for activation quantization. Default is None.
+ act_sym (bool): Whether to use symmetric activation quantization. Default is None.
+ act_dynamic (bool): Whether to use dynamic activation quantization. Default is True.
+ to_quant_block_names (str|list): A string or list whose elements are list of
+ block's layer names to be quantized.
+ enable_torch_compile (bool): Whether to enable torch compile to optimize quant_block/layer
+ **kwargs: Additional keyword arguments.
+ """
- loss.backward()
- if is_optimum_habana_available():
- htcore.mark_step()
- return loss
+ bits: int | None
+ group_size: int | None
+ sym: bool | None
+ data_type: str | None
+ act_bits: int | None
+ act_group_size: int | None
+ act_sym: bool | None
+ act_data_type: str | None
+ act_dynamic: bool | None
+ super_bits: int | None
+ super_group_size: int | None
- def _step(self, scaler, optimizer, lr_schedule):
- if scaler is not None:
- scaler.step(optimizer)
- optimizer.zero_grad()
- lr_schedule.step()
- scaler.update()
- else:
- optimizer.step()
- optimizer.zero_grad()
- lr_schedule.step()
- if is_optimum_habana_available():
- htcore.mark_step()
+ def __init__(
+ self,
+ model: Union[torch.nn.Module, str],
+ tokenizer=None,
+ processor=None,
+ image_processor=None,
+ scheme: Union[str, dict, QuantizationScheme] = "W4A16",
+ layer_config: dict[str, Union[str, dict, QuantizationScheme]] = None,
+ dataset: Union[str, list, tuple, torch.utils.data.DataLoader] = "NeelNanda/pile-10k",
+ quant_nontext_module: bool = False,
+ iters: int = 200,
+ seqlen: int = 2048,
+ nsamples: int = 128,
+ batch_size: int = 8,
+ gradient_accumulate_steps: int = 1,
+ low_gpu_mem_usage: bool = False,
+ device_map: Union[str, torch.device, int, dict] = 0,
+ enable_torch_compile: bool = False,
+ seed: int = 42,
+ **kwargs,
+ ):
+ super().__init__(
+ model=model,
+ tokenizer=tokenizer,
+ processor=processor,
+ image_processor=image_processor,
+ scheme=scheme,
+ layer_config=layer_config,
+ dataset=dataset,
+ quant_nontext_module=quant_nontext_module,
+ iters=iters,
+ seqlen=seqlen,
+ nsamples=nsamples,
+ batch_size=batch_size,
+ gradient_accumulate_steps=gradient_accumulate_steps,
+ low_gpu_mem_usage=low_gpu_mem_usage,
+ device_map=device_map,
+ enable_torch_compile=enable_torch_compile,
+ seed=seed,
+ **kwargs,
+ )
diff --git a/auto_round/script/__init__.py b/auto_round/compressors/__init__.py
similarity index 66%
rename from auto_round/script/__init__.py
rename to auto_round/compressors/__init__.py
index 28f108cb6..35bbc5666 100644
--- a/auto_round/script/__init__.py
+++ b/auto_round/compressors/__init__.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2024 Intel Corporation
+# Copyright (c) 2025 Intel Corporation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -11,3 +11,12 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
+
+from auto_round.compressors.base import *
+from auto_round.compressors.mllm.compressor import MLLMCompressor
+from auto_round.compressors.config import (
+ ExtraConfig,
+ MLLMExtraConfig,
+ SchemeExtraConfig,
+ TuningExtraConfig,
+)
diff --git a/auto_round/compressors/base.py b/auto_round/compressors/base.py
new file mode 100644
index 000000000..9ed346bb8
--- /dev/null
+++ b/auto_round/compressors/base.py
@@ -0,0 +1,3468 @@
+# Copyright (c) 2023 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import copy
+import os
+import re
+import sys
+import time
+import traceback
+from dataclasses import asdict, fields
+from enum import Enum
+from typing import Any, Callable, Union
+
+import accelerate
+import torch
+from accelerate.big_modeling import dispatch_model, infer_auto_device_map
+from torch import autocast
+from tqdm import tqdm
+from transformers import set_seed
+
+from auto_round.data_type import QUANT_FUNC_WITH_DTYPE
+from auto_round.data_type.utils import reshape_pad_tensor_by_group_size
+from auto_round.export.export_to_autoround import AutoRoundFormat
+from auto_round.export.export_to_gguf.config import GGUF_CONFIG, GGUF_INNER_CONFIG, ModelType
+from auto_round.logger import logger
+from auto_round.low_cpu_mem.utils import get_layers_before_block
+from auto_round.schemes import QuantizationScheme, preset_name_to_scheme
+from auto_round.sign_sgd import SignSGD
+from auto_round.special_model_handler import _handle_moe_model
+from auto_round.utils import (
+ INNER_SUPPORTED_LAYER_TYPES,
+ SUPPORTED_DTYPES,
+ SUPPORTED_FORMATS,
+ SUPPORTED_LAYER_TYPES,
+ TORCH_VERSION_AT_LEAST_2_6,
+ CpuInfo,
+ _gguf_args_check,
+ _is_fp8_linear,
+ _is_fp8_model,
+ block_forward,
+ check_and_mark_fp8_model,
+ check_is_cpu,
+ check_need_act_calibration,
+ check_seqlen_compatible,
+ check_skippable_keywords,
+ check_to_quantized,
+ clear_memory,
+ collect_best_params,
+ compile_func,
+ convert_dtype_str2torch,
+ convert_fp8_layer_to_linear,
+ convert_fp8_model_to_16b_model,
+ copy_python_files_from_model_cache,
+ detect_device,
+ estimate_tuning_block_mem,
+ find_matching_blocks,
+ flatten_list,
+ get_block_names,
+ get_device_memory,
+ get_fp_layer_names,
+ get_layer_config_by_gguf_format,
+ get_layer_features,
+ get_layer_names_in_block,
+ get_lm_head_name,
+ get_max_vram,
+ get_module,
+ get_quant_keys,
+ get_shared_keys,
+ htcore,
+ infer_bits_by_data_type,
+ init_cache,
+ is_debug_mode,
+ is_mx_fp,
+ is_nv_fp,
+ is_optimum_habana_available,
+ is_standard_fp,
+ is_static_wfp8afp8,
+ is_wfp8afp8,
+ llm_load_model,
+ mv_module_from_gpu,
+ reset_params,
+ set_amax_for_all_moe_layers,
+ set_module,
+ to_device,
+ to_dtype,
+ unsupport_meta_device,
+)
+from auto_round.wrapper import WrapperLinear, WrapperMultiblock, unwrapper_block, unwrapper_layer, wrapper_block
+
+
+class BaseCompressor(object):
+ """Base compressor for LLM quantization
+
+ Attributes:
+ model (torch.nn.Module): The loaded PyTorch model in eval mode.
+ tokenizer: Tokenizer used to prepare input text for calibration/tuning.
+ bits (int): Weight quantization bits.
+ group_size (int): Per-group size for weight quantization.
+ sym (bool): Whether to use symmetric weight quantization.
+ layer_config (dict): Per-layer quantization configuration.
+ nsamples (int): Number of calibration samples.
+ enable_torch_compile (bool): Whether to enable torch.compile for quant blocks/layers.
+ """
+
+ bits: int | None
+ group_size: int | None
+ sym: bool | None
+ data_type: str | None
+ act_bits: int | None
+ act_group_size: int | None
+ act_sym: bool | None
+ act_data_type: str | None
+ act_dynamic: bool | None
+ super_bits: int | None
+ super_group_size: int | None
+
+ def __init__(
+ self,
+ model: Union[torch.nn.Module, str],
+ tokenizer=None,
+ scheme: Union[str, dict, QuantizationScheme] = "W4A16",
+ layer_config: dict[str, Union[str, dict, QuantizationScheme]] = None,
+ dataset: Union[str, list, tuple, torch.utils.data.DataLoader] = "NeelNanda/pile-10k",
+ iters: int = 200,
+ seqlen: int = 2048,
+ nsamples: int = 128,
+ batch_size: int = 8,
+ gradient_accumulate_steps: int = 1,
+ low_gpu_mem_usage: bool = False,
+ device_map: Union[str, torch.device, int, dict] = 0,
+ enable_torch_compile: bool = False,
+ seed: int = 42,
+ **kwargs,
+ ):
+ """Initialize AutoRound with quantization and tuning configuration.
+
+ Args:
+ model (torch.nn.Module | str): Model object or model name to load.
+ tokenizer: Tokenizer for text processing. Required if `model` is not a string and `iters > 0`.
+ scheme (str| dict | QuantizationScheme ): A preset scheme that defines the quantization configurations
+ bits (int, optional): Weight quantization bits. Defaults to 4.
+ group_size (int, optional): Weight quantization group size. Defaults to 128.
+ sym (bool, optional): Symmetric weight quantization. Defaults to True.
+ layer_config (dict, optional): Layer-wise quantization config. Defaults to None.
+ batch_size (int, optional): Calibration batch size. Defaults to 8.
+ amp (bool, optional): Use AMP for tuning. Defaults to True.
+ device (str | torch.device | int, optional): Compute device. Defaults to 0.
+ dataset (str | list | tuple | DataLoader, optional): Calibration data. Defaults to "NeelNanda/pile-10k".
+ enable_minmax_tuning (bool, optional): Enable weight min-max tuning. Defaults to True.
+ lr (float, optional): Learning rate; if None, set to 1.0 / iters except when iters==0.
+ minmax_lr (float, optional): Learning rate for min-max tuning; defaults to `lr`.
+ low_gpu_mem_usage (bool, optional): Lower GPU memory mode. Defaults to False.
+ iters (int, optional): Optimization iterations. Defaults to 200.
+ seqlen (int, optional): Calibration sequence length. Defaults to 2048.
+ nsamples (int, optional): Number of calibration samples. Defaults to 128.
+ seed (int, optional): Random seed. Defaults to 42.
+ gradient_accumulate_steps (int, optional): Gradient accumulation steps. Defaults to 1.
+ data_type (str, optional): Weight data type string, e.g., "int". Defaults to "int".
+ act_bits (int, optional): Activation quantization bits. Defaults to 16.
+ act_group_size (int, optional): Activation group size. Defaults to None.
+ act_sym (bool, optional): Symmetric activation quantization. Defaults to None.
+ act_data_type (str, optional): Activation data type; inherits weight dtype if None and act_bits < 16.
+ act_dynamic (bool, optional): Dynamic activation quantization. Defaults to True.
+ enable_torch_compile (bool, optional): Enable torch.compile for quant blocks/layers. Defaults to False.
+ device_map (str | dict, optional): Device placement map. Defaults to None.
+ disable_opt_rtn (bool, optional): Disable RTN-mode optimization (iters=0). Defaults to False.
+ enable_alg_ext (bool, optional): Enable algorithm extension (primarily for INT2). Defaults to False.
+ **kwargs: Backward compatible options:
+ - enable_alg_ext, quant_lm_head, lr, lr_scheduler, sampler, not_use_best_mse, dynamic_max_gap,
+ super_group_size, super_bits, scale_dtype ("fp16" etc.),
+ nblocks, low_cpu_mem_usage, to_quant_block_names,
+ enable_norm_bias_tuning, enable_quanted_input,
+ disable_deterministic_algorithms, mllm, static_kv_dtype
+ Raises:
+ ValueError: If invalid device is provided or tokenizer is missing for non-str model with iters > 0.
+ RuntimeError: If model parameters are on meta device.
+ Example:
+ Layer-wise configuration structure:
+
+ >>> layer_config = {
+ ... "layer1": {
+ ... "data_type": "int",
+ ... "bits": 4,
+ ... "group_size": 128,
+ ... "sym": True,
+ ... "act_data_type": None,
+ ... "act_bits": 16,
+ ... "act_group_size": None,
+ ... "act_sym": None,
+ ... },
+ ... # ...
+ ... }
+ """
+ self.scheme = None
+ self._parse_and_set_scheme(scheme, kwargs)
+
+ # Extra/legacy kwargs for backward compatibility
+ # Major version releases may pack them with extra configuration options
+ amp = kwargs.pop("amp", True)
+ lr = kwargs.pop("lr", None)
+ enable_alg_ext = kwargs.pop("enable_alg_ext", False)
+ enable_minmax_tuning = kwargs.pop("enable_minmax_tuning", True)
+ minmax_lr = kwargs.pop("minmax_lr", None)
+ disable_opt_rtn = kwargs.pop("disable_opt_rtn", False)
+ lr_scheduler = kwargs.pop("lr_scheduler", None)
+ sampler = kwargs.pop("sampler", "rand")
+ not_use_best_mse = kwargs.pop("not_use_best_mse", False)
+ dynamic_max_gap = kwargs.pop("dynamic_max_gap", -1)
+ scale_dtype = kwargs.pop("scale_dtype", "fp16")
+ nblocks = kwargs.pop("nblocks", 1)
+ low_cpu_mem_usage = kwargs.pop("low_cpu_mem_usage", False)
+ to_quant_block_names: Union[str, list, None] = kwargs.pop("to_quant_block_names", None)
+ enable_norm_bias_tuning: bool = kwargs.pop("enable_norm_bias_tuning", False)
+ enable_quanted_input: bool = kwargs.pop("enable_quanted_input", True)
+ disable_deterministic_algorithms = kwargs.pop("disable_deterministic_algorithms", True)
+ enable_deterministic_algorithms = kwargs.pop("enable_deterministic_algorithms", False)
+ static_kv_dtype = kwargs.pop("static_kv_dtype", None)
+ device = kwargs.pop("device", None)
+ self.quant_lm_head = kwargs.pop("quant_lm_head", False)
+ self.mllm = kwargs.pop("mllm") if "mllm" in kwargs else False
+ # Scale factor for RAM usage per parameter.
+ self.mem_per_param_scale = kwargs.pop("mem_per_param_scale", None)
+ fp_layers = kwargs.pop("fp_layers", None)
+
+ if kwargs:
+ logger.warning(f"unrecognized keys {list(kwargs.keys())} were passed. Please check them.")
+ if "CUBLAS_WORKSPACE_CONFIG" not in os.environ:
+ os.environ["CUBLAS_WORKSPACE_CONFIG"] = ":4096:8"
+ # deprecated, default not to use torch.use_deterministic_algorithms
+ if not disable_deterministic_algorithms or enable_deterministic_algorithms:
+ if not disable_deterministic_algorithms:
+ logger.warning(
+ "default not use deterministic_algorithms. disable_deterministic_algorithms is deprecated,"
+ " please use enable_deterministic_algorithms instead. "
+ )
+
+ torch.use_deterministic_algorithms(True, warn_only=False)
+ else:
+ torch.use_deterministic_algorithms(True, warn_only=True)
+
+ if device is not None:
+ logger.warning("`device` is deprecated, please use `device_map` instead")
+
+ if device_map is None:
+ device_map = 0
+
+ # Set device, must place after model loading
+ self._set_device(device_map)
+
+ if (isinstance(device_map, dict) and device_map) or device_map == "auto":
+ self.device_map = device_map
+ elif isinstance(device_map, str) and "," in device_map:
+ device_map = device_map.replace(" ", "") # Remove any spaces
+ self.device_list = [int(dev) for dev in device_map.split(",") if dev.isdigit()]
+ self.device_map = "auto"
+ else:
+ self.device_map = None
+ self._set_device_map_in_blocks(self.device_map)
+
+ # Model related
+ self.quantized = False
+ if isinstance(model, str):
+ model, tokenizer, low_cpu_mem_usage = llm_load_model(
+ model,
+ device="cpu",
+ low_cpu_mem_mode=low_cpu_mem_usage, # always load cpu first
+ )
+ elif tokenizer is None and iters > 0:
+ raise ValueError("A tokenizer must be set for non-str model input")
+ self.low_cpu_mem_usage = bool(low_cpu_mem_usage)
+ if unsupport_meta_device(model):
+ raise RuntimeError(
+ "AutoRound does not support parameters on meta device. "
+ "Please use more GPUs by setting `--device 0,1,2,3` or just place the model on CPU."
+ )
+ check_and_mark_fp8_model(model)
+ self.model = model.eval()
+ self.tokenizer = tokenizer
+ self.shared_cache_keys = get_shared_keys(self.model)
+
+ not_quantize_layer_names = get_fp_layer_names(self.model, fp_layers)
+ if len(not_quantize_layer_names) > 0:
+ logger.info(f"{not_quantize_layer_names} will not be quantized.")
+ if layer_config is None:
+ layer_config = {}
+ for name in not_quantize_layer_names:
+ layer_config[name] = {"bits": 16, "act_bits": 16, "data_type": "float", "act_data_type": "float"}
+ self._parse_layer_config(layer_config) # must place after model init
+
+ self.to_quant_block_names = to_quant_block_names
+
+ # Tuning hyperparameters
+ self.seed = seed
+ set_seed(self.seed)
+ self.amp = amp
+ self.enable_quanted_input = enable_quanted_input
+ self.enable_minmax_tuning = enable_minmax_tuning
+ self.nsamples = nsamples
+ self.enable_norm_bias_tuning = enable_norm_bias_tuning
+ self.low_gpu_mem_usage = low_gpu_mem_usage
+ self.seqlen = seqlen
+ self.batch_size, self.gradient_accumulate_steps = batch_size, gradient_accumulate_steps
+ self.nblocks = nblocks
+ self.dataset = dataset
+ self.iters = iters
+ if self.iters < 0:
+ logger.warning("`iters` must be non-negative, reset it to 200")
+ self.iters = 200
+ if self.iters == 0:
+ self.lr = 5e-3
+ else:
+ self.lr = lr or (1.0 / self.iters) # must place after iter setting
+ self.minmax_lr = minmax_lr or self.lr
+ self.enable_alg_ext = enable_alg_ext
+ self.sampler = sampler
+ self.not_use_best_mse = not_use_best_mse
+ self.dynamic_max_gap = dynamic_max_gap
+ self.lr_scheduler = lr_scheduler
+ self.optimizer = self._get_optimizer(None)
+ self.disable_opt_rtn = disable_opt_rtn
+ self.is_packing_immediate = False # whether to pack the layer immediately after tuning
+
+ # KV cache, this one does not affect tuning but will collect some infos during tuning
+ self.static_kv_dtype = static_kv_dtype
+ if self.static_kv_dtype is not None:
+ logger.warning("The static kv is experimental and currently has limited support.")
+
+ # Model related
+ self.quantized = False
+ if isinstance(model, str):
+ model, tokenizer, low_cpu_mem_usage = llm_load_model(
+ model, device=device, low_cpu_mem_mode=low_cpu_mem_usage
+ )
+ elif tokenizer is None and iters > 0:
+ raise ValueError("A tokenizer must be set for non-str model input")
+ self.low_cpu_mem_usage = bool(low_cpu_mem_usage)
+ if unsupport_meta_device(model):
+ raise RuntimeError(
+ "AutoRound does not support parameters on meta device. "
+ "Please use more GPUs by setting `--device_map 0,1,2,3` or just place the model on CPU."
+ )
+ self.model = model.eval()
+ self.tokenizer = tokenizer
+ self.shared_cache_keys = get_shared_keys(self.model)
+ if not hasattr(self, "quant_block_list"):
+ all_blocks = get_block_names(model)
+ self.quant_block_list = find_matching_blocks(model, all_blocks, self.to_quant_block_names)
+
+ self.scale_dtype = convert_dtype_str2torch(scale_dtype)
+ self._set_amp_dtype()
+ self.cache_device = torch.device("cpu") if self.low_gpu_mem_usage else self.device
+ if self.act_bits <= 8 and self.amp_dtype == torch.float16:
+ logger.warning("force to use bf16 to for quantization tuning when enabling activation quantization")
+ self.amp_dtype = torch.bfloat16
+ if self.model.dtype != torch.bfloat16: # keep the model's buffer dtype unchanged
+ self.model = self.model.to(torch.bfloat16)
+ else:
+ logger.info(f"using {self.model.dtype} for quantization tuning")
+
+ # Some helpers
+ self.supported_types = SUPPORTED_LAYER_TYPES
+ self.inner_supported_types = INNER_SUPPORTED_LAYER_TYPES
+ if "hpu" in str(self.device):
+ self.inner_supported_types = tuple(x for x in INNER_SUPPORTED_LAYER_TYPES if x != "FP8Linear")
+ self.batch_dim = None
+ self.infer_bs_coeff = 1
+ self.enable_torch_compile = enable_torch_compile
+ self._adjust_torch_compile(enable_torch_compile)
+ self._check_configs()
+ torch.set_printoptions(precision=3, sci_mode=True)
+
+ if is_optimum_habana_available():
+ logger.info("optimum Habana is available, import htcore explicitly.")
+ import habana_frameworks.torch.core as htcore # pylint: disable=E0401
+ import habana_frameworks.torch.hpu as hthpu # pylint: disable=E0401]
+
+ def _set_device(self, device_map):
+ if hasattr(self, "device") and self.device is not None:
+ return
+ if isinstance(device_map, (str, torch.device, int)):
+ self.device = detect_device(device_map)
+
+ elif isinstance(device_map, dict) and device_map:
+ tmp_devices = []
+ for val in device_map.values():
+ if isinstance(val, (str, torch.device, int)): # could optimize
+ tmp_device = detect_device(val)
+ tmp_device = tmp_device.split(":")[0]
+ tmp_devices.append(tmp_device)
+ tmp_devices = list(set(tmp_devices))
+ if len(tmp_devices) > 1:
+ logger.warning(
+ f"there are multiple device types in the device_map, "
+ f"please make sure they are correct,use the first device {tmp_devices[0]} as the core device "
+ )
+
+ self.device = tmp_devices[0]
+ else:
+ raise TypeError(f"device_map should be [str, torch.device, int, dict], but got {type(device_map)}")
+
+ def _parse_layer_config(self, layer_config: dict[str, Union[str, dict, QuantizationScheme]]) -> None:
+ """Parse and set the layer-wise quantization configuration."""
+ # Some other quantization configs
+ self.layer_config = {} if layer_config is None else layer_config
+ scheme_keys = [f.name for f in fields(QuantizationScheme)]
+ for key, item in self.layer_config.items():
+ if isinstance(item, str):
+ item = asdict(preset_name_to_scheme(item.upper()))
+ self.layer_config[key] = item
+
+ if isinstance(item, QuantizationScheme):
+ config = asdict(item)
+ tmp_keys = copy.deepcopy(list(config.keys()))
+ for tmp_key in tmp_keys: ## Pop None value to be overridden
+ if config[tmp_key] is None:
+ config.pop(tmp_key)
+ self.layer_config[key] = config
+ elif isinstance(item, dict):
+ item_keys = item.keys()
+ if item_keys not in scheme_keys:
+ for item_key in item_keys:
+ if item_key not in scheme_keys:
+ raise ValueError(
+ f"the key {item_key} in layer_config for layer {key} is invalid,"
+ f" only {scheme_keys} are supported"
+ )
+
+ if not self.quant_lm_head or (isinstance(self.scheme, str) and self.scheme.lower().startswith("gguf")):
+ return
+ for n, _ in self.model.named_modules():
+ lm_head_layer_name = n
+
+ if (
+ hasattr(self.model, "config")
+ and self.model.config.tie_word_embeddings
+ and hasattr(self.model, "_tied_weights_keys")
+ ):
+ tied_keys = self.model._tied_weights_keys
+ for item in tied_keys:
+ if lm_head_layer_name in item: # TODO extend to encoder-decoder layer, seq classification model
+ self.quant_lm_head = False
+ logger.warning(
+ "reset `quant_lm_head` to `False` as quantizing lm_head with tied weights has not been "
+ "supported currently"
+ )
+ break
+
+ lm_head_layer_config = self.layer_config[lm_head_layer_name] if lm_head_layer_name in self.layer_config else {}
+
+ for key in scheme_keys:
+ if key not in lm_head_layer_config:
+ lm_head_layer_config[key] = getattr(self, key)
+
+ def _parse_and_set_scheme(self, scheme: Union[str, dict, QuantizationScheme], kwargs) -> None:
+ """Parse and set the quantization scheme."""
+ if isinstance(scheme, QuantizationScheme):
+ scheme = asdict(scheme)
+ elif isinstance(scheme, dict):
+ scheme = scheme
+ elif isinstance(scheme, str):
+ scheme = scheme.upper()
+ self.scheme = scheme
+ scheme = asdict(preset_name_to_scheme(scheme))
+ scheme_keys = [f.name for f in fields(QuantizationScheme)]
+ for key in scheme_keys:
+ if key in kwargs and kwargs[key] is not None:
+ setattr(self, key, kwargs[key])
+ else:
+ setattr(self, key, scheme.get(key, None))
+ kwargs.pop(key, None)
+ if self.act_dynamic is None:
+ self.act_dynamic = True
+
+ tmp_bits = infer_bits_by_data_type(self.data_type)
+ if tmp_bits is not None and tmp_bits < 16 and tmp_bits != self.bits:
+ logger.warning(f"'data_type' do not match the specified 'bits' setting. Resetting 'bits' to {tmp_bits}.")
+ self.bits = tmp_bits
+ if tmp_bits is not None and tmp_bits < 16:
+ for supported_dtype in SUPPORTED_DTYPES: # to easily handle dtype mx_fp4 and layer_config={xxx:{bits:8}}
+ if self.data_type.startswith(supported_dtype):
+ if supported_dtype + str(tmp_bits) == self.data_type: # could not replace FP8_e4m3
+ self.data_type = supported_dtype
+ break
+
+ self.act_group_size = self.act_group_size if self.act_group_size is not None else self.group_size
+ self.act_bits = self.act_bits if self.act_bits is not None else 16
+ self.act_sym = self.act_sym if self.act_sym is not None else self.sym
+
+ if self.act_data_type is None:
+ if self.data_type in SUPPORTED_DTYPES and self.act_bits < 16:
+ self.act_data_type = self.data_type
+ logger.info(f"activation adopts {self.data_type}")
+ else:
+ self.act_data_type = "float"
+ tmp_act_bits = infer_bits_by_data_type(self.act_data_type)
+ if tmp_act_bits is not None and tmp_act_bits < 16 and tmp_act_bits != self.act_bits:
+ self.act_bits = tmp_act_bits
+ logger.warning(
+ f"`act_data_type` do not"
+ f" match the specified 'act_bits' setting. Resetting 'act_bits' to {tmp_act_bits}."
+ )
+ if tmp_act_bits is not None and tmp_act_bits < 16:
+ for supported_dtype in SUPPORTED_DTYPES: # to easily handle dtype mx_fp4 and layer_config={xxx:{bits:8}}
+ if self.act_data_type.startswith(supported_dtype):
+ if supported_dtype + str(tmp_act_bits) == self.act_data_type: # could not replace FP8_e4m3
+ self.act_data_type = supported_dtype
+ break
+
+ def _adjust_torch_compile(self, enable_torch_compile: bool) -> None:
+ """Sets the torch compile configuration for the tuning."""
+ self.enable_torch_compile = enable_torch_compile
+ if (
+ not self.enable_torch_compile
+ and TORCH_VERSION_AT_LEAST_2_6
+ and self.act_bits > 8
+ and not is_debug_mode()
+ and not self.low_cpu_mem_usage
+ and "fp8" not in self.data_type
+ and "fp8" not in self.act_data_type
+ ):
+ logger.info(
+ "'enable_torch_compile' is set to `False` by default. "
+ "Enabling it can reduce tuning cost by 20%, but it might throw an exception."
+ )
+
+ if self.low_cpu_mem_usage and self.enable_torch_compile:
+ self.enable_torch_compile = False
+ logger.warning("reset enable_torch_compile to `False` as low_cpu_mem_usage is enabled")
+
+ if is_debug_mode() and self.enable_torch_compile:
+ self.enable_torch_compile = False
+ logger.warning("reset enable_torch_compile to `False` as debug mode is enabled")
+
+ if ("fp8" in self.data_type or "fp8" in self.act_data_type) and self.enable_torch_compile:
+ self.enable_torch_compile = False
+ logger.warning("reset enable_torch_compile to `False` as fp8 is enabled")
+
+ def _set_device_map_in_blocks(self, device_map: Union[str, dict, None]) -> None:
+ """Sets the device map for specific blocks in the model.
+
+ Args:
+ device_map (Union[str, dict, None]): A mapping of module names to devices.
+ If provided as a string, it should be in the format
+ "module_name:device,module_name:device". Devices can be integers
+ (GPU IDs) or strings (e.g., 'cpu', 'cuda:0').
+ """
+ if self.device_map is None or len(self.device_map) == 0:
+ self.device_map = None
+ if not device_map:
+ return
+ if self.device_map == "auto" and device_map == "auto":
+ return
+ if isinstance(device_map, str):
+ device_map = device_map.replace(" ", "")
+ infos = device_map.split(",")
+ device_map_dict = {}
+ for info in infos:
+ index = info.find(":")
+ key = info[:index]
+ value = info[index + 1 :]
+ device_map_dict[key] = value
+ device_map = device_map_dict
+
+ names = [n for n, m in self.model.named_modules() if len(list(m.children())) == 0]
+
+ for key, device in device_map.items():
+ if isinstance(device, str) and device.isdigit():
+ device = int(device)
+ device = detect_device(device)
+ try:
+ module = get_module(self.model, key)
+ module.tuning_device = device
+ except:
+ matching_names = [name for name in names if re.match(key, name)]
+ if len(matching_names) > 0:
+ for name in matching_names:
+ self._set_device_for_matching_module(name, device)
+ else:
+ for name in names:
+ if key in name:
+ self._set_device_for_matching_module(name, device)
+
+ def _set_device_for_matching_module(self, name: str, device: str) -> None:
+ """Sets the device for a module if it matches the given name."""
+ module = get_module(self.model, name)
+ if hasattr(module, "tuning_device") and module.tuning_device != device:
+ logger.warning(
+ f"multiple devices have been set for layer {name}, keeping original device {module.tuning_device}"
+ )
+ else:
+ module.tuning_device = device
+
+ def _set_auto_device_map_in_block(self, block: torch.nn.Module, input_ids: list[torch.Tensor]) -> None:
+ """Automatically sets the device map for the block based on available GPUs and memory constraints."""
+ if torch.cuda.is_available():
+ num_gpus = torch.cuda.device_count()
+ elif torch.xpu.is_available():
+ logger.warning_once("XPU does not support auto device map yet, using device 0 for tuning.")
+ return
+ else:
+ raise RuntimeError("No CUDA or XPU devices found.")
+ if num_gpus <= 1:
+ self.device_map = None
+ return
+
+ if hasattr(self, "device_list") and self.device_list:
+ cuda_devices = [f"cuda:{i}" for i in self.device_list]
+ device_0 = cuda_devices[0]
+ else:
+ cuda_devices = [f"cuda:{i}" for i in range(num_gpus)]
+ device_0 = "cuda:0"
+
+ device_0_memory = get_device_memory(
+ self.device_list[0] if hasattr(self, "device_list") and self.device_list else 0
+ )
+ block_memory, input_ouput_memory = estimate_tuning_block_mem(block, input_ids)
+ if self.low_gpu_mem_usage:
+ input_ouput_memory = 0
+
+ mem_per_param_scale = 13 if self.mem_per_param_scale is None else self.mem_per_param_scale
+ if self.iters == 0:
+ mem_per_param_scale = 1 # for rtn
+
+ if (block_memory * mem_per_param_scale + input_ouput_memory) < device_0_memory:
+ return # fit in one GPU
+
+ device_map = {}
+ device_memory = {device: get_device_memory(int(device.split(":")[1])) for device in cuda_devices}
+ device_memory[device_0] = device_0_memory - input_ouput_memory
+
+ device_idx = 0
+ # First, fill device 0 to its maximum capacity, then distribute the remaining layers evenly across other devices
+ for n, m in block.named_modules():
+ if check_to_quantized(m):
+ layer_name = block.tmp_name + "." + n
+ layer_memory = m.weight.nbytes / 1024**3
+ if device_idx == 0 and layer_memory * mem_per_param_scale < device_memory[cuda_devices[device_idx]]:
+ device_map[layer_name] = cuda_devices[device_idx]
+ device_memory[cuda_devices[device_idx]] -= layer_memory * mem_per_param_scale
+ elif device_idx == 0:
+ device_idx += 1 # Move to the next device once device 0 is full
+ device_map[layer_name] = cuda_devices[device_idx]
+ device_memory[cuda_devices[device_idx]] -= layer_memory * mem_per_param_scale
+ else:
+ # Calculate the target device index based on even distribution
+ sorted_devices = sorted(cuda_devices, key=lambda d: device_memory[d], reverse=True)
+ device_idx = sorted_devices[0]
+ if layer_memory * mem_per_param_scale < device_memory[device_idx]:
+ device_map[layer_name] = device_idx
+ device_memory[device_idx] -= layer_memory * mem_per_param_scale
+ else:
+ logger.warning_once(
+ f"Block {block.tmp_name} not fit in available GPU memory. "
+ "Consider using more GPUs or reducing mem_per_param_scale if OOM occurs."
+ )
+ self._set_device_map_in_blocks(device_map)
+
+ def _dq_check(self) -> None:
+ """Reset the default value of super_bits and super_group_size"""
+ if self.data_type.endswith("_dq"):
+ gguf_config = GGUF_INNER_CONFIG[f"gguf:q{self.bits}_k"]
+ self.super_bits = gguf_config["super_bits"] if self.super_bits is None else self.super_bits
+ self.super_group_size = (
+ gguf_config["super_group_size"] if self.super_group_size is None else self.super_group_size
+ )
+
+ def _check_configs(self) -> None:
+ """Checks if the configurations are valid.
+
+ Raises:
+ ValueError, TypeError: If any of the configurations are invalid.
+ """
+ if not isinstance(self.model, torch.nn.Module):
+ raise TypeError("model must be an instance of torch.nn.Module")
+ if self.bits <= 0:
+ raise ValueError("`bits` must be positive")
+ if self.act_bits <= 0:
+ raise ValueError("`act_bits` must be positive")
+ if not (self.group_size == -1 or self.group_size >= 0):
+ raise ValueError("`group_size` must be -1 (per channel) or 0 (per-tensor) or a positive integer")
+ if not (self.act_group_size == -1 or self.act_group_size >= 0):
+ raise ValueError("`act_group_size` must be -1 (per channel) or 0 (per-tensor) or a positive integer")
+ if self.batch_size <= 0:
+ raise ValueError("`batch_size` must be positive")
+ if self.iters < 0:
+ raise ValueError("`iters` must be non-negative")
+ if self.seqlen <= 0:
+ raise ValueError("`seqlen` must be positive")
+ if self.nblocks <= 0:
+ raise ValueError("`nblocks` must be positive")
+ if self.gradient_accumulate_steps <= 0:
+ raise ValueError("`gradient_accumulate_steps` must be positive")
+
+ if (
+ self.act_bits <= 8
+ and (not is_nv_fp(self.act_data_type) or "static_gs" not in self.act_data_type)
+ and not is_mx_fp(self.act_data_type)
+ ):
+ logger.warning(
+ "activation quantization is an experimental feature with limited support and a complex API. "
+ "And please save the quantized model to fake format as real deployment is not supported currently"
+ )
+
+ if is_mx_fp(self.data_type) and self.group_size != 32:
+ logger.warning("dtype mx_fp should only support group_size of 32 in real deployment")
+
+ if is_nv_fp(self.data_type) and (self.group_size != 16):
+ logger.warning("dtype nv_fp should only support group_size of 16 in real deployment")
+
+ if self.nsamples < self.gradient_accumulate_steps * self.batch_size:
+ if self.batch_size > self.nsamples:
+ if self.iters > 0: # GGUF should log this warning, but we don't know the format here
+ logger.warning(
+ f"reset `batch_size` to {self.nsamples} as `nsamples`({self.nsamples})"
+ f" is smaller than batch_size({self.batch_size})"
+ )
+ self.batch_size = self.nsamples
+ if self.gradient_accumulate_steps > self.nsamples // self.batch_size:
+ self.gradient_accumulate_steps = self.nsamples // self.batch_size
+ logger.warning(
+ f"reset `gradient_accumulate_steps` to {self.gradient_accumulate_steps}"
+ f" as nsamples must equal or greater"
+ f" than gradient_accumulate_steps * batch_size"
+ )
+
+ if self.enable_norm_bias_tuning:
+ logger.warning("the `enable_norm_bias_tuning` feature is experimental and currently has limited support.")
+
+ self._dq_check()
+
+ def _check_compatibility(self) -> None:
+ """Checks compatibility of the configurations and model."""
+ # Check gguf and others
+ has_gguf = False
+ if hasattr(self, "formats"):
+ has_besides_gguf = False
+ for format_ in self.formats:
+ if "gguf" in format_:
+ has_gguf = True
+ elif format_ != "fake":
+ has_besides_gguf = True
+ if has_gguf and has_besides_gguf:
+ raise ValueError("Gguf format is not compatible with other formats, please choose only one of them")
+ if has_gguf and self.iters != 0 and self.bits != 3:
+ logger.warning(
+ "`iters=0` is recommended when exporting to GGUF format except for bits 3,"
+ " as we have optimized the RTN method for this case."
+ " We are likely to release new algorithm for certain configurations in the future."
+ )
+
+ # Check group_size 32 for auto_round
+ if (
+ self.data_type == "int"
+ and hasattr(self, "formats")
+ and any(key in fmt for fmt in self.formats for key in ("auto_round", "auto_gptq", "auto_awq"))
+ ):
+ for n, m in self.model.named_modules():
+ if isinstance(m, self.supported_types):
+ if m.weight.shape[0] % 32 != 0 or m.weight.shape[1] % 32 != 0:
+ self.layer_config[n] = {"bits": 16}
+ logger.info(
+ f"{n} will not be quantized due to its shape not being divisible by 32,"
+ " resulting in an exporting issue to autogptq"
+ )
+
+ if (
+ self.seqlen is not None
+ and hasattr(self.model, "config")
+ and hasattr(self.model.config, "max_position_embeddings")
+ ):
+ if self.model.config.max_position_embeddings < self.seqlen:
+ logger.warning(
+ f"Change sequence length to {self.model.config.max_position_embeddings} "
+ "due to the limitation of max_position_embeddings"
+ )
+ self.seqlen = min(self.seqlen, self.model.config.max_position_embeddings)
+
+ if self.seqlen is not None and hasattr(self.tokenizer, "model_max_length"):
+ if self.tokenizer.model_max_length < self.seqlen:
+ logger.warning(
+ f"Change sequence length to {self.tokenizer.model_max_length} "
+ "due to the limitation of model_max_length. "
+ "You can also try to increase the model_max_length to avoid this issue."
+ )
+ self.seqlen = min(self.seqlen, self.tokenizer.model_max_length)
+
+ if self.group_size == 0 and "fp8" not in self.data_type:
+ logger.warning("`group_size==0` is not supported for data_type other than fp8 ")
+
+ def _parse_format_to_list(self, format: str) -> list:
+ """Parses the format string into a list of formats.
+
+ This method checks the requested format(s) against the model's
+ quantization settings and adjusts them if necessary. It ensures that
+ the formats are compatible with the model's data type, bit width,
+ and activation quantization settings.
+
+ Args:
+ format (str): The requested format(s) for quantization, separated by commas.
+
+ Returns:
+ list: A list of validated and updated formats.
+ """
+
+ # Remove duplicates from formats list
+ def remove_duplicates(lst):
+ seen = set()
+ return [x for x in lst if not (x in seen or seen.add(x))]
+
+ formats = format.replace("q*_", f"q{self.bits}_").replace(" ", "").split(",")
+ formats = remove_duplicates(formats) # need the keep origin order
+
+ if isinstance(self.scheme, str) and self.scheme.lower().startswith("gguf"):
+ for i in range(len(formats)):
+ if formats[i] != "fake" and formats[i] != self.scheme.lower().startswith("gguf"):
+ logger.warning(
+ f"reset format {formats[i]} to {self.scheme.lower()} "
+ f"since scheme {self.scheme} can only be exported to format {self.scheme.lower()}"
+ )
+ formats[i] = self.scheme.lower()
+
+ _gguf_args_check(self, formats, model_type=ModelType.TEXT)
+ if self.mllm:
+ _gguf_args_check(self, formats, model_type=ModelType.MMPROJ)
+
+ for format_ in formats:
+ if format_ not in SUPPORTED_FORMATS:
+ logger.error(f"Unsupported format {format_}, please choose from {SUPPORTED_FORMATS}")
+ exit(-1)
+ if self.scale_dtype != torch.float32:
+ only_gguf = True
+ for format_ in formats:
+ if not ("gguf" in format_ or "fake" in format_):
+ only_gguf = False
+ break
+ if len(formats) == 1 and "fake" == formats[0]:
+ only_gguf = False
+ if only_gguf:
+ self.scale_dtype = torch.float32
+ logger.info("change `scale_dtype` to `torch.float32`")
+
+ # Adjust format settings based on compatibility
+ for index in range(len(formats)):
+ format = formats[index]
+ if format == "auto_round":
+ if self.sym and "int" in self.data_type:
+ format = "auto_round:auto_gptq"
+ elif self.bits == 4 and not self.sym and "int" in self.data_type:
+ enable_awq = all(
+ config["bits"] == self.bits or config["bits"] >= 16 for config in self.layer_config.values()
+ )
+ if enable_awq:
+ format = "auto_round:auto_awq"
+ elif is_nv_fp(self.data_type) or is_mx_fp(self.data_type):
+ format = f"auto_round:{self.data_type}"
+ elif is_static_wfp8afp8(self): # staic wfp8afp8
+ format = f"auto_round:{AutoRoundFormat.TORCH_FP8_STATIC.value}"
+ elif self.data_type == "fp" and self.bits == 8 and self.act_bits >= 16: # woq fp8
+ format = "auto_round:fp8"
+ elif self.act_bits < 16:
+ raise ValueError(
+ "AutoRound format does not support exporting "
+ "for the current quantization configuration, "
+ "please change to `fake` format for research purpose"
+ )
+ formats[index] = format
+ elif format == "llm_compressor":
+ from auto_round.export.export_to_llmcompressor import check_compressed_tensors_supported
+
+ if is_nv_fp(self.data_type) or is_mx_fp(self.data_type):
+ check_compressed_tensors_supported()
+ format = format.replace("llm_compressor", f"llm_compressor:{self.data_type}")
+ formats[index] = format
+ elif not is_wfp8afp8(self):
+ logger.error(
+ "Currently, the llm_compressor format only supports MXFP/NVFP/FP8. "
+ "Please change format to fake or auto_round etc."
+ )
+ elif "auto_awq" in format:
+ from auto_round.utils import check_awq_gemm_compatibility
+
+ awq_supported, info = check_awq_gemm_compatibility(
+ self.model, self.bits, self.group_size, self.sym, self.layer_config
+ )
+ if not awq_supported:
+ logger.warning(f"The AutoAWQ format may not be supported due to {info}")
+ else:
+ if (is_nv_fp(self.data_type) or is_mx_fp(self.data_type)) and format != "fake":
+ logger.warning(f"nv_fp and mx_fp dtypes are not supported for export format: {format}")
+
+ formats = remove_duplicates(formats)
+ for i in range(len(formats)):
+ formats[i] = self._check_supported_format(formats[i])
+ formats = remove_duplicates(formats)
+ return formats
+
+ def _check_supported_format(self, format: str) -> bool:
+ """Checks if the specified format is supported.
+
+ This method validates the requested format against the model's bit width,
+ group size, symmetry, and activation quantization settings. It raises an
+ error if the format is incompatible with the current model configuration.
+
+ Args:
+ format (str): The requested format for quantization.
+
+ Returns:
+ bool: True if the format is supported, False otherwise.
+ """
+ if format == "fake":
+ return format
+ format = format.replace("q*_", f"q{self.bits}_")
+
+ # format check for fp8
+ w_fp8 = self.data_type == "fp" and self.bits == 8
+ act_fp8 = self.act_data_type == "fp" and self.act_bits == 8
+ if (w_fp8 or act_fp8) and re.search("^auto_round|^llm_compressor", format) is None:
+ error_msg = (
+ f"is only supported to export auto_round or llm_compressor format," f" but got {format}, please check."
+ )
+ error_msg = ("act_data_type<fp8> " + error_msg) if act_fp8 else error_msg
+ error_msg = ("data_type<fp8> " + error_msg) if w_fp8 else error_msg
+ logger.error(error_msg)
+ sys.exit(-1)
+
+ # Only support to export afp8/nv_fp/mx_fp
+ if self.act_bits <= 8:
+ if not is_standard_fp(self.act_data_type) or self.act_dynamic:
+ if "llm_compressor" in format:
+ if (is_nv_fp(self.act_data_type) and "static_gs" in self.act_data_type) or (
+ is_mx_fp(self.act_data_type)
+ ):
+ return format
+ bits, group_size, sym, act_bits = 8, -1, True, 8
+ assert (
+ self.bits == bits
+ and self.group_size == group_size
+ and self.sym == sym
+ and self.act_bits == act_bits
+ and self.act_dynamic
+ ), (
+ f"Currently only support to export llm_compressor format for sym dynamic quantized"
+ f" W{self.bits}A{self.act_bits} model with group_size={group_size},"
+ f" but got bits={self.bits}, group_size={self.group_size}, sym={self.sym},"
+ f" act_bits={self.act_bits}"
+ )
+ elif "auto_round" in format and (
+ is_mx_fp(self.act_data_type) or (is_nv_fp(format) and "static_gs" in self.act_data_type)
+ ):
+ logger.warning(
+ f"AutoRound supports exporting to format '{format}', "
+ "but loading quantized models in this format is not yet supported. "
+ "It is currently recommended to export to the 'llm_compressor' format."
+ )
+ elif format != "fake":
+ logger.warning(
+ "Currently only support to export auto_round format quantized model"
+ " with fp8, mx_fp and nv_fp4 dtype activation for activation quantization."
+ " Change format to fake and save."
+ )
+ format = "fake"
+ else:
+ if not (format == "auto_round" or format == f"auto_round:{AutoRoundFormat.TORCH_FP8_STATIC.value}"):
+ logger.warning(
+ f"Currently only support to export auto_round or fake format for static W{self.bits}AFP8 model,"
+ f" change format {format} to auto_round"
+ )
+ format = "auto_round"
+ if self.act_group_size != 0 and not self.act_dynamic and format == "auto_round:fp8":
+ logger.warning(
+ f"Please note that quantize activation with act_group_size={self.act_group_size}"
+ " may result in failure to export or import normally."
+ )
+ if re.search(r"q\d_k", format) and not self.data_type.endswith("_dq"):
+ logger.error(
+ f"datatype<{self.data_type}> not support to export {format} format."
+ " Please change export format or `data_type`."
+ )
+ sys.exit(-1)
+
+ return format
+
+ def quantize_and_save(
+ self, output_dir: str = "tmp_autoround", format: str = "auto_round", inplace: bool = True, **kwargs
+ ) -> tuple[torch.nn.Module, dict[str, Any]]:
+ """Quantizes the model and saves it in the specified format(s).
+
+ This function checks the validity of the requested format(s), quantizes
+ the model accordingly, and saves it to the specified output directory.
+ If multiple formats are provided, the model is saved separately for each format.
+
+ Args:
+ output_dir (str, optional): The directory where the quantized model
+ will be saved. Defaults to "tmp_autoround".
+ format (str, optional): The quantization format(s) to use, separated
+ by commas if multiple. Defaults to "auto_round".
+ inplace (bool, optional): Whether to modify the model in place if only
+ one format is used. Defaults to True.
+ **kwargs: Additional arguments for the quantization and saving process.
+
+ Returns:
+ model: A qdq model or packed model based on the configurations
+ folders: The folder paths where the quantized models are saved.
+
+ Raises:
+ ValueError: If an unsupported format is specified.
+ """
+ # Validate and process the specified formats
+ self.orig_output_dir = output_dir
+
+ # check and update the format based on the current configuration
+ format_list = self._parse_format_to_list(format)
+ self.formats = format_list
+
+ # If multiple formats are specified, enforce inplace=False
+ if len(format_list) > 1:
+ inplace = False
+ self.inplace = kwargs.get("inplace", inplace)
+ kwargs.pop("inplace", None)
+
+ # Perform model quantization
+ if self.static_kv_dtype is not None:
+ from auto_round.experimental.kv_cache import kvcache_quant_context
+
+ with kvcache_quant_context(self.model, static_kv_dtype=self.static_kv_dtype):
+ model, _ = self.quantize()
+ else:
+ model, _ = self.quantize()
+ # Save the quantized model in the specified format_list
+ folders = []
+ for format in format_list:
+ if "gptq" in format and not self.sym:
+ logger.warning(
+ "The asymmetrical kernel of the GPTQ format may result in a noticeable accuracy drop,"
+ " particularly for 2-bit quantization and smaller models."
+ " We recommend exporting to either the AutoAWQ format ( only 4 bits) or "
+ "the AutoRound format(2/3/4/8 bits)."
+ )
+ save_folder = self._get_save_folder_name(format)
+ self.save_quantized(save_folder, format=format, inplace=inplace, **kwargs)
+
+ folders.append(save_folder)
+
+ return model, folders
+
+ def _get_save_folder_name(self, format_str: str) -> str:
+ """Generates the save folder name based on the provided format string.
+
+ If there are multiple formats to handle, the function creates a subfolder
+ named after the format string with special characters replaced. If there's
+ only one format, it returns the original output directory directly.
+
+ Args:
+ format_str (str): The format identifier (e.g., 'gguf:q2_k_s').
+
+ Returns:
+ str: The path to the folder where results should be saved.
+ """
+ # Replace special characters to make the folder name filesystem-safe
+ sanitized_format = format_str.replace(":", "-").replace("_", "-")
+
+ # Use a subfolder only if there are multiple formats
+ if len(self.formats) > 1:
+ return os.path.join(self.orig_output_dir, sanitized_format)
+
+ return self.orig_output_dir
+
+ @torch.inference_mode()
+ def _quantize_embedding_layer(self):
+ """Quantizes embedding layers in the model according to the configuration.
+
+ This method iterates through all modules in the model, identifies embedding
+ layers specified in `self.layer_config`, and applies the appropriate quantization
+ function based on bit precision, grouping strategy, and dtype.
+
+ Returns:
+ bool: True if the quantization process completes without critical errors.
+ """
+ is_quantized = False
+ for name, module in self.model.named_modules():
+ # Skip non-Embedding modules or layers not in config
+ if not isinstance(module, torch.nn.Embedding) or name not in self.layer_config:
+ continue
+
+ config = self.layer_config[name]
+
+ # Skip layers that are not marked for quantization
+ if not check_to_quantized(config):
+ continue
+ is_quantized = True
+ config["scale_dtype"] = self.scale_dtype
+ dtype = config["data_type"]
+
+ # Determine quantization function key with symmetry/asymmetry
+ if dtype not in QUANT_FUNC_WITH_DTYPE:
+ dtype = f"{dtype}_{'sym' if config['sym'] else 'asym'}"
+
+ # Optionally use optimized rounding (RTN) variant
+ if not self.disable_opt_rtn and f"rtn_{dtype}" in QUANT_FUNC_WITH_DTYPE:
+ dtype = f"rtn_{dtype}"
+
+ quant_func = QUANT_FUNC_WITH_DTYPE[dtype]
+
+ # Attempt quantization on GPU, fall back to CPU if OOM
+ try:
+ weight, scale, zp = quant_func(
+ module.weight.to(self.device),
+ **{k: config[k] for k in ["bits", "group_size", "super_bits", "super_group_size", "scale_dtype"]},
+ )
+ except RuntimeError as e:
+ cuda_error_msg = traceback.format_exc()
+ try:
+ logger.error(cuda_error_msg)
+ logger.warning("falling back to CPU")
+ weight, scale, zp = quant_func(
+ module.weight.to("cpu"),
+ **{
+ k: config[k]
+ for k in ["bits", "group_size", "super_bits", "super_group_size", "scale_dtype"]
+ },
+ )
+ except Exception as e:
+ raise
+
+ # Overwrite the module's weights with the quantized version
+ module.weight.data.copy_(weight.cpu())
+
+ # Attach scale and zero point (zp) to the module
+ for param_name, value in zip(["scale", "zp"], [scale, zp]):
+ if isinstance(value, dict):
+ for k, v in value.items():
+ setattr(module, k if k == "scale" else f"w_{k}", v.cpu())
+ elif isinstance(value, torch.Tensor):
+ setattr(module, param_name, value.cpu())
+ else:
+ setattr(module, param_name, value)
+
+ # Update config
+ self.layer_config.setdefault(name, {}).update(config)
+
+ # Release memory
+ clear_memory()
+
+ return is_quantized
+
+ def _quant_rtn_with_imatrix(self, all_to_quantized_module_names: list[str]) -> None:
+ """Performs RTN quantization using input activation statistics (imatrix).
+
+ This method accumulates per-channel second-moment activation statistics (imatrix)
+ via forward hooks and uses them to perform RTN quantization. If CUDA memory runs out,
+ it falls back to CPU-based blockwise quantization.
+
+ Args:
+ all_to_quantized_module_names (list[str]):
+ A list of module names (e.g., 'model.layers.0.self_attn.q_proj') to be quantized.
+
+ Returns:
+ None
+ """
+ logger.info("start to compute imatrix for GGUF quantization")
+
+ # Load dataset
+ from auto_round.calib_dataset import get_dataloader
+
+ if _is_fp8_model(self.model):
+ convert_fp8_model_to_16b_model(self.model, self.amp_dtype)
+
+ if isinstance(self.dataset, str):
+ if self.tokenizer is None:
+ raise ValueError("A tokenizer must be set for the model when using a dataset string.")
+ dataset_name = self.dataset.replace(" ", "")
+ self.dataloader = get_dataloader(
+ self.tokenizer, self.seqlen, dataset_name, self.seed, self.batch_size, self.nsamples
+ )
+ else:
+ self.dataloader = self.dataset
+
+ model = self.model
+
+ # Dispatch multi-GPU model if necessary
+ if hasattr(model, "hf_device_map") and len(model.hf_device_map) > 1:
+ dispatch_model(model, model.hf_device_map)
+
+ def register_act_hook(model):
+ """Registers hooks to accumulate activation squared norms into `imatrix`."""
+
+ def get_imatrix_hook(module, input, output):
+ input = input[0] if isinstance(input, (tuple, list)) else input
+ flattened = input.reshape(-1, input.shape[-1]).to(torch.float32)
+ squared = torch.sum(flattened**2, dim=0).to(torch.float32)
+
+ if not hasattr(module, "imatrix"):
+ module.imatrix = squared
+ module.imatrix_cnt = input.shape[0]
+ else:
+ module.imatrix += squared.to(module.imatrix.device)
+ module.imatrix_cnt += input.shape[0]
+
+ hook_handles = []
+ for name, module in model.named_modules():
+ if isinstance(module, self.supported_types) and check_to_quantized(module):
+ hook = module.register_forward_hook(get_imatrix_hook)
+ hook_handles.append(hook)
+ return hook_handles
+
+ hooks = register_act_hook(model)
+
+ try:
+ # Move model to target device
+ if hasattr(self.model, "hf_device_map") and len(self.model.hf_device_map) > 1:
+ dispatch_model(self.model, self.model.hf_device_map)
+ else:
+ model = model.to(self.device)
+ cnt = 0
+
+ # Run forward pass to accumulate imatrix
+ for data in self.dataloader:
+ cnt += data["input_ids"].shape[0]
+ data = to_device(data, self.device)
+ model(**data)
+ if cnt >= self.nsamples:
+ break
+
+ # Remove hooks after data collection
+ for hook in hooks:
+ hook.remove()
+
+ # Normalize imatrix by count
+ for _, module in model.named_modules():
+ if hasattr(module, "imatrix"):
+ module.imatrix /= module.imatrix_cnt
+ if hasattr(model, "hf_device_map") and len(model.hf_device_map) > 1:
+ import accelerate
+
+ accelerate.hooks.remove_hook_from_submodules(model)
+ # Perform quantization using RTN
+ pbar = tqdm(all_to_quantized_module_names)
+ block_names_cnt = len(flatten_list(get_block_names(self.model, True)))
+ clear_mem_freq = len(all_to_quantized_module_names) // block_names_cnt
+ if clear_mem_freq == 0:
+ clear_mem_freq = 1
+ cnt = 1
+ for name in pbar:
+ pbar.set_description(f"Quantizing {name}")
+ self._quantize_layer_via_rtn(name)
+ if cnt % clear_mem_freq == 0:
+ clear_memory()
+ cnt = 1
+ cnt += 1
+ except RuntimeError as e:
+ try:
+ if hasattr(model, "hf_device_map") and len(model.hf_device_map) > 1:
+ import accelerate
+
+ accelerate.hooks.remove_hook_from_submodules(model)
+ # Fallback: out-of-memory → try CPU blockwise quantization
+ logger.warning("Out of VRAM, falling back to blockwise quantization. Accuracy may degrade.")
+ model = model.to("cpu")
+ clear_memory()
+ self._quantize_via_rtn_blockwise(all_to_quantized_module_names)
+ except RuntimeError as e:
+ cuda_error_msg = traceback.format_exc()
+ try:
+ logger.error(cuda_error_msg)
+ # Final fallback: warn and use CPU-only quantization
+ logger.warning(
+ "Fallback to CPU. "
+ "Consider enabling `low_gpu_mem_usage` or using more GPUs via `--device 0,1,2,3`."
+ )
+ model = model.to("cpu")
+ clear_memory()
+ if hasattr(model, "hf_device_map") and len(model.hf_device_map) > 1:
+ import accelerate
+
+ accelerate.hooks.remove_hook_from_submodules(model)
+
+ orig_device = self.device
+ self.device = "cpu"
+ self._quantize_via_rtn_blockwise(all_to_quantized_module_names)
+ self.device = orig_device
+ except Exception as e:
+ raise
+ finally:
+ # Always remove hooks
+ for hook in hooks:
+ hook.remove()
+
+ def _check_need_to_quantize_lm_head_embedding(self) -> bool:
+ """Checks if LM head and embedding layers need quantization for GGUF format.
+
+ This function inspects the current model's formats and determines whether
+ it needs to apply quantization settings to the embedding and LM head layers.
+ The function modifies `self.layer_config` in-place and updates the model modules.
+
+ Returns:
+ bool: True if the LM head needs quantization, otherwise False.
+
+ Raises:
+ NotImplementedError: If multiple non-fake GGUF formats are specified.
+ """
+ gguf_scheme = False
+ if isinstance(self.scheme, str) and "gguf" in self.scheme.lower():
+ gguf_scheme = True
+
+ if not hasattr(self, "formats") and not gguf_scheme:
+ return False
+
+ has_gguf: bool = gguf_scheme or any("gguf" in fmt for fmt in self.formats)
+ if not has_gguf:
+ return False
+ if hasattr(self, "formats"):
+ formats: list[str] = [fmt for fmt in self.formats if "fake" not in fmt]
+ if not (len(formats) == 1 and "gguf" in formats[0]):
+ raise NotImplementedError("Only one GGUF format can be set at a time.")
+ target_format: str = formats[0]
+
+ else:
+ target_format = self.scheme.lower()
+
+ tie_word_embeddings: bool = getattr(getattr(self.model, "config", None), "tie_word_embeddings", True)
+ for name, module in self.model.named_modules():
+ if isinstance(module, torch.nn.Embedding):
+ key: str = "lm_head" if tie_word_embeddings else "embedding"
+ config: dict[str, Any] = GGUF_INNER_CONFIG[GGUF_CONFIG[target_format][key]]
+ self._apply_config_to_layer(name, config, True)
+
+ if not tie_word_embeddings:
+ lm_head_name: str = get_lm_head_name(self.model)
+ config: dict[str, Any] = GGUF_CONFIG[GGUF_CONFIG[target_format]["lm_head"]]
+ check_fixed_by_user = (
+ self.layer_config[lm_head_name].get("fixed_by_user", False)
+ if lm_head_name in self.layer_config
+ else None
+ )
+ self._apply_config_to_layer(lm_head_name, config, check_fixed_by_user=check_fixed_by_user)
+ return True
+
+ return False
+
+ def _apply_config_to_layer(
+ self,
+ layer_name: str,
+ config: dict[str, Any],
+ check_fixed_by_user: bool = False,
+ ) -> None:
+ """Applies GGUF quantization configuration to a given layer.
+
+ Args:
+ layer_name (str): Name of the layer to configure.
+ config (dict[str, Any]): GGUF layer configuration.
+ check_fixed_by_user (bool): If True, preserve user-defined settings.
+ """
+ act_bits: int = 16
+ scale_dtype: Any = self.scale_dtype
+ keys: list[str] = ["bits", "group_size", "super_bits", "super_group_size", "data_type", "sym"]
+
+ self.layer_config[layer_name] = self.layer_config.get(layer_name, {})
+
+ for key in keys:
+ if (
+ key in self.layer_config[layer_name]
+ and check_fixed_by_user
+ # and self.layer_config[layer_name].get("fixed_by_user", False)
+ ):
+ continue
+ self.layer_config[layer_name][key] = config.get(key)
+ setattr(get_module(self.model, layer_name), key, config.get(key))
+
+ self.layer_config[layer_name]["act_bits"] = act_bits
+ self.layer_config[layer_name]["scale_dtype"] = scale_dtype
+ setattr(get_module(self.model, layer_name), "act_bits", act_bits)
+ setattr(get_module(self.model, layer_name), "scale_dtype", scale_dtype)
+
+ def _quantize_layer_via_rtn(self, name: str) -> None:
+ """Quantizes a layer using RTN (Round-To-Nearest) if available.
+
+ This function attempts to quantize a layer by switching its data type to a
+ `rtn_*` version if supported, then wraps and unwraps the module to apply
+ quantization. If GPU memory is insufficient, it falls back to CPU.
+
+ If packing is enabled (`is_packing_immediate`), the function will also export
+ the quantized layer to the appropriate backend format.
+
+ Args:
+ name (str): Name of the layer to quantize.
+
+ Raises:
+ RuntimeError: If quantization fails for reasons unrelated to memory.
+ """
+ m = get_module(self.model, name)
+
+ # if m.__class__.__name__ == "FP8Linear":
+ if _is_fp8_linear(m):
+ m = convert_fp8_layer_to_linear(m, self.amp_dtype)
+ set_module(self.model, name, m)
+
+ # Step 1: Use optimized RTN data type if available
+ if not self.disable_opt_rtn and not m.data_type.startswith("rtn_"):
+ from auto_round.data_type import QUANT_FUNC_WITH_DTYPE
+
+ rtn_dtype = "rtn_" + m.data_type
+ if rtn_dtype in QUANT_FUNC_WITH_DTYPE:
+ m.data_type = rtn_dtype
+ self.layer_config[name]["data_type"] = m.data_type
+
+ # Step 2: Try quantization on GPU first, fall back to CPU if OOM
+ # if only export gguf, using gguf-packing instead of rtn
+ if self.is_packing_immediate and self.iters == 0 and "gguf" in self.formats[0] and not self.disable_opt_rtn:
+ m.scale = None
+ m.zp = None
+ else:
+ try:
+ m.to(self.device)
+ m = WrapperLinear(
+ m,
+ enable_minmax_tuning=False,
+ enable_norm_bias_tuning=False,
+ enable_round_tuning=False,
+ )
+ m = m.unwrapper({})
+ m.to("cpu")
+ except RuntimeError as e:
+ cuda_error_msg = traceback.format_exc()
+ m = m.orig_layer if hasattr(m, "orig_layer") else m
+ try:
+ logger.error(cuda_error_msg)
+ logger.warning("falling back to CPU.")
+ m.to("cpu")
+ m = WrapperLinear(
+ m,
+ enable_minmax_tuning=False,
+ enable_norm_bias_tuning=False,
+ enable_round_tuning=False,
+ )
+ m = m.unwrapper({})
+ except Exception as e:
+ raise
+
+ # Step 3: Optional immediate packing/export
+ if self.is_packing_immediate:
+ from auto_round.export import PACKING_LAYER_WITH_FORMAT
+
+ if check_to_quantized(m):
+ target_backend = self.formats[0].split(":")[0] if ":" in self.formats[0] else self.formats[0]
+ has_gguf = any("gguf" in fmt for fmt in self.formats)
+
+ if has_gguf:
+ from auto_round.export.export_to_gguf.export import pack_gguf_layer
+
+ output_dir = self._get_save_folder_name(self.formats[0])
+ model_type = ModelType.MMPROJ if self.mllm else ModelType.TEXT
+ pack_gguf_layer(
+ name,
+ self.model,
+ self.formats[0],
+ output_dir,
+ self.layer_config,
+ self.tokenizer,
+ processor=self.processor if hasattr(self, "processor") else None,
+ image_processor=self.image_processor if hasattr(self, "image_processor") else None,
+ model_type=model_type,
+ )
+ else:
+ PACKING_LAYER_WITH_FORMAT[target_backend](name, self.model, self.formats[0], device=self.device)
+
+ # if self.low_gpu_mem_usage:
+ # clear_memory()
+ else:
+ set_module(self.model, name, m)
+
+ @torch.inference_mode()
+ def _quantize_rtn(self) -> tuple[torch.nn.Module, dict[str, Any]]:
+ """Quantize all modules in the model using RTN (Round-To-Nearest) strategy.
+
+ If the target format includes GGUF with `k`, and optimized RTN is enabled,
+ blockwise quantization with input caching and imatrix is used.
+
+ Returns:
+ tuple[nn.Module, Dict[str, Any]]: The quantized model and the layer configuration.
+ """
+ if self.amp and self.model.dtype != self.amp_dtype:
+ self.model.to(self.amp_dtype)
+
+ all_to_quantized_module_names: list[str] = [n for n, m in self.model.named_modules() if check_to_quantized(m)]
+
+ if is_nv_fp(self.data_type):
+ from auto_round.data_type.nvfp import calculate_gparam
+ from auto_round.data_type.utils import update_fused_layer_global_scales
+
+ pbar = tqdm(all_to_quantized_module_names)
+ for name in pbar:
+ pbar.set_description(f"Calculate weight global scale: {name}")
+ m = get_module(self.model, name)
+ weight_global_scale = calculate_gparam(m.weight, self.group_size)
+ setattr(m, "weight_global_scale", weight_global_scale)
+
+ modules = list(self.model.modules())
+ for module in tqdm(modules, desc="Update weight global scale for fuse module"):
+ update_fused_layer_global_scales(module)
+
+ has_gguf_k = any("gguf" in fmt and "k" in fmt for fmt in getattr(self, "formats", []))
+
+ self._quantize_embedding_layer()
+
+ self.model.to("cpu")
+ if has_gguf_k and not self.disable_opt_rtn:
+ self._quant_rtn_with_imatrix(all_to_quantized_module_names)
+ elif self.act_bits <= 8 and check_need_act_calibration(
+ self.act_dynamic, self.act_data_type, self.act_bits
+ ): # TODO, mixed datatype has bug
+ hook_handles = self._register_act_max_hook(self.model)
+ try:
+ self._quantize_via_rtn_blockwise(all_to_quantized_module_names)
+ except RuntimeError as e:
+ logger.warning("Fallback to CPU. Consider using more GPUs via `--device 0,1,2,3`.")
+ self.model = self.model.to("cpu")
+ clear_memory()
+ if hasattr(self.model, "hf_device_map") and len(self.model.hf_device_map) > 1:
+ import accelerate
+
+ accelerate.hooks.remove_hook_from_submodules(self.model)
+ orig_device = self.device
+ self.device = "cpu"
+ self._quantize_via_rtn_blockwise(all_to_quantized_module_names)
+ self.device = orig_device
+ for handle in hook_handles:
+ handle.remove()
+ else:
+ block_names_cnt = len(flatten_list(get_block_names(self.model, True)))
+ clear_mem_freq = len(all_to_quantized_module_names) // block_names_cnt
+ if clear_mem_freq == 0:
+ clear_mem_freq = 1
+ pbar = tqdm(all_to_quantized_module_names)
+ cnt = 1
+ for name in pbar:
+ pbar.set_description(f"Quantizing {name}")
+ self._quantize_layer_via_rtn(name)
+ if cnt % clear_mem_freq == 0:
+ clear_memory()
+ cnt = 1
+ cnt += 1
+ # Convert remaining fp8
+ if _is_fp8_model(self.model):
+ convert_fp8_model_to_16b_model(self.model, self.amp_dtype)
+ self.quantized = True
+ return self.model, self.layer_config
+
+ def _quantize_via_rtn_blockwise(self, all_to_quantized_module_names: list[str]) -> None:
+ """Quantize model layers block by block using cached inputs and imatrix.
+
+ Args:
+ all_to_quantized_module_names (list[str]): Names of layers to be quantized.
+ """
+ all_to_quantized_module_names = list(set(all_to_quantized_module_names))
+
+ all_blocks = self.quant_block_list if self.quant_block_list else get_block_names(self.model)
+ if not all_blocks:
+ raise ValueError("Could not find any blocks. Check the model or quant_block_list.")
+
+ all_first_block_names = [block[0] for block in all_blocks]
+ if self.act_bits < 16 and not self.act_dynamic:
+ layer_names = self._get_quantized_layer_names_outside_blocks()
+ if len(layer_names) > 0:
+ logger.warning(
+ "quantize layers outside blocks for static activation quantizaiton"
+ " will significantly increase calibration time"
+ )
+ all_inputs = self.try_cache_inter_data_gpucpu(all_first_block_names, self.nsamples, layer_names)
+ else:
+ all_inputs = self.cache_inter_data(all_first_block_names, self.nsamples)
+
+ # Clear hooks for multi-GPU setups
+ if hasattr(self.model, "hf_device_map") and len(self.model.hf_device_map) > 1:
+ accelerate.hooks.remove_hook_from_submodules(self.model)
+
+ pbar = tqdm(range(sum(len(block) for block in all_blocks)))
+
+ for block_names in all_blocks:
+ first_block = block_names[0]
+ inputs = all_inputs.pop(first_block)
+ input_keys = [k for k in inputs if k.startswith("hidden_state")]
+ if len(input_keys) != 1:
+ raise RuntimeError(
+ "hidden_states arg mismatch. Please file an issue at https://github.com/intel/auto-round/issues"
+ )
+ inputs["input_ids"] = inputs.pop(input_keys[0])
+
+ clear_memory(self.inputs)
+
+ total_samples = len(inputs["input_ids"])
+ if total_samples < self.batch_size:
+ self.batch_size = total_samples
+ logger.warning(f"Forcing batch size to {total_samples}")
+
+ input_ids = to_device(inputs.pop("input_ids"), self.cache_device)
+ input_others = to_device(inputs, self.cache_device)
+
+ tmp_dtype = self.amp_dtype if self.amp else torch.float32
+ input_ids = [id_.to(tmp_dtype) for id_ in input_ids]
+
+ for key, val in input_others.items():
+ if isinstance(val, torch.Tensor) and val.dtype in (torch.float16, torch.bfloat16):
+ input_others[key] = val.to(tmp_dtype)
+ elif isinstance(val, list):
+ input_others[key] = [to_dtype(v, tmp_dtype) for v in val]
+
+ for block_name in block_names:
+ pbar.set_description(f"Quantizing {block_name}")
+ block = get_module(self.model, block_name)
+ block = block.to(self.device)
+ if _is_fp8_model(self.model):
+ convert_fp8_model_to_16b_model(block, dtype=self.amp_dtype)
+
+ if self.device_map == "auto":
+ self._set_auto_device_map_in_block(block, input_ids)
+
+ # Dispatch model if needed
+ if self.device_map is not None:
+ from accelerate.hooks import AlignDevicesHook, add_hook_to_module
+
+ for _, m in block.named_modules():
+ if len(list(m.children())) != 0 or not hasattr(m, "tuning_device"):
+ continue
+ hook = AlignDevicesHook(m.tuning_device, io_same_device=True)
+ add_hook_to_module(m, hook, True)
+ else:
+ block = block.to(self.device)
+ input_ids = self._get_block_outputs(
+ block,
+ input_ids,
+ input_others,
+ self.batch_size * self.infer_bs_coeff,
+ self.device,
+ self.cache_device,
+ )
+ if self.device_map is not None:
+ accelerate.hooks.remove_hook_from_submodules(block)
+
+ if (
+ is_nv_fp(self.act_data_type) and any("nv_fp" in format_ for format_ in self.formats)
+ ) or is_static_wfp8afp8(self):
+ # enable moe experts act_max automatic generation for Linear
+ set_amax_for_all_moe_layers(block, attr_name="act_max")
+ # Normalize imatrix and quantize layers
+ for _, m in block.named_modules():
+ if hasattr(m, "imatrix"):
+ m.imatrix /= m.imatrix_cnt
+ if hasattr(m, "tmp_name") and m.tmp_name in all_to_quantized_module_names:
+ self._quantize_layer_via_rtn(m.tmp_name)
+ all_to_quantized_module_names.remove(m.tmp_name)
+
+ mv_module_from_gpu(block, self.low_cpu_mem_usage)
+ pbar.update(1)
+
+ pbar.close()
+ cnt = 1
+ block_names_cnt = len(flatten_list(get_block_names(self.model, True)))
+ clear_mem_freq = len(all_to_quantized_module_names) // block_names_cnt
+ if clear_mem_freq == 0:
+ clear_mem_freq = 1
+ # Process remaining layers not in blocks
+ for name in all_to_quantized_module_names:
+ self._quantize_layer_via_rtn(name)
+ if cnt % clear_mem_freq == 0:
+ clear_memory()
+ cnt = 1
+ cnt += 1
+
+ def quantize(self) -> tuple[torch.nn.Module, dict[str, Any]]:
+ """Quantize the model and return the quantized model along with layer configurations.The entry of AutoRound.
+ Returns:
+ The quantized model and layer configurations.
+ """
+ for n, m in self.model.named_modules():
+ m.tmp_name = n
+ self._check_compatibility()
+ formats = self.formats if hasattr(self, "formats") else None
+ # It is best to modify the model structure in the quantize function and check the format,
+ # because it may cause the gguf format to not be exported normally.
+ self.model = _handle_moe_model(self.model, formats=formats)
+ self.has_qlayer_outside_block = self._set_layerwise_config(self.layer_config)
+ if not hasattr(self, "formats"):
+ logger.warning("this API is deprecated, please use `quantize_and_save` instead")
+ else:
+ only_gguf = True
+ for format_ in self.formats:
+ if not ("gguf" in format_ or "fake" in format_):
+ only_gguf = False
+ break
+ if len(self.formats) == 1 and self.formats[0] == "fake":
+ only_gguf = False
+ if only_gguf:
+ self.layer_config, gguf_format_config = get_layer_config_by_gguf_format(
+ self.layer_config, self.formats, self.model, model_type=ModelType.TEXT
+ )
+ if self.mllm:
+ self.layer_config, gguf_format_config = get_layer_config_by_gguf_format(
+ self.layer_config, self.formats, self.model, model_type=ModelType.MMPROJ
+ )
+ # Determine if immediate packing is required
+ formats = self.formats
+ if (
+ len(formats) == 1
+ and (
+ "awq" in formats[0]
+ or "gptq" in formats[0]
+ or "auto_round" in formats[0]
+ or "gguf" in formats[0]
+ or "llm_compressor" in formats[0]
+ )
+ and self.inplace
+ ):
+ self.is_packing_immediate = True
+ if self.iters == 0:
+ return self._quantize_rtn()
+
+ if bool(self.quant_block_list):
+ all_blocks = self.quant_block_list
+ else:
+ all_blocks = get_block_names(self.model)
+
+ if len(all_blocks) == 0:
+ logger.warning("could not find blocks, exit with original model")
+ return self.model, self.layer_config
+
+ if self.amp and self.model.dtype != self.amp_dtype:
+ self.model = self.model.to(self.amp_dtype)
+
+ layer_names = self._get_quantized_layer_names_outside_blocks()
+ self.start_time = time.time()
+ all_first_block_names = [block[0] for block in all_blocks]
+ if len(layer_names) > 0:
+ logger.info(
+ "Starting to cache block inputs. This may be slow due to external block layers: %s", layer_names
+ )
+ else:
+ logger.info("start to cache block inputs")
+ all_inputs = self.try_cache_inter_data_gpucpu(all_first_block_names, self.nsamples, layer_names=layer_names)
+ is_quantized_embedding = self._quantize_embedding_layer()
+ all_q_inputs = None
+ if is_quantized_embedding:
+ all_inputs = copy.deepcopy(self.inputs)
+ clear_memory(self.inputs)
+ all_q_inputs = self.try_cache_inter_data_gpucpu(
+ all_first_block_names, self.nsamples, layer_names=layer_names
+ )
+ self.model = mv_module_from_gpu(self.model, self.low_cpu_mem_usage)
+ clear_memory()
+ if hasattr(self.model, "hf_device_map") and len(self.model.hf_device_map) > 1:
+ accelerate.hooks.remove_hook_from_submodules(self.model) # self.model.hf_device_map has not been changed
+ self.model = mv_module_from_gpu(self.model, self.low_cpu_mem_usage)
+ logger.info("caching done")
+ if len(all_blocks) > 1:
+ pbar = tqdm(range(0, sum([len(i) for i in all_blocks]), self.nblocks))
+ else:
+ pbar = None # move the alg warning outside pbar
+
+ for block_names in all_blocks:
+ inputs = all_inputs[block_names[0]]
+ all_inputs.pop(block_names[0])
+ q_inputs = None
+ if all_q_inputs is not None:
+ q_inputs = all_q_inputs[block_names[0]]
+ all_q_inputs.pop(block_names[0])
+ keys = inputs.keys()
+ input_id_str = [key for key in keys if key.startswith("hidden_state")]
+ if len(input_id_str) != 1:
+ raise RuntimeError(
+ "hidden_states arg mismatch error,"
+ "please raise an issue in https://github.com/intel/auto-round/issues"
+ )
+ inputs["input_ids"] = inputs.pop(input_id_str[0], None)
+ if q_inputs is not None:
+ q_inputs["input_ids"] = q_inputs.pop(input_id_str[0], None)
+
+ clear_memory(self.inputs)
+
+ if "input_ids" in inputs.keys():
+ total_samples = len(inputs["input_ids"])
+ if total_samples < self.batch_size:
+ self.batch_size = total_samples
+ logger.warning(f"force the train batch size to {total_samples}")
+
+ self._quantize_blocks(
+ self.model,
+ inputs,
+ block_names,
+ q_input=q_inputs["input_ids"] if q_inputs is not None else None,
+ nblocks=self.nblocks,
+ device=self.device,
+ pbar=pbar,
+ )
+ if self.is_packing_immediate and len(self.formats) != 1:
+ raise ValueError(
+ f"Expected exactly one packing format when 'is_packing_immediate' is True, "
+ f"but got {len(self.formats)} formats."
+ )
+
+ self._quantize_layers(layer_names, all_inputs) ##TODO pack layer immediately
+
+ if _is_fp8_model(self.model):
+ for n, m in self.model.named_modules():
+ if _is_fp8_linear(m):
+ new_layer = convert_fp8_layer_to_linear(m, self.amp_dtype).to("cpu")
+ set_module(self.model, n, new_layer)
+
+ end_time = time.time()
+ cost_time = end_time - self.start_time
+ logger.info(f"quantization tuning time {cost_time}")
+
+ ## dump a summary
+ quantized_layers = []
+ unquantized_layers = []
+ for n, m in self.model.named_modules():
+ if isinstance(m, tuple(self.supported_types)):
+ if check_to_quantized(m):
+ quantized_layers.append(n)
+ else:
+ unquantized_layers.append(n)
+ elif hasattr(m, "scales") or hasattr(m, "scale"): ##packing_immediately
+ quantized_layers.append(n)
+ summary_info = (
+ f"Summary: quantized {len(quantized_layers)}/{len(quantized_layers) + len(unquantized_layers)} in the model"
+ )
+ if len(unquantized_layers) > 0:
+ summary_info += f", {unquantized_layers} have not been quantized"
+ logger.info(summary_info)
+
+ self.quantized = True
+ return self.model, self.layer_config
+
+ def _quantize_layers(self, layer_names: list, layer_inputs: dict) -> None:
+ """Quantizes specified layers based on inputs and configuration.
+
+ Args:
+ layer_names (list): list of layer names to quantize.
+ layer_inputs (dict): Dictionary mapping layer names to input data.
+
+ Returns:
+ None
+ """
+ ##TODO currently we take all the layers outside blocks as post block layers which is not optimal
+ ## if there is no input for layer, we use rtn
+
+ for layer_name in copy.deepcopy(layer_names):
+ if layer_name not in layer_inputs:
+ logger.info(f"using rtn to quantize {layer_name}")
+ from auto_round.data_type import QUANT_FUNC_WITH_DTYPE
+
+ layer = get_module(self.model, layer_name)
+ if _is_fp8_model(self.model):
+ new_layer = convert_fp8_layer_to_linear(layer, self.amp_dtype).to(self.device)
+ set_module(self.model, layer_name, new_layer)
+ layer = new_layer
+
+ if not self.disable_opt_rtn and "rtn_" + layer.data_type in QUANT_FUNC_WITH_DTYPE:
+ layer.data_type = "rtn_" + layer.data_type
+ logger.info("using optimized rtn method for quantizing %s", layer_name)
+ self.layer_config[layer_name]["data_type"] = layer.data_type
+ wrapper_layer = WrapperLinear(
+ layer,
+ enable_round_tuning=False,
+ enable_minmax_tuning=False,
+ enable_norm_bias_tuning=False,
+ device=self.device,
+ )
+ new_layer = wrapper_layer.unwrapper({})
+ set_module(self.model, layer_name, new_layer)
+ layer.cpu()
+ layer_names.remove(layer_name)
+ if len(layer_names) == 0:
+ return
+ q_layer_inputs = None
+ enable_quanted_input = self.enable_quanted_input
+ has_gguf = False
+ if hasattr(self, "formats"):
+ has_gguf = any("gguf" in format_ for format_ in self.formats)
+ if has_gguf and self.is_packing_immediate:
+ enable_quanted_input = False
+
+ if hasattr(self.model, "hf_device_map") and len(self.model.hf_device_map) > 1 and enable_quanted_input:
+ dispatch_model(self.model, self.model.hf_device_map)
+
+ if enable_quanted_input:
+ logger.info("starting to cache layer inputs for %s, this may be quite slow ", layer_names)
+ q_layer_inputs = self.try_cache_inter_data_gpucpu([], self.nsamples, layer_names=layer_names)
+ if hasattr(self.model, "hf_device_map") and len(self.model.hf_device_map) > 1:
+ accelerate.hooks.remove_hook_from_submodules(
+ self.model
+ ) ##self.model.hf_device_map has not been changed
+
+ self.model = mv_module_from_gpu(self.model, self.low_cpu_mem_usage)
+ clear_memory()
+ if self.enable_torch_compile:
+ quant_layer = compile_func(self._quantize_layer, self.device)
+ else:
+ quant_layer = self._quantize_layer
+ for layer_name in layer_names:
+ layer_input = layer_inputs[layer_name]
+ layer_input = to_device(layer_input, self.cache_device)
+ q_layer_input = q_layer_inputs.get(layer_name, None) if q_layer_inputs is not None else None
+ q_layer_input = to_device(q_layer_input, self.cache_device)
+ quant_layer(layer_name, layer_input, q_layer_input, device=self.device)
+ del layer_input
+ clear_memory(q_layer_input)
+
+ def _set_layerwise_config(self, layer_config: dict) -> bool:
+ """
+ Sets the layer-wise configuration based on the provided `layer_config`.
+ By default, only quantize layers in blocks.
+
+ Args:
+ layer_config (dict): The configuration dictionary for each layer containing various configuration options.
+
+ Returns:
+ bool: Returns True if there are quantized layers outside the blocks (e.g., lm-head),
+ otherwise returns False.
+ """
+ # Get the names of layers in quantization blocks
+ supported_types = self.supported_types
+ layers_in_blocks = get_layer_names_in_block(
+ self.model, supported_types, self.quant_block_list, self.inner_supported_types
+ )
+ ##process regex in layer_config
+ all_supported_layer_names = []
+ # List of configuration keys
+ keys = get_quant_keys()
+
+ for n, m in self.model.named_modules():
+ # Delete previous configuration to avoid conflicts with prior tuning
+ for key in keys:
+ if hasattr(m, key):
+ delattr(m, key)
+
+ if not isinstance(m, supported_types) and m.__class__.__name__ not in self.inner_supported_types:
+ continue
+ all_supported_layer_names.append(n)
+
+ names_in_layer_config = list(layer_config.keys())
+ for name in names_in_layer_config:
+ if name in all_supported_layer_names:
+ continue
+ matched_names = []
+ for layer_name in all_supported_layer_names:
+ if re.search(re.compile(name), layer_name) is not None:
+ matched_names.append(layer_name)
+ if len(matched_names) > 0:
+ val = layer_config[name]
+ layer_config.pop(name)
+ for match_name in matched_names:
+ layer_config[match_name] = val
+ else:
+ tmp_m = get_module(self.model, name)
+ if not isinstance(tmp_m, torch.nn.Embedding): # TODO not good code style
+ raise ValueError(f"key {name} in layer_config is invalid, please have a double check")
+
+ has_qlayer_outside_block = False # Flag to track if there are quantized layers outside blocks (e.g., lm-head)
+
+ # Iterate through all modules in the model
+ is_gguf = hasattr(self, "formats") and any("gguf" in format_ for format_ in self.formats)
+ for n, m in self.model.named_modules():
+ # Skip unsupported types
+ if not isinstance(m, supported_types) and m.__class__.__name__ not in self.inner_supported_types:
+ if n in self.layer_config:
+ if not isinstance(m, torch.nn.Embedding):
+ logger.warning(f"{n} is not supported, layer_config {n}: {layer_config[n]} will be ignored.")
+ self.layer_config.pop(n)
+ continue
+ if not is_gguf:
+ if not check_to_quantized(layer_config[n]):
+ self.layer_config.pop(n)
+ continue
+ else:
+ continue
+
+ # If the layer is not in the config and is part of a quantization block, use default configuration
+ if n not in layer_config.keys() and n in layers_in_blocks:
+ layer_config[n] = {}
+ for key in keys:
+ layer_config[n][key] = getattr(self, key)
+
+ # If the layer is partially configured, fill in missing values
+ elif n in layer_config.keys():
+ if "data_type" in layer_config[n] and "bits" not in layer_config[n]:
+ tmp_bits = infer_bits_by_data_type(layer_config[n]["data_type"])
+ if tmp_bits is not None and tmp_bits != self.bits:
+ logger.warning(
+ f"'data_type' do not match the specified 'bits' setting for {n}."
+ f" Resetting 'bits' to {tmp_bits}."
+ )
+ layer_config[n]["bits"] = tmp_bits
+ if "act_data_type" in layer_config[n] and "act_bits" not in layer_config[n]:
+ tmp_bits = infer_bits_by_data_type(layer_config[n]["act_data_type"])
+ if tmp_bits is not None and tmp_bits != self.act_bits:
+ logger.warning(
+ f"'act_data_type' do not match the specified 'act_bits' setting for {n}."
+ f" Resetting 'act_bits' to {tmp_bits}."
+ )
+ layer_config[n]["act_bits"] = tmp_bits
+
+ for key in keys:
+ if key not in layer_config[n].keys():
+ layer_config[n][key] = getattr(self, key)
+ layer_config[n]["fixed_by_user"] = True
+
+ # If the layer is not in the config and not part of a quantization block,
+ # use default configuration and set specific values
+ else:
+ layer_config[n] = {}
+ for key in keys:
+ layer_config[n][key] = getattr(self, key)
+ layer_config[n]["bits"] = 16
+ layer_config[n]["act_bits"] = 16
+
+ if n in layers_in_blocks:
+ layer_config[n]["in_blocks"] = True
+ else:
+ layer_config[n]["in_blocks"] = False
+
+ # If the layer is outside a block and requires quantization, mark it as a quantized layer outside the block
+ if (
+ n not in layers_in_blocks
+ and check_to_quantized(layer_config[n])
+ and not isinstance(m, torch.nn.Embedding)
+ ):
+ has_qlayer_outside_block = True
+
+ in_features, out_features = get_layer_features(m)
+ if in_features <= layer_config[n]["group_size"]:
+ layer_config[n]["group_size"] = -1
+
+ # Apply the configuration to the corresponding layer in the model
+ for key in keys:
+ setattr(m, key, layer_config[n][key])
+ need_to_quantize_lm_head = self._check_need_to_quantize_lm_head_embedding()
+ if need_to_quantize_lm_head:
+ has_qlayer_outside_block = True
+
+ # Return whether there are quantized layers outside the blocks
+ return has_qlayer_outside_block
+
+ @torch.no_grad()
+ def _get_block_outputs(
+ self,
+ block: torch.nn.Module,
+ input_ids: torch.Tensor,
+ input_others: torch.Tensor,
+ bs: int,
+ device: Union[str, torch.device],
+ cache_device: Union[str, torch.device],
+ save_output: bool = True,
+ ):
+ """Compute the output of a given block of the model for a given input.
+
+ Args:
+ block: The block of the model.
+ input_ids: The input tensor containing tokenized input ids.
+ input_others: A dictionary containing additional input data.
+ bs: The batch size for computing the output.
+ device: The device for computation.
+ cache_device: The device for storing the output.
+ batch_dim: The batch dimension of the output tensor.
+
+ Returns:
+ The output tensor of the block.
+ """
+
+ output = []
+ nsamples = len(input_ids)
+ for i in range(0, nsamples, bs):
+ end_index = min(nsamples, i + bs)
+ indices = torch.arange(i, end_index).to(torch.long)
+ tmp_input_ids, tmp_input_others = self._sampling_inputs(
+ input_ids, input_others, indices, self.seqlen, self.batch_dim, share_cache_keys=self.shared_cache_keys
+ )
+ tmp_output = block_forward(block, tmp_input_ids, tmp_input_others, self.amp, self.amp_dtype, device).to(
+ cache_device
+ )
+ if save_output:
+ if self.batch_size == 1:
+ output.append(tmp_output)
+ else:
+ output.extend(list(torch.split(tmp_output, 1, dim=self.batch_dim)))
+ if self.low_gpu_mem_usage:
+ clear_memory()
+
+ return output
+
+ @torch.no_grad()
+ def calib(self, nsamples, bs):
+ """Perform calibration for quantization.
+
+ This method calibrates the model for quantization by processing a specified
+ number of samples from the calibration dataset. It ensures that the data is
+ properly formatted and feeds it to the model. If the number of samples processed
+ is less than the specified number, it logs a warning. If no samples are processed,
+ it logs an error and exits.
+ Args:
+ nsamples (int): The number of samples to use for calibration.
+ bs (int): The number of samples to use for calibration
+ """
+ from auto_round.calib_dataset import get_dataloader
+
+ if isinstance(self.dataset, str):
+ dataset = self.dataset.replace(" ", "") ##remove all whitespaces
+
+ # slow here
+ self.dataloader = get_dataloader(
+ self.tokenizer,
+ self.seqlen,
+ dataset,
+ self.seed,
+ bs,
+ self.nsamples,
+ )
+ else:
+ self.dataloader = self.dataset
+ total_cnt = 0
+
+ # load embed weight if use low_cpu_mem_usage
+ if self.low_cpu_mem_usage:
+ embed_layers = get_layers_before_block(self.model)
+ for n, m in embed_layers:
+ m = m.to(self.device)
+
+ for data in self.dataloader:
+ if data is None:
+ continue
+ if isinstance(data, torch.Tensor):
+ input_ids = data.to(self.model.device)
+ data_new = input_ids
+ elif isinstance(data, str):
+ if self.tokenizer is None:
+ logger.error("please provide tokenizer for string input")
+ exit(-1)
+ data = self.tokenizer(data, truncation=True, max_length=self.seqlen, return_tensors="pt").data
+ data_new = {}
+ for key in data.keys():
+ data_new[key] = data[key].to(self.model.device)
+ input_ids = data_new["input_ids"]
+ elif isinstance(data, tuple) or isinstance(data, list):
+ data_new = to_device(data)
+ input_ids = data_new[0]
+ else:
+ data_new = {}
+ for key in data.keys():
+ data_new[key] = to_device(data[key], self.model.device)
+ if key == "images":
+ data_new[key] = to_dtype(data_new[key], self.model.dtype)
+ input_ids = data_new["input_ids"]
+ if input_ids.shape[-1] < self.seqlen:
+ continue
+ try:
+ if isinstance(data_new, torch.Tensor):
+ self.model(data_new, use_cache=False)
+ elif isinstance(data_new, tuple) or isinstance(data_new, list):
+ self.model(*data_new, use_cache=False)
+ else:
+ self.model(**data_new, use_cache=False)
+ except NotImplementedError:
+ pass
+ except RuntimeError as error:
+ error_msg = str(error)
+ if "The expanded size of the tensor" in str(error_msg) and "must match the existing size" in error_msg:
+ check_seqlen_compatible(self.seqlen, self.tokenizer, self.model)
+ logger.warning(
+ "When quantization encounters tensor shape mismatch error, "
+ "you can try to avoid it with batch_size=1"
+ )
+ raise error
+ except Exception as error:
+ raise error
+ total_cnt += input_ids.shape[0] if len(input_ids.shape) > 1 else 1
+ if total_cnt >= nsamples:
+ break
+ if total_cnt == 0:
+ logger.error(
+ f"no data has been cached, please provide more data with sequence length >={self.seqlen} in the "
+ f"dataset or decease the sequence length"
+ )
+ exit(-1)
+ elif total_cnt < nsamples:
+ logger.warning(
+ f"An insufficient number of samples likely reduces the accuracy of the quantized model. "
+ f"Target samples count is {nsamples}, while valid samples count is {total_cnt}"
+ )
+
+ # clean embed weight to save memory
+ if self.low_cpu_mem_usage:
+ for n, m in embed_layers:
+ m = m.to("meta")
+
+ @torch.no_grad()
+ def try_cache_inter_data_gpucpu(self, block_names, nsamples, layer_names=None, last_cache_name=None):
+ """Attempts to cache intermediate data on GPU, if failed, then using CPU.
+
+ Args:
+ block_names (list): List of block names to cache data for.
+ nsamples (int): Number of samples to use for caching.
+ layer_names (list, optional): List of layer names to cache data for. Defaults to [].
+ last_cache_name (str, optional): Name of the last cache. Defaults to None.
+
+ Returns:
+ all_inputs: Cached intermediate data.
+
+ Raises:
+ Exception: If caching on GPU fails, switches to CPU and caches there.
+ """
+ if _is_fp8_model(self.model):
+ layer_names = []
+ if layer_names is None:
+ layer_names = []
+
+ if self.low_gpu_mem_usage or (
+ len(block_names) == 1
+ and len(layer_names) == 0
+ and not self.has_qlayer_outside_block
+ and (last_cache_name is None or last_cache_name in block_names)
+ ):
+ # low_gpu_mem_usage or calibrate only the embedding layer, which is also very fast on CPU
+ all_inputs = self.cache_inter_data(block_names, nsamples, layer_names=[], last_cache_name=last_cache_name)
+ else:
+ try:
+ if not self.model.device.type == "meta":
+ if hasattr(self.model, "hf_device_map") and len(self.model.hf_device_map) > 1:
+ self.model = dispatch_model(self.model, device_map=self.model.hf_device_map)
+ else:
+ # Change this if new device is supported
+ if str(self.model.device) == "cpu" and (
+ self.device.startswith("xpu") or self.device.startswith("cuda")
+ ):
+ max_memory = get_max_vram() # TODO model is not evenly split
+ no_split_modules = getattr(self.model, "_no_split_modules", [])
+ device_map = infer_auto_device_map(
+ self.model, max_memory=max_memory, no_split_module_classes=no_split_modules
+ )
+
+ self.model = dispatch_model(self.model, device_map=device_map)
+ else:
+ self.model = self.model.to(self.device)
+
+ all_inputs = self.cache_inter_data(
+ block_names, nsamples, layer_names=layer_names, last_cache_name=last_cache_name
+ )
+ if hasattr(self.model, "hf_device_map") and len(self.model.hf_device_map) > 1:
+ accelerate.hooks.remove_hook_from_submodules(self.model)
+
+ except RuntimeError as e:
+ cuda_error_msg = traceback.format_exc()
+ try:
+ logger.info("switch to cpu to cache block inputs")
+ if self.has_qlayer_outside_block or self.__class__.__name__ == "AutoRoundMLLM":
+ logger.warning(
+ "we recommend using more GPUs in calibration."
+ " Otherwise, some layers may fall back to `rtn` mode, which can affect accuracy."
+ )
+ if hasattr(self.model, "hf_device_map") and len(self.model.hf_device_map) > 1:
+ accelerate.hooks.remove_hook_from_submodules(
+ self.model
+ ) ##self.model.hf_device_map has not been changed
+ self.model = mv_module_from_gpu(self.model, self.low_cpu_mem_usage)
+ clear_memory()
+ ## Important change after v0.51, on cpu, we use rtn mode for layers in layer_names
+ all_inputs = self.cache_inter_data(
+ block_names, nsamples, layer_names=[], last_cache_name=last_cache_name
+ )
+ except Exception as e:
+ logger.error(cuda_error_msg)
+ raise
+ return all_inputs
+
+ @torch.no_grad()
+ def cache_inter_data(self, block_names, nsamples, layer_names=None, last_cache_name=None):
+ """Save the inputs of block_name for calibration.
+
+ This method temporarily replaces the forward method of the model to capture
+ the inputs passing through the specified block. It then calibrates the model
+ using a specified number of samples. Finally, it restores the original forward
+ method and returns the inputs for the specified block.
+ Args:
+ block_names (list): The names of the blocks for which inputs are to be saved.
+ layer_names (list):The names of the layers for which inputs are to be saved.
+ nsamples (int): The number of samples to use for calibration.
+ last_cache_name (str, optional): The name of the last layer to be cached,
+ we could break the forward in this layer to save time
+
+ Returns:
+ dict: A dictionary containing the inputs for the specified block.
+ """
+ if layer_names is None:
+ layer_names = []
+ self.inputs = {}
+ self.to_cached_layers = block_names + layer_names
+
+ tmp_dtype = None # TODO delete this as most model is not fp32 now
+ ## have bug if block name is not the first block
+ if (len(block_names) > 1 or len(layer_names) > 0) and self.low_gpu_mem_usage:
+ tmp_dtype = self.model.dtype
+ if self.amp:
+ if self.model.dtype != self.model.dtype:
+ self.model = self.model.to(torch.bfloat16)
+ else:
+ self.model = self.model.to(torch.float32) ##model on cpu
+
+ self.last_cache_name = last_cache_name
+ if last_cache_name is None and len(block_names) + len(layer_names) == 1:
+ self.last_cache_name = block_names[0] if len(block_names) == 1 else layer_names[0]
+ # do not set last_cache_name for multimodal models
+ calib_bs = self.batch_size
+ self.hook_handles = []
+ self._replace_forward()
+ self.calib(nsamples, calib_bs)
+ self._recover_forward()
+ res = self.inputs
+ del self.last_cache_name
+ del self.to_cached_layers
+ if tmp_dtype is not None:
+ self.model = self.model.to(tmp_dtype)
+
+ return res
+
+ @torch.no_grad()
+ def _get_block_forward_func(self, name: str) -> Callable:
+ """Gets the forward function.
+
+ Args:
+ name (str): The name of the function.
+ Returns:
+ function: The forward function.
+ """
+
+ def post_process_cache_data(batch_size, data, data_name):
+ """
+ Processes store data for batch handling, reshaping if necessary.
+
+ Args:
+ batch_size (int): The size of the batch.
+ data: The data value to store, potentially for caching.
+ data_name (str): Name of the data.
+
+ Returns:
+ Processed data or None
+ """
+ new_data = data
+ if batch_size <= 1:
+ return new_data
+ if data_name in self.shared_cache_keys:
+ return None
+ if "alibi" in data_name:
+ if isinstance(data, torch.Tensor):
+ alibi = data
+ alibi = alibi.reshape(batch_size, -1, alibi.shape[1], alibi.shape[2])
+ new_data = alibi
+ return new_data
+
+ def forward(m, hidden_states=None, *positional_inputs, **kwargs):
+ """Rewrite forward function, process and collect input data.
+
+ Args:
+ hidden_states (torch.Tensor): The hidden states tensor.
+ *positional_inputs: Variable number of positional arguments.
+ **kwargs: Variable number of keyword arguments.
+
+ Returns:
+ NotImplementedError: Getting the first layer inputs and then raise the error to save runtime.
+ """
+ if name not in self.inputs:
+ self.inputs[name] = {}
+ init_cache(positional_inputs, self.inputs[name])
+
+ if self.batch_dim is None:
+ self.batch_dim = 0
+ if hidden_states is not None and self.batch_size > 1:
+ if hidden_states.shape[0] > self.batch_size:
+ self.batch_dim = 1
+ if len(hidden_states.shape) > 1 and hidden_states.shape[1] > self.batch_size:
+ logger.error(
+ "this model has not been supported, "
+ "please raise an issue in https://github.com/intel/auto-round/issues"
+ " or try to set the `batch_size` to 1 and "
+ "`gradient_accumulate_steps` to your current batch size."
+ )
+ exit(-1)
+
+ if hidden_states is not None:
+ kwargs["hidden_states"] = hidden_states
+
+ for key in kwargs.keys():
+ if (
+ isinstance(kwargs[key], torch.Tensor)
+ or isinstance(kwargs[key], list)
+ or isinstance(kwargs[key], tuple)
+ ):
+ if key not in self.inputs[name].keys(): # initialization
+ data = to_device(kwargs[key], device=torch.device("cpu"))
+ if data is None or (self.batch_size > 1 and key in self.shared_cache_keys):
+ self.inputs[name][key] = data
+ continue
+ if self.batch_size <= 1:
+ self.inputs[name][key] = [data]
+ else:
+ data = post_process_cache_data(self.batch_size, data, key)
+ self.inputs[name][key] = list(torch.split(data, 1, dim=self.batch_dim))
+ else: # append cache inputs
+ new_data = post_process_cache_data(self.batch_size, kwargs[key], key)
+ if new_data is None: # shareable args or NoneType
+ continue
+ new_data = to_device(new_data, device=torch.device("cpu"))
+ if self.batch_size <= 1:
+ self.inputs[name][key].append(new_data)
+ else:
+ self.inputs[name][key].extend(list(torch.split(new_data, 1, dim=self.batch_dim)))
+ elif isinstance(kwargs[key], (str, bool, type(None))):
+ if key not in self.inputs[name].keys():
+ self.inputs[name][key] = kwargs[key]
+ else:
+ # Parameters not to be cached
+ if check_skippable_keywords(key):
+ logger.warning_once(
+ f"Please note that '{key}' key" " is not currently used in quantization fine-tuning."
+ )
+ reset_params(self.inputs[name])
+ if name == self.last_cache_name:
+ raise NotImplementedError
+ else:
+ if hidden_states is not None:
+ kwargs.pop("hidden_states")
+ return m.orig_forward(hidden_states, *positional_inputs, **kwargs)
+ else:
+ # Currently only for Llama-3.2-Vision-Instruct Series
+ return m.orig_forward(*positional_inputs, **kwargs)
+
+ return forward
+
+ @torch.no_grad()
+ def _get_cache_data_hook_for_layer(self, name):
+ """A forward hook to save input max of a module
+ :param name: the module name
+ :return: A hook function."""
+
+ def cache_input_hook(module, inputs, outputs):
+ input = inputs
+ if isinstance(inputs, tuple) or isinstance(input, list):
+ input = inputs[0]
+ if name in self.inputs:
+ self.inputs[name].extend(list(torch.split(input.to("cpu"), 1, dim=0)))
+ else:
+ self.inputs[name] = list(torch.split(input.to("cpu"), 1, dim=0))
+
+ return cache_input_hook
+
+ def _recover_forward(self):
+ """Recovers the forward function."""
+ for n, m in self.model.named_modules():
+ if hasattr(m, "orig_forward"):
+ m.forward = m.orig_forward
+ delattr(m, "orig_forward")
+ for hook_handle in self.hook_handles:
+ hook_handle.remove()
+ self.hook_handles = []
+
+ def _replace_forward(self):
+ """Replaces the forward function."""
+ from functools import partial
+
+ for n, m in self.model.named_modules():
+ if n in self.to_cached_layers and not isinstance(m, tuple(self.supported_types)): ##block
+ m.orig_forward = m.forward
+ m.forward = partial(self._get_block_forward_func(n), m)
+ elif n in self.to_cached_layers: ##linear layer or conv1d layer
+ hook_func = self._get_cache_data_hook_for_layer(n)
+ hook_handle = m.register_forward_hook(hook_func)
+ self.hook_handles.append(hook_handle)
+
+ def _quantize_layer(
+ self, layer_name: str, inputs: torch.Tensor, q_inputs: torch.Tensor = None, device: str = "cpu"
+ ):
+ """Quantize a specific layer of the model using the provided inputs.
+
+ Args:
+ layer_name (str): The name of the layer to quantize.
+ inputs (torch.Tensor): Input data for quantization.
+ q_inputs (torch.Tensor, optional): Quantized input data. Defaults to None.
+ device (torch.device, optional): The device to use for quantization. Defaults to torch.device("cpu").
+
+ Returns:
+ None
+ """
+ logger.info(f"quantizing layer {layer_name}")
+ layer = get_module(self.model, layer_name)
+ if hasattr(layer, "tuning_device"):
+ device = layer.tuning_device
+
+ layer = layer.to(device)
+ for i in range(len(inputs)):
+ inputs[i] = inputs[i].to(layer.weight.dtype)
+ if q_inputs is not None:
+ q_inputs[i] = q_inputs[i].to(layer.weight.dtype)
+
+ wrapper_linear = WrapperLinear(layer, enable_minmax_tuning=self.enable_minmax_tuning, device=device).to(device)
+ round_params = []
+ minmax_params = []
+ for key in wrapper_linear.params.keys():
+ if "min" in key or "max" in key:
+ minmax_params.append(wrapper_linear.params[key])
+ else:
+ round_params.append(wrapper_linear.value)
+ if len(round_params) + len(minmax_params) <= 0:
+ dump_info = f"quantized {layer_name}"
+ logger.info(dump_info)
+ with torch.no_grad():
+ unwrapper_layer(self.model, wrapper_linear, layer_name, {})
+ mv_module_from_gpu(layer, self.low_cpu_mem_usage)
+
+ lr = torch.tensor(self.lr)
+ minmax_lr = torch.tensor(self.minmax_lr)
+ if self.enable_minmax_tuning:
+ optimizer = self.optimizer(
+ [{"params": round_params}, {"params": minmax_params, "lr": minmax_lr}], lr=lr, weight_decay=0
+ )
+ else:
+ optimizer = self.optimizer(round_params, lr=lr, weight_decay=0)
+
+ if self.lr_scheduler is None:
+ lr_schedule = torch.optim.lr_scheduler.LinearLR(
+ optimizer, start_factor=1.0, end_factor=0.0, total_iters=self.iters
+ )
+ else:
+ lr_schedule = copy.deepcopy(self.lr_scheduler)
+ nsamples = len(inputs)
+ last_best_iter = 0
+ best_loss = torch.finfo(torch.float).max
+ scaler = self._get_scaler() # pylint: disable=assignment-from-none
+ init_loss = None
+ gradient_accumulate_steps = self.batch_size ##Force to low gpu
+ batch_size = 1 ##Force to low gpu
+ pick_samples = batch_size * gradient_accumulate_steps
+ pick_samples = min(nsamples, pick_samples)
+ if self.sampler != "rand":
+ whole_indices = torch.randperm(nsamples)[:pick_samples]
+ total_loss = 0
+ num_elm = 1
+ mse_reduction = "mean"
+ if gradient_accumulate_steps != 1:
+ mse_reduction = "sum"
+ mse_loss = torch.nn.MSELoss(reduction=mse_reduction).to(device)
+
+ for i in range(self.iters):
+ total_loss = 0
+ if self.sampler == "rand":
+ whole_indices = torch.randperm(nsamples)[:pick_samples]
+ if gradient_accumulate_steps != 1:
+ if q_inputs is not None:
+ current_input = [q_inputs[i] for i in whole_indices]
+ else:
+ current_input = [inputs[i] for i in whole_indices]
+ num_elm = sum(id.numel() for id in current_input)
+ for tmp_step in range(gradient_accumulate_steps):
+ indices = whole_indices[tmp_step * batch_size : (tmp_step + 1) * batch_size]
+ if q_inputs is not None:
+ current_input = [q_inputs[i] for i in indices]
+ current_input = torch.cat(current_input, dim=0).to(device)
+ org_input = [inputs[i] for i in indices]
+ org_input = torch.cat(org_input, dim=0).to(device)
+ else:
+ current_input = [inputs[i] for i in indices]
+ current_input = torch.cat(current_input, dim=0).to(device)
+ org_input = current_input
+ with torch.no_grad():
+ current_output = layer(org_input)
+
+ if self.amp:
+ with autocast(device_type=device.split(":")[0], dtype=self.amp_dtype):
+ output_q = wrapper_linear(current_input) # pylint: disable=not-callable
+ loss = mse_loss(output_q, current_output) # pylint: disable=not-callable
+ else:
+ output_q = wrapper_linear(current_input) # pylint: disable=not-callable
+ loss = mse_loss( # pylint: disable=not-callable
+ output_q.to(torch.float32), current_output.to(torch.float32)
+ )
+ total_loss += loss.item() / num_elm
+
+ self._scale_loss_and_backward(scaler, loss)
+ if i == 0:
+ init_loss = total_loss
+
+ if total_loss < best_loss:
+ best_loss = total_loss
+ if not self.not_use_best_mse:
+ best_params = collect_best_params(wrapper_linear)
+ last_best_iter = i
+ if self.not_use_best_mse and i == self.iters - 1:
+ best_params = collect_best_params(wrapper_linear)
+
+ if not self.not_use_best_mse:
+ if 0 < self.dynamic_max_gap <= i - last_best_iter:
+ break
+ self._step(scaler, optimizer, lr_schedule)
+
+ last_loss = total_loss
+ best_iter = self.iters
+ if not self.not_use_best_mse:
+ last_loss = best_loss
+ best_iter = last_best_iter
+ with torch.no_grad():
+ unwrapper_layer(self.model, wrapper_linear, layer_name, best_params)
+ mv_module_from_gpu(layer, self.low_cpu_mem_usage)
+ dump_info = f"quantized {layer_name}, loss iter 0: {init_loss:.6f} -> iter {best_iter}: {last_loss:.6f}"
+ logger.info(dump_info)
+
+ def _register_act_max_hook(self, model):
+ def get_act_max_hook(module, input, output):
+ if isinstance(input, (tuple, list)):
+ input = input[0]
+ if input.numel() == 0:
+ return # as no needs for act_max update
+ input, _, _ = reshape_pad_tensor_by_group_size(input, self.act_group_size)
+ act_max = torch.max(torch.abs(input), dim=-1).values
+ if not hasattr(module, "act_max") or module.act_max.numel() == 0:
+ module.act_max = act_max
+ else:
+ act_max = act_max.to(module.act_max.device)
+ if is_nv_fp(self.act_data_type): ## for nvfp per-tensor input_global_scale calculation usage
+ module.act_max = torch.max(
+ torch.tensor([act_max.max(), module.act_max.max()], device=act_max.device)
+ )
+ else:
+ module.act_max = torch.max(act_max, module.act_max)
+
+ hook_handles = []
+
+ for n, m in model.named_modules():
+ if (
+ hasattr(m, "act_dynamic")
+ and check_need_act_calibration(m.act_dynamic, m.act_data_type, m.act_bits)
+ and check_to_quantized(m)
+ ):
+ hook = m.register_forward_hook(get_act_max_hook)
+ hook_handles.append(hook)
+ continue
+
+ # for whole model, RTN
+ if n in self.layer_config:
+ config = self.layer_config[n]
+ act_dynamic = config.get("act_dynamic", True)
+ act_data_type = config.get("act_data_type", None)
+ act_bits = config.get("act_data_type", 16)
+ if (
+ config["bits"] <= 8
+ and check_need_act_calibration(act_dynamic, act_data_type, act_bits)
+ and check_to_quantized(config)
+ ):
+ hook = m.register_forward_hook(get_act_max_hook)
+ hook_handles.append(hook)
+ continue
+ return hook_handles
+
+ def _quantize_block(
+ self,
+ block: torch.nn.Module,
+ input_ids: list[torch.Tensor],
+ input_others: dict,
+ q_input: Union[None, torch.Tensor] = None,
+ device: Union[str, torch.device] = "cpu",
+ ):
+ """Quantize the weights of a given block of the model.
+
+ Args:
+ block: The block of the model to be quantized.
+ input_ids: The input tensor containing tokenized input ids.
+ input_others: A dictionary containing additional input data.
+ q_input: The quantized input tensor.
+ device: The device for quantization.
+
+ Returns:
+ Tuple: (q_outputs, output) if self.enable_quanted_input is True, else (None, output)
+ """
+ if _is_fp8_model(self.model):
+ for n, m in block.named_modules():
+ if _is_fp8_linear(m):
+ new_layer = convert_fp8_layer_to_linear(m, self.amp_dtype).to(device)
+ set_module(block, n, new_layer)
+
+ if self.device_map == "auto":
+ self._set_auto_device_map_in_block(block, input_ids)
+
+ if self.device_map is not None:
+ for n, m in block.named_modules():
+ if len(list(m.children())) != 0 or not hasattr(m, "tuning_device"):
+ continue
+ from accelerate.hooks import AlignDevicesHook, add_hook_to_module
+
+ hook = AlignDevicesHook(m.tuning_device, io_same_device=True)
+ add_hook_to_module(m, hook, True)
+
+ if q_input is None:
+ hook_handles = self._register_act_max_hook(block)
+
+ output = self._get_block_outputs(
+ block, input_ids, input_others, self.batch_size * self.infer_bs_coeff, device, self.cache_device
+ )
+
+ for handle in hook_handles:
+ handle.remove()
+ else:
+ output = self._get_block_outputs(
+ block, input_ids, input_others, self.batch_size * self.infer_bs_coeff, device, self.cache_device
+ )
+ hook_handles = self._register_act_max_hook(block)
+ if hook_handles:
+ self._get_block_outputs(
+ block,
+ q_input,
+ input_others,
+ self.batch_size * self.infer_bs_coeff,
+ device,
+ self.cache_device,
+ save_output=False,
+ )
+
+ for handle in hook_handles:
+ handle.remove()
+
+ if q_input is not None:
+ if input_ids is not q_input:
+ clear_memory(input_ids)
+ else:
+ clear_memory()
+ input_ids = q_input
+
+ quantized_layer_names, unquantized_layer_names = wrapper_block(
+ block, self.enable_minmax_tuning, self.enable_norm_bias_tuning, device=self.device
+ )
+ if is_nv_fp(self.data_type): # enable qkv and moe structure global_scale fuse
+ from auto_round.data_type.utils import update_fused_layer_global_scales
+
+ modules = block.modules()
+ for module in modules:
+ update_fused_layer_global_scales(module)
+ round_params = []
+ minmax_params = []
+ for n, m in block.named_modules():
+ if hasattr(m, "orig_layer"):
+ for key in m.params.keys():
+ if "min" in key or "max" in key:
+ minmax_params.append(m.params[key])
+ else:
+ round_params.append(m.params[key])
+
+ lr = torch.tensor(self.lr)
+ minmax_lr = torch.tensor(self.minmax_lr)
+ if self.enable_minmax_tuning:
+ optimizer = self.optimizer(
+ [{"params": round_params}, {"params": minmax_params, "lr": minmax_lr}], lr=lr, weight_decay=0
+ )
+ else:
+ optimizer = self.optimizer(round_params, lr=lr, weight_decay=0)
+
+ if len(round_params) + len(minmax_params) <= 0:
+ dump_info = (
+ f"quantized {len(quantized_layer_names)}/{(len(quantized_layer_names) + len(unquantized_layer_names))} "
+ f"layers in the block"
+ )
+ logger.info(dump_info)
+ unwrapper_block(block, {}) # TODO Quant layer should change
+ mv_module_from_gpu(block, self.low_cpu_mem_usage)
+ return output, output
+
+ if self.lr_scheduler is None:
+ lr_schedule = torch.optim.lr_scheduler.LinearLR(
+ optimizer, start_factor=1.0, end_factor=0.0, total_iters=self.iters
+ )
+ else:
+ lr_schedule = copy.deepcopy(self.lr_scheduler)
+
+ nsamples = len(input_ids)
+ pick_samples = self.batch_size * self.gradient_accumulate_steps
+ pick_samples = min(nsamples, pick_samples)
+ if self.sampler != "rand":
+ whole_indices = torch.randperm(nsamples)[:pick_samples]
+ last_best_iter = 0
+ best_loss = torch.finfo(torch.float).max
+ num_elm = 1
+ mse_reduction = "mean"
+ if self.gradient_accumulate_steps != 1:
+ mse_reduction = "sum"
+ mse_loss = torch.nn.MSELoss(reduction=mse_reduction).to(device)
+ scaler = self._get_scaler() # pylint: disable=assignment-from-none
+ init_loss = None
+ best_params = {}
+ total_loss = 0
+ for i in range(self.iters):
+ total_loss = 0
+ if self.sampler == "rand":
+ whole_indices = torch.randperm(nsamples)[:pick_samples]
+ # We assume the block input and output shape is same
+ if self.gradient_accumulate_steps != 1:
+ current_input_ids = [input_ids[i] for i in whole_indices]
+ num_elm = sum(id.numel() for id in current_input_ids)
+
+ for tmp_step in range(self.gradient_accumulate_steps):
+ indices = whole_indices[tmp_step * self.batch_size : (tmp_step + 1) * self.batch_size]
+ current_input_ids, current_input_others = self._sampling_inputs(
+ input_ids,
+ input_others,
+ indices,
+ seqlen=self.seqlen,
+ batch_dim=self.batch_dim,
+ share_cache_keys=self.shared_cache_keys,
+ )
+
+ current_output = [output[x] for x in indices]
+ current_output = torch.cat(current_output, dim=self.batch_dim)
+ current_output = to_device(current_output, device)
+
+ output_q = block_forward(
+ block, current_input_ids, current_input_others, self.amp, self.amp_dtype, device
+ )
+ if self.amp:
+ with autocast(device_type=device.split(":")[0], dtype=self.amp_dtype):
+ loss = mse_loss(output_q, current_output) # pylint: disable=not-callable
+ else:
+ loss = mse_loss( # pylint: disable=not-callable
+ output_q.to(torch.float32), current_output.to(torch.float32)
+ )
+
+ total_loss += loss.item() / num_elm
+ self._scale_loss_and_backward(scaler, loss)
+
+ if i == 0:
+ init_loss = total_loss
+
+ if total_loss < best_loss:
+ best_loss = total_loss
+ if not self.not_use_best_mse:
+ best_params = collect_best_params(block)
+ # print(f"get better result at iter {i}, the loss is {total_loss}", flush=True)
+
+ last_best_iter = i
+ if self.not_use_best_mse and i == self.iters - 1:
+ best_params = collect_best_params(block)
+
+ if not self.not_use_best_mse:
+ if 0 < self.dynamic_max_gap <= i - last_best_iter:
+ break
+ self._step(scaler, optimizer, lr_schedule)
+
+ last_loss = total_loss
+ best_iter = self.iters
+ if not self.not_use_best_mse:
+ last_loss = best_loss
+ best_iter = last_best_iter
+ dump_info = (
+ f"quantized {len(quantized_layer_names)}/{(len(quantized_layer_names) + len(unquantized_layer_names))} "
+ f"layers in the block, loss iter 0: {init_loss:.6f} -> iter {best_iter}: {last_loss:.6f}"
+ )
+ logger.info(dump_info)
+ if len(unquantized_layer_names) != 0:
+ logger.info(f"{unquantized_layer_names} have not been quantized")
+ with torch.no_grad():
+ unwrapper_block(block, best_params)
+
+ if (
+ is_nv_fp(self.act_data_type)
+ and hasattr(self, "formats")
+ and any("nv_fp" in format_ for format_ in self.formats)
+ ):
+ # enable moe experts act_max automatic generation for WrapperWALayer
+ set_amax_for_all_moe_layers(block, attr_name="orig_layer.act_max")
+
+ if self.enable_quanted_input:
+ if self.low_cpu_mem_usage:
+ block = block.to(device)
+ clear_memory()
+ q_outputs = self._get_block_outputs(
+ block,
+ input_ids,
+ input_others,
+ self.batch_size * self.infer_bs_coeff,
+ device,
+ cache_device=self.cache_device,
+ )
+ if self.device_map is not None:
+ accelerate.hooks.remove_hook_from_submodules(block)
+ mv_module_from_gpu(block, self.low_cpu_mem_usage)
+ clear_memory(input_ids)
+
+ return q_outputs, output
+
+ else:
+ if self.device_map is not None:
+ accelerate.hooks.remove_hook_from_submodules(block)
+ mv_module_from_gpu(block, self.low_cpu_mem_usage)
+ clear_memory(input_ids)
+ return None, output
+
+ def _quantize_blocks(
+ self,
+ model: torch.nn.Module,
+ inputs: dict,
+ block_names: list,
+ q_input: torch.Tensor = None,
+ nblocks: int = 1,
+ device: str = "cpu",
+ pbar: tqdm = None,
+ ):
+ """Quantize and dequantize the weights of the specified blocks in the model.
+
+ Args:
+ model: The PyTorch model to be quantized.
+ inputs: The input data for quantization.
+ block_names: The names of the blocks to be quantized and dequantized.
+ nblocks: The number of blocks to quantize and dequantize.
+ device: The device for quantization and dequantization.
+
+ Returns:
+ None
+ """
+ clear_memory()
+ for n, m in model.named_parameters():
+ m.requires_grad_(False)
+ input_ids = inputs["input_ids"]
+ inputs.pop("input_ids", None)
+ input_others = inputs
+ clear_memory()
+ input_ids = to_device(input_ids, self.cache_device)
+ input_others = to_device(input_others, self.cache_device)
+ # As in calibration phase, we may use bf16 for calibration due to low_gpu_memory usage
+ tmp_dtype = self.amp_dtype if self.amp else torch.float32
+ for i in range(len(input_ids)):
+ input_ids[i] = input_ids[i].to(tmp_dtype)
+
+ for key in input_others.keys():
+ if isinstance(input_others[key], torch.Tensor) and (
+ input_others[key].dtype == torch.float16 or input_others[key].dtype == torch.bfloat16
+ ):
+ input_others[key] = input_others[key].to(tmp_dtype)
+ elif isinstance(input_others[key], list):
+ for i in range(len(input_others[key])):
+ to_dtype(input_others[key][i], tmp_dtype)
+
+ if (
+ self.sym
+ and self.enable_alg_ext
+ and self.super_group_size is None
+ and (
+ (self.data_type.startswith("int") and self.act_bits >= 8)
+ or self.data_type.startswith("mx")
+ or self.data_type.startswith("nv")
+ )
+ ):
+ try:
+ from auto_round.alg_ext import quantize_block_ext
+
+ BaseCompressor.quantize_block_ext = quantize_block_ext
+ quantize_block = self.quantize_block_ext # must use self.quantize_block_ext
+ if self.bits > 2 and (not self.data_type.startswith("mx") or not self.data_type.startswith("nv")):
+ logger.warning(
+ "algorithm extension has only undergone limited validation on "
+ "INT2,mxfp4 and nvfp4; use with caution."
+ )
+ else:
+ logger.info("using algorithm extension for quantization.")
+ except (ImportError, ModuleNotFoundError):
+ quantize_block = self._quantize_block
+ if self.enable_torch_compile:
+ quantize_block = compile_func(quantize_block, device)
+ else:
+ quantize_block = quantize_block
+ else:
+ quantize_block = self._quantize_block
+ if self.enable_torch_compile:
+ quantize_block = compile_func(quantize_block, device)
+
+ if pbar is None:
+ pbar = tqdm(range(0, len(block_names), nblocks))
+
+ for i in range(0, len(block_names), nblocks):
+ if i != 0:
+ pbar.update(1)
+ if nblocks == 1:
+ n = block_names[i]
+ pbar.set_description(f"Quantizing {n}")
+ m = get_module(model, n)
+ else:
+ names = block_names[i : min(i + nblocks, len(block_names))]
+ pbar.set_description(f"Quantizing [{i + 1}-{min(i + nblocks, len(block_names))}]/{len(block_names)}")
+ modules = [get_module(model, n) for n in names]
+ m = WrapperMultiblock(modules)
+
+ if not self.model.device.type == "meta" or self.low_cpu_mem_usage:
+ m = m.to(device)
+
+ q_input, input_ids = quantize_block(
+ m,
+ input_ids,
+ input_others,
+ q_input=q_input,
+ device=device,
+ )
+ if self.is_packing_immediate:
+ from auto_round.export import PACKING_LAYER_WITH_FORMAT
+
+ for _, tmp_m in m.named_modules():
+ if not (hasattr(tmp_m, "bits") and check_to_quantized(tmp_m)):
+ continue
+ target_backend = self.formats[0].split(":")[0] if ":" in self.formats[0] else self.formats[0]
+ has_gguf = any("gguf" in format_ for format_ in self.formats)
+ if has_gguf:
+ from auto_round.export.export_to_gguf.export import pack_gguf_layer
+
+ output_dir = self._get_save_folder_name(self.formats[0])
+ model_type = ModelType.MMPROJ if self.mllm else ModelType.TEXT
+ pack_gguf_layer(
+ tmp_m.tmp_name,
+ self.model,
+ self.formats[0],
+ output_dir,
+ self.layer_config,
+ self.tokenizer,
+ processor=self.processor if hasattr(self, "processor") else None,
+ image_processor=self.image_processor if hasattr(self, "image_processor") else None,
+ model_type=model_type,
+ )
+ else:
+ PACKING_LAYER_WITH_FORMAT[target_backend](
+ tmp_m.tmp_name, self.model, self.formats[0], device=self.device
+ )
+ pbar.set_description("Quantizing done")
+ pbar.update(1)
+ pbar.close()
+ self.model = mv_module_from_gpu(self.model, self.low_cpu_mem_usage)
+ for n, m in self.model.named_modules():
+ if hasattr(m, "name"):
+ delattr(m, "name")
+
+ del q_input
+ del input_ids
+ del input_others
+ del inputs
+
+ clear_memory()
+
+ def save_quantized(
+ self, output_dir: str = None, format: str = "auto_round", inplace: bool = True, **kwargs
+ ) -> torch.nn.Module:
+ """Save the quantized model to the specified output directory in the specified format.
+
+ Args:
+ output_dir (str, optional): The directory to save the quantized model. Defaults to None.
+ format (str, optional): The format in which to save the model. Defaults to "auto_round".
+ inplace (bool, optional): Whether to modify the model in place. Defaults to True.
+ **kwargs: Additional keyword arguments specific to the export format.
+
+ Returns:
+ object: The compressed model object.
+ """
+ format = self._check_supported_format(format)
+
+ if self.low_cpu_mem_usage:
+ self.model = self.model.to("cpu")
+
+ if not self.quantized:
+ logger.warning("please run autoround.quantize first")
+ return
+ if format == "fake" or format == "qdq": # TODO fix act quantization later
+ self.model = self.model.to("cpu")
+ self.model.save_pretrained(output_dir)
+ if self.tokenizer is not None:
+ self.tokenizer.save_pretrained(output_dir)
+ processor = kwargs.get("processor", None)
+ if processor is not None:
+ processor.save_pretrained(output_dir)
+ try:
+ copy_python_files_from_model_cache(self.model, output_dir)
+ except Exception as e:
+ logger.warning("Skipping source model Python file copy due to error: %s", e)
+ return
+ if self.act_bits <= 8 and format == "qdq":
+ logger.warning(
+ "Support for exporting activation quantization is limited. "
+ "Please ensure that your configuration is supported."
+ )
+ if format == "llm_compressor" and (is_nv_fp(self.data_type) or is_mx_fp(self.data_type)):
+ format = format.replace("llm_compressor", f"llm_compressor:{self.data_type}")
+
+ from auto_round.export import EXPORT_FORMAT
+
+ backend = format
+ format = format.split(":")[0]
+ if format not in EXPORT_FORMAT:
+ logger.error(f"export format only supports {EXPORT_FORMAT.keys()}")
+ raise ValueError(f"export format only supports {EXPORT_FORMAT.keys()}, but got {format}")
+ save_quantized_as_format = EXPORT_FORMAT.get(format)
+ if "gptq" in format and not self.sym:
+ logger.warning(
+ "the asymmetrical kernel of the GPTQ format may result in a noticeable accuracy drop,"
+ " particularly for 2-bit quantization and smaller models."
+ " We recommend exporting to either the AutoAWQ format ( only 4 bits) or "
+ "the AutoRound format(2/3/4/8 bits)."
+ )
+ if "awq" in format and not self.bits == 4:
+ raise ValueError("The AWQ format only supports W4 quantization ")
+ serialization_keys = [
+ "bits",
+ "group_size",
+ "sym",
+ "data_type",
+ "enable_quanted_input",
+ "enable_minmax_tuning",
+ "seqlen",
+ "batch_size",
+ "scale_dtype",
+ "lr",
+ "minmax_lr",
+ "gradient_accumulate_steps",
+ "iters",
+ "amp",
+ "nsamples",
+ "low_gpu_mem_usage",
+ "to_quant_block_names",
+ "enable_norm_bias_tuning",
+ "act_bits",
+ "act_group_size",
+ "act_sym",
+ "act_dynamic",
+ "act_data_type",
+ "super_bits",
+ "super_group_size",
+ ]
+ if isinstance(self.dataset, str):
+ serialization_keys.append("dataset")
+ serialization_dict = {}
+ for key in serialization_keys:
+ serialization_dict[key] = getattr(self, key)
+ from auto_round.version import __version__
+
+ serialization_dict["autoround_version"] = __version__
+ if "scale_dtype" in serialization_dict.keys():
+ serialization_dict["scale_dtype"] = str(serialization_dict["scale_dtype"])
+
+ compressed_model = save_quantized_as_format( # TODO refine the code
+ output_dir,
+ model=self.model,
+ layer_config=self.layer_config,
+ inplace=inplace,
+ bits=self.bits,
+ act_bits=self.act_bits,
+ group_size=self.group_size,
+ sym=self.sym,
+ iters=self.iters,
+ lr=self.lr,
+ minmax_lr=self.minmax_lr,
+ enable_minmax_tuning=self.enable_minmax_tuning,
+ enable_quanted_input=self.enable_quanted_input,
+ scale_dtype=self.scale_dtype,
+ tokenizer=self.tokenizer,
+ supported_types=self.supported_types,
+ data_type=self.data_type,
+ act_data_type=self.act_data_type,
+ serialization_dict=serialization_dict,
+ backend=backend,
+ to_quant_block_names=self.to_quant_block_names,
+ quant_block_list=self.quant_block_list,
+ device=self.device,
+ **kwargs,
+ )
+ return compressed_model
+
+ def _get_quantized_layer_names_outside_blocks(self) -> list:
+ """Gets the names of quantized layers outside blocks in the model.
+
+ Returns:
+ list: List of layer names outside blocks.
+ """
+ if self.layer_config is None or len(self.layer_config) == 0:
+ return []
+
+ layer_names = []
+ all_layers_in_block = get_layer_names_in_block(self.model, self.supported_types, self.quant_block_list)
+
+ for key in self.layer_config.keys():
+ if key in all_layers_in_block:
+ continue
+ layer = get_module(self.model, key)
+ if layer is None:
+ logger.error(f"could not find layer {key} in the model, exit...")
+ exit(-1)
+ if isinstance(layer, tuple(self.supported_types)) and check_to_quantized(self.layer_config[key]):
+ layer_names.append(key)
+
+ return layer_names
+
+ def _set_amp_dtype(self) -> None:
+ """Sets the automatic mixed precision (AMP) data type for the model based on the device and configuration."""
+ self.amp_dtype = torch.bfloat16
+ if self.model.dtype != torch.float32:
+ self.amp_dtype = self.model.dtype
+ if self.device == "cpu" or "hpu" in self.device:
+ self.amp_dtype = torch.bfloat16
+ if self.amp:
+ if self.device == "cpu" and not CpuInfo().bf16:
+ self.amp = False
+ self.amp_dtype = torch.float32
+ self.model = self.model.to(torch.float32)
+ logger.warning(
+ f"amp is set to FALSE as the current {self.device} device does not support the 'bf16' data type."
+ )
+ else:
+ if self.model.dtype != self.amp_dtype:
+ self.model = self.model.to(self.amp_dtype)
+ else:
+ self.amp_dtype = torch.float32
+ self.model = self.model.to(torch.float32)
+
+ def _get_optimizer(self, optimizer: Any):
+ """Returns the specified optimizer. In SignRound, we fix the optimizer.
+
+ Args:
+ optimizer: The optimizer to be used.
+
+ Returns:
+ The specified optimizer.
+ """
+ return SignSGD
+
+ def _get_scaler(self):
+ """Returns scaler, in SignRound, no need to use scaler."""
+ return None
+
+ def _scale_loss_and_backward(self, scaler: Any, loss: torch.Tensor) -> torch.Tensor:
+ """Scales the loss and performs backward pass.
+
+ Args:
+ scaler: The scaler to be used.
+ loss: The loss to be scaled.
+
+ Returns:
+ The scaled loss.
+ """
+ scale_loss = loss * 1000
+ scale_loss.backward()
+ if is_optimum_habana_available():
+ htcore.mark_step()
+ return scale_loss
+
+ def _step(self, scaler: Any, optimizer: Any, lr_schedule: Any):
+ """Performs a step in the optimization process.
+
+ Args:
+ scaler: The scaler to be used.
+ optimizer: The optimizer for the step.
+ lr_schedule: The learning rate schedule.
+
+ Returns:
+ None
+ """
+ optimizer.step()
+ # for hpu
+ if is_optimum_habana_available():
+ htcore.mark_step()
+ optimizer.zero_grad()
+ lr_schedule.step()
+
+ @classmethod
+ @torch.no_grad()
+ def _sampling_inputs(
+ cls,
+ input_ids: list[torch.Tensor],
+ input_others: dict,
+ indices: list[int],
+ seqlen: int,
+ batch_dim: int = 0,
+ share_cache_keys: tuple = (),
+ ):
+ """Samples inputs based on the given indices and sequence length.
+
+ Args:
+ input_ids: The list of input tensor containing input_ids.
+ input_others: A dictionary containing other input data.
+ indices: The indices to sample from the input.
+ seqlen: The sequence length.
+
+ Returns:
+ current_input_ids: The sampled input IDs.
+ current_input_others: The sampled other input data.
+ """
+ current_input_ids = [input_ids[i] for i in indices]
+
+ current_input_ids = torch.cat(current_input_ids, dim=batch_dim)
+
+ current_input_others = {"positional_inputs": input_others["positional_inputs"]}
+ for key in input_others.keys():
+ if "positional_inputs" in key:
+ continue
+ if (key not in share_cache_keys or len(indices) == 1) and not isinstance(
+ input_others[key], (str, bool, type(None))
+ ):
+ current_input_others[key] = None
+ if input_others[key] is not None:
+ current_input_others[key] = [input_others[key][i] for i in indices]
+ if len(indices) == 1:
+ current_input_others[key] = current_input_others[key][0]
+ else:
+ try:
+ current_input_others[key] = torch.cat(current_input_others[key], dim=0)
+ except TypeError as err:
+ logger.warning_once("Please check the model cache inputs or try setting batch_size to 1.")
+ else:
+ current_input_others[key] = input_others[key]
+
+ return current_input_ids, current_input_others
+
+
+class LLMCompressor(BaseCompressor):
+ pass
+
+
+class AdamCompressor(BaseCompressor):
+ """Class for quantization with optimizers like adamw of a PyTorch model.
+
+ Args:
+ model: The PyTorch model to be quantized.
+ tokenizer: An optional tokenizer for processing input data.
+ scheme (str| dict | QuantizationScheme ): A preset scheme that defines the quantization configurations
+ bits (int): Number of bits for quantization (default is 4).
+ group_size (int): Size of the quantization group (default is 128).
+ sym (bool): Whether sym to be used (default is True).
+ layer_config (dict): Configuration for weight quantization (default is None).
+ batch_size (int): Batch size for training (default is 8).
+ amp (bool): Whether to use automatic mixed precision (default is True).
+ device: The device to be used for training (default is "auto").
+ lr_scheduler: The learning rate scheduler to be used.
+ dataset: The default dataset name (default is "NeelNanda/pile-10k").
+ enable_quanted_input (bool): Whether to use quantized input data (default is True).
+ enable_minmax_tuning (bool): Whether to enable min-max tuning (default is True).
+ lr (float): The learning rate (default is 0.005).
+ minmax_lr (float): The learning rate for min-max tuning (default is None).
+ low_gpu_mem_usage (bool): Whether to use low GPU memory (default is False).
+ low_cpu_mem_usage (bool): Whether to use low CPU memory (default is False).
+ iters (int): Number of iterations (default is 200).
+ seqlen (int): Length of the sequence.
+ nsamples (int): Number of samples (default is 128).
+ sampler (str): The sampling method (default is "rand").
+ seed (int): The random seed (default is 42).
+ nblocks (int): Number of blocks (default is 1).
+ gradient_accumulate_steps (int): Number of gradient accumulation steps (default is 1).
+ not_use_best_mse (bool): Whether to use mean squared error (default is False).
+ dynamic_max_gap (int): The dynamic maximum gap (default is -1).
+ data_type (str): The data type to be used (default is "int").
+ scale_dtype (str): The data type of quantization scale to be used (default is "float16"), different kernels
+ have different choices.
+ act_bits (int): Number of bits for activation quantization. Default is 16.
+ act_group_size (int): Group size for activation quantization. Default is None.
+ act_sym (bool): Whether to use symmetric activation quantization. Default is None.
+ act_data_type (str): Specifies the data type for activations.
+ Defaults to None, in which case it inherits the weight data type.
+ act_dynamic (bool): Whether to use dynamic activation quantization. Default is True.
+ to_quant_block_names (str|list): A string or list whose elements are list of
+ block's layer names to be quantized.
+ enable_norm_bias_tuning (bool): Whether to enable fast norm/layer_bias tuning
+ enable_torch_compile (bool): Whether to enable torch compile to optimize quant_block/layer function
+ **kwargs: Additional keyword arguments.
+
+ Returns:
+ The quantized model.
+ """
+
+ bits: int | None
+ group_size: int | None
+ sym: bool | None
+ data_type: str | None
+ act_bits: int | None
+ act_group_size: int | None
+ act_sym: bool | None
+ act_data_type: str | None
+ act_dynamic: bool | None
+ super_bits: int | None
+ super_group_size: int | None
+
+ def __init__(
+ self,
+ model: Union[torch.nn.Module, str],
+ tokenizer=None,
+ scheme: Union[str, dict, QuantizationScheme] = "W4A16",
+ layer_config: dict[str, Union[str, dict, QuantizationScheme]] = None,
+ dataset: Union[str, list, tuple, torch.utils.data.DataLoader] = "NeelNanda/pile-10k",
+ iters: int = 200,
+ seqlen: int = 2048,
+ nsamples: int = 128,
+ batch_size: int = 8,
+ gradient_accumulate_steps: int = 1,
+ low_gpu_mem_usage: bool = False,
+ device_map: Union[str, int, torch.device, dict] = 0,
+ enable_torch_compile: bool = False,
+ seed: int = 42,
+ optimizer="AdamW",
+ **kwargs,
+ ):
+ super(AdamCompressor, self).__init__(
+ model=model,
+ tokenizer=tokenizer,
+ scheme=scheme,
+ layer_config=layer_config,
+ batch_size=batch_size,
+ dataset=dataset,
+ low_gpu_mem_usage=low_gpu_mem_usage,
+ iters=iters,
+ seqlen=seqlen,
+ nsamples=nsamples,
+ seed=seed,
+ gradient_accumulate_steps=gradient_accumulate_steps,
+ enable_torch_compile=enable_torch_compile,
+ device_map=device_map,
+ **kwargs,
+ )
+
+ self.optimizer = self._get_optimizer(optimizer)
+
+ def _get_optimizer(self, optimizer):
+ if optimizer is None:
+ optimizer = torch.optim.AdamW
+ elif isinstance(optimizer, str):
+ optimizer = getattr(torch.optim, optimizer)
+ else:
+ optimizer = optimizer
+ return optimizer
+
+ def _get_scaler(self):
+ scaler = None
+ if self.amp and not check_is_cpu(self.device):
+ from torch.cuda.amp import GradScaler
+
+ scaler = GradScaler(init_scale=1024, growth_interval=100000)
+ return scaler
+
+ def _scale_loss_and_backward(self, scaler, loss):
+ if scaler is not None:
+ loss = scaler.scale(loss)
+
+ loss.backward()
+ if is_optimum_habana_available():
+ htcore.mark_step()
+ return loss
+
+ def _step(self, scaler, optimizer, lr_schedule):
+ if scaler is not None:
+ scaler.step(optimizer)
+ optimizer.zero_grad()
+ lr_schedule.step()
+ scaler.update()
+ else:
+ optimizer.step()
+ optimizer.zero_grad()
+ lr_schedule.step()
+ if is_optimum_habana_available():
+ htcore.mark_step()
diff --git a/auto_round/compressors/config.py b/auto_round/compressors/config.py
new file mode 100644
index 000000000..cc9081ce9
--- /dev/null
+++ b/auto_round/compressors/config.py
@@ -0,0 +1,262 @@
+# Copyright (c) 2025 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from __future__ import annotations
+
+from dataclasses import dataclass, fields
+from typing import Any, Callable, Union
+
+import torch
+
+
+class ExtraConfig:
+ """Class for extra or legacy configs."""
+
+ _model_config = None
+ _scheme_config = None
+ _tuning_config = None
+ _mllm_config = None
+
+ def __init__(
+ self,
+ # tuning
+ amp: bool = True,
+ disable_opt_rtn: bool = True,
+ enable_alg_ext: bool = False,
+ enable_minmax_tuning: bool = True,
+ enable_norm_bias_tuning: bool = False,
+ enable_quanted_input: bool = True,
+ enable_deterministic_algorithms: bool = False,
+ lr: float = None,
+ lr_scheduler: Callable = None,
+ minmax_lr: float = None,
+ mem_per_param_scale: int = None,
+ nblocks: int = 1,
+ quant_lm_head: bool = False,
+ to_quant_block_names: Union[str, list, None] = None,
+ scale_dtype: str = "fp16",
+ # scheme
+ bits: int = None,
+ group_size: int = None,
+ sym: bool = None,
+ data_type: str = None,
+ act_bits: int = None,
+ act_group_size: int = None,
+ act_sym: bool = None,
+ act_data_type: str = None,
+ act_dynamic: bool = None,
+ super_bits: int = None,
+ super_group_size: int = None,
+ static_kv_dtype: Union[str, torch.dtype] = None,
+ # mllm
+ processor: Callable = None,
+ image_processor: Callable = None,
+ quant_nontext_module: bool = False,
+ extra_data_dir: str = None,
+ template: str = None,
+ ):
+ """Initialize
+
+ Args:
+ amp (bool): Whether to use automatic mixed precision (default is True).
+ disable_opt_rtn (bool, optional): Disable RTN-mode optimization (iters=0). Defaults to False.
+ enable_alg_ext (bool, optional): Enable algorithm extension (primarily for INT2). Defaults to False.
+ enable_minmax_tuning (bool, optional): Enable weight min-max tuning. Defaults to True.
+ enable_norm_bias_tuning (bool): Whether to enable fast norm/layer_bias tuning.
+ enable_quanted_input (bool): Whether to use quantized input data (default is True).
+ enable_deterministic_algorithms (bool): Whether to use deterministic_algorithms.
+ lr (float): The learning rate (default is 0.005).
+ lr_scheduler: The learning rate scheduler to be used.
+ minmax_lr (float): The learning rate for min-max tuning (default is None).
+ mem_per_param_scale (int): Scale factor for memory per parameter,
+ used to adjust memory usage estimation for tuning.
+ nblocks (int): Number of blocks (default is 1).
+ quant_lm_head (bool): Whether to quant lm_head.
+ to_quant_block_names (str|list): Names of quantitative blocks, please use commas to separate them.
+ scale_dtype (str): The data type of quantization scale to be used (default is "float16"), different kernels
+ bits (int, optional): Weight quantization bits. Defaults to 4.
+ group_size (int, optional): Weight quantization group size. Defaults to 128.
+ sym (bool, optional): Symmetric weight quantization. Defaults to True.
+ data_type (str, optional): Weight data type string, e.g., "int". Defaults to "int".
+ act_bits (int, optional): Activation quantization bits. Defaults to 16.
+ act_group_size (int, optional): Activation group size. Defaults to None.
+ act_sym (bool, optional): Symmetric activation quantization. Defaults to None.
+ act_data_type (str, optional): Activation data type; inherits weight dtype if None and act_bits < 16.
+ act_dynamic (bool, optional): Dynamic activation quantization. Defaults to True.
+ super_bits (int): number of scale and mins quant bits for double quant.
+ super_group_size (int): the number of super group size when use double quant.
+ static_kv_dtype (str): The data type of kv-cache to be used.
+ processor: Any multi-modal model will require an object to encode or
+ decode the data that groups several modalities (among text, vision and audio).
+ image_processor: Image processor for special model like llava.
+ quant_nontext_module: Whether to quantize nontext module.
+ extra_data_dir: The path of extra data such as images, audio and videos.
+ template: The path or name of template used to specify process for different MLLMs.
+ """
+ self.tuning_config = TuningExtraConfig(
+ amp=amp,
+ disable_opt_rtn=disable_opt_rtn,
+ enable_alg_ext=enable_alg_ext,
+ enable_minmax_tuning=enable_minmax_tuning,
+ enable_norm_bias_tuning=enable_norm_bias_tuning,
+ enable_quanted_input=enable_quanted_input,
+ enable_deterministic_algorithms=enable_deterministic_algorithms,
+ lr=lr,
+ lr_scheduler=lr_scheduler,
+ minmax_lr=minmax_lr,
+ mem_per_param_scale=mem_per_param_scale,
+ nblocks=nblocks,
+ quant_lm_head=quant_lm_head,
+ to_quant_block_names=to_quant_block_names,
+ scale_dtype=scale_dtype,
+ )
+ self.scheme_config = SchemeExtraConfig(
+ bits=bits,
+ group_size=group_size,
+ sym=sym,
+ data_type=data_type,
+ act_bits=act_bits,
+ act_group_size=act_group_size,
+ act_sym=act_sym,
+ act_data_type=act_data_type,
+ act_dynamic=act_dynamic,
+ super_bits=super_bits,
+ super_group_size=super_group_size,
+ static_kv_dtype=static_kv_dtype,
+ )
+ self.mllm_config = MLLMExtraConfig(
+ processor=processor,
+ image_processor=image_processor,
+ quant_nontext_module=quant_nontext_module,
+ extra_data_dir=extra_data_dir,
+ template=template,
+ )
+
+ @property
+ def tuning_config(self):
+ return self._tuning_config
+
+ @tuning_config.setter
+ def tuning_config(self, config: TuningExtraConfig):
+ assert isinstance(
+ config, TuningExtraConfig
+ ), f"tuning_config should be ModelExtraConfig, but got {config.__class__.__name__}"
+ self._tuning_config = config
+
+ @property
+ def scheme_config(self):
+ return self._scheme_config
+
+ @scheme_config.setter
+ def scheme_config(self, config: SchemeExtraConfig):
+ assert isinstance(
+ config, SchemeExtraConfig
+ ), f"scheme_config should be SchemeExtraConfig, but got {config.__class__.__name__}"
+ self._scheme_config = config
+
+ @property
+ def mllm_config(self):
+ return self._mllm_config
+
+ @mllm_config.setter
+ def mllm_config(self, config: MLLMExtraConfig):
+ if config is None:
+ self._mllm_config = None
+ else:
+ assert isinstance(
+ config, MLLMExtraConfig
+ ), f"mllm_config should be MLLMExtraConfig, but got {config.__class__.__name__}"
+ self._mllm_config = config
+
+ def to_dict(self):
+ output_dict = {}
+ for config in self.__dict__.values():
+ if config:
+ output_dict.update(config.to_dict())
+ return output_dict
+
+
+@dataclass
+class BaseExtraConfig:
+
+ @classmethod
+ def get_attributes(cls: "BaseExtraConfig") -> list[str]:
+ return [field.name for field in fields(cls)]
+
+ def __getitem__(self, key: str):
+ if key not in self.get_attributes():
+ raise KeyError(f"{key} is not a valid attribute")
+ return getattr(self, key)
+
+ def __setitem__(self, key: str, value: None | int | str):
+ if key not in self.get_attributes():
+ raise KeyError(f"{key} is not a valid attribute")
+ setattr(self, key, value)
+
+ def __contains__(self, item):
+ return item in self.get_attributes()
+
+ def to_dict(self):
+ return self.__dict__
+
+ def is_default(self):
+ for field in fields(self):
+ default_value = field.default
+ current_value = getattr(self, field.name)
+ if current_value != default_value:
+ return False
+ return True
+
+
+@dataclass
+class TuningExtraConfig(BaseExtraConfig):
+ amp: bool = True
+ disable_opt_rtn: bool = True
+ enable_alg_ext: bool = False
+ enable_minmax_tuning: bool = True
+ enable_norm_bias_tuning: bool = False
+ enable_quanted_input: bool = True
+ enable_deterministic_algorithms: bool = False
+ lr: float = None
+ lr_scheduler: Callable = None
+ minmax_lr: float = None
+ mem_per_param_scale: int = None
+ nblocks: int = 1
+ quant_lm_head: bool = False
+ to_quant_block_names: Union[str, list, None] = None
+ scale_dtype: str = "fp16"
+
+
+@dataclass
+class SchemeExtraConfig(BaseExtraConfig):
+ bits: int = None
+ group_size: int = None
+ sym: bool = None
+ data_type: str = None
+ act_bits: int = None
+ act_group_size: int = None
+ act_sym: bool = None
+ act_data_type: str = None
+ act_dynamic: bool = None
+ super_bits: int = None
+ super_group_size: int = None
+ static_kv_dtype: Union[str, torch.dtype] = None
+
+
+@dataclass
+class MLLMExtraConfig(BaseExtraConfig):
+ processor: Callable = None
+ image_processor: Callable = None
+ quant_nontext_module: bool = False
+ extra_data_dir: str = None
+ template: str = None
diff --git a/auto_round/mllm/README.md b/auto_round/compressors/mllm/README.md
similarity index 100%
rename from auto_round/mllm/README.md
rename to auto_round/compressors/mllm/README.md
diff --git a/auto_round/mllm/__init__.py b/auto_round/compressors/mllm/__init__.py
similarity index 67%
rename from auto_round/mllm/__init__.py
rename to auto_round/compressors/mllm/__init__.py
index d77883b7e..e92f378d4 100644
--- a/auto_round/mllm/__init__.py
+++ b/auto_round/compressors/mllm/__init__.py
@@ -12,8 +12,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-from .mllm_dataset import get_mllm_dataloader
-from .template import Template, get_template, TEMPLATES
-from .autoround_mllm import AutoRoundMLLM
-from ..utils import LazyImport
-from .eval import mllm_eval, lmms_eval
+from auto_round.compressors.mllm.dataset import get_mllm_dataloader
+from auto_round.compressors.mllm.template import Template, get_template, TEMPLATES
+from auto_round.compressors.mllm.compressor import MLLMCompressor
+from auto_round.compressors.mllm.eval import mllm_eval, lmms_eval
diff --git a/auto_round/mllm/autoround_mllm.py b/auto_round/compressors/mllm/compressor.py
similarity index 97%
rename from auto_round/mllm/autoround_mllm.py
rename to auto_round/compressors/mllm/compressor.py
index f6be55851..63199b081 100644
--- a/auto_round/mllm/autoround_mllm.py
+++ b/auto_round/compressors/mllm/compressor.py
@@ -18,11 +18,11 @@ from typing import Union
import torch
from tqdm import tqdm
-from auto_round.autoround import AutoRound
+from auto_round.compressors.base import BaseCompressor
+from auto_round.compressors.mllm.dataset import get_mllm_dataloader
+from auto_round.compressors.mllm.template import Template, get_template
from auto_round.logger import logger
from auto_round.low_cpu_mem.utils import get_layers_before_block
-from auto_round.mllm.mllm_dataset import get_mllm_dataloader
-from auto_round.mllm.template import Template, get_template
from auto_round.schemes import QuantizationScheme
from auto_round.special_model_handler import (
NOT_SUPPORT_ONLY_TEXT_MODELS,
@@ -76,7 +76,7 @@ def _only_text_test(model, tokenizer, device, model_type):
return False
-class AutoRoundMLLM(AutoRound):
+class MLLMCompressor(BaseCompressor):
"""Class for automatic rounding-based quantization with MLLMs.
Args:
@@ -123,8 +123,6 @@ class AutoRoundMLLM(AutoRound):
block's layer names to be quantized.
enable_torch_compile (bool): Whether to enable torch compile to optimize quant_block/layer
**kwargs: Additional keyword arguments.
-
-
"""
bits: int | None
@@ -205,8 +203,9 @@ class AutoRoundMLLM(AutoRound):
model = _handle_special_model(model)
- from ..calib_dataset import CALIB_DATASETS
- from .mllm_dataset import MLLM_DATASET
+ from auto_round.calib_dataset import CALIB_DATASETS
+
+ from .dataset import MLLM_DATASET
if iters > 0 and isinstance(dataset, str) and dataset in CALIB_DATASETS.keys():
if quant_nontext_module:
@@ -251,7 +250,8 @@ class AutoRoundMLLM(AutoRound):
nsamples = (nsamples // batch_size + 1) * batch_size
logger.warning(f"'nsamples' is not divisible by 'batch_size', will adjusted to {nsamples}")
- super(AutoRoundMLLM, self).__init__(
+ kwargs["mllm"] = True
+ super(MLLMCompressor, self).__init__(
model=model,
tokenizer=tokenizer,
scheme=scheme,
@@ -266,7 +266,6 @@ class AutoRoundMLLM(AutoRound):
device_map=device_map,
enable_torch_compile=enable_torch_compile,
seed=seed,
- vlm=True,
to_quant_block_names=to_quant_block_names,
**kwargs,
)
@@ -435,7 +434,7 @@ class AutoRoundMLLM(AutoRound):
def _check_quant_nontext(self, layer_config, quant_nontext_module):
if not layer_config:
return quant_nontext_module
- from auto_round.mllm.utils import VISUAL_KEYS
+ from auto_round.compressors.mllm.utils import VISUAL_KEYS
for layer_name in layer_config.keys():
for vlm_key in VISUAL_KEYS:
diff --git a/auto_round/mllm/mllm_dataset.py b/auto_round/compressors/mllm/dataset.py
similarity index 98%
rename from auto_round/mllm/mllm_dataset.py
rename to auto_round/compressors/mllm/dataset.py
index 267b8eb4a..21da6ba25 100644
--- a/auto_round/mllm/mllm_dataset.py
+++ b/auto_round/compressors/mllm/dataset.py
@@ -20,8 +20,9 @@ import torch
from torch.utils.data import DataLoader, Dataset
from transformers import set_seed
-from ..special_model_handler import check_mllm_model_batch
-from ..utils import logger
+from auto_round.logger import logger
+from auto_round.special_model_handler import check_mllm_model_batch
+
from .template import Template
from .utils import _extract_data_dir
@@ -246,7 +247,7 @@ def get_mllm_dataloader(
return DataLoader(dataset, **dataloader_params), bs, gradient_accumulate_steps
else:
# try to load text calibration dataset
- from ..calib_dataset import get_dataloader
+ from auto_round.calib_dataset import get_dataloader
dataloader = get_dataloader(tokenizer, seqlen, dataset, seed, bs, nsamples)
if quant_nontext_module:
diff --git a/auto_round/mllm/eval.py b/auto_round/compressors/mllm/eval.py
similarity index 99%
rename from auto_round/mllm/eval.py
rename to auto_round/compressors/mllm/eval.py
index e08c88ebd..d9e7bdeaf 100644
--- a/auto_round/mllm/eval.py
+++ b/auto_round/compressors/mllm/eval.py
@@ -53,7 +53,8 @@ from functools import partial
import numpy as np
-from auto_round.utils import LazyImport, logger
+from auto_round.logger import logger
+from auto_round.utils import LazyImport
vlmeval = LazyImport("vlmeval")
diff --git a/auto_round/mllm/processor.py b/auto_round/compressors/mllm/processor.py
similarity index 99%
rename from auto_round/mllm/processor.py
rename to auto_round/compressors/mllm/processor.py
index 62e76e488..9d98be1e9 100644
--- a/auto_round/mllm/processor.py
+++ b/auto_round/compressors/mllm/processor.py
@@ -262,7 +262,7 @@ class CogVLM2Processor(BasicProcessor):
return fetch_image(image_path_or_url).convert("RGB")
-from ..utils import LazyImport
+from auto_round.utils import LazyImport
llava_train = LazyImport("llava.train.train")
diff --git a/auto_round/mllm/template.py b/auto_round/compressors/mllm/template.py
similarity index 99%
rename from auto_round/mllm/template.py
rename to auto_round/compressors/mllm/template.py
index 7f4531e07..b3f9802ba 100644
--- a/auto_round/mllm/template.py
+++ b/auto_round/compressors/mllm/template.py
@@ -18,7 +18,8 @@ from dataclasses import dataclass
from enum import Enum, unique
from typing import Dict, List, Optional
-from ..utils import logger
+from auto_round.logger import logger
+
from .processor import PROCESSORS, BasicProcessor
TEMPLATES: Dict[str, "Template"] = {}
diff --git a/auto_round/mllm/templates/cogvlm2.json b/auto_round/compressors/mllm/templates/cogvlm2.json
similarity index 100%
rename from auto_round/mllm/templates/cogvlm2.json
rename to auto_round/compressors/mllm/templates/cogvlm2.json
diff --git a/auto_round/mllm/templates/default.json b/auto_round/compressors/mllm/templates/default.json
similarity index 100%
rename from auto_round/mllm/templates/default.json
rename to auto_round/compressors/mllm/templates/default.json
diff --git a/auto_round/mllm/templates/llava.json b/auto_round/compressors/mllm/templates/llava.json
similarity index 100%
rename from auto_round/mllm/templates/llava.json
rename to auto_round/compressors/mllm/templates/llava.json
diff --git a/auto_round/mllm/templates/phi3_v.json b/auto_round/compressors/mllm/templates/phi3_v.json
similarity index 100%
rename from auto_round/mllm/templates/phi3_v.json
rename to auto_round/compressors/mllm/templates/phi3_v.json
diff --git a/auto_round/mllm/utils.py b/auto_round/compressors/mllm/utils.py
similarity index 97%
rename from auto_round/mllm/utils.py
rename to auto_round/compressors/mllm/utils.py
index f9a012627..39256f704 100644
--- a/auto_round/mllm/utils.py
+++ b/auto_round/compressors/mllm/utils.py
@@ -16,7 +16,7 @@ import os
import requests
-from ..utils import LazyImport
+from auto_round.utils import LazyImport
PIL = LazyImport("PIL")
from PIL import Image # pylint: disable=E0401
diff --git a/auto_round/eval/eval_cli.py b/auto_round/eval/eval_cli.py
new file mode 100644
index 000000000..82164b5a6
--- /dev/null
+++ b/auto_round/eval/eval_cli.py
@@ -0,0 +1,331 @@
+# Copyright (c) 2025 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import argparse
+import os
+
+from auto_round.utils import (
+ clear_memory,
+ get_device_and_parallelism,
+ get_model_dtype,
+ set_cuda_visible_devices,
+)
+
+
+class EvalArgumentParser(argparse.ArgumentParser):
+
+ def __init__(self, *args, **kwargs):
+ super().__init__(*args, **kwargs)
+ self.add_argument(
+ "--model", "--model_name", "--model_name_or_path", default="facebook/opt-125m", help="model name or path"
+ )
+ self.add_argument("--mllm", action="store_true", help="whether to eval multi-modal model.")
+ self.add_argument(
+ "--device_map",
+ "--device",
+ "--devices",
+ default="0",
+ type=str,
+ help="the device to be used for tuning. "
+ "Currently, device settings support CPU, GPU, and HPU."
+ "The default is set to cuda:0,"
+ "allowing for automatic detection and switch to HPU or CPU."
+ "set --device 0,1,2 to use multiple cards.",
+ )
+
+ self.add_argument(
+ "--tasks",
+ "--task",
+ default="lambada_openai,hellaswag,winogrande,piqa,mmlu,wikitext,truthfulqa_mc1,"
+ "truthfulqa_mc2,openbookqa,boolq,rte,arc_easy,arc_challenge",
+ help="lm-eval tasks",
+ )
+ self.add_argument(
+ "--disable_trust_remote_code", action="store_true", help="whether to disable trust_remote_code"
+ )
+ self.add_argument("--eval_bs", "--bs", "--batch_size", default=None, type=int, help="batch size in evaluation")
+ self.add_argument("--eval_task_by_task", action="store_true", help="whether to eval task by task.")
+ self.add_argument(
+ "--eval_model_dtype", default=None, type=str, help="the torch_dytpe to load the model for evaluation."
+ )
+ self.add_argument(
+ "--limit",
+ type=float,
+ default=None,
+ metavar="N|0<N<1",
+ help="Limit the number of examples per task. "
+ "If <1, limit is a percentage of the total number of examples.",
+ )
+ # vllm related arguments
+ self.add_argument("--revision", default=None, type=str, help="model revision for vllm")
+ self.add_argument("--tokenizer", default=None, type=str, help="tokenizer to use with vllm")
+ self.add_argument(
+ "--tokenizer_mode", default="auto", type=str, help="tokenizer mode for vllm (e.g. auto/fast/slow)"
+ )
+ self.add_argument("--tokenizer_revision", default=None, type=str, help="tokenizer revision for vllm")
+ self.add_argument("--add_bos_token", action="store_true", help="add BOS token when using vllm")
+ self.add_argument("--prefix_token_id", default=None, type=int, help="prefix token id for vllm")
+ self.add_argument("--tensor_parallel_size", default=1, type=int, help="tensor parallel size for vllm")
+ self.add_argument("--data_parallel_size", default=1, type=int, help="data parallel size for vllm")
+ self.add_argument("--quantization", default=None, type=str, help="quantization setting for vllm")
+ self.add_argument("--max_gen_toks", default=256, type=int, help="max generation tokens for vllm")
+ self.add_argument("--swap_space", default=4, type=float, help="swap space (GB) for vllm")
+ self.add_argument("--max_batch_size", default=None, type=int, help="max batch size for vllm")
+ self.add_argument("--max_length", default=None, type=int, help="max generation length for vllm")
+ self.add_argument("--max_model_len", default=None, type=int, help="maximum model sequence length for vllm")
+ self.add_argument(
+ "--gpu_memory_utilization", default=0.9, type=float, help="target GPU memory utilization for vllm"
+ )
+ self.add_argument("--lora_local_path", default=None, type=str, help="local LoRA path for vllm")
+
+
+def _eval_init(tasks, model_path, device, disable_trust_remote_code=False, dtype="auto"):
+ set_cuda_visible_devices(device)
+ device_str, parallelism = get_device_and_parallelism(device)
+ if dtype != "auto":
+ dtype = get_model_dtype(model_dtype=dtype)
+ # ,add_bos_token={True}
+ model_args = f"pretrained={model_path},trust_remote_code={not disable_trust_remote_code},dtype={dtype}"
+ if parallelism:
+ model_args += ",parallelize=True"
+ if isinstance(tasks, str):
+ tasks = tasks.split(",")
+ return tasks, model_args, device_str
+
+
+def eval(args):
+ import time
+
+ tasks, model_args, device_str = _eval_init(
+ args.tasks, args.model, args.device_map, args.disable_trust_remote_code, args.eval_model_dtype
+ )
+
+ # load after _eval_int in order to make sure import torch after set CUDA_VISIBLE_DEVICES
+ from auto_round.eval.evaluation import simple_evaluate, simple_evaluate_user_model
+ from auto_round.utils import logger
+
+ if (batch_size := args.eval_bs) is None:
+ batch_size = "auto:8"
+ is_gguf_file = False
+ if os.path.exists(args.model):
+ if os.path.isfile(args.model) and args.model.endswith(".gguf"):
+ is_gguf_file = True
+ gguf_file = os.path.basename(args.model)
+ model = os.path.dirname(args.model)
+ else:
+ for file in os.listdir(args.model):
+ if file.endswith(".gguf"):
+ is_gguf_file = True
+ gguf_file = file
+ eval_model_dtype = get_model_dtype(args.eval_model_dtype)
+ if is_gguf_file:
+ import torch
+ from lm_eval.utils import make_table # pylint: disable=E0401
+ from transformers import AutoModelForCausalLM, AutoTokenizer
+
+ tokenizer = AutoTokenizer.from_pretrained(model, gguf_file=gguf_file)
+
+ logger.warning("evaluating gguf model is an experimental feature, the accuracy may be not correct.")
+ if eval_model_dtype == "float32" or eval_model_dtype == "auto":
+ logger.warning(
+ "set '--eval_model_dtype bf16' can significantly speed up evaluation for gguf model,"
+ " but may affect accuracy."
+ )
+ model = AutoModelForCausalLM.from_pretrained(
+ model, gguf_file=gguf_file, device_map="auto", torch_dtype=eval_model_dtype
+ )
+ model.eval()
+ st = time.time()
+ res = simple_evaluate_user_model(
+ model, tokenizer, tasks=tasks, batch_size=batch_size, device=device_str, limit=args.limit
+ )
+ print(make_table(res))
+ print("evaluation running time=%ds" % (time.time() - st))
+ else:
+ st = time.time()
+ if "auto" in str(batch_size) and args.mllm:
+ logger.warning("Batch size 'auto' is not yet supported for hf-multimodal models, reset to 16")
+ batch_size = 16
+ res = simple_evaluate(
+ model="hf" if not args.mllm else "hf-multimodal",
+ model_args=model_args,
+ tasks=tasks,
+ device=device_str,
+ batch_size=batch_size,
+ limit=args.limit,
+ )
+ from lm_eval.utils import make_table # pylint: disable=E0401
+
+ print(make_table(res))
+ print("evaluation running time=%ds" % (time.time() - st))
+
+
+def eval_task_by_task(
+ model,
+ device=None,
+ tasks=None,
+ tokenizer=None,
+ batch_size=None,
+ limit=None,
+ max_batch_size=64,
+ trust_remote_code=True,
+ eval_model_dtype=None,
+ retry_times=3,
+):
+ set_cuda_visible_devices(device)
+ device_str, parallelism = get_device_and_parallelism(device)
+
+ # load after _eval_int in order to make sure import torch after set CUDA_VISIBLE_DEVICES
+ import traceback
+
+ from lm_eval import simple_evaluate as lm_simple_evaluate # pylint: disable=E0611
+ from lm_eval.models.huggingface import HFLM
+ from transformers import AutoModelForCausalLM, AutoTokenizer
+
+ from auto_round.utils import logger
+
+ if batch_size is None:
+ batch_size = "auto:8"
+ is_gguf_file = False
+ if not isinstance(model, str):
+ parallelism = False
+ else:
+ if os.path.isfile(model) and model.endswith(".gguf"):
+ is_gguf_file = True
+ gguf_file = os.path.basename(model)
+ model = os.path.dirname(model)
+ else:
+ for file in os.listdir(model):
+ if file.endswith(".gguf"):
+ is_gguf_file = True
+ gguf_file = file
+ eval_model_dtype = get_model_dtype(eval_model_dtype)
+ if is_gguf_file:
+ tokenizer = AutoTokenizer.from_pretrained(model, gguf_file=gguf_file)
+ logger.warning("evaluating gguf model is an experimental feature, the accuracy may be not correct.")
+ if eval_model_dtype == "float32" or eval_model_dtype == "auto":
+ logger.warning(
+ "set '--eval_model_dtype bf16' can significantly speed up evaluation for gguf model,"
+ " but may affect accuracy."
+ )
+
+ model = AutoModelForCausalLM.from_pretrained(
+ model, gguf_file=gguf_file, device_map="auto", torch_dtype=eval_model_dtype
+ )
+ model.eval()
+ parallelism = False
+ hflm = HFLM(
+ pretrained=model,
+ tokenizer=tokenizer,
+ device=device_str,
+ batch_size=batch_size,
+ max_batch_size=max_batch_size,
+ parallelize=parallelism,
+ trust_remote_code=trust_remote_code,
+ dtype=eval_model_dtype,
+ )
+
+ if isinstance(tasks, str):
+ tasks = tasks.replace(" ", "").split(",")
+
+ from lm_eval.utils import make_table # pylint: disable=E0611
+
+ res_all = {}
+ res_keys = ["results", "versions", "n-shot", "higher_is_better"]
+ import time
+
+ st = time.time()
+ for task in tasks:
+ while retry_times:
+ try:
+ res = lm_simple_evaluate(
+ model=hflm, model_args=None, device=device_str, tasks=task, batch_size=batch_size, limit=limit
+ )
+ break
+ except Exception as e:
+ cuda_error_msg = traceback.format_exc()
+ try:
+ ori_batch_sizes = hflm.batch_sizes if hflm.batch_sizes else {"0": 64}
+ try:
+ for k, v in hflm.batch_sizes.items():
+ hflm.batch_sizes[k] = max(v // 2, 1)
+ logger.warning(f"Out of memory, reset batch_size to {hflm.batch_sizes} and re-try.")
+ res = lm_simple_evaluate(
+ model=hflm, model_args=None, device=device_str, tasks=task, batch_size=1, limit=limit
+ )
+ hflm.batch_sizes = ori_batch_sizes
+ except Exception as e:
+ traceback.print_exc()
+ pass
+ except Exception as e:
+ logger.error(cuda_error_msg)
+ traceback.print_exc()
+ break
+ retry_times -= 1
+
+ if not res_all:
+ res_all = res
+ else:
+ for key in res_keys:
+ res_all[key].update(res[key])
+ print(make_table(res_all))
+
+ print("total eval time:", time.time() - st)
+
+
+def eval_with_vllm(args):
+ import time
+
+ from lm_eval import evaluator # pylint: disable=E0401
+ from lm_eval.models.vllm_causallms import VLLM # pylint: disable=E0401
+ from lm_eval.utils import make_table # pylint: disable=E0401
+
+ st = time.time()
+ os.environ["TOKENIZERS_PARALLELISM"] = "false"
+ device_str, _ = get_device_and_parallelism(args.device)
+ eval_model_dtype = get_model_dtype(args.eval_model_dtype, "auto")
+ if (batch_size := args.eval_bs) is None:
+ batch_size = "auto:8"
+
+ vllm_lm = VLLM(
+ pretrained=args.model,
+ dtype=eval_model_dtype,
+ revision=args.revision,
+ trust_remote_code=not args.disable_trust_remote_code,
+ tokenizer=args.tokenizer,
+ tokenizer_mode=args.tokenizer_mode,
+ tokenizer_revision=args.tokenizer_revision,
+ add_bos_token=args.add_bos_token,
+ prefix_token_id=args.prefix_token_id,
+ tensor_parallel_size=args.tensor_parallel_size,
+ quantization=args.quantization,
+ max_gen_toks=args.max_gen_toks,
+ swap_space=args.swap_space,
+ batch_size=batch_size,
+ max_batch_size=args.max_batch_size,
+ max_length=args.max_length,
+ max_model_len=args.max_model_len,
+ seed=args.seed,
+ gpu_memory_utilization=args.gpu_memory_utilization,
+ device=device_str,
+ data_parallel_size=args.data_parallel_size,
+ lora_local_path=args.lora_local_path,
+ )
+ res = evaluator.simple_evaluate(
+ model=vllm_lm,
+ tasks=args.tasks,
+ limit=args.limit,
+ )
+
+ print(make_table(res))
+ print("evaluation running time=%ds" % (time.time() - st))
diff --git a/auto_round/export/export_to_autoround/export.py b/auto_round/export/export_to_autoround/export.py
index 8877f4d4b..26d721ec0 100644
--- a/auto_round/export/export_to_autoround/export.py
+++ b/auto_round/export/export_to_autoround/export.py
@@ -275,7 +275,6 @@ def save_quantized_as_autoround(output_dir, inplace=True, backend="auto_round:ex
from auto_round.export.export_to_autoround.export_to_fp8 import save_quantized_as_autoround
return save_quantized_as_autoround(output_dir, inplace=inplace, backend="auto_round", **kwargs)
- from auto_round.autoround import AutoRoundFormat
##if using sym, we change to gptq sym kernel to avoid compiling from auto_round source
if (
diff --git a/auto_round/logger.py b/auto_round/logger.py
index 73f891315..a43be3167 100644
--- a/auto_round/logger.py
+++ b/auto_round/logger.py
@@ -13,10 +13,14 @@
# limitations under the License.
import logging
-from functools import lru_cache
+import warnings
+from functools import lru_cache, wraps
+from typing import TYPE_CHECKING, Any, Callable, Dict, List, Mapping, Optional, TypeVar
import auto_round.envs as envs
+T = TypeVar("T", bound="Callable") # used by `deprecated`
+
@lru_cache(maxsize=None)
def warning_once(self, msg, *args):
@@ -84,3 +88,29 @@ logger.propagate = False
fh = logging.StreamHandler()
fh.setFormatter(AutoRoundFormatter())
logger.addHandler(fh)
+
+
+def deprecated(future_name: Optional[str] = None, message: Optional[str] = None) -> Callable[[T], T]:
+ """
+ Decorator to mark functions as deprecated
+
+ :param new_function: Function called in place of deprecated function
+ :param message: Deprecation message, replaces default deprecation message
+ """
+
+ def decorator(func: T) -> T:
+ nonlocal message
+
+ if message is None:
+ message = f"{func.__name__} is deprecated and will be removed in a future release"
+ if future_name is not None:
+ message += f". Please use {future_name} instead."
+
+ @wraps(func)
+ def wrapped(*args, **kwargs):
+ logger.warning_once(message)
+ return func(*args, **kwargs)
+
+ return wrapped
+
+ return decorator
diff --git a/auto_round/script/llm.py b/auto_round/script/llm.py
deleted file mode 100644
index f489bd12e..000000000
--- a/auto_round/script/llm.py
+++ /dev/null
@@ -1,1021 +0,0 @@
-# Copyright (c) 2024 Intel Corporation
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import argparse
-import logging
-
-# Copyright (c) 2024 Intel Corporation
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-import os
-import re
-import sys
-
-from auto_round.schemes import PRESET_SCHEMES
-from auto_round.utils import (
- clear_memory,
- get_device_and_parallelism,
- get_fp_layer_names,
- get_model_dtype,
- infer_bits_by_data_type,
- set_cuda_visible_devices,
-)
-
-
-class BasicArgumentParser(argparse.ArgumentParser):
-
- def __init__(self, *args, **kwargs):
- super().__init__(*args, **kwargs)
- self.add_argument(
- "--model", "--model_name", "--model_name_or_path", default="facebook/opt-125m", help="model name or path"
- )
-
- self.add_argument("--mllm", action="store_true", help="whether to quant multi-modal model.")
-
- self.add_argument("--eval", action="store_true", help="whether to use eval only mode")
-
- self.add_argument(
- "--scheme",
- default="W4A16",
- type=str,
- # choices=["W4A16", "W2A16", "W3A16", "W8A16", "MXFP4", "MXFP8", "NVFP4", "FPW8A16", "FP8_STATIC"],
- help="quantization scheme",
- )
-
- self.add_argument("--bits", default=None, type=int, help="number of weight bits")
- self.add_argument("--group_size", default=None, type=int, help="group size")
- self.add_argument("--asym", action="store_true", help="whether to use asym quantization")
- self.add_argument("--data_type", "--dtype", default=None, help="data type for tuning, 'int', 'mx_fp' and etc")
- self.add_argument("--act_bits", default=None, type=int, help="activation bits")
- self.add_argument("--act_group_size", default=None, type=int, help="activation group size")
- self.add_argument(
- "--super_group_size", default=None, type=int, help="the number of super group size when use double quant."
- )
-
- self.add_argument(
- "--super_bits", default=None, type=int, help="number of scale and mins quant bits for double quant."
- )
- self.add_argument("--act_data_type", "--act_dtype", default=None, type=str, help="activation data type")
-
- self.add_argument("--disable_act_dynamic", action="store_true", help="activation static quantization")
-
- self.add_argument(
- "--device_map",
- "--device",
- "--devices",
- default="0",
- type=str,
- help="the device to be used for tuning. "
- "Currently, device settings support CPU, GPU, and HPU."
- "The default is set to cuda:0,"
- "allowing for automatic detection and switch to HPU or CPU."
- "set --device 0,1,2 to use multiple cards.",
- )
-
- self.add_argument(
- "--dataset", default="NeelNanda/pile-10k", type=str, help="the dataset for quantization training"
- )
-
- self.add_argument(
- "--minmax_lr",
- default=None,
- type=float,
- help="minmax learning rate, if None, it will beset to be the same with lr",
- )
-
- self.add_argument(
- "--mem_per_param_scale",
- default=13,
- type=float,
- help="Scale factor for memory per parameter, used to adjust memory usage estimation for tuning",
- )
-
- self.add_argument("--seed", default=42, type=int, help="random seed")
-
- self.add_argument("--adam", action="store_true", help="whether to use adam optimizer instead of SignSGD")
-
- self.add_argument("--gradient_accumulate_steps", default=1, type=int, help="gradient accumulate steps")
-
- self.add_argument("--nblocks", default=1, type=int, help="how many blocks to tune together")
-
- self.add_argument("--low_gpu_mem_usage", action="store_true", help="offload intermediate features to cpu")
-
- self.add_argument("--format", default="auto_round", type=str, help="the format to save the model")
-
- self.add_argument(
- "--scale_dtype",
- default="fp16",
- choices=["fp16", "float16", "bf16", "bfloat16", "fp32", "float32"],
- help="scale data type to use for quantization",
- )
-
- self.add_argument(
- "--output_dir", default="./tmp_autoround", type=str, help="the directory to save quantized model"
- )
-
- self.add_argument("--disable_amp", action="store_true", help="disable amp")
-
- self.add_argument(
- "--disable_minmax_tuning", action="store_true", help="whether to disable enable weight minmax tuning"
- )
-
- self.add_argument("--enable_norm_bias_tuning", action="store_true", help="whether to enable norm bias tuning")
-
- self.add_argument(
- "--disable_trust_remote_code", action="store_true", help="whether to disable trust_remote_code"
- )
-
- self.add_argument(
- "--disable_quanted_input",
- action="store_true",
- help="whether to disuse the output of quantized block to tune the next block",
- )
-
- self.add_argument("--quant_lm_head", action="store_true", help="whether to quant lm_head")
-
- self.add_argument(
- "--low_cpu_mem_mode",
- default=0,
- type=int,
- choices=[0, 1, 2],
- help="choose which low cpu memory mode to use. "
- "Can significantly reduce cpu memory footprint but cost more time."
- "1 means choose block-wise mode, load the weights of each block"
- " from disk when tuning and release the memory of the block after tuning."
- "2 means choose layer-wise mode, load the weights of each layer from disk when tuning,"
- " minimum memory consumption and also slowest running speed."
- "others means not use low cpu memory. Default to 0, not use low cpu memory.",
- )
-
- self.add_argument(
- "--low_cpu_mem_tmp_dir",
- default=None,
- type=str,
- help="temporary work space to store the temporary files "
- "when using low cpu memory mode. Will remove after tuning.",
- )
-
- self.add_argument(
- "--model_dtype",
- default=None,
- type=str,
- choices=["fp16", "float16", "bf16", "bfloat16", "fp32", "float32"],
- help="force to convert the dtype, some backends supports fp16 dtype better",
- )
-
- self.add_argument(
- "--fp_layers", default="", type=str, help="list of Layer names to maintain original data type"
- )
-
- self.add_argument(
- "--not_use_best_mse",
- action="store_true",
- help="whether to use the iter of best mes loss in the tuning phase",
- )
-
- self.add_argument(
- "--to_quant_block_names",
- default=None,
- type=str,
- help="Names of quantitative blocks, please use commas to separate them.",
- )
-
- self.add_argument("--enable_torch_compile", action="store_true", help="whether to enable torch compile")
-
- self.add_argument("--enable_alg_ext", action="store_true", help="whether to enable probably better algorithm")
-
- self.add_argument(
- "--disable_deterministic_algorithms",
- action="store_true",
- help="deprecated, disable torch deterministic algorithms.",
- )
- self.add_argument(
- "--enable_deterministic_algorithms", action="store_true", help="enable torch deterministic algorithms."
- )
-
- self.add_argument(
- "--disable_opt_rtn",
- action="store_true",
- help="whether to disable optimization of the RTN mode(iters=0) (default is False).",
- )
-
- ## ======================= MLLM =======================
- self.add_argument(
- "--quant_nontext_module",
- action="store_true",
- help="whether to quantize non-text module, e.g. vision component",
- )
-
- self.add_argument(
- "--extra_data_dir",
- default=None,
- type=str,
- help="dataset dir for storing images/audio/videos. "
- "Can be a dir path or multiple dir path with format as "
- "'image=path_to_image,video=path_to_video,audio=path_to_audio'"
- "By default, it will search in the relative path, "
- "and if not find, will automatic download.",
- )
-
- self.add_argument(
- "--template",
- default=None,
- type=str,
- help="the template for building training dataset. It can be a custom one.",
- )
-
- ## ======================= eval =======================
- self.add_argument(
- "--disable_eval", action="store_true", help="whether to disable lm-eval evaluation after tuning"
- )
-
- self.add_argument(
- "--tasks",
- "--task",
- nargs="?",
- const="lambada_openai,hellaswag,winogrande,piqa,mmlu,wikitext,truthfulqa_mc1,"
- "openbookqa,boolq,arc_easy,arc_challenge",
- default=None,
- help="lm-eval tasks",
- )
-
- self.add_argument("--eval_bs", default=None, type=int, help="batch size in evaluation")
-
- self.add_argument(
- "--limit",
- type=float,
- default=None,
- metavar="N|0<N<1",
- help="Limit the number of examples per task. "
- "If <1, limit is a percentage of the total number of examples.",
- )
-
- self.add_argument("--eval_task_by_task", action="store_true", help="whether to eval task by task.")
-
- self.add_argument(
- "--eval_model_dtype", default=None, type=str, help="the torch_dytpe to load the model for evaluation."
- )
-
-
-class EvalArgumentParser(argparse.ArgumentParser):
-
- def __init__(self, *args, **kwargs):
- super().__init__(*args, **kwargs)
- self.add_argument(
- "--model", "--model_name", "--model_name_or_path", default="facebook/opt-125m", help="model name or path"
- )
- self.add_argument("--mllm", action="store_true", help="whether to eval multi-modal model.")
- self.add_argument(
- "--device_map",
- "--device",
- "--devices",
- default="0",
- type=str,
- help="the device to be used for tuning. "
- "Currently, device settings support CPU, GPU, and HPU."
- "The default is set to cuda:0,"
- "allowing for automatic detection and switch to HPU or CPU."
- "set --device 0,1,2 to use multiple cards.",
- )
-
- self.add_argument(
- "--tasks",
- "--task",
- default="lambada_openai,hellaswag,winogrande,piqa,mmlu,wikitext,truthfulqa_mc1,"
- "truthfulqa_mc2,openbookqa,boolq,rte,arc_easy,arc_challenge",
- help="lm-eval tasks",
- )
- self.add_argument(
- "--disable_trust_remote_code", action="store_true", help="whether to disable trust_remote_code"
- )
- self.add_argument("--eval_bs", "--bs", "--batch_size", default=None, type=int, help="batch size in evaluation")
- self.add_argument("--eval_task_by_task", action="store_true", help="whether to eval task by task.")
- self.add_argument(
- "--eval_model_dtype", default=None, type=str, help="the torch_dytpe to load the model for evaluation."
- )
- self.add_argument(
- "--limit",
- type=float,
- default=None,
- metavar="N|0<N<1",
- help="Limit the number of examples per task. "
- "If <1, limit is a percentage of the total number of examples.",
- )
- # vllm related arguments
- self.add_argument("--revision", default=None, type=str, help="model revision for vllm")
- self.add_argument("--tokenizer", default=None, type=str, help="tokenizer to use with vllm")
- self.add_argument(
- "--tokenizer_mode", default="auto", type=str, help="tokenizer mode for vllm (e.g. auto/fast/slow)"
- )
- self.add_argument("--tokenizer_revision", default=None, type=str, help="tokenizer revision for vllm")
- self.add_argument("--add_bos_token", action="store_true", help="add BOS token when using vllm")
- self.add_argument("--prefix_token_id", default=None, type=int, help="prefix token id for vllm")
- self.add_argument("--tensor_parallel_size", default=1, type=int, help="tensor parallel size for vllm")
- self.add_argument("--data_parallel_size", default=1, type=int, help="data parallel size for vllm")
- self.add_argument("--quantization", default=None, type=str, help="quantization setting for vllm")
- self.add_argument("--max_gen_toks", default=256, type=int, help="max generation tokens for vllm")
- self.add_argument("--swap_space", default=4, type=float, help="swap space (GB) for vllm")
- self.add_argument("--max_batch_size", default=None, type=int, help="max batch size for vllm")
- self.add_argument("--max_length", default=None, type=int, help="max generation length for vllm")
- self.add_argument("--max_model_len", default=None, type=int, help="maximum model sequence length for vllm")
- self.add_argument(
- "--gpu_memory_utilization", default=0.9, type=float, help="target GPU memory utilization for vllm"
- )
- self.add_argument("--lora_local_path", default=None, type=str, help="local LoRA path for vllm")
-
-
-def setup_parser():
- parser = BasicArgumentParser()
-
- parser.add_argument("--batch_size", "--train_bs", "--bs", default=8, type=int, help="train batch size")
-
- parser.add_argument("--iters", "--iter", default=200, type=int, help="iteration to tune each block")
-
- parser.add_argument(
- "--seqlen", "--seq_len", default=2048, type=int, help="sequence length of the calibration samples"
- )
-
- parser.add_argument("--nsamples", "--nsample", default=128, type=int, help="number of samples")
-
- parser.add_argument(
- "--lr", default=None, type=float, help="learning rate, if None, it will be set to 1.0/iters automatically"
- )
-
- args = parser.parse_args()
- return args
-
-
-def setup_best_parser():
- parser = BasicArgumentParser()
-
- parser.add_argument("--batch_size", "--train_bs", "--bs", default=8, type=int, help="train batch size")
-
- parser.add_argument("--iters", "--iter", default=1000, type=int, help="iterations to tune each block")
-
- parser.add_argument(
- "--seqlen", "--seq_len", default=2048, type=int, help="sequence length of the calibration samples"
- )
-
- parser.add_argument("--nsamples", "--nsample", default=512, type=int, help="number of samples")
-
- parser.add_argument(
- "--lr", default=None, type=float, help="learning rate, if None, it will be set to 1.0/iters automatically"
- )
-
- args = parser.parse_args()
- args.low_gpu_mem_usage = True
-
- return args
-
-
-def setup_light_parser():
- parser = BasicArgumentParser()
-
- parser.add_argument("--batch_size", "--train_bs", "--bs", default=8, type=int, help="train batch size")
-
- parser.add_argument("--iters", "--iter", default=50, type=int, help="iterations to tune each block")
-
- parser.add_argument(
- "--seqlen", "--seq_len", default=2048, type=int, help="sequence length of the calibration samples"
- )
-
- parser.add_argument("--nsamples", "--nsample", default=128, type=int, help="number of samples")
-
- parser.add_argument(
- "--lr", default=5e-3, type=float, help="learning rate, if None, it will be set to 1.0/iters automatically"
- )
-
- args = parser.parse_args()
-
- return args
-
-
-def setup_fast_parser():
- parser = BasicArgumentParser()
-
- parser.add_argument("--batch_size", "--train_bs", "--bs", default=4, type=int, help="train batch size")
-
- parser.add_argument("--iters", default=200, type=int, help="iterations to tune each block")
-
- parser.add_argument(
- "--seqlen", "--seq_len", default=512, type=int, help="sequence length of the calibration samples"
- )
-
- parser.add_argument("--nsamples", "--nsample", default=128, type=int, help="number of samples")
-
- parser.add_argument(
- "--lr", default=None, type=float, help="learning rate, if None, it will be set to 1.0/iters automatically"
- )
-
- args = parser.parse_args()
-
- return args
-
-
-def setup_eval_parser():
- parser = EvalArgumentParser()
- args = parser.parse_args()
- return args
-
-
-def tune(args):
- if args.disable_eval:
- logging.warning("`disable_eval` is deprecated and is now set by default.")
-
- if args.eval_bs is None:
- args.eval_bs = "auto"
- from transformers import AutoConfig, AutoModelForCausalLM, AutoTokenizer
-
- from auto_round.utils import detect_device, get_library_version, logger
-
- if args.format is None:
- args.format = "auto_round"
-
- formats = args.format.lower().replace(" ", "").split(",")
- from auto_round.utils import SUPPORTED_FORMATS
-
- for format in formats:
- if format not in SUPPORTED_FORMATS:
- raise ValueError(f"{format} is not supported, we only support {SUPPORTED_FORMATS}")
-
- if "auto_gptq" in args.format and args.asym is True:
- logger.warning(
- "the auto_gptq kernel has issues with asymmetric quantization. "
- "It is recommended to use sym quantization or --format='auto_round'"
- )
-
- if "marlin" in args.format and args.asym is True:
- raise RuntimeError("marlin backend only supports sym quantization, please remove --asym")
-
- # Must set this before import torch
- # set_cuda_visible_devices(args.device_map)
- device_str, use_auto_mapping = get_device_and_parallelism(args.device_map)
-
- import torch
-
- if args.enable_torch_compile:
- logger.info(
- "`torch.compile` is enabled to reduce tuning costs. "
- "If it causes issues, you can disable it by removing `--enable_torch_compile` argument."
- )
-
- model_name = args.model
- if model_name[-1] == "/":
- model_name = model_name[:-1]
- logger.info(f"start to quantize {model_name}")
-
- from auto_round.utils import llm_load_model, mllm_load_model
-
- if args.mllm:
- model, processor, tokenizer, image_processor = mllm_load_model(
- model_name,
- device="cpu",
- use_auto_mapping=False,
- trust_remote_code=not args.disable_trust_remote_code,
- model_dtype=args.model_dtype,
- )
- low_cpu_mem_usage = False
- else:
- model, tokenizer, low_cpu_mem_usage = llm_load_model(
- model_name,
- use_auto_mapping=False,
- trust_remote_code=not args.disable_trust_remote_code,
- device="cpu",
- low_cpu_mem_mode=args.low_cpu_mem_mode,
- low_cpu_mem_tmp_dir=args.low_cpu_mem_tmp_dir,
- model_dtype=args.model_dtype,
- )
-
- from auto_round import AutoRound, AutoRoundAdam, AutoRoundMLLM
-
- if "bloom" in model_name:
- args.low_gpu_mem_usage = False
-
- round = AutoRound
- mllm_kwargs = {}
- if args.adam:
- round = AutoRoundAdam
- if args.mllm:
- round = AutoRoundMLLM
- mllm_kwargs = {
- "processor": processor,
- "image_processor": image_processor,
- "extra_data_dir": args.extra_data_dir,
- "quant_nontext_module": args.quant_nontext_module,
- "template": args.template,
- }
-
- layer_config = {}
- not_quantize_layer_names = get_fp_layer_names(model, args.fp_layers)
- for name in not_quantize_layer_names:
- layer_config[name] = {"bits": 16, "act_bits": 16, "data_type": "float", "act_data_type": "float"}
- if len(not_quantize_layer_names) > 0:
- logger.info(f"{not_quantize_layer_names} will not be quantized.")
- for format in formats:
- if (
- "auto_round" not in format
- and "fake" not in format
- and "awq" not in format
- and "llm_compressor" not in format
- ):
- # TODO gptq could support some mixed precision config
- logger.warning(f"mixed precision exporting does not support {format} currently")
-
- if args.quant_lm_head:
- for format in formats:
- if "auto_round" not in format and "fake" not in format:
- auto_round_formats = [s for s in SUPPORTED_FORMATS if s.startswith("auto_round")]
- raise ValueError(
- f"{format} is not supported for lm-head quantization, please change to {auto_round_formats}"
- )
-
- if "auto_awq" in args.format:
- from auto_round.utils import check_awq_gemm_compatibility
-
- awq_supported, info = check_awq_gemm_compatibility(
- model, args.bits, args.group_size, not args.asym, layer_config
- )
- if not awq_supported:
- logger.warning(f"The AutoAWQ format may not be supported due to {info}")
-
- enable_torch_compile = True if "--enable_torch_compile" in sys.argv else False
- sym = None # the default value should be None now
- if args.asym: # if the scheme is asym, how to set it to sym is an issue
- sym = False
- act_dynamic = None
- if args.disable_act_dynamic:
- act_dynamic = False
- scheme = args.scheme.upper()
- if scheme not in PRESET_SCHEMES:
- raise ValueError(f"{scheme} is not supported. only {PRESET_SCHEMES.keys()} are supported ")
- if args.disable_deterministic_algorithms:
- logger.warning(
- "default not use deterministic_algorithms. disable_deterministic_algorithms is deprecated,"
- " please use enable_deterministic_algorithms instead. "
- )
- autoround = round(
- model=model,
- tokenizer=tokenizer,
- scheme=scheme,
- bits=args.bits,
- group_size=args.group_size,
- sym=sym,
- batch_size=args.batch_size,
- dataset=args.dataset,
- seqlen=args.seqlen,
- nblocks=args.nblocks,
- iters=args.iters,
- lr=args.lr,
- minmax_lr=args.minmax_lr,
- enable_quanted_input=not args.disable_quanted_input,
- amp=not args.disable_amp,
- nsamples=args.nsamples,
- seed=args.seed,
- low_gpu_mem_usage=args.low_gpu_mem_usage,
- scale_dtype=args.scale_dtype,
- gradient_accumulate_steps=args.gradient_accumulate_steps,
- layer_config=layer_config,
- enable_minmax_tuning=not args.disable_minmax_tuning,
- act_bits=args.act_bits,
- act_group_size=args.act_group_size,
- low_cpu_mem_usage=args.low_cpu_mem_mode,
- data_type=args.data_type,
- enable_norm_bias_tuning=args.enable_norm_bias_tuning,
- not_use_best_mse=args.not_use_best_mse,
- to_quant_block_names=args.to_quant_block_names,
- enable_torch_compile=enable_torch_compile,
- act_data_type=args.act_data_type,
- act_dynamic=act_dynamic,
- device_map=args.device_map,
- super_group_size=args.super_group_size,
- super_bits=args.super_bits,
- disable_opt_rtn=args.disable_opt_rtn,
- enable_deterministic_algorithms=args.enable_deterministic_algorithms,
- enable_alg_ext=args.enable_alg_ext,
- **mllm_kwargs,
- )
-
- model_name = args.model.rstrip("/")
-
- if model_name.split("/")[-1].strip(".") == "" and "gguf" not in args.format:
- if autoround.group_size <= 0:
- if "fp" in autoround.act_data_type:
- suffix = f"afp{autoround.act_bits}"
- else:
- suffix = f"a{autoround.act_bits}"
- else:
- suffix = f"g{autoround.group_size}"
- export_dir = os.path.join(args.output_dir, f"w{autoround.bits}{suffix}")
- elif model_name.split("/")[-1].strip(".") == "" and "gguf" in args.format:
- export_dir = args.output_dir
- elif model_name.split("./")[-1].strip("./") != "" and "gguf" in args.format:
- export_dir = os.path.join(args.output_dir, model_name.split("/")[-1] + "-gguf")
- else:
- if autoround.group_size <= 0:
- if "fp" in autoround.act_data_type:
- suffix = f"afp{autoround.act_bits}"
- else:
- suffix = f"a{autoround.act_bits}"
- else:
- suffix = f"g{autoround.group_size}"
- export_dir = os.path.join(args.output_dir, model_name.split("/")[-1] + f"-w{autoround.bits}{suffix}")
-
- model, folders = autoround.quantize_and_save(export_dir, format=args.format)
-
- if args.low_cpu_mem_mode == 1 or args.low_cpu_mem_mode == 2:
- import shutil
-
- shutil.rmtree(args.low_cpu_mem_tmp_dir, ignore_errors=True)
-
- model.eval()
- clear_memory()
-
- lm_eval_version = get_library_version("lm-eval")
-
- eval_folder = folders[-1]
- if args.tasks is None or args.tasks == "" or eval_folder is None:
- return
-
- tasks = args.tasks
- if isinstance(tasks, str):
- tasks = tasks.split(",")
-
- from lm_eval.utils import make_table # pylint: disable=E0401
-
- logger.info(f"Using lm-eval version {lm_eval_version}")
- eval_gguf_model = False
- for file in os.listdir(eval_folder):
- if file.endswith("gguf"):
- eval_gguf_model = True
- break
-
- import time
-
- eval_model_dtype = get_model_dtype(args.eval_model_dtype, "auto")
-
- if autoround.act_bits <= 8 or eval_gguf_model:
- if eval_gguf_model:
- # for file in os.listdir(eval_folder):
- # gguf_file = file
- gguf_file = None
- gguf_format = None # Initialize gguf_format to None
- # gguf folder only contains one file
- for format in formats:
- if format.startswith("gguf"):
- gguf_format = format.split(":")[-1].upper()
- if gguf_format is None: # Validate gguf_format after the loop
- logger.error("No valid gguf format found in formats. Please check the input.")
- sys.exit(-1)
- for file in os.listdir(eval_folder):
- if gguf_format in file:
- gguf_file = file
-
- logger.warning("evaluating gguf model is an experimental feature, the accuracy may be not correct.")
- if eval_model_dtype == "float32" or eval_model_dtype == "auto":
- logger.warning(
- "set '--eval_model_dtype bf16' can significantly speed up evaluation for gguf model,"
- " but may affect accuracy."
- )
- if gguf_file is None:
- logger.error("Cannot find correct gguf file for evaluation, please check.")
- sys.exit(-1)
- model = AutoModelForCausalLM.from_pretrained(
- eval_folder, gguf_file=gguf_file, device_map="auto", torch_dtype=eval_model_dtype
- )
- model.eval()
- tokenizer = AutoTokenizer.from_pretrained(eval_folder, gguf_file=gguf_file)
- else:
- if hasattr(model, "hf_device_map") and len(model.hf_device_map) > 1:
- from accelerate.big_modeling import dispatch_model
-
- dispatch_model(model, model.hf_device_map)
- else:
- device_str = detect_device(device_str)
- model = model.to(device_str)
- if model.dtype != eval_model_dtype and eval_model_dtype != "auto":
- model.to(getattr(torch, eval_model_dtype))
-
- if args.eval_task_by_task:
- eval_task_by_task(
- model,
- tokenizer=tokenizer,
- device=device_str,
- tasks=args.tasks,
- limit=args.limit,
- batch_size=args.eval_bs,
- eval_model_dtype=eval_model_dtype,
- )
- else:
- if args.eval_bs is None or args.eval_bs == "auto":
- logger.warning("This API does not support auto currently, reset eval_bs to 16")
- args.eval_bs = 16
- from auto_round.eval.evaluation import simple_evaluate_user_model
-
- st = time.time()
- add_bos_token = False
- if "llama" in args.model.lower():
- add_bos_token = True
- res = simple_evaluate_user_model(
- model,
- tokenizer,
- tasks=tasks,
- batch_size=args.eval_bs,
- limit=args.limit,
- device=device_str,
- eval_model_dtype=eval_model_dtype,
- add_bos_token=add_bos_token,
- )
- print(make_table(res))
- print("evaluation running time=%ds" % (time.time() - st))
- else:
- if args.eval_task_by_task:
- eval_task_by_task(
- eval_folder,
- device=device_str,
- tasks=args.tasks,
- batch_size=args.eval_bs,
- limit=args.limit,
- eval_model_dtype=eval_model_dtype,
- )
- else:
- from auto_round.eval.evaluation import simple_evaluate
-
- tasks, model_args, device_str = _eval_init(
- args.tasks, eval_folder, args.device_map, args.disable_trust_remote_code, dtype=eval_model_dtype
- )
- st = time.time()
- if "llama" in args.model.lower():
- model_args += ",add_bos_token=True"
- res = simple_evaluate(
- model="hf",
- model_args=model_args,
- tasks=tasks,
- device=device_str,
- batch_size=args.eval_bs,
- limit=args.limit,
- )
- print(make_table(res))
- print("evaluation running time=%ds" % (time.time() - st))
-
-
-def _eval_init(tasks, model_path, device, disable_trust_remote_code=False, dtype="auto"):
- set_cuda_visible_devices(device)
- device_str, parallelism = get_device_and_parallelism(device)
- if dtype != "auto":
- dtype = get_model_dtype(model_dtype=dtype)
- # ,add_bos_token={True}
- model_args = f"pretrained={model_path},trust_remote_code={not disable_trust_remote_code},dtype={dtype}"
- if parallelism:
- model_args += ",parallelize=True"
- if isinstance(tasks, str):
- tasks = tasks.split(",")
- return tasks, model_args, device_str
-
-
-def eval(args):
- import time
-
- tasks, model_args, device_str = _eval_init(
- args.tasks, args.model, args.device_map, args.disable_trust_remote_code, args.eval_model_dtype
- )
-
- # load after _eval_int in order to make sure import torch after set CUDA_VISIBLE_DEVICES
- from auto_round.eval.evaluation import simple_evaluate, simple_evaluate_user_model
- from auto_round.utils import logger
-
- if (batch_size := args.eval_bs) is None:
- batch_size = "auto:8"
- is_gguf_file = False
- if os.path.exists(args.model):
- if os.path.isfile(args.model) and args.model.endswith(".gguf"):
- is_gguf_file = True
- gguf_file = os.path.basename(args.model)
- model = os.path.dirname(args.model)
- else:
- for file in os.listdir(args.model):
- if file.endswith(".gguf"):
- is_gguf_file = True
- gguf_file = file
- eval_model_dtype = get_model_dtype(args.eval_model_dtype)
- if is_gguf_file:
- import torch
- from lm_eval.utils import make_table # pylint: disable=E0401
- from transformers import AutoModelForCausalLM, AutoTokenizer
-
- tokenizer = AutoTokenizer.from_pretrained(model, gguf_file=gguf_file)
-
- logger.warning("evaluating gguf model is an experimental feature, the accuracy may be not correct.")
- if eval_model_dtype == "float32" or eval_model_dtype == "auto":
- logger.warning(
- "set '--eval_model_dtype bf16' can significantly speed up evaluation for gguf model,"
- " but may affect accuracy."
- )
- model = AutoModelForCausalLM.from_pretrained(
- model, gguf_file=gguf_file, device_map="auto", torch_dtype=eval_model_dtype
- )
- model.eval()
- st = time.time()
- res = simple_evaluate_user_model(
- model, tokenizer, tasks=tasks, batch_size=batch_size, device=device_str, limit=args.limit
- )
- print(make_table(res))
- print("evaluation running time=%ds" % (time.time() - st))
- else:
- st = time.time()
- if "auto" in str(batch_size) and args.mllm:
- logger.warning("Batch size 'auto' is not yet supported for hf-multimodal models, reset to 16")
- batch_size = 16
- res = simple_evaluate(
- model="hf" if not args.mllm else "hf-multimodal",
- model_args=model_args,
- tasks=tasks,
- device=device_str,
- batch_size=batch_size,
- limit=args.limit,
- )
- from lm_eval.utils import make_table # pylint: disable=E0401
-
- print(make_table(res))
- print("evaluation running time=%ds" % (time.time() - st))
-
-
-def eval_task_by_task(
- model,
- device=None,
- tasks=None,
- tokenizer=None,
- batch_size=None,
- limit=None,
- max_batch_size=64,
- trust_remote_code=True,
- eval_model_dtype=None,
- retry_times=3,
-):
- set_cuda_visible_devices(device)
- device_str, parallelism = get_device_and_parallelism(device)
-
- # load after _eval_int in order to make sure import torch after set CUDA_VISIBLE_DEVICES
- import traceback
-
- from lm_eval import simple_evaluate as lm_simple_evaluate # pylint: disable=E0611
- from lm_eval.models.huggingface import HFLM
- from transformers import AutoModelForCausalLM, AutoTokenizer
-
- from auto_round.utils import logger
-
- if batch_size is None:
- batch_size = "auto:8"
- is_gguf_file = False
- if not isinstance(model, str):
- parallelism = False
- else:
- if os.path.isfile(model) and model.endswith(".gguf"):
- is_gguf_file = True
- gguf_file = os.path.basename(model)
- model = os.path.dirname(model)
- else:
- for file in os.listdir(model):
- if file.endswith(".gguf"):
- is_gguf_file = True
- gguf_file = file
- eval_model_dtype = get_model_dtype(eval_model_dtype)
- if is_gguf_file:
- tokenizer = AutoTokenizer.from_pretrained(model, gguf_file=gguf_file)
- logger.warning("evaluating gguf model is an experimental feature, the accuracy may be not correct.")
- if eval_model_dtype == "float32" or eval_model_dtype == "auto":
- logger.warning(
- "set '--eval_model_dtype bf16' can significantly speed up evaluation for gguf model,"
- " but may affect accuracy."
- )
-
- model = AutoModelForCausalLM.from_pretrained(
- model, gguf_file=gguf_file, device_map="auto", torch_dtype=eval_model_dtype
- )
- model.eval()
- parallelism = False
- hflm = HFLM(
- pretrained=model,
- tokenizer=tokenizer,
- device=device_str,
- batch_size=batch_size,
- max_batch_size=max_batch_size,
- parallelize=parallelism,
- trust_remote_code=trust_remote_code,
- dtype=eval_model_dtype,
- )
-
- if isinstance(tasks, str):
- tasks = tasks.replace(" ", "").split(",")
-
- from lm_eval.utils import make_table # pylint: disable=E0611
-
- res_all = {}
- res_keys = ["results", "versions", "n-shot", "higher_is_better"]
- import time
-
- st = time.time()
- for task in tasks:
- while retry_times:
- try:
- res = lm_simple_evaluate(
- model=hflm, model_args=None, device=device_str, tasks=task, batch_size=batch_size, limit=limit
- )
- break
- except Exception as e:
- cuda_error_msg = traceback.format_exc()
- try:
- ori_batch_sizes = hflm.batch_sizes if hflm.batch_sizes else {"0": 64}
- try:
- for k, v in hflm.batch_sizes.items():
- hflm.batch_sizes[k] = max(v // 2, 1)
- logger.warning(f"Out of memory, reset batch_size to {hflm.batch_sizes} and re-try.")
- res = lm_simple_evaluate(
- model=hflm, model_args=None, device=device_str, tasks=task, batch_size=1, limit=limit
- )
- hflm.batch_sizes = ori_batch_sizes
- except Exception as e:
- traceback.print_exc()
- pass
- except Exception as e:
- logger.error(cuda_error_msg)
- traceback.print_exc()
- break
- retry_times -= 1
-
- if not res_all:
- res_all = res
- else:
- for key in res_keys:
- res_all[key].update(res[key])
- print(make_table(res_all))
-
- print("total eval time:", time.time() - st)
-
-
-def eval_with_vllm(args):
- import time
-
- from lm_eval import evaluator # pylint: disable=E0401
- from lm_eval.models.vllm_causallms import VLLM # pylint: disable=E0401
- from lm_eval.utils import make_table # pylint: disable=E0401
-
- st = time.time()
- os.environ["TOKENIZERS_PARALLELISM"] = "false"
- device_str, _ = get_device_and_parallelism(args.device)
- eval_model_dtype = get_model_dtype(args.eval_model_dtype, "auto")
- if (batch_size := args.eval_bs) is None:
- batch_size = "auto:8"
-
- vllm_lm = VLLM(
- pretrained=args.model,
- dtype=eval_model_dtype,
- revision=args.revision,
- trust_remote_code=not args.disable_trust_remote_code,
- tokenizer=args.tokenizer,
- tokenizer_mode=args.tokenizer_mode,
- tokenizer_revision=args.tokenizer_revision,
- add_bos_token=args.add_bos_token,
- prefix_token_id=args.prefix_token_id,
- tensor_parallel_size=args.tensor_parallel_size,
- quantization=args.quantization,
- max_gen_toks=args.max_gen_toks,
- swap_space=args.swap_space,
- batch_size=batch_size,
- max_batch_size=args.max_batch_size,
- max_length=args.max_length,
- max_model_len=args.max_model_len,
- seed=args.seed,
- gpu_memory_utilization=args.gpu_memory_utilization,
- device=device_str,
- data_parallel_size=args.data_parallel_size,
- lora_local_path=args.lora_local_path,
- )
- res = evaluator.simple_evaluate(
- model=vllm_lm,
- tasks=args.tasks,
- limit=args.limit,
- )
-
- print(make_table(res))
- print("evaluation running time=%ds" % (time.time() - st))
diff --git a/auto_round/script/mllm.py b/auto_round/script/mllm.py
deleted file mode 100644
index cb3edd93b..000000000
--- a/auto_round/script/mllm.py
+++ /dev/null
@@ -1,610 +0,0 @@
-# Copyright (c) 2024 Intel Corporation
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import argparse
-import os
-import sys
-
-from auto_round.logger import logger
-from auto_round.schemes import PRESET_SCHEMES
-from auto_round.utils import (
- clear_memory,
- get_device_and_parallelism,
- get_fp_layer_names,
- is_debug_mode,
- set_cuda_visible_devices,
-)
-
-
-class BasicArgumentParser(argparse.ArgumentParser):
-
- def __init__(self, *args, **kwargs):
- super().__init__(*args, **kwargs)
- self.add_argument(
- "--model",
- "--model_name",
- "--model_name_or_path",
- default="Qwen/Qwen2-VL-2B-Instruct",
- help="model name or path",
- )
-
- self.add_argument("--eval", action="store_true", help="whether to use eval only mode.")
-
- self.add_argument(
- "--scheme",
- default="W4A16",
- type=str,
- help="quantization scheme",
- )
-
- self.add_argument("--bits", default=None, type=int, help="number of weight bits")
- self.add_argument("--group_size", default=None, type=int, help="group size")
- self.add_argument("--asym", action="store_true", help="whether to use asym quantization")
- self.add_argument("--data_type", "--dtype", default=None, help="data type for tuning, 'int', 'mx_fp' and etc")
- self.add_argument("--act_bits", default=None, type=int, help="activation bits")
-
- self.add_argument("--eval_bs", default=None, type=int, help="batch size in evaluation")
-
- self.add_argument(
- "--device",
- "--devices",
- default="0",
- type=str,
- help="the device to be used for tuning. "
- "Currently, device settings support CPU, GPU, and HPU."
- "The default is set to cuda:0,"
- "allowing for automatic detection and switch to HPU or CPU."
- "set --device 0,1,2 to use multiple cards.",
- )
-
- self.add_argument(
- "--dataset",
- type=str,
- default=None,
- help="the dataset for quantization training."
- " current support NeelNanda/pile-10k,liuhaotian/llava_conv_58k,"
- "liuhaotian/llava_instruct_80k,liuhaotian/llava_instruct_150k"
- "It can be a custom one. Default is NeelNanda/pile-10k",
- )
-
- self.add_argument(
- "--lr", default=None, type=float, help="learning rate, if None, it will be set to 1.0/iters automatically"
- )
-
- self.add_argument(
- "--minmax_lr",
- default=None,
- type=float,
- help="minmax learning rate, if None,it will beset to be the same with lr",
- )
-
- self.add_argument("--seed", default=42, type=int, help="random seed")
-
- self.add_argument("--adam", action="store_true", help="whether to use adam optimizer instead of SignSGD")
-
- self.add_argument("--gradient_accumulate_steps", default=1, type=int, help="gradient accumulate steps")
-
- self.add_argument("--nblocks", default=1, type=int, help="how many blocks to tune together")
-
- self.add_argument("--low_gpu_mem_usage", action="store_true", help="offload intermediate features to cpu")
-
- self.add_argument("--format", default="auto_round", type=str, help="the format to save the model")
-
- self.add_argument(
- "--scale_dtype",
- default="fp16",
- choices=["fp16", "float16", "bf16", "bfloat16", "fp32", "float32"],
- help="scale data type to use for quantization",
- )
-
- self.add_argument(
- "--output_dir", default="./tmp_autoround", type=str, help="the directory to save quantized model"
- )
-
- self.add_argument("--disable_amp", action="store_true", help="disable amp")
-
- self.add_argument(
- "--disable_minmax_tuning", action="store_true", help="whether disable enable weight minmax tuning"
- )
-
- self.add_argument("--enable_norm_bias_tuning", action="store_true", help="whether enable norm bias tuning")
-
- self.add_argument(
- "--disable_trust_remote_code", action="store_true", help="whether to disable trust_remote_code"
- )
-
- self.add_argument(
- "--disable_quanted_input",
- action="store_true",
- help="whether to disuse the output of quantized block to tune the next block",
- )
-
- self.add_argument("--quant_lm_head", action="store_true", help="whether to quant lm_head")
-
- self.add_argument(
- "--low_cpu_mem_mode",
- default=0,
- type=int,
- choices=[0, 1, 2],
- help="choose which low cpu memory mode to use. "
- "Can significantly reduce cpu memory footprint but cost more time."
- "1 means choose block-wise mode, load the weights of each block"
- " from disk when tuning and release the memory of the block after tuning."
- "2 means choose layer-wise mode, load the weights of each layer from disk when tuning,"
- " minimum memory consumption and also slowest running speed."
- "others means not use low cpu memory. Default to 0, not use low cpu memory.",
- )
-
- self.add_argument(
- "--low_cpu_mem_tmp_dir",
- default=None,
- type=str,
- help="temporary work space to store the temporary files "
- "when using low cpu memory mode. Will remove after tuning.",
- )
-
- self.add_argument(
- "--model_dtype",
- default=None,
- type=str,
- choices=["fp16", "float16", "bf16", "bfloat16", "fp32", "float32"],
- help="force to convert the dtype, some backends supports fp16 dtype better",
- )
-
- self.add_argument("--fp_layers", default="", type=str, help="layers to maintain original data type")
-
- self.add_argument(
- "--not_use_best_mse",
- action="store_true",
- help="whether to use the iter of best mes loss in the tuning phase",
- )
-
- self.add_argument("--enable_torch_compile", action="store_true", help="whether to enable torch compile")
-
- self.add_argument(
- "--disable_deterministic_algorithms",
- action="store_true",
- help="deprecated, disable torch deterministic algorithms.",
- )
-
- self.add_argument(
- "--enable_deterministic_algorithms", action="store_true", help="enable torch deterministic algorithms."
- )
-
- ## ======================= VLM =======================
- self.add_argument(
- "--quant_nontext_module",
- action="store_true",
- help="whether to quantize non-text module, e.g. vision component",
- )
-
- self.add_argument(
- "--extra_data_dir",
- default=None,
- type=str,
- help="dataset dir for storing images/audio/videos. "
- "Can be a dir path or multiple dir path with format as "
- "'image=path_to_image,video=path_to_video,audio=path_to_audio'"
- "By default, it will search in the relative path, "
- "and if not find, will automatic download.",
- )
-
- self.add_argument(
- "--template",
- default=None,
- type=str,
- help="the template for building training dataset. It can be a custom one.",
- )
-
- self.add_argument(
- "--truncation",
- action="store_true",
- help="whether to truncate sequences at the maximum length."
- " Default True for pile and False for llava dataset.",
- )
-
- self.add_argument(
- "--to_quant_block_names",
- default=None,
- type=str,
- help="Names of quantitative blocks, please use commas to separate them.",
- )
-
- self.add_argument("--device_map", default=None, type=str, help="device_map for block in tuning phase")
-
- self.add_argument(
- "--disable_opt_rtn",
- action="store_true",
- help="whether to disable optimization of the RTN mode(iters=0) (default is False).",
- )
-
-
-def setup_parser():
- parser = BasicArgumentParser()
-
- parser.add_argument("--batch_size", "--train_bs", "--bs", default=8, type=int, help="train batch size")
-
- parser.add_argument("--iters", "--iter", default=200, type=int, help=" iters")
-
- parser.add_argument(
- "--seqlen",
- "--seq_len",
- default=None,
- type=int,
- help="sequence length, default 2048 for text-only, 512 for liuhaotian/llava",
- )
-
- parser.add_argument("--nsamples", "--nsample", default=128, type=int, help="number of samples")
-
- args = parser.parse_args()
- return args
-
-
-def setup_lmeval_parser():
- parser = argparse.ArgumentParser()
- parser.add_argument("--model", "--model_name", "--model_name_or_path", help="model name or path")
- parser.add_argument(
- "--device",
- "--devices",
- default="0",
- type=str,
- help="the device to be used for tuning. "
- "Currently, device settings support CPU, GPU, and HPU."
- "The default is set to cuda:0,"
- "allowing for automatic detection and switch to HPU or CPU."
- "set --device 0,1,2 to use multiple cards.",
- )
- parser.add_argument(
- "--tasks",
- type=str,
- default="MMBench_DEV_EN_V11,ScienceQA_VAL,TextVQA_VAL,POPE",
- help="eval tasks for VLMEvalKit.",
- )
- # Args that only apply to Video Dataset
- parser.add_argument(
- "--nframe",
- type=int,
- default=8,
- help="the number of frames to sample from a video," " only applicable to the evaluation of video benchmarks.",
- )
- parser.add_argument(
- "--pack",
- action="store_true",
- help="a video may associate with multiple questions, if pack==True,"
- " will ask all questions for a video in a single",
- )
- parser.add_argument("--fps", type=float, default=-1, help="set the fps for a video.")
- # Work Dir
- # Infer + Eval or Infer Only
- parser.add_argument(
- "--mode",
- type=str,
- default="all",
- choices=["all", "infer"],
- help="when mode set to 'all', will perform both inference and evaluation;"
- " when set to 'infer' will only perform the inference.",
- )
- parser.add_argument(
- "--eval_data_dir",
- type=str,
- default=None,
- help="path for VLMEvalKit to store the eval data. Default will store in ~/LMUData",
- )
- # API Kwargs, Apply to API VLMs and Judge API LLMs
- parser.add_argument("--retry", type=int, default=None, help="retry numbers for API VLMs")
- # Explicitly Set the Judge Model
- parser.add_argument("--judge", type=str, default=None, help="whether is a judge model.")
- # Logging Utils
- parser.add_argument("--verbose", action="store_true", help="whether to display verbose information.")
- # Configuration for Resume
- # Ignore: will not rerun failed VLM inference
- parser.add_argument("--ignore", action="store_true", help="ignore failed indices. ")
- # Rerun: will remove all evaluation temp files
- parser.add_argument(
- "--rerun", action="store_true", help="if true, will remove all evaluation temp files and rerun."
- )
- parser.add_argument("--output_dir", default="./eval_result", type=str, help="the directory to save quantized model")
- args = parser.parse_args()
- return args
-
-
-def tune(args):
- import transformers
- from transformers import AutoConfig, AutoModelForCausalLM, AutoProcessor, AutoTokenizer
-
- if args.format is None:
- args.format = "auto_round"
-
- formats = args.format.replace(" ", "").split(",")
- from auto_round.utils import SUPPORTED_FORMATS
-
- for format in formats:
- if format not in SUPPORTED_FORMATS:
- raise ValueError(f"{format} is not supported, we only support {SUPPORTED_FORMATS}")
-
- # Must set this before import torch
- # set_cuda_visible_devices(args.device_map)
- device_str, use_auto_mapping = get_device_and_parallelism(args.device_map)
-
- import torch
-
- model_name = args.model
- if model_name[-1] == "/":
- model_name = model_name[:-1]
- logger.info(f"start to quantize {model_name}")
- torch_dtype = "auto"
- if device_str is not None and "hpu" in device_str:
- torch_dtype = torch.bfloat16
-
- # load_model
- from auto_round.utils import mllm_load_model
-
- model, processor, tokenizer, image_processor = mllm_load_model(
- model_name,
- device="cpu",
- torch_dtype=torch_dtype,
- use_auto_mapping=False,
- trust_remote_code=not args.disable_trust_remote_code,
- model_dtype=args.model_dtype,
- )
-
- from auto_round import AutoRoundMLLM
-
- seqlen = args.seqlen
- if seqlen is not None and hasattr(model, "config") and hasattr(model.config, "max_position_embeddings"):
- seqlen = min(seqlen, model.config.max_position_embeddings)
-
- if seqlen is not None and hasattr(tokenizer, "model_max_length"):
- if tokenizer.model_max_length < seqlen:
- logger.info(
- f"change sequence length to {tokenizer.model_max_length} due to the limitation of model_max_length"
- )
- seqlen = min(seqlen, tokenizer.model_max_length)
- args.seqlen = seqlen
-
- model = model.eval()
-
- round = AutoRoundMLLM
-
- layer_config = {}
- not_quantize_layer_names = get_fp_layer_names(model, args.fp_layers)
- for name in not_quantize_layer_names:
- layer_config[name] = {"bits": 16, "act_bits": 16}
- if len(not_quantize_layer_names) > 0:
- logger.info(f"{not_quantize_layer_names} will not be quantized.")
- for format in formats:
- if "auto_round" not in format and "fake" not in format and "awq" not in format:
- ##TODO gptq could support some mixed precision config
- logger.warning(f"mixed precision exporting does not support {format} currently")
-
- if args.fp_layers != "":
- fp_layers = args.fp_layers.replace(" ", "").split(",")
- for n, m in model.named_modules():
- if not isinstance(m, (torch.nn.Linear, transformers.pytorch_utils.Conv1D)):
- continue
- for fp_layer in fp_layers:
- if fp_layer in n:
- layer_config[n] = {"bits": 16}
- logger.info(f"{n} will not be quantized.")
- if len(layer_config) > 0:
- for format in formats:
- if "auto_round" not in format and "fake" not in format:
- ##TODO gptq, awq could support some mixed precision config
- logger.warning(f"mixed precision exporting does not support {format} currently")
-
- if args.quant_lm_head:
- for format in formats:
- if "auto_round" not in format and "fake" not in format:
- auto_round_formats = [s for s in SUPPORTED_FORMATS if s.startswith("auto_round")]
- raise ValueError(
- f"{format} is not supported for lm-head quantization, please change to {auto_round_formats}"
- )
-
- if args.quant_lm_head and args.low_gpu_mem_usage:
- print("warning, low_gpu_mem_usage=False is strongly recommended if the whole model could be loaded to " "gpu")
-
- if "--truncation" not in sys.argv:
- args.truncation = None
-
- if "auto_awq" in args.format:
- from auto_round.utils import check_awq_gemm_compatibility
-
- awq_supported, info = check_awq_gemm_compatibility(
- model, args.bits, args.group_size, not args.asym, layer_config
- )
- if not awq_supported:
- logger.warning(f"The AutoAWQ format may not be supported due to {info}")
-
- enable_torch_compile = True if "--enable_torch_compile" in sys.argv else False
-
- model_kwargs = {
- "use_auto_mapping": use_auto_mapping,
- "trust_remote_code": not args.disable_trust_remote_code,
- "model_dtype": args.model_dtype,
- }
-
- sym = None # the default value should be None now
- if args.asym: # if the scheme is asym, how to set it to sym is an issue
- sym = False
-
- scheme = args.scheme.upper()
- if scheme not in PRESET_SCHEMES:
- raise ValueError(f"{scheme} is not supported. only {PRESET_SCHEMES.keys()} are supported ")
- if args.disable_deterministic_algorithms:
- logger.warning(
- "default not use deterministic_algorithms. disable_deterministic_algorithms is deprecated,"
- " please use enable_deterministic_algorithms instead. "
- )
- autoround = round(
- model,
- tokenizer,
- scheme=args.scheme,
- processor=processor,
- image_processor=image_processor,
- dataset=args.dataset,
- extra_data_dir=args.extra_data_dir,
- bits=args.bits,
- group_size=args.group_size,
- sym=sym,
- batch_size=args.batch_size,
- seqlen=seqlen,
- nblocks=args.nblocks,
- iters=args.iters,
- lr=args.lr,
- minmax_lr=args.minmax_lr,
- amp=not args.disable_amp,
- enable_quanted_input=not args.disable_quanted_input,
- truncation=args.truncation,
- nsamples=args.nsamples,
- low_gpu_mem_usage=args.low_gpu_mem_usage,
- seed=args.seed,
- gradient_accumulate_steps=args.gradient_accumulate_steps,
- scale_dtype=args.scale_dtype,
- layer_config=layer_config,
- template=args.template,
- enable_minmax_tuning=not args.disable_minmax_tuning,
- act_bits=args.act_bits,
- quant_nontext_module=args.quant_nontext_module,
- not_use_best_mse=args.not_use_best_mse,
- to_quant_block_names=args.to_quant_block_names,
- enable_torch_compile=enable_torch_compile,
- device_map=args.device_map,
- model_kwargs=model_kwargs,
- data_type=args.data_type,
- disable_opt_rtn=args.disable_opt_rtn,
- enable_deterministic_algorithms=args.enable_deterministic_algorithms,
- )
-
- model_name = args.model.rstrip("/")
-
- if model_name.split("/")[-1].strip(".") == "" and "gguf" not in args.format:
- export_dir = os.path.join(args.output_dir, f"w{autoround.bits}g{autoround.group_size}")
- elif model_name.split("/")[-1].strip(".") == "" and "gguf" in args.format:
- export_dir = args.output_dir
- elif model_name.split("./")[-1].strip("./") != "" and "gguf" in args.format:
- export_dir = os.path.join(args.output_dir, model_name.split("/")[-1] + "-gguf")
- else:
- export_dir = os.path.join(
- args.output_dir, model_name.split("/")[-1] + f"-w{autoround.bits}g{autoround.group_size}"
- )
-
- model, folders = autoround.quantize_and_save(export_dir, format=args.format)
-
- if args.low_cpu_mem_mode == 1 or args.low_cpu_mem_mode == 2:
- import shutil
-
- shutil.rmtree(args.low_cpu_mem_tmp_dir, ignore_errors=True)
-
- model.eval()
- clear_memory()
-
-
-def vlmeval(args):
- set_cuda_visible_devices(args.device_map)
- device_str, parallelism = get_device_and_parallelism(args.device_map)
- if parallelism:
- os.environ["AUTO_SPLIT"] = "1"
- if isinstance(args.tasks, str):
- args.tasks = args.tasks.replace(" ", "").split(",")
- from auto_round.mllm import mllm_eval
-
- mllm_eval(
- args.model,
- work_dir=args.output_dir,
- data_store_dir=args.eval_data_dir,
- dataset=args.tasks,
- pack=args.pack,
- fps=args.fps,
- nframe=args.nframe,
- rerun=args.rerun,
- judge=args.judge,
- verbose=args.verbose,
- mode=args.mode,
- ignore=args.ignore,
- )
-
-
-def setup_lmms_parser():
- parser = argparse.ArgumentParser()
- parser.add_argument("--model", "--model_name", "--model_name_or_path", help="model name or path")
- parser.add_argument(
- "--tasks",
- default="pope,textvqa_val,scienceqa,mmbench_en",
- help="To get full list of tasks, use the command lmms-eval --tasks list",
- )
- parser.add_argument("--output_dir", default="./eval_result", type=str, help="the directory to save quantized model")
- parser.add_argument(
- "--num_fewshot",
- type=int,
- default=None,
- help="Number of examples in few-shot context",
- )
- parser.add_argument(
- "--batch_size",
- "--bs",
- "-b",
- type=str,
- default=1,
- metavar="auto|auto:N|N",
- help="Acceptable values are 'auto', 'auto:N' or N, where N is an integer. Default 1.",
- )
- parser.add_argument(
- "--max_batch_size",
- type=int,
- default=None,
- metavar="N",
- help="Maximal batch size to try with --batch_size auto.",
- )
- parser.add_argument(
- "--device",
- "--devices",
- default="0",
- type=str,
- help="the device to be used for tuning. "
- "Currently, device settings support CPU, GPU, and HPU."
- "The default is set to cuda:0,"
- "allowing for automatic detection and switch to HPU or CPU."
- "set --device 0,1,2 to use multiple cards.",
- )
- parser.add_argument(
- "--limit",
- type=float,
- default=None,
- help="Limit the number of examples per task. "
- "If <1, limit is a percentage of the total"
- " number of examples.",
- )
- args = parser.parse_args()
- return args
-
-
-def lmms_eval(args):
- set_cuda_visible_devices(args.device_map)
- device_str, parallelism = get_device_and_parallelism(args.device_map)
-
- from auto_round.mllm import lmms_eval
-
- results = lmms_eval(
- model=args.model,
- tasks=args.tasks,
- output_dir=args.output_dir,
- num_fewshot=args.num_fewshot,
- limit=args.limit,
- batch_size=args.batch_size,
- max_batch_size=args.max_batch_size,
- device=device_str,
- use_cache=None,
- apply_chat_template=False,
- )
- return results
diff --git a/auto_round/utils.py b/auto_round/utils.py
index dcb794521..2d98ae121 100644
--- a/auto_round/utils.py
+++ b/auto_round/utils.py
@@ -1042,6 +1042,8 @@ def get_fp_layer_names(model, fp_layers):
list: A list of layer names that match the specified FP layers or are
subcomponents of those layers.
"""
+ if not fp_layers:
+ return []
fp_layers = fp_layers.replace(" ", "").split(",")
all_layer_names = []
for n, m in model.named_modules():
@@ -2686,3 +2688,45 @@ def copy_python_files_from_model_cache(model, save_path: str):
if file.endswith(".py") and os.path.isfile(full_file_name):
logger.debug(f"Transferring {full_file_name} to {save_path}")
shutil.copy(full_file_name, save_path)
+
+
+def is_mllm_model(model_or_path: Union[str, torch.nn.Module]):
+ MM_KEYS = [
+ "multi_modal_projector",
+ "vision_tower",
+ "multimodal_projector",
+ "thinker",
+ "visual",
+ "audio",
+ "talker",
+ "token2wav",
+ "vision_model",
+ "audio_tower",
+ "vision_encoder",
+ "vision_language_adapter",
+ "patch_merger",
+ "pre_mm_projector_norm",
+ "vision",
+ ]
+
+ model_path = model_or_path if isinstance(model_or_path, str) else model_or_path.name_or_path
+ if not os.path.isdir(model_path):
+ model_path = download_hf_model(model_path)
+
+ if isinstance(model_path, str):
+ if os.path.exists(os.path.join(model_path, "preprocessor_config.json")):
+ return True
+ if os.path.exists(os.path.join(model_path, "processor_config.json")):
+ return True
+ with open(os.path.join(model_path, "config.json")) as f:
+ config = json.load(f)
+ for key in config.keys():
+ if any([k in key for k in MM_KEYS]):
+ return True
+
+ if isinstance(model_or_path, torch.nn.Module):
+ for name, module in model_or_path.named_modules():
+ if any([k in name for k in MM_KEYS]):
+ return True
+
+ return False
diff --git a/test/test_cpu/test_autoround.py b/test/test_cpu/test_autoround.py
index ea53aaf94..827688812 100644
--- a/test/test_cpu/test_autoround.py
+++ b/test/test_cpu/test_autoround.py
@@ -677,6 +677,18 @@ class TestAutoRound(unittest.TestCase):
ar = AutoRound(model_name, quant_lm_head=True, iters=1, nsamples=1, seqlen=32)
ar.quantize()
+ def test_compressor(self):
+ model_name = "Qwen/Qwen2-VL-2B-Instruct"
+ ar = AutoRound(model_name, enable_adam=True)
+ self.assertEqual(ar.optimizer, torch.optim.AdamW)
+ self.assertTrue(ar.mllm)
+
+ # test old api
+ from auto_round import AutoRoundMLLM
+
+ ar = AutoRoundMLLM(model_name)
+ self.assertTrue(ar.mllm)
+
if __name__ == "__main__":
unittest.main()
diff --git a/test/test_cpu/test_gguf_format.py b/test/test_cpu/test_gguf_format.py
index eb146d574..2a25b0f5f 100644
--- a/test/test_cpu/test_gguf_format.py
+++ b/test/test_cpu/test_gguf_format.py
@@ -312,7 +312,7 @@ class TestGGUF(unittest.TestCase):
def test_vlm_gguf(self):
model_name = "Qwen/Qwen2-VL-2B-Instruct"
- from auto_round.mllm.autoround_mllm import AutoRoundMLLM
+ from auto_round import AutoRoundMLLM
from auto_round.utils import mllm_load_model
model, processor, tokenizer, image_processor = mllm_load_model(model_name)
diff --git a/test/test_cpu/test_mllm.py b/test/test_cpu/test_mllm.py
index 0fd102407..10ccd9d37 100644
--- a/test/test_cpu/test_mllm.py
+++ b/test/test_cpu/test_mllm.py
@@ -97,7 +97,7 @@ class TestAutoRoundMLLM(unittest.TestCase):
assert target_blocks == blocks
def test_dataset_check(self):
- from auto_round.mllm.mllm_dataset import MLLM_DATASET
+ from auto_round.compressors.mllm.dataset import MLLM_DATASET
class Myclass:
model_type = None
diff --git a/test/test_cuda/test_gguf.py b/test/test_cuda/test_gguf.py
index 33473a116..50bb23aa6 100644
--- a/test/test_cuda/test_gguf.py
+++ b/test/test_cuda/test_gguf.py
@@ -183,7 +183,7 @@ class TestAutoRound(unittest.TestCase):
@require_gguf
def test_vlm_gguf(self):
model_name = "/models/Qwen2.5-VL-7B-Instruct"
- from auto_round.mllm.autoround_mllm import AutoRoundMLLM
+ from auto_round import AutoRoundMLLM
from auto_round.utils import mllm_load_model
model, processor, tokenizer, image_processor = mllm_load_model(model_name)
@@ -205,7 +205,7 @@ class TestAutoRound(unittest.TestCase):
shutil.rmtree("./saved", ignore_errors=True)
model_name = "/models/gemma-3-12b-it"
- from auto_round.mllm.autoround_mllm import AutoRoundMLLM
+ from auto_round import AutoRoundMLLM
from auto_round.utils import mllm_load_model
model, processor, tokenizer, image_processor = mllm_load_model(model_name)
@@ -230,7 +230,7 @@ class TestAutoRound(unittest.TestCase):
@require_gguf
def test_llama_4(self):
model_name = "/dataset/Llama-4-Scout-17B-16E-Instruct/"
- from auto_round.mllm.autoround_mllm import AutoRoundMLLM
+ from auto_round import AutoRoundMLLM
from auto_round.utils import mllm_load_model
model, processor, tokenizer, image_processor = mllm_load_model(model_name, use_auto_mapping=False)
diff --git a/test/test_cuda/test_vlms.py b/test/test_cuda/test_vlms.py
index d18c53a66..b3bf0f0fd 100644
--- a/test/test_cuda/test_vlms.py
+++ b/test/test_cuda/test_vlms.py
@@ -130,6 +130,33 @@ class TestAutoRound(unittest.TestCase):
self.assertTrue(len(block_name) == 1)
self.assertTrue(get_block_names(model) == block_name)
+ def test_mllm_detect(self):
+ from auto_round.utils import is_mllm_model, llm_load_model, mllm_load_model
+
+ for model_name in [
+ "/models/Llama-3.2-11B-Vision-Instruct/",
+ "/models/deepseek-vl2-tiny/",
+ "/models/gemma-3-12b-it/",
+ "/models/Phi-3.5-vision-instruct",
+ "/models/Qwen2-VL-2B-Instruct",
+ "/models/SmolVLM-256M-Instruct",
+ "/models/Llama-4-Maverick-17B-128E-Instruct",
+ "/models/Mistral-Small-3.1-24B-Instruct-2503",
+ "/models/InternVL3-1B",
+ "/models/pixtral-12b",
+ ]:
+ self.assertTrue(is_mllm_model(model_name))
+ try:
+ model, _, _, _ = mllm_load_model(model_name)
+ except:
+ continue
+ self.assertTrue(is_mllm_model(model))
+
+ for model_name in ["/models/glm-4-9b-chat", "/models/Qwen2.5-1.5B-Instruct/"]:
+ self.assertFalse(is_mllm_model(model_name))
+ model, _, _ = llm_load_model(model_name)
+ self.assertFalse(is_mllm_model(model))
+
if __name__ == "__main__":
unittest.main()