PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
Add black and blacken-docs in pre-commit (#692)
Sun, Xuehaod424d3b4f7ce
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yamlindex b81bdeaa0..a6650bf3a 100644--- a/.pre-commit-config.yaml+++ b/.pre-commit-config.yaml@@ -29,6 +29,20 @@ repos:--skip-license-insertion-comment=Copyright,]+ - repo: https://github.com/psf/black.git+ rev: 25.1.0+ hooks:+ - id: black+ files: (.*\.py)$++ - repo: https://github.com/asottile/blacken-docs+ rev: 1.19.1+ hooks:+ - id: blacken-docs+ args: [--line-length=120, --skip-errors]+ additional_dependencies:+ - black==25.1.0+- repo: https://github.com/codespell-project/codespellrev: v2.4.1hooks:@@ -43,7 +57,7 @@ repos:- id: isort- repo: https://github.com/astral-sh/ruff-pre-commit- rev: v0.12.4+ rev: v0.12.5hooks:- id: ruffargs: [--fix, --exit-non-zero-on-fix, --no-cache]diff --git a/README.md b/README.mdindex e0abbd949..3416b09b0 100644--- a/README.md+++ b/README.md@@ -175,7 +175,7 @@ autoround = AutoRound(model, tokenizer, bits=bits, group_size=group_size, sym=syoutput_dir = "./tmp_autoround"## format= 'auto_round'(default), 'auto_gptq', 'auto_awq'-autoround.quantize_and_save(output_dir, format='auto_round')+autoround.quantize_and_save(output_dir, format="auto_round")```<details>@@ -252,20 +252,18 @@ from transformers import Qwen2VLForConditionalGeneration, AutoProcessor, AutoTok## load the modelmodel_name = "Qwen/Qwen2-VL-2B-Instruct"-model = Qwen2VLForConditionalGeneration.from_pretrained(- model_name, trust_remote_code=True, torch_dtype="auto")+model = Qwen2VLForConditionalGeneration.from_pretrained(model_name, trust_remote_code=True, torch_dtype="auto")tokenizer = AutoTokenizer.from_pretrained(model_name)processor = AutoProcessor.from_pretrained(model_name, trust_remote_code=True)## quantize the modelbits, group_size, sym = 4, 128, True-autoround = AutoRoundMLLM(model, tokenizer, processor,- bits=bits, group_size=group_size, sym=sym)+autoround = AutoRoundMLLM(model, tokenizer, processor, bits=bits, group_size=group_size, sym=sym)autoround.quantize()# save the quantized model, set format='auto_gptq' or 'auto_awq' to use other formatsoutput_dir = "./tmp_autoround"-autoround.save_quantized(output_dir, format='auto_round', inplace=True)+autoround.save_quantized(output_dir, format="auto_round", inplace=True)```</details>@@ -335,8 +333,7 @@ this may cause unexpected exceptions.from transformers import AutoModelForCausalLM, AutoTokenizerquantized_model_path = "./tmp_autoround"-model = AutoModelForCausalLM.from_pretrained(quantized_model_path,- device_map="auto", torch_dtype="auto")+model = AutoModelForCausalLM.from_pretrained(quantized_model_path, device_map="auto", torch_dtype="auto")tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)text = "There is a girl who likes adventure,"inputs = tokenizer(text, return_tensors="pt").to(model.device)@@ -370,9 +367,9 @@ from transformers import AutoRoundConfigquantized_model_path = "./tmp_autoround"quantization_config = AutoRoundConfig(backend="auto")-model = AutoModelForCausalLM.from_pretrained(quantized_model_path, device_map="auto",- torch_dtype="auto",- quantization_config=quantization_config)+model = AutoModelForCausalLM.from_pretrained(+ quantized_model_path, device_map="auto", torch_dtype="auto", quantization_config=quantization_config+)tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)text = "There is a girl who likes adventure,"inputs = tokenizer(text, return_tensors="pt").to(model.device)@@ -390,8 +387,9 @@ from transformers import AutoRoundConfigmodel_name = "ybelkada/opt-125m-gptq-4bit"quantization_config = AutoRoundConfig()-model = AutoModelForCausalLM.from_pretrained(model_name, device_map="cpu", torch_dtype="auto",- quantization_config=quantization_config)+model = AutoModelForCausalLM.from_pretrained(+ model_name, device_map="cpu", torch_dtype="auto", quantization_config=quantization_config+)tokenizer = AutoTokenizer.from_pretrained(model_name)text = "There is a girl who likes adventure,"inputs = tokenizer(text, return_tensors="pt").to(model.device)diff --git a/auto_round/__init__.py b/auto_round/__init__.pyindex af0aa1834..7d5215f68 100644--- a/auto_round/__init__.py+++ b/auto_round/__init__.py@@ -15,14 +15,18 @@ from auto_round.autoround import AutoRound, AutoRoundAdam, AutoRoundOPTfrom auto_round.mllm import AutoRoundMLLMfrom auto_round.utils import LazyImport+def __getattr__(name):- if name == 'AutoHfQuantizer':+ if name == "AutoHfQuantizer":from auto_round.inference.auto_quantizer import AutoHfQuantizer+return AutoHfQuantizer- if name == 'AutoRoundConfig':+ if name == "AutoRoundConfig":from auto_round.inference.auto_quantizer import AutoRoundConfig+return AutoRoundConfigraise AttributeError(f"auto-round has no attribute '{name}'")+from .version import __version__diff --git a/auto_round/__main__.py b/auto_round/__main__.pyindex 003035922..8c682ec3a 100644--- a/auto_round/__main__.py+++ b/auto_round/__main__.py@@ -16,21 +16,24 @@ import sysdef run_eval():from auto_round.script.llm import setup_eval_parser+args = setup_eval_parser()if args.eval_task_by_task:from auto_round.script.llm import eval_task_by_task+eval_task_by_task(model=args.model,device=args.device,tasks=args.tasks,batch_size=args.eval_bs,trust_remote_code=not args.disable_trust_remote_code,- eval_model_dtype=args.eval_model_dtype- )+ eval_model_dtype=args.eval_model_dtype,+ )else:from auto_round.script.llm import eval+eval(args)-+def run():if "--eval" in sys.argv:@@ -38,21 +41,28 @@ def run():run_eval()else:from auto_round.script.llm import setup_parser, tune+args = setup_parser()tune(args)+def run_best():from auto_round.script.llm import setup_best_parser, tune+args = setup_best_parser()tune(args)-++def run_light():from auto_round.script.llm import setup_light_parser, tune+args = setup_light_parser()tune(args)+def run_fast():from auto_round.script.llm import setup_fast_parser, tune+args = setup_fast_parser()tune(args)@@ -60,6 +70,7 @@ def run_fast():def run_mllm():if "--eval" in sys.argv:from auto_round.script.llm import eval, setup_eval_parser+sys.argv.remove("--eval")args = setup_eval_parser()args.mllm = True@@ -72,17 +83,22 @@ def run_mllm():run_lmms()else:from auto_round.script.mllm import setup_parser, tune+args = setup_parser()tune(args)+def run_lmms():# from auto_round.script.lmms_eval import setup_lmms_args, evalfrom auto_round.script.mllm import lmms_eval, setup_lmms_parser+args = setup_lmms_parser()lmms_eval(args)+def run_vlmeavl():from auto_round.script.mllm import setup_lmeval_parser, vlmeval+args = setup_lmeval_parser()vlmeval(args)@@ -94,6 +110,6 @@ def switch():else:run()-if __name__ == '__main__':- switch()+if __name__ == "__main__":+ switch()diff --git a/auto_round/autoround.py b/auto_round/autoround.pyindex 056308f41..87770d30c 100644--- a/auto_round/autoround.py+++ b/auto_round/autoround.py@@ -137,49 +137,49 @@ class AutoRound(object):"""def __init__(- self,- model: torch.nn.Module,- tokenizer,- bits: int = 4,- group_size: int = 128,- sym: bool = True,- layer_config: dict = None,- batch_size: int = 8,- amp: bool = True,- device: str = None,- lr_scheduler=None,- dataset: Union[str, list, tuple, torch.utils.data.DataLoader] = "NeelNanda/pile-10k",- enable_quanted_input: bool = True,- enable_minmax_tuning: bool = True,- lr: float = None,- minmax_lr: float = None,- low_gpu_mem_usage: bool = False,- low_cpu_mem_usage: bool = False,- iters: int = 200,- seqlen: int = 2048,- nsamples: int = 128,- sampler: str = "rand",- seed: int = 42,- nblocks: int = 1,- gradient_accumulate_steps: int = 1,- not_use_best_mse: bool = False,- dynamic_max_gap: int = -1,- data_type: str = "int",- scale_dtype: Union[str, torch.dtype] = "fp16",- act_bits: int = 16,- act_group_size: int = None,- act_sym: bool = None,- act_data_type: str = None,- act_dynamic: bool = True,- to_quant_block_names: Union[str, list] = None,- enable_norm_bias_tuning: bool = False,- enable_torch_compile: bool = False,- device_map: Union[str, dict] = None,- super_bits: int = None,- super_group_size: int = None,- disable_opt_rtn: bool = False,- model_kwargs: dict = None,- **kwargs,+ self,+ model: torch.nn.Module,+ tokenizer,+ bits: int = 4,+ group_size: int = 128,+ sym: bool = True,+ layer_config: dict = None,+ batch_size: int = 8,+ amp: bool = True,+ device: str = None,+ lr_scheduler=None,+ dataset: Union[str, list, tuple, torch.utils.data.DataLoader] = "NeelNanda/pile-10k",+ enable_quanted_input: bool = True,+ enable_minmax_tuning: bool = True,+ lr: float = None,+ minmax_lr: float = None,+ low_gpu_mem_usage: bool = False,+ low_cpu_mem_usage: bool = False,+ iters: int = 200,+ seqlen: int = 2048,+ nsamples: int = 128,+ sampler: str = "rand",+ seed: int = 42,+ nblocks: int = 1,+ gradient_accumulate_steps: int = 1,+ not_use_best_mse: bool = False,+ dynamic_max_gap: int = -1,+ data_type: str = "int",+ scale_dtype: Union[str, torch.dtype] = "fp16",+ act_bits: int = 16,+ act_group_size: int = None,+ act_sym: bool = None,+ act_data_type: str = None,+ act_dynamic: bool = True,+ to_quant_block_names: Union[str, list] = None,+ enable_norm_bias_tuning: bool = False,+ enable_torch_compile: bool = False,+ device_map: Union[str, dict] = None,+ super_bits: int = None,+ super_group_size: int = None,+ disable_opt_rtn: bool = False,+ model_kwargs: dict = None,+ **kwargs,):self.vlm = kwargs.pop("vlm") if "vlm" in kwargs else Falseif kwargs:@@ -225,7 +225,8 @@ class AutoRound(object):tmp_bits = infer_bits_by_data_type(self.data_type)if tmp_bits < 16 and tmp_bits != bits:logger.warning(- f"bits set in 'data_type' do not match the specified 'bits' setting. Resetting 'bits' to {tmp_bits}.")+ f"bits set in 'data_type' do not match the specified 'bits' setting. Resetting 'bits' to {tmp_bits}."+ )self.bits = tmp_bitsself.supported_types = SUPPORTED_LAYER_TYPESself.model = model.eval()@@ -234,7 +235,7 @@ class AutoRound(object):self.scale_dtype = convert_dtype_str2torch(scale_dtype)self.set_amp_dtype()self.to_quant_block_names = to_quant_block_names- if not hasattr(self, 'quant_block_list'):+ if not hasattr(self, "quant_block_list"):all_blocks = get_block_names(model)self.quant_block_list = find_matching_blocks(model, all_blocks, self.to_quant_block_names)self.cache_device = torch.device("cpu") if self.low_gpu_mem_usage else self.device@@ -279,10 +280,19 @@ class AutoRound(object):logger.info(f"using {self.model.dtype} for quantization tuning")self.enable_torch_compile = enable_torch_compile- if not self.enable_torch_compile and TORCH_VERSION_AT_LEAST_2_6 and self.act_bits > 8 and not is_debug_mode() \- and not self.low_cpu_mem_usage and "fp8" not in self.data_type and "fp8" not in self.act_data_type:- logger.info("'enable_torch_compile' is set to `False` by default. " \- "Enabling it can reduce tuning cost by 20%, but it might throw an exception.")+ if (+ not self.enable_torch_compile+ and TORCH_VERSION_AT_LEAST_2_6+ and self.act_bits > 8+ and not is_debug_mode()+ and not self.low_cpu_mem_usage+ and "fp8" not in self.data_type+ and "fp8" not in self.act_data_type+ ):+ logger.info(+ "'enable_torch_compile' is set to `False` by default. "+ "Enabling it can reduce tuning cost by 20%, but it might throw an exception."+ )if self.act_bits <= 8 and self.enable_torch_compile:self.enable_torch_compile = False@@ -335,7 +345,7 @@ class AutoRound(object):"act_dynamic","act_data_type","super_bits",- "super_group_size"+ "super_group_size",]self.shared_cache_keys = get_shared_keys(self.model)@@ -358,9 +368,9 @@ class AutoRound(object):infos = device_map.split(",")device_map_dict = {}for info in infos:- index = info.find(':')+ index = info.find(":")key = info[:index]- value = info[index + 1:]+ value = info[index + 1 :]device_map_dict[key] = valuedevice_map = device_map_dict@@ -387,7 +397,8 @@ class AutoRound(object):module = get_module(self.model, name)if hasattr(module, "tuning_device") and module.tuning_device != device:logger.warning(- f"Multiple devices have been set for layer {name}, keeping original device {module.tuning_device}")+ f"Multiple devices have been set for layer {name}, keeping original device {module.tuning_device}"+ )else:module.tuning_device = device@@ -396,11 +407,11 @@ class AutoRound(object):if self.data_type.endswith("_dq"):gguf_config = GGUF_INNER_CONFIG[f"gguf:q{self.bits}_k"]self.super_bits = gguf_config["super_bits"] if self.super_bits is None else self.super_bits- self.super_group_size = gguf_config["super_group_size"] \- if self.super_group_size is None else self.super_group_size+ self.super_group_size = (+ gguf_config["super_group_size"] if self.super_group_size is None else self.super_group_size+ )def check_configs(self):-"""Checks if the configurations are valid.Raises:@@ -430,12 +441,14 @@ class AutoRound(object):if self.act_bits <= 8:logger.warning("activation quantization is an experimental feature with limited support and a complex API. "- "And please save the quantized model to fake format as real deployment is not supported currently")+ "And please save the quantized model to fake format as real deployment is not supported currently"+ )if "mx_fp" in self.data_type or "nv_fp" in self.data_type:logger.warning("please save the quantized model to fake format "- "as real deployment is not supported for mx_fp/nv_fp datatype currently")+ "as real deployment is not supported for mx_fp/nv_fp datatype currently"+ )if "mx_fp" in self.data_type and self.group_size != 32:logger.warning("mx_fp should only support group_size of 32 in real deployment")@@ -445,15 +458,18 @@ class AutoRound(object):if self.nsamples < self.gradient_accumulate_steps * self.batch_size:if self.batch_size > self.nsamples:- logger.warning(f"reset batch_size to {self.nsamples} as nsamples({self.nsamples})"- f" is smaller than batch_size({self.batch_size})")+ logger.warning(+ f"reset batch_size to {self.nsamples} as nsamples({self.nsamples})"+ f" is smaller than batch_size({self.batch_size})"+ )self.batch_size = self.nsamplesif self.gradient_accumulate_steps > self.nsamples // self.batch_size:self.gradient_accumulate_steps = self.nsamples // self.batch_sizelogger.warning(f"reset gradient_accumulate_steps to {self.gradient_accumulate_steps}"f" as nsamples must equal or greater"- f" than gradient_accumulate_steps * batch_size")+ f" than gradient_accumulate_steps * batch_size"+ )if self.enable_norm_bias_tuning:logger.warning("the `enable_norm_bias_tuning` feature is experimental and currently has limited support.")@@ -480,30 +496,39 @@ class AutoRound(object):)##check group_size 32 for auto_round- if self.data_type == "int" and hasattr(self, "formats") and any(- key in fmt for fmt in self.formats for key in ("auto_round", "auto_gptq", "auto_awq")):+ if (+ self.data_type == "int"+ and hasattr(self, "formats")+ and any(key in fmt for fmt in self.formats for key in ("auto_round", "auto_gptq", "auto_awq"))+ ):for n, m in self.model.named_modules():if isinstance(m, self.supported_types):if m.weight.shape[0] % 32 != 0 or m.weight.shape[1] % 32 != 0:self.layer_config[n] = {"bits": 16}logger.info(f"{n} will not be quantized due to its shape not being divisible by 32,"- " resulting in an exporting issue to autogptq")-- if self.seqlen is not None and hasattr(self.model, "config") and \- hasattr(self.model.config, "max_position_embeddings"):+ " resulting in an exporting issue to autogptq"+ )++ if (+ self.seqlen is not None+ and hasattr(self.model, "config")+ and hasattr(self.model.config, "max_position_embeddings")+ ):if self.model.config.max_position_embeddings < self.seqlen:logger.warning(- f"change sequence length to {self.model.config.max_position_embeddings} " \- "due to the limitation of max_position_embeddings")+ f"change sequence length to {self.model.config.max_position_embeddings} "+ "due to the limitation of max_position_embeddings"+ )self.seqlen = min(self.seqlen, self.model.config.max_position_embeddings)if self.seqlen is not None and hasattr(self.tokenizer, "model_max_length"):if self.tokenizer.model_max_length < self.seqlen:logger.warning(- f"change sequence length to {self.tokenizer.model_max_length} " \- "due to the limitation of model_max_length. " \- "You can also try to increase the model_max_length to avoid this issue.")+ f"change sequence length to {self.tokenizer.model_max_length} "+ "due to the limitation of model_max_length. "+ "You can also try to increase the model_max_length to avoid this issue."+ )self.seqlen = min(self.seqlen, self.tokenizer.model_max_length)if self.group_size == 0 and "fp8" not in self.data_type:@@ -527,8 +552,9 @@ class AutoRound(object):if self.vlm:_gguf_args_check(self, format, model_type=ModelType.MMPROJ)- formats = format.replace("q*_", f"q{self.bits}_").replace(' ', '').split(',')+ formats = format.replace("q*_", f"q{self.bits}_").replace(" ", "").split(",")from auto_round.utils import SUPPORTED_FORMATS+for format_ in formats:if format_ not in SUPPORTED_FORMATS:logger.error(f"Unsupported format {format_}, please choose from {SUPPORTED_FORMATS}")@@ -550,12 +576,11 @@ class AutoRound(object):format = formats[index]if format == "auto_round":if self.sym and "int" in self.data_type:- format = format.replace('auto_round', 'auto_round:auto_gptq')+ format = format.replace("auto_round", "auto_round:auto_gptq")formats[index] = formatif self.bits == 4 and not self.sym and "int" in self.data_type:enable_awq = all(- config["bits"] == self.bits or config["bits"] >= 16- for config in self.layer_config.values()+ config["bits"] == self.bits or config["bits"] >= 16 for config in self.layer_config.values())if enable_awq:formats[index] = format.replace("auto_round", "auto_round:auto_awq")@@ -593,11 +618,17 @@ class AutoRound(object):if "fp8" not in self.act_data_type or self.act_dynamic:if format == "llmcompressor":bits, group_size, sym, act_bits = 8, -1, True, 8- assert self.bits == bits and self.group_size == group_size and self.sym == sym \- and self.act_bits == act_bits and self.act_dynamic, \- f"Currently only support to export llmcompressor format for dynamic quantized" \- f" W{self.bits}A{self.act_bits} model, but got bits={self.bits}," \+ assert (+ self.bits == bits+ and self.group_size == group_size+ and self.sym == sym+ and self.act_bits == act_bits+ and self.act_dynamic+ ), (+ f"Currently only support to export llmcompressor format for dynamic quantized"+ f" W{self.bits}A{self.act_bits} model, but got bits={self.bits},"f" group_size={self.group_size}, sym={self.sym}, act_bits={self.act_bits}"+ )elif format != "fake":logger.warning("Currently only support to export auto_round format quantized model"@@ -621,6 +652,7 @@ class AutoRound(object):sys.exit(-1)return format+def quantize_and_save(self, output_dir: str = "tmp_autoround", format: str = "auto_round", inplace=True, **kwargs):"""Quantizes the model and saves it in the specified format(s).@@ -739,20 +771,17 @@ class AutoRound(object):try:weight, scale, zp = quant_func(module.weight.to(self.device),- **{k: config[k] for k in [- "bits", "group_size", "super_bits",- "super_group_size", "scale_dtype"- ]}+ **{k: config[k] for k in ["bits", "group_size", "super_bits", "super_group_size", "scale_dtype"]},)except RuntimeError as e:if "CUDA out of memory" in str(e) or "MODULE:PT_DEVMEM" in str(e):logger.info("out of VRAM, falling back to CPU.")weight, scale, zp = quant_func(module.weight.to("cpu"),- **{k: config[k] for k in [- "bits", "group_size", "super_bits",- "super_group_size", "scale_dtype"- ]}+ **{+ k: config[k]+ for k in ["bits", "group_size", "super_bits", "super_group_size", "scale_dtype"]+ },)else:raise@@ -794,11 +823,11 @@ class AutoRound(object):# Load datasetfrom .calib_dataset import get_dataloader+if isinstance(self.dataset, str):dataset_name = self.dataset.replace(" ", "")self.dataloader = get_dataloader(- self.tokenizer, self.seqlen, dataset_name,- self.seed, self.batch_size, self.nsamples+ self.tokenizer, self.seqlen, dataset_name, self.seed, self.batch_size, self.nsamples)else:self.dataloader = self.dataset@@ -808,6 +837,7 @@ class AutoRound(object):# Dispatch multi-GPU model if necessaryif hasattr(model, "hf_device_map") and len(model.hf_device_map) > 1:from accelerate.big_modeling import dispatch_model+dispatch_model(model, model.hf_device_map)def register_act_hook(model):@@ -816,7 +846,7 @@ class AutoRound(object):def get_act_max_hook(module, input, output):input = input[0] if isinstance(input, (tuple, list)) else inputflattened = input.reshape(-1, input.shape[-1]).to(torch.float32)- squared = torch.sum(flattened ** 2, dim=0).to(torch.float32)+ squared = torch.sum(flattened**2, dim=0).to(torch.float32)if not hasattr(module, "imatrix"):module.imatrix = squared@@ -862,19 +892,21 @@ class AutoRound(object):module.imatrix /= module.imatrix_cntif hasattr(model, "hf_device_map") and len(model.hf_device_map) > 1:import accelerate+accelerate.hooks.remove_hook_from_submodules(model)# Perform quantization using RTNfrom tqdm import tqdm+pbar = tqdm(all_to_quantized_module_names)- block_names_cnt = len(flatten_list(get_block_names(self.model,True)))- clear_mem_freq = len(all_to_quantized_module_names)//block_names_cnt+ block_names_cnt = len(flatten_list(get_block_names(self.model, True)))+ clear_mem_freq = len(all_to_quantized_module_names) // block_names_cntif clear_mem_freq == 0:clear_mem_freq = 1cnt = 1for name in pbar:pbar.set_description(f"Quantizing {name}")self.quantize_layer_via_rtn(name)- if cnt % clear_mem_freq == 0:+ if cnt % clear_mem_freq == 0:clear_memory()cnt = 1cnt += 1@@ -883,6 +915,7 @@ class AutoRound(object):try:if hasattr(model, "hf_device_map") and len(model.hf_device_map) > 1:import accelerate+accelerate.hooks.remove_hook_from_submodules(model)# Fallback: out-of-memory → try CPU blockwise quantizationlogger.warning("Out of VRAM, falling back to blockwise quantization. Accuracy may degrade.")@@ -892,12 +925,15 @@ class AutoRound(object):except RuntimeError as e:if "CUDA out of memory" in str(e) or "MODULE:PT_DEVMEM" in str(e):# Final fallback: warn and use CPU-only quantization- logger.warning("Fallback to CPU. "- "Consider enabling `low_gpu_mem_usage` or using more GPUs via `--device 0,1,2,3`.")+ logger.warning(+ "Fallback to CPU. "+ "Consider enabling `low_gpu_mem_usage` or using more GPUs via `--device 0,1,2,3`."+ )model = model.to("cpu")clear_memory()if hasattr(model, "hf_device_map") and len(model.hf_device_map) > 1:import accelerate+accelerate.hooks.remove_hook_from_submodules(model)orig_device = self.device@@ -940,11 +976,7 @@ class AutoRound(object):raise NotImplementedError("Only one GGUF format can be set at a time.")target_format: str = formats[0]- tie_word_embeddings: bool = getattr(- getattr(self.model, "config", None),- "tie_word_embeddings",- True- )+ tie_word_embeddings: bool = getattr(getattr(self.model, "config", None), "tie_word_embeddings", True)for name, module in self.model.named_modules():if isinstance(module, torch.nn.Embedding):@@ -955,18 +987,21 @@ class AutoRound(object):if not tie_word_embeddings:lm_head_name: str = get_lm_head_name(self.model)config: dict[str, Any] = GGUF_CONFIG[GGUF_CONFIG[target_format]["lm_head"]]- check_fixed_by_user = self.layer_config[lm_head_name].get(- "fixed_by_user", False) if lm_head_name in self.layer_config else None+ check_fixed_by_user = (+ self.layer_config[lm_head_name].get("fixed_by_user", False)+ if lm_head_name in self.layer_config+ else None+ )self._apply_config_to_layer(lm_head_name, config, check_fixed_by_user=check_fixed_by_user)return Truereturn Falsedef _apply_config_to_layer(- self,- layer_name: str,- config: dict[str, Any],- check_fixed_by_user: bool = False,+ self,+ layer_name: str,+ config: dict[str, Any],+ check_fixed_by_user: bool = False,) -> None:"""Applies GGUF quantization configuration to a given layer.@@ -977,18 +1012,15 @@ class AutoRound(object):"""act_bits: int = 16scale_dtype: Any = self.scale_dtype- keys: list[str] = [- "bits", "group_size", "super_bits",- "super_group_size", "data_type", "sym"- ]+ keys: list[str] = ["bits", "group_size", "super_bits", "super_group_size", "data_type", "sym"]self.layer_config[layer_name] = self.layer_config.get(layer_name, {})for key in keys:if (- key in self.layer_config[layer_name]- and check_fixed_by_user- # and self.layer_config[layer_name].get("fixed_by_user", False)+ key in self.layer_config[layer_name]+ and check_fixed_by_user+ # and self.layer_config[layer_name].get("fixed_by_user", False)):continueself.layer_config[layer_name][key] = config.get(key)@@ -1020,6 +1052,7 @@ class AutoRound(object):# Step 1: Use optimized RTN data type if availableif not self.disable_opt_rtn and not m.data_type.startswith("rtn_"):from auto_round.data_type import QUANT_FUNC_WITH_DTYPE+rtn_dtype = "rtn_" + m.data_typeif rtn_dtype in QUANT_FUNC_WITH_DTYPE:m.data_type = rtn_dtype@@ -1065,6 +1098,7 @@ class AutoRound(object):if has_gguf:from auto_round.export.export_to_gguf.export import pack_gguf_layer+output_dir = self.get_save_folder_name(self.formats[0])model_type = ModelType.MMPROJ if self.vlm else ModelType.TEXTpack_gguf_layer(@@ -1076,11 +1110,10 @@ class AutoRound(object):self.tokenizer,processor=self.processor if hasattr(self, "processor") else None,image_processor=self.image_processor if hasattr(self, "image_processor") else None,- model_type=model_type)- else:- PACKING_LAYER_WITH_FORMAT[target_backend](- name, self.model, self.formats[0]+ model_type=model_type,)+ else:+ PACKING_LAYER_WITH_FORMAT[target_backend](name, self.model, self.formats[0])# if self.low_gpu_mem_usage:# clear_memory()@@ -1100,9 +1133,7 @@ class AutoRound(object):if self.amp:self.model.to(self.amp_dtype)- all_to_quantized_module_names: list[str] = [- n for n, m in self.model.named_modules() if check_to_quantized(m)- ]+ all_to_quantized_module_names: list[str] = [n for n, m in self.model.named_modules() if check_to_quantized(m)]has_gguf_k = any("gguf" in fmt and "k" in fmt for fmt in getattr(self, "formats", []))@@ -1111,7 +1142,7 @@ class AutoRound(object):self.model.to("cpu")if has_gguf_k and not self.disable_opt_rtn:self.quant_rtn_with_imatrix(all_to_quantized_module_names)- elif self.act_bits <=8 and self.act_dynamic is False:+ elif self.act_bits <= 8 and self.act_dynamic is False:hook_handles = self.register_act_max_hook(self.model)try:self.quantize_via_rtn_blockwise(all_to_quantized_module_names)@@ -1121,6 +1152,7 @@ class AutoRound(object):clear_memory()if hasattr(self.model, "hf_device_map") and len(self.model.hf_device_map) > 1:import accelerate+accelerate.hooks.remove_hook_from_submodules(self.model)orig_device = self.deviceself.device = "cpu"@@ -1138,7 +1170,7 @@ class AutoRound(object):for name in pbar:pbar.set_description(f"Quantizing {name}")self.quantize_layer_via_rtn(name)- if cnt % clear_mem_freq == 0:+ if cnt % clear_mem_freq == 0:clear_memory()cnt = 1cnt += 1@@ -1204,6 +1236,7 @@ class AutoRound(object):if self.device_map is not None:from accelerate import dispatch_modelfrom accelerate.hooks import AlignDevicesHook, add_hook_to_module+for _, m in block.named_modules():if len(list(m.children())) != 0 or not hasattr(m, "tuning_device"):continue@@ -1243,7 +1276,7 @@ class AutoRound(object):# Process remaining layers not in blocksfor name in all_to_quantized_module_names:self.quantize_layer_via_rtn(name)- if cnt % clear_mem_freq == 0:+ if cnt % clear_mem_freq == 0:clear_memory()cnt = 1cnt += 1@@ -1269,15 +1302,19 @@ class AutoRound(object):only_gguf = Falseif only_gguf:self.layer_config, gguf_format_config = get_layer_config_by_gguf_format(- self.layer_config, self.formats, self.model, model_type=ModelType.TEXT)+ self.layer_config, self.formats, self.model, model_type=ModelType.TEXT+ )if self.vlm:self.layer_config, gguf_format_config = get_layer_config_by_gguf_format(- self.layer_config, self.formats, self.model, model_type=ModelType.MMPROJ)+ self.layer_config, self.formats, self.model, model_type=ModelType.MMPROJ+ )# Determine if immediate packing is requiredformats = self.formats- if (len(formats) == 1 and- ("awq" in formats[0] or "gptq" in formats[0] or- "auto_round" in formats[0] or "gguf" in formats[0]) and self.inplace):+ if (+ len(formats) == 1+ and ("awq" in formats[0] or "gptq" in formats[0] or "auto_round" in formats[0] or "gguf" in formats[0])+ and self.inplace+ ):self.is_packing_immediate = Trueif self.iters == 0:return self.quantize_rtn()@@ -1298,8 +1335,9 @@ class AutoRound(object):self.start_time = time.time()all_first_block_names = [block[0] for block in all_blocks]if len(layer_names) > 0:- logger.info("Starting to cache block inputs. This may be slow due to external block layers: %s",- layer_names)+ logger.info(+ "Starting to cache block inputs. This may be slow due to external block layers: %s", layer_names+ )else:logger.info("start to cache block inputs")all_inputs = self.try_cache_inter_data_gpucpu(all_first_block_names, self.nsamples, layer_names=layer_names)@@ -1308,8 +1346,9 @@ class AutoRound(object):if is_quantized_embedding:all_inputs = copy.deepcopy(self.inputs)clear_memory(self.inputs)- all_q_inputs = self.try_cache_inter_data_gpucpu(all_first_block_names, self.nsamples,- layer_names=layer_names)+ all_q_inputs = self.try_cache_inter_data_gpucpu(+ all_first_block_names, self.nsamples, layer_names=layer_names+ )self.model = mv_module_from_gpu(self.model, self.low_cpu_mem_usage)clear_memory()if hasattr(self.model, "hf_device_map") and len(self.model.hf_device_map) > 1:@@ -1326,10 +1365,12 @@ class AutoRound(object):q_inputs = all_q_inputs[block_names[0]]all_q_inputs.pop(block_names[0])keys = inputs.keys()- input_id_str = [key for key in keys if key.startswith('hidden_state')]+ input_id_str = [key for key in keys if key.startswith("hidden_state")]if len(input_id_str) != 1:- raise RuntimeError("hidden_states arg mismatch error,"- "please raise an issue in https://github.com/intel/auto-round/issues")+ raise RuntimeError(+ "hidden_states arg mismatch error,"+ "please raise an issue in https://github.com/intel/auto-round/issues"+ )inputs["input_ids"] = inputs.pop(input_id_str[0], None)if q_inputs is not None:q_inputs["input_ids"] = q_inputs.pop(input_id_str[0], None)@@ -1349,7 +1390,7 @@ class AutoRound(object):q_input=q_inputs["input_ids"] if q_inputs is not None else None,nblocks=self.nblocks,device=self.device,- pbar=pbar+ pbar=pbar,)if self.is_packing_immediate and len(self.formats) != 1:raise ValueError(@@ -1407,8 +1448,13 @@ class AutoRound(object):logger.info("using optimized rtn method for quantizing %s", layer_name)self.layer_config[layer_name]["data_type"] = layer.data_typelayer.to(self.device)- wrapper_layer = WrapperLinear(layer, enable_round_tuning=False, enable_minmax_tuning=False,- enable_norm_bias_tuning=False, device=self.device)+ wrapper_layer = WrapperLinear(+ layer,+ enable_round_tuning=False,+ enable_minmax_tuning=False,+ enable_norm_bias_tuning=False,+ device=self.device,+ )new_layer = wrapper_layer.unwrapper({})set_module(self.model, layer_name, new_layer)layer.cpu()@@ -1429,13 +1475,12 @@ class AutoRound(object):dispatch_model(self.model, self.model.hf_device_map)if enable_quanted_input:- logger.info(- "starting to cache layer inputs for %s, this may be quite slow ",- layer_names)+ logger.info("starting to cache layer inputs for %s, this may be quite slow ", layer_names)q_layer_inputs = self.try_cache_inter_data_gpucpu([], self.nsamples, layer_names=layer_names)if hasattr(self.model, "hf_device_map") and len(self.model.hf_device_map) > 1:accelerate.hooks.remove_hook_from_submodules(- self.model) ##self.model.hf_device_map has not been changed+ self.model+ ) ##self.model.hf_device_map has not been changedself.model = mv_module_from_gpu(self.model, self.low_cpu_mem_usage)clear_memory()@@ -1481,7 +1526,7 @@ class AutoRound(object):"act_dynamic","act_data_type","super_bits",- "super_group_size"+ "super_group_size",]for n, m in self.model.named_modules():@@ -1550,8 +1595,11 @@ class AutoRound(object):layer_config[n]["in_blocks"] = False# If the layer is outside a block and requires quantization, mark it as a quantized layer outside the block- if (n not in layers_in_blocks and check_to_quantized(layer_config[n])- and not isinstance(m, torch.nn.Embedding)):+ if (+ n not in layers_in_blocks+ and check_to_quantized(layer_config[n])+ and not isinstance(m, torch.nn.Embedding)+ ):has_qlayer_outside_block = Truein_features, out_features = get_layer_features(m)@@ -1591,12 +1639,7 @@ class AutoRound(object):end_index = min(nsamples, i + bs)indices = torch.arange(i, end_index).to(torch.long)tmp_input_ids, tmp_input_others = AutoRound.sampling_inputs(- input_ids,- input_others,- indices,- self.seqlen,- self.batch_dim,- share_cache_keys=self.shared_cache_keys+ input_ids, input_others, indices, self.seqlen, self.batch_dim, share_cache_keys=self.shared_cache_keys)tmp_output = block_forward(block, tmp_input_ids, tmp_input_others, self.amp, self.amp_dtype, device).to(cache_device@@ -1625,6 +1668,7 @@ class AutoRound(object):bs (int): The number of samples to use for calibration"""from .calib_dataset import get_dataloader+if isinstance(self.dataset, str):dataset = self.dataset.replace(" ", "") ##remove all whitespaces@@ -1669,7 +1713,7 @@ class AutoRound(object):data_new = {}for key in data.keys():data_new[key] = to_device(data[key], self.model.device)- if key == 'images':+ if key == "images":data_new[key] = to_dtype(data_new[key], self.model.dtype)input_ids = data_new["input_ids"]if input_ids.shape[-1] < self.seqlen:@@ -1688,8 +1732,10 @@ class AutoRound(object):error_msg = str(error)if "The expanded size of the tensor" in str(error_msg) and "must match the existing size" in error_msg:check_seqlen_compatible(self.seqlen, self.tokenizer, self.model)- logger.warning("When quantization encounters tensor shape mismatch error, " \- "you can try to avoid it with batch_size=1")+ logger.warning(+ "When quantization encounters tensor shape mismatch error, "+ "you can try to avoid it with batch_size=1"+ )raise errorexcept Exception as error:raise error@@ -1743,13 +1789,15 @@ class AutoRound(object):except RuntimeError as e:if "CUDA out of memory" in str(e) or "MODULE:PT_DEVMEM" in str(e):logger.info("switch to cpu to cache block inputs")- if (self.has_qlayer_outside_block or- self.__class__.__name__ == "AutoRoundMLLM"):- logger.warning("we strongly recommend using more GPUs in calibration."- " Otherwise, some layers may fall back to `rtn` mode, which can affect accuracy.")+ if self.has_qlayer_outside_block or self.__class__.__name__ == "AutoRoundMLLM":+ logger.warning(+ "we strongly recommend using more GPUs in calibration."+ " Otherwise, some layers may fall back to `rtn` mode, which can affect accuracy."+ )if hasattr(self.model, "hf_device_map") and len(self.model.hf_device_map) > 1:accelerate.hooks.remove_hook_from_submodules(- self.model) ##self.model.hf_device_map has not been changed+ self.model+ ) ##self.model.hf_device_map has not been changedself.model = mv_module_from_gpu(self.model, self.low_cpu_mem_usage)clear_memory()## Important change after v0.51, on cpu, we use rtn mode for layers in layer_names@@ -1864,15 +1912,19 @@ class AutoRound(object):"this model has not been supported, ""please raise an issue in https://github.com/intel/auto-round/issues"" or try to set the `batch_size` to 1 and "- "`gradient_accumulate_steps` to your current batch size.")+ "`gradient_accumulate_steps` to your current batch size."+ )exit(-1)if hidden_states is not None:- kwargs['hidden_states'] = hidden_states+ kwargs["hidden_states"] = hidden_statesfor key in kwargs.keys():- if isinstance(kwargs[key], torch.Tensor) or isinstance(kwargs[key], list) \- or isinstance(kwargs[key], tuple):+ if (+ isinstance(kwargs[key], torch.Tensor)+ or isinstance(kwargs[key], list)+ or isinstance(kwargs[key], tuple)+ ):if key not in self.inputs[name].keys(): # initializationdata = to_device(kwargs[key], device=torch.device("cpu"))if data is None or (self.batch_size > 1 and key in self.shared_cache_keys):@@ -1898,14 +1950,15 @@ class AutoRound(object):else:# Parameters not to be cachedif check_skippable_keywords(key):- logger.warning_once(f"Please note that '{key}' key" \- " is not currently used in quantization fine-tuning.")+ logger.warning_once(+ f"Please note that '{key}' key" " is not currently used in quantization fine-tuning."+ )reset_params(self.inputs[name])if name == self.last_cache_name:raise NotImplementedErrorelse:if hidden_states is not None:- kwargs.pop('hidden_states')+ kwargs.pop("hidden_states")return m.orig_forward(hidden_states, *positional_inputs, **kwargs)else:# Currently only for Llama-3.2-Vision-Instruct Series@@ -1976,8 +2029,7 @@ class AutoRound(object):if q_inputs is not None:q_inputs[i] = q_inputs[i].to(layer.weight.dtype)- wrapper_linear = WrapperLinear(- layer, enable_minmax_tuning=self.enable_minmax_tuning, device=device).to(device)+ wrapper_linear = WrapperLinear(layer, enable_minmax_tuning=self.enable_minmax_tuning, device=device).to(device)round_params = []minmax_params = []for key in wrapper_linear.params.keys():@@ -1986,9 +2038,7 @@ class AutoRound(object):else:round_params.append(wrapper_linear.value)if len(round_params) + len(minmax_params) <= 0:- dump_info = (- f"quantized {layer_name}"- )+ dump_info = f"quantized {layer_name}"logger.info(dump_info)with torch.no_grad():unwrapper_layer(self.model, wrapper_linear, layer_name, {})@@ -2038,7 +2088,7 @@ class AutoRound(object):current_input = [inputs[i] for i in whole_indices]num_elm = sum(id.numel() for id in current_input)for tmp_step in range(gradient_accumulate_steps):- indices = whole_indices[tmp_step * batch_size: (tmp_step + 1) * batch_size]+ indices = whole_indices[tmp_step * batch_size : (tmp_step + 1) * batch_size]if q_inputs is not None:current_input = [q_inputs[i] for i in indices]current_input = torch.cat(current_input, dim=0).to(device)@@ -2110,8 +2160,12 @@ class AutoRound(object):# for whole model, RTNif n in self.layer_config:config = self.layer_config[n]- if config["bits"] <= 8 and "act_dynamic" in config and config[- "act_dynamic"] is False and check_to_quantized(config):+ if (+ config["bits"] <= 8+ and "act_dynamic" in config+ and config["act_dynamic"] is False+ and check_to_quantized(config)+ ):hook = m.register_forward_hook(get_act_max_hook)hook_handles.append(hook)continue@@ -2132,30 +2186,39 @@ class AutoRound(object):"""if self.device_map is not None:from accelerate import dispatch_model+for n, m in block.named_modules():if len(list(m.children())) != 0 or not hasattr(m, "tuning_device"):continuefrom accelerate.hooks import AlignDevicesHook, add_hook_to_module+hook = AlignDevicesHook(m.tuning_device, io_same_device=True)add_hook_to_module(m, hook, True)if q_input is None:hook_handles = self.register_act_max_hook(block)- output = self.get_block_outputs(block, input_ids, input_others, self.batch_size * self.infer_bs_coeff,- device,- self.cache_device)+ output = self.get_block_outputs(+ block, input_ids, input_others, self.batch_size * self.infer_bs_coeff, device, self.cache_device+ )for handle in hook_handles:handle.remove()else:- output = self.get_block_outputs(block, input_ids, input_others, self.batch_size * self.infer_bs_coeff,- device,- self.cache_device)+ output = self.get_block_outputs(+ block, input_ids, input_others, self.batch_size * self.infer_bs_coeff, device, self.cache_device+ )hook_handles = self.register_act_max_hook(block)if hook_handles:- self.get_block_outputs(block, q_input, input_others, self.batch_size * self.infer_bs_coeff,- device, self.cache_device, save_output=False)+ self.get_block_outputs(+ block,+ q_input,+ input_others,+ self.batch_size * self.infer_bs_coeff,+ device,+ self.cache_device,+ save_output=False,+ )for handle in hook_handles:handle.remove()@@ -2168,7 +2231,8 @@ class AutoRound(object):input_ids = q_inputquantized_layer_names, unquantized_layer_names = wrapper_block(- block, self.enable_minmax_tuning, self.enable_norm_bias_tuning, device=self.device)+ block, self.enable_minmax_tuning, self.enable_norm_bias_tuning, device=self.device+ )round_params = []minmax_params = []@@ -2230,14 +2294,14 @@ class AutoRound(object):current_input_ids = [input_ids[i] for i in whole_indices]num_elm = sum(id.numel() for id in current_input_ids)for tmp_step in range(self.gradient_accumulate_steps):- indices = whole_indices[tmp_step * self.batch_size: (tmp_step + 1) * self.batch_size]+ indices = whole_indices[tmp_step * self.batch_size : (tmp_step + 1) * self.batch_size]current_input_ids, current_input_others = AutoRound.sampling_inputs(input_ids,input_others,indices,seqlen=self.seqlen,batch_dim=self.batch_dim,- share_cache_keys=self.shared_cache_keys+ share_cache_keys=self.shared_cache_keys,)current_output = [output[x] for x in indices]@@ -2296,12 +2360,15 @@ class AutoRound(object):block = block.to(device)clear_memory()q_outputs = self.get_block_outputs(- block, input_ids, input_others, self.batch_size * self.infer_bs_coeff, device,- cache_device=self.cache_device+ block,+ input_ids,+ input_others,+ self.batch_size * self.infer_bs_coeff,+ device,+ cache_device=self.cache_device,)if self.device_map is not None:- accelerate.hooks.remove_hook_from_submodules(- block)+ accelerate.hooks.remove_hook_from_submodules(block)mv_module_from_gpu(block, self.low_cpu_mem_usage)clear_memory(input_ids)@@ -2309,21 +2376,13 @@ class AutoRound(object):else:if self.device_map is not None:- accelerate.hooks.remove_hook_from_submodules(- block)+ accelerate.hooks.remove_hook_from_submodules(block)mv_module_from_gpu(block, self.low_cpu_mem_usage)clear_memory(input_ids)return None, outputdef quant_blocks(- self,- model: torch.nn.Module,- inputs,- block_names,- q_input=None,- nblocks=1,- device="cpu",- pbar=None+ self, model: torch.nn.Module, inputs, block_names, q_input=None, nblocks=1, device="cpu", pbar=None):"""Quantize and dequantize the weights of the specified blocks in the model.@@ -2353,7 +2412,7 @@ class AutoRound(object):for key in input_others.keys():if isinstance(input_others[key], torch.Tensor) and (- input_others[key].dtype == torch.float16 or input_others[key].dtype == torch.bfloat16+ input_others[key].dtype == torch.float16 or input_others[key].dtype == torch.bfloat16):input_others[key] = input_others[key].to(tmp_dtype)elif isinstance(input_others[key], list):@@ -2375,7 +2434,7 @@ class AutoRound(object):pbar.set_description(f"Quantizing {n}")m = get_module(model, n)else:- names = block_names[i: min(i + nblocks, len(block_names))]+ names = block_names[i : min(i + nblocks, len(block_names))]pbar.set_description(f"Quantizing [{i + 1}-{min(i + nblocks, len(block_names))}]/{len(block_names)}")modules = [get_module(model, n) for n in names]m = WrapperMultiblock(modules)@@ -2392,12 +2451,14 @@ class AutoRound(object):)if self.is_packing_immediate:from auto_round.export import PACKING_LAYER_WITH_FORMAT+for _, tmp_m in m.named_modules():if hasattr(tmp_m, "bits") and check_to_quantized(tmp_m):target_backend = self.formats[0].split(":")[0] if ":" in self.formats[0] else self.formats[0]has_gguf = any("gguf" in format_ for format_ in self.formats)if has_gguf:from auto_round.export.export_to_gguf.export import pack_gguf_layer+output_dir = self.get_save_folder_name(self.formats[0])model_type = ModelType.MMPROJ if self.vlm else ModelType.TEXTpack_gguf_layer(@@ -2409,7 +2470,8 @@ class AutoRound(object):self.tokenizer,processor=self.processor if hasattr(self, "processor") else None,image_processor=self.image_processor if hasattr(self, "image_processor") else None,- model_type=model_type)+ model_type=model_type,+ )else:PACKING_LAYER_WITH_FORMAT[target_backend](tmp_m.tmp_name, self.model, self.formats[0])pbar.set_description("Quantizing done")@@ -2443,7 +2505,7 @@ class AutoRound(object):format = self._check_supported_format(format)if self.low_cpu_mem_usage:- self.model = self.model.to('cpu')+ self.model = self.model.to("cpu")if not self.quantized:logger.warning("please run autoround.quantize first")@@ -2460,9 +2522,11 @@ class AutoRound(object):if self.act_bits <= 8 and format == "qdq":logger.warning("Support for exporting activation quantization is limited. "- "Please ensure that your configuration is supported.")+ "Please ensure that your configuration is supported."+ )from auto_round.export import EXPORT_FORMAT+backend = formatformat = format.split(":")[0]if format not in EXPORT_FORMAT:@@ -2511,7 +2575,7 @@ class AutoRound(object):backend=backend,to_quant_block_names=self.to_quant_block_names,quant_block_list=self.quant_block_list,- **kwargs+ **kwargs,)return compressed_model@@ -2612,8 +2676,7 @@ class AutoRound(object):@classmethod@torch.no_grad()- def sampling_inputs(cls, input_ids, input_others, indices, seqlen,- batch_dim=0, share_cache_keys=()):+ def sampling_inputs(cls, input_ids, input_others, indices, seqlen, batch_dim=0, share_cache_keys=()):"""Samples inputs based on the given indices and sequence length.Args:@@ -2634,8 +2697,9 @@ class AutoRound(object):for key in input_others.keys():if "positional_inputs" in key:continue- if (key not in share_cache_keys or len(indices) == 1) \- and not isinstance(input_others[key], (str, bool, type(None))):+ if (key not in share_cache_keys or len(indices) == 1) and not isinstance(+ input_others[key], (str, bool, type(None))+ ):current_input_others[key] = Noneif input_others[key] is not None:current_input_others[key] = [input_others[key][i] for i in indices]@@ -2702,49 +2766,49 @@ class AutoRoundOPT(AutoRound):"""def __init__(- self,- model,- tokenizer=None,- bits: int = 4,- group_size: int = 128,- sym: bool = True,- layer_config=None,- batch_size: int = 8,- amp: bool = True,- device=None,- lr_scheduler=None,- dataset: Union[str, list, tuple, torch.utils.data.DataLoader] = "NeelNanda/pile-10k",- enable_quanted_input: bool = True,- enable_minmax_tuning: bool = True,- lr: float = None,- minmax_lr: float = None,- low_gpu_mem_usage: bool = False,- low_cpu_mem_usage: bool = False,- iters: int = 200,- seqlen: int = 2048,- nsamples: int = 128,- sampler: str = "rand",- seed: int = 42,- nblocks: int = 1,- gradient_accumulate_steps: int = 1,- not_use_best_mse: bool = False,- dynamic_max_gap: int = -1,- data_type: str = "int",- scale_dtype: str = "fp16",- act_bits: int = 16,- act_group_size: int = None,- act_sym: bool = None,- act_data_type: str = None,- act_dynamic: bool = True,- to_quant_block_names: Union[str, list] = None,- enable_norm_bias_tuning: bool = False,- enable_torch_compile: bool = False,- device_map: Union[str, dict] = None,- optimizer="AdamW",- super_bits: int = None,- super_group_size: int = None,- disable_opt_rtn: bool = False,- **kwargs,+ self,+ model,+ tokenizer=None,+ bits: int = 4,+ group_size: int = 128,+ sym: bool = True,+ layer_config=None,+ batch_size: int = 8,+ amp: bool = True,+ device=None,+ lr_scheduler=None,+ dataset: Union[str, list, tuple, torch.utils.data.DataLoader] = "NeelNanda/pile-10k",+ enable_quanted_input: bool = True,+ enable_minmax_tuning: bool = True,+ lr: float = None,+ minmax_lr: float = None,+ low_gpu_mem_usage: bool = False,+ low_cpu_mem_usage: bool = False,+ iters: int = 200,+ seqlen: int = 2048,+ nsamples: int = 128,+ sampler: str = "rand",+ seed: int = 42,+ nblocks: int = 1,+ gradient_accumulate_steps: int = 1,+ not_use_best_mse: bool = False,+ dynamic_max_gap: int = -1,+ data_type: str = "int",+ scale_dtype: str = "fp16",+ act_bits: int = 16,+ act_group_size: int = None,+ act_sym: bool = None,+ act_data_type: str = None,+ act_dynamic: bool = True,+ to_quant_block_names: Union[str, list] = None,+ enable_norm_bias_tuning: bool = False,+ enable_torch_compile: bool = False,+ device_map: Union[str, dict] = None,+ optimizer="AdamW",+ super_bits: int = None,+ super_group_size: int = None,+ disable_opt_rtn: bool = False,+ **kwargs,):super(AutoRoundOPT, self).__init__(model=model,@@ -2881,49 +2945,49 @@ class AutoRoundAdam(AutoRoundOPT):"""def __init__(- self,- model,- tokenizer=None,- bits: int = 4,- group_size: int = 128,- sym: bool = True,- layer_config=None,- batch_size: int = 8,- amp: bool = True,- device=None,- lr_scheduler=None,- dataset: Union[str, list, tuple, torch.utils.data.DataLoader] = "NeelNanda/pile-10k",- enable_quanted_input: bool = True,- enable_minmax_tuning: bool = True,- lr: float = None,- minmax_lr: float = None,- low_gpu_mem_usage: bool = False,- low_cpu_mem_usage: bool = False,- iters: int = 200,- seqlen: int = 2048,- nsamples: int = 128,- sampler: str = "rand",- seed: int = 42,- nblocks: int = 1,- gradient_accumulate_steps: int = 1,- not_use_best_mse: bool = False,- dynamic_max_gap: int = -1,- data_type: str = "int",- scale_dtype: str = "fp16",- act_bits: int = 16,- act_group_size: int = None,- act_sym: bool = None,- act_data_type: str = None,- act_dynamic: bool = True,- to_quant_block_names: Union[str, list] = None,- enable_norm_bias_tuning: bool = False,- enable_torch_compile: bool = False,- device_map: Union[str, dict] = None,- optimizer="AdamW",- super_bits: int = None,- super_group_size: int = None,- disable_opt_rtn: bool = False,- **kwargs,+ self,+ model,+ tokenizer=None,+ bits: int = 4,+ group_size: int = 128,+ sym: bool = True,+ layer_config=None,+ batch_size: int = 8,+ amp: bool = True,+ device=None,+ lr_scheduler=None,+ dataset: Union[str, list, tuple, torch.utils.data.DataLoader] = "NeelNanda/pile-10k",+ enable_quanted_input: bool = True,+ enable_minmax_tuning: bool = True,+ lr: float = None,+ minmax_lr: float = None,+ low_gpu_mem_usage: bool = False,+ low_cpu_mem_usage: bool = False,+ iters: int = 200,+ seqlen: int = 2048,+ nsamples: int = 128,+ sampler: str = "rand",+ seed: int = 42,+ nblocks: int = 1,+ gradient_accumulate_steps: int = 1,+ not_use_best_mse: bool = False,+ dynamic_max_gap: int = -1,+ data_type: str = "int",+ scale_dtype: str = "fp16",+ act_bits: int = 16,+ act_group_size: int = None,+ act_sym: bool = None,+ act_data_type: str = None,+ act_dynamic: bool = True,+ to_quant_block_names: Union[str, list] = None,+ enable_norm_bias_tuning: bool = False,+ enable_torch_compile: bool = False,+ device_map: Union[str, dict] = None,+ optimizer="AdamW",+ super_bits: int = None,+ super_group_size: int = None,+ disable_opt_rtn: bool = False,+ **kwargs,):super(AutoRoundAdam, self).__init__(model=model,@@ -2968,5 +3032,3 @@ class AutoRoundAdam(AutoRoundOPT):super_group_size=super_group_size,**kwargs,)--diff --git a/auto_round/calib_dataset.py b/auto_round/calib_dataset.pyindex 6f6ec957b..1f02fe79d 100644--- a/auto_round/calib_dataset.py+++ b/auto_round/calib_dataset.py@@ -69,15 +69,12 @@ def apply_chat_template_to_samples(samples, tokenizer, seqlen, system_prompt=Nonadd_generation_prompt=True,)except:- logger.warning(- "Failed to apply chat template. removing the system role in chat history."- )+ logger.warning("Failed to apply chat template. removing the system role in chat history.")message_modified = [msg for msg in message if msg["role"] != "system"]chat_templated = tokenizer.apply_chat_template(message_modified,tokenize=False,add_generation_prompt=True,-)rendered_messages.append(chat_templated)@@ -108,8 +105,15 @@ def get_tokenizer_function(tokenizer, seqlen, apply_chat_template=False, system_@register_dataset(["NeelNanda/pile-10k", "pile-10k"])-def get_pile_dataset(tokenizer, seqlen, dataset_name="NeelNanda/pile-10k", split=None, seed=42,- apply_chat_template=False, system_prompt=None):+def get_pile_dataset(+ tokenizer,+ seqlen,+ dataset_name="NeelNanda/pile-10k",+ split=None,+ seed=42,+ apply_chat_template=False,+ system_prompt=None,+):"""Returns a dataloader for the specified dataset and split.Args:@@ -127,18 +131,22 @@ def get_pile_dataset(tokenizer, seqlen, dataset_name="NeelNanda/pile-10k", splitsplit = "train"- tokenizer_function = get_tokenizer_function(tokenizer, seqlen, apply_chat_template=apply_chat_template,- system_prompt=system_prompt)+ tokenizer_function = get_tokenizer_function(+ tokenizer, seqlen, apply_chat_template=apply_chat_template, system_prompt=system_prompt+ )try:calib_dataset = load_dataset("NeelNanda/pile-10k", split=split)except Exception as e:import ssl+error_message = str(e)# Check for proxy or SSL errorif "proxy" in error_message.lower() or isinstance(e, ssl.SSLError) or "SSL" in error_message.upper():- logger.error("Network error detected, please checking proxy settings." \- "Error: {error_message}. Or consider using a backup dataset by `pip install modelscope`" \- " and set '--dataset swift/pile-val-backup' in AutoRound API.")+ logger.error(+ "Network error detected, please checking proxy settings."+ "Error: {error_message}. Or consider using a backup dataset by `pip install modelscope`"+ " and set '--dataset swift/pile-val-backup' in AutoRound API."+ )else:logger.error(f"Failed to load the dataset: {error_message}")sys.exit(1)@@ -149,8 +157,15 @@ def get_pile_dataset(tokenizer, seqlen, dataset_name="NeelNanda/pile-10k", split@register_dataset(["swift/pile-val-backup", "pile-val-backup"])-def get_pile_val_dataset(tokenizer, seqlen, dataset_name="swift/pile-val-backup", split=None, seed=42,- apply_chat_template=False, system_prompt=None):+def get_pile_val_dataset(+ tokenizer,+ seqlen,+ dataset_name="swift/pile-val-backup",+ split=None,+ seed=42,+ apply_chat_template=False,+ system_prompt=None,+):"""Returns a dataloader for the specified dataset and split.Args:@@ -167,14 +182,20 @@ def get_pile_val_dataset(tokenizer, seqlen, dataset_name="swift/pile-val-backup"split = "validation"- tokenizer_function = get_tokenizer_function(tokenizer, seqlen, apply_chat_template=apply_chat_template,- system_prompt=system_prompt)+ tokenizer_function = get_tokenizer_function(+ tokenizer, seqlen, apply_chat_template=apply_chat_template, system_prompt=system_prompt+ )from transformers.utils.versions import require_version- require_version("modelscope", "Loading 'swift/pile-val-backup' dataset requires modelscope to be installed, " \- "`pip install modelscope`")++ require_version(+ "modelscope",+ "Loading 'swift/pile-val-backup' dataset requires modelscope to be installed, " "`pip install modelscope`",+ )from modelscope import MsDataset # pylint: disable=E0401- calib_dataset = MsDataset.load('swift/pile-val-backup',- 'default', split=split).to_iterable_dataset() # , use_streaming=True++ calib_dataset = MsDataset.load(+ "swift/pile-val-backup", "default", split=split+ ).to_iterable_dataset() # , use_streaming=Truecalib_dataset = calib_dataset.shuffle(seed=seed).take(10000)calib_dataset = calib_dataset.map(tokenizer_function, batched=True)@@ -182,8 +203,9 @@ def get_pile_val_dataset(tokenizer, seqlen, dataset_name="swift/pile-val-backup"@register_dataset(["BAAI/CCI3-HQ", "CCI3-HQ"])-def get_cci3_hq_dataset(tokenizer, seqlen, dataset_name="BAAI/CCI3-HQ", split=None, seed=42, apply_chat_template=False,- system_prompt=None):+def get_cci3_hq_dataset(+ tokenizer, seqlen, dataset_name="BAAI/CCI3-HQ", split=None, seed=42, apply_chat_template=False, system_prompt=None+):"""Returns a dataloader for the specified dataset and split.Args:@@ -199,10 +221,11 @@ def get_cci3_hq_dataset(tokenizer, seqlen, dataset_name="BAAI/CCI3-HQ", split=No"""from datasets import load_dataset- tokenizer_function = get_tokenizer_function(tokenizer, seqlen, apply_chat_template=apply_chat_template,- system_prompt=system_prompt)+ tokenizer_function = get_tokenizer_function(+ tokenizer, seqlen, apply_chat_template=apply_chat_template, system_prompt=system_prompt+ )- calib_dataset = load_dataset("BAAI/CCI3-HQ", split='train', streaming=True)+ calib_dataset = load_dataset("BAAI/CCI3-HQ", split="train", streaming=True)calib_dataset = calib_dataset.shuffle(seed=seed).take(10000)calib_dataset = calib_dataset.map(tokenizer_function, batched=True)@@ -210,8 +233,15 @@ def get_cci3_hq_dataset(tokenizer, seqlen, dataset_name="BAAI/CCI3-HQ", split=No@register_dataset(["codeparrot/github-code-clean", "github-code-clean"])-def get_github_code_clean_dataset(tokenizer, seqlen, dataset_name="codeparrot/github-code-clean", split=None, seed=42,- apply_chat_template=False, system_prompt=None):+def get_github_code_clean_dataset(+ tokenizer,+ seqlen,+ dataset_name="codeparrot/github-code-clean",+ split=None,+ seed=42,+ apply_chat_template=False,+ system_prompt=None,+):"""Returns a dataloader for the specified dataset and split.Args:@@ -242,18 +272,21 @@ def get_github_code_clean_dataset(tokenizer, seqlen, dataset_name="codeparrot/giif not apply_chat_template:example = tokenizer(examples["code"], truncation=True, max_length=seqlen)else:- example = apply_chat_template_to_samples(examples["code"], tokenizer, seqlen,- system_prompt=system_prompt)+ example = apply_chat_template_to_samples(+ examples["code"], tokenizer, seqlen, system_prompt=system_prompt+ )return examplereturn default_tokenizer_functiontokenizer_function = get_default_tokenizer_function()try:- dataset_mit = load_dataset("codeparrot/github-code-clean", "all-mit", split='train',- trust_remote_code=True, streaming=True).shuffle(seed=seed)- dataset_apache = load_dataset("codeparrot/github-code-clean", "all-apache-2.0", split='train',- trust_remote_code=True, streaming=True).shuffle(seed=seed)+ dataset_mit = load_dataset(+ "codeparrot/github-code-clean", "all-mit", split="train", trust_remote_code=True, streaming=True+ ).shuffle(seed=seed)+ dataset_apache = load_dataset(+ "codeparrot/github-code-clean", "all-apache-2.0", split="train", trust_remote_code=True, streaming=True+ ).shuffle(seed=seed)except RuntimeError as error:if "Dataset scripts are no longer supported" in str(error):raise RuntimeError(@@ -263,7 +296,7 @@ def get_github_code_clean_dataset(tokenizer, seqlen, dataset_name="codeparrot/gielse:raise errorcalib_dataset = concatenate_datasets([dataset_mit, dataset_apache])- calib_dataset = calib_dataset.shuffle(seed=seed).take(10000) ##TODO concat data'shuffle may have bugs+ calib_dataset = calib_dataset.shuffle(seed=seed).take(10000) ##TODO concat data'shuffle may have bugscalib_dataset = calib_dataset.map(tokenizer_function, batched=True)return calib_dataset@@ -271,13 +304,13 @@ def get_github_code_clean_dataset(tokenizer, seqlen, dataset_name="codeparrot/gi@register_dataset(["HuggingFaceH4/ultrachat_200k", "ultrachat_200k"])def get_ultrachat_dataset(- tokenizer,- seqlen,- dataset_name="HuggingFaceH4/ultrachat_200k",- split=None,- seed=42,- apply_chat_template=True,- system_prompt=None,+ tokenizer,+ seqlen,+ dataset_name="HuggingFaceH4/ultrachat_200k",+ split=None,+ seed=42,+ apply_chat_template=True,+ system_prompt=None,):if split is None:split = "train_sft"@@ -285,8 +318,7 @@ def get_ultrachat_dataset(if split not in all_splits:raise ValueError("split must be one of {} for ultrachat_200k ".format(all_splits))- dataset = load_dataset("HuggingFaceH4/ultrachat_200k", split=split,- streaming=True, trust_remote_code=True)+ dataset = load_dataset("HuggingFaceH4/ultrachat_200k", split=split, streaming=True, trust_remote_code=True)dataset = dataset.shuffle(seed=seed).take(20000)def is_instruct_tokenizer(tokenizer):@@ -313,9 +345,7 @@ def get_ultrachat_dataset(texts.append(combined)return tokenizer(texts, truncation=True, max_length=seqlen)else:- return apply_chat_template_to_samples(- examples["messages"], tokenizer, seqlen, system_prompt=system_prompt- )+ return apply_chat_template_to_samples(examples["messages"], tokenizer, seqlen, system_prompt=system_prompt)dataset = dataset.map(tokenize_example_batch, batched=True)return dataset@@ -323,28 +353,25 @@ def get_ultrachat_dataset(@register_dataset(["openbmb/Ultra-FineWeb", "openbmb/Ultra-FineWeb"])def get_ultrafinweb_dataset(- tokenizer,- seqlen,- dataset_name="openbmb/Ultra-FineWeb",- split=None,- seed=42,- apply_chat_template=True,- system_prompt=None,+ tokenizer,+ seqlen,+ dataset_name="openbmb/Ultra-FineWeb",+ split=None,+ seed=42,+ apply_chat_template=True,+ system_prompt=None,):if split is not None:if split not in ["en", "zh"]:raise ValueError("split must be one of ['en', 'zh'] for Ultra-FineWeb dataset")- calib_dataset = load_dataset("openbmb/Ultra-FineWeb", split=split,- streaming=True, trust_remote_code=True)+ calib_dataset = load_dataset("openbmb/Ultra-FineWeb", split=split, streaming=True, trust_remote_code=True)else:- calib_dataset = load_dataset("openbmb/Ultra-FineWeb", split='en',- streaming=True, trust_remote_code=True)+ calib_dataset = load_dataset("openbmb/Ultra-FineWeb", split="en", streaming=True, trust_remote_code=True)# dataset_ch = load_dataset("openbmb/Ultra-FineWeb", split='zh',# streaming=True, trust_remote_code=True).shuffle(seed=seed).take(2000)# calib_dataset = concatenate_datasets([dataset_en, dataset_ch]) ##concat dasetset could not shuffle-calib_dataset = calib_dataset.shuffle(seed=seed).take(20000)def get_default_tokenizer_function():@@ -352,8 +379,9 @@ def get_ultrafinweb_dataset(if not apply_chat_template:example = tokenizer(examples["content"], truncation=True, max_length=seqlen)else:- example = apply_chat_template_to_samples(examples["content"], tokenizer, seqlen,- system_prompt=system_prompt)+ example = apply_chat_template_to_samples(+ examples["content"], tokenizer, seqlen, system_prompt=system_prompt+ )return examplereturn default_tokenizer_function@@ -366,13 +394,13 @@ def get_ultrafinweb_dataset(@register_dataset(["madao33/new-title-chinese", "new-title-chinese"])def get_new_chinese_title_dataset(- tokenizer,- seqlen,- dataset_name="madao33/new-title-chinese",- split=None,- seed=42,- apply_chat_template=False,- system_prompt=None+ tokenizer,+ seqlen,+ dataset_name="madao33/new-title-chinese",+ split=None,+ seed=42,+ apply_chat_template=False,+ system_prompt=None,):"""Returns a tokenized dataset for the specified parameters.@@ -405,8 +433,9 @@ def get_new_chinese_title_dataset(if not apply_chat_template:example = tokenizer(examples["content"], truncation=True, max_length=seqlen)else:- example = apply_chat_template_to_samples(examples["content"], tokenizer, seqlen,- system_prompt=system_prompt)+ example = apply_chat_template_to_samples(+ examples["content"], tokenizer, seqlen, system_prompt=system_prompt+ )return examplereturn default_tokenizer_function@@ -424,8 +453,9 @@ def get_new_chinese_title_dataset(@register_dataset("mbpp")-def get_mbpp_dataset(tokenizer, seqlen, dataset_name="mbpp", split=None, seed=42, apply_chat_template=False,- system_prompt=None):+def get_mbpp_dataset(+ tokenizer, seqlen, dataset_name="mbpp", split=None, seed=42, apply_chat_template=False, system_prompt=None+):"""Returns a dataloader for the specified dataset and split.Args:@@ -441,8 +471,9 @@ def get_mbpp_dataset(tokenizer, seqlen, dataset_name="mbpp", split=None, seed=42"""from datasets import load_dataset- tokenizer_function = get_tokenizer_function(tokenizer, seqlen, apply_chat_template=apply_chat_template,- system_prompt=system_prompt)+ tokenizer_function = get_tokenizer_function(+ tokenizer, seqlen, apply_chat_template=apply_chat_template, system_prompt=system_prompt+ )samples = []splits = split@@ -465,8 +496,9 @@ def get_mbpp_dataset(tokenizer, seqlen, dataset_name="mbpp", split=None, seed=42@register_dataset("local")-def get_local_dataset(tokenizer, seqlen, dataset_name="./tmp.json", split=None, seed=42, apply_chat_template=False,- system_prompt=None):+def get_local_dataset(+ tokenizer, seqlen, dataset_name="./tmp.json", split=None, seed=42, apply_chat_template=False, system_prompt=None+):"""Returns a dataloader for a custom dataset and split.We allow the input of a json or text file containing a processed text sample each line.@@ -481,8 +513,9 @@ def get_local_dataset(tokenizer, seqlen, dataset_name="./tmp.json", split=None,Returns:A dataloader for a custom dataset and split, using the provided tokenizer and sequence length."""- tokenizer_function = get_tokenizer_function(tokenizer, seqlen, apply_chat_template=apply_chat_template,- system_prompt=system_prompt)+ tokenizer_function = get_tokenizer_function(+ tokenizer, seqlen, apply_chat_template=apply_chat_template, system_prompt=system_prompt+ )def load_local_data(data_path):if data_path.endswith(".json"):@@ -590,18 +623,19 @@ def select_dataset(dataset, indices):except:list_data = list(select(dataset, indices))import pandas as pd+df = pd.DataFrame(list_data)dataset = Dataset.from_pandas(df)return datasetdef get_dataloader(- tokenizer,- seqlen,- dataset_name="NeelNanda/pile-10k",- seed=42,- bs=8,- nsamples=512,+ tokenizer,+ seqlen,+ dataset_name="NeelNanda/pile-10k",+ seed=42,+ bs=8,+ nsamples=512,):"""Generate a DataLoader for calibration using specified parameters.@@ -635,7 +669,7 @@ def get_dataloader(return Truedef concat_dataset_element(dataset):- input_ids, concat_input_ids = [eg['input_ids'] for eg in dataset], []+ input_ids, concat_input_ids = [eg["input_ids"] for eg in dataset], []attention_mask_list, attention_mask = [], torch.ones([1, seqlen]).to(torch.int64)buffer_input_id = torch.Tensor().to(torch.int64)bos_token_id, eos_token_id = tokenizer.bos_token_id, tokenizer.eos_token_id@@ -672,8 +706,9 @@ def get_dataloader(concat_input_ids.append(torch.cat(input_id_to_append).to(torch.int64))attention_mask_list.append(attention_mask)buffer_input_id = torch.Tensor().to(torch.int64)- data = [{'input_ids': a, 'attention_mask': b} for a, b in zip(concat_input_ids, attention_mask_list)]+ data = [{"input_ids": a, "attention_mask": b} for a, b in zip(concat_input_ids, attention_mask_list)]import datasets+dataset_new = datasets.Dataset.from_list(data)return dataset_new@@ -687,15 +722,15 @@ def get_dataloader(if ":" in name:name, split_list = name.split(":")[0], name.split(":")[1:]for ele in split_list:- key, values = ele.split('=')[0], ele.split('=')[1:]+ key, values = ele.split("=")[0], ele.split("=")[1:]if key == "split":- split = values[0].split('+')+ split = values[0].split("+")if key == "num":data_lens[name] = int(values[0])if key == "concat":- do_concat = False if (len(values) > 0 and values[0].lower() == 'false') else True+ do_concat = False if (len(values) > 0 and values[0].lower() == "false") else Trueif key == "apply_chat_template":- apply_chat_template = False if (len(values) > 0 and values[0].lower() == 'false') else True+ apply_chat_template = False if (len(values) > 0 and values[0].lower() == "false") else Trueif key == "system_prompt":system_prompt = values[0]apply_chat_template = True@@ -704,14 +739,14 @@ def get_dataloader(else:calib_name = nameif name not in CALIB_DATASETS.keys():- calib_name = name.split('/')[-1]+ calib_name = name.split("/")[-1]for key in CALIB_DATASETS.keys():if calib_name in key:calib_name = keybreakget_dataset = CALIB_DATASETS.get(calib_name)if get_dataset is None:- filtered_keys = [k for k in CALIB_DATASETS.keys() if '/' not in k]+ filtered_keys = [k for k in CALIB_DATASETS.keys() if "/" not in k]raise ValueError(f"Dataset '{name}' is not found. Please choose from the supported datasets: {filtered_keys}.")@@ -722,7 +757,7 @@ def get_dataloader(split=split,dataset_name=name,apply_chat_template=apply_chat_template,- system_prompt=system_prompt+ system_prompt=system_prompt,)if do_concat:dataset = concat_dataset_element(dataset)@@ -735,9 +770,9 @@ def get_dataloader(new_features = {}for k, v in dataset.features.items():if k == "input_ids":- new_features[k] = Sequence(Value('int64'))+ new_features[k] = Sequence(Value("int64"))elif k == "attention_mask":- new_features[k] = Sequence(Value('int8'))+ new_features[k] = Sequence(Value("int8"))else:new_features[k] = v@@ -764,10 +799,13 @@ def get_dataloader(cnt = 0for i in range(len(datasets)):- name = dataset_names[i].split(':')[0]+ name = dataset_names[i].split(":")[0]if name not in data_lens:- target_cnt = (nsamples - cnt) // (len(datasets) - len(data_lens)) if data_lens \+ target_cnt = (+ (nsamples - cnt) // (len(datasets) - len(data_lens))+ if data_lenselse (nsamples - cnt) // (len(datasets) - i)+ )target_cnt = min(target_cnt, lens[i])cnt += target_cntelse:@@ -811,4 +849,3 @@ def get_dataloader(calib_dataloader = DataLoader(dataset_final, batch_size=bs, shuffle=False, collate_fn=collate_batch)return calib_dataloader-diff --git a/auto_round/data_type/__init__.py b/auto_round/data_type/__init__.pyindex 5502fc659..03e9160e1 100644--- a/auto_round/data_type/__init__.py+++ b/auto_round/data_type/__init__.py@@ -20,4 +20,3 @@ import auto_round.data_type.w4fp8from auto_round.data_type.utils import get_quant_funcimport auto_round.data_type.nvfpimport auto_round.data_type.gguf-diff --git a/auto_round/data_type/fp8.py b/auto_round/data_type/fp8.pyindex 6253d662e..0fbd20020 100644--- a/auto_round/data_type/fp8.py+++ b/auto_round/data_type/fp8.py@@ -24,8 +24,8 @@ from auto_round.data_type.utils import ()-@register_dtype(("fp8_sym","fp8","fp8_e4m3"))-def quant_fp8_sym(tensor, max_scale=1.0, tensor_max=None, group_size=-1, v=0,**kwargs):+@register_dtype(("fp8_sym", "fp8", "fp8_e4m3"))+def quant_fp8_sym(tensor, max_scale=1.0, tensor_max=None, group_size=-1, v=0, **kwargs):"""Symmetric quantization using float8 format.Allows both dynamic per-token scaling and tensor-wide quantization depending on input.@@ -44,14 +44,13 @@ def quant_fp8_sym(tensor, max_scale=1.0, tensor_max=None, group_size=-1, v=0,**k"""info = torch.finfo(torch.float8_e4m3fn)orig_dtype = tensor.dtype- tensor,orig_shape,pad_len = reshape_pad_tensor_by_group_size(tensor, group_size)+ tensor, orig_shape, pad_len = reshape_pad_tensor_by_group_size(tensor, group_size)if isinstance(max_scale, torch.Tensor):max_scale = max_scale.to(tensor.device)if isinstance(v, torch.Tensor):v = v.to(tensor.device)if tensor_max is None: ##dynamic per-token- max_tensor = torch.max(torch.abs(tensor), dim=-1)[- 0] * max_scale+ max_tensor = torch.max(torch.abs(tensor), dim=-1)[0] * max_scaleelif isinstance(tensor_max, torch.Tensor):max_tensor = tensor_max.to(tensor.device) * max_scaleelse:@@ -62,7 +61,7 @@ def quant_fp8_sym(tensor, max_scale=1.0, tensor_max=None, group_size=-1, v=0,**kif tensor.dtype == torch.float16: ## Avoid NaN gradients with float16tensor = tensor.to(torch.bfloat16)scale = scale.unsqueeze(dim=-1)- fp8_res = (tensor / scale+v)+ fp8_res = tensor / scale + vfp8_res = torch.clip(fp8_res, info.min, info.max)fp8_res = float8_e4m3fn_ste(fp8_res)qdq_res = fp8_res * scale@@ -70,8 +69,9 @@ def quant_fp8_sym(tensor, max_scale=1.0, tensor_max=None, group_size=-1, v=0,**kqdq_res = qdq_res.to(orig_dtype)return qdq_res, scale, None+@register_dtype("fp8_e5m2")-def quant_fp8_e5m2(tensor, max_scale=1.0, tensor_max=None, group_size=-1, v=0,**kwargs):+def quant_fp8_e5m2(tensor, max_scale=1.0, tensor_max=None, group_size=-1, v=0, **kwargs):"""Symmetric quantization using float8 format.Allows both dynamic per-token scaling and tensor-wide quantization depending on input.@@ -90,10 +90,9 @@ def quant_fp8_e5m2(tensor, max_scale=1.0, tensor_max=None, group_size=-1, v=0,**"""info = torch.finfo(torch.float8_e5m2)orig_dtype = tensor.dtype- tensor,orig_shape,pad_len = reshape_pad_tensor_by_group_size(tensor, group_size)+ tensor, orig_shape, pad_len = reshape_pad_tensor_by_group_size(tensor, group_size)if tensor_max is None: ##dynamic per-token- max_tensor = torch.max(torch.abs(tensor), dim=-1)[- 0] * max_scale+ max_tensor = torch.max(torch.abs(tensor), dim=-1)[0] * max_scaleelif isinstance(tensor_max, torch.Tensor):max_tensor = tensor_max.to(tensor.device) * max_scaleelse:@@ -104,7 +103,7 @@ def quant_fp8_e5m2(tensor, max_scale=1.0, tensor_max=None, group_size=-1, v=0,**if tensor.dtype == torch.float16: ## Avoid NaN gradients with float16tensor = tensor.to(torch.bfloat16)scale = scale.unsqueeze(dim=-1)- fp8_res = (tensor / scale+v)+ fp8_res = tensor / scale + vfp8_res = torch.clip(fp8_res, info.min, info.max)fp8_res = float8_e5m2_ste(fp8_res)qdq_res = fp8_res * scale@@ -114,7 +113,7 @@ def quant_fp8_e5m2(tensor, max_scale=1.0, tensor_max=None, group_size=-1, v=0,**@register_dtype("fp8_unit_scale")-def quant_fp8_unit_scale(tensor, max_scale=1.0, tensor_max=None, group_size=-1, v=0,**kwargs):+def quant_fp8_unit_scale(tensor, max_scale=1.0, tensor_max=None, group_size=-1, v=0, **kwargs):"""Symmetric quantization using float8 format.Allows both dynamic per-token scaling and tensor-wide quantization depending on input.@@ -133,11 +132,11 @@ def quant_fp8_unit_scale(tensor, max_scale=1.0, tensor_max=None, group_size=-1,"""info = torch.finfo(torch.float8_e4m3fn)orig_dtype = tensor.dtype- tensor,orig_shape,pad_len = reshape_pad_tensor_by_group_size(tensor, group_size)+ tensor, orig_shape, pad_len = reshape_pad_tensor_by_group_size(tensor, group_size)if tensor.dtype == torch.float16: ## Avoid NaN gradients with float16tensor = tensor.to(torch.bfloat16)scale = torch.ones((1), device=tensor.device)- fp8_res = (tensor / scale+v)+ fp8_res = tensor / scale + vfp8_res = torch.clip(fp8_res, info.min, info.max)fp8_res = float8_e4m3fn_ste(fp8_res)qdq_res = fp8_res * scale@@ -147,7 +146,7 @@ def quant_fp8_unit_scale(tensor, max_scale=1.0, tensor_max=None, group_size=-1,@register_dtype("fp8_e5m2_unit_scale")-def quant_fp8_e5m2_unit_scale(tensor, max_scale=1.0, tensor_max=None, group_size=-1, v=0,**kwargs):+def quant_fp8_e5m2_unit_scale(tensor, max_scale=1.0, tensor_max=None, group_size=-1, v=0, **kwargs):"""Symmetric quantization using float8 format.Allows both dynamic per-token scaling and tensor-wide quantization depending on input.@@ -166,11 +165,11 @@ def quant_fp8_e5m2_unit_scale(tensor, max_scale=1.0, tensor_max=None, group_size"""info = torch.finfo(torch.float8_e5m2)orig_dtype = tensor.dtype- tensor,orig_shape,pad_len = reshape_pad_tensor_by_group_size(tensor, group_size)+ tensor, orig_shape, pad_len = reshape_pad_tensor_by_group_size(tensor, group_size)if tensor.dtype == torch.float16: ## Avoid NaN gradients with float16tensor = tensor.to(torch.bfloat16)scale = torch.ones((1), device=tensor.device)- fp8_res = (tensor / scale+v)+ fp8_res = tensor / scale + vfp8_res = torch.clip(fp8_res, info.min, info.max)fp8_res = float8_e5m2_ste(fp8_res)qdq_res = fp8_res * scale@@ -179,7 +178,6 @@ def quant_fp8_e5m2_unit_scale(tensor, max_scale=1.0, tensor_max=None, group_sizereturn qdq_res, scale, None-@register_dtype("fp8_gaudi3_sym")def quant_fp8_sym_gaudi3(tensor, max_scale=1.0, tensor_max=None, **kwargs):"""Symmetric quantization using float8 format.@@ -204,8 +202,7 @@ def quant_fp8_sym_gaudi3(tensor, max_scale=1.0, tensor_max=None, **kwargs):if tensor_max is None: ##dynamic per-tetensor = tensor.reshape(-1, orig_shape[-1])- max_tensor = torch.max(torch.abs(tensor), dim=-1)[- 0] * max_scale+ max_tensor = torch.max(torch.abs(tensor), dim=-1)[0] * max_scaleelif isinstance(tensor_max, torch.Tensor):max_tensor = tensor_max.clone().detach().to(tensor.device) * max_scaleelse:@@ -216,7 +213,7 @@ def quant_fp8_sym_gaudi3(tensor, max_scale=1.0, tensor_max=None, **kwargs):if tensor.dtype == torch.float16: ## Avoid NaN gradients with float16tensor = tensor.to(torch.bfloat16)scale = scale.unsqueeze(dim=-1)- fp8_res = (tensor / scale)+ fp8_res = tensor / scalefp8_res = torch.clip(fp8_res, -fp8_max, fp8_max)float8_e4m3fn_ste_gaudi = get_gaudi_fp8_ste_func()fp8_res = float8_e4m3fn_ste_gaudi(fp8_res)diff --git a/auto_round/data_type/gguf.py b/auto_round/data_type/gguf.pyindex 0d094f41d..fa120938a 100644--- a/auto_round/data_type/gguf.py+++ b/auto_round/data_type/gguf.py@@ -21,19 +21,20 @@ from auto_round.utils import get_reciprocal@register_dtype("int_sym_dq")def quant_tensor_sym_dq(- tensor,- bits=4,- group_size=-1,- v=0,- min_scale=1.0,- max_scale=1.0,- scale_dtype=torch.float16,- tensor_min=None,- tensor_max=None,- q_scale_thresh=1e-5,- super_group_size=16,- super_bits=6,- **kwargs):+ tensor,+ bits=4,+ group_size=-1,+ v=0,+ min_scale=1.0,+ max_scale=1.0,+ scale_dtype=torch.float16,+ tensor_min=None,+ tensor_max=None,+ q_scale_thresh=1e-5,+ super_group_size=16,+ super_bits=6,+ **kwargs,+):"""Quantize and de-quantize tensor symmetrically. full range, credict goes to llamacpp communityArgs:@@ -72,7 +73,7 @@ def quant_tensor_sym_dq(scale = scale.view(-1, 1)zp = torch.full_like(scale, maxq) # pylint: disable=E1130int_w = round_ste(tensor * get_reciprocal(scale) + v)- q = torch.clamp(int_w + zp, 0, 2 ** bits - 1)+ q = torch.clamp(int_w + zp, 0, 2**bits - 1)qdq_result = (scale * (q - zp)).to(tensor.dtype)qdq_result = revert_tensor_by_pad(qdq_result, orig_shape=orig_shape, pad_len=pad_len)return qdq_result, {"scale": scale, "d_scale": d_scale}, zp@@ -80,17 +81,18 @@ def quant_tensor_sym_dq(@register_dtype("int_asym_float_zp")def quant_tensor_asym_float_zp(- tensor,- bits=4,- group_size=-1,- v=0,- min_scale=1.0,- max_scale=1.0,- scale_dtype=torch.float16,- tensor_min=None,- tensor_max=None,- q_scale_thresh=1e-5,- **kwargs):+ tensor,+ bits=4,+ group_size=-1,+ v=0,+ min_scale=1.0,+ max_scale=1.0,+ scale_dtype=torch.float16,+ tensor_min=None,+ tensor_max=None,+ q_scale_thresh=1e-5,+ **kwargs,+):"""Quantize and de-quantize tensor asymmetrically.Args:@@ -109,7 +111,7 @@ def quant_tensor_asym_float_zp(Quantized and de-quantized tensor, scale, zero-point"""tensor, orig_shape, pad_len = reshape_pad_tensor_by_group_size(tensor, group_size)- maxq = 2 ** bits - 1+ maxq = 2**bits - 1if tensor_min is None or tensor_max is None:wmin_tmp = torch.clamp(tensor.min(-1)[0], max=0)wmax_tmp = torch.clamp(tensor.max(-1)[0], min=0)@@ -137,7 +139,7 @@ def quant_tensor_asym_float_zp(## the values should be positivedef double_quant_tensor(tensor, bits):tensor = tensor.to(torch.float32) # Ensure tensor is in float32 for precision- maxq = 2 ** bits - 1+ maxq = 2**bits - 1wmax = torch.clamp(tensor.max(-1)[0], min=0)scale = wmax / maxqscale = scale.view(-1, 1)@@ -210,9 +212,21 @@ def make_qp_quants(nmax, data, quant_weights):@register_dtype("int_asym_dq")-def quant_tensor_asym_dq(tensor, bits=4, group_size=-1, v=0, min_scale=1.0, max_scale=1.0, scale_dtype=torch.float16,- tensor_min=None, tensor_max=None, q_scale_thresh=1e-5, super_group_size=8, super_bits=6,- **kwargs):+def quant_tensor_asym_dq(+ tensor,+ bits=4,+ group_size=-1,+ v=0,+ min_scale=1.0,+ max_scale=1.0,+ scale_dtype=torch.float16,+ tensor_min=None,+ tensor_max=None,+ q_scale_thresh=1e-5,+ super_group_size=8,+ super_bits=6,+ **kwargs,+):"""Quantize and de-quantize tensor asymmetrically.Args:@@ -231,7 +245,7 @@ def quant_tensor_asym_dq(tensor, bits=4, group_size=-1, v=0, min_scale=1.0, max_Quantized and de-quantized tensor, scale, zero-point"""tensor, orig_shape, pad_len = reshape_pad_tensor_by_group_size(tensor, group_size)- maxq = 2 ** bits - 1+ maxq = 2**bits - 1if tensor_min is None or tensor_max is None:wmin_tmp = torch.clamp(tensor.min(-1)[0], max=0)wmax_tmp = torch.clamp(tensor.max(-1)[0], min=0)@@ -270,17 +284,18 @@ def quant_tensor_asym_dq(tensor, bits=4, group_size=-1, v=0, min_scale=1.0, max_@register_dtype("rtn_int_asym_dq")def quant_tensor_gguf_asym_dq(- tensor,- bits=4,- v=0,- min_scale=1.0,- max_scale=1.0,- scale_dtype=torch.float16,- tensor_min=None,- tensor_max=None,- q_scale_thresh=1e-5,- imatrix=None,- **kwargs):+ tensor,+ bits=4,+ v=0,+ min_scale=1.0,+ max_scale=1.0,+ scale_dtype=torch.float16,+ tensor_min=None,+ tensor_max=None,+ q_scale_thresh=1e-5,+ imatrix=None,+ **kwargs,+):"""Quantizes and dequantizes a tensor using asymmetric integer quantization for formats like Q2_K, Q4_K, and Q5_K.Only fit for iters 0@@ -303,7 +318,7 @@ def quant_tensor_gguf_asym_dq(Tuple: (Quantized-dequantized tensor, scale dictionary, zero-point dictionary)"""orig_dtype = tensor.dtype- maxq = 2 ** bits - 1+ maxq = 2**bits - 1group_size = 16 if bits == 2 else 32super_bits = 4 if bits == 2 else 6super_group_size = 16 if bits == 2 else 8@@ -316,18 +331,23 @@ def quant_tensor_gguf_asym_dq(search_kwargs = {2: {"rmin": -0.5, "rdelta": 0.1, "nstep": 15, "use_mad": True},4: {"rmin": -1, "rdelta": 0.1, "nstep": 20, "use_mad": False},- 5: {"rmin": -0.5, "rdelta": 0.1, "nstep": 15, "use_mad": False}+ 5: {"rmin": -0.5, "rdelta": 0.1, "nstep": 15, "use_mad": False},}if bits == 2:quant_weights = torch.abs(tensor)elif bits == 4 or bits == 5:- sigma2 = torch.sum(tensor ** 2, dim=-1, keepdim=True) / 32 ##Note 32 is different from QK_K+ sigma2 = torch.sum(tensor**2, dim=-1, keepdim=True) / 32 ##Note 32 is different from QK_Kav_x = torch.sqrt(sigma2)quant_weights = torch.abs(tensor) + av_xparams = search_kwargs[bits]scale, wmin = iterative_wls_quant_search(- tensor, bits=bits, rrmin=params["rmin"], rdelta=params["rdelta"], nstep=params["nstep"],- use_mad=params["use_mad"], weights=quant_weights+ tensor,+ bits=bits,+ rrmin=params["rmin"],+ rdelta=params["rdelta"],+ nstep=params["nstep"],+ use_mad=params["use_mad"],+ weights=quant_weights,)scale = scale.to(scale_dtype)scale = torch.where(torch.abs(scale) < 1e-30, torch.zeros_like(scale), scale)@@ -353,7 +373,8 @@ def quant_tensor_gguf_asym_dq(if torch.min(quant_weights) == 0:logger.warning_once(- "please use more data via setting `nsamples` to improve accuracy as calibration activations contain 0")+ "please use more data via setting `nsamples` to improve accuracy as calibration activations contain 0"+ )zero_cnt = torch.sum(quant_weights == 0, dim=-1)replace_index = zero_cnt > group_size // 2@@ -362,7 +383,7 @@ def quant_tensor_gguf_asym_dq(if bits == 2:tmp_quant_weights = torch.abs(tensor)elif bits == 4 or bits == 5:- sigma2 = torch.sum(tensor ** 2, dim=-1, keepdim=True) / 32 ## Note 32 is different from QK_K+ sigma2 = torch.sum(tensor**2, dim=-1, keepdim=True) / 32 ## Note 32 is different from QK_Kav_x = torch.sqrt(sigma2)tmp_quant_weights = torch.abs(tensor) + av_xquant_weights[replace_index, :] = tmp_quant_weights[replace_index, :]@@ -384,12 +405,17 @@ def quant_tensor_gguf_asym_dq(params = search_kwargs[bits]scale, wmin_0 = iterative_wls_quant_search(- tensor, bits=bits, rrmin=params["rmin"], rdelta=params["rdelta"], nstep=params["nstep"],- use_mad=params["use_mad"], weights=quant_weights+ tensor,+ bits=bits,+ rrmin=params["rmin"],+ rdelta=params["rdelta"],+ nstep=params["nstep"],+ use_mad=params["use_mad"],+ weights=quant_weights,)scale = scale.to(scale_dtype)scale = torch.where(torch.abs(scale) < 1e-30, torch.zeros_like(scale), scale)- nmax = 2 ** super_bits - 1+ nmax = 2**super_bits - 1scale = scale.reshape(-1, super_group_size)wmin = wmin_0.reshape(-1, super_group_size)sum_quant_weights = quant_weights.sum(-1, keepdim=True).reshape(-1, super_group_size)@@ -426,7 +452,7 @@ def iterative_wls_quant_search(data, bits=4, rrmin=-1.0, rdelta=0.1, nstep=20, u"""dtype = torch.float32data = data.to(dtype)- maxq = 2 ** bits - 1+ maxq = 2**bits - 1minq = 0weights = 1.0 if weights is None else weights.to(dtype)@@ -443,7 +469,7 @@ def iterative_wls_quant_search(data, bits=4, rrmin=-1.0, rdelta=0.1, nstep=20, uquant_data = torch.clamp(torch.round(iscale * (data - rmin)), minq, maxq)diff = scale * quant_data + rmin - data- best_mad = torch.sum((weights * torch.abs(diff)) if use_mad else weights * diff ** 2, dim=1, keepdim=True)+ best_mad = torch.sum((weights * torch.abs(diff)) if use_mad else weights * diff**2, dim=1, keepdim=True)for is_ in range(nstep):factor = rrmin + rdelta * is_ + maxq - minq@@ -457,7 +483,7 @@ def iterative_wls_quant_search(data, bits=4, rrmin=-1.0, rdelta=0.1, nstep=20, usum_l2 = torch.sum(mul_weights_quant_data * quant_data_new, dim=-1, keepdim=True)sum_xl = torch.sum(mul_weights_quant_data * data, dim=-1, keepdim=True)- D = sum_w * sum_l2 - sum_l ** 2+ D = sum_w * sum_l2 - sum_l**2this_scale = (sum_w * sum_xl - sum_x * sum_l) / Dthis_min = (sum_l2 * sum_x - sum_l * sum_xl) / Dthis_min[this_min > 0] = 0@@ -467,7 +493,7 @@ def iterative_wls_quant_search(data, bits=4, rrmin=-1.0, rdelta=0.1, nstep=20, uquant_data = torch.clamp(torch.round(reverse_this_scale * (data - this_min)), minq, maxq)diff = this_scale * quant_data + this_min - data# diff = this_scale * quant_data_new + this_min - data- mad = torch.sum((weights * torch.abs(diff)) if use_mad else weights * diff ** 2, dim=-1, keepdim=True)+ mad = torch.sum((weights * torch.abs(diff)) if use_mad else weights * diff**2, dim=-1, keepdim=True)idx_to_replace = torch.where((mad < best_mad) & (D > 0))[0]best_mad[idx_to_replace] = mad[idx_to_replace]@@ -479,17 +505,18 @@ def iterative_wls_quant_search(data, bits=4, rrmin=-1.0, rdelta=0.1, nstep=20, u@register_dtype("rtn_int_sym_dq")def quant_tensor_gguf_sym_dq(- tensor,- bits=3,- v=0,- min_scale=1.0,- max_scale=1.0,- scale_dtype=torch.float16,- tensor_min=None,- tensor_max=None,- q_scale_thresh=1e-5,- imatrix=None,- **kwargs):+ tensor,+ bits=3,+ v=0,+ min_scale=1.0,+ max_scale=1.0,+ scale_dtype=torch.float16,+ tensor_min=None,+ tensor_max=None,+ q_scale_thresh=1e-5,+ imatrix=None,+ **kwargs,+):"""Quantize and de-quantize tensor asymmetrically. For Q3_K, Q6_K.Args:@@ -552,7 +579,8 @@ def quant_tensor_gguf_sym_dq(quant_weights = weights.reshape(tensor.shape)if torch.min(quant_weights) == 0:logger.warning_once(- "please use more data via setting `nsamples` to improve accuracy as calibration activations contain 0")+ "please use more data via setting `nsamples` to improve accuracy as calibration activations contain 0"+ )zero_cnt = torch.sum(quant_weights == 0, dim=-1)replace_index = zero_cnt > group_size // 2@@ -560,7 +588,7 @@ def quant_tensor_gguf_sym_dq(if bits == 6:quant_weights[replace_index, :] = 1.0else:- sigma2 = 2 * torch.sum(tensor ** 2, dim=-1, keepdim=True) / QK_K+ sigma2 = 2 * torch.sum(tensor**2, dim=-1, keepdim=True) / QK_Ktmp_quant_weights = torch.sqrt(sigma2 + tensor * tensor)quant_weights[replace_index, :] = tmp_quant_weights[replace_index, :]mean_replace_index = (zero_cnt > 0) & (zero_cnt <= group_size // 2)diff --git a/auto_round/data_type/int.py b/auto_round/data_type/int.pyindex be5d7cf33..3a50b870f 100644--- a/auto_round/data_type/int.py+++ b/auto_round/data_type/int.py@@ -19,9 +19,19 @@ from auto_round.data_type.utils import reshape_pad_tensor_by_group_size, revert_@register_dtype("int_sym")-def quant_tensor_sym(tensor, bits=4, group_size=-1, v=0, min_scale=1.0, max_scale=1.0, scale_dtype=torch.float16,- tensor_min=None,- tensor_max=None, q_scale_thresh=1e-5, **kwargs):+def quant_tensor_sym(+ tensor,+ bits=4,+ group_size=-1,+ v=0,+ min_scale=1.0,+ max_scale=1.0,+ scale_dtype=torch.float16,+ tensor_min=None,+ tensor_max=None,+ q_scale_thresh=1e-5,+ **kwargs+):"""Quantize and de-quantize tensor asymmetrically. full range, credict goes to llamacpp communityArgs:@@ -58,15 +68,26 @@ def quant_tensor_sym(tensor, bits=4, group_size=-1, v=0, min_scale=1.0, max_scalscale = scale.unsqueeze(dim=-1)zp = zp.unsqueeze(dim=-1)int_w = round_ste(tensor / scale + v)- q = torch.clamp(int_w + zp, 0, 2 ** bits - 1)+ q = torch.clamp(int_w + zp, 0, 2**bits - 1)qdq_result = (scale * (q - zp)).to(tensor.dtype)qdq_result = revert_tensor_by_pad(qdq_result, orig_shape=orig_shape, pad_len=pad_len)return qdq_result, scale, zp@register_dtype("int_asym")-def quant_tensor_asym(tensor, bits=4, group_size=-1, v=0, min_scale=1.0, max_scale=1.0, scale_dtype=torch.float16,- tensor_min=None, tensor_max=None, q_scale_thresh=1e-5, **kwargs):+def quant_tensor_asym(+ tensor,+ bits=4,+ group_size=-1,+ v=0,+ min_scale=1.0,+ max_scale=1.0,+ scale_dtype=torch.float16,+ tensor_min=None,+ tensor_max=None,+ q_scale_thresh=1e-5,+ **kwargs+):"""Quantize and de-quantize tensor asymmetrically.Args:@@ -85,7 +106,7 @@ def quant_tensor_asym(tensor, bits=4, group_size=-1, v=0, min_scale=1.0, max_scaQuantized and de-quantized tensor, scale, zero-point"""tensor, orig_shape, pad_len = reshape_pad_tensor_by_group_size(tensor, group_size)- maxq = 2 ** bits - 1+ maxq = 2**bits - 1if tensor_min is None or tensor_max is None:wmin_tmp = torch.clamp(tensor.min(-1)[0], max=0)wmax_tmp = torch.clamp(tensor.max(-1)[0], min=0)@@ -111,9 +132,19 @@ def quant_tensor_asym(tensor, bits=4, group_size=-1, v=0, min_scale=1.0, max_sca@register_dtype("int_sym_gptq")-def quant_tensor_sym_gptq(tensor, bits=4, group_size=-1, v=0, min_scale=1.0, max_scale=1.0, scale_dtype=torch.float16,- tensor_min=None,- tensor_max=None, q_scale_thresh=1e-5, **kwargs):+def quant_tensor_sym_gptq(+ tensor,+ bits=4,+ group_size=-1,+ v=0,+ min_scale=1.0,+ max_scale=1.0,+ scale_dtype=torch.float16,+ tensor_min=None,+ tensor_max=None,+ q_scale_thresh=1e-5,+ **kwargs+):"""Quantize and de-quantize tensor asymmetrically.Args:@@ -132,7 +163,7 @@ def quant_tensor_sym_gptq(tensor, bits=4, group_size=-1, v=0, min_scale=1.0, maxQuantized and de-quantized tensor, scale, zero-point"""tensor, orig_shape, pad_len = reshape_pad_tensor_by_group_size(tensor, group_size)- maxq = 2 ** bits - 1+ maxq = 2**bits - 1if tensor_min is None or tensor_max is None:wmin_tmp = torch.clamp(tensor.min(-1)[0], max=0)wmax_tmp = torch.clamp(tensor.max(-1)[0], min=0)@@ -164,9 +195,19 @@ def quant_tensor_sym_gptq(tensor, bits=4, group_size=-1, v=0, min_scale=1.0, maxreturn qdq_result, scale, zp-def quant_tensor_asym_wo_round(tensor, bits=4, group_size=-1, v=0, min_scale=1.0, max_scale=1.0,- scale_dtype=torch.float16,- tensor_min=None, tensor_max=None, q_scale_thresh=1e-5, **kwargs):+def quant_tensor_asym_wo_round(+ tensor,+ bits=4,+ group_size=-1,+ v=0,+ min_scale=1.0,+ max_scale=1.0,+ scale_dtype=torch.float16,+ tensor_min=None,+ tensor_max=None,+ q_scale_thresh=1e-5,+ **kwargs+):"""Quantize and de-quantize tensor asymmetrically without rounding, this is mainly for tuning bias, norm.Args:@@ -185,7 +226,7 @@ def quant_tensor_asym_wo_round(tensor, bits=4, group_size=-1, v=0, min_scale=1.0Quantized and de-quantize tensor, scale, zero-point"""tensor, orig_shape, pad_len = reshape_pad_tensor_by_group_size(tensor, group_size)- maxq = 2 ** bits - 1+ maxq = 2**bits - 1if tensor_min is None or tensor_max is None:wmin_tmp = torch.clamp(tensor.min(-1)[0], max=0)wmax_tmp = torch.clamp(tensor.max(-1)[0], min=0)diff --git a/auto_round/data_type/mxfp.py b/auto_round/data_type/mxfp.pyindex 6c6290445..e8a9258d5 100644--- a/auto_round/data_type/mxfp.py+++ b/auto_round/data_type/mxfp.py@@ -38,8 +38,8 @@ MXFP_FORMAT_CACHE = {"mx_fp4e2m1": (2, 3, 2, 6.0, 1.0),"mx_float16": (5, 12, 15, 65504.0, 6.103515625e-05),"mx_fp16": (5, 12, 15, 65504.0, 6.103515625e-05),- "mx_bfloat16": (8, 9, 127, 3.3895313892515355e+38, 1.1754943508222875e-38),- "mx_bf16": (8, 9, 127, 3.3895313892515355e+38, 1.1754943508222875e-38),+ "mx_bfloat16": (8, 9, 127, 3.3895313892515355e38, 1.1754943508222875e-38),+ "mx_bf16": (8, 9, 127, 3.3895313892515355e38, 1.1754943508222875e-38),}FP32_EXPONENT_BIAS = 127@@ -56,7 +56,7 @@ def quant_element(tensor, ebits, mbits, max_norm, mantissa_rounding="even"):private_exp = None# Scale up so appropriate number of mbits are in the integer portion of the number- tensor = tensor * (2 ** (mbits - 2)) if private_exp is None else tensor / (2 ** private_exp) * (2 ** (mbits - 2))+ tensor = tensor * (2 ** (mbits - 2)) if private_exp is None else tensor / (2**private_exp) * (2 ** (mbits - 2))if mantissa_rounding == "even":abs_tensor = torch.abs(tensor)mask_tensor = ((abs_tensor - 0.5) % 2 == torch.zeros_like(abs_tensor)).type(tensor.dtype)@@ -69,15 +69,14 @@ def quant_element(tensor, ebits, mbits, max_norm, mantissa_rounding="even"):raise ValueError("mantissa_rounding only supports even, nearest or floor.")# Undo scaling- tensor = tensor / (2 ** (mbits - 2)) if private_exp is None else tensor / (2 ** (mbits - 2)) * (2 ** private_exp)+ tensor = tensor / (2 ** (mbits - 2)) if private_exp is None else tensor / (2 ** (mbits - 2)) * (2**private_exp)tensor = torch.clamp(tensor, min=-max_norm, max=max_norm)return tensor@torch.compile()-def quant_mx(tensor, bits=4, group_size=-1, v=0, max_scale=1.0,- mantissa_rounding="even", data_type="mx_fp", **kwargs):+def quant_mx(tensor, bits=4, group_size=-1, v=0, max_scale=1.0, mantissa_rounding="even", data_type="mx_fp", **kwargs):"""Quantize the given tensor using the specified parameters.This function performs quantization on the `tensor` tensor according to the@@ -128,8 +127,9 @@ def quant_mx(tensor, bits=4, group_size=-1, v=0, max_scale=1.0,@torch.compile()-def quant_mx_rceil(tensor, bits=4, group_size=-1, v=0, max_scale=1.0,- mantissa_rounding="even", data_type="mx_fp", **kwargs):+def quant_mx_rceil(+ tensor, bits=4, group_size=-1, v=0, max_scale=1.0, mantissa_rounding="even", data_type="mx_fp", **kwargs+):"""Quantize the given tensor using the specified parameters.This function performs quantization on the `tensor` tensor according to the@@ -187,8 +187,8 @@ if __name__ == "__main__":data = torch.tensor([0.0, 0.25, 0.4, 0.75, 1.25, 1.4, 1.75, 2.5, 2.9, 3.5, 5.0, 5.1])data1 = quant_element(data, 2, 3, 6.0)gt = torch.tensor([0.0, 0.0, 0.5, 1.0, 1.0, 1.5, 2.0, 2.0, 3.0, 4.0, 4.0, 6.0])- assert (torch.sum(torch.abs(data1 - gt)) < 1e-6)+ assert torch.sum(torch.abs(data1 - gt)) < 1e-6data_neg = data * -1data2 = quant_element(data_neg, 2, 3, 6.0)- assert (torch.sum(torch.abs(data2 - gt * -1)) < 1e-6)+ assert torch.sum(torch.abs(data2 - gt * -1)) < 1e-6diff --git a/auto_round/data_type/nvfp.py b/auto_round/data_type/nvfp.pyindex 147264c68..b39320100 100644--- a/auto_round/data_type/nvfp.py+++ b/auto_round/data_type/nvfp.py@@ -91,7 +91,7 @@ def float_to_e5m3_frexp(x: torch.Tensor) -> torch.Tensor:x_masked = x[mask]# 正常数:x >= 2^-14- normal_mask = x_masked >= 2 ** -14+ normal_mask = x_masked >= 2**-14x_normal = x_masked[normal_mask]mantissa, exponent = torch.frexp(x_normal)@@ -103,7 +103,7 @@ def float_to_e5m3_frexp(x: torch.Tensor) -> torch.Tensor:# sumnorm:0 < x < 2^-14subnormal_mask = ~normal_maskx_subnormal = x_masked[subnormal_mask]- m_sub = torch.clamp(torch.round(x_subnormal / (2 ** -14) * 8), 1, 7).to(torch.uint8) # exponent = 0+ m_sub = torch.clamp(torch.round(x_subnormal / (2**-14) * 8), 1, 7).to(torch.uint8) # exponent = 0e5m3_sub = m_sub # top 5 bits = 0out_vals = torch.zeros_like(x_masked, dtype=torch.uint8)@@ -123,20 +123,20 @@ def e5m3_to_float_tensor(e5m3: torch.Tensor) -> torch.Tensor:m = (e5m3[mask_nonzero] & 0x07).to(torch.int32)is_nan = (e == 31) & (m == 7)- is_subnormal = (e == 0)+ is_subnormal = e == 0is_normal = (e > 0) & (~is_nan)out = torch.zeros_like(e, dtype=torch.float32)# subnormal: exponent = -14, no implicit leading 1- out[is_subnormal] = (m[is_subnormal].float() / 8.0) * (2 ** -14)+ out[is_subnormal] = (m[is_subnormal].float() / 8.0) * (2**-14)# normal: exponent = e - 15, implicit leading 1mant = 1.0 + m[is_normal].float() / 8.0exp = e[is_normal] - 15out[is_normal] = torch.ldexp(mant, exp)- out[is_nan] = float('nan')+ out[is_nan] = float("nan")x[mask_nonzero] = outreturn x@@ -175,7 +175,7 @@ def ref_fp4_quant(x, global_scale, block_size=16, v=0, max_scale=1.0):@register_dtype("fp4_v2_with_global_scale")@torch.compile()def fp4_v2_with_global_scale(tensor, bits=4, group_size=16, v=0, max_scale=1.0, **kwargs):- assert (group_size == 32 or group_size == 16)+ assert group_size == 32 or group_size == 16orig_dtype = tensor.dtypetensor, orig_shape, pad_len = reshape_pad_tensor_by_group_size(tensor, group_size)tensor_amax = tensor.abs().max().to(torch.float32)@@ -188,7 +188,7 @@ def fp4_v2_with_global_scale(tensor, bits=4, group_size=16, v=0, max_scale=1.0,@register_dtype("fp4_v2")@torch.compile()def fp4_v2(tensor, bits=4, group_size=32, v=0, max_scale=1.0, **kwargs):- assert (group_size == 32 or group_size == 16)+ assert group_size == 32 or group_size == 16orig_dtype = tensor.dtypetensor, orig_shape, pad_len = reshape_pad_tensor_by_group_size(tensor, group_size)global_scale = 1.0@@ -200,11 +200,25 @@ def fp4_v2(tensor, bits=4, group_size=32, v=0, max_scale=1.0, **kwargs):if __name__ == "__main__":test = torch.tensor(- [0.0, 1e-38, 2 ** (-17), (2 ** -14) * 0.875, 2 ** -14, 2 ** -13, 2 ** -6,- 1e-6, 2.7657e-05, 0.1, 1.0, 3.14, 1000.0,- 114688,- 1e10],- dtype=torch.float32)+ [+ 0.0,+ 1e-38,+ 2 ** (-17),+ (2**-14) * 0.875,+ 2**-14,+ 2**-13,+ 2**-6,+ 1e-6,+ 2.7657e-05,+ 0.1,+ 1.0,+ 3.14,+ 1000.0,+ 114688,+ 1e10,+ ],+ dtype=torch.float32,+ )encoded = float_to_e5m3_frexp(test)decoded = e5m3_to_float_tensor(encoded)decoded_bf16 = decoded.to(torch.bfloat16)@@ -213,4 +227,5 @@ if __name__ == "__main__":for i in range(len(test)):print(f"{test[i].item():.6g} -> {encoded[i].item():3d} -> {decoded[i].item():.6g} "- f"(error={abs(test[i] - decoded[i]).item():.3g})")+ f"(error={abs(test[i] - decoded[i]).item():.3g})"+ )diff --git a/auto_round/data_type/register.py b/auto_round/data_type/register.pyindex 6f22528ca..12c4406a4 100644--- a/auto_round/data_type/register.py+++ b/auto_round/data_type/register.py@@ -30,7 +30,7 @@ def register_dtype(names):"""def register(dtype):- if isinstance(names,(tuple,list)):+ if isinstance(names, (tuple, list)):for name in names:QUANT_FUNC_WITH_DTYPE[name] = dtypeelse:diff --git a/auto_round/data_type/utils.py b/auto_round/data_type/utils.pyindex ea8c56ffa..09178ed94 100644--- a/auto_round/data_type/utils.py+++ b/auto_round/data_type/utils.py@@ -40,8 +40,8 @@ def reshape_pad_tensor_by_group_size(data: torch.Tensor, group_size: int):"""orig_shape = data.shapepad_len = 0- if group_size==0:- data = data.reshape(1,-1)+ if group_size == 0:+ data = data.reshape(1, -1)return data, orig_shape, pad_lenif len(data.shape) > 2:data = data.reshape(-1, orig_shape[-1])@@ -87,17 +87,17 @@ def revert_tensor_by_pad(data: torch.Tensor, orig_shape: tuple, pad_len: int):def get_quant_func(dtype, bits, sym):"""Retrieve the quantization function based on data type, bit width, and symmetry.- This function returns the appropriate quantization function from the QUANT_FUNC_WITH_DTYPE- dictionary based on the provided data type (`dtype`), bit width (`bits`), and whether- the quantization is symmetric (`sym`). If the function does not exist, raise ValueError.+ This function returns the appropriate quantization function from the QUANT_FUNC_WITH_DTYPE+ dictionary based on the provided data type (`dtype`), bit width (`bits`), and whether+ the quantization is symmetric (`sym`). If the function does not exist, raise ValueError.- Args:- dtype (str): The data type for the quantization (e.g., 'int', 'mxfp4').- bits (int): The bit width for the quantization (e.g., 2,4,8).- sym (bool): A flag indicating whether the quantization is symmetric (True) or asymmetric (False).+ Args:+ dtype (str): The data type for the quantization (e.g., 'int', 'mxfp4').+ bits (int): The bit width for the quantization (e.g., 2,4,8).+ sym (bool): A flag indicating whether the quantization is symmetric (True) or asymmetric (False).- Returns:- function: The quantization function corresponding to the specified parameters.+ Returns:+ function: The quantization function corresponding to the specified parameters."""key = dtypeif key in QUANT_FUNC_WITH_DTYPE.keys():@@ -121,9 +121,9 @@ def get_quant_func(dtype, bits, sym):return QUANT_FUNC_WITH_DTYPE[key], keyif sym:- key = dtype + "_sym"+ key = dtype + "_sym"else:- key = dtype + "_asym"+ key = dtype + "_asym"if key in QUANT_FUNC_WITH_DTYPE.keys():return QUANT_FUNC_WITH_DTYPE[key], key@@ -237,8 +237,3 @@ def get_gaudi_fp8_ste_func():fn = float8_e4m3fn_stelogger.warning_once("Using CUDA/CPU STE for FP8")return fn-----diff --git a/auto_round/data_type/w4fp8.py b/auto_round/data_type/w4fp8.pyindex 1dd550b22..d7e722f29 100644--- a/auto_round/data_type/w4fp8.py+++ b/auto_round/data_type/w4fp8.py@@ -194,8 +194,9 @@ from auto_round.data_type.utils import float8_e4m3fn_ste, get_gaudi_fp8_ste_func@register_dtype("fp8_to_int_sym")-def progressive_quant_fp8_int4(tensor, bits=4, group_size=-1, v=0, min_scale=1.0, max_scale=1.0,- q_scale_thresh=1e-5, **kwargs):+def progressive_quant_fp8_int4(+ tensor, bits=4, group_size=-1, v=0, min_scale=1.0, max_scale=1.0, q_scale_thresh=1e-5, **kwargs+):"""Two-stage quantization: quantize tensor to fp8 by per tensor, then quantize fp8 to w4g128This method first quantizes the input tensor into float8 format and then performs@@ -231,13 +232,18 @@ def progressive_quant_fp8_int4(tensor, bits=4, group_size=-1, v=0, min_scale=1.0fp8_res_using_16bit = fp8_res.to(tensor.dtype)##convert to int4from auto_round.data_type.int import quant_tensor_sym- qdq_int4_tensor, scale_fp8_to_int4, zp_fp8_to_int4 = quant_tensor_sym(fp8_res_using_16bit, bits=bits,- group_size=group_size, v=v,- min_scale=min_scale,- max_scale=max_scale,- scale_dtype=torch.bfloat16,- q_scale_thresh=q_scale_thresh)++ qdq_int4_tensor, scale_fp8_to_int4, zp_fp8_to_int4 = quant_tensor_sym(+ fp8_res_using_16bit,+ bits=bits,+ group_size=group_size,+ v=v,+ min_scale=min_scale,+ max_scale=max_scale,+ scale_dtype=torch.bfloat16,+ q_scale_thresh=q_scale_thresh,+ )qdq_tensor = qdq_int4_tensor * bf16_to_fp8_scalebf16_to_int4_scale = scale_fp8_to_int4 * bf16_to_fp8_scale- return qdq_tensor, {"scale": bf16_to_int4_scale, "bf16_to_fp8_scale": bf16_to_fp8_scale}, zp_fp8_to_int4\ No newline at end of file+ return qdq_tensor, {"scale": bf16_to_int4_scale, "bf16_to_fp8_scale": bf16_to_fp8_scale}, zp_fp8_to_int4diff --git a/auto_round/eval/__init__.py b/auto_round/eval/__init__.pyindex e3fdc07b1..28f108cb6 100644--- a/auto_round/eval/__init__.py+++ b/auto_round/eval/__init__.py@@ -10,4 +10,4 @@# distributed under the License is distributed on an "AS IS" BASIS,# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.# See the License for the specific language governing permissions and-# limitations under the License.\ No newline at end of file+# limitations under the License.diff --git a/auto_round/eval/evaluation.py b/auto_round/eval/evaluation.pyindex 293379bab..79fbafb23 100644--- a/auto_round/eval/evaluation.py+++ b/auto_round/eval/evaluation.py@@ -23,13 +23,13 @@ from lm_eval.models.huggingface import HFLMdef simple_evaluate_user_model(- user_model,- tokenizer,- batch_size: Optional[int] = 1,- max_batch_size: Optional[int] = 64,- eval_model_dtype = "auto",- add_bos_token: bool = False,- **kwargs+ user_model,+ tokenizer,+ batch_size: Optional[int] = 1,+ max_batch_size: Optional[int] = 64,+ eval_model_dtype="auto",+ add_bos_token: bool = False,+ **kwargs):hflm = HFLM(pretrained=user_model,@@ -37,18 +37,21 @@ def simple_evaluate_user_model(batch_size=batch_size,max_batch_size=max_batch_size,dtype=eval_model_dtype,- add_bos_token=add_bos_token)+ add_bos_token=add_bos_token,+ )return lm_simple_evaluate(- model=hflm, model_args=None, batch_size=batch_size, max_batch_size=max_batch_size, **kwargs)+ model=hflm, model_args=None, batch_size=batch_size, max_batch_size=max_batch_size, **kwargs+ )def simple_evaluate(- model,- model_args: Optional[Union[str, dict]] = None,- batch_size: Optional[int] = None,- max_batch_size: Optional[int] = None,- device: Optional[str] = None,- **kwargs):+ model,+ model_args: Optional[Union[str, dict]] = None,+ batch_size: Optional[int] = None,+ max_batch_size: Optional[int] = None,+ device: Optional[str] = None,+ **kwargs+):try:from auto_round import AutoRoundConfigexcept:@@ -60,4 +63,5 @@ def simple_evaluate(batch_size=batch_size,max_batch_size=max_batch_size,device=device,- **kwargs)+ **kwargs+ )diff --git a/auto_round/export/__init__.py b/auto_round/export/__init__.pyindex 42196d115..99c19822a 100644--- a/auto_round/export/__init__.py+++ b/auto_round/export/__init__.py@@ -12,7 +12,7 @@# See the License for the specific language governing permissions and# limitations under the License.-from auto_round.export.register import EXPORT_FORMAT,PACKING_LAYER_WITH_FORMAT, register_format,register_layer_packing+from auto_round.export.register import EXPORT_FORMAT, PACKING_LAYER_WITH_FORMAT, register_format, register_layer_packing@register_format("auto_gptq")@@ -49,9 +49,11 @@ def _save_quantized_as_autoawq(*args, **kwargs):return save_quantized_as_autoawq(*args, **kwargs)+@register_format("gguf")def _save_quantized_as_gguf(*args, **kwargs):from auto_round.export.export_to_gguf.export import save_quantized_as_gguf+return save_quantized_as_gguf(*args, **kwargs)@@ -81,4 +83,3 @@ def _save_quantized_as_llmcompressor(*args, **kwargs):from auto_round.export.export_to_llmcompressor.export import save_quantized_as_llmcompressorreturn save_quantized_as_llmcompressor(*args, **kwargs)-diff --git a/auto_round/export/export_to_autogptq/__init__.py b/auto_round/export/export_to_autogptq/__init__.pyindex 2045808ac..28f108cb6 100644--- a/auto_round/export/export_to_autogptq/__init__.py+++ b/auto_round/export/export_to_autogptq/__init__.py@@ -11,4 +11,3 @@# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.# See the License for the specific language governing permissions and# limitations under the License.-diff --git a/auto_round/export/export_to_autogptq/export.py b/auto_round/export/export_to_autogptq/export.pyindex d74040419..4ccfae2d3 100644--- a/auto_round/export/export_to_autogptq/export.py+++ b/auto_round/export/export_to_autogptq/export.py@@ -121,19 +121,17 @@ def pack_layer(name, model, backend):else:qlayer.pack(layer, scale, zero, None)qlayer.to(device)- if hasattr(layer,"weight"):+ if hasattr(layer, "weight"):layer.weight = None- if hasattr(layer,"bias"):+ if hasattr(layer, "bias"):layer.bias = None--def save_quantized_as_autogptq(output_dir, inplace=True, backend="auto_gptq:exllamav2",- **kwargs):+def save_quantized_as_autogptq(output_dir, inplace=True, backend="auto_gptq:exllamav2", **kwargs):"""Export the model to autogptq format to easily leverage cuda kernel."""model = kwargs["model"]- safe_serialization = True if 'safe_serialization' not in kwargs.keys() else kwargs["safe_serialization"]+ safe_serialization = True if "safe_serialization" not in kwargs.keys() else kwargs["safe_serialization"]quant_block_list = kwargs.get("quant_block_list", get_block_names(model))tokenizer = kwargs.get("tokenizer", None)processor = kwargs.get("processor", None)@@ -147,10 +145,10 @@ def save_quantized_as_autogptq(output_dir, inplace=True, backend="auto_gptq:exllquantization_config = kwargs["serialization_dict"]all_blocks = quant_block_listflattened_list = [item for sublist in all_blocks for item in sublist]- common_prefix = os.path.commonprefix(flattened_list).rstrip('.')+ common_prefix = os.path.commonprefix(flattened_list).rstrip(".")if common_prefix not in BLOCK_PATTERNS:logger.error("auto-gptq format may not support loading this quantized model")- quantization_config['block_name_to_quantize'] = common_prefix+ quantization_config["block_name_to_quantize"] = common_prefixquantization_config.pop("to_quant_block_names", None)## as layers maybe already packed, we need to check in layer_config@@ -187,6 +185,7 @@ def save_quantized_as_autogptq(output_dir, inplace=True, backend="auto_gptq:exllwith ThreadPoolExecutor(max_workers=max_workers) as executor:with tqdm(total=len(names), leave=True) as pbar:+def wrapper(name):pbar.set_description(f"packing {name}")with tctl.threadpool_limits(limits=1):@@ -214,8 +213,9 @@ def save_quantized_as_autogptq(output_dir, inplace=True, backend="auto_gptq:exllreturn model-def save(model: torch.nn.Module, save_dir: str, max_shard_size: str = "5GB", safe_serialization: bool = True,- dtype=None):+def save(+ model: torch.nn.Module, save_dir: str, max_shard_size: str = "5GB", safe_serialization: bool = True, dtype=None+):"""Save model state dict and configs.Args:@@ -240,16 +240,13 @@ def save(model: torch.nn.Module, save_dir: str, max_shard_size: str = "5GB", safmodel.save_pretrained(save_dir, max_shard_size=max_shard_size, safe_serialization=safe_serialization)config_path = os.path.join(save_dir, "config.json")if dtype is not None and dtype != model.dtype and os.path.exists(os.path.join(save_dir, "config.json")):- with open(config_path, 'r') as file:+ with open(config_path, "r") as file:data = json.load(file)data["torch_dtype"] = str(dtype).split(".")[-1]- with open(config_path, 'w') as file:+ with open(config_path, "w") as file:json.dump(data, file, indent=2)config_file = "quantize_config.json"if hasattr(model, "config") and hasattr(model.config, "quantization_config"):with open(os.path.join(save_dir, config_file), "w", encoding="utf-8") as f:json.dump(model.config.quantization_config, f, indent=2)---diff --git a/auto_round/export/export_to_autogptq/qlinear_triton.py b/auto_round/export/export_to_autogptq/qlinear_triton.pyindex 671737d3b..5c0c07032 100644--- a/auto_round/export/export_to_autogptq/qlinear_triton.py+++ b/auto_round/export/export_to_autogptq/qlinear_triton.py@@ -39,7 +39,7 @@ class QuantLinear(nn.Module, TritonModuleMixin):self.outfeatures = outfeaturesself.bits = bitsself.group_size = group_size if group_size != -1 else infeatures- self.maxq = 2 ** self.bits - 1+ self.maxq = 2**self.bits - 1self.register_buffer("qweight",@@ -98,12 +98,12 @@ class QuantLinear(nn.Module, TritonModuleMixin):repeat_scales = scales.to(device).repeat_interleave(self.group_size, 1)if isinstance(zeros, torch.Tensor):repeat_zeros = zeros.to(device).repeat_interleave(self.group_size, 1)- intweight = torch.round(W.to(device) / repeat_scales[:, :W.shape[1]] + repeat_zeros[:, :W.shape[1]]).to(- torch.int32)+ intweight = torch.round(W.to(device) / repeat_scales[:, : W.shape[1]] + repeat_zeros[:, : W.shape[1]]).to(+ torch.int32+ )else:repeat_zeros = zeros- intweight = torch.round(W.to(device) / repeat_scales[:, :W.shape[1]] + repeat_zeros).to(- torch.int32)+ intweight = torch.round(W.to(device) / repeat_scales[:, : W.shape[1]] + repeat_zeros).to(torch.int32)del repeat_scalesintweight = intweight.reshape(-1, intweight.shape[1] // 32 * self.bits, 32 // self.bits)@@ -140,4 +140,4 @@ class QuantLinear(nn.Module, TritonModuleMixin):self.qzeros = torch.from_numpy(qzeros)-__all__ = ["QuantLinear"]\ No newline at end of file+__all__ = ["QuantLinear"]diff --git a/auto_round/export/export_to_autoround/__init__.py b/auto_round/export/export_to_autoround/__init__.pyindex afa7c3fe6..0c036d831 100644--- a/auto_round/export/export_to_autoround/__init__.py+++ b/auto_round/export/export_to_autoround/__init__.py@@ -13,4 +13,3 @@# limitations under the License.from .export import save_quantized_as_autoround-diff --git a/auto_round/export/export_to_autoround/export.py b/auto_round/export/export_to_autoround/export.pyindex a0ac9ae8b..7fa583848 100644--- a/auto_round/export/export_to_autoround/export.py+++ b/auto_round/export/export_to_autoround/export.py@@ -52,11 +52,7 @@ def check_neq_config(config, data_type, bits, group_size, sym):Returns:list: A list of strings indicating which configuration parameters do not match."""- expected_config = {"data_type": data_type,- "bits": bits,- "group_size": group_size,- "sym": sym- }+ expected_config = {"data_type": data_type, "bits": bits, "group_size": group_size, "sym": sym}return [key for key, expected_value in expected_config.items() if config.get(key) != expected_value]@@ -79,23 +75,27 @@ def dynamic_import_quant_linear_for_packing(backend, bits, group_size, sym, act_if "auto_round" in backend and "awq" not in backend and "gptq" not in backend:if act_bits <= 8: ##easily have bug for other configuration, need to refine code laterimport auto_round.export.export_to_autoround.qlinear_triton_act+return auto_round.export.export_to_autoround.qlinear_triton_act.QuantLinearfrom auto_round_extension.torch.qlinear_torch import QuantLinear+return QuantLinearelif "gptqmodel" in backend:from auto_round_extension.torch.qlinear_torch import QuantLinear+return QuantLinearelif "auto_round" in backend and "gptq" in backend and "gptqmodel" not in backend:from auto_round_extension.torch.qlinear_torch_zp import QuantLinear+return QuantLinearelif "awq" in backend:from ..export_to_awq.utils import WQLinear_GEMM+return WQLinear_GEMMelif "gptq" in backend and "gptqmodel" not in backend: ## have g_idxreturn get_autogptq_packing_qlinear(backend, bits, group_size, sym)else:- raise ValueError(- f"unsupported backend: '{backend}'. Supported backends are: {', '.join(SUPPORTED_FORMATS)}")+ raise ValueError(f"unsupported backend: '{backend}'. Supported backends are: {', '.join(SUPPORTED_FORMATS)}")def pack_qact_layer(name, model):@@ -116,6 +116,7 @@ def pack_qact_layer(name, model):scale = layer.scalezp = layer.zpimport auto_round.export.export_to_autoround.qlinear_triton_act+QuantLinear = auto_round.export.export_to_autoround.qlinear_triton_act.QuantLinearif isinstance(layer, nn.Linear):@@ -168,7 +169,8 @@ def pack_layer(layer_name, model, backend):if "fp8" in backend:from auto_round.export.export_to_autoround.export_to_fp8_woq import pack_layer- return pack_layer(layer_name,model,backend)++ return pack_layer(layer_name, model, backend)if int(layer.act_bits) <= 8:return pack_qact_layer(layer_name, model)@@ -205,6 +207,7 @@ def pack_layer(layer_name, model, backend):set_module(model, layer_name, new_layer)qlayer = new_layerimport auto_round_extension.torch.qlinear_torch+if sym and isinstance(QuantLinear, (auto_round_extension.torch.qlinear_torch.QuantLinear)):zp = int(zp.flatten()[0])@@ -236,9 +239,9 @@ def pack_layer(layer_name, model, backend):)qlayer.to(device)set_module(model, layer_name, qlayer)- if hasattr(layer,"weight"):+ if hasattr(layer, "weight"):layer.weight = None- if hasattr(layer,"bias"):+ if hasattr(layer, "bias"):layer.bias = None@@ -266,14 +269,15 @@ def save_quantized_as_autoround(output_dir, inplace=True, backend="auto_round:ex"""if "fp8" in kwargs.get("data_type", None) and kwargs.get("act_bits", 16) >= 16:from auto_round.export.export_to_autoround.export_to_fp8_woq import save_quantized_as_autoround+return save_quantized_as_autoround(output_dir, inplace=inplace, backend="auto_round", **kwargs)##if using sym, we change to gptq sym kernel to avoid compiling from auto_round sourceif (kwargs.get("sym") is None or kwargs.get("sym")) and ("gptq" not in backend and "awq" not in backend):- backend = backend.replace('auto_round', 'auto_round:auto_gptq')+ backend = backend.replace("auto_round", "auto_round:auto_gptq")model = kwargs["model"]- safe_serialization = True if 'safe_serialization' not in kwargs.keys() else kwargs["safe_serialization"]+ safe_serialization = True if "safe_serialization" not in kwargs.keys() else kwargs["safe_serialization"]if not inplace:model = copy.deepcopy(model.to("cpu"))@@ -291,18 +295,20 @@ def save_quantized_as_autoround(output_dir, inplace=True, backend="auto_round:exblock_name_to_quantize = block_name_to_quantize.split(",")elif isinstance(block_name_to_quantize, list):for i in range(len(block_name_to_quantize)):- block_name_to_quantize[i] = os.path.commonprefix(block_name_to_quantize[i]).rstrip('.')+ block_name_to_quantize[i] = os.path.commonprefix(block_name_to_quantize[i]).rstrip(".")for layer_name in layer_config:- if not layer_config[layer_name]["in_blocks"] and layer_config[layer_name][- "bits"] <= 8: ##lm head ##TODO fix act and so on+ if (+ not layer_config[layer_name]["in_blocks"] and layer_config[layer_name]["bits"] <= 8+ ): ##lm head ##TODO fix act and so onextra_config[layer_name] = {}extra_config[layer_name]["bits"] = layer_config[layer_name]["bits"]extra_config[layer_name]["data_type"] = layer_config[layer_name]["data_type"]extra_config[layer_name]["group_size"] = layer_config[layer_name]["group_size"]extra_config[layer_name]["sym"] = layer_config[layer_name]["sym"]elif layer_config[layer_name]["in_blocks"] or (- block_name_to_quantize is not None and check_start_with_block_name(layer_name, block_name_to_quantize)):+ block_name_to_quantize is not None and check_start_with_block_name(layer_name, block_name_to_quantize)+ ):neq_keys = check_neq_config(layer_config[layer_name],data_type=quantization_config["data_type"],@@ -323,6 +329,7 @@ def save_quantized_as_autoround(output_dir, inplace=True, backend="auto_round:exmax_workers = 2 ## 2 with cuda packing will cause hang occasionallywith ThreadPoolExecutor(max_workers=max_workers) as executor:with tqdm(total=len(names), leave=True) as pbar:+def wrapper(name):pbar.set_description(f"packing {name}")with tctl.threadpool_limits(limits=1):@@ -382,14 +389,12 @@ def save(model: nn.Module, save_dir: str, max_shard_size: str = "5GB", safe_serimodel.save_pretrained(save_dir, max_shard_size=max_shard_size, safe_serialization=safe_serialization)config_path = os.path.join(save_dir, "config.json")if dtype is not None and dtype != model.dtype and os.path.exists(os.path.join(save_dir, "config.json")):- with open(config_path, 'r') as file:+ with open(config_path, "r") as file:data = json.load(file)data["torch_dtype"] = str(dtype).split(".")[-1]- with open(config_path, 'w') as file:+ with open(config_path, "w") as file:json.dump(data, file, indent=2)config_file = "quantization_config.json"if hasattr(model, "config") and hasattr(model.config, "quantization_config"):with open(os.path.join(save_dir, config_file), "w", encoding="utf-8") as f:json.dump(model.config.quantization_config, f, indent=2)--diff --git a/auto_round/export/export_to_autoround/export_to_fp8_woq.py b/auto_round/export/export_to_autoround/export_to_fp8_woq.pyindex 02ef4d755..6589f243f 100644--- a/auto_round/export/export_to_autoround/export_to_fp8_woq.py+++ b/auto_round/export/export_to_autoround/export_to_fp8_woq.py@@ -47,26 +47,23 @@ def check_neq_config(config, data_type, bits, group_size, sym):Returns:list: A list of strings indicating which configuration parameters do not match."""- expected_config = {"data_type": data_type,- "bits": bits,- "group_size": group_size,- "sym": sym- }+ expected_config = {"data_type": data_type, "bits": bits, "group_size": group_size, "sym": sym}return [key for key, expected_value in expected_config.items() if config.get(key) != expected_value]class FP8WOQLinear(torch.nn.Module):def __init__(- self,- in_features,- out_features,- weight,- weight_scale,- bias=None,- weight_zp=None,- act_scale=None,- dtype=torch.bfloat16):+ self,+ in_features,+ out_features,+ weight,+ weight_scale,+ bias=None,+ weight_zp=None,+ act_scale=None,+ dtype=torch.bfloat16,+ ):super().__init__()self.in_features = in_featuresself.out_features = out_features@@ -77,13 +74,13 @@ class FP8WOQLinear(torch.nn.Module):else:self.register_parameter("bias", None)- self.register_buffer('weight_scale', weight_scale.to(dtype))+ self.register_buffer("weight_scale", weight_scale.to(dtype))if weight_zp:- self.register_buffer('weight_zp', weight_zp.to(dtype))+ self.register_buffer("weight_zp", weight_zp.to(dtype))if act_scale:- self.register_buffer('act_scale', act_scale.to(dtype))+ self.register_buffer("act_scale", act_scale.to(dtype))def pack_layer(layer_name, model, data_type, packing_device=None):@@ -152,7 +149,8 @@ def pack_layer(layer_name, model, data_type, packing_device=None):bias=bias,weight_zp=zp,act_scale=act_scale,- dtype=model.dtype)+ dtype=model.dtype,+ )my_linear.to(device)set_module(model, layer_name, my_linear)@@ -160,7 +158,7 @@ def pack_layer(layer_name, model, data_type, packing_device=None):def save_quantized_as_autoround(output_dir, inplace=True, backend="auto_round", **kwargs):model = kwargs["model"]- safe_serialization = True if 'safe_serialization' not in kwargs.keys() else kwargs["safe_serialization"]+ safe_serialization = True if "safe_serialization" not in kwargs.keys() else kwargs["safe_serialization"]if not inplace:model = copy.deepcopy(model.to("cpu"))@@ -172,7 +170,7 @@ def save_quantized_as_autoround(output_dir, inplace=True, backend="auto_round",quantization_config["fmt"] = "e5m2"else:quantization_config["fmt"] = "e4m3"- quantization_config["activation_scheme"] = "dynamic" if quantization_config['act_dynamic'] else "static"+ quantization_config["activation_scheme"] = "dynamic" if quantization_config["act_dynamic"] else "static"tokenizer = kwargs.get("tokenizer", None)processor = kwargs.get("processor", None)@@ -182,18 +180,20 @@ def save_quantized_as_autoround(output_dir, inplace=True, backend="auto_round",block_name_to_quantize = block_name_to_quantize.split(",")elif isinstance(block_name_to_quantize, list):for i in range(len(block_name_to_quantize)):- block_name_to_quantize[i] = os.path.commonprefix(block_name_to_quantize[i]).rstrip('.')+ block_name_to_quantize[i] = os.path.commonprefix(block_name_to_quantize[i]).rstrip(".")for layer_name in layer_config:- if not layer_config[layer_name]["in_blocks"] and layer_config[layer_name][- "bits"] <= 8: ##lm head ##TODO fix act and so on+ if (+ not layer_config[layer_name]["in_blocks"] and layer_config[layer_name]["bits"] <= 8+ ): ##lm head ##TODO fix act and so onextra_config[layer_name] = {}extra_config[layer_name]["bits"] = layer_config[layer_name]["bits"]extra_config[layer_name]["data_type"] = layer_config[layer_name]["data_type"]extra_config[layer_name]["group_size"] = layer_config[layer_name]["group_size"]extra_config[layer_name]["sym"] = layer_config[layer_name]["sym"]elif layer_config[layer_name]["in_blocks"] or (- block_name_to_quantize is not None and check_start_with_block_name(layer_name, block_name_to_quantize)):+ block_name_to_quantize is not None and check_start_with_block_name(layer_name, block_name_to_quantize)+ ):neq_keys = check_neq_config(layer_config[layer_name],data_type=quantization_config["data_type"],@@ -219,6 +219,7 @@ def save_quantized_as_autoround(output_dir, inplace=True, backend="auto_round",packing_device = "xpu"with ThreadPoolExecutor(max_workers=max_workers) as executor:with tqdm(total=len(names), leave=True) as pbar:+def wrapper(name):pbar.set_description(f"packing {name}")with tctl.threadpool_limits(limits=1):@@ -254,8 +255,9 @@ def save_quantized_as_autoround(output_dir, inplace=True, backend="auto_round",return model-def save(model: torch.nn.Module, save_dir: str, max_shard_size: str = "5GB", safe_serialization: bool = True,- dtype=None):+def save(+ model: torch.nn.Module, save_dir: str, max_shard_size: str = "5GB", safe_serialization: bool = True, dtype=None+):"""Save model state dict and configs.Args:@@ -279,13 +281,12 @@ def save(model: torch.nn.Module, save_dir: str, max_shard_size: str = "5GB", safmodel.save_pretrained(save_dir, max_shard_size=max_shard_size, safe_serialization=safe_serialization)config_path = os.path.join(save_dir, "config.json")if dtype is not None and dtype != model.dtype and os.path.exists(os.path.join(save_dir, "config.json")):- with open(config_path, 'r') as file:+ with open(config_path, "r") as file:data = json.load(file)data["torch_dtype"] = str(dtype).split(".")[-1]- with open(config_path, 'w') as file:+ with open(config_path, "w") as file:json.dump(data, file, indent=2)config_file = "quantization_config.json"if hasattr(model, "config") and hasattr(model.config, "quantization_config"):with open(os.path.join(save_dir, config_file), "w", encoding="utf-8") as f:json.dump(model.config.quantization_config, f, indent=2)-diff --git a/auto_round/export/export_to_autoround/qlinear_triton_act.py b/auto_round/export/export_to_autoround/qlinear_triton_act.pyindex 3fd951302..21fc99642 100644--- a/auto_round/export/export_to_autoround/qlinear_triton_act.py+++ b/auto_round/export/export_to_autoround/qlinear_triton_act.py@@ -57,7 +57,7 @@ class QuantLinear(nn.Module):self.outfeatures = outfeaturesself.bits = bitsself.group_size = group_size if group_size != -1 else infeatures- self.maxq = 2 ** self.bits - 1+ self.maxq = 2**self.bits - 1self.register_buffer("qweight",@@ -97,7 +97,6 @@ class QuantLinear(nn.Module):),)-if bias:self.register_buffer("bias", torch.zeros((outfeatures), dtype=torch.float16))else:diff --git a/auto_round/export/export_to_awq/__init__.py b/auto_round/export/export_to_awq/__init__.pyindex 0bdb4d35e..8f4caae54 100644--- a/auto_round/export/export_to_awq/__init__.py+++ b/auto_round/export/export_to_awq/__init__.py@@ -13,5 +13,3 @@# limitations under the License.from .export import save_quantized_as_autoawq--diff --git a/auto_round/export/export_to_awq/export.py b/auto_round/export/export_to_awq/export.pyindex 22c13048e..115d211b7 100644--- a/auto_round/export/export_to_awq/export.py+++ b/auto_round/export/export_to_awq/export.py@@ -73,13 +73,12 @@ def pack_layer(name, model, backend):zeros=zp,)set_module(model, name, q_linear)- if hasattr(layer,"weight"):+ if hasattr(layer, "weight"):layer.weight = None- if hasattr(layer,"bias"):+ if hasattr(layer, "bias"):layer.bias = None-def save_quantized_as_autoawq(output_dir, inplace=True, **kwargs):"""Export the model to autogptq format to easily leverage cuda kernel."""model = kwargs["model"]@@ -100,8 +99,8 @@ def save_quantized_as_autoawq(output_dir, inplace=True, **kwargs):# mllm modelsall_blocks = get_block_names(model, quant_vision=True)all_block_names = extract_block_names_to_str(all_blocks)- all_block_names = all_block_names.split(',')- to_quant_block_names = to_quant_block_names.split(',')+ all_block_names = all_block_names.split(",")+ to_quant_block_names = to_quant_block_names.split(",")modules_to_not_convert = list(set(all_block_names) - set(to_quant_block_names))if inplace:@@ -113,10 +112,11 @@ def save_quantized_as_autoawq(output_dir, inplace=True, **kwargs):backend = Nonemax_workers = 1- if not torch.cuda.is_available() and not torch.xpu.is_available():+ if not torch.cuda.is_available() and not torch.xpu.is_available():max_workers = 2 ## 2 with cuda packing will cause hang occasionallywith ThreadPoolExecutor(max_workers=max_workers) as executor:with tqdm(total=len(names), leave=True) as pbar:+def wrapper(name):pbar.set_description(f"packing {name}")with tctl.threadpool_limits(limits=1):@@ -133,11 +133,9 @@ def save_quantized_as_autoawq(output_dir, inplace=True, **kwargs):if output_dir is None:return compressed_model-layer_config = kwargs["layer_config"]for key in layer_config.keys():- if not check_to_quantized(layer_config[key]) and \- not any(name in key for name in modules_to_not_convert):+ if not check_to_quantized(layer_config[key]) and not any(name in key for name in modules_to_not_convert):modules_to_not_convert.append(key)quantization_config["quant_method"] = "awq"@@ -149,7 +147,7 @@ def save_quantized_as_autoawq(output_dir, inplace=True, **kwargs):filter_quantization_config(quantization_config)if hasattr(compressed_model, "config"):compressed_model.config.quantization_config = quantization_config- safe_serialization = kwargs.get('safe_serialization', True)+ safe_serialization = kwargs.get("safe_serialization", True)dtype = torch.float16 ##force dtype to fp16save(compressed_model, output_dir, safe_serialization=safe_serialization, dtype=dtype)@@ -180,13 +178,12 @@ def save(model: nn.Module, save_dir: str, max_shard_size: str = "5GB", safe_serimodel.save_pretrained(save_dir, max_shard_size=max_shard_size, safe_serialization=safe_serialization)config_path = os.path.join(save_dir, "config.json")if dtype is not None and dtype != model.dtype and os.path.exists(os.path.join(save_dir, "config.json")):- with open(config_path, 'r') as file:+ with open(config_path, "r") as file:data = json.load(file)data["torch_dtype"] = str(dtype).split(".")[-1]- with open(config_path, 'w') as file:+ with open(config_path, "w") as file:json.dump(data, file, indent=2)config_file = "quantization_config.json"if hasattr(model, "config") and hasattr(model.config, "quantization_config"):with open(os.path.join(save_dir, config_file), "w", encoding="utf-8") as f:json.dump(model.config.quantization_config, f, indent=2)-diff --git a/auto_round/export/export_to_awq/utils.py b/auto_round/export/export_to_awq/utils.pyindex 72e218df5..a570b2a9c 100644--- a/auto_round/export/export_to_awq/utils.py+++ b/auto_round/export/export_to_awq/utils.py@@ -88,8 +88,8 @@ def dequantize_gemm(qweight, qzeros, scales, bits, group_size):iweight, izeros = reverse_awq_order(iweight, izeros, bits)# overflow checks- iweight = torch.bitwise_and(iweight, (2 ** bits) - 1)- izeros = torch.bitwise_and(izeros, (2 ** bits) - 1)+ iweight = torch.bitwise_and(iweight, (2**bits) - 1)+ izeros = torch.bitwise_and(izeros, (2**bits) - 1)# fp16 weightsscales = scales.repeat_interleave(group_size, dim=0)@@ -103,15 +103,15 @@ class WQLinearMMFunction(Function):@staticmethod# ctx is the first argument to forwarddef forward(- ctx,- x,- qweight,- qzeros,- scales,- w_bit=4,- group_size=128,- bias=None,- out_features=0,+ ctx,+ x,+ qweight,+ qzeros,+ scales,+ w_bit=4,+ group_size=128,+ bias=None,+ out_features=0,):# The forward pass can use ctx.ctx.save_for_backward(x, qweight, qzeros, scales, bias)@@ -134,9 +134,7 @@ class WQLinearMMFunction(Function):class WQLinear_GEMM(nn.Module):- def __init__(- self, w_bit, group_size, in_features, out_features, bias, dev, training=False- ):+ def __init__(self, w_bit, group_size, in_features, out_features, bias, dev, training=False):super().__init__()if w_bit not in [4]:@@ -192,9 +190,7 @@ class WQLinear_GEMM(nn.Module):self.bias = None@classmethod- def from_linear(- cls, linear, w_bit, group_size, init_only=False, scales=None, zeros=None- ):+ def from_linear(cls, linear, w_bit, group_size, init_only=False, scales=None, zeros=None):awq_linear = cls(w_bit,group_size,@@ -224,15 +220,24 @@ class WQLinear_GEMM(nn.Module):repeat_scales = scales.to(device).t().repeat_interleave(repeat_size, 1)if isinstance(zeros, torch.Tensor):repeat_zeros = zeros.to(device).t().repeat_interleave(repeat_size, 1)- intweight = torch.round(- linear.weight.to(device) / repeat_scales[:, :linear.weight.shape[1]] +- repeat_zeros[:, :linear.weight.shape[1]]).to(torch.int).t().contiguous()+ intweight = (+ torch.round(+ linear.weight.to(device) / repeat_scales[:, : linear.weight.shape[1]]+ + repeat_zeros[:, : linear.weight.shape[1]]+ )+ .to(torch.int)+ .t()+ .contiguous()+ )else:repeat_zeros = zeros- intweight = torch.round(- linear.weight.to(device) / repeat_scales[:, :linear.weight.shape[1]] + repeat_zeros).to(- torch.int).t().contiguous()+ intweight = (+ torch.round(linear.weight.to(device) / repeat_scales[:, : linear.weight.shape[1]] + repeat_zeros)+ .to(torch.int)+ .t()+ .contiguous()+ )intweight = intweight.to(dtype=torch.int32)del repeat_scales@@ -254,11 +259,14 @@ class WQLinear_GEMM(nn.Module):value = 0for i in range(pack_num):value |= zeros << (i * awq_linear.w_bit)- qzeros = torch.ones(- (scales.shape[0], scales.shape[1] // pack_num),- dtype=torch.int32,- device=device,- ) * value+ qzeros = (+ torch.ones(+ (scales.shape[0], scales.shape[1] // pack_num),+ dtype=torch.int32,+ device=device,+ )+ * value+ )awq_linear.qzeros = qzeros.to("cpu")@@ -301,14 +309,12 @@ class WQLinear_GEMM(nn.Module):return out.reshape(out_shape)def extra_repr(self) -> str:- return (- "in_features={}, out_features={}, bias={}, w_bit={}, group_size={}".format(- self.in_features,- self.out_features,- self.bias is not None,- self.w_bit,- self.group_size,- )+ return "in_features={}, out_features={}, bias={}, w_bit={}, group_size={}".format(+ self.in_features,+ self.out_features,+ self.bias is not None,+ self.w_bit,+ self.group_size,)@@ -317,4 +323,3 @@ def clear_memory(weight=None):del weightgc.collect()torch.cuda.empty_cache()-diff --git a/auto_round/export/export_to_gguf/config.py b/auto_round/export/export_to_gguf/config.pyindex e6b71586b..d4e53ed5b 100644--- a/auto_round/export/export_to_gguf/config.py+++ b/auto_round/export/export_to_gguf/config.py@@ -18,6 +18,7 @@ class ModelType(IntEnum):TEXT = 1MMPROJ = 2+GGUF_INNER_CONFIG = {}GGUF_INNER_CONFIG["gguf:q4_0"] = {@@ -89,7 +90,7 @@ GGUF_INNER_CONFIG["gguf:q2_k"] = {"sym": False,"data_type": "int_asym_dq","embedding": "gguf:q2_k",- "lm_head": "gguf:q6_k"+ "lm_head": "gguf:q6_k",}GGUF_INNER_CONFIG["gguf:q3_k"] = {@@ -101,10 +102,10 @@ GGUF_INNER_CONFIG["gguf:q3_k"] = {"sym": True,"data_type": "int_sym_dq","embedding": "gguf:q3_k",- "lm_head": "gguf:q6_k"+ "lm_head": "gguf:q6_k",}-GGUF_INNER_CONFIG["gguf:q4_k"] = {+GGUF_INNER_CONFIG["gguf:q4_k"] = {"bits": 4,"act_bits": 16,"super_group_size": 8,@@ -113,7 +114,7 @@ GGUF_INNER_CONFIG["gguf:q4_k"] = {"sym": False,"data_type": "int_asym_dq","embedding": "gguf:q4_k",- "lm_head": "gguf:q6_k"+ "lm_head": "gguf:q6_k",}GGUF_INNER_CONFIG["gguf:q5_k"] = {@@ -125,10 +126,10 @@ GGUF_INNER_CONFIG["gguf:q5_k"] = {"sym": False,"data_type": "int_asym_dq","embedding": "gguf:q5_k",- "lm_head": "gguf:q6_k"+ "lm_head": "gguf:q6_k",}-GGUF_INNER_CONFIG["gguf:q6_k"] = {+GGUF_INNER_CONFIG["gguf:q6_k"] = {"bits": 6,"act_bits": 16,"super_group_size": 16,@@ -137,10 +138,10 @@ GGUF_INNER_CONFIG["gguf:q6_k"] = {"sym": True,"data_type": "int_sym_dq","embedding": "gguf:q6_k",- "lm_head": "gguf:q6_k"+ "lm_head": "gguf:q6_k",}-GGUF_INNER_CONFIG["gguf:bf16"]=GGUF_INNER_CONFIG["gguf:fp16"] = {+GGUF_INNER_CONFIG["gguf:bf16"] = GGUF_INNER_CONFIG["gguf:fp16"] = {"bits": 16,"act_bits": 16,"super_group_size": None,@@ -153,18 +154,17 @@ GGUF_INNER_CONFIG["gguf:bf16"]=GGUF_INNER_CONFIG["gguf:fp16"] = {}-GGUF_CONFIG = {}GGUF_CONFIG["gguf:q4_0"] = GGUF_INNER_CONFIG["gguf:q4_0"]GGUF_CONFIG["gguf:q4_0"]["mostly"] = "gguf:q4_0"GGUF_CONFIG["gguf:q4_1"] = GGUF_INNER_CONFIG["gguf:q4_1"]-GGUF_CONFIG["gguf:q4_1"]["mostly"]= "gguf:q4_1"+GGUF_CONFIG["gguf:q4_1"]["mostly"] = "gguf:q4_1"GGUF_CONFIG["gguf:q5_0"] = GGUF_INNER_CONFIG["gguf:q5_0"]-GGUF_CONFIG["gguf:q5_0"]["mostly"]= "gguf:q5_0"+GGUF_CONFIG["gguf:q5_0"]["mostly"] = "gguf:q5_0"GGUF_CONFIG["gguf:q5_1"] = GGUF_INNER_CONFIG["gguf:q5_1"]GGUF_CONFIG["gguf:q5_1"]["mostly"] = "gguf:q5_1"GGUF_CONFIG["gguf:q2_k_s"] = GGUF_INNER_CONFIG["gguf:q2_k"]-GGUF_CONFIG["gguf:q2_k_s"]["mostly"]= "gguf:q2_k"+GGUF_CONFIG["gguf:q2_k_s"]["mostly"] = "gguf:q2_k"# GGUF_CONFIG["gguf:q3_k"] = GGUF_INNER_CONFIG["gguf:q3_k"]# GGUF_CONFIG["gguf:q3_k"]["mostly"] = "gguf:q3_k"GGUF_CONFIG["gguf:q3_k_s"] = GGUF_INNER_CONFIG["gguf:q3_k"]@@ -172,11 +172,11 @@ GGUF_CONFIG["gguf:q3_k_s"]["mostly"] = "gguf:q3_k"GGUF_CONFIG["gguf:q3_k_m"] = GGUF_INNER_CONFIG["gguf:q3_k"]GGUF_CONFIG["gguf:q3_k_m"]["mostly"] = "gguf:q3_k"GGUF_CONFIG["gguf:q3_k_l"] = GGUF_INNER_CONFIG["gguf:q3_k"]-GGUF_CONFIG["gguf:q3_k_l"]["mostly"]= "gguf:q3_k"+GGUF_CONFIG["gguf:q3_k_l"]["mostly"] = "gguf:q3_k"# GGUF_CONFIG["gguf:q4_k"] = GGUF_INNER_CONFIG["gguf:q4_k"]# GGUF_CONFIG["gguf:q4_k"]["mostly"]= "gguf:q4_k"GGUF_CONFIG["gguf:q4_k_s"] = GGUF_INNER_CONFIG["gguf:q4_k"]-GGUF_CONFIG["gguf:q4_k_s"]["mostly"]= "gguf:q4_k"+GGUF_CONFIG["gguf:q4_k_s"]["mostly"] = "gguf:q4_k"GGUF_CONFIG["gguf:q4_k_m"] = GGUF_INNER_CONFIG["gguf:q4_k"]GGUF_CONFIG["gguf:q4_k_m"]["mostly"] = "gguf:q4_k"# GGUF_CONFIG["gguf:q5_k"] = GGUF_INNER_CONFIG["gguf:q5_k"]@@ -186,16 +186,15 @@ GGUF_CONFIG["gguf:q5_k_s"]["mostly"] = "gguf:q5_k"GGUF_CONFIG["gguf:q5_k_m"] = GGUF_INNER_CONFIG["gguf:q5_k"]GGUF_CONFIG["gguf:q5_k_m"]["mostly"] = "gguf:q5_k"GGUF_CONFIG["gguf:q6_k"] = GGUF_INNER_CONFIG["gguf:q6_k"]-GGUF_CONFIG["gguf:q6_k"]["mostly"]= "gguf:q6_k"+GGUF_CONFIG["gguf:q6_k"]["mostly"] = "gguf:q6_k"GGUF_CONFIG["gguf:q8_0"] = GGUF_INNER_CONFIG["gguf:q8_0"]-GGUF_CONFIG["gguf:q8_0"]["mostly"]= "gguf:q8_0"+GGUF_CONFIG["gguf:q8_0"]["mostly"] = "gguf:q8_0"# GGUF_CONFIG["gguf:fp16"] = GGUF_INNER_CONFIG["gguf:fp16"]# GGUF_CONFIG["gguf:fp16"]["mostly"]= "gguf:fp16"# GGUF_CONFIG["gguf:bf16"] = GGUF_INNER_CONFIG["gguf:fp16"]# GGUF_CONFIG["gguf:bf16"]["mostly"]= "gguf:bf16"-QK_K = 256K_SCALE_SIZE = 12GGML_QUANT_SIZES = {@@ -205,10 +204,10 @@ GGML_QUANT_SIZES = {"q5_0": (32, 2 + 4 + 16),"q5_1": (32, 2 + 2 + 4 + 16),"q8_0": (32, 2 + 32),- "q2_k": (256, 2 + 2 + QK_K//16 + QK_K//4),+ "q2_k": (256, 2 + 2 + QK_K // 16 + QK_K // 4),"q3_k": (256, 2 + QK_K // 4 + QK_K // 8 + 12),- "q4_k": (256, 2 + 2 + QK_K//2 + 12),+ "q4_k": (256, 2 + 2 + QK_K // 2 + 12),"q5_k": (256, 2 + 2 + QK_K // 2 + QK_K // 8 + 12),"q6_k": (256, 2 + QK_K // 2 + QK_K // 4 + QK_K // 16),- "q8_k": (256, 4 + QK_K + QK_K // 8)-}\ No newline at end of file+ "q8_k": (256, 4 + QK_K + QK_K // 8),+}diff --git a/auto_round/export/export_to_gguf/convert.py b/auto_round/export/export_to_gguf/convert.pyindex ea69b0bbd..cec587d33 100644--- a/auto_round/export/export_to_gguf/convert.py+++ b/auto_round/export/export_to_gguf/convert.py@@ -63,8 +63,8 @@ gguf = LazyImport("gguf")if TYPE_CHECKING:from torch import Tensor-if 'NO_LOCAL_GGUF' not in os.environ:- sys.path.insert(1, str(Path(__file__).parent / 'gguf-py'))+if "NO_LOCAL_GGUF" not in os.environ:+ sys.path.insert(1, str(Path(__file__).parent / "gguf-py"))###### MODEL DEFINITIONS ######@@ -117,25 +117,24 @@ class OriModel:tensor_map: gguf.TensorNameMapdef __init__(- self,- dir_model: Path,- ftype: gguf.LlamaFileType,- fname_out: Path,- *,- is_big_endian: bool = False,- use_temp_file: bool = False,- eager: bool = False,- metadata_override: Path | None = None,- model_name: str | None = None,- split_max_tensors: int = 0,- split_max_size: int = 0,- dry_run: bool = False,- small_first_shard: bool = False,- hparams: dict[str, Any] | None = None,- remote_hf_model_id: str | None = None):- if type(self) is ModelBase or \- type(self) is TextModel or \- type(self) is MmprojModel:+ self,+ dir_model: Path,+ ftype: gguf.LlamaFileType,+ fname_out: Path,+ *,+ is_big_endian: bool = False,+ use_temp_file: bool = False,+ eager: bool = False,+ metadata_override: Path | None = None,+ model_name: str | None = None,+ split_max_tensors: int = 0,+ split_max_size: int = 0,+ dry_run: bool = False,+ small_first_shard: bool = False,+ hparams: dict[str, Any] | None = None,+ remote_hf_model_id: str | None = None,+ ):+ if type(self) is ModelBase or type(self) is TextModel or type(self) is MmprojModel:raise TypeError(f"{type(self).__name__!r} should not be directly instantiated")self.dir_model = dir_model@@ -154,7 +153,8 @@ class OriModel:remote_tensors = gguf.utility.SafetensorRemote.get_list_tensors_hf_model(remote_hf_model_id)self.tensor_names = set(name for name in remote_tensors.keys())for name, remote_tensor in gguf.utility.SafetensorRemote.get_list_tensors_hf_model(- remote_hf_model_id).items():+ remote_hf_model_id+ ).items():yield (name, LazyTorchTensor.from_remote_tensor(remote_tensor))self.get_tensors = get_remote_tensors@@ -189,7 +189,8 @@ class OriModel:split_max_tensors=split_max_tensors,split_max_size=split_max_size,dry_run=dry_run,- small_first_shard=small_first_shard)+ small_first_shard=small_first_shard,+ )@classmethoddef add_prefix_to_filename(cls, path: Path, prefix: str) -> Path:@@ -230,10 +231,12 @@ class OriModel:ctx: ContextManager[Any]if self.is_safetensors:from safetensors import safe_open+ctx = cast(ContextManager[Any], safe_open(self.dir_model / part_name, framework="pt", device="cpu"))else:ctx = contextlib.nullcontext(- torch.load(str(self.dir_model / part_name), map_location="cpu", mmap=True, weights_only=True))+ torch.load(str(self.dir_model / part_name), map_location="cpu", mmap=True, weights_only=True)+ )with ctx as model_part:tensor_names_from_parts.update(model_part.keys())@@ -257,13 +260,13 @@ class OriModel:extra = sorted(tensor_names_from_parts.difference(self.tensor_names))missing_files = sorted(set(weight_map[n] for n in missing if n in weight_map))if len(extra) == 0 and len(missing_files) > 0:- raise ValueError(f"Missing or incomplete model files: {missing_files}\n"- f"Missing tensors: {missing}")+ raise ValueError(f"Missing or incomplete model files: {missing_files}\n" f"Missing tensors: {missing}")else:raise ValueError("Mismatch between weight map and model parts for tensor names:\n"f"Missing tensors: {missing}\n"- f"Extra tensors: {extra}")+ f"Extra tensors: {extra}"+ )def format_tensor_name(self, key: gguf.MODEL_TENSOR, bid: int | None = None, suffix: str = ".weight") -> str:if key not in gguf.MODEL_TENSORS[self.model_arch]:@@ -275,7 +278,8 @@ class OriModel:return name + suffixdef match_model_tensor_name(- self, name: str, key: gguf.MODEL_TENSOR, bid: int | None, suffix: str = ".weight") -> bool:+ self, name: str, key: gguf.MODEL_TENSOR, bid: int | None, suffix: str = ".weight"+ ) -> bool:if key not in gguf.MODEL_TENSORS[self.model_arch]:return Falsekey_name: str = gguf.TENSOR_NAMES[key]@@ -303,7 +307,8 @@ class OriModel:return [(self.map_tensor_name(name), data_torch)]def tensor_force_quant(- self, name: str, new_name: str, bid: int | None, n_dims: int) -> gguf.GGMLQuantizationType | bool:+ self, name: str, new_name: str, bid: int | None, n_dims: int+ ) -> gguf.GGMLQuantizationType | bool:del name, new_name, bid, n_dims # unusedreturn False@@ -333,7 +338,7 @@ class OriModel:bid = int(part)break- for new_name, data_torch in (self.modify_tensors(data_torch, name, bid)):+ for new_name, data_torch in self.modify_tensors(data_torch, name, bid):# TODO: why do we squeeze here?# data = data_torch.squeeze().numpy()data = data_torch.numpy()@@ -351,38 +356,47 @@ class OriModel:# Conditions should closely match those in llama_model_quantize_internal in llama.cpp# Some tensor types are always in float32- if data_qtype is False and (any(self.match_model_tensor_name(new_name, key, bid) for key in (- gguf.MODEL_TENSOR.FFN_GATE_INP,- gguf.MODEL_TENSOR.POS_EMBD,- gguf.MODEL_TENSOR.TOKEN_TYPES,- gguf.MODEL_TENSOR.SSM_CONV1D,- gguf.MODEL_TENSOR.SHORTCONV_CONV,- gguf.MODEL_TENSOR.TIME_MIX_FIRST,- gguf.MODEL_TENSOR.TIME_MIX_W1,- gguf.MODEL_TENSOR.TIME_MIX_W2,- gguf.MODEL_TENSOR.TIME_MIX_DECAY_W1,- gguf.MODEL_TENSOR.TIME_MIX_DECAY_W2,- gguf.MODEL_TENSOR.TIME_MIX_LERP_FUSED,- gguf.MODEL_TENSOR.POSNET_NORM1,- gguf.MODEL_TENSOR.POSNET_NORM2,- gguf.MODEL_TENSOR.V_ENC_EMBD_POS,- gguf.MODEL_TENSOR.A_ENC_EMBD_POS,- gguf.MODEL_TENSOR.ALTUP_CORRECT_COEF,- gguf.MODEL_TENSOR.ALTUP_PREDICT_COEF,- )) or not new_name.endswith(".weight")):+ if data_qtype is False and (+ any(+ self.match_model_tensor_name(new_name, key, bid)+ for key in (+ gguf.MODEL_TENSOR.FFN_GATE_INP,+ gguf.MODEL_TENSOR.POS_EMBD,+ gguf.MODEL_TENSOR.TOKEN_TYPES,+ gguf.MODEL_TENSOR.SSM_CONV1D,+ gguf.MODEL_TENSOR.SHORTCONV_CONV,+ gguf.MODEL_TENSOR.TIME_MIX_FIRST,+ gguf.MODEL_TENSOR.TIME_MIX_W1,+ gguf.MODEL_TENSOR.TIME_MIX_W2,+ gguf.MODEL_TENSOR.TIME_MIX_DECAY_W1,+ gguf.MODEL_TENSOR.TIME_MIX_DECAY_W2,+ gguf.MODEL_TENSOR.TIME_MIX_LERP_FUSED,+ gguf.MODEL_TENSOR.POSNET_NORM1,+ gguf.MODEL_TENSOR.POSNET_NORM2,+ gguf.MODEL_TENSOR.V_ENC_EMBD_POS,+ gguf.MODEL_TENSOR.A_ENC_EMBD_POS,+ gguf.MODEL_TENSOR.ALTUP_CORRECT_COEF,+ gguf.MODEL_TENSOR.ALTUP_PREDICT_COEF,+ )+ )+ or not new_name.endswith(".weight")+ ):data_qtype = gguf.GGMLQuantizationType.F32- if data_qtype is False and any(self.match_model_tensor_name(new_name, key, bid) for key in (+ if data_qtype is False and any(+ self.match_model_tensor_name(new_name, key, bid)+ for key in (gguf.MODEL_TENSOR.TOKEN_EMBD,gguf.MODEL_TENSOR.PER_LAYER_TOKEN_EMBD,gguf.MODEL_TENSOR.OUTPUT,gguf.MODEL_TENSOR.ALTUP_ROUTER,gguf.MODEL_TENSOR.LAUREL_L,gguf.MODEL_TENSOR.LAUREL_R,- )):+ )+ ):if self.ftype in (- gguf.LlamaFileType.MOSTLY_TQ1_0,- gguf.LlamaFileType.MOSTLY_TQ2_0,+ gguf.LlamaFileType.MOSTLY_TQ1_0,+ gguf.LlamaFileType.MOSTLY_TQ2_0,):# TODO: use Q4_K and Q6_Kdata_qtype = gguf.GGMLQuantizationType.F16@@ -411,15 +425,17 @@ class OriModel:data_qtype = gguf.GGMLQuantizationType.F16data = gguf.quants.quantize(data, data_qtype)- shape = gguf.quant_shape_from_byte_shape(- data.shape, data_qtype) if data.dtype == np.uint8 else data.shape+ shape = (+ gguf.quant_shape_from_byte_shape(data.shape, data_qtype) if data.dtype == np.uint8 else data.shape+ )# reverse shape to make it similar to the internal ggml dimension ordershape_str = f"{{{', '.join(str(n) for n in reversed(shape))}}}"# n_dims is implicit in the shapelogger.info(- f"{f'%-{max_name_len}s' % f'{new_name},'} {old_dtype} --> {data_qtype.name}, shape = {shape_str}")+ f"{f'%-{max_name_len}s' % f'{new_name},'} {old_dtype} --> {data_qtype.name}, shape = {shape_str}"+ )self.gguf_writer.add_tensor(new_name, data, raw_dtype=data_qtype)@@ -520,29 +536,30 @@ class OriModel:try:return cls._model_classes[model_type][arch]except KeyError:- raise NotImplementedError(f'Architecture {arch!r} not supported!') from None+ raise NotImplementedError(f"Architecture {arch!r} not supported!") from Noneclass ModelBase(OriModel):def __init__(- self,- model,- layer_config: dict,- dir_model: Path,- ftype: gguf.LlamaFileType,- fname_out: Path,- low_cpu_mem_usage: bool = False,- is_big_endian: bool = False,- use_temp_file: bool = False,- eager: bool = False,- metadata_override: Path | None = None,- model_name: str | None = None,- split_max_tensors: int = 0,- split_max_size: int = 0,- dry_run: bool = False,- small_first_shard: bool = False,- hparams: dict[str, Any] | None = None):+ self,+ model,+ layer_config: dict,+ dir_model: Path,+ ftype: gguf.LlamaFileType,+ fname_out: Path,+ low_cpu_mem_usage: bool = False,+ is_big_endian: bool = False,+ use_temp_file: bool = False,+ eager: bool = False,+ metadata_override: Path | None = None,+ model_name: str | None = None,+ split_max_tensors: int = 0,+ split_max_size: int = 0,+ dry_run: bool = False,+ small_first_shard: bool = False,+ hparams: dict[str, Any] | None = None,+ ):if self.model_arch == gguf.MODEL_ARCH.MMPROJ and fname_out.is_dir():fname_out = fname_out / "mmproj-model.gguf"self.model = model@@ -561,7 +578,8 @@ class ModelBase(OriModel):split_max_size=split_max_size,dry_run=dry_run,small_first_shard=small_first_shard,- hparams=hparams)+ hparams=hparams,+ )def _need_low_cpu_mem(self, low_cpu_mem_usage):if not low_cpu_mem_usage:@@ -582,11 +600,11 @@ class ModelBase(OriModel):"FFN_DOWN_EXP": ["down_proj", "w2", "linear_1"],"FFN_UP_EXP": ["up_proj", "w3", "linear_v"],}- nums = re.findall('\d+', name)+ nums = re.findall("\d+", name)if len(nums) != 2:return name- name_tmp = name[:-len(".weight")].replace(f'.{nums[1]}', '')- new_name_tmp = new_name[:-len(".weight")]+ name_tmp = name[: -len(".weight")].replace(f".{nums[1]}", "")+ new_name_tmp = new_name[: -len(".weight")]if self.tensor_map.get_type(name_tmp) == self.tensor_map.get_type(new_name_tmp):return name@@ -604,7 +622,8 @@ class ModelBase(OriModel):yield name, tensordef _quant_data_with_args(- self, data_torch, data_qtype, scale, zp, d_scale=None, wmin=None, d_wmin=None, imatrix=None):+ self, data_torch, data_qtype, scale, zp, d_scale=None, wmin=None, d_wmin=None, imatrix=None+ ):if torch.cuda.is_available():device = "cuda"elif torch.xpu.is_available():@@ -628,11 +647,12 @@ class ModelBase(OriModel):d_scale=d_scale,d_wmin=d_wmin,imatrix=imatrix,- device=device)+ device=device,+ )return datadef _quant_data(self, data_torch, data_qtype, name, modify_name, bid):- suffix = '.weight'+ suffix = ".weight"if torch.cuda.is_available():device = "cuda"elif torch.xpu.is_available():@@ -640,7 +660,7 @@ class ModelBase(OriModel):else:device = "cpu"if suffix in name:- layer_name = name[:-len(suffix)]+ layer_name = name[: -len(suffix)]module = get_module(self.model, layer_name)if hasattr(module, "scale"):if hasattr(self, "permute"):@@ -672,7 +692,8 @@ class ModelBase(OriModel):d_scale=d_scale,d_wmin=d_wmin,imatrix=imatrix,- device=device)+ device=device,+ )else:data = ggml_quant(data_torch, data_qtype.name.lower(), scale, zp, device=device)else:@@ -695,10 +716,10 @@ class ModelBase(OriModel):return data, data_qtypedef get_qtype_by_layer_config(self, layer_config, name, data_qtype):- name = name[:-len('.weight')]- if name not in layer_config or layer_config[name]['bits'] >= 16:+ name = name[: -len(".weight")]+ if name not in layer_config or layer_config[name]["bits"] >= 16:return data_qtype- bits = layer_config[name]['bits']+ bits = layer_config[name]["bits"]super_bits = layer_config[name]["super_bits"]sym = layer_config[name]["sym"]if bits == 2:@@ -759,9 +780,10 @@ class ModelBase(OriModel):name = ".".join(name.split(".")[1:])breakreturn name+if self.model_arch == gguf.MODEL_ARCH.QWEN2VL:name = name.replace("language_model.", "")- visual_keys = ["thinker", "visual", "audio", "talker", "token2wav"]+ visual_keys = ["thinker", "visual", "audio", "talker", "token2wav"]name = remove_prefix(name, visual_keys)if isinstance(self, Qwen2VLVisionModel):if "visual." in name and not name.startswith("visual."):@@ -769,7 +791,7 @@ class ModelBase(OriModel):# # for gemma3if self.model_arch == gguf.MODEL_ARCH.GEMMA3 or isinstance(self, Gemma3VisionModel):- visual_keys = ["multi_modal_projector", "vision_tower", "multimodal_projector"]+ visual_keys = ["multi_modal_projector", "vision_tower", "multimodal_projector"]name = remove_prefix(name, visual_keys)# for LlavaForConditionalGeneration@@ -777,12 +799,12 @@ class ModelBase(OriModel):name = name.replace("language_model.", "")# name = name.replace("model.", "")if isinstance(self, LlavaVisionModel):- visual_keys = ["multi_modal_projector", "vision_tower"]+ visual_keys = ["multi_modal_projector", "vision_tower"]name = remove_prefix(name, visual_keys)# for InternVisionModelif isinstance(self, InternVisionModel):- visual_keys = ["vision_model"]+ visual_keys = ["vision_model"]name = remove_prefix(name, visual_keys)return name@@ -796,11 +818,16 @@ class ModelBase(OriModel):# we don't need theseif name.endswith((".attention.masked_bias", ".attention.bias", ".rotary_emb.inv_freq")):continue- if hasattr(self, "current_packing_block") and self.current_packing_block is not None: # pylint: disable=E1101- current_packing_block_split = self.current_packing_block.split('.') # pylint: disable=E1101- name_split = name.split('.')- if (len(name_split) < len(current_packing_block_split) or- name_split[:len(current_packing_block_split)] != current_packing_block_split):+ if (+ hasattr(self, "current_packing_block")+ and self.current_packing_block is not None # pylint: disable=E1101+ ):+ current_packing_block_split = self.current_packing_block.split(".") # pylint: disable=E1101+ name_split = name.split(".")+ if (+ len(name_split) < len(current_packing_block_split)+ or name_split[: len(current_packing_block_split)] != current_packing_block_split+ ):continueold_dtype = data_torch.dtype@@ -819,7 +846,7 @@ class ModelBase(OriModel):clean_weight_list = []modify_name = self._special_name_handle(name)- for new_name, data_torch in (self.modify_tensors(data_torch, modify_name, bid)):+ for new_name, data_torch in self.modify_tensors(data_torch, modify_name, bid):data = data_torch.squeeze().cpu().numpy()# if data ends up empty, it means data_torch was a scalar tensor -> restore@@ -835,26 +862,35 @@ class ModelBase(OriModel):# Conditions should closely match those in llama_model_quantize_internal in llama.cpp# Some tensor types are always in float32- if data_qtype is False and (any(self.match_model_tensor_name(new_name, key, bid) for key in (- gguf.MODEL_TENSOR.FFN_GATE_INP,- gguf.MODEL_TENSOR.POS_EMBD,- gguf.MODEL_TENSOR.TOKEN_TYPES,- gguf.MODEL_TENSOR.SSM_CONV1D,- gguf.MODEL_TENSOR.TIME_MIX_FIRST,- gguf.MODEL_TENSOR.TIME_MIX_W1,- gguf.MODEL_TENSOR.TIME_MIX_W2,- gguf.MODEL_TENSOR.TIME_MIX_DECAY_W1,- gguf.MODEL_TENSOR.TIME_MIX_DECAY_W2,- )) or not new_name.endswith(".weight")):+ if data_qtype is False and (+ any(+ self.match_model_tensor_name(new_name, key, bid)+ for key in (+ gguf.MODEL_TENSOR.FFN_GATE_INP,+ gguf.MODEL_TENSOR.POS_EMBD,+ gguf.MODEL_TENSOR.TOKEN_TYPES,+ gguf.MODEL_TENSOR.SSM_CONV1D,+ gguf.MODEL_TENSOR.TIME_MIX_FIRST,+ gguf.MODEL_TENSOR.TIME_MIX_W1,+ gguf.MODEL_TENSOR.TIME_MIX_W2,+ gguf.MODEL_TENSOR.TIME_MIX_DECAY_W1,+ gguf.MODEL_TENSOR.TIME_MIX_DECAY_W2,+ )+ )+ or not new_name.endswith(".weight")+ ):data_qtype = gguf.GGMLQuantizationType.F32- if data_qtype is False and any(self.match_model_tensor_name(new_name, key, bid) for key in (+ if data_qtype is False and any(+ self.match_model_tensor_name(new_name, key, bid)+ for key in (gguf.MODEL_TENSOR.TOKEN_EMBD,gguf.MODEL_TENSOR.OUTPUT,- )):+ )+ ):if self.ftype in (- gguf.LlamaFileType.MOSTLY_TQ1_0,- gguf.LlamaFileType.MOSTLY_TQ2_0,+ gguf.LlamaFileType.MOSTLY_TQ1_0,+ gguf.LlamaFileType.MOSTLY_TQ2_0,):# TODO: use Q4_K and Q6_Kdata_qtype = gguf.GGMLQuantizationType.F16@@ -885,18 +921,21 @@ class ModelBase(OriModel):data_qtype = gguf.GGMLQuantizationType.Q5_0elif self.ftype == gguf.LlamaFileType.MOSTLY_Q5_1:data_qtype = gguf.GGMLQuantizationType.Q5_1- elif self.ftype == gguf.LlamaFileType.MOSTLY_Q2_K_S or \- self.ftype == gguf.LlamaFileType.MOSTLY_Q2_K:+ elif self.ftype == gguf.LlamaFileType.MOSTLY_Q2_K_S or self.ftype == gguf.LlamaFileType.MOSTLY_Q2_K:data_qtype = gguf.GGMLQuantizationType.Q2_K- elif self.ftype == gguf.LlamaFileType.MOSTLY_Q3_K_S or \- self.ftype == gguf.LlamaFileType.MOSTLY_Q3_K_M or \- self.ftype == gguf.LlamaFileType.MOSTLY_Q3_K_L:+ elif (+ self.ftype == gguf.LlamaFileType.MOSTLY_Q3_K_S+ or self.ftype == gguf.LlamaFileType.MOSTLY_Q3_K_M+ or self.ftype == gguf.LlamaFileType.MOSTLY_Q3_K_L+ ):data_qtype = gguf.GGMLQuantizationType.Q3_K- elif self.ftype == gguf.LlamaFileType.MOSTLY_Q4_K_S or \- self.ftype == gguf.LlamaFileType.MOSTLY_Q4_K_M:+ elif (+ self.ftype == gguf.LlamaFileType.MOSTLY_Q4_K_S or self.ftype == gguf.LlamaFileType.MOSTLY_Q4_K_M+ ):data_qtype = gguf.GGMLQuantizationType.Q4_K- elif self.ftype == gguf.LlamaFileType.MOSTLY_Q5_K_S \- or self.ftype == gguf.LlamaFileType.MOSTLY_Q5_K_M:+ elif (+ self.ftype == gguf.LlamaFileType.MOSTLY_Q5_K_S or self.ftype == gguf.LlamaFileType.MOSTLY_Q5_K_M+ ):data_qtype = gguf.GGMLQuantizationType.Q5_Kelif self.ftype == gguf.LlamaFileType.MOSTLY_Q6_K:data_qtype = gguf.GGMLQuantizationType.Q6_K@@ -904,8 +943,11 @@ class ModelBase(OriModel):raise ValueError(f"Unknown file type: {self.ftype.name}")if data_qtype.name.endswith("_K") and data_torch.shape[-1] % 256 != 0:- if data_qtype in [gguf.GGMLQuantizationType.Q2_K, gguf.GGMLQuantizationType.Q3_K,- gguf.GGMLQuantizationType.Q4_K]:+ if data_qtype in [+ gguf.GGMLQuantizationType.Q2_K,+ gguf.GGMLQuantizationType.Q3_K,+ gguf.GGMLQuantizationType.Q4_K,+ ]:data_qtype = gguf.GGMLQuantizationType.Q5_0elif data_qtype == gguf.GGMLQuantizationType.Q5_K:data_qtype = gguf.GGMLQuantizationType.Q5_0@@ -913,7 +955,9 @@ class ModelBase(OriModel):data_qtype = gguf.GGMLQuantizationType.Q8_0if isinstance(data_qtype, bool) or data_qtype in [- gguf.GGMLQuantizationType.F16, gguf.GGMLQuantizationType.BF16, gguf.GGMLQuantizationType.F32+ gguf.GGMLQuantizationType.F16,+ gguf.GGMLQuantizationType.BF16,+ gguf.GGMLQuantizationType.F32,]:try:data = gguf.quants.quantize(data, data_qtype)@@ -923,8 +967,8 @@ class ModelBase(OriModel):data = gguf.quants.quantize(data, data_qtype)else:# for deepseek v2- if name.endswith("kv_b_proj.weight") and self.model_arch.name == 'DEEPSEEK2':- layer_name = name[:-len('.weight')]+ if name.endswith("kv_b_proj.weight") and self.model_arch.name == "DEEPSEEK2":+ layer_name = name[: -len(".weight")]module = get_module(self.model, layer_name)n_head_kv = self.hparams["num_key_value_heads"]v_head_dim = self.hparams["v_head_dim"]@@ -933,7 +977,7 @@ class ModelBase(OriModel):attr_list = {"scale": None, "zp": None, "d_scale": None, "wmin": None, "d_wmin": None}for attr in attr_list:if hasattr(module, attr) or hasattr(module, "w_" + attr):- if attr in ['scale', "zp"]:+ if attr in ["scale", "zp"]:attr_tensor = getattr(module, attr)else:attr_tensor = getattr(module, "w_" + attr)@@ -955,7 +999,7 @@ class ModelBase(OriModel):elif len(data_torch.shape) == 3 and len(re.findall("\d+", name)) == 2:new_data = []for idx, arr in enumerate(data_torch):- arr_name = name.split('.')+ arr_name = name.split(".")for i in range(len(arr_name) - 1, -1, -1):if arr_name[i].isdecimal() and int(arr_name[i]) == (data_torch.shape[0] - 1):arr_name[i] = str(idx)@@ -967,8 +1011,9 @@ class ModelBase(OriModel):else:data, data_qtype = self._quant_data(data_torch, data_qtype, name, modify_name, bid)- shape = gguf.quant_shape_from_byte_shape(- data.shape, data_qtype) if data.dtype == np.uint8 else data.shape+ shape = (+ gguf.quant_shape_from_byte_shape(data.shape, data_qtype) if data.dtype == np.uint8 else data.shape+ )# reverse shape to make it similar to the internal ggml dimension ordershape_str = f"{{{', '.join(str(n) for n in reversed(shape))}}}"@@ -976,7 +1021,8 @@ class ModelBase(OriModel):# n_dims is implicit in the shapelogger.info(f"{f'%-{max_name_len}s' % f'{new_name},'} {old_dtype}"- f" --> {data_qtype.name}, shape = {shape_str}")+ f" --> {data_qtype.name}, shape = {shape_str}"+ )self.gguf_writer.add_tensor(new_name, data, raw_dtype=data_qtype)@@ -1037,7 +1083,8 @@ class TextModel(ModelBase):self.metadata.version,self.metadata.size_label,output_type,- model_type="LoRA" if total_params < 0 else None)+ model_type="LoRA" if total_params < 0 else None,+ )else:fname_default: str = gguf.naming_convention(self.metadata.name,@@ -1046,7 +1093,8 @@ class TextModel(ModelBase):self.metadata.version,size_label=None,output_type=None,- model_type="vocab")+ model_type="vocab",+ )# Use the default filenameself.fname_out = self.fname_out / f"{fname_default}.gguf"@@ -1064,8 +1112,9 @@ class TextModel(ModelBase):def set_gguf_parameters(self):self.gguf_writer.add_block_count(self.block_count)- if (n_ctx := self.find_hparam(["max_position_embeddings", "n_ctx", "n_positions", "max_length"],- optional=True)) is not None:+ if (+ n_ctx := self.find_hparam(["max_position_embeddings", "n_ctx", "n_positions", "max_length"], optional=True)+ ) is not None:self.gguf_writer.add_context_length(n_ctx)logger.info(f"gguf: context length = {n_ctx}")@@ -1091,8 +1140,9 @@ class TextModel(ModelBase):if (f_rms_eps := self.hparams.get("rms_norm_eps")) is not None:self.gguf_writer.add_layer_norm_rms_eps(f_rms_eps)logger.info(f"gguf: rms norm epsilon = {f_rms_eps}")- if (f_norm_eps := self.find_hparam(["layer_norm_eps", "layer_norm_epsilon", "norm_epsilon"],- optional=True)) is not None:+ if (+ f_norm_eps := self.find_hparam(["layer_norm_eps", "layer_norm_epsilon", "norm_epsilon"], optional=True)+ ) is not None:self.gguf_writer.add_layer_norm_eps(f_norm_eps)logger.info(f"gguf: layer norm epsilon = {f_norm_eps}")if (n_experts := self.hparams.get("num_local_experts")) is not None:@@ -1111,7 +1161,7 @@ class TextModel(ModelBase):def write_vocab(self):if len(self.gguf_writer.tensors) != 1:- raise ValueError('Splitting the vocabulary is not supported')+ raise ValueError("Splitting the vocabulary is not supported")self.prepare_metadata(vocab_only=True)self.gguf_writer.write_header_to_file(path=self.fname_out)@@ -1149,6 +1199,7 @@ class TextModel(ModelBase):toktypes: list[int] = []from transformers import AutoTokenizer+tokenizer = AutoTokenizer.from_pretrained(self.dir_model)vocab_size = self.hparams.get("vocab_size", len(tokenizer.vocab))assert max(tokenizer.vocab.values()) < vocab_size@@ -1183,7 +1234,7 @@ class TextModel(ModelBase):else:# NOTE: this was added for Gemma.# Encoding and decoding the tokens above isn't sufficient for this case.- token = token.replace(b"\xe2\x96\x81".decode("utf-8"), " ")+ token = token.replace(b"\xe2\x96\x81".decode("utf-8"), " ")toktypes.append(gguf.TokenType.USER_DEFINED)else:toktypes.append(gguf.TokenType.NORMAL)@@ -1201,7 +1252,7 @@ class TextModel(ModelBase):# we will use this unique identifier to write a "tokenizer.ggml.pre" entry in the GGUF file which we can# use in llama.cpp to implement the same pre-tokenizer- chktxt = '\n \n\n \n\n\n \t \t\t \t\n \n \n \n \n🚀 (normal) 😶\u200d🌫️ (multiple emojis concatenated) ✅ 🦙🦙 3 33 333 3333 33333 333333 3333333 33333333 3.3 3..3 3...3 កាន់តែពិសេសអាច😁 ?我想在apple工作1314151天~ ------======= нещо на Български \'\'\'\'\'\'```````""""......!!!!!!?????? I\'ve been \'told he\'s there, \'RE you sure? \'M not sure I\'ll make it, \'D you like some tea? We\'Ve a\'lL' # pylint: disable=C0301+ chktxt = "\n \n\n \n\n\n \t \t\t \t\n \n \n \n \n🚀 (normal) 😶\u200d🌫️ (multiple emojis concatenated) ✅ 🦙🦙 3 33 333 3333 33333 333333 3333333 33333333 3.3 3..3 3...3 កាន់តែពិសេសអាច😁 ?我想在apple工作1314151天~ ------======= нещо на Български ''''''```````\"\"\"\"......!!!!!!?????? I've been 'told he's there, 'RE you sure? 'M not sure I'll make it, 'D you like some tea? We'Ve a'lL" # pylint: disable=C0301chktok = tokenizer.encode(chktxt)chkhsh = sha256(str(chktok).encode()).hexdigest()@@ -1400,7 +1451,8 @@ class TextModel(ModelBase):logger.warning("** - the model has not been added to convert_hf_to_gguf_update.py yet")logger.warning("** - the pre-tokenization config has changed upstream")logger.warning(- "** Check your model files and convert_hf_to_gguf_update.py and update them accordingly.")+ "** Check your model files and convert_hf_to_gguf_update.py and update them accordingly."+ )logger.warning("** ref: https://github.com/ggml-org/llama.cpp/pull/6920")logger.warning("**")logger.warning(f"** chkhsh: {chkhsh}")@@ -1434,6 +1486,7 @@ class TextModel(ModelBase):toktypes: list[int] = []from transformers import AutoTokenizer+tokenizer = AutoTokenizer.from_pretrained(dir_model, trust_remote_code=True)vocab_size = hparams["vocab_size"]assert max(tokenizer.get_vocab().values()) < vocab_size@@ -1449,7 +1502,7 @@ class TextModel(ModelBase):continuemerged = QwenModel.bpe(mergeable_ranks, token, max_rank=rank)assert len(merged) == 2- merges.append(' '.join(map(QwenModel.token_bytes_to_string, merged)))+ merges.append(" ".join(map(QwenModel.token_bytes_to_string, merged)))# for this kind of tokenizer, added_vocab is not a subset of vocab, so they need to be combinedadded_vocab = tokenizer.special_tokens@@ -1496,7 +1549,7 @@ class TextModel(ModelBase):def _create_vocab_sentencepiece(self):from sentencepiece import SentencePieceProcessor- tokenizer_path = self.dir_model / 'tokenizer.model'+ tokenizer_path = self.dir_model / "tokenizer.model"if not tokenizer_path.is_file():raise FileNotFoundError(f"File not found: {tokenizer_path}")@@ -1504,12 +1557,16 @@ class TextModel(ModelBase):tokenizer = SentencePieceProcessor()tokenizer.LoadFromFile(str(tokenizer_path))- vocab_size = self.find_hparam(- [- "vocab_size_per_layer_input", # gemma3n- "vocab_size",- ],- optional=True) or tokenizer.vocab_size()+ vocab_size = (+ self.find_hparam(+ [+ "vocab_size_per_layer_input", # gemma3n+ "vocab_size",+ ],+ optional=True,+ )+ or tokenizer.vocab_size()+ )tokens: list[bytes] = [f"[PAD{i}]".encode("utf-8") for i in range(vocab_size)]scores: list[float] = [-10000.0] * vocab_size@@ -1517,7 +1574,7 @@ class TextModel(ModelBase):for token_id in range(tokenizer.vocab_size()):if token_id >= vocab_size:- logger.warning(f'ignore tokens from {token_id}: id is out of range, max={vocab_size - 1}')+ logger.warning(f"ignore tokens from {token_id}: id is out of range, max={vocab_size - 1}")breakpiece = tokenizer.IdToPiece(token_id)@@ -1538,21 +1595,21 @@ class TextModel(ModelBase):scores[token_id] = scoretoktypes[token_id] = toktype- added_tokens_file = self.dir_model / 'added_tokens.json'+ added_tokens_file = self.dir_model / "added_tokens.json"if added_tokens_file.is_file():with open(added_tokens_file, "r", encoding="utf-8") as f:added_tokens_json = json.load(f)for key in added_tokens_json:token_id = added_tokens_json[key]if token_id >= vocab_size:- logger.warning(f'ignore token {token_id}: id is out of range, max={vocab_size - 1}')+ logger.warning(f"ignore token {token_id}: id is out of range, max={vocab_size - 1}")continuetokens[token_id] = key.encode("utf-8")scores[token_id] = -1000.0toktypes[token_id] = SentencePieceTokenTypes.USER_DEFINED- tokenizer_config_file = self.dir_model / 'tokenizer_config.json'+ tokenizer_config_file = self.dir_model / "tokenizer_config.json"if tokenizer_config_file.is_file():with open(tokenizer_config_file, "r", encoding="utf-8") as f:tokenizer_config_json = json.load(f)@@ -1561,12 +1618,13 @@ class TextModel(ModelBase):token_id = int(token_id)token: str = token_data["content"]if token_id >= vocab_size:- logger.warning(f'ignore token {token_id}: id is out of range, max={vocab_size - 1}')+ logger.warning(f"ignore token {token_id}: id is out of range, max={vocab_size - 1}")continueif toktypes[token_id] != SentencePieceTokenTypes.UNUSED:if tokens[token_id] != token.encode("utf-8"):logger.warning(- f'replacing token {token_id}: {tokens[token_id].decode("utf-8")!r} -> {token!r}')+ f'replacing token {token_id}: {tokens[token_id].decode("utf-8")!r} -> {token!r}'+ )if token_data.get("special") or self.does_token_look_special(token):toktypes[token_id] = SentencePieceTokenTypes.CONTROLelse:@@ -1612,15 +1670,15 @@ class TextModel(ModelBase):assert (self.dir_model / "rwkv_vocab_v20230424.txt").is_file()vocab_size = self.hparams.get("vocab_size", 65536)- tokens: list[bytes] = ['<s>'.encode("utf-8")]+ tokens: list[bytes] = ["<s>".encode("utf-8")]toktypes: list[int] = [gguf.TokenType.CONTROL]with open(self.dir_model / "rwkv_vocab_v20230424.txt", "r", encoding="utf-8") as f:lines = f.readlines()for line in lines:- parts = line.split(' ')+ parts = line.split(" ")assert len(parts) >= 3- token, token_len = ast.literal_eval(' '.join(parts[1:-1])), int(parts[-1])+ token, token_len = ast.literal_eval(" ".join(parts[1:-1])), int(parts[-1])token = token.encode("utf-8") if isinstance(token, str) else tokenassert isinstance(token, bytes)assert len(token) == token_len@@ -1757,6 +1815,7 @@ class MmprojModel(ModelBase):# move vision config to the top level, while preserving the original hparams in global_configimport copy+self.global_config = copy.deepcopy(self.hparams)self.hparams_vision = self.get_vision_config()self.hparams_audio = self.get_audio_config()@@ -1849,7 +1908,8 @@ class GPTNeoXModel(TextModel):self.gguf_writer.add_block_count(block_count)self.gguf_writer.add_feed_forward_length(self.hparams["intermediate_size"])self.gguf_writer.add_rope_dimension_count(- int(self.hparams["rotary_pct"] * (self.hparams["hidden_size"] // self.hparams["num_attention_heads"])),)+ int(self.hparams["rotary_pct"] * (self.hparams["hidden_size"] // self.hparams["num_attention_heads"])),+ )self.gguf_writer.add_head_count(self.hparams["num_attention_heads"])self.gguf_writer.add_parallel_residual(self.hparams.get("use_parallel_residual", True))self.gguf_writer.add_layer_norm_eps(self.hparams["layer_norm_eps"])@@ -1912,7 +1972,7 @@ class BloomModel(TextModel):n_head = self.hparams.get("n_head", self.hparams.get("num_attention_heads"))n_embed = self.hparams.get("hidden_size", self.hparams.get("n_embed"))- name = re.sub(r'transformer\.', '', name)+ name = re.sub(r"transformer\.", "", name)tensors: list[tuple[str, Tensor]] = []@@ -2069,11 +2129,13 @@ class BaichuanModel(TextModel):logger.info(f"Unpacking and permuting layer {bid}")tensors = [(- self.format_tensor_name(gguf.MODEL_TENSOR.ATTN_Q,- bid), self._reverse_hf_permute_part(data_torch, 0, head_count, head_count)),+ self.format_tensor_name(gguf.MODEL_TENSOR.ATTN_Q, bid),+ self._reverse_hf_permute_part(data_torch, 0, head_count, head_count),+ ),(self.format_tensor_name(gguf.MODEL_TENSOR.ATTN_K, bid),- self._reverse_hf_permute_part(data_torch, 1, head_count, head_count_kv)),+ self._reverse_hf_permute_part(data_torch, 1, head_count, head_count_kv),+ ),(self.format_tensor_name(gguf.MODEL_TENSOR.ATTN_V, bid), self._reverse_hf_part(data_torch, 2)),]else:@@ -2086,8 +2148,10 @@ class BaichuanModel(TextModel):n_head //= n_kv_headreturn (- weights.reshape(n_head, 2, weights.shape[0] // n_head // 2,- *weights.shape[1:]).swapaxes(1, 2).reshape(weights.shape))+ weights.reshape(n_head, 2, weights.shape[0] // n_head // 2, *weights.shape[1:])+ .swapaxes(1, 2)+ .reshape(weights.shape)+ )def _reverse_hf_permute_part(self,@@ -2097,11 +2161,11 @@ class BaichuanModel(TextModel):n_head_kv: int | None = None,) -> Tensor:r = weights.shape[0] // 3- return self._reverse_hf_permute(weights[r * n_part:r*n_part + r, ...], n_head, n_head_kv)+ return self._reverse_hf_permute(weights[r * n_part : r * n_part + r, ...], n_head, n_head_kv)def _reverse_hf_part(self, weights: Tensor, n_part: int) -> Tensor:r = weights.shape[0] // 3- return weights[r * n_part:r*n_part + r, ...]+ return weights[r * n_part : r * n_part + r, ...]@ModelBase.register("XverseForCausalLM")@@ -2117,6 +2181,7 @@ class XverseModel(TextModel):toktypes: list[int] = []from transformers import AutoTokenizer+tokenizer = AutoTokenizer.from_pretrained(dir_model)vocab_size = hparams.get("vocab_size", len(tokenizer.vocab))# Since we are checking the maximum index, we need to ensure it's strictly less than vocab_size,@@ -2129,12 +2194,12 @@ class XverseModel(TextModel):added_vocab = tokenizer.get_added_vocab()for token_id in range(vocab_size):- token_text = reverse_vocab[token_id].encode('utf-8')+ token_text = reverse_vocab[token_id].encode("utf-8")# replace "\x00" to string with length > 0if token_text == b"\x00":toktype = gguf.TokenType.BYTE # special- token_text = f"<{token_text}>".encode('utf-8')- elif re.fullmatch(br"<0x[0-9A-Fa-f]{2}>", token_text):+ token_text = f"<{token_text}>".encode("utf-8")+ elif re.fullmatch(rb"<0x[0-9A-Fa-f]{2}>", token_text):toktype = gguf.TokenType.BYTE # specialelif reverse_vocab[token_id] in added_vocab:if tokenizer.added_tokens_decoder[token_id].special:@@ -2205,8 +2270,10 @@ class XverseModel(TextModel):n_head //= n_kv_headreturn (- weights.reshape(n_head, 2, weights.shape[0] // n_head // 2,- *weights.shape[1:]).swapaxes(1, 2).reshape(weights.shape))+ weights.reshape(n_head, 2, weights.shape[0] // n_head // 2, *weights.shape[1:])+ .swapaxes(1, 2)+ .reshape(weights.shape)+ )@ModelBase.register("FalconForCausalLM", "RWForCausalLM")@@ -2254,7 +2321,7 @@ class FalconModel(TextModel):n_head_kv = self.find_hparam(["num_kv_heads", "n_head_kv"], optional=True) or 1head_dim = self.hparams["hidden_size"] // n_head- qkv = data_torch.view(n_head_kv, n_head//n_head_kv + 2, head_dim, head_dim * n_head)+ qkv = data_torch.view(n_head_kv, n_head // n_head_kv + 2, head_dim, head_dim * n_head)q = qkv[:, :-2].reshape(n_head * head_dim, head_dim * n_head)k = qkv[:, [-2]].reshape(n_head_kv * head_dim, head_dim * n_head)v = qkv[:, [-1]].reshape(n_head_kv * head_dim, head_dim * n_head)@@ -2289,7 +2356,8 @@ class RefactModel(TextModel):# TODO: how to determine special FIM tokens automatically?special_vocab = gguf.SpecialVocab(- self.dir_model, load_merges=False, special_token_types=['prefix', 'suffix', 'middle', 'eot'])+ self.dir_model, load_merges=False, special_token_types=["prefix", "suffix", "middle", "eot"]+ )special_vocab._set_special_token("prefix", 1)special_vocab._set_special_token("suffix", 3)special_vocab._set_special_token("middle", 2)@@ -2301,7 +2369,7 @@ class RefactModel(TextModel):inner_dim = 4 * hidden_dimhidden_dim = int(2 * inner_dim / 3)multiple_of = 256- ff_dim = multiple_of * ((hidden_dim+multiple_of-1) // multiple_of)+ ff_dim = multiple_of * ((hidden_dim + multiple_of - 1) // multiple_of)block_count = self.hparams["n_layer"]@@ -2321,7 +2389,7 @@ class RefactModel(TextModel):inner_dim = 4 * hidden_dimhidden_dim = int(2 * inner_dim / 3)multiple_of = 256- ff_dim = multiple_of * ((hidden_dim+multiple_of-1) // multiple_of)+ ff_dim = multiple_of * ((hidden_dim + multiple_of - 1) // multiple_of)n_head = self.hparams["n_head"]n_head_kv = 1head_dim = self.hparams["n_embd"] // n_head@@ -2331,9 +2399,11 @@ class RefactModel(TextModel):if bid is not None:if name == f"transformer.h.{bid}.attn.kv.weight":tensors.append(- (self.format_tensor_name(gguf.MODEL_TENSOR.ATTN_K, bid), data_torch[:n_head_kv * head_dim]))+ (self.format_tensor_name(gguf.MODEL_TENSOR.ATTN_K, bid), data_torch[: n_head_kv * head_dim])+ )tensors.append(- (self.format_tensor_name(gguf.MODEL_TENSOR.ATTN_V, bid), data_torch[n_head_kv * head_dim:]))+ (self.format_tensor_name(gguf.MODEL_TENSOR.ATTN_V, bid), data_torch[n_head_kv * head_dim :])+ )elif name == f"transformer.h.{bid}.attn.q.weight":tensors.append((self.format_tensor_name(gguf.MODEL_TENSOR.ATTN_Q, bid), data_torch))elif name == f"transformer.h.{bid}.mlp.gate_up_proj.weight":@@ -2367,11 +2437,13 @@ class StableLMModel(TextModel):self.gguf_writer.add_feed_forward_length(hparams["intermediate_size"])rotary_factor = self.find_hparam(["partial_rotary_factor", "rope_pct"])self.gguf_writer.add_rope_dimension_count(- int(rotary_factor * (hparams["hidden_size"] // hparams["num_attention_heads"])))+ int(rotary_factor * (hparams["hidden_size"] // hparams["num_attention_heads"]))+ )self.gguf_writer.add_head_count(hparams["num_attention_heads"])self.gguf_writer.add_head_count_kv(hparams["num_key_value_heads"])self.gguf_writer.add_parallel_residual(- hparams["use_parallel_residual"] if "use_parallel_residual" in hparams else True)+ hparams["use_parallel_residual"] if "use_parallel_residual" in hparams else True+ )self.gguf_writer.add_layer_norm_eps(self.find_hparam(["layer_norm_eps", "norm_eps"]))self.gguf_writer.add_file_type(self.ftype)@@ -2429,15 +2501,22 @@ class StableLMModel(TextModel):if self._q_norms is not None or self._k_norms is not None:# flatten two `list[dict[str, Tensor]]` into a single `list[str]`- norms = ([k for d in self._q_norms for k in d.keys()] if self._q_norms is not None else- []) + ([k for d in self._k_norms for k in d.keys()] if self._k_norms is not None else [])+ norms = ([k for d in self._q_norms for k in d.keys()] if self._q_norms is not None else []) + (+ [k for d in self._k_norms for k in d.keys()] if self._k_norms is not None else []+ )if len(norms) > 0:raise ValueError(f"Unprocessed norms: {norms}")@ModelBase.register(- "LLaMAForCausalLM", "LlamaForCausalLM", "MistralForCausalLM", "MixtralForCausalLM", "VLlama3ForCausalLM",- "LlavaForConditionalGeneration", "LlamaModel")+ "LLaMAForCausalLM",+ "LlamaForCausalLM",+ "MistralForCausalLM",+ "MixtralForCausalLM",+ "VLlama3ForCausalLM",+ "LlavaForConditionalGeneration",+ "LlamaModel",+)class LlamaModel(TextModel):model_arch = gguf.MODEL_ARCH.LLAMAundo_permute = True@@ -2461,14 +2540,15 @@ class LlamaModel(TextModel):# Apply to CodeLlama only (and ignore for Llama 3 with a vocab size of 128256)if self.hparams.get("vocab_size", 32000) == 32016:special_vocab = gguf.SpecialVocab(- self.dir_model, load_merges=False, special_token_types=['prefix', 'suffix', 'middle', 'eot'])+ self.dir_model, load_merges=False, special_token_types=["prefix", "suffix", "middle", "eot"]+ )special_vocab._set_special_token("prefix", 32007)special_vocab._set_special_token("suffix", 32008)special_vocab._set_special_token("middle", 32009)special_vocab._set_special_token("eot", 32010)special_vocab.add_to_gguf(self.gguf_writer)- tokenizer_config_file = self.dir_model / 'tokenizer_config.json'+ tokenizer_config_file = self.dir_model / "tokenizer_config.json"if tokenizer_config_file.is_file():with open(tokenizer_config_file, "r", encoding="utf-8") as f:tokenizer_config_json = json.load(f)@@ -2498,18 +2578,22 @@ class LlamaModel(TextModel):if n_head_kv is not None and n_head != n_head_kv:n_head = n_head_kvreturn (- weights.reshape(n_head, 2, weights.shape[0] // n_head // 2,- *weights.shape[1:]).swapaxes(1, 2).reshape(weights.shape))+ weights.reshape(n_head, 2, weights.shape[0] // n_head // 2, *weights.shape[1:])+ .swapaxes(1, 2)+ .reshape(weights.shape)+ )_experts: list[dict[str, Tensor]] | None = Nonedef modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:n_head = self.hparams["num_attention_heads"]n_kv_head = self.hparams.get("num_key_value_heads")- is_vision_tensor = "vision_tower" in name \- or "vision_model" in name \- or "model.connector" in name \+ is_vision_tensor = (+ "vision_tower" in name+ or "vision_model" in name+ or "model.connector" in nameor "multi_modal_projector" in name+ )if is_vision_tensor:return [] # skip vision tensors@@ -2564,11 +2648,11 @@ class LlamaModel(TextModel):def generate_extra_tensors(self) -> Iterable[tuple[str, Tensor]]:if rope_scaling := self.find_hparam(["rope_scaling"], optional=True):- if rope_scaling.get("rope_type", '').lower() == "llama3":+ if rope_scaling.get("rope_type", "").lower() == "llama3":base = self.hparams.get("rope_theta", 10000.0)if (dim := self.hparams.get("head_dim")) is None:dim = self.hparams["hidden_size"] // self.hparams["num_attention_heads"]- freqs = 1.0 / (base**(torch.arange(0, dim, 2, dtype=torch.float32) / dim))+ freqs = 1.0 / (base ** (torch.arange(0, dim, 2, dtype=torch.float32) / dim))factor = rope_scaling.get("factor", 8.0)low_freq_factor = rope_scaling.get("low_freq_factor", 1.0)@@ -2587,12 +2671,13 @@ class LlamaModel(TextModel):elif wavelen > low_freq_wavelen:rope_factors.append(factor)else:- smooth = (old_context_len/wavelen - low_freq_factor) / (high_freq_factor-low_freq_factor)- rope_factors.append(1 / ((1-smooth) / factor + smooth))+ smooth = (old_context_len / wavelen - low_freq_factor) / (high_freq_factor - low_freq_factor)+ rope_factors.append(1 / ((1 - smooth) / factor + smooth))yield (self.format_tensor_name(gguf.MODEL_TENSOR.ROPE_FREQS),- torch.tensor(rope_factors, dtype=torch.float32))+ torch.tensor(rope_factors, dtype=torch.float32),+ )def prepare_tensors(self):super().prepare_tensors()@@ -2636,9 +2721,9 @@ class LlavaVisionModel(MmprojModel):raise ValueError(f"Unsupported model type: {self.hparams['model_type']}")def get_token_id(self, token: str) -> int:- tokenizer_config_file = self.dir_model / 'tokenizer_config.json'+ tokenizer_config_file = self.dir_model / "tokenizer_config.json"with open(tokenizer_config_file, "r", encoding="utf-8") as f:- added_tokens_decoder = json.load(f)['added_tokens_decoder']+ added_tokens_decoder = json.load(f)["added_tokens_decoder"]for id_, token_data in added_tokens_decoder.items():if token_data["content"] == token:return int(id_)@@ -2750,8 +2835,10 @@ class Llama4Model(LlamaModel):name_gate = name.replace("gate_up_proj", "gate_proj.weight")dim_half = data_torch.shape[-1] // 2gate_proj_weight, up_proj_weight = data_torch.transpose(-1, -2).split(dim_half, dim=-2)- return [(self.map_tensor_name(name_gate), gate_proj_weight),- (self.map_tensor_name(name_up), up_proj_weight)]+ return [+ (self.map_tensor_name(name_gate), gate_proj_weight),+ (self.map_tensor_name(name_up), up_proj_weight),+ ]if name.endswith("down_proj"):name += ".weight"@@ -2781,7 +2868,7 @@ class Llama4VisionModel(MmprojModel):name += ".weight"if "multi_modal_projector.linear_1" in name:# despite the name with number postfix, this is a single fully connected layer- return [(gguf.TENSOR_NAMES[gguf.MODEL_TENSOR.V_MMPROJ_FC] + '.weight', data_torch)]+ return [(gguf.TENSOR_NAMES[gguf.MODEL_TENSOR.V_MMPROJ_FC] + ".weight", data_torch)]return [(self.map_tensor_name(name), data_torch)]return []@@ -2812,7 +2899,7 @@ class DeciModel(TextModel):# DeciLM-specific codeif n % k == 0:return n- return n + k - (n%k)+ return n + k - (n % k)def __init__(self, *args, **kwargs):super().__init__(*args, **kwargs)@@ -2846,7 +2933,8 @@ class DeciModel(TextModel):self._num_heads.append(0)else:self._num_kv_heads.append(- self.hparams["num_attention_heads"] // _block_configs[il]["attention"]["n_heads_in_group"])+ self.hparams["num_attention_heads"] // _block_configs[il]["attention"]["n_heads_in_group"]+ )self._num_heads.append(self.hparams["num_attention_heads"])if _block_configs[il]["ffn"]["ffn_mult"] is None: # dummy layer_ffn_multipliers.append(0.0)@@ -2919,8 +3007,10 @@ class DeciModel(TextModel):if n_head_kv is not None and n_head != n_head_kv:n_head = n_head_kvreturn (- weights.reshape(n_head, 2, weights.shape[0] // n_head // 2,- *weights.shape[1:]).swapaxes(1, 2).reshape(weights.shape))+ weights.reshape(n_head, 2, weights.shape[0] // n_head // 2, *weights.shape[1:])+ .swapaxes(1, 2)+ .reshape(weights.shape)+ )def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:n_head = self.hparams["num_attention_heads"]@@ -2943,11 +3033,11 @@ class DeciModel(TextModel):def generate_extra_tensors(self) -> Iterable[tuple[str, Tensor]]:if rope_scaling := self.find_hparam(["rope_scaling"], optional=True):- if rope_scaling.get("rope_type", '').lower() == "llama3":+ if rope_scaling.get("rope_type", "").lower() == "llama3":base = self.hparams.get("rope_theta", 10000.0)if (dim := self.hparams.get("head_dim")) is None:dim = self.hparams["hidden_size"] // self.hparams["num_attention_heads"]- freqs = 1.0 / (base**(torch.arange(0, dim, 2, dtype=torch.float32) / dim))+ freqs = 1.0 / (base ** (torch.arange(0, dim, 2, dtype=torch.float32) / dim))factor = rope_scaling.get("factor", 8.0)low_freq_factor = rope_scaling.get("low_freq_factor", 1.0)@@ -2966,12 +3056,13 @@ class DeciModel(TextModel):elif wavelen > low_freq_wavelen:rope_factors.append(factor)else:- smooth = (old_context_len/wavelen - low_freq_factor) / (high_freq_factor-low_freq_factor)- rope_factors.append(1 / ((1-smooth) / factor + smooth))+ smooth = (old_context_len / wavelen - low_freq_factor) / (high_freq_factor - low_freq_factor)+ rope_factors.append(1 / ((1 - smooth) / factor + smooth))yield (self.format_tensor_name(gguf.MODEL_TENSOR.ROPE_FREQS),- torch.tensor(rope_factors, dtype=torch.float32))+ torch.tensor(rope_factors, dtype=torch.float32),+ )def prepare_tensors(self):super().prepare_tensors()@@ -3000,7 +3091,9 @@ class BitnetModel(TextModel):def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:new_name = self.map_tensor_name(name)- if any(self.match_model_tensor_name(new_name, key, bid) for key in [+ if any(+ self.match_model_tensor_name(new_name, key, bid)+ for key in [gguf.MODEL_TENSOR.ATTN_Q,gguf.MODEL_TENSOR.ATTN_K,gguf.MODEL_TENSOR.ATTN_V,@@ -3008,7 +3101,8 @@ class BitnetModel(TextModel):gguf.MODEL_TENSOR.FFN_UP,gguf.MODEL_TENSOR.FFN_DOWN,gguf.MODEL_TENSOR.FFN_GATE,- ]):+ ]+ ):# transform weight into 1/0/-1 (in fp32)data_torch = self.weight_quant(data_torch)@@ -3111,7 +3205,7 @@ class DbrxModel(TextModel):exp_tensor_names = {"ffn.experts.mlp.w1": None, # LLM_TENSOR_FFN_GATE_EXPS ggml_tensor->ne{n_embd, n_ff, n_expert}"ffn.experts.mlp.w2": (0, 2, 1), # LLM_TENSOR_FFN_DOWN_EXPS ggml_tensor->ne{n_ff, n_embd, n_expert}- "ffn.experts.mlp.v1": None+ "ffn.experts.mlp.v1": None,} # LLM_TENSOR_FFN_UP_EXPS ggml_tensor->ne{n_embd, n_ff, n_expert}experts = False@@ -3134,7 +3228,8 @@ class DbrxModel(TextModel):return [(new_name, data_torch)]def tensor_force_quant(- self, name: str, new_name: str, bid: int | None, n_dims: int) -> gguf.GGMLQuantizationType | bool:+ self, name: str, new_name: str, bid: int | None, n_dims: int+ ) -> gguf.GGMLQuantizationType | bool:del name, new_name, bid # unusedreturn n_dims > 1@@ -3149,7 +3244,7 @@ class MiniCPMModel(TextModel):embedding_scale = float(self.hparams["scale_emb"])self.gguf_writer.add_embedding_scale(embedding_scale)logger.info(f"gguf: (minicpm) embedding_scale = {embedding_scale}")- residual_scale = self.hparams["scale_depth"] / self.hparams["num_hidden_layers"]**0.5+ residual_scale = self.hparams["scale_depth"] / self.hparams["num_hidden_layers"] ** 0.5self.gguf_writer.add_residual_scale(residual_scale)logger.info(f"gguf: (minicpm) residual_scale = {residual_scale}")logit_scale = self.hparams["hidden_size"] / self.hparams["dim_model_base"]@@ -3163,23 +3258,25 @@ class MiniCPMModel(TextModel):def generate_extra_tensors(self) -> Iterable[tuple[str, Tensor]]:rope_dims = self.hparams["hidden_size"] // self.hparams["num_attention_heads"]- rope_scaling = self.find_hparam(['rope_scaling'], True)+ rope_scaling = self.find_hparam(["rope_scaling"], True)if rope_scaling is not None:- long_factors = rope_scaling.get('long_factor', None)- short_factors = rope_scaling.get('short_factor', None)+ long_factors = rope_scaling.get("long_factor", None)+ short_factors = rope_scaling.get("short_factor", None)if long_factors is None or short_factors is None:- raise KeyError('Missing the required key rope_scaling.long_factor or rope_scaling_short_factor')+ raise KeyError("Missing the required key rope_scaling.long_factor or rope_scaling_short_factor")if len(long_factors) != len(short_factors) or len(long_factors) != rope_dims / 2:- raise ValueError(f'The length of rope long and short factors must be {rope_dims / 2}')+ raise ValueError(f"The length of rope long and short factors must be {rope_dims / 2}")yield (self.format_tensor_name(gguf.MODEL_TENSOR.ROPE_FACTORS_LONG),- torch.tensor(long_factors, dtype=torch.float32))+ torch.tensor(long_factors, dtype=torch.float32),+ )yield (self.format_tensor_name(gguf.MODEL_TENSOR.ROPE_FACTORS_SHORT),- torch.tensor(short_factors, dtype=torch.float32))+ torch.tensor(short_factors, dtype=torch.float32),+ )def set_vocab(self):self._set_vocab_sentencepiece()@@ -3222,25 +3319,27 @@ class MiniCPM3Model(TextModel):self.gguf_writer.add_rope_dimension_count(hparams["qk_rope_head_dim"])def generate_extra_tensors(self) -> Iterable[tuple[str, Tensor]]:- rope_scaling = self.find_hparam(['rope_scaling'], True)+ rope_scaling = self.find_hparam(["rope_scaling"], True)if rope_scaling is not None:rope_dims = self.hparams["qk_rope_head_dim"]- long_factors = rope_scaling.get('long_factor', None)- short_factors = rope_scaling.get('short_factor', None)+ long_factors = rope_scaling.get("long_factor", None)+ short_factors = rope_scaling.get("short_factor", None)if long_factors is None or short_factors is None:- raise KeyError('Missing the required key rope_scaling.long_factor or rope_scaling_short_factor')+ raise KeyError("Missing the required key rope_scaling.long_factor or rope_scaling_short_factor")if len(long_factors) != len(short_factors) or len(long_factors) != rope_dims / 2:- raise ValueError(f'The length of rope long and short factors must be {rope_dims / 2}')+ raise ValueError(f"The length of rope long and short factors must be {rope_dims / 2}")yield (self.format_tensor_name(gguf.MODEL_TENSOR.ROPE_FACTORS_LONG),- torch.tensor(long_factors, dtype=torch.float32))+ torch.tensor(long_factors, dtype=torch.float32),+ )yield (self.format_tensor_name(gguf.MODEL_TENSOR.ROPE_FACTORS_SHORT),- torch.tensor(short_factors, dtype=torch.float32))+ torch.tensor(short_factors, dtype=torch.float32),+ )def set_vocab(self):self._set_vocab_sentencepiece()@@ -3250,8 +3349,10 @@ class MiniCPM3Model(TextModel):n_head //= n_kv_headreturn (- weights.reshape(n_head, 2, weights.shape[0] // n_head // 2,- *weights.shape[1:]).swapaxes(1, 2).reshape(weights.shape))+ weights.reshape(n_head, 2, weights.shape[0] // n_head // 2, *weights.shape[1:])+ .swapaxes(1, 2)+ .reshape(weights.shape)+ )@ModelBase.register("QWenLMHeadModel")@@ -3261,8 +3362,9 @@ class QwenModel(TextModel):@staticmethoddef token_bytes_to_string(b):from transformers.models.gpt2.tokenization_gpt2 import bytes_to_unicode+byte_encoder = bytes_to_unicode()- return ''.join([byte_encoder[ord(char)] for char in b.decode('latin-1')])+ return "".join([byte_encoder[ord(char)] for char in b.decode("latin-1")])@staticmethoddef bpe(mergeable_ranks: dict[bytes, int], token: bytes, max_rank: int | None = None) -> list[bytes]:@@ -3278,7 +3380,7 @@ class QwenModel(TextModel):if min_rank is None or (max_rank is not None and min_rank >= max_rank):breakassert min_idx is not None- parts = parts[:min_idx] + [parts[min_idx] + parts[min_idx + 1]] + parts[min_idx + 2:]+ parts = parts[:min_idx] + [parts[min_idx] + parts[min_idx + 1]] + parts[min_idx + 2 :]return partsdef set_vocab(self):@@ -3320,8 +3422,12 @@ class Qwen2Model(TextModel):name = f"model.{name}" # map to Qwen2ForCausalLM tensorsif "language_model." in name:name = name.replace("language_model.", "") # for InternVL- if name.startswith("mlp") or name.startswith("multi_modal_projector") \- or name.startswith("vision_model") or name.startswith("audio_tower"):+ if (+ name.startswith("mlp")+ or name.startswith("multi_modal_projector")+ or name.startswith("vision_model")+ or name.startswith("audio_tower")+ ):# skip vision and audio tensorsreturn []yield from super().modify_tensors(data_torch, name, bid)@@ -3336,6 +3442,7 @@ class DreamModel(TextModel):toktypes: list[int] = []from transformers import AutoTokenizer+tokenizer = AutoTokenizer.from_pretrained(self.dir_model, trust_remote_code=True)vocab_dict = tokenizer.get_vocab()@@ -3354,7 +3461,7 @@ class DreamModel(TextModel):elif reverse_vocab[i] in added_vocab:tokens.append(reverse_vocab[i])# Check if it's a special token - treat special tokens as CONTROL tokens- if hasattr(tokenizer, 'added_tokens_decoder') and i in tokenizer.added_tokens_decoder:+ if hasattr(tokenizer, "added_tokens_decoder") and i in tokenizer.added_tokens_decoder:if tokenizer.added_tokens_decoder[i].special:toktypes.append(gguf.TokenType.CONTROL)else:@@ -3424,10 +3531,14 @@ class Ernie4_5Model(TextModel):total_q_dim = num_heads * head_dimtotal_k_dim = num_kv_heads * head_dimtotal_v_dim = num_kv_heads * head_dim- q_proj_weight, k_proj_weight, v_proj_weight = data_torch.split([total_q_dim, total_k_dim, total_v_dim],- dim=0)- return [(self.map_tensor_name(name_q), q_proj_weight), (self.map_tensor_name(name_k), k_proj_weight),- (self.map_tensor_name(name_v), v_proj_weight)]+ q_proj_weight, k_proj_weight, v_proj_weight = data_torch.split(+ [total_q_dim, total_k_dim, total_v_dim], dim=0+ )+ return [+ (self.map_tensor_name(name_q), q_proj_weight),+ (self.map_tensor_name(name_k), k_proj_weight),+ (self.map_tensor_name(name_v), v_proj_weight),+ ]# split the up_gate_proj into gate and up# up_gate_proj shape: [2 * intermediate_size, hidden_size]if "up_gate_proj" in name:@@ -3435,8 +3546,10 @@ class Ernie4_5Model(TextModel):name_gate = name.replace("up_gate_proj.weight", "gate_proj.weight")dim_half = data_torch.shape[0] // 2gate_proj_weight, up_proj_weight = data_torch.split(dim_half, dim=0)- return [(self.map_tensor_name(name_gate), gate_proj_weight),- (self.map_tensor_name(name_up), up_proj_weight)]+ return [+ (self.map_tensor_name(name_gate), gate_proj_weight),+ (self.map_tensor_name(name_up), up_proj_weight),+ ]return [(self.map_tensor_name(name), data_torch)]@@ -3457,13 +3570,16 @@ class Ernie4_5MoeModel(Ernie4_5Model):self.gguf_writer.add_leading_dense_block_count(self.hparams["moe_layer_start_index"])if (moe_intermediate_size := self.hparams.get("moe_intermediate_size")) is not None:self.gguf_writer.add_expert_feed_forward_length(moe_intermediate_size)- if (shared_expert_count := self.hparams.get('moe_num_shared_experts')) is not None:+ if (shared_expert_count := self.hparams.get("moe_num_shared_experts")) is not None:self.gguf_writer.add_expert_shared_count(shared_expert_count)- if shared_expert_count > 0 and (shared_expert_intermediate_size := self.hparams.get('intermediate_size')- ) is not None and (num_key_value_heads :=- self.hparams.get('num_key_value_heads')) is not None:+ if (+ shared_expert_count > 0+ and (shared_expert_intermediate_size := self.hparams.get("intermediate_size")) is not None+ and (num_key_value_heads := self.hparams.get("num_key_value_heads")) is not None+ ):self.gguf_writer.add_expert_shared_feed_forward_length(- shared_expert_intermediate_size // num_key_value_heads)+ shared_expert_intermediate_size // num_key_value_heads+ )def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:# Modify correction bias name as in DeepseekV2@@ -3555,8 +3671,12 @@ class Qwen2VLModel(TextModel):del bid # unusedif name.startswith("thinker."):name = name.replace("thinker.", "")- if name.startswith("visual") or name.startswith("audio") or \- name.startswith("talker") or name.startswith("token2wav"):+ if (+ name.startswith("visual")+ or name.startswith("audio")+ or name.startswith("talker")+ or name.startswith("token2wav")+ ):# skip multimodal tensorsreturn []return [(self.map_tensor_name(name), data_torch)]@@ -3580,11 +3700,11 @@ class Qwen2VLVisionModel(MmprojModel):super().set_gguf_parameters()assert self.hparams_vision is not Nonehparams = self.hparams_vision- model_type = self.global_config['model_type']- if model_type == 'qwen2_vl':+ model_type = self.global_config["model_type"]+ if model_type == "qwen2_vl":self.gguf_writer.add_clip_projector_type(gguf.VisionProjectorType.QWEN2VL)- elif model_type == 'qwen2_5_vl' or model_type == 'qwen2_5_omni':- if model_type == 'qwen2_5_omni':+ elif model_type == "qwen2_5_vl" or model_type == "qwen2_5_omni":+ if model_type == "qwen2_5_omni":self.gguf_writer.add_clip_projector_type(gguf.VisionProjectorType.QWEN25O)else:self.gguf_writer.add_clip_projector_type(gguf.VisionProjectorType.QWEN25VL)@@ -3624,14 +3744,14 @@ class Qwen2VLVisionModel(MmprojModel):assert c3 % 3 == 0c = c3 // 3wq = data_torch[:c]- wk = data_torch[c:c * 2]- wv = data_torch[c * 2:]+ wk = data_torch[c : c * 2]+ wv = data_torch[c * 2 :]return [(self.map_tensor_name(name.replace("qkv", "q")), wq),(self.map_tensor_name(name.replace("qkv", "k")), wk),(self.map_tensor_name(name.replace("qkv", "v")), wv),]- elif 'patch_embed.proj.weight' in name:+ elif "patch_embed.proj.weight" in name:# split Conv3D into Conv2Dsc1, c2, kt, kh, kw = data_torch.shapedel c1, c2, kh, kw # unused@@ -3675,7 +3795,7 @@ class Qwen25OmniModel(Qwen2VLVisionModel):max_timescale = 10000length = 1500channels = self.hparams_audio["hidden_size"]- log_timescale_increment = np.log(max_timescale) / (channels//2 - 1)+ log_timescale_increment = np.log(max_timescale) / (channels // 2 - 1)inv_timescales = torch.exp(-log_timescale_increment * torch.arange(channels // 2).float())scaled_time = torch.arange(length)[:, np.newaxis] * inv_timescales[np.newaxis, :]pos_embd = torch.cat([torch.sin(scaled_time), torch.cos(scaled_time)], dim=1).to(dtype=torch.float32)@@ -3749,8 +3869,8 @@ class InternVisionModel(MmprojModel):assert c3 % 3 == 0c = c3 // 3wq = data_torch[:c]- wk = data_torch[c:c * 2]- wv = data_torch[c * 2:]+ wk = data_torch[c : c * 2]+ wv = data_torch[c * 2 :]return [(self.map_tensor_name(name.replace("attn.qkv", "self_attn.q_proj")), wq),(self.map_tensor_name(name.replace("attn.qkv", "self_attn.k_proj")), wk),@@ -3767,10 +3887,11 @@ class WavTokenizerDecModel(TextModel):def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:del bid # unused- if \- name.endswith("codebook.cluster_size") or \- name.endswith("codebook.embed_avg") or \- name.endswith("codebook.inited"):+ if (+ name.endswith("codebook.cluster_size")+ or name.endswith("codebook.embed_avg")+ or name.endswith("codebook.inited")+ ):logger.debug(f"Skipping {name!r}")return []@@ -3809,7 +3930,7 @@ class Qwen2MoeModel(TextModel):if (moe_intermediate_size := self.hparams.get("moe_intermediate_size")) is not None:self.gguf_writer.add_expert_feed_forward_length(moe_intermediate_size)logger.info(f"gguf: expert feed forward length = {moe_intermediate_size}")- if (shared_expert_intermediate_size := self.hparams.get('shared_expert_intermediate_size')) is not None:+ if (shared_expert_intermediate_size := self.hparams.get("shared_expert_intermediate_size")) is not None:self.gguf_writer.add_expert_shared_feed_forward_length(shared_expert_intermediate_size)logger.info(f"gguf: expert shared feed forward length = {shared_expert_intermediate_size}")# YaRN is not enabled by default@@ -3940,25 +4061,25 @@ class Phi3MiniModel(TextModel):def set_vocab(self):# Phi-4 model uses GPT2Tokenizer- tokenizer_config_file = self.dir_model / 'tokenizer_config.json'+ tokenizer_config_file = self.dir_model / "tokenizer_config.json"if tokenizer_config_file.is_file():with open(tokenizer_config_file, "r", encoding="utf-8") as f:tokenizer_config_json = json.load(f)- tokenizer_class = tokenizer_config_json['tokenizer_class']- if tokenizer_class == 'GPT2Tokenizer':+ tokenizer_class = tokenizer_config_json["tokenizer_class"]+ if tokenizer_class == "GPT2Tokenizer":return self._set_vocab_gpt2()from sentencepiece import SentencePieceProcessor- tokenizer_path = self.dir_model / 'tokenizer.model'+ tokenizer_path = self.dir_model / "tokenizer.model"if not tokenizer_path.is_file():- raise ValueError(f'Error: Missing {tokenizer_path}')+ raise ValueError(f"Error: Missing {tokenizer_path}")tokenizer = SentencePieceProcessor()tokenizer.LoadFromFile(str(tokenizer_path))- vocab_size = self.hparams.get('vocab_size', tokenizer.vocab_size())+ vocab_size = self.hparams.get("vocab_size", tokenizer.vocab_size())tokens: list[bytes] = [f"[PAD{i}]".encode("utf-8") for i in range(vocab_size)]scores: list[float] = [-10000.0] * vocab_size@@ -3984,7 +4105,7 @@ class Phi3MiniModel(TextModel):scores[token_id] = scoretoktypes[token_id] = toktype- added_tokens_file = self.dir_model / 'added_tokens.json'+ added_tokens_file = self.dir_model / "added_tokens.json"if added_tokens_file.is_file():with open(added_tokens_file, "r", encoding="utf-8") as f:added_tokens_json = json.load(f)@@ -3992,14 +4113,14 @@ class Phi3MiniModel(TextModel):for key in added_tokens_json:token_id = added_tokens_json[key]if token_id >= vocab_size:- logger.debug(f'ignore token {token_id}: id is out of range, max={vocab_size - 1}')+ logger.debug(f"ignore token {token_id}: id is out of range, max={vocab_size - 1}")continuetokens[token_id] = key.encode("utf-8")scores[token_id] = -1000.0toktypes[token_id] = SentencePieceTokenTypes.USER_DEFINED- tokenizer_config_file = self.dir_model / 'tokenizer_config.json'+ tokenizer_config_file = self.dir_model / "tokenizer_config.json"if tokenizer_config_file.is_file():with open(tokenizer_config_file, "r", encoding="utf-8") as f:tokenizer_config_json = json.load(f)@@ -4019,7 +4140,7 @@ class Phi3MiniModel(TextModel):if foken_data.get("special"):toktypes[token_id] = SentencePieceTokenTypes.CONTROL- tokenizer_file = self.dir_model / 'tokenizer.json'+ tokenizer_file = self.dir_model / "tokenizer.json"if tokenizer_file.is_file():with open(tokenizer_file, "r", encoding="utf-8") as f:tokenizer_json = json.load(f)@@ -4086,43 +4207,45 @@ class Phi3MiniModel(TextModel):rope_dims = int(rot_pct * n_embd) // n_head# write rope scaling for long context (128k) model- rope_scaling = self.find_hparam(['rope_scaling'], True)+ rope_scaling = self.find_hparam(["rope_scaling"], True)if rope_scaling is None:returnscale = max_pos_embds / orig_max_pos_embds- rope_scaling_type = rope_scaling.get('rope_type', rope_scaling.get('type', '')).lower()+ rope_scaling_type = rope_scaling.get("rope_type", rope_scaling.get("type", "")).lower()if len(rope_scaling_type) == 0:- raise KeyError('Missing the required key rope_scaling.type')+ raise KeyError("Missing the required key rope_scaling.type")- if rope_scaling_type == 'su' or rope_scaling_type == 'longrope':+ if rope_scaling_type == "su" or rope_scaling_type == "longrope":attn_factor = math.sqrt(1 + math.log(scale) / math.log(orig_max_pos_embds)) if scale > 1.0 else 1.0- elif rope_scaling_type == 'yarn':+ elif rope_scaling_type == "yarn":attn_factor = 0.1 * math.log(scale) + 1.0 if scale > 1.0 else 1.0else:- raise NotImplementedError(f'The rope scaling type {rope_scaling_type} is not supported yet')+ raise NotImplementedError(f"The rope scaling type {rope_scaling_type} is not supported yet")self.gguf_writer.add_rope_scaling_attn_factors(attn_factor)- long_factors = rope_scaling.get('long_factor', None)- short_factors = rope_scaling.get('short_factor', None)+ long_factors = rope_scaling.get("long_factor", None)+ short_factors = rope_scaling.get("short_factor", None)if long_factors is None or short_factors is None:- raise KeyError('Missing the required key rope_scaling.long_factor or rope_scaling_short_factor')+ raise KeyError("Missing the required key rope_scaling.long_factor or rope_scaling_short_factor")if len(long_factors) != len(short_factors) or len(long_factors) != rope_dims / 2:raise ValueError(- f'The length of rope long and short factors must be {rope_dims / 2}.'- f' long_factors = {len(long_factors)}, short_factors = {len(short_factors)}.'+ f"The length of rope long and short factors must be {rope_dims / 2}."+ f" long_factors = {len(long_factors)}, short_factors = {len(short_factors)}.")yield (self.format_tensor_name(gguf.MODEL_TENSOR.ROPE_FACTORS_LONG),- torch.tensor(long_factors, dtype=torch.float32))+ torch.tensor(long_factors, dtype=torch.float32),+ )yield (self.format_tensor_name(gguf.MODEL_TENSOR.ROPE_FACTORS_SHORT),- torch.tensor(short_factors, dtype=torch.float32))+ torch.tensor(short_factors, dtype=torch.float32),+ )@ModelBase.register("PhiMoEForCausalLM")@@ -4244,7 +4367,7 @@ class Plamo2Model(TextModel):raise FileNotFoundError(f"PLaMo 2 tokenizer file not found: {tokenizer_jsonl_path}")# Load tokenizer config- with open(tokenizer_config_path, 'r', encoding='utf-8') as f:+ with open(tokenizer_config_path, "r", encoding="utf-8") as f:tokenizer_config = json.load(f)# Load tokens from JSONL file (actually a list format)@@ -4252,7 +4375,7 @@ class Plamo2Model(TextModel):scores = []toktypes = []- with open(tokenizer_jsonl_path, 'r', encoding='utf-8') as f:+ with open(tokenizer_jsonl_path, "r", encoding="utf-8") as f:for line_num, line in enumerate(f):if line.strip():token_data = json.loads(line)@@ -4333,7 +4456,7 @@ class Plamo2Model(TextModel):for i in range(block_count):if block_count <= (mamba_step // 2):# use attention in last layer- is_mamba = (i != block_count - 1)+ is_mamba = i != block_count - 1else:is_mamba = (i % mamba_step) != (mamba_step // 2)if is_mamba:@@ -4452,14 +4575,14 @@ class InternLM2Model(TextModel):from sentencepiece import SentencePieceProcessorfrom sentencepiece import sentencepiece_model_pb2 as model- tokenizer_path = self.dir_model / 'tokenizer.model'+ tokenizer_path = self.dir_model / "tokenizer.model"tokens: list[bytes] = []scores: list[float] = []toktypes: list[int] = []if not tokenizer_path.is_file():- logger.error(f'Error: Missing {tokenizer_path}')+ logger.error(f"Error: Missing {tokenizer_path}")sys.exit(1)sentencepiece_model = model.ModelProto() # pylint: disable=E1101@@ -4469,7 +4592,7 @@ class InternLM2Model(TextModel):tokenizer = SentencePieceProcessor()tokenizer.LoadFromFile(str(tokenizer_path))- vocab_size = self.hparams.get('vocab_size', tokenizer.vocab_size())+ vocab_size = self.hparams.get("vocab_size", tokenizer.vocab_size())for token_id in range(vocab_size):piece = tokenizer.IdToPiece(token_id)@@ -4491,14 +4614,14 @@ class InternLM2Model(TextModel):elif tokenizer.IsByte(token_id):toktype = SentencePieceTokenTypes.BYTE# take care of unused raw token- if piece.startswith('[UNUSED'):+ if piece.startswith("[UNUSED"):toktype = SentencePieceTokenTypes.UNUSEDtokens.append(text)scores.append(score)toktypes.append(toktype)- added_tokens_file = self.dir_model / 'added_tokens.json'+ added_tokens_file = self.dir_model / "added_tokens.json"if added_tokens_file.is_file():with open(added_tokens_file, "r", encoding="utf-8") as f:added_tokens_json = json.load(f)@@ -4508,10 +4631,10 @@ class InternLM2Model(TextModel):scores.append(-1000.0)toktypes.append(SentencePieceTokenTypes.USER_DEFINED)- chat_eos_token = '<|im_end|>'+ chat_eos_token = "<|im_end|>"chat_eos_token_id = None- tokenizer_config_file = self.dir_model / 'tokenizer_config.json'+ tokenizer_config_file = self.dir_model / "tokenizer_config.json"if tokenizer_config_file.is_file():with open(tokenizer_config_file, "r", encoding="utf-8") as f:tokenizer_config_json = json.load(f)@@ -4534,7 +4657,7 @@ class InternLM2Model(TextModel):if foken_data.get("special"):toktypes[token_id] = SentencePieceTokenTypes.CONTROL- tokenizer_file = self.dir_model / 'tokenizer.json'+ tokenizer_file = self.dir_model / "tokenizer.json"if tokenizer_file.is_file():with open(tokenizer_file, "r", encoding="utf-8") as f:tokenizer_json = json.load(f)@@ -4573,7 +4696,8 @@ class InternLM2Model(TextModel):special_vocab.special_token_ids["eos"] = chat_eos_token_idlogger.warning(f"Replace eos:{old_eos} with a special token:{chat_eos_token_id}"- " in chat mode so that the conversation can end normally.")+ " in chat mode so that the conversation can end normally."+ )special_vocab.add_to_gguf(self.gguf_writer)@@ -4640,7 +4764,7 @@ class InternLM3Model(TextModel):special_vocab = gguf.SpecialVocab(self.dir_model, n_vocab=len(tokens))- tokenizer_config_file = self.dir_model / 'tokenizer_config.json'+ tokenizer_config_file = self.dir_model / "tokenizer_config.json"if tokenizer_config_file.is_file():with open(tokenizer_config_file, "r", encoding="utf-8") as f:tokenizer_config_json = json.load(f)@@ -4654,7 +4778,7 @@ class InternLM3Model(TextModel):token = token_data["content"]special_vocab._set_special_token(token, token_id)# update eos token- if token == '<|im_end|>' and "eos" in special_vocab.special_token_ids:+ if token == "<|im_end|>" and "eos" in special_vocab.special_token_ids:special_vocab.special_token_ids["eos"] = token_idspecial_vocab.add_to_gguf(self.gguf_writer)@@ -4785,13 +4909,13 @@ class BertModel(TextModel):from sentencepiece import SentencePieceProcessorfrom sentencepiece import sentencepiece_model_pb2 as model- tokenizer_path = self.dir_model / 'sentencepiece.bpe.model'+ tokenizer_path = self.dir_model / "sentencepiece.bpe.model"tokenizer_json = {}tokenizer_config_json = {}if not tokenizer_path.is_file():- tokenizer_path = self.dir_model / 'tokenizer.json'- tokenizer_config_path = self.dir_model / 'tokenizer_config.json'+ tokenizer_path = self.dir_model / "tokenizer.json"+ tokenizer_config_path = self.dir_model / "tokenizer_config.json"if not tokenizer_path.is_file():raise FileNotFoundError(f"File not found: {tokenizer_path}")@@ -4799,6 +4923,7 @@ class BertModel(TextModel):from base64 import b64decodefrom transformers import AutoTokenizer+tokenizer = AutoTokenizer.from_pretrained(self.dir_model)with open(tokenizer_path, "r", encoding="utf-8") as fp:@@ -4878,7 +5003,7 @@ class BertModel(TextModel):if isinstance(tokenizer, SentencePieceProcessor):# realign tokens (see HF tokenizer code)- tokens = [b'<s>', b'<pad>', b'</s>', b'<unk>'] + tokens[3:-1]+ tokens = [b"<s>", b"<pad>", b"</s>", b"<unk>"] + tokens[3:-1]scores = [0.0, 0.0, 0.0, 0.0] + scores[3:-1]toktypes = [SentencePieceTokenTypes.CONTROL,@@ -4889,7 +5014,7 @@ class BertModel(TextModel):if self.model_arch == gguf.MODEL_ARCH.NOMIC_BERT_MOE:# Add mask token missing from sentencepiece.bpe.model- tokens[250001] = b'<mask>'+ tokens[250001] = b"<mask>"scores[250001] = 0.0toktypes[250001] = SentencePieceTokenTypes.CONTROL@@ -4964,7 +5089,7 @@ class RobertaModel(BertModel):# position embeddings start at pad_token_id + 1, so just chop down the weight tensorif name == "embeddings.position_embeddings.weight":if self._position_offset is not None:- data_torch = data_torch[self._position_offset:, :]+ data_torch = data_torch[self._position_offset :, :]return super().modify_tensors(data_torch, name, bid)@@ -5016,8 +5141,9 @@ class NomicBertModel(BertModel):return self._xlmroberta_set_vocab()return super().set_vocab()- def modify_tensors(self, data_torch: torch.Tensor, name: str,- bid: int | None) -> Iterable[tuple[str, torch.Tensor]]:+ def modify_tensors(+ self, data_torch: torch.Tensor, name: str, bid: int | None+ ) -> Iterable[tuple[str, torch.Tensor]]:# If the tensor is an experts bias tensor, skip it by returning an empty list.if "mlp.experts.bias" in name:return [] # Explicitly return an empty list.@@ -5098,7 +5224,7 @@ class XLMRobertaModel(BertModel):# position embeddings start at pad_token_id + 1, so just chop down the weight tensorif name == "embeddings.position_embeddings.weight":if self._position_offset is not None:- data_torch = data_torch[self._position_offset:, :]+ data_torch = data_torch[self._position_offset :, :]return super().modify_tensors(data_torch, name, bid)@@ -5112,7 +5238,8 @@ class GemmaModel(TextModel):# TODO: these special tokens should be exported only for the CodeGemma familyspecial_vocab = gguf.SpecialVocab(- self.dir_model, load_merges=False, special_token_types=['prefix', 'suffix', 'middle', 'fsep', 'eot'])+ self.dir_model, load_merges=False, special_token_types=["prefix", "suffix", "middle", "fsep", "eot"]+ )special_vocab._set_special_token("prefix", 67)special_vocab._set_special_token("suffix", 69)special_vocab._set_special_token("middle", 68)@@ -5133,7 +5260,8 @@ class GemmaModel(TextModel):self.gguf_writer.add_feed_forward_length(hparams["intermediate_size"])self.gguf_writer.add_head_count(hparams["num_attention_heads"])self.gguf_writer.add_head_count_kv(- self.hparams["num_key_value_heads"] if "num_key_value_heads" in hparams else hparams["num_attention_heads"])+ self.hparams["num_key_value_heads"] if "num_key_value_heads" in hparams else hparams["num_attention_heads"]+ )self.gguf_writer.add_layer_norm_rms_eps(self.hparams["rms_norm_eps"])self.gguf_writer.add_key_length(hparams["head_dim"])self.gguf_writer.add_value_length(hparams["head_dim"])@@ -5173,7 +5301,8 @@ class Gemma2Model(TextModel):self.gguf_writer.add_feed_forward_length(hparams["intermediate_size"])self.gguf_writer.add_head_count(hparams["num_attention_heads"])self.gguf_writer.add_head_count_kv(- self.hparams["num_key_value_heads"] if "num_key_value_heads" in hparams else hparams["num_attention_heads"])+ self.hparams["num_key_value_heads"] if "num_key_value_heads" in hparams else hparams["num_attention_heads"]+ )self.gguf_writer.add_layer_norm_rms_eps(self.hparams["rms_norm_eps"])self.gguf_writer.add_key_length(hparams["head_dim"])self.gguf_writer.add_value_length(hparams["head_dim"])@@ -5238,15 +5367,19 @@ class Gemma3Model(TextModel):if "language_model." in name:name = name.replace("language_model.", "")- elif name.startswith("multi_modal_projector.") or name.startswith("vision_tower.") \- or name.startswith("multimodal_projector.") or name.startswith("vision_model."):+ elif (+ name.startswith("multi_modal_projector.")+ or name.startswith("vision_tower.")+ or name.startswith("multimodal_projector.")+ or name.startswith("vision_model.")+ ):return [] # skip vision tensors# remove OOV (out-of-vocabulary) rows in token_embdif "embed_tokens.weight" in name:vocab = self._create_vocab_sentencepiece()tokens = vocab[0]- data_torch = data_torch[:len(tokens)]+ data_torch = data_torch[: len(tokens)]# ref code in Gemma3RMSNorm# output = output * (1.0 + self.weight.float())@@ -5272,7 +5405,7 @@ class Gemma3VisionModel(MmprojModel):n_per_side = int(image_seq_length**0.5)image_size = self.hparams["image_size"]patch_size = self.hparams["patch_size"]- proj_scale_factor = (image_size//patch_size) // n_per_side+ proj_scale_factor = (image_size // patch_size) // n_per_sideif proj_scale_factor > 0 and proj_scale_factor != 4:# we only need to write this if it's not the default value# in this case, we are converting a test model@@ -5293,8 +5426,12 @@ class Gemma3VisionModel(MmprojModel):if "vision_model.head." in name:return [] # skip redundant tensors for tinygemma3- if name.startswith("multi_modal_projector.") or name.startswith("vision_tower.") \- or name.startswith("multimodal_projector.") or name.startswith("vision_model."):+ if (+ name.startswith("multi_modal_projector.")+ or name.startswith("vision_tower.")+ or name.startswith("multimodal_projector.")+ or name.startswith("vision_model.")+ ):# process vision tensorsname = name.replace("_weight", ".weight")@@ -5423,8 +5560,11 @@ class Rwkv6Model(TextModel):hidden_size = self.hparams["hidden_size"]layer_norm_eps = self.hparams["layer_norm_epsilon"]rescale_every_n_layers = self.hparams["rescale_every"]- intermediate_size = self.hparams["intermediate_size"] if self.hparams["intermediate_size"] is not None else int(- (hidden_size*3.5) // 32 * 32)+ intermediate_size = (+ self.hparams["intermediate_size"]+ if self.hparams["intermediate_size"] is not None+ else int((hidden_size * 3.5) // 32 * 32)+ )time_mix_extra_dim = 64 if hidden_size == 4096 else 32time_decay_extra_dim = 128 if hidden_size == 4096 else 64@@ -5451,8 +5591,11 @@ class Rwkv6Model(TextModel):if not (new_name.endswith(".weight") or new_name.endswith(".bias")):new_name += ".weight"- if new_name.endswith("time_mix_w1.weight") or new_name.endswith(- "time_mix_decay_w1.weight") or new_name.endswith("time_mix_decay_w2.weight"):+ if (+ new_name.endswith("time_mix_w1.weight")+ or new_name.endswith("time_mix_decay_w1.weight")+ or new_name.endswith("time_mix_decay_w2.weight")+ ):data_torch = data_torch.transpose(0, 1)if new_name.endswith("time_mix_w2.weight"):@@ -5465,7 +5608,7 @@ class Rwkv6Model(TextModel):rescale_every_n_layers = self.hparams["rescale_every"]if rescale_every_n_layers > 0:if new_name.endswith("time_mix_output.weight") or new_name.endswith("channel_mix_value.weight"):- data_torch = data_torch.div_(2**int(bid // rescale_every_n_layers))+ data_torch = data_torch.div_(2 ** int(bid // rescale_every_n_layers))except KeyError:pass@@ -5476,14 +5619,18 @@ class Rwkv6Model(TextModel):self.lerp_weights[bid][new_name] = data_torchexcept KeyError:self.lerp_weights[bid] = {new_name: data_torch}- if all(f"blk.{bid}.time_mix_lerp_{i}.weight" in self.lerp_weights[bid].keys()- for i in ["w", "k", "v", "r", "g"]):+ if all(+ f"blk.{bid}.time_mix_lerp_{i}.weight" in self.lerp_weights[bid].keys()+ for i in ["w", "k", "v", "r", "g"]+ ):new_name = f"blk.{bid}.time_mix_lerp_fused.weight"- data = torch.stack([- self.lerp_weights[bid][f"blk.{bid}.time_mix_lerp_{i}.weight"].unsqueeze(0)- for i in ["w", "k", "v", "r", "g"]- ],- dim=0).unsqueeze(1)+ data = torch.stack(+ [+ self.lerp_weights[bid][f"blk.{bid}.time_mix_lerp_{i}.weight"].unsqueeze(0)+ for i in ["w", "k", "v", "r", "g"]+ ],+ dim=0,+ ).unsqueeze(1)yield (new_name, data)return@@ -5563,28 +5710,53 @@ class Rwkv7Model(TextModel):head_size = self.hparams["head_dim"]layer_norm_eps = self.hparams["norm_eps"]hidden_size = self.hparams["hidden_size"]- intermediate_size = self.hparams["intermediate_size"] if self.hparams["intermediate_size"] is not None else (- hidden_size * 4)+ intermediate_size = (+ self.hparams["intermediate_size"] if self.hparams["intermediate_size"] is not None else (hidden_size * 4)+ )# ICLR: In-Context-Learning-Ratetry:- lora_rank_decay = self.hparams["lora_rank_decay"] if self.hparams[- "lora_rank_decay"] is not None else self.calc_lora_rank(hidden_size, 0.5, 1.8)- lora_rank_iclr = self.hparams["lora_rank_iclr"] if self.hparams[- "lora_rank_iclr"] is not None else self.calc_lora_rank(hidden_size, 0.5, 1.8)- lora_rank_value_residual_mix = self.hparams["lora_rank_value_residual_mix"] if self.hparams[- "lora_rank_value_residual_mix"] is not None else self.calc_lora_rank(hidden_size, 0.5, 1.3)- lora_rank_gate = self.hparams["lora_rank_gate"] if self.hparams[- "lora_rank_gate"] is not None else self.calc_lora_rank(hidden_size, 0.8, 0.6)+ lora_rank_decay = (+ self.hparams["lora_rank_decay"]+ if self.hparams["lora_rank_decay"] is not None+ else self.calc_lora_rank(hidden_size, 0.5, 1.8)+ )+ lora_rank_iclr = (+ self.hparams["lora_rank_iclr"]+ if self.hparams["lora_rank_iclr"] is not None+ else self.calc_lora_rank(hidden_size, 0.5, 1.8)+ )+ lora_rank_value_residual_mix = (+ self.hparams["lora_rank_value_residual_mix"]+ if self.hparams["lora_rank_value_residual_mix"] is not None+ else self.calc_lora_rank(hidden_size, 0.5, 1.3)+ )+ lora_rank_gate = (+ self.hparams["lora_rank_gate"]+ if self.hparams["lora_rank_gate"] is not None+ else self.calc_lora_rank(hidden_size, 0.8, 0.6)+ )except KeyError:- lora_rank_decay = self.hparams["decay_low_rank_dim"] if self.hparams[- "decay_low_rank_dim"] is not None else self.calc_lora_rank(hidden_size, 0.5, 1.8)- lora_rank_iclr = self.hparams["a_low_rank_dim"] if self.hparams[- "a_low_rank_dim"] is not None else self.calc_lora_rank(hidden_size, 0.5, 1.8)- lora_rank_value_residual_mix = self.hparams["v_low_rank_dim"] if self.hparams[- "v_low_rank_dim"] is not None else self.calc_lora_rank(hidden_size, 0.5, 1.3)- lora_rank_gate = self.hparams["gate_low_rank_dim"] if self.hparams[- "gate_low_rank_dim"] is not None else self.calc_lora_rank(hidden_size, 0.8, 0.6)+ lora_rank_decay = (+ self.hparams["decay_low_rank_dim"]+ if self.hparams["decay_low_rank_dim"] is not None+ else self.calc_lora_rank(hidden_size, 0.5, 1.8)+ )+ lora_rank_iclr = (+ self.hparams["a_low_rank_dim"]+ if self.hparams["a_low_rank_dim"] is not None+ else self.calc_lora_rank(hidden_size, 0.5, 1.8)+ )+ lora_rank_value_residual_mix = (+ self.hparams["v_low_rank_dim"]+ if self.hparams["v_low_rank_dim"] is not None+ else self.calc_lora_rank(hidden_size, 0.5, 1.3)+ )+ lora_rank_gate = (+ self.hparams["gate_low_rank_dim"]+ if self.hparams["gate_low_rank_dim"] is not None+ else self.calc_lora_rank(hidden_size, 0.8, 0.6)+ )# RWKV isn't context limitedself.gguf_writer.add_context_length(1048576)@@ -5642,7 +5814,8 @@ class Rwkv7Model(TextModel):if all(f"model.layers.{bid}.attention.x_{i}" in self.lerp_weights[bid].keys() for i in lerp_list):new_name = f"blk.{bid}.time_mix_lerp_fused.weight"data = torch.stack(- [self.lerp_weights[bid][f"model.layers.{bid}.attention.x_{i}"] for i in lerp_list], dim=0)+ [self.lerp_weights[bid][f"model.layers.{bid}.attention.x_{i}"] for i in lerp_list], dim=0+ )yield (new_name, data)returnelse:@@ -5652,7 +5825,9 @@ class Rwkv7Model(TextModel):if not (new_name.endswith(".weight") or new_name.endswith(".bias")):new_name += ".weight"- if self.lora_needs_transpose and any(new_name.endswith(t) for t in [+ if self.lora_needs_transpose and any(+ new_name.endswith(t)+ for t in ["time_mix_w1.weight","time_mix_w2.weight","time_mix_a1.weight",@@ -5661,10 +5836,11 @@ class Rwkv7Model(TextModel):"time_mix_v2.weight","time_mix_g1.weight","time_mix_g2.weight",- ]):+ ]+ ):data_torch = data_torch.transpose(0, 1)- if 'r_k' in new_name:+ if "r_k" in new_name:data_torch = data_torch.flatten()if bid == 0 and "time_mix_a" in new_name:@@ -5772,7 +5948,8 @@ class MambaModel(TextModel):self.gguf_writer.add_ssm_time_step_rank(dt_rank)self.gguf_writer.add_layer_norm_rms_eps(rms_norm_eps)self.gguf_writer.add_ssm_dt_b_c_rms(- use_dt_b_c_norm) # For classic Mamba we don't apply rms norm on B / DT layers+ use_dt_b_c_norm+ ) # For classic Mamba we don't apply rms norm on B / DT layersself.gguf_writer.add_file_type(self.ftype)_tok_embd = None@@ -5815,8 +5992,9 @@ class Mamba2Model(TextModel):hparams = json.load(f)super().__init__(dir_model, *args, hparams=hparams, **kwargs)self.d_model = self.find_hparam(["hidden_size", "d_model", "dim"])- self.d_inner = self.find_hparam(["mamba_d_ssm", "intermediate_size", "d_inner"],- optional=True) or 2 * self.d_model+ self.d_inner = (+ self.find_hparam(["mamba_d_ssm", "intermediate_size", "d_inner"], optional=True) or 2 * self.d_model+ )self.n_group = self.find_hparam(["n_groups"], optional=True) or 1def set_vocab(self):@@ -5833,7 +6011,8 @@ class Mamba2Model(TextModel):elif (self.dir_model / "tokenizer.model.v3").is_file():# mamba-codestralraise NotImplementedError(- f"Please rename {self.dir_model / 'tokenizer.model.v3'} to {self.dir_model / 'tokenizer.model'}")+ f"Please rename {self.dir_model / 'tokenizer.model.v3'} to {self.dir_model / 'tokenizer.model'}"+ )elif (self.dir_model / "tokenizer.json").is_file():self._set_vocab_gpt2()else:@@ -5880,10 +6059,13 @@ class Mamba2Model(TextModel):if self.match_model_tensor_name(new_name, gguf.MODEL_TENSOR.SSM_CONV1D, bid):data_torch = data_torch.squeeze()- elif any(self.match_model_tensor_name(new_name, t, bid, suffix="") for t in [+ elif any(+ self.match_model_tensor_name(new_name, t, bid, suffix="")+ for t in [gguf.MODEL_TENSOR.SSM_A,gguf.MODEL_TENSOR.SSM_D,- ]):+ ]+ ):# unsqueeze A to use similar shape semantics as Mamba-1# (D is also unsqueezed, but for more straightforward broadcast internally)data_torch = data_torch.reshape((*data_torch.shape, 1))@@ -5927,7 +6109,7 @@ class JambaModel(TextModel):attn_offset = self.hparams["attn_layer_offset"]attn_period = self.hparams["attn_layer_period"]n_kv_vec = [0 for _ in range(attn_offset)] + [- n_kv_head if (i-attn_offset) % attn_period == 0 else 0 for i in range(attn_offset, self.block_count)+ n_kv_head if (i - attn_offset) % attn_period == 0 else 0 for i in range(attn_offset, self.block_count)]self.gguf_writer.add_block_count(self.block_count)@@ -5955,7 +6137,7 @@ class JambaModel(TextModel):moe_offset = self.hparams["expert_layer_offset"]moe_period = self.hparams["expert_layer_period"]- if not (bid >= moe_offset and (bid-moe_offset) % moe_period == 0):+ if not (bid >= moe_offset and (bid - moe_offset) % moe_period == 0):name = name.replace(".experts.0.", ".")# process the experts separately@@ -6043,7 +6225,7 @@ class Cohere2Model(TextModel):rotary_pct = self.hparams["rotary_pct"]hidden_size = self.hparams["hidden_size"]num_attention_heads = self.hparams["num_attention_heads"]- self.gguf_writer.add_rope_dimension_count(int(rotary_pct * (hidden_size//num_attention_heads)))+ self.gguf_writer.add_rope_dimension_count(int(rotary_pct * (hidden_size // num_attention_heads)))self.gguf_writer.add_rope_scaling_type(gguf.RopeScalingType.NONE)@@ -6145,17 +6327,17 @@ class JinaBertV2Model(BertModel):model_arch = gguf.MODEL_ARCH.JINA_BERT_V2def set_vocab(self):- tokenizer_class = 'BertTokenizer'+ tokenizer_class = "BertTokenizer"with open(self.dir_model / "tokenizer_config.json", "r", encoding="utf-8") as f:- tokenizer_class = json.load(f)['tokenizer_class']+ tokenizer_class = json.load(f)["tokenizer_class"]- if tokenizer_class == 'BertTokenizer':+ if tokenizer_class == "BertTokenizer":super().set_vocab()- elif tokenizer_class == 'RobertaTokenizer':+ elif tokenizer_class == "RobertaTokenizer":self._set_vocab_gpt2()self.gguf_writer.add_token_type_count(2)else:- raise NotImplementedError(f'Tokenizer {tokenizer_class} is not supported for JinaBertModel')+ raise NotImplementedError(f"Tokenizer {tokenizer_class} is not supported for JinaBertModel")@ModelBase.register("OpenELMForCausalLM")@@ -6164,7 +6346,7 @@ class OpenELMModel(TextModel):@staticmethoddef _make_divisible(v: float | int, divisor: int) -> int:- new_v = max(divisor, int(v + divisor/2) // divisor * divisor)+ new_v = max(divisor, int(v + divisor / 2) // divisor * divisor)# Make sure that round down does not go down by more than 10%.if new_v < 0.9 * v:new_v += divisor@@ -6241,17 +6423,17 @@ class ArcticModel(TextModel):# tokenizer.model and used them as BOS and EOS instead of adding new tokens.from sentencepiece import SentencePieceProcessor- tokenizer_path = self.dir_model / 'tokenizer.model'+ tokenizer_path = self.dir_model / "tokenizer.model"if not tokenizer_path.is_file():- logger.error(f'Error: Missing {tokenizer_path}')+ logger.error(f"Error: Missing {tokenizer_path}")sys.exit(1)# Read the whole vocabulary from the tokenizer.model filetokenizer = SentencePieceProcessor()tokenizer.LoadFromFile(str(tokenizer_path))- vocab_size = self.hparams.get('vocab_size', tokenizer.vocab_size())+ vocab_size = self.hparams.get("vocab_size", tokenizer.vocab_size())tokens: list[bytes] = [f"[PAD{i}]".encode("utf-8") for i in range(vocab_size)]scores: list[float] = [-10000.0] * vocab_size@@ -6279,7 +6461,7 @@ class ArcticModel(TextModel):# Use the added_tokens_decoder field from tokeniser_config.json as the source# of information about added/redefined tokens and modify them accordingly.- tokenizer_config_file = self.dir_model / 'tokenizer_config.json'+ tokenizer_config_file = self.dir_model / "tokenizer_config.json"if tokenizer_config_file.is_file():with open(tokenizer_config_file, "r", encoding="utf-8") as f:tokenizer_config_json = json.load(f)@@ -6289,7 +6471,7 @@ class ArcticModel(TextModel):for token_id, token_json in added_tokens_decoder.items():token_id = int(token_id)if token_id >= vocab_size:- logger.debug(f'ignore token {token_id}: id is out of range, max={vocab_size - 1}')+ logger.debug(f"ignore token {token_id}: id is out of range, max={vocab_size - 1}")continuetoken_content = token_json["content"]@@ -6417,8 +6599,10 @@ class DeepseekModel(TextModel):if n_head_kv is not None and n_head != n_head_kv:n_head = n_head_kvreturn (- weights.reshape(n_head, 2, weights.shape[0] // n_head // 2,- *weights.shape[1:]).swapaxes(1, 2).reshape(weights.shape))+ weights.reshape(n_head, 2, weights.shape[0] // n_head // 2, *weights.shape[1:])+ .swapaxes(1, 2)+ .reshape(weights.shape)+ )def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:n_head = self.hparams["num_attention_heads"]@@ -6487,6 +6671,7 @@ class DeepseekV2Model(TextModel):passfrom transformers import AutoTokenizer+tokenizer = AutoTokenizer.from_pretrained(self.dir_model, trust_remote_code=True)tokpre = self.get_vocab_base_pre(tokenizer)@@ -6501,7 +6686,7 @@ class DeepseekV2Model(TextModel):continuemerged = QwenModel.bpe(mergeable_ranks, token, max_rank=rank)if len(merged) == 2:- merges.append(' '.join(map(QwenModel.token_bytes_to_string, merged)))+ merges.append(" ".join(map(QwenModel.token_bytes_to_string, merged)))# Build token listvocab_size = self.hparams["vocab_size"]@@ -6630,7 +6815,7 @@ class DeepseekV2Model(TextModel):v_head_dim = self.hparams["v_head_dim"]qk_nope_head_dim = self.hparams["qk_nope_head_dim"]- assert data_torch.shape[0] == n_head_kv * (v_head_dim+qk_nope_head_dim)+ assert data_torch.shape[0] == n_head_kv * (v_head_dim + qk_nope_head_dim)kv_b = data_torch.view(n_head_kv, v_head_dim + qk_nope_head_dim, data_torch.shape[-1])k_b, v_b = torch.split(kv_b, [qk_nope_head_dim, v_head_dim], dim=1)@@ -6719,11 +6904,11 @@ class T5Model(TextModel):from sentencepiece import SentencePieceProcessorfrom sentencepiece import sentencepiece_model_pb2 as model- tokenizer_path = self.dir_model / 'tokenizer.model'+ tokenizer_path = self.dir_model / "tokenizer.model"# many older models use spiece.model tokenizer model filenameif not tokenizer_path.is_file():- tokenizer_path = self.dir_model / 'spiece.model'+ tokenizer_path = self.dir_model / "spiece.model"if not tokenizer_path.is_file():raise FileNotFoundError(f"File not found: {tokenizer_path}")@@ -6734,7 +6919,7 @@ class T5Model(TextModel):# some models like Pile-T5 family use BPE tokenizer instead of Unigramif sentencepiece_model.trainer_spec.model_type == 2: # BPE# assure the tokenizer model file name is correct- assert tokenizer_path.name == 'tokenizer.model'+ assert tokenizer_path.name == "tokenizer.model"return self._set_vocab_sentencepiece()else:assert sentencepiece_model.trainer_spec.model_type == 1 # UNIGRAM@@ -6746,7 +6931,7 @@ class T5Model(TextModel):tokenizer = SentencePieceProcessor()tokenizer.LoadFromFile(str(tokenizer_path))- vocab_size = self.hparams.get('vocab_size', tokenizer.vocab_size())+ vocab_size = self.hparams.get("vocab_size", tokenizer.vocab_size())tokens: list[bytes] = [f"[PAD{i}]".encode("utf-8") for i in range(vocab_size)]scores: list[float] = [-10000.0] * vocab_size@@ -6771,14 +6956,14 @@ class T5Model(TextModel):scores[token_id] = scoretoktypes[token_id] = toktype- added_tokens_file = self.dir_model / 'added_tokens.json'+ added_tokens_file = self.dir_model / "added_tokens.json"if added_tokens_file.is_file():with open(added_tokens_file, "r", encoding="utf-8") as f:added_tokens_json = json.load(f)for key in added_tokens_json:token_id = added_tokens_json[key]if token_id >= vocab_size:- logger.warning(f'ignore token {token_id}: id is out of range, max={vocab_size - 1}')+ logger.warning(f"ignore token {token_id}: id is out of range, max={vocab_size - 1}")continuetokens[token_id] = key.encode("utf-8")@@ -6852,11 +7037,11 @@ class T5EncoderModel(TextModel):from sentencepiece import SentencePieceProcessorfrom sentencepiece import sentencepiece_model_pb2 as model- tokenizer_path = self.dir_model / 'tokenizer.model'+ tokenizer_path = self.dir_model / "tokenizer.model"# many older models use spiece.model tokenizer model filenameif not tokenizer_path.is_file():- tokenizer_path = self.dir_model / 'spiece.model'+ tokenizer_path = self.dir_model / "spiece.model"if not tokenizer_path.is_file():raise FileNotFoundError(f"File not found: {tokenizer_path}")@@ -6867,7 +7052,7 @@ class T5EncoderModel(TextModel):# some models like Pile-T5 family use BPE tokenizer instead of Unigramif sentencepiece_model.trainer_spec.model_type == 2: # BPE# assure the tokenizer model file name is correct- assert tokenizer_path.name == 'tokenizer.model'+ assert tokenizer_path.name == "tokenizer.model"return self._set_vocab_sentencepiece()else:assert sentencepiece_model.trainer_spec.model_type == 1 # UNIGRAM@@ -6879,7 +7064,7 @@ class T5EncoderModel(TextModel):tokenizer = SentencePieceProcessor()tokenizer.LoadFromFile(str(tokenizer_path))- vocab_size = self.hparams.get('vocab_size', tokenizer.vocab_size())+ vocab_size = self.hparams.get("vocab_size", tokenizer.vocab_size())tokens: list[bytes] = [f"[PAD{i}]".encode("utf-8") for i in range(vocab_size)]scores: list[float] = [-10000.0] * vocab_size@@ -6904,14 +7089,14 @@ class T5EncoderModel(TextModel):scores[token_id] = scoretoktypes[token_id] = toktype- added_tokens_file = self.dir_model / 'added_tokens.json'+ added_tokens_file = self.dir_model / "added_tokens.json"if added_tokens_file.is_file():with open(added_tokens_file, "r", encoding="utf-8") as f:added_tokens_json = json.load(f)for key in added_tokens_json:token_id = added_tokens_json[key]if token_id >= vocab_size:- logger.warning(f'ignore token {token_id}: id is out of range, max={vocab_size - 1}')+ logger.warning(f"ignore token {token_id}: id is out of range, max={vocab_size - 1}")continuetokens[token_id] = key.encode("utf-8")@@ -6983,19 +7168,19 @@ class JaisModel(TextModel):# Embeddings scaleself.embeddings_scale = 1.0- if 'mup_embeddings_scale' in self.hparams:- self.embeddings_scale = self.hparams['mup_embeddings_scale']- elif 'embeddings_scale' in self.hparams:- self.embeddings_scale = self.hparams['embeddings_scale']+ if "mup_embeddings_scale" in self.hparams:+ self.embeddings_scale = self.hparams["mup_embeddings_scale"]+ elif "embeddings_scale" in self.hparams:+ self.embeddings_scale = self.hparams["embeddings_scale"]else:assert Falseself.width_scale = 1.0- if 'mup_output_alpha' in self.hparams:- assert 'mup_width_scale' in self.hparams- self.width_scale = self.hparams['mup_output_alpha'] * self.hparams['mup_width_scale']- elif 'width_scale' in self.hparams:- self.width_scale = self.hparams['width_scale']+ if "mup_output_alpha" in self.hparams:+ assert "mup_width_scale" in self.hparams+ self.width_scale = self.hparams["mup_output_alpha"] * self.hparams["mup_width_scale"]+ elif "width_scale" in self.hparams:+ self.width_scale = self.hparams["width_scale"]else:assert False@@ -7027,7 +7212,7 @@ class JaisModel(TextModel):# Some other models has max_alibi_bias spelled out explicitly in the hyperparams,# but Jais's PyTorch model simply precalculates the slope values and places them# in relative_pes.slopes- n_head_closest_log2 = 2**math.floor(math.log2(self.hparams["n_head"]))+ n_head_closest_log2 = 2 ** math.floor(math.log2(self.hparams["n_head"]))first_val = float(data_torch[0].item())self.max_alibi_bias = -round(math.log2(first_val) * n_head_closest_log2)@@ -7058,6 +7243,7 @@ class Glm4Model(TextModel):def set_vocab(self):from transformers import AutoTokenizer+tokenizer = AutoTokenizer.from_pretrained(self.dir_model, trust_remote_code=True)special_vocab = gguf.SpecialVocab(self.dir_model, load_merges=True)tokens, toktypes, tokpre = self.get_vocab_base()@@ -7095,6 +7281,7 @@ class ChatGLMModel(TextModel):scores: list[float] = []from transformers import AutoTokenizer+tokenizer = AutoTokenizer.from_pretrained(dir_model, trust_remote_code=True)vocab_size = hparams.get("padded_vocab_size", len(tokenizer.get_vocab()))assert max(tokenizer.get_vocab().values()) < vocab_size@@ -7155,8 +7342,9 @@ class ChatGLMModel(TextModel):@staticmethoddef token_bytes_to_string(b):from transformers.models.gpt2.tokenization_gpt2 import bytes_to_unicode+byte_encoder = bytes_to_unicode()- return ''.join([byte_encoder[ord(char)] for char in b.decode('latin-1')])+ return "".join([byte_encoder[ord(char)] for char in b.decode("latin-1")])@staticmethoddef bpe(mergeable_ranks: dict[bytes, int], token: bytes, max_rank: int | None = None) -> list[bytes]:@@ -7172,7 +7360,7 @@ class ChatGLMModel(TextModel):if min_rank is None or (max_rank is not None and min_rank >= max_rank):breakassert min_idx is not None- parts = parts[:min_idx] + [parts[min_idx] + parts[min_idx + 1]] + parts[min_idx + 2:]+ parts = parts[:min_idx] + [parts[min_idx] + parts[min_idx + 1]] + parts[min_idx + 2 :]return partsdef set_vocab(self):@@ -7186,6 +7374,7 @@ class ChatGLMModel(TextModel):toktypes: list[int] = []from transformers import AutoTokenizer+tokenizer = AutoTokenizer.from_pretrained(dir_model, trust_remote_code=True)vocab_size = hparams.get("padded_vocab_size", hparams["vocab_size"])assert max(tokenizer.get_vocab().values()) < vocab_size@@ -7210,7 +7399,8 @@ class ChatGLMModel(TextModel):self.gguf_writer.add_context_length(self.hparams.get("seq_length", n_embed))self.gguf_writer.add_embedding_length(n_embed)self.gguf_writer.add_feed_forward_length(- self.hparams.get("ffn_hidden_size", self.hparams.get("intermediate_size", 4 * n_embed)))+ self.hparams.get("ffn_hidden_size", self.hparams.get("intermediate_size", 4 * n_embed))+ )self.gguf_writer.add_block_count(self.hparams.get("num_layers", self.hparams["num_hidden_layers"]))self.gguf_writer.add_head_count(n_head)self.gguf_writer.add_head_count_kv(n_head_kv)@@ -7281,7 +7471,7 @@ class ExaoneModel(TextModel):def set_gguf_parameters(self):hparams = self.hparams- assert (hparams["activation_function"] == "silu")+ assert hparams["activation_function"] == "silu"max_position_embeddings = hparams["max_position_embeddings"]embed_dim = hparams["hidden_size"]@@ -7308,7 +7498,8 @@ class ExaoneModel(TextModel):rotary_factor = self.find_hparam(["partial_rotary_factor", "rope_pct"], optional=True)rotary_factor = rotary_factor if rotary_factor is not None else 1.0self.gguf_writer.add_rope_dimension_count(- int(rotary_factor * (hparams["hidden_size"] // hparams["num_attention_heads"])))+ int(rotary_factor * (hparams["hidden_size"] // hparams["num_attention_heads"]))+ )rope_scaling = self.hparams.get("rope_scaling") or {}if rope_scaling.get("rope_type", rope_scaling.get("type")) == "linear" and "factor" in rope_scaling:self.gguf_writer.add_rope_scaling_type(gguf.RopeScalingType.LINEAR)@@ -7316,11 +7507,11 @@ class ExaoneModel(TextModel):def generate_extra_tensors(self) -> Iterable[tuple[str, Tensor]]:if rope_scaling := self.find_hparam(["rope_scaling"], optional=True):- if rope_scaling.get("rope_type", '').lower() == "llama3":+ if rope_scaling.get("rope_type", "").lower() == "llama3":base = self.hparams.get("rope_theta", 10000.0)if (dim := self.hparams.get("head_dim")) is None:dim = self.hparams["hidden_size"] // self.hparams["num_attention_heads"]- freqs = 1.0 / (base**(torch.arange(0, dim, 2, dtype=torch.float32) / dim))+ freqs = 1.0 / (base ** (torch.arange(0, dim, 2, dtype=torch.float32) / dim))factor = rope_scaling.get("factor", 8.0)low_freq_factor = rope_scaling.get("low_freq_factor", 1.0)@@ -7339,12 +7530,13 @@ class ExaoneModel(TextModel):elif wavelen > low_freq_wavelen:rope_factors.append(factor)else:- smooth = (old_context_len/wavelen - low_freq_factor) / (high_freq_factor-low_freq_factor)- rope_factors.append(1 / ((1-smooth) / factor + smooth))+ smooth = (old_context_len / wavelen - low_freq_factor) / (high_freq_factor - low_freq_factor)+ rope_factors.append(1 / ((1 - smooth) / factor + smooth))yield (self.format_tensor_name(gguf.MODEL_TENSOR.ROPE_FREQS),- torch.tensor(rope_factors, dtype=torch.float32))+ torch.tensor(rope_factors, dtype=torch.float32),+ )@ModelBase.register("Exaone4ForCausalLM")@@ -7375,10 +7567,11 @@ class Exaone4Model(TextModel):if isinstance(hparams["sliding_window_pattern"], str): # e.g. LLLGfor i in range(hparams["num_hidden_layers"]):sliding_window_pattern.append(- hparams["sliding_window_pattern"][i % len(hparams["sliding_window_pattern"])] == "L")+ hparams["sliding_window_pattern"][i % len(hparams["sliding_window_pattern"])] == "L"+ )if isinstance(hparams["sliding_window_pattern"], int): # e.g. 4for i in range(hparams["num_hidden_layers"]):- sliding_window_pattern.append((i+1) % hparams["sliding_window_pattern"] != 0)+ sliding_window_pattern.append((i + 1) % hparams["sliding_window_pattern"] != 0)if len(sliding_window_pattern) == hparams["num_hidden_layers"]:self.gguf_writer.add_sliding_window_pattern(sliding_window_pattern)@@ -7389,11 +7582,11 @@ class Exaone4Model(TextModel):def generate_extra_tensors(self) -> Iterable[tuple[str, Tensor]]:if rope_scaling := self.find_hparam(["rope_scaling"], optional=True):- if rope_scaling.get("rope_type", '').lower() == "llama3":+ if rope_scaling.get("rope_type", "").lower() == "llama3":base = self.hparams.get("rope_theta", 10_000.0)if (dim := self.hparams.get("head_dim")) is None:dim = self.hparams["hidden_size"] // self.hparams["num_attention_heads"]- freqs = 1.0 / (base**(torch.arange(0, dim, 2, dtype=torch.float32) / dim))+ freqs = 1.0 / (base ** (torch.arange(0, dim, 2, dtype=torch.float32) / dim))factor = rope_scaling.get("factor", 16.0)low_freq_factor = rope_scaling.get("low_freq_factor", 1.0)@@ -7411,17 +7604,19 @@ class Exaone4Model(TextModel):elif wavelen > low_freq_wavelen:rope_factors.append(factor)else:- smooth = (old_context_len/wavelen - low_freq_factor) / (high_freq_factor-low_freq_factor)- rope_factors.append(1 / ((1-smooth) / factor + smooth))+ smooth = (old_context_len / wavelen - low_freq_factor) / (high_freq_factor - low_freq_factor)+ rope_factors.append(1 / ((1 - smooth) / factor + smooth))yield (self.format_tensor_name(gguf.MODEL_TENSOR.ROPE_FREQS),- torch.tensor(rope_factors, dtype=torch.float32))+ torch.tensor(rope_factors, dtype=torch.float32),+ )@ModelBase.register("GraniteForCausalLM")class GraniteModel(LlamaModel):"""Conversion for IBM's GraniteForCausalLM"""+model_arch = gguf.MODEL_ARCH.GRANITEdef set_gguf_parameters(self):@@ -7456,6 +7651,7 @@ class GraniteModel(LlamaModel):@ModelBase.register("GraniteMoeForCausalLM", "GraniteMoeSharedForCausalLM")class GraniteMoeModel(GraniteModel):"""Conversion for IBM's GraniteMoeForCausalLM"""+model_arch = gguf.MODEL_ARCH.GRANITE_MOEdef set_gguf_parameters(self):@@ -7483,7 +7679,7 @@ class GraniteMoeModel(GraniteModel):(self.format_tensor_name(gguf.MODEL_TENSOR.FFN_UP_EXP, bid), up),]- has_experts = bool(self.hparams.get('num_local_experts'))+ has_experts = bool(self.hparams.get("num_local_experts"))if name.endswith("shared_mlp.input_linear.weight"):ffn_dim = self.hparams["shared_intermediate_size"]@@ -7509,6 +7705,7 @@ class GraniteMoeModel(GraniteModel):class GraniteHybridModel(Mamba2Model, GraniteMoeModel):"""GraniteHybrid is a hybrid SSM + Attention model that uses Mamba2 SSMlayers and optionally uses MoE w/ a shared expert"""+model_arch = gguf.MODEL_ARCH.GRANITE_HYBRIDundo_permute = True@@ -7552,7 +7749,7 @@ class GraniteHybridModel(Mamba2Model, GraniteMoeModel):return Mamba2Model.find_hparam(self, keys, *args, **kwargs)def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:- if (name.endswith("block_sparse_moe.input_linear.weight") or "shared_mlp" in name):+ if name.endswith("block_sparse_moe.input_linear.weight") or "shared_mlp" in name:return GraniteMoeModel.modify_tensors(self, data_torch, name, bid)# Determine whether this is a mamba layer or an attention layer@@ -7640,8 +7837,10 @@ class BailingMoeModel(TextModel):if n_head_kv is not None and n_head != n_head_kv:n_head = n_head_kvreturn (- weights.reshape(n_head, 2, weights.shape[0] // n_head // 2,- *weights.shape[1:]).swapaxes(1, 2).reshape(weights.shape))+ weights.reshape(n_head, 2, weights.shape[0] // n_head // 2, *weights.shape[1:])+ .swapaxes(1, 2)+ .reshape(weights.shape)+ )def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:n_head = self.hparams["num_attention_heads"]@@ -7660,7 +7859,7 @@ class BailingMoeModel(TextModel):return [(self.format_tensor_name(gguf.MODEL_TENSOR.ATTN_Q, bid), BailingMoeModel.permute(q, n_head, n_head)),(self.format_tensor_name(gguf.MODEL_TENSOR.ATTN_K, bid), BailingMoeModel.permute(k, n_head, n_kv_head)),- (self.format_tensor_name(gguf.MODEL_TENSOR.ATTN_V, bid), v)+ (self.format_tensor_name(gguf.MODEL_TENSOR.ATTN_V, bid), v),]elif name.find("mlp.experts") != -1:n_experts = self.hparams["num_experts"]@@ -7881,11 +8080,12 @@ class FalconH1Model(Mamba2Model):intermediate_size = self.hparams["mamba_d_ssm"]groups_time_state_size = self.hparams["mamba_n_groups"] * self.hparams["mamba_d_state"]tensor[:intermediate_size, :] *= zxbcdt_multipliers[0]- tensor[intermediate_size:2 * intermediate_size, :] *= zxbcdt_multipliers[1]- tensor[2 * intermediate_size:2*intermediate_size + groups_time_state_size, :] *= zxbcdt_multipliers[2]- tensor[2*intermediate_size + groups_time_state_size:2*intermediate_size +- 2*groups_time_state_size, :] *= zxbcdt_multipliers[3]- tensor[2*intermediate_size + 2*groups_time_state_size:, :] *= zxbcdt_multipliers[4]+ tensor[intermediate_size : 2 * intermediate_size, :] *= zxbcdt_multipliers[1]+ tensor[2 * intermediate_size : 2 * intermediate_size + groups_time_state_size, :] *= zxbcdt_multipliers[2]+ tensor[+ 2 * intermediate_size + groups_time_state_size : 2 * intermediate_size + 2 * groups_time_state_size, :+ ] *= zxbcdt_multipliers[3]+ tensor[2 * intermediate_size + 2 * groups_time_state_size :, :] *= zxbcdt_multipliers[4]elif "lm_head" in name:tensor = tensor * self.hparams["lm_head_multiplier"]elif "embed_tokens" in name:@@ -7914,8 +8114,9 @@ class FalconH1Model(Mamba2Model):## Validation ##assert self.hparams.get("hidden_act") in [None, "silu"], "Only SILU activation supported"- assert self.d_inner % self.d_head == 0, \- f"SSM inner size {self.d_inner} not a multiple of head dim {self.d_head}"+ assert (+ self.d_inner % self.d_head == 0+ ), f"SSM inner size {self.d_inner} not a multiple of head dim {self.d_head}"# Add any other Falcon Mamba2 specific configurationself.gguf_writer.add_rope_freq_base(self.find_hparam(["rope_theta"]))@@ -7932,6 +8133,7 @@ class HunYuanMoEModel(TextModel):def set_vocab(self):from transformers import AutoTokenizer+tokenizer = AutoTokenizer.from_pretrained(self.dir_model, trust_remote_code=True)# 1. Get the pre-tokenizer identifier hash@@ -7947,7 +8149,7 @@ class HunYuanMoEModel(TextModel):continuemerged = QwenModel.bpe(mergeable_ranks, token, max_rank=rank)if len(merged) == 2: # todo this is an assert in Qwen, why?- merges.append(' '.join(map(QwenModel.token_bytes_to_string, merged)))+ merges.append(" ".join(map(QwenModel.token_bytes_to_string, merged)))# 3. Generate the tokens and toktypes listsvocab_size = self.hparams["vocab_size"]@@ -8005,8 +8207,8 @@ class HunYuanMoEModel(TextModel):if rope_scaling.get("type") == "dynamic":alpha = rope_scaling.get("alpha", 1000)base = hparams.get("rope_theta", 10000.0)- dim = (hparams["hidden_size"] // hparams["num_attention_heads"]) # 128- scaled_base = base * (alpha**(dim / (dim-2))) # 10000 * (1000 ** (128 / 126)) = 11158839.9251+ dim = hparams["hidden_size"] // hparams["num_attention_heads"] # 128+ scaled_base = base * (alpha ** (dim / (dim - 2))) # 10000 * (1000 ** (128 / 126)) = 11158839.9251self.gguf_writer.add_rope_freq_base(scaled_base)self.gguf_writer.add_rope_scaling_type(gguf.RopeScalingType.NONE)self.gguf_writer.add_rope_scaling_factor(1)@@ -8014,9 +8216,12 @@ class HunYuanMoEModel(TextModel):self.gguf_writer.add_rope_scaling_orig_ctx_len(256 * 1024) # 256k context lengthself.gguf_writer.add_context_length(256 * 1024) # 256k context length- assert alpha == 1000 and base == 10000.0 and dim == 128 \- and self.hparams["max_position_embeddings"] in [32 * 1024, 256 * 1024] , \- "HunYuan dynamic RoPE scaling assumptions changed, please update the logic or context length manually"+ assert (+ alpha == 1000+ and base == 10000.0+ and dim == 128+ and self.hparams["max_position_embeddings"] in [32 * 1024, 256 * 1024]+ ), "HunYuan dynamic RoPE scaling assumptions changed, please update the logic or context length manually"_experts: list[dict[str, Tensor]] | None = None@@ -8077,6 +8282,7 @@ class SmolLM3Model(LlamaModel):# remove unsupported array slicing in chat template# ref: https://huggingface.co/ggml-org/SmolLM3-3B-GGUF/discussions/1from transformers import AutoTokenizer+tokenizer = AutoTokenizer.from_pretrained(self.dir_model)if tokenizer.chat_template is not None:chat_template = tokenizer.chat_template.replace("[:]", "")@@ -8101,7 +8307,7 @@ class LFM2Model(TextModel):# custom dim factor multiplierif ffn_dim_multiplier is not None:ff_dim = int(ffn_dim_multiplier * ff_dim)- ff_dim = multiple_of * ((ff_dim+multiple_of-1) // multiple_of)+ ff_dim = multiple_of * ((ff_dim + multiple_of - 1) // multiple_of)self.gguf_writer.add_feed_forward_length(ff_dim)@@ -8120,7 +8326,7 @@ class LFM2Model(TextModel):def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:# conv op requires 2d tensor- if 'conv.conv' in name:+ if "conv.conv" in name:data_torch = data_torch.squeeze(1)return [(self.map_tensor_name(name), data_torch)]@@ -8165,7 +8371,8 @@ class LazyTorchTensor(gguf.LazyBase):return gguf.LazyNumpyTensor(meta=gguf.LazyNumpyTensor.meta_with_dtype_and_shape(dtype, self.shape),args=(self,),- func=(lambda s: s.numpy()))+ func=(lambda s: s.numpy()),+ )@classmethoddef meta_with_dtype_and_shape(cls, dtype: torch.dtype, shape: tuple[int, ...]) -> Tensor:@@ -8184,7 +8391,8 @@ class LazyTorchTensor(gguf.LazyBase):shape = remote_tensor.shapemeta = cls.meta_with_dtype_and_shape(dtype, shape)lazy = cls(- meta=meta, args=(remote_tensor,), func=lambda r: torch.frombuffer(r.data(), dtype=dtype).reshape(shape))+ meta=meta, args=(remote_tensor,), func=lambda r: torch.frombuffer(r.data(), dtype=dtype).reshape(shape)+ )return cast(torch.Tensor, lazy)@classmethod@@ -8237,4 +8445,4 @@ def get_model_architecture(hparams: dict[str, Any], model_type: ModelType) -> starch = vision_config["architectures"][0]if arch is None:raise ValueError("Failed to detect model architecture")- return arch\ No newline at end of file+ return archdiff --git a/auto_round/export/export_to_gguf/export.py b/auto_round/export/export_to_gguf/export.pyindex e42b49f95..31d2fcbd2 100644--- a/auto_round/export/export_to_gguf/export.py+++ b/auto_round/export/export_to_gguf/export.py@@ -59,7 +59,8 @@ FTYPE_MAP: dict[str, gguf.LlamaFileType] = {def create_model_class(- output_dir, model, layer_config, backend="gguf:q4_0", low_cpu_mem_usage=False, model_type=ModelType.TEXT):+ output_dir, model, layer_config, backend="gguf:q4_0", low_cpu_mem_usage=False, model_type=ModelType.TEXT+):tmp_work_dir = Path(os.path.join(output_dir, TMP_DIR_NAME))with torch.inference_mode():hparams = ModelBase.load_hparams(tmp_work_dir)@@ -79,7 +80,7 @@ def create_model_class(logger.error(f"Model {model_architecture} is not supported")sys.exit(1)model_class = ModelBase.from_model_architecture(model_architecture, model_type=model_type)- model_name = model.name_or_path.split('/')+ model_name = model.name_or_path.split("/")if len(model_name[-1]) == 0:model_name = model_name[-2]else:@@ -102,21 +103,23 @@ def create_model_class(split_max_tensors=False,split_max_size=0,dry_run=False,- small_first_shard=False)+ small_first_shard=False,+ )return model_instance@torch.inference_mode()def pack_gguf_layer(- name,- model,- backend,- output_dir,- layer_config,- tokenizer,- processor=None,- image_processor=None,- model_type=ModelType.TEXT):+ name,+ model,+ backend,+ output_dir,+ layer_config,+ tokenizer,+ processor=None,+ image_processor=None,+ model_type=ModelType.TEXT,+):"""Export the model to gguf format."""global gguf_model_instance_globaltmp_work_dir = Path(os.path.join(output_dir, TMP_DIR_NAME))@@ -135,12 +138,15 @@ def pack_gguf_layer(gguf_model_instance_global = [create_model_class(- output_dir, model, layer_config, backend, low_cpu_mem_usage=True, model_type=ModelType.TEXT)+ output_dir, model, layer_config, backend, low_cpu_mem_usage=True, model_type=ModelType.TEXT+ )]if model_type == ModelType.MMPROJ:gguf_model_instance_global.append(create_model_class(- output_dir, model, layer_config, backend, low_cpu_mem_usage=True, model_type=ModelType.MMPROJ))+ output_dir, model, layer_config, backend, low_cpu_mem_usage=True, model_type=ModelType.MMPROJ+ )+ )if not hasattr(model, "last_layer_name_to_block_name"):block_name_to_last_layer_name = {}block_names = get_block_names(model, quant_vision=True)@@ -149,10 +155,12 @@ def pack_gguf_layer(if not check_to_quantized(m):continuefor block_name in block_names_flatten:- block_name_split = block_name.split('.')- name_split = n.split('.')- if (len(name_split) < len(block_name_split) or- name_split[:len(block_name_split)] != block_name_split):+ block_name_split = block_name.split(".")+ name_split = n.split(".")+ if (+ len(name_split) < len(block_name_split)+ or name_split[: len(block_name_split)] != block_name_split+ ):continueblock_name_to_last_layer_name[block_name] = nlast_layer_name_to_block_name = {v: k for k, v in block_name_to_last_layer_name.items()}@@ -200,13 +208,13 @@ def save_quantized_as_gguf(output_dir, backend="gguf:q4_0", layer_config=None, vif image_processor is not None:image_processor.save_pretrained(tmp_work_dir)-gguf_model_instance_global = [create_model_class(output_dir, model, layer_config, backend, model_type=ModelType.TEXT)]if vlm:gguf_model_instance_global.append(- create_model_class(output_dir, model, layer_config, backend, model_type=ModelType.MMPROJ))+ create_model_class(output_dir, model, layer_config, backend, model_type=ModelType.MMPROJ)+ )for gguf_model in gguf_model_instance_global:gguf_model.write()diff --git a/auto_round/export/export_to_gguf/packing.py b/auto_round/export/export_to_gguf/packing.pyindex 2dfe3b82b..204e3d1c0 100644--- a/auto_round/export/export_to_gguf/packing.py+++ b/auto_round/export/export_to_gguf/packing.py@@ -30,16 +30,17 @@ def register_qtype(name):def ggml_quant(- data,- ggml_type,- scale=None,- zp=None,- wmin=None,- d_scale=None,- d_wmin=None,- imatrix=None,- device="cuda",- original=False):+ data,+ ggml_type,+ scale=None,+ zp=None,+ wmin=None,+ d_scale=None,+ d_wmin=None,+ imatrix=None,+ device="cuda",+ original=False,+):block_size, type_size = GGML_QUANT_SIZES[ggml_type]data = data.to(torch.float32).to(device)@@ -55,7 +56,8 @@ def ggml_quant(quant_func = GGML_QUANT_TYPE[ggml_type]try:new_data = quant_func(- blocks, scale, zp=zp, wmin=wmin, d_scale=d_scale, d_wmin=d_wmin, imatrix=imatrix, original=original)+ blocks, scale, zp=zp, wmin=wmin, d_scale=d_scale, d_wmin=d_wmin, imatrix=imatrix, original=original+ )except Exception:device = "cpu"blocks = blocks.to(device)@@ -65,8 +67,8 @@ def ggml_quant(d_scale = d_scale.to(device) if d_scale is not None else d_scaled_wmin = d_wmin.to(device) if d_wmin is not None else d_wminnew_data = quant_func(- blocks, scale, zp=zp, wmin=wmin, d_scale=d_scale, d_wmin=d_wmin, imatrix=imatrix, original=original)-+ blocks, scale, zp=zp, wmin=wmin, d_scale=d_scale, d_wmin=d_wmin, imatrix=imatrix, original=original+ )assert new_data.shape[-1] == type_sizenew_data = new_data.reshape(*shape[:-1], shape[-1] // block_size * type_size)@@ -104,7 +106,7 @@ def make_qx_quants(data, bits, rmse_type=0, qw=None):if qw is not None:w = qwelif rmse_type == 1:- w = data ** 2+ w = data**2elif rmse_type == 2:w = 1elif rmse_type == 3:@@ -150,16 +152,16 @@ def make_q3_quants(data, bits, do_rmse=False):for itry in range(5):for i in range(sumlx.shape[-1]):- w_tmp, data_tmp, L_tmp = w[:,:, i], data[:,:, i], L[:,:, i]+ w_tmp, data_tmp, L_tmp = w[:, :, i], data[:, :, i], L[:, :, i]slx = sumlx - w_tmp * data_tmp * L_tmpreplace_idx = slx > 0- sl2 = suml2 - w_tmp * L_tmp ** 2+ sl2 = suml2 - w_tmp * L_tmp**2new_L = torch.round(data_tmp * sl2 / slx).clip(-nmax, nmax - 1)tmp_replace_idx = replace_idx & (new_L != L_tmp)slx[tmp_replace_idx] += w_tmp[tmp_replace_idx] * data_tmp[tmp_replace_idx] * new_L[tmp_replace_idx]sl2[tmp_replace_idx] += w_tmp[tmp_replace_idx] * new_L[tmp_replace_idx] * new_L[tmp_replace_idx]replace_idx &= (sl2 > 0) & (slx * slx * suml2 > sumlx * sumlx * sl2)- L[:,:, i][replace_idx] = new_L[replace_idx]+ L[:, :, i][replace_idx] = new_L[replace_idx]sumlx = slxsuml2 = sl2return sumlx * get_reciprocal(suml2), L.to(torch.uint8)@@ -205,7 +207,7 @@ def make_qkx2_quants(data, bits, weights=None, rmin=-1.0, rdelta=0.1, nstep=20,L = torch.clip(l_values, 0, nmax).to(torch.uint8)diffs = scale * L + group_min - data- diffs = torch.abs(diffs) if use_mad else diffs ** 2+ diffs = torch.abs(diffs) if use_mad else diffs**2best_mad = torch.sum(weights * diffs, dim=-1, keepdim=True)if nstep < 1:@@ -219,7 +221,7 @@ def make_qkx2_quants(data, bits, weights=None, rmin=-1.0, rdelta=0.1, nstep=20,Laux = torch.clip(l_values, 0, nmax).to(torch.uint8)sum_l = torch.sum(weights * Laux, dim=-1, keepdim=True)- sum_l2 = torch.sum(weights * Laux ** 2, dim=-1, keepdim=True)+ sum_l2 = torch.sum(weights * Laux**2, dim=-1, keepdim=True)sum_xl = torch.sum(weights * Laux * data, dim=-1, keepdim=True)D = sum_w * sum_l2 - sum_l * sum_l@@ -231,7 +233,7 @@ def make_qkx2_quants(data, bits, weights=None, rmin=-1.0, rdelta=0.1, nstep=20,this_scale[this_min > 0] = (sum_xl / sum_l2)[this_min > 0]diffs = this_scale * Laux + this_min - data- diffs = torch.abs(diffs) if use_mad else diffs ** 2+ diffs = torch.abs(diffs) if use_mad else diffs**2mad = torch.sum(weights * diffs, dim=-1, keepdim=True)replace_idx &= mad < best_mad@@ -301,9 +303,9 @@ def make_qp_quants(nmax, data, quant_weights):def bf16_quant_block(blocks, scale=None, zp=None, **kwargs):n = blocks.view(torch.uint32)# force nan to quiet- n = torch.where((n & 0x7fffffff) > 0x7f800000, (n & 0xffff0000) | (64 << 16), n)+ n = torch.where((n & 0x7FFFFFFF) > 0x7F800000, (n & 0xFFFF0000) | (64 << 16), n)# round to nearest even- n = n.to(torch.uint64) + (0x7fff + ((n >> 16) & 1)) >> 16+ n = n.to(torch.uint64) + (0x7FFF + ((n >> 16) & 1)) >> 16return n.cpu().numpy().astype(np.uint16).view(np.uint8)@@ -448,26 +450,31 @@ def q2_k_quant_block(blocks, scale=None, wmin=None, d_scale=None, d_wmin=None, ireplace_ids = (max_scales > 0).squeeze()output_scale = torch.zeros_like(scales).to(torch.uint8)- output_scale[replace_ids] = torch.round(- id_scales[replace_ids] * scales[replace_ids]).clip(0, 15).to(torch.uint8)+ output_scale[replace_ids] = (+ torch.round(id_scales[replace_ids] * scales[replace_ids]).clip(0, 15).to(torch.uint8)+ )replace_ids = (max_mins > 0).squeeze()- output_scale[replace_ids] |= torch.round(- id_mins[replace_ids] * mins[replace_ids]).clip(0, 15).to(torch.uint8) << 4+ output_scale[replace_ids] |= (+ torch.round(id_mins[replace_ids] * mins[replace_ids]).clip(0, 15).to(torch.uint8) << 4+ )d_tmp = output_d * (output_scale & 0xF)dm_tmp = output_dmin * (output_scale >> 4)replace_ids = d_tmp != 0- all_L[replace_ids] = torch.round(- (blocks[replace_ids] + dm_tmp[replace_ids].unsqueeze(-1)) / \- d_tmp[replace_ids].unsqueeze(-1)).clip(0, 3).to(torch.uint8)+ all_L[replace_ids] = (+ torch.round((blocks[replace_ids] + dm_tmp[replace_ids].unsqueeze(-1)) / d_tmp[replace_ids].unsqueeze(-1))+ .clip(0, 3)+ .to(torch.uint8)+ )else:from auto_round.data_type.gguf import quant_tensor_gguf_asym_dq+blocks.reshape(blocks.shape[0], -1)blocks, scales, mins = quant_tensor_gguf_asym_dq(blocks, bits=2, scale_dtype=torch.float32, imatrix=imatrix)- scales, d_scale = scales['scale'], scales['d_scale']- mins, d_wmin = mins['wmin'], mins['d_wmin']+ scales, d_scale = scales["scale"], scales["d_scale"]+ mins, d_wmin = mins["wmin"], mins["d_wmin"]blocks = blocks.reshape((nb, QK_K // 16, 16))scales = scales.reshape((-1, QK_K // 16))@@ -490,6 +497,7 @@ def q2_k_quant_block(blocks, scale=None, wmin=None, d_scale=None, d_wmin=None, i# [scale, qs, d, dmin]return np.concatenate([output_scale, output_qs, output_d, output_dmin], axis=-1)+@register_qtype("q3_k")def q3_k_quant_block(blocks: np.array, scale=None, d_scale=None, original=False, imatrix=None, **kwargs):nb = blocks.shape[0]@@ -502,7 +510,7 @@ def q3_k_quant_block(blocks: np.array, scale=None, d_scale=None, original=False,dq_scale = d_scale.reshape(-1, 1).to(torch.float32)all_L = (torch.round(blocks * get_reciprocal(qdq_scale.unsqueeze(-1))).clip(-4, 3) + 4).to(torch.uint8)q_scales_offset = torch.round(qdq_scale * get_reciprocal(dq_scale)).clip(-32, 31) + 32- elif original: ## this is correct+ elif original: ## this is correctscales, _ = make_q3_quants(blocks, bits=3, do_rmse=True)scales_abs_max = abs(scales).argmax(dim=-1, keepdim=True)max_scales_mag = torch.take_along_dim(scales, scales_abs_max, dim=-1)@@ -515,22 +523,21 @@ def q3_k_quant_block(blocks: np.array, scale=None, d_scale=None, original=False,q_scales_offset = torch.round(qdq_scale * inverse_dq_scale).clip(-32, 31) + 32else:from auto_round.data_type.gguf import quant_tensor_gguf_sym_dq+blocks = blocks.reshape(blocks.shape[0], -1)blocks, scales, _ = quant_tensor_gguf_sym_dq(blocks, bits=3, scale_dtype=torch.float32, imatrix=imatrix)- scales, d_scale = scales['scale'], scales['d_scale']+ scales, d_scale = scales["scale"], scales["d_scale"]blocks = blocks.reshape((nb, QK_K // 16, 16))qdq_scale = scales.reshape((-1, QK_K // 16)).to(torch.float32)dq_scale = d_scale.reshape(-1, 1).to(torch.float32)all_L = (torch.round(blocks * get_reciprocal(qdq_scale.unsqueeze(-1))).clip(-4, 3) + 4).to(torch.uint8)q_scales_offset = torch.round(qdq_scale * get_reciprocal(dq_scale)).clip(-32, 31) + 32-q_scales_offset = q_scales_offset.cpu().numpy().astype(np.uint8)output_scale[:, :8] = (q_scales_offset[:, :8] & 0xF) | ((q_scales_offset[:, 8:] & 0xF) << 4)hmask = q_scales_offset >> 4output_scale[:, 8:] = hmask[:, :4] | hmask[:, 4:8] << 2 | hmask[:, 8:12] << 4 | hmask[:, 12:] << 6-output_hmask = all_L.reshape(nb, 8, 32) >> 2 << torch.arange(8, device=all_L.device).reshape(1, 8, 1)output_hmask = np.bitwise_or.reduce(output_hmask.cpu().numpy(), axis=1, dtype=np.uint8) # pylint: disable=E1121all_L = torch.where(all_L > 3, all_L - 4, all_L)@@ -544,7 +551,6 @@ def q3_k_quant_block(blocks: np.array, scale=None, d_scale=None, original=False,return np.concatenate([output_hmask, output_qs, output_scale, dq_scale], axis=-1)-@register_qtype("q4_k")def q4_k_quant_block(blocks, scale=None, wmin=None, d_scale=None, d_wmin=None, imatrix=None, original=False, **kwargs):nb = blocks.shape[0]@@ -559,8 +565,11 @@ def q4_k_quant_block(blocks, scale=None, wmin=None, d_scale=None, d_wmin=None, ioutput_dmin = d_wmin.reshape(-1, 1).to(torch.float32)q_scales = torch.round(scales * get_reciprocal(output_d)).clip(0, 63).to(torch.uint8)q_mins = torch.round(mins * get_reciprocal(output_dmin)).clip(0, 63).to(torch.uint8)- all_L = torch.round(- (blocks + mins.unsqueeze(-1)) * get_reciprocal(scales.unsqueeze(-1))).clip(0, 15).to(torch.uint8)+ all_L = (+ torch.round((blocks + mins.unsqueeze(-1)) * get_reciprocal(scales.unsqueeze(-1)))+ .clip(0, 15)+ .to(torch.uint8)+ )elif original:scales, all_L, mins = make_qkx2_quants(blocks, bits=4, rmin=-1, rdelta=0.1, nstep=20, use_mad=False)max_scales = torch.max(scales, dim=-1, keepdim=True)[0]@@ -575,14 +584,18 @@ def q4_k_quant_block(blocks, scale=None, wmin=None, d_scale=None, d_wmin=None, id_tmp = output_d * q_scalesdm_tmp = output_dmin * q_minsreplace_ids = d_tmp != 0- all_L[replace_ids] = torch.round((blocks[replace_ids] + dm_tmp[replace_ids].unsqueeze(-1)) /- d_tmp[replace_ids].unsqueeze(-1)).clip(0, 15).to(torch.uint8)+ all_L[replace_ids] = (+ torch.round((blocks[replace_ids] + dm_tmp[replace_ids].unsqueeze(-1)) / d_tmp[replace_ids].unsqueeze(-1))+ .clip(0, 15)+ .to(torch.uint8)+ )else:from auto_round.data_type.gguf import quant_tensor_gguf_asym_dq+blocks.reshape(blocks.shape[0], -1)blocks, scales, mins = quant_tensor_gguf_asym_dq(blocks, bits=4, scale_dtype=torch.float32, imatrix=imatrix)- scales, d_scale = scales['scale'], scales['d_scale']- mins, d_wmin = mins['wmin'], mins['d_wmin']+ scales, d_scale = scales["scale"], scales["d_scale"]+ mins, d_wmin = mins["wmin"], mins["d_wmin"]blocks = blocks.reshape((nb, QK_K // 32, 32))scales = scales.reshape((-1, QK_K // 32))@@ -591,15 +604,18 @@ def q4_k_quant_block(blocks, scale=None, wmin=None, d_scale=None, d_wmin=None, ioutput_dmin = d_wmin.reshape(-1, 1).to(torch.float32)q_scales = torch.round(scales * get_reciprocal(output_d)).clip(0, 63).to(torch.uint8)q_mins = torch.round(mins * get_reciprocal(output_dmin)).clip(0, 63).to(torch.uint8)- all_L = torch.round(- (blocks + mins.unsqueeze(-1)) * get_reciprocal(scales.unsqueeze(-1))).clip(0, 15).to(torch.uint8)+ all_L = (+ torch.round((blocks + mins.unsqueeze(-1)) * get_reciprocal(scales.unsqueeze(-1)))+ .clip(0, 15)+ .to(torch.uint8)+ )output_scale[:, :4] = q_scales[:, :4]output_scale[:, 4:8] = q_mins[:, :4]output_scale[:, 8:] = (q_scales[:, 4:] & 0xF) | ((q_mins[:, 4:] & 0xF) << 4)- output_scale[:, :4] |= ((q_scales[:, 4:] >> 4) << 6)- output_scale[:, 4:8] |= ((q_mins[:, 4:] >> 4) << 6)+ output_scale[:, :4] |= (q_scales[:, 4:] >> 4) << 6+ output_scale[:, 4:8] |= (q_mins[:, 4:] >> 4) << 6output_qs = all_L[:, ::2] | (all_L[:, 1::2] << 4)@@ -616,7 +632,8 @@ def q4_k_quant_block(blocks, scale=None, wmin=None, d_scale=None, d_wmin=None, i@register_qtype("q5_k")def q5_k_quant_block(- blocks, scale=None, zp=None, wmin=None, d_scale=None, d_wmin=None, imatrix=None, original=False, **kwargs):+ blocks, scale=None, zp=None, wmin=None, d_scale=None, d_wmin=None, imatrix=None, original=False, **kwargs+):nb = blocks.shape[0]blocks = blocks.reshape((nb, QK_K // 32, 32))@@ -629,8 +646,11 @@ def q5_k_quant_block(output_dmin = d_wmin.reshape(-1, 1).to(torch.float32)q_scales = torch.round(scales * get_reciprocal(output_d)).clip(0, 63).to(torch.uint8)q_mins = torch.round(mins * get_reciprocal(output_dmin)).clip(0, 63).to(torch.uint8)- all_L = torch.round(- (blocks + mins.unsqueeze(-1)) * get_reciprocal(scales.unsqueeze(-1))).clip(0, 31).to(torch.uint8)+ all_L = (+ torch.round((blocks + mins.unsqueeze(-1)) * get_reciprocal(scales.unsqueeze(-1)))+ .clip(0, 31)+ .to(torch.uint8)+ )elif original:scales, all_L, mins = make_qkx2_quants(blocks, bits=5, rmin=-0.5, rdelta=0.1, nstep=15, use_mad=False)max_scales = torch.max(scales, dim=-1, keepdim=True)[0]@@ -645,15 +665,18 @@ def q5_k_quant_block(d_tmp = output_d * q_scalesdm_tmp = output_dmin * q_minsreplace_ids = d_tmp != 0- all_L[replace_ids] = torch.round(- (blocks[replace_ids] + dm_tmp[replace_ids].unsqueeze(-1)) \- / d_tmp[replace_ids].unsqueeze(-1)).clip(0, 31).to(torch.uint8)+ all_L[replace_ids] = (+ torch.round((blocks[replace_ids] + dm_tmp[replace_ids].unsqueeze(-1)) / d_tmp[replace_ids].unsqueeze(-1))+ .clip(0, 31)+ .to(torch.uint8)+ )else:from auto_round.data_type.gguf import quant_tensor_gguf_asym_dq+blocks.reshape(blocks.shape[0], -1)blocks, scales, mins = quant_tensor_gguf_asym_dq(blocks, bits=4, scale_dtype=torch.float32, imatrix=imatrix)- scales, d_scale = scales['scale'], scales['d_scale']- mins, d_wmin = mins['wmin'], mins['d_wmin']+ scales, d_scale = scales["scale"], scales["d_scale"]+ mins, d_wmin = mins["wmin"], mins["d_wmin"]blocks = blocks.reshape((nb, QK_K // 32, 32))scales = scales.reshape((-1, QK_K // 32))@@ -662,20 +685,24 @@ def q5_k_quant_block(output_dmin = d_wmin.reshape(-1, 1).to(torch.float32)q_scales = torch.round(scales * get_reciprocal(output_d)).clip(0, 63).to(torch.uint8)q_mins = torch.round(mins * get_reciprocal(output_dmin)).clip(0, 63).to(torch.uint8)- all_L = torch.round(- (blocks + mins.unsqueeze(-1)) * get_reciprocal(scales.unsqueeze(-1))).clip(0, 31).to(torch.uint8)+ all_L = (+ torch.round((blocks + mins.unsqueeze(-1)) * get_reciprocal(scales.unsqueeze(-1)))+ .clip(0, 31)+ .to(torch.uint8)+ )output_scale[:, :4] = q_scales[:, :4]output_scale[:, 4:8] = q_mins[:, :4]output_scale[:, 8:] = (q_scales[:, 4:] & 0xF) | ((q_mins[:, 4:] & 0xF) << 4)- output_scale[:, :4] |= ((q_scales[:, 4:] >> 4) << 6)- output_scale[:, 4:8] |= ((q_mins[:, 4:] >> 4) << 6)+ output_scale[:, :4] |= (q_scales[:, 4:] >> 4) << 6+ output_scale[:, 4:8] |= (q_mins[:, 4:] >> 4) << 6output_qs = all_L[:, ::2] | (all_L[:, 1::2] << 4)output_qh = all_L >> 4 << torch.arange(8, device=all_L.device).reshape(1, 8, 1)- output_qh = np.bitwise_or.reduce(- output_qh.cpu().numpy(), axis=1, dtype=np.uint8).astype(np.uint8) # pylint: disable=E1121+ output_qh = np.bitwise_or.reduce(output_qh.cpu().numpy(), axis=1, dtype=np.uint8).astype(+ np.uint8+ ) # pylint: disable=E1121output_d = output_d.cpu().numpy()output_d = output_d.reshape(-1, 1).astype(np.float16).view(np.uint8)@@ -707,13 +734,15 @@ def q6_k_quant_block(blocks: np.array, scale=None, d_scale=None, original=False,output_scale = torch.round(iscales * scales).clip(max=127).to(torch.int8)d_tmp = output_d * output_scale.to(torch.float32)replace_ids = d_tmp != 0- all_L[replace_ids] = torch.round(blocks[replace_ids] / d_tmp[replace_ids].reshape(-1, 1) + 32) \- .clip(0, 63).to(torch.uint8)+ all_L[replace_ids] = (+ torch.round(blocks[replace_ids] / d_tmp[replace_ids].reshape(-1, 1) + 32).clip(0, 63).to(torch.uint8)+ )else:from auto_round.data_type.gguf import quant_tensor_gguf_sym_dq+blocks = blocks.reshape(blocks.shape[0], -1)blocks, scales, _ = quant_tensor_gguf_sym_dq(blocks, bits=6, scale_dtype=torch.float32, imatrix=imatrix)- scales, d_scale = scales['scale'], scales['d_scale']+ scales, d_scale = scales["scale"], scales["d_scale"]blocks = blocks.reshape((nb, QK_K // 16, 16))scales = scales.reshape((-1, QK_K // 16))output_d = d_scale.reshape(-1, 1).to(torch.float32)@@ -722,11 +751,14 @@ def q6_k_quant_block(blocks: np.array, scale=None, d_scale=None, original=False,tmp_L = all_L.reshape(nb, 4, 64) & 0xFoutput_ql = (tmp_L[:, ::2] | (tmp_L[:, 1::2] << 4)).reshape(nb, QK_K // 2).cpu().numpy().astype(np.uint8)- ouptut_qh = (all_L >> 4).reshape(nb, 2, 4, 32) << torch.tensor(- [0, 2, 4, 6], device=all_L.device).reshape(1, 1, 4, 1)- output_qh = np.bitwise_or.reduce(- ouptut_qh.cpu().numpy(), # pylint: disable=E1121- axis=2, dtype=np.uint8).reshape(nb, QK_K // 4).astype(np.uint8) # pylint: disable=E1121+ ouptut_qh = (all_L >> 4).reshape(nb, 2, 4, 32) << torch.tensor([0, 2, 4, 6], device=all_L.device).reshape(+ 1, 1, 4, 1+ )+ output_qh = (+ np.bitwise_or.reduce(ouptut_qh.cpu().numpy(), axis=2, dtype=np.uint8) # pylint: disable=E1121+ .reshape(nb, QK_K // 4)+ .astype(np.uint8)+ ) # pylint: disable=E1121output_d = output_d.cpu().numpy().reshape(-1, 1).astype(np.float16).view(np.uint8)diff --git a/auto_round/export/export_to_itrex/config.py b/auto_round/export/export_to_itrex/config.pyindex c212dbe82..81c244140 100644--- a/auto_round/export/export_to_itrex/config.py+++ b/auto_round/export/export_to_itrex/config.py@@ -301,4 +301,3 @@ class QuantConfig(PretrainedConfig):for parameter in remove_parameters:if hasattr(self, parameter):delattr(self, parameter)-diff --git a/auto_round/export/export_to_itrex/export.py b/auto_round/export/export_to_itrex/export.pyindex 34aa1aed7..bc233eebe 100644--- a/auto_round/export/export_to_itrex/export.py+++ b/auto_round/export/export_to_itrex/export.py@@ -48,15 +48,15 @@ def quant_weight_w_scale(weight, scale, zp, group_size=-1, device="cpu"):leng = weight.shape[1] // group_sizetail_flag = False if weight.shape[1] % group_size == 0 else Truefor i in range(leng):- int_weight_tmp = weight[:, i * group_size: (i + 1) * group_size] / scale[:, i].unsqueeze(1)+ int_weight_tmp = weight[:, i * group_size : (i + 1) * group_size] / scale[:, i].unsqueeze(1)if zp is not None:int_weight_tmp += zp[:, i].unsqueeze(1)- int_weight[:, i * group_size: (i + 1) * group_size] = torch.round(int_weight_tmp)+ int_weight[:, i * group_size : (i + 1) * group_size] = torch.round(int_weight_tmp)if tail_flag:- int_weight_tmp = weight[:, leng * group_size:] / scale[:, -1].unsqueeze(1)+ int_weight_tmp = weight[:, leng * group_size :] / scale[:, -1].unsqueeze(1)if zp is not None:int_weight_tmp += zp[:, -1].unsqueeze(1)- int_weight[:, leng * group_size:] = torch.round(int_weight_tmp)+ int_weight[:, leng * group_size :] = torch.round(int_weight_tmp)return int_weight@@ -74,7 +74,7 @@ def save_quantized_as_itrex(output_dir, inplace=True, **kwargs):enable_quanted_input = kwargs["enable_quanted_input"]scale_dtype = kwargs["scale_dtype"]tokenizer = kwargs["tokenizer"]- safe_serialization = True if 'safe_serialization' not in kwargs.keys() else kwargs["safe_serialization"]+ safe_serialization = True if "safe_serialization" not in kwargs.keys() else kwargs["safe_serialization"]compressed_model = pack_model(model, layer_config, inplace=inplace)if output_dir is None:@@ -106,7 +106,6 @@ def save_quantized_as_itrex(output_dir, inplace=True, **kwargs):return compressed_model-def save_quantized_as_itrex_xpu(output_dir, inplace=True, **kwargs):"""Save configure file and weights for XPU backend inference."""model = kwargs["model"]@@ -190,7 +189,7 @@ def pack_model("""# Due to XPU doesn't support tuning yetdevice = "cpu" if device == "xpu" else device- if model.device.type == 'meta':+ if model.device.type == "meta":model = model.to(device)if inplace:compressed_model = model@@ -201,7 +200,7 @@ def pack_model(q_config = json.load(f)else:q_config = layer_config-+for k, v in q_config.items():if check_to_quantized(v) is False:continue@@ -252,6 +251,3 @@ def pack_model(new_module.pack(int_weight, scale, zp, m.bias)set_module(compressed_model, k, new_module)return compressed_model---diff --git a/auto_round/export/export_to_itrex/model_wrapper.py b/auto_round/export/export_to_itrex/model_wrapper.pyindex 05d0ce45e..202770df0 100644--- a/auto_round/export/export_to_itrex/model_wrapper.py+++ b/auto_round/export/export_to_itrex/model_wrapper.py@@ -97,7 +97,7 @@ class WeightOnlyLinear(torch.nn.Module):compression_dim=1,device="cpu",use_optimum_format=True,- use_legacy_pack = False,+ use_legacy_pack=False,):super().__init__()self.use_optimum_format = use_optimum_format@@ -199,16 +199,12 @@ class WeightOnlyLinear(torch.nn.Module):self.register_buffer("bias", torch.zeros(self.out_features, dtype=self.float_type).to(device))else:self.bias = None- self.pack_func = (- self.pack_tensor_native_impl if use_legacy_pack else self.pack_tensor- )+ self.pack_func = self.pack_tensor_native_impl if use_legacy_pack else self.pack_tensordef pack_tensor_native_impl(self, raw_tensor):mask = torch.tensor(2**self.bits - 1, dtype=self.compression_dtype).to(self.device)target_len = math.ceil(raw_tensor.shape[1] / self.n_pack)- packed_tensor = torch.zeros(- raw_tensor.shape[0], target_len, dtype=self.compression_dtype- ).to(raw_tensor.device)+ packed_tensor = torch.zeros(raw_tensor.shape[0], target_len, dtype=self.compression_dtype).to(raw_tensor.device)for j in range(packed_tensor.shape[1]):start = self.n_pack * jend = self.n_pack * (j + 1)@@ -380,12 +376,8 @@ class WeightOnlyLinear(torch.nn.Module):tensor: packed tensor."""target_len = math.ceil(raw_tensor.shape[1] / self.n_pack)- packed_tensor = torch.zeros(- raw_tensor.shape[0], target_len, dtype=self.compression_dtype- ).to(raw_tensor.device)- mask = torch.tensor(2**self.bits - 1, dtype=self.compression_dtype).to(- raw_tensor.device- )+ packed_tensor = torch.zeros(raw_tensor.shape[0], target_len, dtype=self.compression_dtype).to(raw_tensor.device)+ mask = torch.tensor(2**self.bits - 1, dtype=self.compression_dtype).to(raw_tensor.device)for j in range(packed_tensor.shape[1]):start = self.n_pack * jend = self.n_pack * (j + 1)@@ -407,12 +399,8 @@ class WeightOnlyLinear(torch.nn.Module):"""target_dtype = torch.int16target_len = packed_tensor.shape[1] * self.n_pack- unpacked_tensor = torch.zeros(- packed_tensor.shape[0], target_len, dtype=target_dtype- ).to(packed_tensor.device)- mask = torch.tensor(2**self.bits - 1, dtype=self.compression_dtype).to(- packed_tensor.device- )+ unpacked_tensor = torch.zeros(packed_tensor.shape[0], target_len, dtype=target_dtype).to(packed_tensor.device)+ mask = torch.tensor(2**self.bits - 1, dtype=self.compression_dtype).to(packed_tensor.device)for j in range(packed_tensor.shape[1]):for e in range(self.n_pack):index = j * self.n_pack + e@@ -460,14 +448,10 @@ class WeightOnlyLinear(torch.nn.Module):"fallback to torch implementation."))- return (- self.pack_tensor_with_torch(torch.from_numpy(raw_array)).cpu().numpy()- )+ return self.pack_tensor_with_torch(torch.from_numpy(raw_array)).cpu().numpy()out_features, in_features = raw_array.shapenew_in_features = (in_features + n_pack - 1) // n_pack- packed_array = np.zeros(- (out_features, new_in_features), dtype=compression_dtype- )+ packed_array = np.zeros((out_features, new_in_features), dtype=compression_dtype)raw_array = raw_array.astype(compression_dtype)pack_method = bit_packers[pack_func_name]return pack_method(raw_array, packed_array, n_pack, new_in_features)@@ -510,17 +494,11 @@ class WeightOnlyLinear(torch.nn.Module):"""Unpack the packed tensor with numpy."""packed_array = packed_tensor.cpu().numpy()target_dtype = np.int16- if (- self.bits == 8- and self.compression_dtype == torch.int8- and hasattr(self, "qzeros")- ):+ if self.bits == 8 and self.compression_dtype == torch.int8 and hasattr(self, "qzeros"):# special case for unpacking uint8 date from int8 compression_dtypetarget_dtype = np.uint8target_len = packed_array.shape[1] * self.n_pack- unpacked_array = np.zeros(- (packed_array.shape[0], target_len), dtype=target_dtype- )+ unpacked_array = np.zeros((packed_array.shape[0], target_len), dtype=target_dtype)mask = np.uint8(2**self.bits - 1)for j in range(packed_array.shape[1]):for e in range(self.n_pack):@@ -531,9 +509,7 @@ class WeightOnlyLinear(torch.nn.Module):if hasattr(self, "qzeros"):tmp &= maskunpacked_array[:, index] = tmp.astype(target_dtype)- unpacked_tensor = torch.from_numpy(unpacked_array).to(- device=packed_tensor.device- )+ unpacked_tensor = torch.from_numpy(unpacked_array).to(device=packed_tensor.device)return unpacked_tensordef pack_tensor(self, raw_tensor):diff --git a/auto_round/export/export_to_llmcompressor/config.py b/auto_round/export/export_to_llmcompressor/config.pyindex 1e1b9a6a1..6b3145c98 100644--- a/auto_round/export/export_to_llmcompressor/config.py+++ b/auto_round/export/export_to_llmcompressor/config.py@@ -14,43 +14,41 @@quantization_config = {-"config_groups": {- "group_0": {- "input_activations": {- "actorder": None,- "block_structure": None,- "dynamic": True,- "group_size": None,- "num_bits": 8,- "observer": None,- "observer_kwargs": {},- "strategy": "token",- "symmetric": True,- "type": "int"+ "config_groups": {+ "group_0": {+ "input_activations": {+ "actorder": None,+ "block_structure": None,+ "dynamic": True,+ "group_size": None,+ "num_bits": 8,+ "observer": None,+ "observer_kwargs": {},+ "strategy": "token",+ "symmetric": True,+ "type": "int",+ },+ "output_activations": None,+ "targets": ["Linear"],+ "weights": {+ "actorder": None,+ "block_structure": None,+ "dynamic": False,+ "group_size": None,+ "num_bits": 8,+ "observer": "minmax",+ "observer_kwargs": {},+ "strategy": "channel",+ "symmetric": True,+ "type": "int",+ },+ }},- "output_activations": None,- "targets": [- "Linear"- ],- "weights": {- "actorder": None,- "block_structure": None,- "dynamic": False,- "group_size": None,- "num_bits": 8,- "observer": "minmax",- "observer_kwargs": {},- "strategy": "channel",- "symmetric": True,- "type": "int"- }- }-},-"format": "int-quantized",-"global_compression_ratio": 1.5, # not exactly correct, but a placeholder-"ignore": [],-"kv_cache_scheme": None,-"quant_method": "compressed-tensors",-"quantization_status": "compressed",-"provider": "auto-round" # This is the key to indicate the model provider+ "format": "int-quantized",+ "global_compression_ratio": 1.5, # not exactly correct, but a placeholder+ "ignore": [],+ "kv_cache_scheme": None,+ "quant_method": "compressed-tensors",+ "quantization_status": "compressed",+ "provider": "auto-round", # This is the key to indicate the model provider}diff --git a/auto_round/export/export_to_llmcompressor/export.py b/auto_round/export/export_to_llmcompressor/export.pyindex d9e5dc50d..161a44ed1 100644--- a/auto_round/export/export_to_llmcompressor/export.py+++ b/auto_round/export/export_to_llmcompressor/export.py@@ -25,7 +25,7 @@ from auto_round.wrapper import WrapperWALayerdef recover_qweight(qdq_weight, scale):"""Recover the quantized weight to its original floating-point representation.-+Args:qdq_weight (torch.Tensor): The quantized weight tensor.scale (float): The scale factor used for quantization.@@ -36,7 +36,6 @@ def recover_qweight(qdq_weight, scale):return (qdq_weight / scale).to(torch.int8)-@torch.no_grad()def save_quantized_as_llmcompressor(output_dir, model=None, **kwargs):safe_serialization = kwargs.get("safe_serialization", True)@@ -57,8 +56,8 @@ def save_quantized_as_llmcompressor(output_dir, model=None, **kwargs):if isinstance(m, WrapperWALayer):m = m.orig_layerq_weight = recover_qweight(m.weight.to(device), m.scale.to(device)).to("cpu")- delattr(m, 'weight')- setattr(m, 'weight', torch.nn.Buffer(q_weight))+ delattr(m, "weight")+ setattr(m, "weight", torch.nn.Buffer(q_weight))setattr(m, "weight_scale", torch.nn.Buffer(m.scale))# replace WrapperWALayer with orig_layerdiff --git a/auto_round/export/register.py b/auto_round/export/register.pyindex 62a518558..7e186f5b2 100644--- a/auto_round/export/register.py+++ b/auto_round/export/register.py@@ -36,9 +36,9 @@ def register_format(name):return register-PACKING_LAYER_WITH_FORMAT = {}+def register_layer_packing(name):"""Class decorator to register a EXPORT subclass to the registry.diff --git a/auto_round/inference/__init__.py b/auto_round/inference/__init__.pyindex 789a8cdc7..666000c4f 100644--- a/auto_round/inference/__init__.py+++ b/auto_round/inference/__init__.py@@ -12,4 +12,3 @@# See the License for the specific language governing permissions and# limitations under the License.from auto_round.inference.convert_model import convert_hf_model, infer_target_device, post_init-diff --git a/auto_round/inference/auto_quantizer.py b/auto_round/inference/auto_quantizer.pyindex 765a7e150..470c4eefd 100644--- a/auto_round/inference/auto_quantizer.py+++ b/auto_round/inference/auto_quantizer.py@@ -59,6 +59,7 @@ def _is_package_available(pkg_name: str, return_version: bool = False) -> Union[# Check we're not importing a "pkg_name" directory somewhere but the actual library by trying to grab the versiontry: ##TODO remove it laterimport auto_round+return True, auto_round.__version__except:pass@@ -83,6 +84,7 @@ _auto_round_available = _is_package_available("auto_round")def is_auto_round_available():try:import auto_round+return Trueexcept:pass@@ -138,9 +140,9 @@ class AutoHfQuantizer:@classmethoddef merge_quantization_configs(- cls,- quantization_config: Union[dict, QuantizationConfigMixin],- quantization_config_from_args: Optional[QuantizationConfigMixin],+ cls,+ quantization_config: Union[dict, QuantizationConfigMixin],+ quantization_config_from_args: Optional[QuantizationConfigMixin],):"""Handles situations where both quantization_configfrom args and quantization_config from model config are present."""@@ -152,8 +154,9 @@ class AutoHfQuantizer:)else:warning_msg = ""- if (quantization_config_from_args is None or- not hasattr(quantization_config_from_args, "get_loading_attributes")):+ if quantization_config_from_args is None or not hasattr(+ quantization_config_from_args, "get_loading_attributes"+ ):# If the quantization_config_from_args is None or does not have get_loading_attributes method,# we will not use it to load the model.quantization_config_from_args = None@@ -161,14 +164,18 @@ class AutoHfQuantizer:loading_attr_dict = quantization_config_from_args.get_loading_attributes()if isinstance(quantization_config, dict):- if "auto-round" in quantization_config[- "quant_method"] or quantization_config_from_args.__class__.__name__ == "AutoRoundConfig":+ if (+ "auto-round" in quantization_config["quant_method"]+ or quantization_config_from_args.__class__.__name__ == "AutoRoundConfig"+ ):quantization_config = AutoRoundConfig.from_dict(quantization_config)else:quantization_config = AutoQuantizationConfig.from_dict(quantization_config) # pylint: disable=E1101- if isinstance(quantization_config,- (GPTQConfig, AwqConfig, AutoRoundConfig)) and quantization_config_from_args is not None:+ if (+ isinstance(quantization_config, (GPTQConfig, AwqConfig, AutoRoundConfig))+ and quantization_config_from_args is not None+ ):# special case for GPTQ / AWQ config collisionfor attr, val in loading_attr_dict.items():@@ -186,15 +193,16 @@ class AutoHfQuantizer:@staticmethoddef supports_quant_method(quantization_config_dict):from transformers.quantizers.auto import AUTO_QUANTIZATION_CONFIG_MAPPING- AUTO_QUANTIZATION_CONFIG_MAPPING['auto-round'] = AutoRoundConfig- AUTO_QUANTIZATION_CONFIG_MAPPING['auto_round'] = AutoRoundConfig++ AUTO_QUANTIZATION_CONFIG_MAPPING["auto-round"] = AutoRoundConfig+ AUTO_QUANTIZATION_CONFIG_MAPPING["auto_round"] = AutoRoundConfigquant_method = quantization_config_dict.get("quant_method", None)if quantization_config_dict.get("load_in_8bit", False) or quantization_config_dict.get("load_in_4bit", False):suffix = "_4bit" if quantization_config_dict.get("load_in_4bit", False) else "_8bit"quant_method = QuantizationMethod.BITS_AND_BYTES + suffixelif quant_method is None:raise ValueError(- "The model's quantization config from the arguments has no `quant_method` attribute." \+ "The model's quantization config from the arguments has no `quant_method` attribute.""Make sure that the model has been correctly quantized")@@ -229,15 +237,15 @@ class AutoRoundConfig(QuantizationConfigMixin):"""def __init__(- self,- bits: int = 4,- tokenizer: Any = None,- dataset: str = None,- group_size: int = 128,- sym: bool = False,- backend="auto",- layer_config: dict = None,- **kwargs,+ self,+ bits: int = 4,+ tokenizer: Any = None,+ dataset: str = None,+ group_size: int = 128,+ sym: bool = False,+ backend="auto",+ layer_config: dict = None,+ **kwargs,):self.bits = bits@@ -278,17 +286,16 @@ class AutoRoundConfig(QuantizationConfigMixin):)if "gptq" in quant_method and "meta" in config_dict:- raise NotImplementedError(- "Failed to convert gptq format to auto_round format. Only supports `gptqv1`")+ raise NotImplementedError("Failed to convert gptq format to auto_round format. Only supports `gptqv1`")if "awq" in quant_method and config_dict.get("version", "gemm") != "gemm":raise NotImplementedError(- "Failed to convert awq format to auto_round format. Only supports awq format with gemm version")+ "Failed to convert awq format to auto_round format. Only supports awq format with gemm version"+ )if "auto-round" not in quant_method:config_dict["packing_format"] = f"auto_round:{quant_method}"-return super().from_dict(config_dict, return_unused_kwargs=return_unused_kwargs, **kwargs)@@ -306,17 +313,21 @@ class AutoRoundQuantizer(HfQuantizer):def validate_environment(self, *args, **kwargs):self.device_map = kwargs.get("device_map", None)if not is_auto_round_available():- raise ImportError("Loading a AutoRound quantized model requires auto-round library (`pip install "- "auto-round`)")+ raise ImportError(+ "Loading a AutoRound quantized model requires auto-round library (`pip install " "auto-round`)"+ )else:try:import auto_round+autoround_version = version.parse(auto_round.__version__)except:autoround_version = version.parse(importlib.metadata.version("auto_round"))if autoround_version < version.parse("0.2.0"):- raise ImportError("You need a version of auto_round > 0.2.0 to use AutoRound: `pip install --upgrade "- "auto-round` or install from source")+ raise ImportError(+ "You need a version of auto_round > 0.2.0 to use AutoRound: `pip install --upgrade "+ "auto-round` or install from source"+ )def update_torch_dtype(self, torch_dtype: "torch.dtype") -> "torch.dtype":if torch_dtype is None:diff --git a/auto_round/inference/backend.py b/auto_round/inference/backend.pyindex 47bffe6dd..a4f578726 100644--- a/auto_round/inference/backend.py+++ b/auto_round/inference/backend.py@@ -59,6 +59,7 @@ class BackendInfo:alias: An optional list of strings representing alternative names for thebackend. Defaults to None."""+device: List[str]sym: List[bool]packing_format: str@@ -78,205 +79,267 @@ def feature_multiply_checker(in_feature, out_feature, group_size, in_feature_mulreturn in_feature % in_feature_multiplier == 0 and out_feature % out_feature_multiplier == 0-def feature_multiply_checker_group_size(in_feature, out_feature, group_size, in_feature_multiplier,- out_feature_multiplier=None):+def feature_multiply_checker_group_size(+ in_feature, out_feature, group_size, in_feature_multiplier, out_feature_multiplier=None+):if out_feature_multiplier is None:out_feature_multiplier = in_feature_multiplier- return (in_feature % in_feature_multiplier == 0 and out_feature % out_feature_multiplier == 0- and in_feature % group_size == 0)+ return (+ in_feature % in_feature_multiplier == 0+ and out_feature % out_feature_multiplier == 0+ and in_feature % group_size == 0+ )feature_multiply_checker_32 = functools.partial(feature_multiply_checker, in_feature_multiplier=32)-in_output_feature_multiply_checker_32 = functools.partial(feature_multiply_checker, in_feature_multiplier=32,- out_feature_multiplier=32)--exllamav2_feature_check = functools.partial(feature_multiply_checker_group_size, in_feature_multiplier=32,- out_feature_multiplier=32)--gptqmodel_marlin_feature_check = functools.partial(feature_multiply_checker_group_size, in_feature_multiplier=1,- out_feature_multiplier=64)--BackendInfos['auto_gptq:exllamav2'] = BackendInfo(device=["cuda"], sym=[True, False],- packing_format="int32_zp",- bits=[4],- priority=5,- dtype=["float16"],- ##16, 384,768 accuracy issue- group_size=[-1, 32, 64, 128, 256, 512, 1024, 2048],- feature_checks=[exllamav2_feature_check],- alias=['gptq', 'auto_gptq', 'exllamav2', "gptq:exllamav2",- "auto_gptq:exllamav2"],- requirements=["auto-gptq>=0.7.1"]- )--BackendInfos['auto_gptq:tritonv2'] = BackendInfo(device=["cuda"], sym=[True, False],- packing_format="int32_zp",- bits=[2, 4, 8], group_size=None,- dtype=["float16"],- priority=0, feature_checks=[exllamav2_feature_check],- alias=["auto_gptq:tritonv2"],- requirements=["auto-gptq>=0.7.1", "triton>=2.0"]- )--BackendInfos['auto_gptq:cuda'] = BackendInfo(device=["cuda"], sym=[True, False],- packing_format="int32_zp",- bits=[2, 3, 4, 8], group_size=None,- priority=1, feature_checks=[exllamav2_feature_check],- alias=["auto_gptq:cuda"],- dtype=["float16"],- convertable_format=["int32_zp"],- requirements=["auto-gptq>=0.7.1"]- )--BackendInfos['auto_round:tritonv2'] = BackendInfo(device=["cuda", "xpu"], sym=[True, False],- packing_format="int32",- dtype=["float16", "bfloat16"],- bits=[2, 4, 8],- priority=2, feature_checks=[feature_multiply_checker_32],- alias=["auto_round", "tritonv2", "triton"],- requirements=["triton>=2.0","auto-round>=0.5.0"]- )--BackendInfos['auto_round:torch'] = BackendInfo(device=["cuda","xpu", "cpu"], sym=[True, False],- packing_format="int32",- dtype=["float16", "bfloat16"],- bits=[2, 3, 4, 8],- priority=0, feature_checks=[feature_multiply_checker_32],- alias=["auto_round", "torch"],- requirements=["auto-round>=0.5.1"]- )--BackendInfos['auto_round:tritonv2_zp'] = BackendInfo(device=["cuda","xpu"], sym=[True], ## asym has accuracys- # issue- packing_format="int32_zp",- dtype=["float16", "bfloat16"],- bits=[2, 4, 8],- priority=2, feature_checks=[feature_multiply_checker_32],- alias=["tritonv2", "tritonv2_zp", "triton"],- requirements=[ "triton>=2.0","auto-round>=0.5.0"]- )--BackendInfos['auto_round:torch_zp'] = BackendInfo(device=["cuda","xpu", "cpu"], sym=[True],- packing_format="int32_zp",- dtype=["float16", "bfloat16"],- bits=[2, 3, 4, 8],- priority=0, feature_checks=[feature_multiply_checker_32],- alias=["torch", "torch_zp"],- requirements=["auto-round>=0.5.1"]- )--BackendInfos['gptqmodel:marlin'] = BackendInfo(device=["cuda"], sym=[True],- packing_format="int32",- bits=[4, 8],- group_size=[-1, 32, 64, 128],- dtype=["float16", "bfloat16"],- priority=6, feature_checks=[gptqmodel_marlin_feature_check],- alias=["marlin", "gptqmodel"],- requirements=["gptqmodel>=2.0"],- )--BackendInfos['gptqmodel:marlin_zp'] = BackendInfo(device=["cuda"], sym=[True],- packing_format="int32_zp",- bits=[4, 8],- group_size=[-1, 32, 64, 128],- dtype=["float16", "bfloat16"],- priority=6, feature_checks=[gptqmodel_marlin_feature_check],- alias=["marlin", "gptqmodel"],- requirements=["gptqmodel>=2.0"]- )--BackendInfos['gptqmodel:exllamav2'] = BackendInfo(device=["cuda"], sym=[True, False],- packing_format="int32",- bits=[4], group_size=[-1, 32, 64, 128], ##16 seems has accuracy issue- dtype=["float16", "bfloat16"],- priority=5, feature_checks=[exllamav2_feature_check],- alias=["exllamav2"],- requirements=["gptqmodel>=2.0"]- )--BackendInfos['auto_awq:gemm'] = BackendInfo(device=["cuda"], sym=[True, False], ##actually is gemm- packing_format="awq",- bits=[4], group_size=None,- priority=5,- dtype=["float16"],- alias=["auto_awq:gemm", "awq", "awq:gemm",- "auto_awq"],- requirements=["autoawq"]- )--BackendInfos['qbits'] = BackendInfo(device=["cpu"], sym=[True, False],- packing_format="qbits",- bits=[2, 4, 8], group_size=None,- priority=1,- feature_checks=[],- alias=["itrex", "qbits"],- dtype=["float16", "bfloat16"],- convertable_format=["int32"],- requirements=["intel-extension-for-transformers", "torch<2.7.0"])--BackendInfos['qbits_zp'] = BackendInfo(device=["cpu"], sym=[True, False],- packing_format="qbits_zp",- bits=[2, 4, 8], group_size=None,- dtype=["float16", "bfloat16"],- priority=1,- feature_checks=[],- alias=["itrex", "qbits"],- convertable_format=["int32_zp"],- requirements=["intel-extension-for-transformers", "torch<2.7.0"]- )--BackendInfos['auto_round:qbits_awq'] = BackendInfo(device=["cpu"], sym=[True, False], ## for awq, not robust- packing_format="awq",- bits=[2, 4, 8], group_size=None,- dtype=["float16", "bfloat16"],- priority=1,- feature_checks=[],- alias=["itrex", "qbits"],- requirements=["intel-extension-for-transformers", "torch<2.7.0"]- )--BackendInfos['ipex_gptq'] = BackendInfo(device=["cpu", "xpu"], sym=[True, False],- packing_format="ipex_gptq",- bits=[4], group_size=None,- priority=5,- feature_checks=[],- dtype=["float16", "bfloat16"],- convertable_format=["int32_zp"],- alias=["ipex"],- requirements=["intel-extension-for-pytorch>=2.5"]- )--BackendInfos['ipex_awq'] = BackendInfo(device=["cpu", "xpu"], sym=[True, False],- packing_format="ipex_awq",- bits=[4], group_size=None,- priority=1,- dtype=["float16", "bfloat16"],- feature_checks=[],- alias=["ipex"],- convertable_format=["awq"],- requirements=["intel-extension-for-pytorch>=2.6"]- )--BackendInfos['hpu'] = BackendInfo(device=["hpu"], sym=[True, False],- packing_format="hpu",- bits=[4],- dtype=["bfloat16"],- alias=["hpu"],- priority=0,- convertable_format=["int32"]- )--BackendInfos['hpu_zp'] = BackendInfo(device=["hpu"], sym=[True, False],- packing_format="hpu_zp",- bits=[4],- dtype=["bfloat16"],- alias=["hpu"],- priority=0,- convertable_format=["int32_zp"])---def check_compatible(backend_name, device, bits, group_size, sym, packing_format, in_features, out_features,- check_requirements=True):+in_output_feature_multiply_checker_32 = functools.partial(+ feature_multiply_checker, in_feature_multiplier=32, out_feature_multiplier=32+)++exllamav2_feature_check = functools.partial(+ feature_multiply_checker_group_size, in_feature_multiplier=32, out_feature_multiplier=32+)++gptqmodel_marlin_feature_check = functools.partial(+ feature_multiply_checker_group_size, in_feature_multiplier=1, out_feature_multiplier=64+)++BackendInfos["auto_gptq:exllamav2"] = BackendInfo(+ device=["cuda"],+ sym=[True, False],+ packing_format="int32_zp",+ bits=[4],+ priority=5,+ dtype=["float16"],+ ##16, 384,768 accuracy issue+ group_size=[-1, 32, 64, 128, 256, 512, 1024, 2048],+ feature_checks=[exllamav2_feature_check],+ alias=["gptq", "auto_gptq", "exllamav2", "gptq:exllamav2", "auto_gptq:exllamav2"],+ requirements=["auto-gptq>=0.7.1"],+)++BackendInfos["auto_gptq:tritonv2"] = BackendInfo(+ device=["cuda"],+ sym=[True, False],+ packing_format="int32_zp",+ bits=[2, 4, 8],+ group_size=None,+ dtype=["float16"],+ priority=0,+ feature_checks=[exllamav2_feature_check],+ alias=["auto_gptq:tritonv2"],+ requirements=["auto-gptq>=0.7.1", "triton>=2.0"],+)++BackendInfos["auto_gptq:cuda"] = BackendInfo(+ device=["cuda"],+ sym=[True, False],+ packing_format="int32_zp",+ bits=[2, 3, 4, 8],+ group_size=None,+ priority=1,+ feature_checks=[exllamav2_feature_check],+ alias=["auto_gptq:cuda"],+ dtype=["float16"],+ convertable_format=["int32_zp"],+ requirements=["auto-gptq>=0.7.1"],+)++BackendInfos["auto_round:tritonv2"] = BackendInfo(+ device=["cuda", "xpu"],+ sym=[True, False],+ packing_format="int32",+ dtype=["float16", "bfloat16"],+ bits=[2, 4, 8],+ priority=2,+ feature_checks=[feature_multiply_checker_32],+ alias=["auto_round", "tritonv2", "triton"],+ requirements=["triton>=2.0", "auto-round>=0.5.0"],+)++BackendInfos["auto_round:torch"] = BackendInfo(+ device=["cuda", "xpu", "cpu"],+ sym=[True, False],+ packing_format="int32",+ dtype=["float16", "bfloat16"],+ bits=[2, 3, 4, 8],+ priority=0,+ feature_checks=[feature_multiply_checker_32],+ alias=["auto_round", "torch"],+ requirements=["auto-round>=0.5.1"],+)++BackendInfos["auto_round:tritonv2_zp"] = BackendInfo(+ device=["cuda", "xpu"],+ sym=[True], ## asym has accuracys+ # issue+ packing_format="int32_zp",+ dtype=["float16", "bfloat16"],+ bits=[2, 4, 8],+ priority=2,+ feature_checks=[feature_multiply_checker_32],+ alias=["tritonv2", "tritonv2_zp", "triton"],+ requirements=["triton>=2.0", "auto-round>=0.5.0"],+)++BackendInfos["auto_round:torch_zp"] = BackendInfo(+ device=["cuda", "xpu", "cpu"],+ sym=[True],+ packing_format="int32_zp",+ dtype=["float16", "bfloat16"],+ bits=[2, 3, 4, 8],+ priority=0,+ feature_checks=[feature_multiply_checker_32],+ alias=["torch", "torch_zp"],+ requirements=["auto-round>=0.5.1"],+)++BackendInfos["gptqmodel:marlin"] = BackendInfo(+ device=["cuda"],+ sym=[True],+ packing_format="int32",+ bits=[4, 8],+ group_size=[-1, 32, 64, 128],+ dtype=["float16", "bfloat16"],+ priority=6,+ feature_checks=[gptqmodel_marlin_feature_check],+ alias=["marlin", "gptqmodel"],+ requirements=["gptqmodel>=2.0"],+)++BackendInfos["gptqmodel:marlin_zp"] = BackendInfo(+ device=["cuda"],+ sym=[True],+ packing_format="int32_zp",+ bits=[4, 8],+ group_size=[-1, 32, 64, 128],+ dtype=["float16", "bfloat16"],+ priority=6,+ feature_checks=[gptqmodel_marlin_feature_check],+ alias=["marlin", "gptqmodel"],+ requirements=["gptqmodel>=2.0"],+)++BackendInfos["gptqmodel:exllamav2"] = BackendInfo(+ device=["cuda"],+ sym=[True, False],+ packing_format="int32",+ bits=[4],+ group_size=[-1, 32, 64, 128], ##16 seems has accuracy issue+ dtype=["float16", "bfloat16"],+ priority=5,+ feature_checks=[exllamav2_feature_check],+ alias=["exllamav2"],+ requirements=["gptqmodel>=2.0"],+)++BackendInfos["auto_awq:gemm"] = BackendInfo(+ device=["cuda"],+ sym=[True, False], ##actually is gemm+ packing_format="awq",+ bits=[4],+ group_size=None,+ priority=5,+ dtype=["float16"],+ alias=["auto_awq:gemm", "awq", "awq:gemm", "auto_awq"],+ requirements=["autoawq"],+)++BackendInfos["qbits"] = BackendInfo(+ device=["cpu"],+ sym=[True, False],+ packing_format="qbits",+ bits=[2, 4, 8],+ group_size=None,+ priority=1,+ feature_checks=[],+ alias=["itrex", "qbits"],+ dtype=["float16", "bfloat16"],+ convertable_format=["int32"],+ requirements=["intel-extension-for-transformers", "torch<2.7.0"],+)++BackendInfos["qbits_zp"] = BackendInfo(+ device=["cpu"],+ sym=[True, False],+ packing_format="qbits_zp",+ bits=[2, 4, 8],+ group_size=None,+ dtype=["float16", "bfloat16"],+ priority=1,+ feature_checks=[],+ alias=["itrex", "qbits"],+ convertable_format=["int32_zp"],+ requirements=["intel-extension-for-transformers", "torch<2.7.0"],+)++BackendInfos["auto_round:qbits_awq"] = BackendInfo(+ device=["cpu"],+ sym=[True, False], ## for awq, not robust+ packing_format="awq",+ bits=[2, 4, 8],+ group_size=None,+ dtype=["float16", "bfloat16"],+ priority=1,+ feature_checks=[],+ alias=["itrex", "qbits"],+ requirements=["intel-extension-for-transformers", "torch<2.7.0"],+)++BackendInfos["ipex_gptq"] = BackendInfo(+ device=["cpu", "xpu"],+ sym=[True, False],+ packing_format="ipex_gptq",+ bits=[4],+ group_size=None,+ priority=5,+ feature_checks=[],+ dtype=["float16", "bfloat16"],+ convertable_format=["int32_zp"],+ alias=["ipex"],+ requirements=["intel-extension-for-pytorch>=2.5"],+)++BackendInfos["ipex_awq"] = BackendInfo(+ device=["cpu", "xpu"],+ sym=[True, False],+ packing_format="ipex_awq",+ bits=[4],+ group_size=None,+ priority=1,+ dtype=["float16", "bfloat16"],+ feature_checks=[],+ alias=["ipex"],+ convertable_format=["awq"],+ requirements=["intel-extension-for-pytorch>=2.6"],+)++BackendInfos["hpu"] = BackendInfo(+ device=["hpu"],+ sym=[True, False],+ packing_format="hpu",+ bits=[4],+ dtype=["bfloat16"],+ alias=["hpu"],+ priority=0,+ convertable_format=["int32"],+)++BackendInfos["hpu_zp"] = BackendInfo(+ device=["hpu"],+ sym=[True, False],+ packing_format="hpu_zp",+ bits=[4],+ dtype=["bfloat16"],+ alias=["hpu"],+ priority=0,+ convertable_format=["int32_zp"],+)+++def check_compatible(+ backend_name, device, bits, group_size, sym, packing_format, in_features, out_features, check_requirements=True+):"""Checks if the given configuration is compatible with the specified backend.Args:@@ -382,19 +445,24 @@ def dynamic_import_inference_linear(backend, bits, group_size, sym):)if "zp" in backend:import auto_round_extension.qbits.qlinear_qbits_gptq as qlinear_qbits_gptq+return qlinear_qbits_gptq.QuantLinearelif "awq" in backend:import auto_round_extension.qbits.qbits_awq as qlinear_qbits_awq+return qlinear_qbits_awq.QuantLinearelse: # auto_round must be at the endimport auto_round_extension.qbits.qlinear_qbits as qlinear_qbits_autoround+return qlinear_qbits_autoround.QuantLinearif "ipex_gptq" in backend:from auto_round_extension.ipex.qlinear_ipex_gptq import QuantLinear+return QuantLinearif "ipex_awq" in backend:from auto_round_extension.ipex.qlinear_ipex_awq import QuantLinear+return QuantLinearif "hpu" in backend:@@ -405,9 +473,11 @@ def dynamic_import_inference_linear(backend, bits, group_size, sym):if "zp" in backend:from auto_round_extension.hpu.qlinear_hpu_gptq import QuantLinear as QuantLinear_gptq+return QuantLinear_gptqelse: # auto_round must be at the endfrom auto_round_extension.hpu.qlinear_hpu import QuantLinear+return QuantLinearif "gptqmodel" in backend:return get_gptqmodel_infer_linear(backend, bits, group_size, sym)@@ -420,23 +490,28 @@ def dynamic_import_inference_linear(backend, bits, group_size, sym):from awq.modules.linear import WQLinear_GEMM # pylint: disable=E0401except ImportError:raise ImportError(- "autoawq is required. Please install it by 'pip install autoawq' to support auto_awq format.")+ "autoawq is required. Please install it by 'pip install autoawq' to support auto_awq format."+ )return WQLinear_GEMMif backend == "auto_round:tritonv2":from auto_round_extension.triton.qlinear_tritonv2 import QuantLinear+return QuantLinearif backend == "auto_round:tritonv2_zp":from auto_round_extension.triton.qlinear_tritonv2_zp import QuantLinear+return QuantLinearif backend == "auto_round:torch":from auto_round_extension.torch.qlinear_torch import QuantLinear+return QuantLinearif backend == "auto_round:torch_zp":from auto_round_extension.torch.qlinear_torch_zp import QuantLinear+return QuantLinearraise ValueError(f"unsupported backend {backend}, please set it to `auto` and retry")@@ -444,6 +519,7 @@ def dynamic_import_inference_linear(backend, bits, group_size, sym):def get_gptqmodel_infer_linear(backend, bits=4, group_size=128, sym=False):import gptqmodel # pylint: disable=E0401+if "marlin" in backend:return auto_round_extension.cuda.gptqmodel_marlin.get_marlin_layer()# return gptqmodel.nn_modules.qlinear.marlin.MarlinQuantLinear@@ -513,6 +589,7 @@ def get_autogptq_infer_linear(backend, bits=4, group_size=128, sym=False):disable_exllamav1 = Truefrom auto_gptq.utils.import_utils import dynamically_import_QuantLinear # pylint: disable=E0401+version = get_library_version("auto_gptq")from packaging.version import Version@@ -526,7 +603,7 @@ def get_autogptq_infer_linear(backend, bits=4, group_size=128, sym=False):disable_exllama=disable_exllamav1,disable_exllamav2=disable_exllamav2,use_qigen=use_qigen,- disable_marlin=disable_marlin+ disable_marlin=disable_marlin,)else:QuantLinear = dynamically_import_QuantLinear( # pylint: disable=E1123@@ -538,7 +615,7 @@ def get_autogptq_infer_linear(backend, bits=4, group_size=128, sym=False):disable_exllamav2=disable_exllamav2,use_qigen=use_qigen,use_marlin=not disable_marlin,- use_tritonv2=use_tritonv2+ use_tritonv2=use_tritonv2,)return QuantLinear@@ -556,7 +633,8 @@ def find_backend(target_backend: str, orig_backend: str = None):str or None: Matching backend key if found and compatible; otherwise, None."""matched_keys = [- key for key, info in BackendInfos.items()+ key+ for key, info in BackendInfos.items()if key == target_backend or (info.alias and target_backend in info.alias)]@@ -570,13 +648,14 @@ def find_backend(target_backend: str, orig_backend: str = None):for key in matched_keys:target_info = BackendInfos[key]- if (target_info.packing_format == orig_info.packing_format or- orig_info.packing_format in target_info.convertable_format):+ if (+ target_info.packing_format == orig_info.packing_format+ or orig_info.packing_format in target_info.convertable_format+ ):return keyraise ValueError(- f"{target_backend} is not compatible with {orig_backend}. "- f"Please set `backend` to `auto` and retry."+ f"{target_backend} is not compatible with {orig_backend}. " f"Please set `backend` to `auto` and retry.")@@ -586,9 +665,11 @@ def get_all_compatible_backend(device, backend, orig_backend, bits, group_size,# Find compatible backendscompatible_backends = [- key for key in BackendInfos.keys()- if check_compatible(key, device, bits, group_size, sym, packing_format, in_features, out_features,- check_requirements=False)+ key+ for key in BackendInfos.keys()+ if check_compatible(+ key, device, bits, group_size, sym, packing_format, in_features, out_features, check_requirements=False+ )]# Return the first compatible backend or an empty list if none found@@ -632,9 +713,7 @@ def get_layer_backend(device, backend, orig_backend, bits, group_size, sym, in_f# Check if the provided backend is in BackendInfosbackend = find_backend(backend)if backend not in BackendInfos.keys():- raise ValueError(- f"Unsupported backend '{backend}'. Please set it to 'auto' to enable automatic selection."- )+ raise ValueError(f"Unsupported backend '{backend}'. Please set it to 'auto' to enable automatic selection.")packing_format = BackendInfos[orig_backend].packing_format@@ -646,22 +725,25 @@ def get_layer_backend(device, backend, orig_backend, bits, group_size, sym, in_f# Raise an error if no compatible backends are foundif len(supported_backends) == 0:- supported_backends_need_package = get_all_compatible_backend(device, backend, orig_backend, bits, group_size,- sym,- in_features, out_features)+ supported_backends_need_package = get_all_compatible_backend(+ device, backend, orig_backend, bits, group_size, sym, in_features, out_features+ )if len(supported_backends_need_package) > 0:- supported_backends_need_package = sorted(supported_backends_need_package,- key=lambda support_backend: BackendInfos[support_backend].priority,- reverse=True)+ supported_backends_need_package = sorted(+ supported_backends_need_package,+ key=lambda support_backend: BackendInfos[support_backend].priority,+ reverse=True,+ )backend_info = BackendInfos[supported_backends_need_package[0]]process_requirement(backend_info.requirements, target_device=device)return None# Sort the compatible backends by priority and return the one with the highest priority- supported_backends = sorted(supported_backends, key=lambda support_backend: BackendInfos[support_backend].priority,- reverse=True)+ supported_backends = sorted(+ supported_backends, key=lambda support_backend: BackendInfos[support_backend].priority, reverse=True+ )return supported_backends[0]@@ -695,9 +777,9 @@ def get_highest_priority_backend(bits, sym, group_size, device, packing_format):if len(supported_backends) > 0:- supported_backends = sorted(supported_backends,- key=lambda support_backend: BackendInfos[support_backend].priority,- reverse=True)+ supported_backends = sorted(+ supported_backends, key=lambda support_backend: BackendInfos[support_backend].priority, reverse=True+ )return supported_backends[0]else:return None@@ -764,5 +846,3 @@ def process_requirement(requirements: list, target_device="cuda", logger_level="log(joined_cmds)if logger_level == "error":exit(-1)--diff --git a/auto_round/inference/convert_model.py b/auto_round/inference/convert_model.pyindex 6004e8acc..bd6dde836 100644--- a/auto_round/inference/convert_model.py+++ b/auto_round/inference/convert_model.py@@ -124,10 +124,10 @@ def get_keys_to_not_convert(model):def _get_modules_to_not_convert(- model,- skip_modules=None,- keep_in_fp32_modules=None,- add_default_skips: bool = False,+ model,+ skip_modules=None,+ keep_in_fp32_modules=None,+ add_default_skips: bool = False,):if skip_modules is None or add_default_skips:modules_to_not_convert = get_keys_to_not_convert(model)@@ -189,7 +189,7 @@ def parse_target_device_and_backend(target_backend: str):def remove_device_prefix(backend: str, prefix: str) -> str:"""Removes the given device prefix from the backend string, if present."""- return backend[len(prefix):].lstrip(":") if backend.startswith(prefix) else backend+ return backend[len(prefix) :].lstrip(":") if backend.startswith(prefix) else backend# Handle "auto" case explicitlyif target_backend == "auto":@@ -255,7 +255,7 @@ def get_layer_config(model, quantization_config):quant_block_list = find_matching_blocks(model, all_blocks, to_quant_block_names)##speed up the matchingfor i in range(len(quant_block_list)):- quant_block_list[i] = os.path.commonprefix(quant_block_list[i]).rstrip('.')+ quant_block_list[i] = os.path.commonprefix(quant_block_list[i]).rstrip(".")# Get layer names that will be quantizedlayer_names = []@@ -271,7 +271,8 @@ def get_layer_config(model, quantization_config):# Process GPTQ format: identify modules that should be quantizedif getattr(quantization_config, "modules_in_block_to_quantize", None):modules_in_block_to_quantize = flatten_list(- quantization_config.modules_in_block_to_quantize) # Flatten the list+ quantization_config.modules_in_block_to_quantize+ ) # Flatten the listfor layer_name in layer_names:if not any([re.search(re.compile(n), layer_name) is not None for n in modules_in_block_to_quantize]):extra_config[layer_name] = {"bits": 16} # Default to 16-bit for unquantized layers@@ -326,8 +327,7 @@ def _replace_by_quant_layers(module: nn.Module, layer_configs, target_backend, tlayer_backend_must = find_backend(layer_backend, orig_backend)used_backends.append(layer_backend_must)if layer_backend_must is None:- raise ValueError(- f"{target_backend} is not compatible, please change backend to `auto` and retry")+ raise ValueError(f"{target_backend} is not compatible, please change backend to `auto` and retry")devices = BackendInfos[layer_backend_must].deviceif target_device not in devices:raise ValueError(f"{target_backend} does not support {target_device}, please change device or backend")@@ -358,8 +358,9 @@ def _replace_by_quant_layers(module: nn.Module, layer_configs, target_backend, tused_backends.append(layer_backend)else:# Determine backend- layer_backend = _get_layer_backend(target_device, target_backend, orig_backend, config,- in_features, out_features)+ layer_backend = _get_layer_backend(+ target_device, target_backend, orig_backend, config, in_features, out_features+ )backend_cache[key] = layer_backendif not layer_backend:@@ -383,12 +384,19 @@ def _get_layer_features(layer):return None, None # Unsupported layer type-def _get_layer_backend(target_device, target_backend, orig_backend, config, in_features,- out_features):+def _get_layer_backend(target_device, target_backend, orig_backend, config, in_features, out_features):"""Determines the best backend for a given layer."""- backend = get_layer_backend(target_device, target_backend, orig_backend, config["bits"], config["group_size"],- config["sym"], in_features, out_features)+ backend = get_layer_backend(+ target_device,+ target_backend,+ orig_backend,+ config["bits"],+ config["group_size"],+ config["sym"],+ in_features,+ out_features,+ )return backend@@ -412,21 +420,38 @@ def _create_quant_layer(layer, layer_backend, config, in_features, out_features)# Special handling for AWQ layersfrom auto_round_extension.qbits.qbits_awq import QuantLinear as QBitsAWQQuantLinear+if "awq" in layer_backend and isinstance(QuantLinear, QBitsAWQQuantLinear):- return QuantLinear.from_linear(layer, config["bits"], config["group_size"], init_only=True,- has_zero_points=not config["sym"])+ return QuantLinear.from_linear(+ layer, config["bits"], config["group_size"], init_only=True, has_zero_points=not config["sym"]+ )elif "awq" in layer_backend:return QuantLinear.from_linear(layer, config["bits"], config["group_size"], init_only=True)elif "gptqmodel" in layer_backend:- return QuantLinear(bits=config["bits"], group_size=config["group_size"], desc_act=False, sym=config["sym"],- in_features=in_features, out_features=out_features, bias=bias)+ return QuantLinear(+ bits=config["bits"],+ group_size=config["group_size"],+ desc_act=False,+ sym=config["sym"],+ in_features=in_features,+ out_features=out_features,+ bias=bias,+ )# Default quantized layer creationtry:- return QuantLinear(config["bits"], config["group_size"], in_features, out_features, bias,- weight_dtype=layer.weight.dtype, clip=config["clip"])+ return QuantLinear(+ config["bits"],+ config["group_size"],+ in_features,+ out_features,+ bias,+ weight_dtype=layer.weight.dtype,+ clip=config["clip"],+ )except: # Handle cases where `clip` is not a valid argument- return QuantLinear(config["bits"], config["group_size"], in_features, out_features, bias,- weight_dtype=layer.weight.dtype)+ return QuantLinear(+ config["bits"], config["group_size"], in_features, out_features, bias, weight_dtype=layer.weight.dtype+ )def infer_target_device(device_map=None):@@ -464,9 +489,11 @@ def post_init(model, used_backends):need_ipex_itrex_init = Trueif need_autogptq_init:from auto_gptq.modeling._utils import autogptq_post_init as gptq_post_init # pylint: disable=E0401+model = gptq_post_init(model, use_act_order=False)if need_gptqmodel_init:from gptqmodel.utils.model import hf_gptqmodel_post_init as gptq_post_init # pylint: disable=E0401+model = gptq_post_init(model, use_act_order=False)if need_ipex_itrex_init:message = "repacking to CPU/XPU format"@@ -475,8 +502,7 @@ def post_init(model, used_backends):if hasattr(m, "QUANT_TYPE") and ("qbits" in m.QUANT_TYPE or "ipex" in m.QUANT_TYPE):layers.append(m)- for layer in tqdm(layers, desc=message, total=len(layers),- leave=True):+ for layer in tqdm(layers, desc=message, total=len(layers), leave=True):layer.post_init()if used_gptq_exllamav2:@@ -491,7 +517,7 @@ def post_init(model, used_backends):for l in data_types[1:]:common &= set(l)common = list(common)- if len(common) > 0 and str(model.dtype).split('.')[-1] not in common:+ if len(common) > 0 and str(model.dtype).split(".")[-1] not in common:if common[0] == "float16":model = model.to(torch.float16)logger.warning("force model to float16")@@ -525,9 +551,11 @@ def convert_hf_model(model: nn.Module, target_device="cpu"):if hasattr(quantization_config, "desc_act") and quantization_config.desc_act:##check static_groupif (hasattr(quantization_config, "static_groups") and not quantization_config.static_groups) or (- not hasattr(quantization_config, "static_groups")):+ not hasattr(quantization_config, "static_groups")+ ):raise NotImplementedError(- "This GPTQ model may contain a non-dummy g_idx, which is not yet supported by AutoRound")+ "This GPTQ model may contain a non-dummy g_idx, which is not yet supported by AutoRound"+ )if hasattr(quantization_config, "backend"):backend = quantization_config.backend@@ -537,10 +565,11 @@ def convert_hf_model(model: nn.Module, target_device="cpu"):##target_backend could be None_, backend = parse_target_device_and_backend(backend)- if hasattr(quantization_config,- "packing_format") and "auto-round" in quantization_config.quant_method: # pragma: no cover+ if (+ hasattr(quantization_config, "packing_format") and "auto-round" in quantization_config.quant_method+ ): # pragma: no coverpacking_format = quantization_config.packing_format- elif 'gptq' in quantization_config.quant_method: # pragma: no cover+ elif "gptq" in quantization_config.quant_method: # pragma: no coverpacking_format = "auto_gptq"elif "awq" in quantization_config.quant_method:packing_format = "auto_awq"@@ -552,18 +581,22 @@ def convert_hf_model(model: nn.Module, target_device="cpu"):layer_configs = get_layer_config(model, quantization_config)if packing_format.startswith("auto_round:") and ("gptq" in packing_format or "awq" in packing_format):- packing_format = packing_format[len("auto_round:"):]+ packing_format = packing_format[len("auto_round:") :]orig_backend = find_backend(packing_format)if backend.startswith("auto_round:") and ("gptq" in packing_format or "awq" in packing_format):- backend = backend[len("auto_round:"):]+ backend = backend[len("auto_round:") :]used_backends = _replace_by_quant_layers(model, layer_configs, backend, target_device, orig_backend)if backend == "auto" or backend == "":- best_backend = get_highest_priority_backend(quantization_config.bits, quantization_config.sym,- quantization_config.group_size, target_device,- BackendInfos[orig_backend].packing_format)+ best_backend = get_highest_priority_backend(+ quantization_config.bits,+ quantization_config.sym,+ quantization_config.group_size,+ target_device,+ BackendInfos[orig_backend].packing_format,+ )if best_backend is not None and best_backend not in used_backends:requirements = BackendInfos[best_backend].requirementsprocess_requirement(requirements, target_device, "warning")diff --git a/auto_round/low_cpu_mem/modified_pickle.py b/auto_round/low_cpu_mem/modified_pickle.pyindex 6c0b87add..22e6ab5c0 100644--- a/auto_round/low_cpu_mem/modified_pickle.py+++ b/auto_round/low_cpu_mem/modified_pickle.py@@ -1837,4 +1837,4 @@ dump, dumps, load, loads = _dump, _dumps, _load, _loadsdef _test(): # pragma: no coverimport doctest- return doctest.testmod()\ No newline at end of file+ return doctest.testmod()diff --git a/auto_round/low_cpu_mem/utils.py b/auto_round/low_cpu_mem/utils.pyindex 715556f0f..a3e0e6b4a 100644--- a/auto_round/low_cpu_mem/utils.py+++ b/auto_round/low_cpu_mem/utils.py@@ -175,6 +175,7 @@ def get_layers_before_block(model):"""return_layers = []block_name = None+def _forward(module, name, *args, **kwargs):if name == block_name:raise NotImplementedError@@ -184,19 +185,19 @@ def get_layers_before_block(model):for n, m in model.named_modules():if isinstance(m, torch.nn.ModuleList):- block_name = n + '.' + m.named_children().__next__()[0]+ block_name = n + "." + m.named_children().__next__()[0]m.ori_forward = m.forwardm.forward = partial(_forward, m, n)-+try:- if model.device.type == 'meta':- target_device = 'cpu'+ if model.device.type == "meta":+ target_device = "cpu"else:target_device = model.deviceinput = {- "input_ids": torch.zeros((1,1), device=target_device, dtype=torch.int),- "attention_mask": torch.zeros((1,1), device=target_device, dtype=torch.int)- }+ "input_ids": torch.zeros((1, 1), device=target_device, dtype=torch.int),+ "attention_mask": torch.zeros((1, 1), device=target_device, dtype=torch.int),+ }model.forward(**input)except NotImplementedError:pass@@ -204,9 +205,8 @@ def get_layers_before_block(model):for n, m in model.named_modules():m.forward = m.ori_forwarddel m.ori_forward-- return return_layers+ return return_layersdef load_layer_wise_quantized_model(path): # pragma: no cover@@ -264,7 +264,7 @@ def _get_path(pretrained_model_name_or_path):def load_value(model, param_name, path):- logger.debug(f'load value for layer: {param_name}')+ logger.debug(f"load value for layer: {param_name}")if "lm_head" in param_name and getattr(model.config, "tie_word_embeddings", True):input_embeddings = model.get_input_embeddings()modules = get_named_children(model)@@ -305,7 +305,7 @@ def register_weight_hooks(model, path, device="cpu", clean_weight=True, saved_pavalue = load_value(model, param_name, path)set_module_tensor_to_device(model, param_name, device, value)module = module.to(device)-+return hookdef forward_hook(name):@@ -345,6 +345,7 @@ def clean_module_weight(submodule): # pragma: no coverdef convert_model(empty_model, saved_path=LWQ_WORKSPACE):os.makedirs(saved_path, exist_ok=True)+def _get_value(name, n):state_dict = Noneif os.path.exists(os.path.join(saved_path, f"{name}.pt")):@@ -361,14 +362,14 @@ def convert_model(empty_model, saved_path=LWQ_WORKSPACE):if os.path.exists(os.path.join(saved_path, f"{name}.pt")):state_dict = torch.load(os.path.join(saved_path, f"{name}.pt"))for n, p in module.named_parameters():- if str(p.device) != 'meta':+ if str(p.device) != "meta":continueparam_name = name + "." + nif state_dict:value = state_dict[n]else:value = load_value(empty_model, param_name, saved_path)- set_module_tensor_to_device(empty_model, param_name, 'cpu', value)+ set_module_tensor_to_device(empty_model, param_name, "cpu", value)file_path = os.path.join(saved_path, f"{name}.pt")torch.save(module.state_dict(), file_path)@@ -377,9 +378,9 @@ def convert_model(empty_model, saved_path=LWQ_WORKSPACE):if hasattr(module, "scale"):quant_info["scale"] = module.scaleif hasattr(module, "zp"):- quant_info["zp"]= module.zp+ quant_info["zp"] = module.zplogger.debug(f"save quant info for layer: {name}")- f = open(os.path.join(saved_path, f"{name}_quant_info.pkl"), 'wb')+ f = open(os.path.join(saved_path, f"{name}_quant_info.pkl"), "wb")pickle.dump(quant_info, f)f.close()@@ -388,19 +389,19 @@ def convert_model(empty_model, saved_path=LWQ_WORKSPACE):return module.ori_to(device_or_dtype)elif len(module._modules) == 0:# skip method type- if len(module._parameters) == 0 or module.weight.device.type != 'meta':+ if len(module._parameters) == 0 or module.weight.device.type != "meta":return module.ori_to(device_or_dtype)else:for n, _ in module.named_parameters():param_name = name + "." + nvalue = load_value(empty_model, param_name, empty_model.path)dtype = None- if hasattr(module, 'dtype'):+ if hasattr(module, "dtype"):dtype = module.dtypeset_module_tensor_to_device(module, n, device_or_dtype, value, dtype=dtype)if hasattr(module, "scale"):- f = open(os.path.join(saved_path, f"{name}_quant_info.pkl"), 'rb')+ f = open(os.path.join(saved_path, f"{name}_quant_info.pkl"), "rb")quant_info = pickle.load(f)f.close()module.scale = quant_info["scale"].to(device_or_dtype)@@ -414,31 +415,29 @@ def convert_model(empty_model, saved_path=LWQ_WORKSPACE):modules = get_named_children(empty_model)for name, module in modules:- if hasattr(module, 'weight'):+ if hasattr(module, "weight"):# delattr(module, 'weight')# module.weight = partial(_get_value, name, 'weight')()- module.get_weight = partial(_get_value, name, 'weight')- if hasattr(module, 'bias') and module.bias is not None:- module.get_bias = partial(_get_value, name, 'bias')+ module.get_weight = partial(_get_value, name, "weight")+ if hasattr(module, "bias") and module.bias is not None:+ module.get_bias = partial(_get_value, name, "bias")module.update = partial(_update, name, module)-+def _repalce_to(module, name):if len(module._modules) > 0:for n, m in module.named_children():if len(name) > 0:- n = name + '.' + n+ n = name + "." + n_repalce_to(m, n)module.ori_to = module.tomodule.to = partial(_layer_wise_to, module, name)- _repalce_to(empty_model, '')++ _repalce_to(empty_model, "")+def load_model_with_hooks(- pretrained_model_name_or_path,- cls=AutoModelForCausalLM,- device=None,- clean_weight=True,- saved_path=None,- **kwargs):+ pretrained_model_name_or_path, cls=AutoModelForCausalLM, device=None, clean_weight=True, saved_path=None, **kwargs+):if saved_path is None:logger.warning(f"saved_path is not set, use default working space: {LWQ_WORKSPACE}")saved_path = LWQ_WORKSPACE@@ -450,9 +449,9 @@ def load_model_with_hooks(def layer_wise_save(model, path):os.makedirs(path, exist_ok=True)- file_path = os.path.join(path, 'layer_wise_model.bin')+ file_path = os.path.join(path, "layer_wise_model.bin")modules = get_named_children(model)- with open(file_path, 'wb') as f:+ with open(file_path, "wb") as f:for name, module in modules:output = OrderedDict()if hasattr(module, "get_weight"):@@ -460,16 +459,16 @@ def layer_wise_save(model, path):if hasattr(module, "get_bias"):output[f"{name}.bias"] = module.get_bias()output = pickle.dumps(output)- f.write(output + b'split_tag')+ f.write(output + b"split_tag")+def layer_wise_load(path):- file_path = os.path.join(path, 'layer_wise_model.bin')+ file_path = os.path.join(path, "layer_wise_model.bin")state_dict = OrderedDict()- with open(file_path, 'rb') as f:- data = f.read().split(b'split_tag')+ with open(file_path, "rb") as f:+ data = f.read().split(b"split_tag")for d in data:if len(d) > 0:d = pickle.loads(d)state_dict.update(d)return state_dict-diff --git a/auto_round/mllm/README.md b/auto_round/mllm/README.mdindex 58e511a34..f33d8309b 100644--- a/auto_round/mllm/README.md+++ b/auto_round/mllm/README.md@@ -17,8 +17,7 @@ from transformers import Qwen2VLForConditionalGeneration, AutoProcessor, AutoTok## load the modelmodel_name = "Qwen/Qwen2-VL-2B-Instruct"-model = Qwen2VLForConditionalGeneration.from_pretrained(- model_name, trust_remote_code=True)+model = Qwen2VLForConditionalGeneration.from_pretrained(model_name, trust_remote_code=True)tokenizer = AutoTokenizer.from_pretrained(model_name)processor = AutoProcessor.from_pretrained(model_name, trust_remote_code=True)@@ -29,7 +28,7 @@ autoround.quantize()# save the quantized model, set format='auto_gptq' to use AutoGPTQ formatoutput_dir = "./tmp_autoround"-autoround.save_quantized(output_dir, format='auto_round', inplace=True)+autoround.save_quantized(output_dir, format="auto_round", inplace=True)```- `dataset`: the dataset for quantization training. Currently only support NeelNanda/pile-10k, llava_conv_58k,diff --git a/auto_round/mllm/__init__.py b/auto_round/mllm/__init__.pyindex 41858319b..d77883b7e 100644--- a/auto_round/mllm/__init__.py+++ b/auto_round/mllm/__init__.py@@ -16,4 +16,4 @@ from .mllm_dataset import get_mllm_dataloaderfrom .template import Template, get_template, TEMPLATESfrom .autoround_mllm import AutoRoundMLLMfrom ..utils import LazyImport-from .eval import mllm_eval, lmms_eval\ No newline at end of file+from .eval import mllm_eval, lmms_evaldiff --git a/auto_round/mllm/autoround_mllm.py b/auto_round/mllm/autoround_mllm.pyindex ed6d06611..08cd1f1bb 100644--- a/auto_round/mllm/autoround_mllm.py+++ b/auto_round/mllm/autoround_mllm.py@@ -46,7 +46,7 @@ def _only_text_test(model, tokenizer, device, model_type):new_tokenizer = deepcopy(tokenizer)device = detect_device(device)text = ["only text", "test"]- new_tokenizer.padding_side = 'left'+ new_tokenizer.padding_side = "left"if new_tokenizer.pad_token is None:new_tokenizer.pad_token = new_tokenizer.eos_tokeninputs = new_tokenizer(text, return_tensors="pt", padding=True, truncation=True)@@ -71,7 +71,7 @@ def _only_text_test(model, tokenizer, device, model_type):class AutoRoundMLLM(AutoRound):"""Class for automatic rounding-based quantization with MLLMs.-+Args:model: The PyTorch model to be quantized.tokenizer: An optional tokenizer for processing input data.@@ -112,7 +112,7 @@ class AutoRoundMLLM(AutoRound):act_group_size (int): Group size for activation quantization. Default is None.act_sym (bool): Whether to use symmetric activation quantization. Default is None.act_dynamic (bool): Whether to use dynamic activation quantization. Default is True.- to_quant_block_names (str|list): A string or list whose elements are list of+ to_quant_block_names (str|list): A string or list whose elements are list ofblock's layer names to be quantized.enable_torch_compile (bool): Whether to enable torch compile to optimize quant_block/layer**kwargs: Additional keyword arguments.@@ -121,50 +121,50 @@ class AutoRoundMLLM(AutoRound):"""def __init__(- self,- model: torch.nn.Module,- tokenizer,- processor = None,- image_processor = None,- bits: int = 4,- group_size: int = 128,- sym: bool = True,- layer_config: dict = None,- batch_size: int = 8,- amp: bool = True,- device: str = None,- lr_scheduler=None,- dataset: Union[str, list, tuple, torch.utils.data.DataLoader] = None,- extra_data_dir: str = None,- template: Union[str, Template] = None,- quant_nontext_module: bool = False,- enable_quanted_input: bool = True,- enable_minmax_tuning: bool = True,- lr: float = None,- minmax_lr: float = None,- low_gpu_mem_usage: bool = False,- low_cpu_mem_usage: bool = False,- iters: int = 200,- seqlen: int = None,- nsamples: int = 128,- sampler: str = "rand",- seed: int = 42,- nblocks: int = 1,- gradient_accumulate_steps: int = 1,- not_use_best_mse: bool = False,- dynamic_max_gap: int = -1,- data_type: str = "int",- scale_dtype: str = "fp16",- act_bits: int = 32,- act_group_size: int = None,- act_sym: bool = None,- act_dynamic: bool = True,- to_quant_block_names: Union[str, list] = None,- enable_norm_bias_tuning: bool = False,- truncation: bool = None,- enable_torch_compile: bool = False,- model_kwargs: dict = None,- **kwargs,+ self,+ model: torch.nn.Module,+ tokenizer,+ processor=None,+ image_processor=None,+ bits: int = 4,+ group_size: int = 128,+ sym: bool = True,+ layer_config: dict = None,+ batch_size: int = 8,+ amp: bool = True,+ device: str = None,+ lr_scheduler=None,+ dataset: Union[str, list, tuple, torch.utils.data.DataLoader] = None,+ extra_data_dir: str = None,+ template: Union[str, Template] = None,+ quant_nontext_module: bool = False,+ enable_quanted_input: bool = True,+ enable_minmax_tuning: bool = True,+ lr: float = None,+ minmax_lr: float = None,+ low_gpu_mem_usage: bool = False,+ low_cpu_mem_usage: bool = False,+ iters: int = 200,+ seqlen: int = None,+ nsamples: int = 128,+ sampler: str = "rand",+ seed: int = 42,+ nblocks: int = 1,+ gradient_accumulate_steps: int = 1,+ not_use_best_mse: bool = False,+ dynamic_max_gap: int = -1,+ data_type: str = "int",+ scale_dtype: str = "fp16",+ act_bits: int = 32,+ act_group_size: int = None,+ act_sym: bool = None,+ act_dynamic: bool = True,+ to_quant_block_names: Union[str, list] = None,+ enable_norm_bias_tuning: bool = False,+ truncation: bool = None,+ enable_torch_compile: bool = False,+ model_kwargs: dict = None,+ **kwargs,):all_blocks = get_block_names(model, quant_nontext_module)self.quant_block_list = find_matching_blocks(model, all_blocks, to_quant_block_names)@@ -176,6 +176,7 @@ class AutoRoundMLLM(AutoRound):self.processor = processorself.image_processor = image_processorfrom transformers import PreTrainedModel+if model.config.model_type == "llava" and isinstance(model, PreTrainedModel):template = "default"self.template = template if template is not None else model.config.model_type@@ -186,23 +187,30 @@ class AutoRoundMLLM(AutoRound):tokenizer=tokenizer,processor=processor,image_processor=image_processor,- use_rtn=iters == 0)+ use_rtn=iters == 0,+ )dataset = self.template.default_dataset if dataset is None else datasetmodel = _handle_special_model(model)from ..calib_dataset import CALIB_DATASETSfrom .mllm_dataset import MLLM_DATASET+if isinstance(dataset, str):- if quant_nontext_module or \- (dataset in CALIB_DATASETS.keys() and not \- _only_text_test(model, tokenizer, device, self.template.model_type)):+ if quant_nontext_module or (+ dataset in CALIB_DATASETS.keys()+ and not _only_text_test(model, tokenizer, device, self.template.model_type)+ ):if quant_nontext_module:- logger.warning("Text only dataset cannot be used for calibrating non-text modules,"- "switching to liuhaotian/llava_conv_58k")+ logger.warning(+ "Text only dataset cannot be used for calibrating non-text modules,"+ "switching to liuhaotian/llava_conv_58k"+ )else:- logger.warning(f"{model.config.model_type} not support for {dataset},"- " will use liuhaotian/llava_conv_58k with default config as an alternative.")+ logger.warning(+ f"{model.config.model_type} not support for {dataset},"+ " will use liuhaotian/llava_conv_58k with default config as an alternative."+ )dataset = "liuhaotian/llava_conv_58k"if dataset in MLLM_DATASET.keys():@@ -213,7 +221,8 @@ class AutoRoundMLLM(AutoRound):f"reset batch_size({batch_size}) to 1 and "f"gradient_accumulate_steps({gradient_accumulate_steps}) "f"to {batch_size * gradient_accumulate_steps}, "- f"because batch_size={batch_size} cannot be used for {dataset}")+ f"because batch_size={batch_size} cannot be used for {dataset}"+ )gradient_accumulate_steps = batch_size * gradient_accumulate_stepsbatch_size = 1if quant_nontext_module and batch_size != 1:@@ -221,7 +230,8 @@ class AutoRoundMLLM(AutoRound):f"reset batch_size({batch_size}) to 1 and "f"gradient_accumulate_steps({gradient_accumulate_steps}) "f"to {batch_size * gradient_accumulate_steps}, "- f"because batch_size={batch_size} cannot be used for calibrating non-text modules.")+ f"because batch_size={batch_size} cannot be used for calibrating non-text modules."+ )gradient_accumulate_steps = batch_size * gradient_accumulate_stepsbatch_size = 1seqlen = 2048 if seqlen is None else seqlen@@ -300,7 +310,7 @@ class AutoRoundMLLM(AutoRound):truncation=self.truncation,nsamples=self.nsamples,gradient_accumulate_steps=self.gradient_accumulate_steps,- quant_nontext_module=self.quant_nontext_module+ quant_nontext_module=self.quant_nontext_module,)else:self.dataloader = self.dataset@@ -336,7 +346,7 @@ class AutoRoundMLLM(AutoRound):data_new[key] = data[key].to(self.device)input_ids = data_new["input_ids"]elif isinstance(data, dict) and "text" in data.keys():- text = data['text']+ text = data["text"]if isinstance(text, dict):text = [text]input_text = self.template._encode(text)@@ -350,7 +360,7 @@ class AutoRoundMLLM(AutoRound):for key in data.keys():data_new[key] = torch.tensor(data[key])data_new[key] = to_device(data_new[key], self.model.device)- if key == 'images':+ if key == "images":data_new[key] = to_dtype(data_new[key], self.model.dtype)input_ids = data_new["input_ids"]elif isinstance(data, tuple) or isinstance(data, list):@@ -360,7 +370,7 @@ class AutoRoundMLLM(AutoRound):data_new = {}for key in data.keys():data_new[key] = to_device(data[key], self.model.device)- if key in ['images', 'pixel_values']:+ if key in ["images", "pixel_values"]:data_new[key] = to_dtype(data_new[key], self.model.dtype)if "input_ids" in data_new:input_ids = data_new["input_ids"]@@ -398,8 +408,10 @@ class AutoRoundMLLM(AutoRound):f"target samples count is {nsamples}, while valid samples count is {total_cnt}")if total_cnt < self.batch_size:- raise ValueError(f"valid samples is less than batch_size({self.batch_size}),"- " please adjust self.batch_size or seqlen.")+ raise ValueError(+ f"valid samples is less than batch_size({self.batch_size}),"+ " please adjust self.batch_size or seqlen."+ )max_len = (total_cnt // self.batch_size) * self.batch_sizefor k, v in self.inputs.items():for key in v:@@ -427,5 +439,6 @@ class AutoRoundMLLM(AutoRound):if self.processor is not None and not hasattr(self.processor, "chat_template"):self.processor.chat_template = Nonecompressed_model = super().save_quantized(- output_dir=output_dir, format=format, inplace=inplace, processor=self.processor, **kwargs)+ output_dir=output_dir, format=format, inplace=inplace, processor=self.processor, **kwargs+ )return compressed_modeldiff --git a/auto_round/mllm/eval.py b/auto_round/mllm/eval.pyindex dfe4aaa52..db2f7882e 100644--- a/auto_round/mllm/eval.py+++ b/auto_round/mllm/eval.py@@ -73,7 +73,6 @@ MODEL_TYPE_TO_VLMEVAL_MODEL = {"SliME": dict(cls="SliME"),"Eagle": dict(cls="Eagle"),"Molmo": dict(cls="molmo"),-# config.model_type"qwen2_vl": dict(cls="Qwen2VLChat", min_pixels=1280 * 28 * 28, max_pixels=16384 * 28 * 28),"qwen": dict(cls="QwenVL"),@@ -87,19 +86,19 @@ MODEL_TYPE_TO_VLMEVAL_MODEL = {def mllm_eval(- pretrained_model_name_or_path: str,- work_dir: str,- dataset: list,- data_store_dir: None,- pack: bool = False,- use_subtitle: bool = False,- fps: float = -1,- nframe: int = 8,- rerun: bool = False,- judge: bool = False,- verbose: bool = False,- mode: str = 'all',- ignore: bool = False+ pretrained_model_name_or_path: str,+ work_dir: str,+ dataset: list,+ data_store_dir: None,+ pack: bool = False,+ use_subtitle: bool = False,+ fps: float = -1,+ nframe: int = 8,+ rerun: bool = False,+ judge: bool = False,+ verbose: bool = False,+ mode: str = "all",+ ignore: bool = False,):try:from auto_round import AutoRoundConfig@@ -112,7 +111,7 @@ def mllm_eval(oldmask = os.umask(000)os.makedirs(data_store_dir, mode=0o777)os.umask(oldmask)- os.environ['LMUData'] = data_store_dir+ os.environ["LMUData"] = data_store_dirmodel_name = pretrained_model_name_or_pathif "/" in model_name:@@ -131,6 +130,7 @@ def mllm_eval(breakif model_type is None:from transformers import AutoConfig+config = AutoConfig.from_pretrained(pretrained_model_name_or_path, trust_remote_code=True)model_type = config.model_typeif "chat" in model_name.lower():@@ -148,52 +148,53 @@ def mllm_eval(os.makedirs(pred_root, exist_ok=True)st = time.time()- rt_file = open(f'{pred_root}/{model_name}_eval_cost.txt', 'w')+ rt_file = open(f"{pred_root}/{model_name}_eval_cost.txt", "w")for dataset_name in dataset:task_st = time.time()try:dataset_kwargs = {}- if dataset_name in ['MMLongBench_DOC', 'DUDE', 'DUDE_MINI', 'SLIDEVQA', 'SLIDEVQA_MINI']:- dataset_kwargs['model'] = model_name- if dataset_name == 'MMBench-Video':- dataset_kwargs['pack'] = pack- if dataset_name == 'Video-MME':- dataset_kwargs['use_subtitle'] = use_subtitle+ if dataset_name in ["MMLongBench_DOC", "DUDE", "DUDE_MINI", "SLIDEVQA", "SLIDEVQA_MINI"]:+ dataset_kwargs["model"] = model_name+ if dataset_name == "MMBench-Video":+ dataset_kwargs["pack"] = pack+ if dataset_name == "Video-MME":+ dataset_kwargs["use_subtitle"] = use_subtitledataset = vlmeval.dataset.build_dataset(dataset_name, **dataset_kwargs)if dataset is None:- logger.error(f'Dataset {dataset_name} is not valid, will be skipped. ')+ logger.error(f"Dataset {dataset_name} is not valid, will be skipped. ")continue- result_file = f'{pred_root}/{model_name}_{dataset_name}.xlsx'+ result_file = f"{pred_root}/{model_name}_{dataset_name}.xlsx"if fps > 0: # For Video Dataset, set the fps for priority- if dataset_name == 'MVBench':- raise ValueError('MVBench does not support fps setting, please transfer to MVBench_MP4!')+ if dataset_name == "MVBench":+ raise ValueError("MVBench does not support fps setting, please transfer to MVBench_MP4!")nframe = 0- if dataset_name in ['MMBench-Video']:- packstr = 'pack' if pack else 'nopack'+ if dataset_name in ["MMBench-Video"]:+ packstr = "pack" if pack else "nopack"if nframe > 0:- result_file = f'{pred_root}/{model_name}_{dataset_name}_{nframe}frame_{packstr}.xlsx'+ result_file = f"{pred_root}/{model_name}_{dataset_name}_{nframe}frame_{packstr}.xlsx"else:- result_file = f'{pred_root}/{model_name}_{dataset_name}_{fps}fps_{packstr}.xlsx'- elif dataset.MODALITY == 'VIDEO':+ result_file = f"{pred_root}/{model_name}_{dataset_name}_{fps}fps_{packstr}.xlsx"+ elif dataset.MODALITY == "VIDEO":if pack:- logger.info(f'{dataset_name} not support Pack Mode, directly change to unpack')+ logger.info(f"{dataset_name} not support Pack Mode, directly change to unpack")pack = False- packstr = 'pack' if pack else 'nopack'+ packstr = "pack" if pack else "nopack"if nframe > 0:- result_file = f'{pred_root}/{model_name}_{dataset_name}_{nframe}frame_{packstr}.xlsx'+ result_file = f"{pred_root}/{model_name}_{dataset_name}_{nframe}frame_{packstr}.xlsx"else:- result_file = f'{pred_root}/{model_name}_{dataset_name}_{fps}fps_{packstr}.xlsx'- if dataset_name in ['Video-MME']:- subtitlestr = 'subs' if use_subtitle else 'nosubs'- result_file = result_file.replace('.xlsx', f'_{subtitlestr}.xlsx')+ result_file = f"{pred_root}/{model_name}_{dataset_name}_{fps}fps_{packstr}.xlsx"+ if dataset_name in ["Video-MME"]:+ subtitlestr = "subs" if use_subtitle else "nosubs"+ result_file = result_file.replace(".xlsx", f"_{subtitlestr}.xlsx")- if dataset.TYPE == 'MT':- result_file = result_file.replace('.xlsx', '.tsv')+ if dataset.TYPE == "MT":+ result_file = result_file.replace(".xlsx", ".tsv")if os.path.exists(result_file) and rerun:import re+pattern = re.compile(f"{model_name}_{dataset_name}_[(openai)|(gpt)|(auxmatch)].*")for file_name in os.listdir(pred_root):if pattern.match(file_name):@@ -203,7 +204,7 @@ def mllm_eval(model = model_name # which is only a name# Perform the Inference- if dataset.MODALITY == 'VIDEO':+ if dataset.MODALITY == "VIDEO":model = vlmeval.inference_video.infer_data_job_video(model,work_dir=pred_root,@@ -213,15 +214,17 @@ def mllm_eval(pack=pack,verbose=verbose,subtitle=use_subtitle,- fps=fps)- elif dataset.TYPE == 'MT':+ fps=fps,+ )+ elif dataset.TYPE == "MT":model = vlmeval.inference_mt.infer_data_job_mt(model,work_dir=pred_root,model_name=model_name,dataset=dataset,verbose=verbose,- ignore_failed=ignore)+ ignore_failed=ignore,+ )else:model = vlmeval.inference.infer_data_job(model,@@ -229,84 +232,101 @@ def mllm_eval(model_name=model_name,dataset=dataset,verbose=verbose,- ignore_failed=ignore)+ ignore_failed=ignore,+ )# Set the judge kwargs first before evaluation or dumpingjudge_kwargs = {- 'verbose': verbose,+ "verbose": verbose,}if judge is not None:- judge_kwargs['model'] = judge+ judge_kwargs["model"] = judgeelse:- if dataset.TYPE in ['MCQ', 'Y/N'] or vlmeval.smp.listinstr(['MathVerse'], dataset_name):- judge_kwargs['model'] = 'chatgpt-0125'- elif vlmeval.smp.listinstr(['MMVet', 'MathVista', 'LLaVABench', 'MMBench-Video', 'MathVision'],- dataset_name):- judge_kwargs['model'] = 'gpt-4-turbo'- elif vlmeval.smp.listinstr(['MMLongBench', 'MMDU', 'DUDE', 'DUDE_MINI', 'SLIDEVQA', 'SLIDEVQA_MINI'],- dataset_name):- judge_kwargs['model'] = 'gpt-4o'- if 'OPENAI_API_KEY_JUDGE' in os.environ and len(os.environ['OPENAI_API_KEY_JUDGE']):- judge_kwargs['key'] = os.environ['OPENAI_API_KEY_JUDGE']- if 'OPENAI_API_BASE_JUDGE' in os.environ and len(os.environ['OPENAI_API_BASE_JUDGE']):- judge_kwargs['api_base'] = os.environ['OPENAI_API_BASE_JUDGE']-- if dataset_name in ['MMMU_TEST']:+ if dataset.TYPE in ["MCQ", "Y/N"] or vlmeval.smp.listinstr(["MathVerse"], dataset_name):+ judge_kwargs["model"] = "chatgpt-0125"+ elif vlmeval.smp.listinstr(+ ["MMVet", "MathVista", "LLaVABench", "MMBench-Video", "MathVision"], dataset_name+ ):+ judge_kwargs["model"] = "gpt-4-turbo"+ elif vlmeval.smp.listinstr(+ ["MMLongBench", "MMDU", "DUDE", "DUDE_MINI", "SLIDEVQA", "SLIDEVQA_MINI"], dataset_name+ ):+ judge_kwargs["model"] = "gpt-4o"+ if "OPENAI_API_KEY_JUDGE" in os.environ and len(os.environ["OPENAI_API_KEY_JUDGE"]):+ judge_kwargs["key"] = os.environ["OPENAI_API_KEY_JUDGE"]+ if "OPENAI_API_BASE_JUDGE" in os.environ and len(os.environ["OPENAI_API_BASE_JUDGE"]):+ judge_kwargs["api_base"] = os.environ["OPENAI_API_BASE_JUDGE"]++ if dataset_name in ["MMMU_TEST"]:result_json = vlmeval.utils.result_transfer.MMMU_result_transfer(result_file)- logger.info(f'Transfer MMMU_TEST result to json for official evaluation, '- f'json file saved in {result_json}') # noqa: E501+ logger.info(+ f"Transfer MMMU_TEST result to json for official evaluation, " f"json file saved in {result_json}"+ ) # noqa: E501continue- elif 'MMT-Bench_ALL' in dataset_name:+ elif "MMT-Bench_ALL" in dataset_name:submission_file = vlmeval.utils.result_transfer.MMTBench_result_transfer(result_file, **judge_kwargs)- logger.info(f'Extract options from prediction of MMT-Bench FULL split for official evaluation '- f'(https://eval.ai/web/challenges/challenge-page/2328/overview), '- f'submission file saved in {submission_file}') # noqa: E501+ logger.info(+ f"Extract options from prediction of MMT-Bench FULL split for official evaluation "+ f"(https://eval.ai/web/challenges/challenge-page/2328/overview), "+ f"submission file saved in {submission_file}"+ ) # noqa: E501continue- elif 'MLLMGuard_DS' in dataset_name:- logger.info('The evaluation of MLLMGuard_DS is not supported yet. ') # noqa: E501+ elif "MLLMGuard_DS" in dataset_name:+ logger.info("The evaluation of MLLMGuard_DS is not supported yet. ") # noqa: E501continue- elif 'AesBench_TEST' == dataset_name:- logger.info(f'The results are saved in {result_file}. '- f'Please send it to the AesBench Team via [email protected].') # noqa: E501+ elif "AesBench_TEST" == dataset_name:+ logger.info(+ f"The results are saved in {result_file}. "+ f"Please send it to the AesBench Team via [email protected]."+ ) # noqa: E501continue- elif dataset_name in ['DocVQA_TEST', 'InfoVQA_TEST', 'Q-Bench1_TEST', 'A-Bench_TEST']:- logger.info(f'{dataset_name} is a test split without ground-truth. '- 'Thus only the inference part is supported for those datasets. ') # noqa: E501+ elif dataset_name in ["DocVQA_TEST", "InfoVQA_TEST", "Q-Bench1_TEST", "A-Bench_TEST"]:+ logger.info(+ f"{dataset_name} is a test split without ground-truth. "+ "Thus only the inference part is supported for those datasets. "+ ) # noqa: E501if dataset_name in [- 'MMBench_TEST_CN', 'MMBench_TEST_EN', 'MMBench', 'MMBench_CN',- 'MMBench_TEST_CN_V11', 'MMBench_TEST_EN_V11', 'MMBench_V11', 'MMBench_CN_V11'+ "MMBench_TEST_CN",+ "MMBench_TEST_EN",+ "MMBench",+ "MMBench_CN",+ "MMBench_TEST_CN_V11",+ "MMBench_TEST_EN_V11",+ "MMBench_V11",+ "MMBench_CN_V11",]:if not vlmeval.smp.MMBenchOfficialServer(dataset_name):logger.error(- f'Can not evaluate {dataset_name} on non-official servers, '- 'will skip the evaluation. '+ f"Can not evaluate {dataset_name} on non-official servers, " "will skip the evaluation. ")continue- if mode == 'all':+ if mode == "all":eval_results = dataset.evaluate(result_file, **judge_kwargs)if eval_results is not None:if not isinstance(eval_results, dict) and not isinstance(eval_results, pd.DataFrame):raise TypeError("Unsupported eval result type")- logger.info(f'The evaluation of model {model_name} x dataset {dataset_name} has finished! ')- logger.info('Evaluation Results:')+ logger.info(f"The evaluation of model {model_name} x dataset {dataset_name} has finished! ")+ logger.info("Evaluation Results:")if isinstance(eval_results, dict):- logger.info('\n' + json.dumps(eval_results, indent=4))+ logger.info("\n" + json.dumps(eval_results, indent=4))elif isinstance(eval_results, pd.DataFrame):if len(eval_results) < len(eval_results.columns):eval_results = eval_results.Ttry:import tabulate- logger.info('\n' + tabulate.tabulate(eval_results))++ logger.info("\n" + tabulate.tabulate(eval_results))except:logger.info(eval_results.to_string())- rt_file.write('%s cost: %.4fs\n' % (dataset_name, time.time() - task_st))+ rt_file.write("%s cost: %.4fs\n" % (dataset_name, time.time() - task_st))except Exception as e:- logger.exception(f'Model {model_name} x Dataset {dataset_name} combination failed: {e}, '- 'skipping this combination.')+ logger.exception(+ f"Model {model_name} x Dataset {dataset_name} combination failed: {e}, " "skipping this combination."+ )continue- rt_file.write('%d tasks cost: %.4fs\n' % (len(dataset), time.time() - st))+ rt_file.write("%d tasks cost: %.4fs\n" % (len(dataset), time.time() - st))rt_file.close()@@ -319,7 +339,6 @@ MODEL_TYPE_TO_LMMS_MODEL = {"Llama-3.2": "llama_vision","Phi-3-vision": "phi3v","Phi-3.5-vision": "phi3v",-# model_type"qwen2_vl": "qwen2_vl","qwen": "qwen_vl",@@ -341,21 +360,22 @@ def _handle_non_serializable(o):def lmms_eval(- model,- tasks,- output_dir=None,- num_fewshot=None,- limit=None,- batch_size=1,- max_batch_size=None,- device='cpu',- use_cache=None,- apply_chat_template=False+ model,+ tasks,+ output_dir=None,+ num_fewshot=None,+ limit=None,+ batch_size=1,+ max_batch_size=None,+ device="cpu",+ use_cache=None,+ apply_chat_template=False,):from transformers.utils.versions import require_version+require_version("lmms_eval", "lmms_eval need to be installed, `pip install lmms_eval`")if isinstance(tasks, str):- tasks = tasks.replace(' ', '').split(',')+ tasks = tasks.replace(" ", "").split(",")model_name = modelif model_name[-1] == "/":@@ -371,6 +391,7 @@ def lmms_eval(breakif model_type is None:from transformers import AutoConfig+config = AutoConfig.from_pretrained(model, trust_remote_code=True)model_type = config.model_type@@ -398,7 +419,7 @@ def lmms_eval(device=device,use_cache=use_cache,apply_chat_template=apply_chat_template,- cli_args=CliArgs()+ cli_args=CliArgs(),)# print and save result@@ -407,10 +428,9 @@ def lmms_eval(os.makedirs(output_dir, exist_ok=True)from datetime import datetime+now = datetime.now().strftime("%Y%m%d_%H%M%S")output_file = os.path.join(output_dir, f"{model_name}_{now}_result.json")- json.dump(results, open(output_file, 'w'), indent=4, default=_handle_non_serializable)+ json.dump(results, open(output_file, "w"), indent=4, default=_handle_non_serializable)return results--diff --git a/auto_round/mllm/mllm_dataset.py b/auto_round/mllm/mllm_dataset.pyindex 953c3e0c2..267b8eb4a 100644--- a/auto_round/mllm/mllm_dataset.py+++ b/auto_round/mllm/mllm_dataset.py@@ -41,16 +41,18 @@ def register_dataset(name_list):"""def register(dataset):- for name in name_list.replace(' ', '').split(','):+ for name in name_list.replace(" ", "").split(","):MLLM_DATASET[name] = datasetreturn register@register_dataset(- "liuhaotian/llava,liuhaotian/llava_conv_58k,liuhaotian/llava_instruct_80k,liuhaotian/llava_instruct_150k")+ "liuhaotian/llava,liuhaotian/llava_conv_58k,liuhaotian/llava_instruct_80k,liuhaotian/llava_instruct_150k"+)class LlavaDataset(Dataset):"""Dataset for supervised fine-tuning."""+BASE_LLAVA_URL = "https://huggingface.co/datasets/liuhaotian/LLaVA-Instruct-150K/resolve/main/"LLAVA_DATASET = {"llava_conv_58k": BASE_LLAVA_URL + "conversation_58k.json?download=true",@@ -61,16 +63,16 @@ class LlavaDataset(Dataset):IMAGE_TOKEN = "<image>"def __init__(- self,- template,- model,- tokenzier,- dataset_path,- extra_data_dir=None,- seqlen=512,- padding=True,- truncation=True,- nsamples=512+ self,+ template,+ model,+ tokenzier,+ dataset_path,+ extra_data_dir=None,+ seqlen=512,+ padding=True,+ truncation=True,+ nsamples=512,) -> None:super().__init__()self.model = model@@ -78,21 +80,22 @@ class LlavaDataset(Dataset):self.template = templateself.tokenizer = tokenzierif os.path.exists(dataset_path):- logger.info(f'use dataset {dataset_path}, loading from disk...')+ logger.info(f"use dataset {dataset_path}, loading from disk...")self.questions = json.load(open(dataset_path, "r"))else:import requests+if dataset_path == "liuhaotian/llava":dataset_path = "llava_conv_58k"else:dataset_path = dataset_path.split("/")[-1]- dataset_name = dataset_path.split('/')[-1]+ dataset_name = dataset_path.split("/")[-1]if dataset_name in self.LLAVA_DATASET:- logger.info(f'use dataset {dataset_name}, downloading ...')+ logger.info(f"use dataset {dataset_name}, downloading ...")self.questions = requests.get(self.LLAVA_DATASET[dataset_name], stream=True).json()else:raise KeyError(f"{dataset_path} is not support, we support {self.LLAVA_DATASET.keys()}.")-+self.seqlen = seqlenself.questions = self.check(self.questions, self.seqlen, nsamples)self.padding = padding@@ -105,7 +108,7 @@ class LlavaDataset(Dataset):if extra_data_dir is not None:image_fold = _extract_data_dir(self.extra_data_dir)if isinstance(image_fold, dict):- image_fold = image_fold['image']+ image_fold = image_fold["image"]self.image_fold = image_folddef check(self, questions, word_len, nsamples):@@ -114,10 +117,10 @@ class LlavaDataset(Dataset):max_len = 0for source in questions:str_len = 0- for text in source['conversations']:- if self.IMAGE_TOKEN in text['value']:- text['value'] = self.IMAGE_TOKEN + text['value'].replace(self.IMAGE_TOKEN, '')- str_len += len(text['value'].split(' '))+ for text in source["conversations"]:+ if self.IMAGE_TOKEN in text["value"]:+ text["value"] = self.IMAGE_TOKEN + text["value"].replace(self.IMAGE_TOKEN, "")+ str_len += len(text["value"].split(" "))if str_len > max_len:max_len = str_lenif min_word_len <= str_len < max_word_len:@@ -125,12 +128,16 @@ class LlavaDataset(Dataset):if len(new_questions) >= nsamples:return new_questionsif min_word_len > max_len:- logger.debug(f"seqlen={min_word_len} is greater than the max length of dataset {max_len},"- f" will change seqlen to {max_len - 128}")+ logger.debug(+ f"seqlen={min_word_len} is greater than the max length of dataset {max_len},"+ f" will change seqlen to {max_len - 128}"+ )new_min_word_len = max_len - 128else:- logger.debug(f"no enough sample for seqlen greater than {min_word_len},"- f" will decrease to {min_word_len - 128}")+ logger.debug(+ f"no enough sample for seqlen greater than {min_word_len},"+ f" will decrease to {min_word_len - 128}"+ )new_min_word_len = min_word_len - 128return new_questions + _check(questions, new_min_word_len, min_word_len, nsamples - len(new_questions))@@ -148,12 +155,11 @@ class LlavaDataset(Dataset):text = self.covert_conversations(text)if self.image_fold is not None:- image_path = os.path.join(- self.image_fold, os.path.basename(self.questions[i]["image"]))+ image_path = os.path.join(self.image_fold, os.path.basename(self.questions[i]["image"]))else:image_path = self.questions[i]["image"]if not os.path.exists(image_path):- image_path = self._COCO_DATA_URL + self.questions[i]["image"].split('/')[-1]+ image_path = self._COCO_DATA_URL + self.questions[i]["image"].split("/")[-1]# image = self.template.processor.image_processor(image_path)text = self.template._encode(text)@@ -167,7 +173,7 @@ class LlavaDataset(Dataset):truncation=self.truncation,return_tensors="pt",max_length=max_length,- truncation_strategy=truncation_strategy+ truncation_strategy=truncation_strategy,)self.cached_data_dict[i] = retreturn ret@@ -179,30 +185,27 @@ class LlavaDataset(Dataset):if self.template.replace_tokens is not None:for old, new in self.template.replace_tokens:content = content.replace(old, new)- new_data.append({- "role": self.role_mapping.get(d["from"], d["from"]),- "content": content- })+ new_data.append({"role": self.role_mapping.get(d["from"], d["from"]), "content": content})return new_datadef get_mllm_dataloader(- template,- model,- tokenizer,- processor,- image_processor=None,- dataset="liuhaotian/llava_conv_58k",- extra_data_dir=None,- seqlen=512,- bs=1,- split=None,- apply_template=None,- truncation=False,- seed=42,- nsamples=512,- gradient_accumulate_steps=1,- quant_nontext_module=False+ template,+ model,+ tokenizer,+ processor,+ image_processor=None,+ dataset="liuhaotian/llava_conv_58k",+ extra_data_dir=None,+ seqlen=512,+ bs=1,+ split=None,+ apply_template=None,+ truncation=False,+ seed=42,+ nsamples=512,+ gradient_accumulate_steps=1,+ quant_nontext_module=False,):"""Generate a DataLoader for calibration using specified parameters.@@ -217,41 +220,39 @@ def get_mllm_dataloader(bs (int, optional): The batch size. Defaults to 4.split (str, optional): The data split to use. Defaults to None.apply_template: Whether to apply chat template in tokenization.-+Returns:DataLoader: The DataLoader for the calibrated datasets."""if isinstance(template, str):from .template import get_template+template = get_template(- template, model=model, tokenizer=tokenizer,- processor=processor, image_processor=image_processor)+ template, model=model, tokenizer=tokenizer, processor=processor, image_processor=image_processor+ )if os.path.isfile(dataset) or dataset in MLLM_DATASET.keys():- dataset = MLLM_DATASET['liuhaotian/llava'](- template, model, tokenizer, dataset, extra_data_dir,- seqlen=seqlen, truncation=truncation, nsamples=nsamples)+ dataset = MLLM_DATASET["liuhaotian/llava"](+ template, model, tokenizer, dataset, extra_data_dir, seqlen=seqlen, truncation=truncation, nsamples=nsamples+ )bs, gradient_accumulate_steps = check_mllm_model_batch(- model, batch_size=bs, gradient_accumulate_steps=gradient_accumulate_steps)+ model, batch_size=bs, gradient_accumulate_steps=gradient_accumulate_steps+ )set_seed(seed)- dataloader_params = {- "batch_size": bs,- "shuffle": True,- "collate_fn": dataset.template.processor.data_collator- }+ dataloader_params = {"batch_size": bs, "shuffle": True, "collate_fn": dataset.template.processor.data_collator}return DataLoader(dataset, **dataloader_params), bs, gradient_accumulate_stepselse:# try to load text calibration datasetfrom ..calib_dataset import get_dataloader- dataloader = get_dataloader(- tokenizer, seqlen, dataset, seed, bs, nsamples)++ dataloader = get_dataloader(tokenizer, seqlen, dataset, seed, bs, nsamples)if quant_nontext_module:logger.error("Text only dataset cannot be used for calibrating non-text modules,"- " switching to liuhaotian/llava_conv_58k")+ " switching to liuhaotian/llava_conv_58k"+ )exit(-1)return dataloader, bs, gradient_accumulate_steps-diff --git a/auto_round/mllm/processor.py b/auto_round/mllm/processor.pyindex a01ac0847..bf48f5556 100644--- a/auto_round/mllm/processor.py+++ b/auto_round/mllm/processor.py@@ -59,13 +59,8 @@ class BasicProcessor:self.use_rtn = use_rtnself.check_image_processor()- def get_input(- self,- text,- images,- squeeze=True,- **kwargs):- raise NotImplementedError+ def get_input(self, text, images, squeeze=True, **kwargs):+ raise NotImplementedError@staticmethoddef data_collator(batch):@@ -80,7 +75,7 @@ class BasicProcessor:for key in ret:ret[key] = ret[key][0]return ret-+def check_image_processor(self):if not self.use_rtn and self.image_processor is None:raise ValueError("image processor should not be None.")@@ -89,7 +84,8 @@ class BasicProcessor:@register_processor("hf")class HFProcessor(BasicProcessor):# evaluation on: Qwen2-VL, mllama, Mistral-Small- IMAGE_TOKEN = '<image>'+ IMAGE_TOKEN = "<image>"+def __init__(self):self.process_func = self._process_v1@@ -97,16 +93,17 @@ class HFProcessor(BasicProcessor):"""support models: Qwen2-VL, gemma-3, granite-vision-3.2, Aria"""conversation = []for content in messages:- conversation.append({- "role": content['role'],- "content": [- {"text": content["content"].replace(self.IMAGE_TOKEN, ""), "type": "text"}- ]- })- if self.IMAGE_TOKEN in content['content']:+ conversation.append(+ {+ "role": content["role"],+ "content": [{"text": content["content"].replace(self.IMAGE_TOKEN, ""), "type": "text"}],+ }+ )+ if self.IMAGE_TOKEN in content["content"]:conversation[-1]["content"].append({"image": image, "type": "image"})ret = self.processor.apply_chat_template(- conversation, add_generation_prompt=True, tokenize=True, return_dict=True)+ conversation, add_generation_prompt=True, tokenize=True, return_dict=True+ )return retdef _process_v2(self, messages, image):@@ -114,44 +111,45 @@ class HFProcessor(BasicProcessor):conversation = []for content in messages:if content["role"] == "user":- conversation.append({- "role": content['role'],- "content": [- {"text": content["content"].replace(self.IMAGE_TOKEN, ""), "type": "text"}- ]- })- if self.IMAGE_TOKEN in content['content']:+ conversation.append(+ {+ "role": content["role"],+ "content": [{"text": content["content"].replace(self.IMAGE_TOKEN, ""), "type": "text"}],+ }+ )+ if self.IMAGE_TOKEN in content["content"]:conversation[-1]["content"].append({"image": image, "type": "image"})else:- conversation.append({- "role": content['role'],- "content": content["content"]- })+ conversation.append({"role": content["role"], "content": content["content"]})if hasattr(self.processor, "chat_template"):text = self.processor.apply_chat_template(- conversation, add_generation_prompt=True, tokenize=False, return_dict=False)+ conversation, add_generation_prompt=True, tokenize=False, return_dict=False+ )else:continue_final_message = messages[-1]["role"] == "assistant"text = self.tokenizer.apply_chat_template(- messages, tokenize=False, add_generation_prompt=not continue_final_message,- continue_final_message=continue_final_message, )+ messages,+ tokenize=False,+ add_generation_prompt=not continue_final_message,+ continue_final_message=continue_final_message,+ )if image is not None:image = self.default_image_processor(image)# image = self.image_processor(image)- ret = self.processor(- text=text, images=image, return_tensors="pt")+ ret = self.processor(text=text, images=image, return_tensors="pt")return ret-+def get_input(- self,- text,- images,- return_tensors="pt",- squeeze=True,- max_length=None,- truncation=False,- truncation_strategy="text",- **kwargs):+ self,+ text,+ images,+ return_tensors="pt",+ squeeze=True,+ max_length=None,+ truncation=False,+ truncation_strategy="text",+ **kwargs+ ):if isinstance(text, list):try:@@ -164,8 +162,7 @@ class HFProcessor(BasicProcessor):if images is not None:images = self.image_processor(images)- ret = self.processor(- text=text, images=images, return_tensors="pt", add_special_tokens=False)+ ret = self.processor(text=text, images=images, return_tensors="pt", add_special_tokens=False)if squeeze:ret = self.squeeze_result(ret)@@ -185,9 +182,7 @@ class Qwen2VLProcessor(HFProcessor):@register_processor("cogvlm2")class CogVLM2Processor(BasicProcessor):- def get_input(- self, text, images, truncation=False,- squeeze=True, max_length=None, **kwargs):+ def get_input(self, text, images, truncation=False, squeeze=True, max_length=None, **kwargs):if images is not None:images = self.image_processor(images)@@ -197,11 +192,7 @@ class CogVLM2Processor(BasicProcessor):max_length += padding_lenpadding = Falseinput_data = self.model.build_conversation_input_ids(- self.tokenizer,- query=text,- history=None,- images=[images],- template_version='base'+ self.tokenizer, query=text, history=None, images=[images], template_version="base")def pad_to_len(unpadded_tensor, pad_to_length, pad_value=0):@@ -213,35 +204,29 @@ class CogVLM2Processor(BasicProcessor):return unpadded_tensorif padding:return torch.cat(- (unpadded_tensor,- torch.full([pad_to_length - current_length],- fill_value=pad_value,- dtype=unpadded_tensor.dtype,- device=unpadded_tensor.device)), dim=0)+ (+ unpadded_tensor,+ torch.full(+ [pad_to_length - current_length],+ fill_value=pad_value,+ dtype=unpadded_tensor.dtype,+ device=unpadded_tensor.device,+ ),+ ),+ dim=0,+ )else:return unpadded_tensor- input_data['input_ids'] = pad_to_len(- input_data['input_ids'],+ input_data["input_ids"] = pad_to_len(+ input_data["input_ids"],max_length,pad_value=128002,)- input_data['attention_mask'] = pad_to_len(- input_data['attention_mask'],- max_length,- pad_value=0- )- input_data['token_type_ids'] = pad_to_len(- input_data['token_type_ids'],- max_length,- pad_value=0- )- if input_data['labels']:- input_data['labels'] = pad_to_len(- input_data['labels'],- max_length,- pad_value=-100- )+ input_data["attention_mask"] = pad_to_len(input_data["attention_mask"], max_length, pad_value=0)+ input_data["token_type_ids"] = pad_to_len(input_data["token_type_ids"], max_length, pad_value=0)+ if input_data["labels"]:+ input_data["labels"] = pad_to_len(input_data["labels"], max_length, pad_value=-100)return input_data@staticmethod@@ -258,7 +243,7 @@ class CogVLM2Processor(BasicProcessor):@staticmethoddef default_image_processor(image_path_or_url):- return fetch_image(image_path_or_url).convert('RGB')+ return fetch_image(image_path_or_url).convert("RGB")from ..utils import LazyImport@@ -276,12 +261,12 @@ class LlavaProcessor(BasicProcessor):self.check_image_processor()def get_input(- self, text, images, max_length=None,- squeeze=True, truncation=False, truncation_strategy="text", **kwargs):+ self, text, images, max_length=None, squeeze=True, truncation=False, truncation_strategy="text", **kwargs+ ):if images is not None:- images = fetch_image(images).convert('RGB')- images = self.image_processor.preprocess(images, return_tensors='pt')['pixel_values'][0]+ images = fetch_image(images).convert("RGB")+ images = self.image_processor.preprocess(images, return_tensors="pt")["pixel_values"][0]class DataArgs:is_multimodal = True@@ -294,13 +279,13 @@ class LlavaProcessor(BasicProcessor):ret = llava_train.preprocess(input_data, self.tokenizer, has_image=(images is not None))if truncation is True and truncation_strategy == "token":- seqlen = ret['input_ids'].shape[-1]+ seqlen = ret["input_ids"].shape[-1]for key in ret:if ret[key].shape[-1] == seqlen:ret[key] = ret[key][:, :max_length]if squeeze:ret = self.squeeze_result(ret)- ret['image'] = images+ ret["image"] = imagesreturn ret@@ -310,25 +295,20 @@ class LlavaProcessor(BasicProcessor):@register_processor("deepseek_v2")class DeepSeekV2Processor(BasicProcessor):- IMAGE_TOKEN = '<image>'+ IMAGE_TOKEN = "<image>"+def get_input(- self, text, images, max_length=None,- squeeze=True, truncation=False, truncation_strategy="text", **kwargs):+ self, text, images, max_length=None, squeeze=True, truncation=False, truncation_strategy="text", **kwargs+ ):messages = []for content in text:if content["role"] == "user":- messages.append({- "role": content['role'],- "content": content["content"]- })- if self.IMAGE_TOKEN in content['content']:+ messages.append({"role": content["role"], "content": content["content"]})+ if self.IMAGE_TOKEN in content["content"]:messages[-1]["images"] = [images]else:- messages.append({- "role": content['role'],- "content": content["content"]- })+ messages.append({"role": content["role"], "content": content["content"]})if images is not None:pil_image = [self.image_processor(images)]@@ -336,10 +316,7 @@ class DeepSeekV2Processor(BasicProcessor):pil_image = Noneprepare_inputs = self.processor(- conversations=messages,- images=pil_image,- force_batchify=True,- system_prompt=""+ conversations=messages, images=pil_image, force_batchify=True, system_prompt="").to(self.model.device)prepare_inputs = prepare_inputs.to(self.model.device)prepare_inputs = self.squeeze_result(dict(prepare_inputs))diff --git a/auto_round/mllm/template.py b/auto_round/mllm/template.pyindex 1794cca06..0844c4fea 100644--- a/auto_round/mllm/template.py+++ b/auto_round/mllm/template.py@@ -64,13 +64,13 @@ class Template:# if i > 0 and i % 2 ==0:# element += fill_content(self.format_separator)- if source['role'] == Role.USER.value:+ if source["role"] == Role.USER.value:element += fill_content(self.format_user, content=source["content"])- elif source['role'] == Role.ASSISTANT.value:+ elif source["role"] == Role.ASSISTANT.value:element += fill_content(self.format_assistant, content=source["content"])- elif source['role'] == Role.OBSERVATION.value:+ elif source["role"] == Role.OBSERVATION.value:element += fill_content(self.format_observation, content=source["content"])- elif source['role'] == Role.FUNCTION.value:+ elif source["role"] == Role.FUNCTION.value:element += fill_content(self.format_function, content=source["content"])return elementelse:@@ -78,18 +78,18 @@ class Template:def _register_template(- model_type: str,- format_user: Optional[str] = None,- format_assistant: Optional[str] = None,- format_system: Optional[str] = None,- format_function: Optional[str] = None,- format_observation: Optional[str] = None,- format_separator: Optional[str] = None,- default_system: str = "",- replace_tokens: List[tuple] = None,- extra_encode: Optional[bool] = False,- default_dataset: Optional[bool] = "NeelNanda/pile-10k",- processor: "BasicProcessor" = PROCESSORS["basic"],+ model_type: str,+ format_user: Optional[str] = None,+ format_assistant: Optional[str] = None,+ format_system: Optional[str] = None,+ format_function: Optional[str] = None,+ format_observation: Optional[str] = None,+ format_separator: Optional[str] = None,+ default_system: str = "",+ replace_tokens: List[tuple] = None,+ extra_encode: Optional[bool] = False,+ default_dataset: Optional[bool] = "NeelNanda/pile-10k",+ processor: "BasicProcessor" = PROCESSORS["basic"],):"""Registers a chat template."""template_class = Template@@ -111,14 +111,16 @@ def _register_template(replace_tokens=replace_tokens,extra_encode=extra_encode,default_dataset=default_dataset,- processor=processor()+ processor=processor(),)return TEMPLATES[model_type]-_register_template("qwen2_vl", default_dataset="NeelNanda/pile-10k",processor=PROCESSORS["qwen2_vl"])++_register_template("qwen2_vl", default_dataset="NeelNanda/pile-10k", processor=PROCESSORS["qwen2_vl"])_register_template("mllama", default_dataset="liuhaotian/llava", processor=PROCESSORS["hf"])-_register_template("deepseek_vl_v2", default_dataset="NeelNanda/pile-10k",processor=PROCESSORS["deepseek_v2"])-_register_template("mistral3", default_dataset="NeelNanda/pile-10k",processor=PROCESSORS["hf"])+_register_template("deepseek_vl_v2", default_dataset="NeelNanda/pile-10k", processor=PROCESSORS["deepseek_v2"])+_register_template("mistral3", default_dataset="NeelNanda/pile-10k", processor=PROCESSORS["hf"])+def load_template(path: str):"""Load template information from a json file."""@@ -128,25 +130,25 @@ def load_template(path: str):data["model_type"] = "user_define"if "replace_tokens" in data and data["replace_tokens"] is not None:if len(data["replace_tokens"]) % 2 != 0:- raise ValueError("the format of replace_tokens should be " \- "[old_tag1, replace_tag1, old_tag2, replace_tag2]")+ raise ValueError(+ "the format of replace_tokens should be " "[old_tag1, replace_tag1, old_tag2, replace_tag2]"+ )temp = []for i in range(0, len(data["replace_tokens"]), 2):temp.append((data["replace_tokens"][i], data["replace_tokens"][i + 1]))data["replace_tokens"] = tempif "processor" in data:if data["processor"] not in PROCESSORS.keys():- raise ValueError(f"{data['processor']} is not supported, current support: " \- "{','.join(PROCESSORS.keys())}")+ raise ValueError(+ f"{data['processor']} is not supported, current support: " "{','.join(PROCESSORS.keys())}"+ )data["processor"] = PROCESSORS[data["processor"]]- template = _register_template(- **data- )+ template = _register_template(**data)return templatedef _load_preset_template():- dir_path = os.path.join(os.path.dirname(__file__), 'templates')+ dir_path = os.path.join(os.path.dirname(__file__), "templates")for file_name in os.listdir(dir_path):load_template(os.path.join(dir_path, file_name))@@ -155,15 +157,16 @@ _load_preset_template()def get_template(- template_or_path: str, model=None, tokenizer=None, processor=None, image_processor=None, use_rtn=False):+ template_or_path: str, model=None, tokenizer=None, processor=None, image_processor=None, use_rtn=False+):"""Get template by template name or from a json file.Args:template_or_path (str): Template name or a path of the template json file.-+Returns:The Template.-+"""if os.path.isfile(template_or_path):template = load_template(template_or_path)@@ -176,7 +179,7 @@ def get_template(template.model_type = template_or_pathtemplate.processor.post_init(- model=model, tokenizer=tokenizer, processor=processor, image_processor=image_processor, use_rtn=use_rtn)+ model=model, tokenizer=tokenizer, processor=processor, image_processor=image_processor, use_rtn=use_rtn+ )return template-diff --git a/auto_round/mllm/utils.py b/auto_round/mllm/utils.pyindex 90e67e855..029080d68 100644--- a/auto_round/mllm/utils.py+++ b/auto_round/mllm/utils.py@@ -29,8 +29,8 @@ def _extract_data_dir(dir_path: str):result = {}dir_path = dir_path.split(",")for _path in dir_path:- k, v = _path.split('=')- if k in ['image', 'video', 'audio']:+ k, v = _path.split("=")+ if k in ["image", "video", "audio"]:result[k] = vreturn resultelse:diff --git a/auto_round/script/__init__.py b/auto_round/script/__init__.pyindex e3fdc07b1..28f108cb6 100644--- a/auto_round/script/__init__.py+++ b/auto_round/script/__init__.py@@ -10,4 +10,4 @@# distributed under the License is distributed on an "AS IS" BASIS,# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.# See the License for the specific language governing permissions and-# limitations under the License.\ No newline at end of file+# limitations under the License.diff --git a/auto_round/script/llm.py b/auto_round/script/llm.pyindex 7463dbcc4..a50c877ff 100644--- a/auto_round/script/llm.py+++ b/auto_round/script/llm.py@@ -49,9 +49,10 @@ class BasicArgumentParser(argparse.ArgumentParser):def __init__(self, *args, **kwargs):super().__init__(*args, **kwargs)self.add_argument(- "--model", "--model_name", "--model_name_or_path", default="facebook/opt-125m", help="model name or path")+ "--model", "--model_name", "--model_name_or_path", default="facebook/opt-125m", help="model name or path"+ )- self.add_argument('--eval', action='store_true', help="whether to use eval only mode")+ self.add_argument("--eval", action="store_true", help="whether to use eval only mode")self.add_argument("--bits", default=4, type=int, help="number of weight bits")@@ -63,78 +64,85 @@ class BasicArgumentParser(argparse.ArgumentParser):default="0",type=str,help="the device to be used for tuning. "- "Currently, device settings support CPU, GPU, and HPU."- "The default is set to cuda:0,"- "allowing for automatic detection and switch to HPU or CPU."- "set --device 0,1,2 to use multiple cards.")+ "Currently, device settings support CPU, GPU, and HPU."+ "The default is set to cuda:0,"+ "allowing for automatic detection and switch to HPU or CPU."+ "set --device 0,1,2 to use multiple cards.",+ )- self.add_argument("--asym", action='store_true', help="whether to use asym quantization")+ self.add_argument("--asym", action="store_true", help="whether to use asym quantization")self.add_argument(- "--dataset", default="NeelNanda/pile-10k", type=str, help="the dataset for quantization training")+ "--dataset", default="NeelNanda/pile-10k", type=str, help="the dataset for quantization training"+ )self.add_argument("--minmax_lr",default=None,type=float,- help="minmax learning rate, if None, it will beset to be the same with lr")+ help="minmax learning rate, if None, it will beset to be the same with lr",+ )self.add_argument("--seed", default=42, type=int, help="random seed")- self.add_argument("--adam", action='store_true', help="whether to use adam optimizer instead of SignSGD")+ self.add_argument("--adam", action="store_true", help="whether to use adam optimizer instead of SignSGD")self.add_argument("--gradient_accumulate_steps", default=1, type=int, help="gradient accumulate steps")self.add_argument("--nblocks", default=1, type=int, help="how many blocks to tune together")- self.add_argument("--low_gpu_mem_usage", action='store_true', help="offload intermediate features to cpu")+ self.add_argument("--low_gpu_mem_usage", action="store_true", help="offload intermediate features to cpu")self.add_argument("--format", default="auto_round", type=str, help="the format to save the model")- self.add_argument("--data_type", "--dtype", default='int', help="data type for tuning, 'int', 'mx_fp' and etc")+ self.add_argument("--data_type", "--dtype", default="int", help="data type for tuning, 'int', 'mx_fp' and etc")self.add_argument("--scale_dtype",- default='fp16',+ default="fp16",choices=["fp16", "float16", "bf16", "bfloat16", "fp32", "float32"],- help="scale data type to use for quantization")+ help="scale data type to use for quantization",+ )self.add_argument("--tasks","--task",- nargs='?',+ nargs="?",const="lambada_openai,hellaswag,winogrande,piqa,mmlu,wikitext,truthfulqa_mc1,"- "openbookqa,boolq,arc_easy,arc_challenge",+ "openbookqa,boolq,arc_easy,arc_challenge",default=None,- help="lm-eval tasks")+ help="lm-eval tasks",+ )self.add_argument(- "--output_dir", default="./tmp_autoround", type=str, help="the directory to save quantized model")-- self.add_argument("--disable_eval", action='store_true',- help="whether to disable lm-eval evaluation after tuning")+ "--output_dir", default="./tmp_autoround", type=str, help="the directory to save quantized model"+ )self.add_argument(- "--eval_task_by_task",- action="store_true",- help="whether to eval task by task.")+ "--disable_eval", action="store_true", help="whether to disable lm-eval evaluation after tuning"+ )++ self.add_argument("--eval_task_by_task", action="store_true", help="whether to eval task by task.")- self.add_argument("--disable_amp", action='store_true', help="disable amp")+ self.add_argument("--disable_amp", action="store_true", help="disable amp")self.add_argument(- "--disable_minmax_tuning", action='store_true', help="whether to disable enable weight minmax tuning")+ "--disable_minmax_tuning", action="store_true", help="whether to disable enable weight minmax tuning"+ )- self.add_argument("--enable_norm_bias_tuning", action='store_true', help="whether to enable norm bias tuning")+ self.add_argument("--enable_norm_bias_tuning", action="store_true", help="whether to enable norm bias tuning")self.add_argument(- "--disable_trust_remote_code", action='store_true', help="whether to disable trust_remote_code")+ "--disable_trust_remote_code", action="store_true", help="whether to disable trust_remote_code"+ )self.add_argument("--disable_quanted_input",- action='store_true',- help="whether to disuse the output of quantized block to tune the next block")+ action="store_true",+ help="whether to disuse the output of quantized block to tune the next block",+ )- self.add_argument("--quant_lm_head", action='store_true', help="whether to quant lm_head")+ self.add_argument("--quant_lm_head", action="store_true", help="whether to quant lm_head")self.add_argument("--low_cpu_mem_mode",@@ -142,69 +150,78 @@ class BasicArgumentParser(argparse.ArgumentParser):type=int,choices=[0, 1, 2],help="choose which low cpu memory mode to use. "- "Can significantly reduce cpu memory footprint but cost more time."- "1 means choose block-wise mode, load the weights of each block"- " from disk when tuning and release the memory of the block after tuning."- "2 means choose layer-wise mode, load the weights of each layer from disk when tuning,"- " minimum memory consumption and also slowest running speed."- "others means not use low cpu memory. Default to 0, not use low cpu memory.")+ "Can significantly reduce cpu memory footprint but cost more time."+ "1 means choose block-wise mode, load the weights of each block"+ " from disk when tuning and release the memory of the block after tuning."+ "2 means choose layer-wise mode, load the weights of each layer from disk when tuning,"+ " minimum memory consumption and also slowest running speed."+ "others means not use low cpu memory. Default to 0, not use low cpu memory.",+ )self.add_argument("--low_cpu_mem_tmp_dir",default=None,type=str,help="temporary work space to store the temporary files "- "when using low cpu memory mode. Will remove after tuning.")+ "when using low cpu memory mode. Will remove after tuning.",+ )self.add_argument("--model_dtype",default=None,type=str,choices=["fp16", "float16", "bf16", "bfloat16", "fp32", "float32"],- help="force to convert the dtype, some backends supports fp16 dtype better")+ help="force to convert the dtype, some backends supports fp16 dtype better",+ )self.add_argument("--act_bits", default=16, type=int, help="activation bits")self.add_argument(- "--fp_layers", default="", type=str, help="list of Layer names to maintain original data type")+ "--fp_layers", default="", type=str, help="list of Layer names to maintain original data type"+ )self.add_argument("--not_use_best_mse",- action='store_true',- help="whether to use the iter of best mes loss in the tuning phase")+ action="store_true",+ help="whether to use the iter of best mes loss in the tuning phase",+ )self.add_argument("--to_quant_block_names",default=None,type=str,- help="Names of quantitative blocks, please use commas to separate them.")+ help="Names of quantitative blocks, please use commas to separate them.",+ )- self.add_argument("--enable_torch_compile", action='store_true',- help="whether to enable torch compile")+ self.add_argument("--enable_torch_compile", action="store_true", help="whether to enable torch compile")self.add_argument("--act_data_type", "--act_dtype", default=None, type=str, help="activation data type")- self.add_argument("--disable_act_dynamic", action='store_true', help="activation static quantization")+ self.add_argument("--disable_act_dynamic", action="store_true", help="activation static quantization")- self.add_argument("--disable_deterministic_algorithms", action='store_true',- help="disable torch deterministic algorithms.")+ self.add_argument(+ "--disable_deterministic_algorithms", action="store_true", help="disable torch deterministic algorithms."+ )self.add_argument("--device_map", default=None, type=str, help="device_map for block in tuning phase")self.add_argument(- "--super_group_size", default=None, type=int, help="the number of super group size when use double quant.")+ "--super_group_size", default=None, type=int, help="the number of super group size when use double quant."+ )self.add_argument(- "--super_bits", default=None, type=int, help="number of scale and mins quant bits for double quant.")+ "--super_bits", default=None, type=int, help="number of scale and mins quant bits for double quant."+ )self.add_argument(- "--eval_model_dtype",- default=None,- type=str,- help="the torch_dytpe to load the model for evaluation.")+ "--eval_model_dtype", default=None, type=str, help="the torch_dytpe to load the model for evaluation."+ )- self.add_argument("--disable_opt_rtn", action='store_true',- help="whether to disable optimization of the RTN mode(iters=0) (default is False).")+ self.add_argument(+ "--disable_opt_rtn",+ action="store_true",+ help="whether to disable optimization of the RTN mode(iters=0) (default is False).",+ )class EvalArgumentParser(argparse.ArgumentParser):@@ -212,34 +229,36 @@ class EvalArgumentParser(argparse.ArgumentParser):def __init__(self, *args, **kwargs):super().__init__(*args, **kwargs)self.add_argument(- "--model", "--model_name", "--model_name_or_path", default="facebook/opt-125m", help="model name or path")- self.add_argument(- "--mllm", default=False, help="whether to eval multi-modal model."+ "--model", "--model_name", "--model_name_or_path", default="facebook/opt-125m", help="model name or path")+ self.add_argument("--mllm", default=False, help="whether to eval multi-modal model.")self.add_argument("--device","--devices",default="0",type=str,help="the device to be used for tuning. "- "Currently, device settings support CPU, GPU, and HPU."- "The default is set to cuda:0,"- "allowing for automatic detection and switch to HPU or CPU."- "set --device 0,1,2 to use multiple cards.")-- self.add_argument("--tasks", "--task",- default="lambada_openai,hellaswag,winogrande,piqa,mmlu,wikitext,truthfulqa_mc1," \- "truthfulqa_mc2,openbookqa,boolq,rte,arc_easy,arc_challenge",- help="lm-eval tasks")+ "Currently, device settings support CPU, GPU, and HPU."+ "The default is set to cuda:0,"+ "allowing for automatic detection and switch to HPU or CPU."+ "set --device 0,1,2 to use multiple cards.",+ )++ self.add_argument(+ "--tasks",+ "--task",+ default="lambada_openai,hellaswag,winogrande,piqa,mmlu,wikitext,truthfulqa_mc1,"+ "truthfulqa_mc2,openbookqa,boolq,rte,arc_easy,arc_challenge",+ help="lm-eval tasks",+ )self.add_argument(- "--disable_trust_remote_code", action='store_true', help="whether to disable trust_remote_code")+ "--disable_trust_remote_code", action="store_true", help="whether to disable trust_remote_code"+ )self.add_argument("--eval_bs", "--bs", "--batch_size", default=None, type=int, help="batch size in evaluation")- self.add_argument("--eval_task_by_task", action='store_true', help="whether to eval task by task.")+ self.add_argument("--eval_task_by_task", action="store_true", help="whether to eval task by task.")self.add_argument(- "--eval_model_dtype",- default=None,- type=str,- help="the torch_dytpe to load the model for evaluation.")+ "--eval_model_dtype", default=None, type=str, help="the torch_dytpe to load the model for evaluation."+ )def setup_parser():@@ -252,12 +271,14 @@ def setup_parser():parser.add_argument("--iters", "--iter", default=200, type=int, help="iteration to tune each block")parser.add_argument(- "--seqlen", "--seq_len", default=2048, type=int, help="sequence length of the calibration samples")+ "--seqlen", "--seq_len", default=2048, type=int, help="sequence length of the calibration samples"+ )parser.add_argument("--nsamples", "--nsample", default=128, type=int, help="number of samples")parser.add_argument(- "--lr", default=None, type=float, help="learning rate, if None, it will be set to 1.0/iters automatically")+ "--lr", default=None, type=float, help="learning rate, if None, it will be set to 1.0/iters automatically"+ )args = parser.parse_args()return args@@ -273,12 +294,14 @@ def setup_best_parser():parser.add_argument("--iters", "--iter", default=1000, type=int, help="iterations to tune each block")parser.add_argument(- "--seqlen", "--seq_len", default=2048, type=int, help="sequence length of the calibration samples")+ "--seqlen", "--seq_len", default=2048, type=int, help="sequence length of the calibration samples"+ )parser.add_argument("--nsamples", "--nsample", default=512, type=int, help="number of samples")parser.add_argument(- "--lr", default=None, type=float, help="learning rate, if None, it will be set to 1.0/iters automatically")+ "--lr", default=None, type=float, help="learning rate, if None, it will be set to 1.0/iters automatically"+ )args = parser.parse_args()args.low_gpu_mem_usage = True@@ -296,12 +319,14 @@ def setup_light_parser():parser.add_argument("--iters", "--iter", default=50, type=int, help="iterations to tune each block")parser.add_argument(- "--seqlen", "--seq_len", default=2048, type=int, help="sequence length of the calibration samples")+ "--seqlen", "--seq_len", default=2048, type=int, help="sequence length of the calibration samples"+ )parser.add_argument("--nsamples", "--nsample", default=128, type=int, help="number of samples")parser.add_argument(- "--lr", default=5e-3, type=float, help="learning rate, if None, it will be set to 1.0/iters automatically")+ "--lr", default=5e-3, type=float, help="learning rate, if None, it will be set to 1.0/iters automatically"+ )args = parser.parse_args()@@ -318,12 +343,14 @@ def setup_fast_parser():parser.add_argument("--iters", default=200, type=int, help="iterations to tune each block")parser.add_argument(- "--seqlen", "--seq_len", default=512, type=int, help="sequence length of the calibration samples")+ "--seqlen", "--seq_len", default=512, type=int, help="sequence length of the calibration samples"+ )parser.add_argument("--nsamples", "--nsample", default=128, type=int, help="number of samples")parser.add_argument(- "--lr", default=None, type=float, help="learning rate, if None, it will be set to 1.0/iters automatically")+ "--lr", default=None, type=float, help="learning rate, if None, it will be set to 1.0/iters automatically"+ )args = parser.parse_args()@@ -351,15 +378,18 @@ def tune(args):if args.format is None:args.format = "auto_round"- formats = args.format.lower().replace(' ', '').split(",")+ formats = args.format.lower().replace(" ", "").split(",")from auto_round.utils import SUPPORTED_FORMATS+for format in formats:if format not in SUPPORTED_FORMATS:raise ValueError(f"{format} is not supported, we only support {SUPPORTED_FORMATS}")if "auto_gptq" in args.format and args.asym is True:- logger.warning("the auto_gptq kernel has issues with asymmetric quantization. "- "It is recommended to use sym quantization or --format='auto_round'")+ logger.warning(+ "the auto_gptq kernel has issues with asymmetric quantization. "+ "It is recommended to use sym quantization or --format='auto_round'"+ )if "marlin" in args.format and args.asym is True:raise RuntimeError("marlin backend only supports sym quantization, please remove --asym")@@ -369,14 +399,17 @@ def tune(args):device_str, use_auto_mapping = get_device_and_parallelism(args.device)import torch+if not args.disable_deterministic_algorithms:torch.use_deterministic_algorithms(True, warn_only=True)# logger.info("`torch.use_deterministic_algorithms` is enabled by default for reproducibility "# "and can be disabled using the `--disable_deterministic_algorithms` argument.")if args.enable_torch_compile:- logger.info("`torch.compile` is enabled to reduce tuning costs. "- "If it causes issues, you can disable it by removing `--enable_torch_compile` argument.")+ logger.info(+ "`torch.compile` is enabled to reduce tuning costs. "+ "If it causes issues, you can disable it by removing `--enable_torch_compile` argument."+ )model_name = args.modelif model_name[-1] == "/":@@ -387,6 +420,7 @@ def tune(args):torch_dtype = torch.bfloat16from auto_round.utils import llm_load_model+model, tokenizer, low_cpu_mem_usage = llm_load_model(model_name,torch_dtype=torch_dtype,@@ -395,7 +429,8 @@ def tune(args):device=device_str,low_cpu_mem_mode=args.low_cpu_mem_mode,low_cpu_mem_tmp_dir=args.low_cpu_mem_tmp_dir,- model_dtype=args.model_dtype)+ model_dtype=args.model_dtype,+ )from auto_round import AutoRound, AutoRoundAdam@@ -429,7 +464,8 @@ def tune(args):args.quant_lm_head = Falselogger.warning("reset `quant_lm_head` to `False` as quantizing lm_head with tied weights has not been "- "supported currently")+ "supported currently"+ )breakif args.quant_lm_head:@@ -438,12 +474,15 @@ def tune(args):if "auto_round" not in format and "fake" not in format:auto_round_formats = [s for s in SUPPORTED_FORMATS if s.startswith("auto_round")]raise ValueError(- f"{format} is not supported for lm-head quantization, please change to {auto_round_formats}")+ f"{format} is not supported for lm-head quantization, please change to {auto_round_formats}"+ )if "auto_awq" in args.format:from auto_round.utils import check_awq_gemm_compatibility+awq_supported, info = check_awq_gemm_compatibility(- model, args.bits, args.group_size, not args.asym, layer_config)+ model, args.bits, args.group_size, not args.asym, layer_config+ )if not awq_supported:logger.warning(f"The AutoAWQ format may not be supported due to {info}")@@ -489,7 +528,7 @@ def tune(args):model_name = args.model.rstrip("/")- if model_name.split('/')[-1].strip('.') == "" and "gguf" not in args.format:+ if model_name.split("/")[-1].strip(".") == "" and "gguf" not in args.format:if autoround.group_size <= 0:if "fp" in autoround.act_data_type:suffix = f"afp{autoround.act_bits}"@@ -498,11 +537,10 @@ def tune(args):else:suffix = f"g{autoround.group_size}"export_dir = os.path.join(args.output_dir, f"w{autoround.bits}{suffix}")- elif model_name.split('/')[-1].strip('.') == "" and "gguf" in args.format:+ elif model_name.split("/")[-1].strip(".") == "" and "gguf" in args.format:export_dir = args.output_dir- elif model_name.split('./')[-1].strip('./') != "" and "gguf" in args.format:- export_dir = os.path.join(args.output_dir,- model_name.split('/')[-1] + "-gguf")+ elif model_name.split("./")[-1].strip("./") != "" and "gguf" in args.format:+ export_dir = os.path.join(args.output_dir, model_name.split("/")[-1] + "-gguf")else:if autoround.group_size <= 0:if "fp" in autoround.act_data_type:@@ -511,14 +549,13 @@ def tune(args):suffix = f"a{autoround.act_bits}"else:suffix = f"g{autoround.group_size}"- export_dir = os.path.join(- args.output_dir,- model_name.split('/')[-1] + f"-w{autoround.bits}{suffix}")+ export_dir = os.path.join(args.output_dir, model_name.split("/")[-1] + f"-w{autoround.bits}{suffix}")model, folders = autoround.quantize_and_save(export_dir, format=args.format)if args.low_cpu_mem_mode == 1 or args.low_cpu_mem_mode == 2:import shutil+shutil.rmtree(args.low_cpu_mem_tmp_dir, ignore_errors=True)model.eval()@@ -532,7 +569,7 @@ def tune(args):tasks = args.tasksif isinstance(tasks, str):- tasks = tasks.split(',')+ tasks = tasks.split(",")from lm_eval.utils import make_table # pylint: disable=E0401@@ -544,6 +581,7 @@ def tune(args):breakimport time+eval_model_dtype = get_model_dtype(args.eval_model_dtype, "auto")if args.act_bits <= 8 or eval_gguf_model:if eval_gguf_model:@@ -572,7 +610,8 @@ def tune(args):logger.error("Cannot find correct gguf file for evaluation, please check.")sys.exit(-1)model = AutoModelForCausalLM.from_pretrained(- eval_folder, gguf_file=gguf_file, device_map="auto", torch_dtype=eval_model_dtype)+ eval_folder, gguf_file=gguf_file, device_map="auto", torch_dtype=eval_model_dtype+ )model.eval()tokenizer = AutoTokenizer.from_pretrained(eval_folder, gguf_file=gguf_file)else:@@ -593,16 +632,18 @@ def tune(args):device=device_str,tasks=args.tasks,batch_size=args.eval_bs,- eval_model_dtype=eval_model_dtype)+ eval_model_dtype=eval_model_dtype,+ )else:if args.eval_bs is None or args.eval_bs == "auto":logger.warning("This API does not support auto currently, reset eval_bs to 16")args.eval_bs = 16from auto_round.eval.evaluation import simple_evaluate_user_model+st = time.time()- add_bos_token=False+ add_bos_token = Falseif "llama" in args.model.lower():- add_bos_token=True+ add_bos_token = Trueres = simple_evaluate_user_model(model,tokenizer,@@ -610,8 +651,8 @@ def tune(args):batch_size=args.eval_bs,device=device_str,eval_model_dtype=eval_model_dtype,- add_bos_token=add_bos_token- )+ add_bos_token=add_bos_token,+ )print(make_table(res))print("evaluation running time=%ds" % (time.time() - st))else:@@ -621,14 +662,18 @@ def tune(args):device=device_str,tasks=args.tasks,batch_size=args.eval_bs,- eval_model_dtype=eval_model_dtype)+ eval_model_dtype=eval_model_dtype,+ )else:from auto_round.eval.evaluation import simple_evaluate+tasks, model_args, device_str = _eval_init(- args.tasks, eval_folder, args.device, args.disable_trust_remote_code, dtype=eval_model_dtype)+ args.tasks, eval_folder, args.device, args.disable_trust_remote_code, dtype=eval_model_dtype+ )st = time.time()res = simple_evaluate(- model="hf", model_args=model_args, tasks=tasks, device=device_str, batch_size=args.eval_bs)+ model="hf", model_args=model_args, tasks=tasks, device=device_str, batch_size=args.eval_bs+ )print(make_table(res))print("evaluation running time=%ds" % (time.time() - st))@@ -639,18 +684,20 @@ def _eval_init(tasks, model_path, device, disable_trust_remote_code=False, dtypeif dtype != "auto":dtype = get_model_dtype(model_dtype=dtype)# ,add_bos_token={True}- model_args = f'pretrained={model_path},trust_remote_code={not disable_trust_remote_code},dtype={dtype}'+ model_args = f"pretrained={model_path},trust_remote_code={not disable_trust_remote_code},dtype={dtype}"if parallelism:model_args += ",parallelize=True"if isinstance(tasks, str):- tasks = tasks.split(',')+ tasks = tasks.split(",")return tasks, model_args, device_strdef eval(args):import time+tasks, model_args, device_str = _eval_init(- args.tasks, args.model, args.device, args.disable_trust_remote_code, args.eval_model_dtype)+ args.tasks, args.model, args.device, args.disable_trust_remote_code, args.eval_model_dtype+ )# load after _eval_int in order to make sure import torch after set CUDA_VISIBLE_DEVICESfrom auto_round.eval.evaluation import simple_evaluate, simple_evaluate_user_model@@ -673,6 +720,7 @@ def eval(args):import torchfrom lm_eval.utils import make_table # pylint: disable=E0401from transformers import AutoModelForCausalLM, AutoTokenizer+tokenizer = AutoTokenizer.from_pretrained(model, gguf_file=gguf_file)logger.warning("evaluating gguf model is an experimental feature, the accuracy may be not correct.")@@ -682,11 +730,11 @@ def eval(args):" but may affect accuracy.")model = AutoModelForCausalLM.from_pretrained(- model, gguf_file=gguf_file, device_map="auto", torch_dtype=eval_model_dtype)+ model, gguf_file=gguf_file, device_map="auto", torch_dtype=eval_model_dtype+ )model.eval()st = time.time()- res = simple_evaluate_user_model(- model, tokenizer, tasks=tasks, batch_size=batch_size, device=device_str)+ res = simple_evaluate_user_model(model, tokenizer, tasks=tasks, batch_size=batch_size, device=device_str)print(make_table(res))print("evaluation running time=%ds" % (time.time() - st))else:@@ -699,22 +747,25 @@ def eval(args):model_args=model_args,tasks=tasks,device=device_str,- batch_size=batch_size)+ batch_size=batch_size,+ )from lm_eval.utils import make_table # pylint: disable=E0401+print(make_table(res))print("evaluation running time=%ds" % (time.time() - st))def eval_task_by_task(- model,- device=None,- tasks=None,- tokenizer=None,- batch_size=None,- max_batch_size=64,- trust_remote_code=True,- eval_model_dtype=None,- retry_times=3):+ model,+ device=None,+ tasks=None,+ tokenizer=None,+ batch_size=None,+ max_batch_size=64,+ trust_remote_code=True,+ eval_model_dtype=None,+ retry_times=3,+):set_cuda_visible_devices(device)device_str, parallelism = get_device_and_parallelism(device)@@ -727,6 +778,7 @@ def eval_task_by_task(from auto_round import AutoRoundConfig # pylint: disable=E0611from auto_round.utils import logger+if batch_size is None:batch_size = "auto:8"is_gguf_file = False@@ -753,7 +805,8 @@ def eval_task_by_task()model = AutoModelForCausalLM.from_pretrained(- model, gguf_file=gguf_file, device_map="auto", torch_dtype=eval_model_dtype)+ model, gguf_file=gguf_file, device_map="auto", torch_dtype=eval_model_dtype+ )model.eval()parallelism = Falsehflm = HFLM(@@ -764,22 +817,25 @@ def eval_task_by_task(max_batch_size=max_batch_size,parallelize=parallelism,trust_remote_code=trust_remote_code,- dtype=eval_model_dtype+ dtype=eval_model_dtype,)if isinstance(tasks, str):tasks = tasks.replace(" ", "").split(",")from lm_eval.utils import make_table # pylint: disable=E0401+res_all = {}res_keys = ["results", "versions", "n-shot", "higher_is_better"]import time+st = time.time()for task in tasks:while retry_times:try:res = lm_simple_evaluate(- model=hflm, model_args=None, device=device_str, tasks=task, batch_size=batch_size)+ model=hflm, model_args=None, device=device_str, tasks=task, batch_size=batch_size+ )breakexcept Exception as e:if "CUDA out of memory" in str(e) or "MODULE:PT_DEVMEM" in str(e):@@ -787,10 +843,10 @@ def eval_task_by_task(try:for k, v in hflm.batch_sizes.items():hflm.batch_sizes[k] = max(v // 2, 1)- logger.warning(- f"Out of memory, reset batch_size to {hflm.batch_sizes} and re-try.")+ logger.warning(f"Out of memory, reset batch_size to {hflm.batch_sizes} and re-try.")res = lm_simple_evaluate(- model=hflm, model_args=None, device=device_str, tasks=task, batch_size=1)+ model=hflm, model_args=None, device=device_str, tasks=task, batch_size=1+ )hflm.batch_sizes = ori_batch_sizesexcept Exception as e:traceback.print_exc()diff --git a/auto_round/script/mllm.py b/auto_round/script/mllm.pyindex dabbc4014..97fa82a9d 100644--- a/auto_round/script/mllm.py+++ b/auto_round/script/mllm.py@@ -37,9 +37,10 @@ class BasicArgumentParser(argparse.ArgumentParser):"--model_name","--model_name_or_path",default="Qwen/Qwen2-VL-2B-Instruct",- help="model name or path")+ help="model name or path",+ )- self.add_argument('--eval', action='store_true', help="whether to use eval only mode.")+ self.add_argument("--eval", action="store_true", help="whether to use eval only mode.")self.add_argument("--bits", default=4, type=int, help="weight bits")@@ -51,70 +52,79 @@ class BasicArgumentParser(argparse.ArgumentParser):default="0",type=str,help="the device to be used for tuning. "- "Currently, device settings support CPU, GPU, and HPU."- "The default is set to cuda:0,"- "allowing for automatic detection and switch to HPU or CPU."- "set --device 0,1,2 to use multiple cards.")+ "Currently, device settings support CPU, GPU, and HPU."+ "The default is set to cuda:0,"+ "allowing for automatic detection and switch to HPU or CPU."+ "set --device 0,1,2 to use multiple cards.",+ )- self.add_argument("--asym", action='store_true', help="whether to use asym quantization")+ self.add_argument("--asym", action="store_true", help="whether to use asym quantization")self.add_argument("--dataset",type=str,default=None,help="the dataset for quantization training."- " current support NeelNanda/pile-10k,liuhaotian/llava_conv_58k,"- "liuhaotian/llava_instruct_80k,liuhaotian/llava_instruct_150k"- "It can be a custom one. Default is NeelNanda/pile-10k")+ " current support NeelNanda/pile-10k,liuhaotian/llava_conv_58k,"+ "liuhaotian/llava_instruct_80k,liuhaotian/llava_instruct_150k"+ "It can be a custom one. Default is NeelNanda/pile-10k",+ )self.add_argument(- "--lr", default=None, type=float, help="learning rate, if None, it will be set to 1.0/iters automatically")+ "--lr", default=None, type=float, help="learning rate, if None, it will be set to 1.0/iters automatically"+ )self.add_argument("--minmax_lr",default=None,type=float,- help="minmax learning rate, if None,it will beset to be the same with lr")+ help="minmax learning rate, if None,it will beset to be the same with lr",+ )self.add_argument("--seed", default=42, type=int, help="random seed")- self.add_argument("--adam", action='store_true', help="whether to use adam optimizer instead of SignSGD")+ self.add_argument("--adam", action="store_true", help="whether to use adam optimizer instead of SignSGD")self.add_argument("--gradient_accumulate_steps", default=1, type=int, help="gradient accumulate steps")self.add_argument("--nblocks", default=1, type=int, help="how many blocks to tune together")- self.add_argument("--low_gpu_mem_usage", action='store_true', help="offload intermediate features to cpu")+ self.add_argument("--low_gpu_mem_usage", action="store_true", help="offload intermediate features to cpu")self.add_argument("--format", default="auto_round", type=str, help="the format to save the model")- self.add_argument("--data_type", "--dtype", default='int', help="data type for tuning, 'int', 'mx_fp' and etc")+ self.add_argument("--data_type", "--dtype", default="int", help="data type for tuning, 'int', 'mx_fp' and etc")self.add_argument("--scale_dtype",- default='fp16',+ default="fp16",choices=["fp16", "float16", "bf16", "bfloat16", "fp32", "float32"],- help="scale data type to use for quantization")+ help="scale data type to use for quantization",+ )self.add_argument(- "--output_dir", default="./tmp_autoround", type=str, help="the directory to save quantized model")+ "--output_dir", default="./tmp_autoround", type=str, help="the directory to save quantized model"+ )- self.add_argument("--disable_amp", action='store_true', help="disable amp")+ self.add_argument("--disable_amp", action="store_true", help="disable amp")self.add_argument(- "--disable_minmax_tuning", action='store_true', help="whether disable enable weight minmax tuning")+ "--disable_minmax_tuning", action="store_true", help="whether disable enable weight minmax tuning"+ )- self.add_argument("--enable_norm_bias_tuning", action='store_true', help="whether enable norm bias tuning")+ self.add_argument("--enable_norm_bias_tuning", action="store_true", help="whether enable norm bias tuning")self.add_argument(- "--disable_trust_remote_code", action='store_true', help="whether to disable trust_remote_code")+ "--disable_trust_remote_code", action="store_true", help="whether to disable trust_remote_code"+ )self.add_argument("--disable_quanted_input",- action='store_true',- help="whether to disuse the output of quantized block to tune the next block")+ action="store_true",+ help="whether to disuse the output of quantized block to tune the next block",+ )- self.add_argument("--quant_lm_head", action='store_true', help="whether to quant lm_head")+ self.add_argument("--quant_lm_head", action="store_true", help="whether to quant lm_head")self.add_argument("--low_cpu_mem_mode",@@ -122,26 +132,29 @@ class BasicArgumentParser(argparse.ArgumentParser):type=int,choices=[0, 1, 2],help="choose which low cpu memory mode to use. "- "Can significantly reduce cpu memory footprint but cost more time."- "1 means choose block-wise mode, load the weights of each block"- " from disk when tuning and release the memory of the block after tuning."- "2 means choose layer-wise mode, load the weights of each layer from disk when tuning,"- " minimum memory consumption and also slowest running speed."- "others means not use low cpu memory. Default to 0, not use low cpu memory.")+ "Can significantly reduce cpu memory footprint but cost more time."+ "1 means choose block-wise mode, load the weights of each block"+ " from disk when tuning and release the memory of the block after tuning."+ "2 means choose layer-wise mode, load the weights of each layer from disk when tuning,"+ " minimum memory consumption and also slowest running speed."+ "others means not use low cpu memory. Default to 0, not use low cpu memory.",+ )self.add_argument("--low_cpu_mem_tmp_dir",default=None,type=str,help="temporary work space to store the temporary files "- "when using low cpu memory mode. Will remove after tuning.")+ "when using low cpu memory mode. Will remove after tuning.",+ )self.add_argument("--model_dtype",default=None,type=str,choices=["fp16", "float16", "bf16", "bfloat16", "fp32", "float32"],- help="force to convert the dtype, some backends supports fp16 dtype better")+ help="force to convert the dtype, some backends supports fp16 dtype better",+ )self.add_argument("--act_bits", default=16, type=int, help="activation bits")@@ -149,53 +162,62 @@ class BasicArgumentParser(argparse.ArgumentParser):self.add_argument("--not_use_best_mse",- action='store_true',- help="whether to use the iter of best mes loss in the tuning phase")+ action="store_true",+ help="whether to use the iter of best mes loss in the tuning phase",+ )- self.add_argument("--enable_torch_compile", action='store_true',- help="whether to enable torch compile")+ self.add_argument("--enable_torch_compile", action="store_true", help="whether to enable torch compile")- self.add_argument("--disable_deterministic_algorithms", action='store_true',- help="disable torch deterministic algorithms.")+ self.add_argument(+ "--disable_deterministic_algorithms", action="store_true", help="disable torch deterministic algorithms."+ )## ======================= VLM =======================self.add_argument("--quant_nontext_module",- action='store_true',- help="whether to quantize non-text module, e.g. vision component")+ action="store_true",+ help="whether to quantize non-text module, e.g. vision component",+ )self.add_argument("--extra_data_dir",default=None,type=str,help="dataset dir for storing images/audio/videos. "- "Can be a dir path or multiple dir path with format as "- "'image=path_to_image,video=path_to_video,audio=path_to_audio'"- "By default, it will search in the relative path, "- "and if not find, will automatic download.")+ "Can be a dir path or multiple dir path with format as "+ "'image=path_to_image,video=path_to_video,audio=path_to_audio'"+ "By default, it will search in the relative path, "+ "and if not find, will automatic download.",+ )self.add_argument("--template",default=None,type=str,- help="the template for building training dataset. It can be a custom one.")+ help="the template for building training dataset. It can be a custom one.",+ )self.add_argument("--truncation",action="store_true",help="whether to truncate sequences at the maximum length."- " Default True for pile and False for llava dataset.")+ " Default True for pile and False for llava dataset.",+ )self.add_argument("--to_quant_block_names",default=None,type=str,- help="Names of quantitative blocks, please use commas to separate them.")+ help="Names of quantitative blocks, please use commas to separate them.",+ )self.add_argument("--device_map", default=None, type=str, help="device_map for block in tuning phase")- self.add_argument("--disable_opt_rtn", action='store_true',- help="whether to disable optimization of the RTN mode(iters=0) (default is False).")+ self.add_argument(+ "--disable_opt_rtn",+ action="store_true",+ help="whether to disable optimization of the RTN mode(iters=0) (default is False).",+ )def setup_parser():@@ -212,7 +234,8 @@ def setup_parser():"--seq_len",default=None,type=int,- help="sequence length, default 2048 for text-only, 512 for liuhaotian/llava")+ help="sequence length, default 2048 for text-only, 512 for liuhaotian/llava",+ )parser.add_argument("--nsamples", "--nsample", default=128, type=int, help="number of samples")@@ -229,54 +252,60 @@ def setup_lmeval_parser():default="0",type=str,help="the device to be used for tuning. "- "Currently, device settings support CPU, GPU, and HPU."- "The default is set to cuda:0,"- "allowing for automatic detection and switch to HPU or CPU."- "set --device 0,1,2 to use multiple cards.")+ "Currently, device settings support CPU, GPU, and HPU."+ "The default is set to cuda:0,"+ "allowing for automatic detection and switch to HPU or CPU."+ "set --device 0,1,2 to use multiple cards.",+ )parser.add_argument("--tasks",type=str,default="MMBench_DEV_EN_V11,ScienceQA_VAL,TextVQA_VAL,POPE",- help="eval tasks for VLMEvalKit.")+ help="eval tasks for VLMEvalKit.",+ )# Args that only apply to Video Datasetparser.add_argument("--nframe",type=int,default=8,- help="the number of frames to sample from a video,"- " only applicable to the evaluation of video benchmarks.")+ help="the number of frames to sample from a video," " only applicable to the evaluation of video benchmarks.",+ )parser.add_argument("--pack",- action='store_true',+ action="store_true",help="a video may associate with multiple questions, if pack==True,"- " will ask all questions for a video in a single")+ " will ask all questions for a video in a single",+ )parser.add_argument("--fps", type=float, default=-1, help="set the fps for a video.")# Work Dir# Infer + Eval or Infer Onlyparser.add_argument("--mode",type=str,- default='all',- choices=['all', 'infer'],+ default="all",+ choices=["all", "infer"],help="when mode set to 'all', will perform both inference and evaluation;"- " when set to 'infer' will only perform the inference.")+ " when set to 'infer' will only perform the inference.",+ )parser.add_argument(- '--eval_data_dir',+ "--eval_data_dir",type=str,default=None,- help='path for VLMEvalKit to store the eval data. Default will store in ~/LMUData')+ help="path for VLMEvalKit to store the eval data. Default will store in ~/LMUData",+ )# API Kwargs, Apply to API VLMs and Judge API LLMs- parser.add_argument('--retry', type=int, default=None, help='retry numbers for API VLMs')+ parser.add_argument("--retry", type=int, default=None, help="retry numbers for API VLMs")# Explicitly Set the Judge Model- parser.add_argument('--judge', type=str, default=None, help="whether is a judge model.")+ parser.add_argument("--judge", type=str, default=None, help="whether is a judge model.")# Logging Utils- parser.add_argument('--verbose', action='store_true', help="whether to display verbose information.")+ parser.add_argument("--verbose", action="store_true", help="whether to display verbose information.")# Configuration for Resume# Ignore: will not rerun failed VLM inference- parser.add_argument('--ignore', action='store_true', help='ignore failed indices. ')+ parser.add_argument("--ignore", action="store_true", help="ignore failed indices. ")# Rerun: will remove all evaluation temp filesparser.add_argument(- '--rerun', action='store_true', help="if true, will remove all evaluation temp files and rerun.")+ "--rerun", action="store_true", help="if true, will remove all evaluation temp files and rerun."+ )parser.add_argument("--output_dir", default="./eval_result", type=str, help="the directory to save quantized model")args = parser.parse_args()return args@@ -289,8 +318,9 @@ def tune(args):if args.format is None:args.format = "auto_round"- formats = args.format.replace(' ', '').split(",")+ formats = args.format.replace(" ", "").split(",")from auto_round.utils import SUPPORTED_FORMATS+for format in formats:if format not in SUPPORTED_FORMATS:raise ValueError(f"{format} is not supported, we only support {SUPPORTED_FORMATS}")@@ -300,10 +330,13 @@ def tune(args):device_str, use_auto_mapping = get_device_and_parallelism(args.device)import torch+if not args.disable_deterministic_algorithms:torch.use_deterministic_algorithms(True, warn_only=True)- print("'torch.use_deterministic_algorithms' is turned on by default for reproducibility, " \- "and can be turned off by setting the '--disable_deterministic_algorithms' parameter.")+ print(+ "'torch.use_deterministic_algorithms' is turned on by default for reproducibility, "+ "and can be turned off by setting the '--disable_deterministic_algorithms' parameter."+ )model_name = args.modelif model_name[-1] == "/":@@ -315,12 +348,14 @@ def tune(args):# load_modelfrom auto_round.utils import mllm_load_model+model, processor, tokenizer, image_processor = mllm_load_model(model_name,torch_dtype=torch_dtype,use_auto_mapping=use_auto_mapping,trust_remote_code=not args.disable_trust_remote_code,- model_dtype=args.model_dtype)+ model_dtype=args.model_dtype,+ )from auto_round import AutoRoundMLLM@@ -331,7 +366,8 @@ def tune(args):if seqlen is not None and hasattr(tokenizer, "model_max_length"):if tokenizer.model_max_length < seqlen:logger.info(- f"change sequence length to {tokenizer.model_max_length} due to the limitation of model_max_length")+ f"change sequence length to {tokenizer.model_max_length} due to the limitation of model_max_length"+ )seqlen = min(seqlen, tokenizer.model_max_length)args.seqlen = seqlen@@ -377,7 +413,8 @@ def tune(args):args.quant_lm_head = Falseprint("warning, disable quant_lm_head as quantizing lm_head with tied weights has not been "- "supported currently")+ "supported currently"+ )breakif args.quant_lm_head:layer_config[lm_head_layer_name] = {"bits": args.bits}@@ -385,20 +422,21 @@ def tune(args):if "auto_round" not in format and "fake" not in format:auto_round_formats = [s for s in SUPPORTED_FORMATS if s.startswith("auto_round")]raise ValueError(- f"{format} is not supported for lm-head quantization, please change to {auto_round_formats}")+ f"{format} is not supported for lm-head quantization, please change to {auto_round_formats}"+ )if args.quant_lm_head and args.low_gpu_mem_usage:- print(- "warning, low_gpu_mem_usage=False is strongly recommended if the whole model could be loaded to "- "gpu")+ print("warning, low_gpu_mem_usage=False is strongly recommended if the whole model could be loaded to " "gpu")if "--truncation" not in sys.argv:args.truncation = Noneif "auto_awq" in args.format:from auto_round.utils import check_awq_gemm_compatibility+awq_supported, info = check_awq_gemm_compatibility(- model, args.bits, args.group_size, not args.asym, layer_config)+ model, args.bits, args.group_size, not args.asym, layer_config+ )if not awq_supported:logger.warning(f"The AutoAWQ format may not be supported due to {info}")@@ -407,7 +445,7 @@ def tune(args):model_kwargs = {"use_auto_mapping": use_auto_mapping,"trust_remote_code": not args.disable_trust_remote_code,- "model_dtype": args.model_dtype+ "model_dtype": args.model_dtype,}autoround = round(model,@@ -450,21 +488,22 @@ def tune(args):model_name = args.model.rstrip("/")- if model_name.split('/')[-1].strip('.') == "" and "gguf" not in args.format:+ if model_name.split("/")[-1].strip(".") == "" and "gguf" not in args.format:export_dir = os.path.join(args.output_dir, f"w{autoround.bits}g{autoround.group_size}")- elif model_name.split('/')[-1].strip('.') == "" and "gguf" in args.format:+ elif model_name.split("/")[-1].strip(".") == "" and "gguf" in args.format:export_dir = args.output_dir- elif model_name.split('./')[-1].strip('./') != "" and "gguf" in args.format:- export_dir = os.path.join(args.output_dir,- model_name.split('/')[-1] + "-gguf")+ elif model_name.split("./")[-1].strip("./") != "" and "gguf" in args.format:+ export_dir = os.path.join(args.output_dir, model_name.split("/")[-1] + "-gguf")else:- export_dir = os.path.join(args.output_dir,- model_name.split('/')[-1] + f"-w{autoround.bits}g{autoround.group_size}")+ export_dir = os.path.join(+ args.output_dir, model_name.split("/")[-1] + f"-w{autoround.bits}g{autoround.group_size}"+ )model, folders = autoround.quantize_and_save(export_dir, format=args.format)if args.low_cpu_mem_mode == 1 or args.low_cpu_mem_mode == 2:import shutil+shutil.rmtree(args.low_cpu_mem_tmp_dir, ignore_errors=True)model.eval()@@ -475,10 +514,11 @@ def vlmeval(args):set_cuda_visible_devices(args.device)device_str, parallelism = get_device_and_parallelism(args.device)if parallelism:- os.environ['AUTO_SPLIT'] = '1'+ os.environ["AUTO_SPLIT"] = "1"if isinstance(args.tasks, str):- args.tasks = args.tasks.replace(' ', '').split(',')+ args.tasks = args.tasks.replace(" ", "").split(",")from auto_round.mllm import mllm_eval+mllm_eval(args.model,work_dir=args.output_dir,@@ -491,7 +531,8 @@ def vlmeval(args):judge=args.judge,verbose=args.verbose,mode=args.mode,- ignore=args.ignore)+ ignore=args.ignore,+ )def setup_lmms_parser():@@ -531,17 +572,18 @@ def setup_lmms_parser():default="0",type=str,help="the device to be used for tuning. "- "Currently, device settings support CPU, GPU, and HPU."- "The default is set to cuda:0,"- "allowing for automatic detection and switch to HPU or CPU."- "set --device 0,1,2 to use multiple cards.")+ "Currently, device settings support CPU, GPU, and HPU."+ "The default is set to cuda:0,"+ "allowing for automatic detection and switch to HPU or CPU."+ "set --device 0,1,2 to use multiple cards.",+ )parser.add_argument("--limit",type=float,default=None,help="Limit the number of examples per task. "- "If <1, limit is a percentage of the total"- " number of examples.",+ "If <1, limit is a percentage of the total"+ " number of examples.",)args = parser.parse_args()return argsdiff --git a/auto_round/special_model_handler.py b/auto_round/special_model_handler.pyindex 6d9a0ec95..161ffe158 100644--- a/auto_round/special_model_handler.py+++ b/auto_round/special_model_handler.py@@ -24,17 +24,19 @@ SUPPORT_ONLY_TEXT_MODELS = ["chatglm","idefics3","llama4",- "internvl_chat"+ "internvl_chat",]SPECIAL_SHARED_CACHE_KEYS = {- "Gemma3ForConditionalGeneration": ("position_embeddings_global", "position_embeddings_local")}+ "Gemma3ForConditionalGeneration": ("position_embeddings_global", "position_embeddings_local")+}SPECIAL_SHARED_CACHE_KEYS["MiniMaxText01ForCausalLM"] = ("slope_rate",)def _handle_special_model(model):if model.config.model_type == "deepseek_vl_v2":from functools import partial+model.forward = partial(_deepseek_vl2_forward, model)return model@@ -49,31 +51,26 @@ def _get_deepseek_vl2_multimodal_block(model, quant_vision=False):return block_names-SPECIAL_MULTIMODAL_BLOCK = {- "deepseek_vl_v2": _get_deepseek_vl2_multimodal_block-}+SPECIAL_MULTIMODAL_BLOCK = {"deepseek_vl_v2": _get_deepseek_vl2_multimodal_block}def _deepseek_vl2_forward(- model,- input_ids=None,-- position_ids=None,- attention_mask=None,- past_key_values=None,- inputs_embeds=None,-- images=None,- images_seq_mask=None,- images_spatial_crop=None,-- labels=None,- use_cache=None,- output_attentions=None,- output_hidden_states=None,- return_dict=None,- cache_position=None,- **kwargs+ model,+ input_ids=None,+ position_ids=None,+ attention_mask=None,+ past_key_values=None,+ inputs_embeds=None,+ images=None,+ images_seq_mask=None,+ images_spatial_crop=None,+ labels=None,+ use_cache=None,+ output_attentions=None,+ output_hidden_states=None,+ return_dict=None,+ cache_position=None,+ **kwargs,):inputs_embeds = model.prepare_inputs_embeds(input_ids=input_ids,@@ -92,7 +89,8 @@ def _deepseek_vl2_forward(output_attentions=output_attentions,output_hidden_states=output_hidden_states,return_dict=return_dict,- cache_position=cache_position)+ cache_position=cache_position,+ )def check_mllm_model_batch(model, batch_size, gradient_accumulate_steps=1):@@ -102,7 +100,9 @@ def check_mllm_model_batch(model, batch_size, gradient_accumulate_steps=1):for key in mllms_with_limited_bs:if hasattr(model, "config") and key in model.config.model_type and batch_size != 1:accumulate_steps = batch_size * gradient_accumulate_steps- print("To avoid the tensor concat mismatch problem, modified parameters to " \- f"batch_size=1. As an alternative, set the gradient_accumulate_steps={accumulate_steps}")+ print(+ "To avoid the tensor concat mismatch problem, modified parameters to "+ f"batch_size=1. As an alternative, set the gradient_accumulate_steps={accumulate_steps}"+ )return 1, accumulate_stepsreturn batch_size, gradient_accumulate_stepsdiff --git a/auto_round/testing_utils.py b/auto_round/testing_utils.pyindex c859c6188..59d8a5d5c 100644--- a/auto_round/testing_utils.py+++ b/auto_round/testing_utils.py@@ -22,15 +22,19 @@ from transformers.utils.versions import require_versiondef is_gguf_available():return importlib.util.find_spec("gguf") is not None+def is_autogptq_available():return importlib.util.find_spec("auto_gptq") is not None+def is_awq_available():return importlib.util.find_spec("awq") is not None+def is_optimum_available():return importlib.util.find_spec("optimum") is not None+def is_ipex_available():try:require_version("intel-extension-for-pytorch>=2.5")@@ -38,12 +42,15 @@ def is_ipex_available():except ImportError:return False+def is_itrex_available():return importlib.util.find_spec("intel_extension_for_transformers") is not None+def is_flash_attn_avaliable():return importlib.util.find_spec("flash_attn") is not None+def is_gptqmodel_available():try:require_version("gptqmodel>=2.0")@@ -51,9 +58,10 @@ def is_gptqmodel_available():except ImportError:return False+def greater_than_050():try:- require_version("auto-round>=0.5.0")+ require_version("auto-round>=0.5.0")return Trueexcept ImportError:return False@@ -126,6 +134,7 @@ def require_itrex(test_case):"""return unittest.skipUnless(is_itrex_available(), "test requires intel-extension-for-transformers")(test_case)+def require_optimum(test_case):"""Decorator marking a test that optimum.@@ -164,7 +173,8 @@ def multi_card(test_case):"""return unittest.skipUnless(- torch.cuda.is_available() and torch.cuda.device_count() > 1, "test requires multiple cards.")(test_case)+ torch.cuda.is_available() and torch.cuda.device_count() > 1, "test requires multiple cards."+ )(test_case)def require_old_version(test_case):@@ -201,8 +211,3 @@ def require_vlm_env(test_case):env_check &= importlib.util.find_spec("xformers") is not Nonereturn unittest.skipUnless(env_check, "Environment is not satisfactory")(test_case)-----diff --git a/auto_round/utils.py b/auto_round/utils.pyindex 988af3db8..eadc090d6 100644--- a/auto_round/utils.py+++ b/auto_round/utils.py@@ -38,8 +38,16 @@ SHARED_CACHE_KEYS = ("position_ids", "cache_position", "position_embeddings")class SupportedFormats:def __init__(self):self._support_format = (- "auto_round", "auto_gptq", "auto_awq", "auto_round:auto_gptq", "auto_round:gptqmodel",- "auto_round:auto_awq", "itrex", "itrex_xpu", "fake")+ "auto_round",+ "auto_gptq",+ "auto_awq",+ "auto_round:auto_gptq",+ "auto_round:gptqmodel",+ "auto_round:auto_awq",+ "itrex",+ "itrex_xpu",+ "fake",+ )self._gguf_format = tuple(sorted(GGUF_CONFIG.keys()))self._support_list = self._support_format + self._gguf_format@@ -48,7 +56,7 @@ class SupportedFormats:def __str__(self):##return "(%s)" % ', '.join(self._support_format + ("gguf:q*_0", "gguf:q*_1", "gguf:q*_k_s"))- return "(%s)" % ', '.join(self._support_list)+ return "(%s)" % ", ".join(self._support_list)def __getitem__(self, key):return self._support_list[key]@@ -65,7 +73,7 @@ def infer_bits_by_data_type(data_type: str):for supported_dtype in SUPPORTED_DTYPES:if data_type.startswith(supported_dtype) and len(data_type) > len(supported_dtype):##first check the following two bits- suc_2str = data_type[len(supported_dtype):len(supported_dtype) + 2]+ suc_2str = data_type[len(supported_dtype) : len(supported_dtype) + 2]if str.isdigit(suc_2str):return int(suc_2str)if str.isdigit(data_type[len(supported_dtype)]):@@ -91,7 +99,7 @@ class AutoRoundFormatter(logging.Formatter):logging.INFO: grey + _format + reset,logging.WARNING: yellow + _format + reset,logging.ERROR: bold_red + _format + reset,- logging.CRITICAL: bold_red + _format + reset+ logging.CRITICAL: bold_red + _format + reset,}def format(self, record):@@ -195,7 +203,7 @@ def get_scale_shape(weight, group_size):The shape of the scale tensor to be used for quantization."""if group_size == 0:- return (1)+ return 1elif group_size == -1 or weight.shape[1] < group_size:shape = weight.shape[0]else:@@ -211,16 +219,16 @@ def unsupport_meta_device(model):model: The model to be checked.Returns:- bool: True if the model is valid, False otherwise.+ bool: True if the model is valid, False otherwise."""target_device = Nonefor param in model.parameters():if target_device is None:target_device = param.deviceif param.device != target_device:- if param.device.type == 'meta' or target_device.type == 'meta':+ if param.device.type == "meta" or target_device.type == "meta":return True- if target_device.type == 'meta':+ if target_device.type == "meta":if hasattr(model, "path"):return Falseelse:@@ -277,9 +285,9 @@ def mv_module_from_gpu(module, low_cpu_mem_usage=False):return module.to(target_device)else:if low_cpu_mem_usage:- return module.to('meta')+ return module.to("meta")else:- return module.to('cpu')+ return module.to("cpu")def to_dtype(input, dtype=torch.float32):@@ -327,11 +335,11 @@ def check_is_cpu(device):def get_common_prefix(paths):# Split each path into components and find the common prefix- split_paths = [path.split('.') for path in paths]+ split_paths = [path.split(".") for path in paths]common_prefix = split_paths[0]for path in split_paths[1:]:common_prefix = [comp for comp, other in zip(common_prefix, path) if comp == other]- return '.'.join(common_prefix)+ return ".".join(common_prefix)def extract_block_names_to_str(quant_block_list):@@ -340,18 +348,18 @@ def extract_block_names_to_str(quant_block_list):# Extract common prefix for each listprefixes = [get_common_prefix(blocks) for blocks in quant_block_list]# Join prefixes into a single string- return ','.join(prefixes)+ return ",".join(prefixes)def find_matching_blocks(model, all_blocks, to_quant_block_names):"""Find and return matching blocks in the model based on to_quant_block_names.-+Args:model: The model (not used in this specific function but kept for completeness).all_blocks: List of lists, where each inner list contains full block names in the model.to_quant_block_names: Comma-separated string of target block names to match.-+Returns:target_blocks: List of lists containing full paths of matching blocks in the model."""@@ -372,8 +380,10 @@ def find_matching_blocks(model, all_blocks, to_quant_block_names):if matched_sublist:target_blocks.append(matched_sublist)if not target_blocks:- raise ValueError("No block names matched. Please check the input for to_quant_block_name," \- "or set to_quant_block_name to None to automatically match quantizable blocks.")+ raise ValueError(+ "No block names matched. Please check the input for to_quant_block_name,"+ "or set to_quant_block_name to None to automatically match quantizable blocks."+ )return target_blocks@@ -507,6 +517,7 @@ def detect_device_count():else:try:import habana_frameworks.torch.hpu as hthpu # pylint: disable=E0401+return hthpu.device_count()except ImportError:return 0@@ -573,13 +584,13 @@ class CpuInfo(object):max_extension_support = cpuid.get_max_extension_support()if max_extension_support >= 7:ecx = cpuid._run_asm(- b"\x31\xC9", # xor ecx, ecx- b"\xB8\x07\x00\x00\x00" b"\x0f\xa2" b"\x89\xC8" b"\xC3", # mov eax, 7 # cpuid # mov ax, cx # ret+ b"\x31\xc9", # xor ecx, ecx+ b"\xb8\x07\x00\x00\x00" b"\x0f\xa2" b"\x89\xc8" b"\xc3", # mov eax, 7 # cpuid # mov ax, cx # ret)self._vnni = bool(ecx & (1 << 11))eax = cpuid._run_asm(- b"\xB9\x01\x00\x00\x00", # mov ecx, 1- b"\xB8\x07\x00\x00\x00" b"\x0f\xa2" b"\xC3", # mov eax, 7 # cpuid # ret+ b"\xb9\x01\x00\x00\x00", # mov ecx, 1+ b"\xb8\x07\x00\x00\x00" b"\x0f\xa2" b"\xc3", # mov eax, 7 # cpuid # ret)self._bf16 = bool(eax & (1 << 5))if "arch" in info and "ARM" in info["arch"]: # pragma: no cover@@ -613,11 +624,11 @@ class CpuInfo(object):cmd = "sysctl -n machdep.cpu.core_count"with subprocess.Popen(- args=cmd,- shell=True,- stdout=subprocess.PIPE,- stderr=subprocess.STDOUT,- universal_newlines=False,+ args=cmd,+ shell=True,+ stdout=subprocess.PIPE,+ stderr=subprocess.STDOUT,+ universal_newlines=False,) as proc:proc.wait()if proc.stdout:@@ -635,7 +646,10 @@ def is_local_path(path):Returns:bool: True if the path exists locally, False otherwise."""- format_list = ("json", "txt",)+ format_list = (+ "json",+ "txt",+ )flag = Nonefor x in format_list:flag = True if x in path else flag@@ -765,8 +779,9 @@ def check_memory_availability(device, inputs, weight, org_seqlen, org_bs):return False, seqlen, bs-def get_layer_names_in_block(model, supported_types=(torch.nn.Linear,- transformers.pytorch_utils.Conv1D), quant_block_list=None):+def get_layer_names_in_block(+ model, supported_types=(torch.nn.Linear, transformers.pytorch_utils.Conv1D), quant_block_list=None+):"""Retrieves the names of layers within each block of the model.Returns:@@ -816,11 +831,14 @@ def is_hpu_supported(): # pragma: no coverdef get_library_version(library_name):from packaging.version import Version+python_vesion = Version(sys.version.split()[0])if python_vesion < Version("3.8"):import warnings- warnings.filterwarnings('ignore', category=DeprecationWarning)++ warnings.filterwarnings("ignore", category=DeprecationWarning)import pkg_resources # pylint: disable=E0401+try:version = pkg_resources.get_distribution(library_name).versionreturn version@@ -828,6 +846,7 @@ def get_library_version(library_name):return f"{library_name} is not installed"else:import importlib.metadata # pylint: disable=E0401+try:version = importlib.metadata.version(library_name)return version@@ -874,6 +893,7 @@ def get_autogptq_packing_qlinear(backend, bits=4, group_size=128, sym=False):disable_exllamav1 = Trueif use_triton:from auto_round.export.export_to_autogptq.qlinear_triton import QuantLinear+return QuantLineartry:import auto_gptq # pylint: disable=E0401@@ -882,8 +902,10 @@ def get_autogptq_packing_qlinear(backend, bits=4, group_size=128, sym=False):exit()from auto_gptq.utils.import_utils import dynamically_import_QuantLinear # pylint: disable=E0401+version = get_library_version("auto_gptq")from packaging.version import Version+if Version(version) < Version("0.7.2"):QuantLinear = dynamically_import_QuantLinear(use_triton=use_triton,@@ -1103,8 +1125,11 @@ def check_awq_gemm_compatibility(model, bits, group_size, sym, layer_configs=Nonlayer_names = get_layer_names_in_block(model)for layer_name in layer_names:- if layer_configs is not None and layer_name in layer_configs.keys() and layer_configs[layer_name].get("bits",- bits) > 8:+ if (+ layer_configs is not None+ and layer_name in layer_configs.keys()+ and layer_configs[layer_name].get("bits", bits) > 8+ ):continuelayer = get_module(model, layer_name)@@ -1118,7 +1143,8 @@ def check_awq_gemm_compatibility(model, bits, group_size, sym, layer_configs=Nondef get_device_and_parallelism(device):from auto_round.utils import detect_device- devices = device.replace(" ", "").split(',')++ devices = device.replace(" ", "").split(",")if all(s.isdigit() for s in devices) and len(devices) > 1 and torch.cuda.is_available():device = "cuda"parallelism = True@@ -1136,11 +1162,11 @@ def get_device_and_parallelism(device):def set_cuda_visible_devices(device):- devices = device.replace(" ", "").split(',')+ devices = device.replace(" ", "").split(",")if all(s.isdigit() for s in devices):if "CUDA_VISIBLE_DEVICES" in os.environ:current_visible_devices = os.environ["CUDA_VISIBLE_DEVICES"]- current_visible_devices = current_visible_devices.split(',')+ current_visible_devices = current_visible_devices.split(",")indices = [int(device) for device in devices]try:pick_device = [current_visible_devices[i] for i in indices]@@ -1148,8 +1174,9 @@ def set_cuda_visible_devices(device):raise ValueError("Invalid '--device' value: It must be smaller than the number of available devices."" For example, with CUDA_VISIBLE_DEVICES=4,5, "- "--device 0,1 is valid, but --device 4,5 is not supported.")- visible_devices = ','.join(pick_device)+ "--device 0,1 is valid, but --device 4,5 is not supported."+ )+ visible_devices = ",".join(pick_device)os.environ["CUDA_VISIBLE_DEVICES"] = visible_deviceselse:os.environ["CUDA_VISIBLE_DEVICES"] = device@@ -1185,7 +1212,7 @@ def _gguf_args_check(args_or_ar, format_str=None, model_type=ModelType.TEXT):format_str = args_or_ar.formatelse:format_str = format_str.replace("q*_", f"q{args_or_ar.bits}_")- formats = format_str.lower().replace(' ', '').split(",")+ formats = format_str.lower().replace(" ", "").split(",")formats = sorted(formats, key=lambda x: len(x))for f in formats:if f.startswith("gguf") and f not in GGUF_CONFIG:@@ -1202,7 +1229,8 @@ def _gguf_args_check(args_or_ar, format_str=None, model_type=ModelType.TEXT):except:raise ImportError(f"Please use the latest gguf-py for {format}, you can use the following command to install it:\n"- "git clone https://github.com/ggml-org/llama.cpp.git && cd llama.cpp/gguf-py && pip install .")+ "git clone https://github.com/ggml-org/llama.cpp.git && cd llama.cpp/gguf-py && pip install ."+ )if re.search(pattern, format):if pre_dq_format and re.search(pattern, format).group() not in pre_dq_format:logger.error(f"Cannot export {pre_dq_format} and {format} at the same time.")@@ -1214,6 +1242,7 @@ def _gguf_args_check(args_or_ar, format_str=None, model_type=ModelType.TEXT):from pathlib import Pathfrom auto_round.export.export_to_gguf.convert import ModelBase+hparams = ModelBase.load_hparams(Path(args_or_ar.model))model_architecture = hparams["architectures"][0]try:@@ -1223,12 +1252,13 @@ def _gguf_args_check(args_or_ar, format_str=None, model_type=ModelType.TEXT):sys.exit(1)if re.search(pattern, format) and ("hidden_size" in hparams and hparams["hidden_size"] % QK_K != 0):- model_name = args_or_ar.model.split('/')+ model_name = args_or_ar.model.split("/")model_name = model_name[-1] if model_name[-1] else model_name[-2]hidden_size = hparams["hidden_size"]logger.error(f"Currently only support pure mode for format: {format}. "- f"{model_name} is not supported, cause hidden_size({hidden_size}) % 256 !=0")+ f"{model_name} is not supported, cause hidden_size({hidden_size}) % 256 !=0"+ )sys.exit(-1)unsupport_list, reset_list = [], []@@ -1249,8 +1279,9 @@ def _gguf_args_check(args_or_ar, format_str=None, model_type=ModelType.TEXT):if len(unsupport_list) > 0:logger.info(f"format {format} does not support for {', '.join(unsupport_list)},"- f" reset to {', '.join(reset_list)}.")-# Removed obsolete commented-out block for improved readability and maintainability.+ f" reset to {', '.join(reset_list)}."+ )+ # Removed obsolete commented-out block for improved readability and maintainability.return args_or_ar@@ -1270,15 +1301,16 @@ def _to_model_dtype(model, model_dtype):def llm_load_model(- pretrained_model_name_or_path,- torch_dtype="auto",- use_auto_mapping=True,- trust_remote_code=True,- model_dtype=None,- device="cpu",- low_cpu_mem_mode=0,- low_cpu_mem_tmp_dir=None,- **kwargs):+ pretrained_model_name_or_path,+ torch_dtype="auto",+ use_auto_mapping=True,+ trust_remote_code=True,+ model_dtype=None,+ device="cpu",+ low_cpu_mem_mode=0,+ low_cpu_mem_tmp_dir=None,+ **kwargs,+):from transformers import AutoModel, AutoModelForCausalLM, AutoTokenizeris_glm = bool(re.search("chatglm", pretrained_model_name_or_path.lower()))@@ -1292,6 +1324,7 @@ def llm_load_model(low_cpu_mem_tmp_dir = "low_cpu_mem_tmp"if low_cpu_mem_mode == 2:from auto_round.low_cpu_mem.utils import load_model_with_hooks+model = load_model_with_hooks(pretrained_model_name_or_path,model_cls,@@ -1299,9 +1332,11 @@ def llm_load_model(clean_weight=True,saved_path=low_cpu_mem_tmp_dir,torch_dtype=torch_dtype,- trust_remote_code=trust_remote_code)+ trust_remote_code=trust_remote_code,+ )elif low_cpu_mem_mode == 1:from auto_round.low_cpu_mem.utils import load_empty_model+low_cpu_mem_usage = Truemodel = load_empty_model(pretrained_model_name_or_path,@@ -1309,28 +1344,35 @@ def llm_load_model(device=device,saved_path=low_cpu_mem_tmp_dir,torch_dtype=torch_dtype,- trust_remote_code=trust_remote_code)+ trust_remote_code=trust_remote_code,+ )else:if _use_hpu_compile_mode():model = model_cls.from_pretrained(- pretrained_model_name_or_path, torch_dtype=torch_dtype,+ pretrained_model_name_or_path,+ torch_dtype=torch_dtype,attn_implementation="eager",- trust_remote_code=trust_remote_code, device_map="auto" if use_auto_mapping else None+ trust_remote_code=trust_remote_code,+ device_map="auto" if use_auto_mapping else None,)else:try:model = model_cls.from_pretrained(- pretrained_model_name_or_path, torch_dtype=torch_dtype,- trust_remote_code=trust_remote_code, device_map="auto" if use_auto_mapping else None+ pretrained_model_name_or_path,+ torch_dtype=torch_dtype,+ trust_remote_code=trust_remote_code,+ device_map="auto" if use_auto_mapping else None,)except OSError as e:logger.warning(- f"fail to load {pretrained_model_name_or_path}, set trust_remote_code to False and retry.")+ f"fail to load {pretrained_model_name_or_path}, set trust_remote_code to False and retry."+ )model = model_cls.from_pretrained(pretrained_model_name_or_path,torch_dtype=torch_dtype,trust_remote_code=False,- device_map="auto" if use_auto_mapping else None)+ device_map="auto" if use_auto_mapping else None,+ )model = model.eval()model = _to_model_dtype(model, model_dtype)@@ -1339,12 +1381,13 @@ def llm_load_model(def mllm_load_model(- pretrained_model_name_or_path,- torch_dtype="auto",- use_auto_mapping=True,- trust_remote_code=True,- model_dtype=None,- **kwargs):+ pretrained_model_name_or_path,+ torch_dtype="auto",+ use_auto_mapping=True,+ trust_remote_code=True,+ model_dtype=None,+ **kwargs,+):import jsonimport transformers@@ -1355,6 +1398,7 @@ def mllm_load_model(config = json.load(open(os.path.join(pretrained_model_name_or_path, "config.json")))else:from huggingface_hub import hf_hub_download, list_repo_files+file_list = list_repo_files(pretrained_model_name_or_path)if "config.json" in file_list:# Load plain JSON@@ -1364,6 +1408,7 @@ def mllm_load_model(elif "config.json.gz" in file_list:# Load gzipped JSONimport gzip+config_path = hf_hub_download(pretrained_model_name_or_path, "config.json.gz")with gzip.open(config_path, "rt", encoding="utf-8") as f:config = json.load(f)@@ -1378,25 +1423,30 @@ def mllm_load_model(processor, image_processor = None, Noneif "deepseek_vl_v2" == model_type:from deepseek_vl2.models import DeepseekVLV2ForCausalLM, DeepseekVLV2Processor # pylint: disable=E0401+processor = DeepseekVLV2Processor.from_pretrained(pretrained_model_name_or_path)tokenizer = processor.tokenizermodel: DeepseekVLV2ForCausalLM = AutoModelForCausalLM.from_pretrained(pretrained_model_name_or_path,trust_remote_code=trust_remote_code,torch_dtype=torch_dtype,- device_map="auto" if use_auto_mapping else None)+ device_map="auto" if use_auto_mapping else None,+ )else:architectures = config["architectures"][0]if architectures == "LlavaLlamaForCausalLM":from llava.model.builder import load_pretrained_model # pylint: disable=E0401+tokenizer, model, image_processor, _ = load_pretrained_model(pretrained_model_name_or_path,model_base=None,model_name=pretrained_model_name_or_path,- torch_dtype=torch_dtype)+ torch_dtype=torch_dtype,+ )else:- if architectures.endswith("Model") \- and hasattr(transformers, n := architectures.replace("Model", "ForConditionalGeneration")):+ if architectures.endswith("Model") and hasattr(+ transformers, n := architectures.replace("Model", "ForConditionalGeneration")+ ):cls = getattr(transformers, n)elif hasattr(transformers, architectures):cls = getattr(transformers, architectures)@@ -1406,15 +1456,20 @@ def mllm_load_model(pretrained_model_name_or_path,trust_remote_code=trust_remote_code,torch_dtype=torch_dtype,- device_map="auto" if use_auto_mapping else None)+ device_map="auto" if use_auto_mapping else None,+ )tokenizer = AutoTokenizer.from_pretrained(- pretrained_model_name_or_path, trust_remote_code=trust_remote_code)+ pretrained_model_name_or_path, trust_remote_code=trust_remote_code+ )processor = AutoProcessor.from_pretrained(- pretrained_model_name_or_path, trust_remote_code=trust_remote_code)+ pretrained_model_name_or_path, trust_remote_code=trust_remote_code+ )try:from transformers import AutoImageProcessor+image_processor = AutoImageProcessor.from_pretrained(- pretrained_model_name_or_path, trust_remote_code=trust_remote_code)+ pretrained_model_name_or_path, trust_remote_code=trust_remote_code+ )except Exception as e:pass@@ -1425,7 +1480,7 @@ def mllm_load_model(def is_pure_text_model(model):- """verify on: phi-3.5, Mistral-Small-3.1, gemma-3, qwen2-vl, """+ """verify on: phi-3.5, Mistral-Small-3.1, gemma-3, qwen2-vl,"""if hasattr(model, "config") and hasattr(model.config, "vision_config"):return Falseif hasattr(model.__class__, "main_input_name") and model.__class__.main_input_name != "input_ids":@@ -1453,7 +1508,7 @@ def reset_params(inputs):inputs (dict): Sets "use_cache" to False if the key is present."""if "use_cache" in inputs.keys(): # Not storing kv cache- inputs['use_cache'] = False+ inputs["use_cache"] = Falsedef check_skippable_keywords(key):@@ -1516,21 +1571,33 @@ def get_model_dtype(model_dtype, default="auto"):def str2bool(v):import argparse+if isinstance(v, bool):return v- if v.lower() in ('yes', 'true', 't', 'y', '1'):+ if v.lower() in ("yes", "true", "t", "y", "1"):return True- elif v.lower() in ('no', 'false', 'f', 'n', '0'):+ elif v.lower() in ("no", "false", "f", "n", "0"):return Falseelse:- raise argparse.ArgumentTypeError('Boolean value expected.')+ raise argparse.ArgumentTypeError("Boolean value expected.")def filter_quantization_config(quantization_config):- default_dict = {"amp": True, "batch_size": 8, "data_type": int, "dataset": "NeelNanda/pile-10k",- "enable_minmax_tuning": True, "enable_norm_bias_tuning": False, "enable_quanted_input": True,- "gradient_accumulate_steps": 1, "iters": 200, "low_gpu_mem_usage": False, "nsamples": 128,- "scale_dtype": "torch.float16", "seqlen": 2048}+ default_dict = {+ "amp": True,+ "batch_size": 8,+ "data_type": int,+ "dataset": "NeelNanda/pile-10k",+ "enable_minmax_tuning": True,+ "enable_norm_bias_tuning": False,+ "enable_quanted_input": True,+ "gradient_accumulate_steps": 1,+ "iters": 200,+ "low_gpu_mem_usage": False,+ "nsamples": 128,+ "scale_dtype": "torch.float16",+ "seqlen": 2048,+ }iters = quantization_config.get("iters", 200)default_dict["lr"] = 1.0 / iters if iters > 0 else 5e-3@@ -1581,14 +1648,18 @@ def check_seqlen_compatible(input_seqlen, tokenizer=None, model=None):Returns:ValueError: if the input length is not valid, riase Error."""- if model is not None and hasattr(model, 'config'):+ if model is not None and hasattr(model, "config"):model_config = model.config- if hasattr(model_config, 'max_position_embeddings') and input_seqlen > model_config.max_position_embeddings:- raise ValueError(f"seqlen({input_seqlen}) exceeds model.config.max_position_embeddings(" \- f"{model_config.max_position_embeddings}). Please lowering '--seqlen'")- if tokenizer is not None and hasattr(tokenizer, 'model_max_length') and input_seqlen > tokenizer.model_max_length:- raise ValueError(f"seqlen({input_seqlen}) exceeds tokenizer.model_max_length({tokenizer.model_max_length}). " \- "Please oncider Consider lowering the '--seqlen' or increasing tokenizer.model_max_length.")+ if hasattr(model_config, "max_position_embeddings") and input_seqlen > model_config.max_position_embeddings:+ raise ValueError(+ f"seqlen({input_seqlen}) exceeds model.config.max_position_embeddings("+ f"{model_config.max_position_embeddings}). Please lowering '--seqlen'"+ )+ if tokenizer is not None and hasattr(tokenizer, "model_max_length") and input_seqlen > tokenizer.model_max_length:+ raise ValueError(+ f"seqlen({input_seqlen}) exceeds tokenizer.model_max_length({tokenizer.model_max_length}). "+ "Please oncider Consider lowering the '--seqlen' or increasing tokenizer.model_max_length."+ )def _use_more_bits(i_layer: int, n_layer: int):@@ -1603,6 +1674,7 @@ def _get_digital_in_layer_name(layer_name):else:return None+def _search_gguf_type(gguf_type):if gguf_type in GGUF_INNER_CONFIG:return gguf_type@@ -1618,6 +1690,7 @@ def _search_gguf_type(gguf_type):return tmp_typereturn None+def _gguf_type_fallback(gguf_type):if gguf_type in ("gguf:q2_k", "gguf:q3_k", "gguf:q4_k"):gguf_type = "gguf:q5_0"@@ -1627,6 +1700,7 @@ def _gguf_type_fallback(gguf_type):gguf_type = "gguf:q8_0"return gguf_type+##https://github.com/ggml-org/llama.cpp/blob/9e31bec4fd53634c9e5b04650488a09a055f5dab/src/llama-quant.cpp#L129def get_layer_config_by_gguf_format(layer_config, gguf_format, model, model_type=ModelType.TEXT):# TODO: support for other format later@@ -1635,6 +1709,7 @@ def get_layer_config_by_gguf_format(layer_config, gguf_format, model, model_typeimport gguf # pylint: disable=E0401from auto_round.export.export_to_gguf.convert import ModelBase, get_model_architecture+model_architecture = get_model_architecture(hparams=model.config.to_dict(), model_type=model_type)try:model_class = ModelBase.from_model_architecture(model_architecture, model_type=model_type)@@ -1674,8 +1749,11 @@ def get_layer_config_by_gguf_format(layer_config, gguf_format, model, model_typetie_word_embeddings = model.config.tie_word_embeddingsn_gqa = 1- if hasattr(model, "config") and hasattr(model.config, "num_attention_heads") and hasattr(model.config,- "num_key_value_heads"):+ if (+ hasattr(model, "config")+ and hasattr(model.config, "num_attention_heads")+ and hasattr(model.config, "num_key_value_heads")+ ):n_gqa = model.config.num_attention_heads // model.config.num_key_value_headsn_expert = 0for name in ["num_experts", "num_local_experts", "n_routed_experts"]:@@ -1701,10 +1779,11 @@ def get_layer_config_by_gguf_format(layer_config, gguf_format, model, model_typei_layer = _get_digital_in_layer_name(layer_name)if lm_head_name is not None and layer_name == lm_head_name:- target_bits = int(re.search("gguf:q([0-9]{1,})_[01k]", GGUF_CONFIG[target_gguf_format]['lm_head']).group(1))+ target_bits = int(re.search("gguf:q([0-9]{1,})_[01k]", GGUF_CONFIG[target_gguf_format]["lm_head"]).group(1))if isinstance(layer, torch.nn.Embedding):target_bits = int(- re.search("gguf:q([0-9]{1,})_[01k]", GGUF_CONFIG[target_gguf_format]['embedding']).group(1))+ re.search("gguf:q([0-9]{1,})_[01k]", GGUF_CONFIG[target_gguf_format]["embedding"]).group(1)+ )gguf_name = tensor_map.get_name(layer_name)bits_index = 6@@ -1712,16 +1791,16 @@ def get_layer_config_by_gguf_format(layer_config, gguf_format, model, model_typeif "bits" not in config:logger.warning(f"Setting layer_config requires providing bits, {layer_name} has not bits,"- f" using bits={target_bits} instead.")- new_type = new_type[:bits_index] + target_bits + new_type[bits_index + 1:]+ f" using bits={target_bits} instead."+ )+ new_type = new_type[:bits_index] + target_bits + new_type[bits_index + 1 :]else:- new_type = new_type[:bits_index] + str(config["bits"]) + new_type[bits_index + 1:]+ new_type = new_type[:bits_index] + str(config["bits"]) + new_type[bits_index + 1 :]new_type = _search_gguf_type(new_type)if new_type is None:raise ValueError(f"invalid bit setting for {layer_name}")- elif (target_bits is not None and- "bits" in config and config["bits"] != target_bits):- new_type = new_type[:bits_index] + str(config["bits"]) + new_type[bits_index + 1:]+ elif target_bits is not None and "bits" in config and config["bits"] != target_bits:+ new_type = new_type[:bits_index] + str(config["bits"]) + new_type[bits_index + 1 :]new_type = _search_gguf_type(new_type)if new_type is None:raise ValueError(f"invalid bit setting for {layer_name}")@@ -1750,7 +1829,8 @@ def get_layer_config_by_gguf_format(layer_config, gguf_format, model, model_typeelif target_gguf_format == "gguf:q3_k_l":new_type = "gguf:q5_k"elif (target_gguf_format == "gguf:q4_k_m" or target_gguf_format == "gguf:q5_k_m") and _use_more_bits(- i_layer, n_layer):+ i_layer, n_layer+ ):new_type = "gguf:q6_k"elif target_gguf_format == "gguf:q4_k_s" and i_attention_wv < 4:new_type = "gguf:q5_k"@@ -1803,8 +1883,11 @@ def get_layer_config_by_gguf_format(layer_config, gguf_format, model, model_typenew_type = "gguf:q6_k"elif target_gguf_format == "gguf:q5_k_m" and _use_more_bits(i_layer, n_layer):new_type = "gguf:q6_k"- elif (target_gguf_format == "gguf:q4_k_s" and- model_class.model_arch != gguf.MODEL_ARCH.FALCON and i_layer < n_layer / 8):+ elif (+ target_gguf_format == "gguf:q4_k_s"+ and model_class.model_arch != gguf.MODEL_ARCH.FALCON+ and i_layer < n_layer / 8+ ):new_type = "gguf:q5_k"elif (target_gguf_format == "gguf:q4_0" or target_gguf_format == "gguf:q5_0") and i_layer < n_layer / 8:if target_gguf_format == "gguf:q4_0":@@ -1817,8 +1900,14 @@ def get_layer_config_by_gguf_format(layer_config, gguf_format, model, model_typeelif "attn_output" in gguf_name:if gguf.MODEL_ARCH.FALCON != model_class.model_arch:if n_expert == 8:- if target_gguf_format in ("gguf:q2_k", "gguf:q3_k_s", "gguf:q3_k_m", "gguf:q4_k_s", "gguf:q4_k_m",- "gguf:q5_k"):+ if target_gguf_format in (+ "gguf:q2_k",+ "gguf:q3_k_s",+ "gguf:q3_k_m",+ "gguf:q4_k_s",+ "gguf:q4_k_m",+ "gguf:q5_k",+ ):new_type = "gguf:q5_k"elif target_gguf_format == "gguf:q2_k":new_type = "gguf:q3_k"@@ -1845,23 +1934,28 @@ def get_layer_config_by_gguf_format(layer_config, gguf_format, model, model_typenew_type = "gguf:bf16"logger.warning(f"fallback {layer_name} to {new_type}, "- f"because input_features({input_features}) % block_size({block_size}) != 0")+ f"because input_features({input_features}) % block_size({block_size}) != 0"+ )# for deepseek v2- if layer_name.endswith("kv_b_proj") and new_type.endswith("_k") \- and 'Deepseek' in model.config.architectures[0]:+ if layer_name.endswith("kv_b_proj") and new_type.endswith("_k") and "Deepseek" in model.config.architectures[0]:fallback = False# calc if need fallbackqk_nope_head_dim = model.config.qk_nope_head_dimkv_b_shape = get_module(model, layer_name).weight.shape- if qk_nope_head_dim < QK_K or qk_nope_head_dim % QK_K != 0 \- or kv_b_shape[-1] < QK_K or kv_b_shape[-1] % QK_K != 0:+ if (+ qk_nope_head_dim < QK_K+ or qk_nope_head_dim % QK_K != 0+ or kv_b_shape[-1] < QK_K+ or kv_b_shape[-1] % QK_K != 0+ ):fallback = Trueif fallback:tmp_type = _gguf_type_fallback(new_type)logger.warning_once(- f"self_attn.kv_b_proj does not support the use of {new_type}, replace it with {tmp_type}")+ f"self_attn.kv_b_proj does not support the use of {new_type}, replace it with {tmp_type}"+ )new_type = tmp_typetarget_config = GGUF_INNER_CONFIG[new_type]@@ -1889,9 +1983,10 @@ def get_lm_head_name(model):def get_gguf_qtype_by_layer_config(layer_config):import gguf # pylint: disable=E0401- if layer_config['bits'] >= 16:++ if layer_config["bits"] >= 16:return None- bits = layer_config['bits']+ bits = layer_config["bits"]super_bits = layer_config.get("super_bits", None)sym = layer_config["sym"]group_size = layer_config.get("group_size", None)@@ -1920,6 +2015,7 @@ def get_gguf_qtype_by_layer_config(layer_config):return gguf.GGMLQuantizationType.Q8_0raise ValueError("Unknown layer config")+def flatten_list(nested_list):flattened = []for item in nested_list:@@ -1929,6 +2025,7 @@ def flatten_list(nested_list):flattened.append(item)return flattened+def clean_module_parameter(submodule, parameter):if submodule is None:return@@ -1939,9 +2036,10 @@ def clean_module_parameter(submodule, parameter):else:submodule._parameters[parameter] = None+def get_reciprocal(tensor):if torch.dtype is torch.float16:- tensor = torch.sign(tensor) * torch.clamp(torch.abs(tensor), min=1e-5)+ tensor = torch.sign(tensor) * torch.clamp(torch.abs(tensor), min=1e-5)else:tensor = torch.where(torch.abs(tensor) < 1e-30, 0, tensor)- return torch.where(tensor != 0, 1 / tensor, torch.zeros_like(tensor))\ No newline at end of file+ return torch.where(tensor != 0, 1 / tensor, torch.zeros_like(tensor))diff --git a/auto_round/version.py b/auto_round/version.pyindex 3a8c9f1ae..d9f21f781 100644--- a/auto_round/version.py+++ b/auto_round/version.py@@ -15,4 +15,3 @@supporting popular model weight only compression based on signround."""__version__ = "0.6.0"-diff --git a/auto_round/wrapper.py b/auto_round/wrapper.pyindex 12eb573fc..0614ebd0f 100644--- a/auto_round/wrapper.py+++ b/auto_round/wrapper.py@@ -57,8 +57,15 @@ class WrapperLinear(torch.nn.Module):device (str): Device on which to run computations (e.g., 'cpu' or 'cuda')."""- def __init__(self, orig_layer, enable_minmax_tuning=True, enable_norm_bias_tuning=False, device='cpu',- enable_round_tuning=True, **kwargs):+ def __init__(+ self,+ orig_layer,+ enable_minmax_tuning=True,+ enable_norm_bias_tuning=False,+ device="cpu",+ enable_round_tuning=True,+ **kwargs,+ ):"""Initializes the WrapperLinear module.Args:@@ -75,7 +82,7 @@ class WrapperLinear(torch.nn.Module):self.enable_round_tuning = enable_round_tuningself.enable_norm_bias_tuning = enable_norm_bias_tuning and (orig_layer.bias is not None)self.enable_act_quant = self.orig_layer.act_bits <= 8- if (hasattr(self.orig_layer, "scale_dtype") and self.orig_layer.scale_dtype == torch.float32):+ if hasattr(self.orig_layer, "scale_dtype") and self.orig_layer.scale_dtype == torch.float32:self.q_scale_thresh = 1e-8else:self.q_scale_thresh = 1e-5@@ -85,9 +92,9 @@ class WrapperLinear(torch.nn.Module):def _init_tuning_params_and_quant_func(self):"""Initializes tuning parameters and quantization functions.- This method sets up required parameters and functions for weight quantization,- activation quantization, and bias/normalization.- """+ This method sets up required parameters and functions for weight quantization,+ activation quantization, and bias/normalization.+ """self.params = {}p_dtype = torch.float32 ##parameter dtype@@ -98,27 +105,26 @@ class WrapperLinear(torch.nn.Module):weight_reshape = reshape_and_pad_tensor(orig_weight.data, orig_layer.group_size)self.weight_min = torch.clamp(weight_reshape.min(1)[0], max=0)self.weight_max = torch.clamp(weight_reshape.max(1)[0], min=0)- self._init_params("value", p_dtype, weight_reshape.shape, 0,- self.enable_round_tuning and self.orig_layer.bits < 16)+ self._init_params(+ "value", p_dtype, weight_reshape.shape, 0, self.enable_round_tuning and self.orig_layer.bits < 16+ )# Min-max scale initializationshape = get_scale_shape(orig_weight, orig_layer.group_size)- self._init_params("min_scale", p_dtype, shape, 1.0, (self.enable_minmax_tuning- and self.orig_layer.bits < 16))- self._init_params("max_scale", p_dtype, shape, 1.0, (self.enable_minmax_tuning- and self.orig_layer.bits < 16))+ self._init_params("min_scale", p_dtype, shape, 1.0, (self.enable_minmax_tuning and self.orig_layer.bits < 16))+ self._init_params("max_scale", p_dtype, shape, 1.0, (self.enable_minmax_tuning and self.orig_layer.bits < 16))- self.weight_quant_func, self.data_type = get_quant_func(orig_layer.data_type, orig_layer.bits,- orig_layer.sym)+ self.weight_quant_func, self.data_type = get_quant_func(orig_layer.data_type, orig_layer.bits, orig_layer.sym)if self.enable_act_quant:- self.act_quant_func, self.act_data_type = get_quant_func(orig_layer.act_data_type,- orig_layer.act_bits,- orig_layer.act_sym)+ self.act_quant_func, self.act_data_type = get_quant_func(+ orig_layer.act_data_type, orig_layer.act_bits, orig_layer.act_sym+ )self._init_params("act_max_scale", p_dtype, (1), 1.0, not orig_layer.act_dynamic)## bias tuningif self.enable_norm_bias_tuning:self._init_params("bias_v", p_dtype, self.orig_layer.bias.shape, 0, True)from auto_round.data_type.int import quant_tensor_asym_wo_round+self.bias_quant_func = quant_tensor_asym_wo_roundself.params["bias_v"] = self.bias_v@@ -156,7 +162,7 @@ class WrapperLinear(torch.nn.Module):min_scale.data.clamp_(0, 1.0)max_scale.data.clamp_(0, 1.0)weight = self.orig_layer.weight- if weight.device.type == 'meta':+ if weight.device.type == "meta":weight = self.orig_layer.get_weight().to(self.device)if isinstance(self.orig_layer, transformers.pytorch_utils.Conv1D):weight = weight.t()@@ -179,7 +185,7 @@ class WrapperLinear(torch.nn.Module):data_type=self.data_type,q_scale_thresh=self.q_scale_thresh,imatrix=self.orig_layer.imatrix if hasattr(self.orig_layer, "imatrix") else None,- **quant_kwargs+ **quant_kwargs,)weight_q = weight_q.to(weight.dtype)if isinstance(self.orig_layer, transformers.pytorch_utils.Conv1D):@@ -198,9 +204,16 @@ class WrapperLinear(torch.nn.Module):tuple: Quantized activation, scale, and zero point."""act_max_scale.data.clamp_(0, 1.0)- x, scale, zp = self.act_quant_func(x, bits=self.orig_layer.act_bits, group_size=self.orig_layer.act_group_size,- scale_dtype=self.orig_layer.scale_dtype, q_scale_thresh=self.q_scale_thresh,- data_type=self.act_data_type, max_scale=act_max_scale, tensor_max=act_max)+ x, scale, zp = self.act_quant_func(+ x,+ bits=self.orig_layer.act_bits,+ group_size=self.orig_layer.act_group_size,+ scale_dtype=self.orig_layer.scale_dtype,+ q_scale_thresh=self.q_scale_thresh,+ data_type=self.act_data_type,+ max_scale=act_max_scale,+ tensor_max=act_max,+ )return x, scale, zpdef _qdq_bias(self, bias, bias_v):@@ -215,8 +228,9 @@ class WrapperLinear(torch.nn.Module):"""bias_bits = 4 ## hard codebias_group_size = -1- bias, scale, zp = self.bias_quant_func(bias, bits=bias_bits, group_size=bias_group_size, v=bias_v,- q_scale_thresh=self.q_scale_thresh)+ bias, scale, zp = self.bias_quant_func(+ bias, bits=bias_bits, group_size=bias_group_size, v=bias_v, q_scale_thresh=self.q_scale_thresh+ )return bias, scale, zpdef unwrapper(self, best_params):@@ -229,11 +243,11 @@ class WrapperLinear(torch.nn.Module):torch.nn.Module: The unwrapped and restored original layer."""best_params = best_params or {}- v = best_params.get('value', torch.tensor(0.0)).to(self.device)- min_scale = best_params.get('min_scale', torch.tensor(1.0)).to(self.device)- max_scale = best_params.get('max_scale', torch.tensor(1.0)).to(self.device)+ v = best_params.get("value", torch.tensor(0.0)).to(self.device)+ min_scale = best_params.get("min_scale", torch.tensor(1.0)).to(self.device)+ max_scale = best_params.get("max_scale", torch.tensor(1.0)).to(self.device)- if self.orig_layer.weight.device.type == 'meta':+ if self.orig_layer.weight.device.type == "meta":self.orig_layer.to(self.device)##unwrapper weightqdq_weight, scale, zp = self._qdq_weight(v, min_scale, max_scale)@@ -280,27 +294,28 @@ class WrapperLinear(torch.nn.Module):if self.enable_norm_bias_tuning and "bias_v" in best_params.keys(): ##fake quantbias_v = best_params["bias_v"].to(self.device)bias = self.orig_layer.bias- if bias is not None and bias.device.type == 'meta':+ if bias is not None and bias.device.type == "meta":bias = self.orig_layer.get_bias().to(self.device)bias, _, _ = self._qdq_bias(bias, bias_v)self.orig_layer.bias.grad = Noneself.orig_layer.bias.data.copy_(bias)- if hasattr(self.orig_layer, 'update'):+ if hasattr(self.orig_layer, "update"):self.orig_layer.update()- self.orig_layer.to('meta')+ self.orig_layer.to("meta")##unwrapper actif self.enable_act_quant:if not self.orig_layer.act_dynamic:- act_max_scale = best_params.get('act_max_scale', torch.tensor(1.0)).to(self.device)+ act_max_scale = best_params.get("act_max_scale", torch.tensor(1.0)).to(self.device)act_max = self.orig_layer.act_max if hasattr(self.orig_layer, "act_max") else None- tmp_shape = (1)+ tmp_shape = 1if self.orig_layer.act_group_size > 1:tmp_shape = (1, self.orig_layer.act_group_size)if act_max is not None:- _, act_scale, _ = self._qdq_act(torch.zeros(tmp_shape).to(self.device),- act_max_scale=self.act_max_scale, act_max=act_max)+ _, act_scale, _ = self._qdq_act(+ torch.zeros(tmp_shape).to(self.device), act_max_scale=self.act_max_scale, act_max=act_max+ )self.orig_layer.act_max = torch.tensor(self.orig_layer.act_max * act_max_scale.item()).to("cpu")else:act_scale = torch.ones_like(act_max_scale, dtype=self.act_data_type)@@ -363,7 +378,7 @@ class WrapperLinear(torch.nn.Module):# pylint: disable=not-callablebias = self.orig_layer.bias- if bias is not None and bias.device.type == 'meta':+ if bias is not None and bias.device.type == "meta":bias = self.orig_layer.get_bias().to(self.device)if self.enable_norm_bias_tuning:bias, _, _ = self._qdq_bias(bias, self.bias_v)@@ -380,21 +395,24 @@ class WrapperWALayer(torch.nn.Module):def forward(self, x):act_max = self.orig_layer.act_max if hasattr(self.orig_layer, "act_max") else None- x, _, _ = self.orig_layer.act_quant_func(x, bits=self.orig_layer.act_bits,- group_size=self.orig_layer.group_size,- scale_dtype=self.orig_layer.scale_dtype,- q_scale_thresh=self.orig_layer.q_scale_thresh,- data_type=self.orig_layer.act_data_type,- tensor_max=act_max)+ x, _, _ = self.orig_layer.act_quant_func(+ x,+ bits=self.orig_layer.act_bits,+ group_size=self.orig_layer.group_size,+ scale_dtype=self.orig_layer.scale_dtype,+ q_scale_thresh=self.orig_layer.q_scale_thresh,+ data_type=self.orig_layer.act_data_type,+ tensor_max=act_max,+ )return self.orig_layer.forward(x)class WrapperLayerNorm(torch.nn.Module):"""A wrapper for layer normalization with quantized weights.- This class wraps a given layer normalization module and applies quantization without round- to its weights. The quantization is parameterized by the number of bits and- an optional group size.+ This class wraps a given layer normalization module and applies quantization without round+ to its weights. The quantization is parameterized by the number of bits and+ an optional group size."""def __init__(self, orig_layer, bit=4, group_size=-1, device="cpu"):@@ -408,39 +426,44 @@ class WrapperLayerNorm(torch.nn.Module):self.q_scale_thresh = 1e-5self.v = torch.nn.Parameter(reshape_and_pad_tensor(- torch.zeros(self.orig_layer.weight.shape, device=self.device, dtype=weight_dtype),- self.group_size),- requires_grad=True)+ torch.zeros(self.orig_layer.weight.shape, device=self.device, dtype=weight_dtype), self.group_size+ ),+ requires_grad=True,+ )self.params = {"v": self.v}from auto_round.data_type.int import quant_tensor_asym_wo_round+self.quant_func = quant_tensor_asym_wo_rounddef unwrapper(self, best_params):if best_params is None:return self.orig_layer- v = best_params['v']- weight_q, _, _ = self.quant_func(self.orig_layer.weight, self.bits, self.group_size,- v, q_scale_thresh=self.q_scale_thresh)+ v = best_params["v"]+ weight_q, _, _ = self.quant_func(+ self.orig_layer.weight, self.bits, self.group_size, v, q_scale_thresh=self.q_scale_thresh+ )self.orig_layer.q_scale_thresh = self.q_scale_threshself.orig_layer.weight.data.copy_(weight_q)return self.orig_layerdef forward(self, input):input = input.to(self.device)- weight_q, _, _ = self.quant_func(self.orig_layer.weight, self.bits, self.group_size,- self.v, q_scale_thresh=self.q_scale_thresh)+ weight_q, _, _ = self.quant_func(+ self.orig_layer.weight, self.bits, self.group_size, self.v, q_scale_thresh=self.q_scale_thresh+ )import torch.nn.functional as F+return F.layer_norm(- input, self.orig_layer.normalized_shape, weight_q, self.orig_layer.bias, self.orig_layer.eps).to(- self.output_device)+ input, self.orig_layer.normalized_shape, weight_q, self.orig_layer.bias, self.orig_layer.eps+ ).to(self.output_device)class WrapperLlamaNorm(torch.nn.Module):"""A wrapper for Llama normalization in HF with fake quantized weights without rounding.- This class wraps a given layer normalization module and applies quantization without rounding- to its weights. The quantization is parameterized by the number of bits and- an optional group size.+ This class wraps a given layer normalization module and applies quantization without rounding+ to its weights. The quantization is parameterized by the number of bits and+ an optional group size."""def __init__(self, orig_layer, bit=4, group_size=-1, device="cpu"):@@ -454,27 +477,31 @@ class WrapperLlamaNorm(torch.nn.Module):self.q_scale_thresh = 1e-5self.v = torch.nn.Parameter(reshape_and_pad_tensor(- torch.zeros(self.orig_layer.weight.shape, device=self.device, dtype=weight_dtype),- self.group_size),- requires_grad=True)+ torch.zeros(self.orig_layer.weight.shape, device=self.device, dtype=weight_dtype), self.group_size+ ),+ requires_grad=True,+ )self.params = {"v": self.v}from auto_round.data_type.int import quant_tensor_asym_wo_round+self.quant_func = quant_tensor_asym_wo_rounddef unwrapper(self, best_params):if best_params is None:return self.orig_layer- v = best_params['v']- weight_q, _, _ = self.quant_func(self.orig_layer.weight, self.bits, self.group_size,- v, q_scale_thresh=self.q_scale_thresh)+ v = best_params["v"]+ weight_q, _, _ = self.quant_func(+ self.orig_layer.weight, self.bits, self.group_size, v, q_scale_thresh=self.q_scale_thresh+ )self.orig_layer.q_scale_thresh = self.q_scale_threshself.orig_layer.weight.data.copy_(weight_q)return self.orig_layerdef forward(self, hidden_states):hidden_states = hidden_states.to(self.device)- weight_q, _, _ = self.quant_func(self.orig_layer.weight, self.bits, self.group_size,- self.v, q_scale_thresh=self.q_scale_thresh)+ weight_q, _, _ = self.quant_func(+ self.orig_layer.weight, self.bits, self.group_size, self.v, q_scale_thresh=self.q_scale_thresh+ )input_dtype = hidden_states.dtypehidden_states = hidden_states.to(torch.float32)variance = hidden_states.pow(2).mean(-1, keepdim=True)@@ -512,7 +539,7 @@ class WrapperMultiblock(torch.nn.Module):return hidden_states-def wrapper_block(block, enable_minmax_tuning, enable_norm_bias_tuning, device='cpu', **kwargs):+def wrapper_block(block, enable_minmax_tuning, enable_norm_bias_tuning, device="cpu", **kwargs):"""Wraps the layers in the given block with a custom Wrapper module.Args:@@ -547,7 +574,8 @@ def wrapper_block(block, enable_minmax_tuning, enable_norm_bias_tuning, device='set_module(block, n, new_m)elif "RMSNorm" in m.__class__.__name__:logger.warning_once(- f"use LlamaRMSNorm to wrap {m.__class__.__name__}, please check the correctness yourself")+ f"use LlamaRMSNorm to wrap {m.__class__.__name__}, please check the correctness yourself"+ )wrapper_layer_class = NORM_MAPPING["LlamaRMSNorm"]new_m = wrapper_layer_class(m, device=device)set_module(block, n, new_m)diff --git a/auto_round_extension/__init__.py b/auto_round_extension/__init__.pyindex 2b414eb72..0d2740cdb 100644--- a/auto_round_extension/__init__.py+++ b/auto_round_extension/__init__.py@@ -11,4 +11,3 @@# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.# See the License for the specific language governing permissions and# limitations under the License.-diff --git a/auto_round_extension/cuda/__init__.py b/auto_round_extension/cuda/__init__.pyindex 2b414eb72..0d2740cdb 100644--- a/auto_round_extension/cuda/__init__.py+++ b/auto_round_extension/cuda/__init__.py@@ -11,4 +11,3 @@# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.# See the License for the specific language governing permissions and# limitations under the License.-diff --git a/auto_round_extension/cuda/gptqmodel_marlin.py b/auto_round_extension/cuda/gptqmodel_marlin.pyindex 42bc2c75a..8bf31ab8c 100644--- a/auto_round_extension/cuda/gptqmodel_marlin.py+++ b/auto_round_extension/cuda/gptqmodel_marlin.py@@ -42,8 +42,8 @@ def get_marlin_layer(): ##use an ugly wrapper to import gptqmodel on demandGPTQ_MARLIN_MAX_PARALLEL = 16def set_weight_attrs(- weight: torch.Tensor,- weight_attrs: Optional[Dict[str, Any]],+ weight: torch.Tensor,+ weight_attrs: Optional[Dict[str, Any]],):"""Set attributes on a weight tensor.@@ -57,51 +57,40 @@ def get_marlin_layer(): ##use an ugly wrapper to import gptqmodel on demandif weight_attrs is None:returnfor key, value in weight_attrs.items():- assert not hasattr(- weight, key), (f"Overwriting existing tensor attribute: {key}")+ assert not hasattr(weight, key), f"Overwriting existing tensor attribute: {key}"setattr(weight, key, value)def marlin_is_k_full(act_order: bool, is_row_parallel: bool) -> bool:return (not act_order) or (act_order and not is_row_parallel)- def marlin_repeat_scales_on_all_ranks(act_order: bool, group_size: int,- is_row_parallel: bool) -> bool:+ def marlin_repeat_scales_on_all_ranks(act_order: bool, group_size: int, is_row_parallel: bool) -> bool:# Need to repeat scales on every rank if act_ordering or# channelwise and RowParallelLinearis_channelwise = group_size == -1return act_order or (is_channelwise and is_row_parallel)- def marlin_make_workspace(output_size_per_partition: int,- device: torch.device) -> torch.Tensor:- max_workspace_size = (output_size_per_partition //- GPTQ_MARLIN_MIN_THREAD_N) * GPTQ_MARLIN_MAX_PARALLEL+ def marlin_make_workspace(output_size_per_partition: int, device: torch.device) -> torch.Tensor:+ max_workspace_size = (output_size_per_partition // GPTQ_MARLIN_MIN_THREAD_N) * GPTQ_MARLIN_MAX_PARALLEL- return torch.zeros(max_workspace_size,- dtype=torch.int,- device=device,- requires_grad=False)+ return torch.zeros(max_workspace_size, dtype=torch.int, device=device, requires_grad=False)- def marlin_sort_g_idx(- g_idx: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]:+ def marlin_sort_g_idx(g_idx: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]:g_idx_sort_indices = torch.argsort(g_idx).to(torch.int)return g_idx[g_idx_sort_indices], g_idx_sort_indicesdef marlin_make_empty_g_idx(device: torch.device) -> torch.Tensor:- return torch.nn.Parameter(torch.empty(0, dtype=torch.int, device=device),- requires_grad=False)+ return torch.nn.Parameter(torch.empty(0, dtype=torch.int, device=device), requires_grad=False)# Newly generated tensors need to replace existing tensors that are# already registered as parameters by vLLM (and won't be freed)- def replace_tensor(layer: torch.nn.Module, name: str,- new_t: torch.Tensor) -> None:+ def replace_tensor(layer: torch.nn.Module, name: str, new_t: torch.Tensor) -> None:# It is important to use resize_() here since it ensures# the same buffer is reusedgetattr(layer, name).resize_(new_t.shape)getattr(layer, name).copy_(new_t)del new_t- def marlin_permute_scales(s: torch.Tensor, size_k: int, size_n: int,- group_size: int) -> torch.Tensor:+ def marlin_permute_scales(s: torch.Tensor, size_k: int, size_n: int, group_size: int) -> torch.Tensor:scale_perm, scale_perm_single = get_scale_perms()if group_size < size_k and group_size != -1:@@ -118,24 +107,23 @@ def get_marlin_layer(): ##use an ugly wrapper to import gptqmodel on demandscale_perm.extend([i + 8 * j for j in range(8)])scale_perm_single: List[int] = []for i in range(4):- scale_perm_single.extend(- [2 * i + j for j in [0, 1, 8, 9, 16, 17, 24, 25]])+ scale_perm_single.extend([2 * i + j for j in [0, 1, 8, 9, 16, 17, 24, 25]])return scale_perm, scale_perm_singledef apply_gptq_marlin_linear(- input: torch.Tensor,- weight: torch.Tensor,- weight_scale: torch.Tensor,- weight_zp: torch.Tensor,- g_idx: torch.Tensor,- g_idx_sort_indices: torch.Tensor,- workspace: torch.Tensor,- num_bits: int,- output_size_per_partition: int,- input_size_per_partition: int,- is_k_full: bool,- bias: torch.Tensor,- fp32: bool,+ input: torch.Tensor,+ weight: torch.Tensor,+ weight_scale: torch.Tensor,+ weight_zp: torch.Tensor,+ g_idx: torch.Tensor,+ g_idx_sort_indices: torch.Tensor,+ workspace: torch.Tensor,+ num_bits: int,+ output_size_per_partition: int,+ input_size_per_partition: int,+ is_k_full: bool,+ bias: torch.Tensor,+ fp32: bool,) -> torch.Tensor:reshaped_x = input.reshape(-1, input.shape[-1])@@ -184,15 +172,17 @@ def get_marlin_layer(): ##use an ugly wrapper to import gptqmodel on demandQUANT_TYPE = "marlin"def __init__(- self, bits: int,- group_size: int,- desc_act: bool,- sym: bool,- in_features: int,- out_features: int,- bias: bool = False,- pack_dtype: torch.dtype = torch.int32,- **kwargs):+ self,+ bits: int,+ group_size: int,+ desc_act: bool,+ sym: bool,+ in_features: int,+ out_features: int,+ bias: bool = False,+ pack_dtype: torch.dtype = torch.int32,+ **kwargs,+ ):if marlin_import_exception is not None:raise ValueError(f"Trying to use the marlin backend, but could not "@@ -219,7 +209,8 @@ def get_marlin_layer(): ##use an ugly wrapper to import gptqmodel on demandbackend=kwargs.pop("backend", BACKEND.MARLIN),adapter=None,register_buffers=False,- **kwargs)+ **kwargs,+ )# toggle fp32 mode depending on MARLIN or MARLIN_FP16 backendself.fp32 = True if self.backend in [BACKEND.MARLIN, BACKEND.AUTO] else False@@ -227,12 +218,11 @@ def get_marlin_layer(): ##use an ugly wrapper to import gptqmodel on demandif not self.fp32:logger.warning_once("Kernel: Marlin FP16 mode is activated with reduced accuracy. "- "Use default Marlin model for improved inference quality.")+ "Use default Marlin model for improved inference quality."+ )# Determine sharding- if marlin_repeat_scales_on_all_ranks(desc_act,- self.group_size,- is_row_parallel=False):+ if marlin_repeat_scales_on_all_ranks(desc_act, self.group_size, is_row_parallel=False):# By setting scale_dim == None, weight_loader will# repeat the scales on each GPU in TP>1 case.scales_and_zp_input_dim = None@@ -360,8 +350,7 @@ def get_marlin_layer(): ##use an ugly wrapper to import gptqmodel on demanddef post_init(self):device = self.qweight.device# Allocate marlin workspace- self.workspace = marlin_make_workspace(- self.out_features, device)+ self.workspace = marlin_make_workspace(self.out_features, device)# Handle sorting for activation reordering if needed.@@ -373,20 +362,14 @@ def get_marlin_layer(): ##use an ugly wrapper to import gptqmodel on demand# Repack weights from autogptq format to marlin format.marlin_qweight = gptqmodel_marlin_kernels.gptq_marlin_repack(- self.qweight,- g_idx_sort_indices,- self.in_features,- self.out_features,- self.bits,- self.pack_dtype_bits)+ self.qweight, g_idx_sort_indices, self.in_features, self.out_features, self.bits, self.pack_dtype_bits+ )replace_tensor(self, "qweight", marlin_qweight)# Permute scales from autogptq format to marlin format.marlin_scales = marlin_permute_scales(- self.scales,- size_k=self.in_features,- size_n=self.out_features,- group_size=self.group_size)+ self.scales, size_k=self.in_features, size_n=self.out_features, group_size=self.group_size+ )replace_tensor(self, "scales", marlin_scales)super().post_init()diff --git a/auto_round_extension/hpu/__init__.py b/auto_round_extension/hpu/__init__.pyindex 2b414eb72..0d2740cdb 100644--- a/auto_round_extension/hpu/__init__.py+++ b/auto_round_extension/hpu/__init__.py@@ -11,4 +11,3 @@# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.# See the License for the specific language governing permissions and# limitations under the License.-diff --git a/auto_round_extension/hpu/qlinear_hpu.py b/auto_round_extension/hpu/qlinear_hpu.pyindex 5bcd06927..a2077b7cf 100644--- a/auto_round_extension/hpu/qlinear_hpu.py+++ b/auto_round_extension/hpu/qlinear_hpu.py@@ -43,6 +43,7 @@ import transformerstry:import habana_frameworks.torch.core as htcore+convert_from_uint4 = torch.ops.hpu.convert_from_uint4except Exception as e:hpu_import_exception = e@@ -57,7 +58,8 @@ except Exception as e:logger = getLogger(__name__)-def pack_tensor(input, bits = 4):++def pack_tensor(input, bits=4):normal = input.to(torch.int32)q = torch.zeros((normal.shape[0], normal.shape[1] // 32 * bits), dtype=torch.int32)i = 0@@ -70,6 +72,7 @@ def pack_tensor(input, bits = 4):q = q.to(torch.int32)return q+class QuantLinear(nn.Module):QUANT_TYPE = "hpu"@@ -85,7 +88,9 @@ class QuantLinear(nn.Module):trainable=False,weight_dtype=torch.float16,):- logger.debug(f"qlinear_hpu QuantLinear::__init__ {bits=}, {group_size=}, {infeatures=}, {outfeatures=}, {bias=}, {use_cuda_fp16=}, {kernel_switch_threshold=}, {trainable=}, {weight_dtype=}")+ logger.debug(+ f"qlinear_hpu QuantLinear::__init__ {bits=}, {group_size=}, {infeatures=}, {outfeatures=}, {bias=}, {use_cuda_fp16=}, {kernel_switch_threshold=}, {trainable=}, {weight_dtype=}"+ )super().__init__()if bits != 4:raise NotImplementedError("Only 4 bits are supported.")@@ -134,7 +139,7 @@ class QuantLinear(nn.Module):self.qweight = self.qweight.cpu()weight = self.unpack_weight_from_cuda_old_format()new_qweight = pack_tensor(weight)- self.qweight = new_qweight.to('hpu')+ self.qweight = new_qweight.to("hpu")# TODO: Support group indexing and remove the checkcolumns = self.qweight.shape[0]@@ -144,13 +149,13 @@ class QuantLinear(nn.Module):zeros = self.unpack_zeros_from_cuda_old_format().cpu()new_qzeros = pack_tensor(zeros)- self.qzeros = new_qzeros.to('hpu')+ self.qzeros = new_qzeros.to("hpu")def post_init(self):self._preprocessing()def pack(self, linear, scales, zeros, g_idx):- #TODO: implement+ # TODO: implementraise NotImplementedError("QuantLinear HPU currently doesn't support packing")def set_packed(self, qlinear_cls):@@ -179,20 +184,21 @@ class QuantLinear(nn.Module):).to(torch.int16 if self.bits == 8 else torch.int8)# zeros = zeros + 1 # This line is commented for auto-round format export. \- # It remains unchanged for gptq format in qlinear_hpu_gptq.py- zeros = torch.bitwise_and(- zeros, (2**self.bits) - 1- ).to(self.scales.dtype) # NOTE: It appears that casting here after the `zeros = zeros + 1` is important.+ # It remains unchanged for gptq format in qlinear_hpu_gptq.py+ zeros = torch.bitwise_and(zeros, (2**self.bits) - 1).to(+ self.scales.dtype+ ) # NOTE: It appears that casting here after the `zeros = zeros + 1` is important.zeros = zeros.reshape(-1, zeros.shape[1] * zeros.shape[2])return zerosdef unpack_weight_from_cuda_old_format(self):weight = torch.bitwise_right_shift(- torch.unsqueeze(self.qweight, 1).expand(-1, 32 // self.bits, -1),- self.wf.unsqueeze(-1),- ).to(torch.int16 if self.bits == 8 else torch.int8)+ torch.unsqueeze(self.qweight, 1).expand(-1, 32 // self.bits, -1),+ self.wf.unsqueeze(-1),+ ).to(torch.int16 if self.bits == 8 else torch.int8)weight = torch.bitwise_and(weight, (2**self.bits) - 1)- weight = weight.reshape((weight.shape[0]*weight.shape[1], weight.shape[2]))+ weight = weight.reshape((weight.shape[0] * weight.shape[1], weight.shape[2]))return weight+__all__ = ["QuantLinear"]diff --git a/auto_round_extension/hpu/qlinear_hpu_gptq.py b/auto_round_extension/hpu/qlinear_hpu_gptq.pyindex 0016ff032..ce807ce7a 100644--- a/auto_round_extension/hpu/qlinear_hpu_gptq.py+++ b/auto_round_extension/hpu/qlinear_hpu_gptq.py@@ -43,6 +43,7 @@ import transformerstry:import habana_frameworks.torch.core as htcore+convert_from_uint4 = torch.ops.hpu.convert_from_uint4except Exception as e:hpu_import_exception = e@@ -57,7 +58,8 @@ except Exception as e:logger = getLogger(__name__)-def pack_tensor(input, bits = 4):++def pack_tensor(input, bits=4):normal = input.to(torch.int32)q = torch.zeros((normal.shape[0], normal.shape[1] // 32 * bits), dtype=torch.int32)i = 0@@ -70,6 +72,7 @@ def pack_tensor(input, bits = 4):q = q.to(torch.int32)return q+class QuantLinear(nn.Module):QUANT_TYPE = "hpu"@@ -85,7 +88,9 @@ class QuantLinear(nn.Module):trainable=False,weight_dtype=torch.float16,):- logger.debug(f"qlinear_hpu QuantLinear::__init__ {bits=}, {group_size=}, {infeatures=}, {outfeatures=}, {bias=}, {use_cuda_fp16=}, {kernel_switch_threshold=}, {trainable=}, {weight_dtype=}")+ logger.debug(+ f"qlinear_hpu QuantLinear::__init__ {bits=}, {group_size=}, {infeatures=}, {outfeatures=}, {bias=}, {use_cuda_fp16=}, {kernel_switch_threshold=}, {trainable=}, {weight_dtype=}"+ )super().__init__()if bits != 4:raise NotImplementedError("Only 4 bits are supported.")@@ -134,7 +139,7 @@ class QuantLinear(nn.Module):self.qweight = self.qweight.cpu()weight = self.unpack_weight_from_cuda_old_format()new_qweight = pack_tensor(weight)- self.qweight = new_qweight.to('hpu')+ self.qweight = new_qweight.to("hpu")# TODO: Support group indexing and remove the checkcolumns = self.qweight.shape[0]@@ -144,13 +149,13 @@ class QuantLinear(nn.Module):zeros = self.unpack_zeros_from_cuda_old_format().cpu()new_qzeros = pack_tensor(zeros)- self.qzeros = new_qzeros.to('hpu')+ self.qzeros = new_qzeros.to("hpu")def post_init(self):self._preprocessing()def pack(self, linear, scales, zeros, g_idx):- #TODO: implement+ # TODO: implementraise NotImplementedError("QuantLinear HPU currently doesn't support packing")def set_packed(self, qlinear_cls):@@ -179,19 +184,20 @@ class QuantLinear(nn.Module):).to(torch.int16 if self.bits == 8 else torch.int8)zeros = zeros + 1- zeros = torch.bitwise_and(- zeros, (2**self.bits) - 1- ).to(self.scales.dtype) # NOTE: It appears that casting here after the `zeros = zeros + 1` is important.+ zeros = torch.bitwise_and(zeros, (2**self.bits) - 1).to(+ self.scales.dtype+ ) # NOTE: It appears that casting here after the `zeros = zeros + 1` is important.zeros = zeros.reshape(-1, zeros.shape[1] * zeros.shape[2])return zerosdef unpack_weight_from_cuda_old_format(self):weight = torch.bitwise_right_shift(- torch.unsqueeze(self.qweight, 1).expand(-1, 32 // self.bits, -1),- self.wf.unsqueeze(-1),- ).to(torch.int16 if self.bits == 8 else torch.int8)+ torch.unsqueeze(self.qweight, 1).expand(-1, 32 // self.bits, -1),+ self.wf.unsqueeze(-1),+ ).to(torch.int16 if self.bits == 8 else torch.int8)weight = torch.bitwise_and(weight, (2**self.bits) - 1)- weight = weight.reshape((weight.shape[0]*weight.shape[1], weight.shape[2]))+ weight = weight.reshape((weight.shape[0] * weight.shape[1], weight.shape[2]))return weight+__all__ = ["QuantLinear"]diff --git a/auto_round_extension/ipex/qlinear_ipex_awq.py b/auto_round_extension/ipex/qlinear_ipex_awq.pyindex 83afdb2ff..b521881ff 100644--- a/auto_round_extension/ipex/qlinear_ipex_awq.py+++ b/auto_round_extension/ipex/qlinear_ipex_awq.py@@ -18,6 +18,7 @@ import torch.nn as nnclass QuantLinear(nn.Module):QUANT_TYPE = "ipex_awq"+def __init__(self, w_bit, group_size, in_features, out_features, bias, dev):super().__init__()assert w_bit == 4, "Only 4 bit are supported for now."@@ -47,7 +48,8 @@ class QuantLinear(nn.Module):(in_features // self.group_size, out_features),dtype=self.compute_dtype,device=dev,- ))+ ),+ )if bias:self.register_buffer("bias",@@ -65,18 +67,19 @@ class QuantLinear(nn.Module):assert self.qweight.device.type == "cpu" or self.qweight.device.type == "xpu"import intel_extension_for_pytorch as ipex- self.ipex_linear = ipex.llm.quantization.IPEXWeightOnlyQuantizedLinear.from_weight(self.qweight,- self.scales,- self.qzeros, \- self.in_features,- self.out_features,- None,- self.bias, \- self.group_size,- None,- 1,- 0- )+ self.ipex_linear = ipex.llm.quantization.IPEXWeightOnlyQuantizedLinear.from_weight(+ self.qweight,+ self.scales,+ self.qzeros,+ self.in_features,+ self.out_features,+ None,+ self.bias,+ self.group_size,+ None,+ 1,+ 0,+ )@classmethoddef from_linear(cls, linear, w_bit, group_size, init_only=False, scales=None):@@ -101,11 +104,10 @@ class QuantLinear(nn.Module):return outputsdef extra_repr(self) -> str:- return ("in_features={}, out_features={}, bias={}, w_bit={}, group_size={}".format(+ return "in_features={}, out_features={}, bias={}, w_bit={}, group_size={}".format(self.in_features,self.out_features,self.bias is not None,self.w_bit,self.group_size,- ))-+ )diff --git a/auto_round_extension/ipex/qlinear_ipex_gptq.py b/auto_round_extension/ipex/qlinear_ipex_gptq.pyindex e308ec5b9..0ed764c74 100644--- a/auto_round_extension/ipex/qlinear_ipex_gptq.py+++ b/auto_round_extension/ipex/qlinear_ipex_gptq.py@@ -46,16 +46,16 @@ class QuantLinear(nn.Module):QUANT_TYPE = "ipex_gptq"def __init__(- self,- bits,- group_size,- infeatures,- outfeatures,- bias,- kernel_switch_threshold=128,- training=False,- weight_dtype=torch.bfloat16,- **kwargs,+ self,+ bits,+ group_size,+ infeatures,+ outfeatures,+ bias,+ kernel_switch_threshold=128,+ training=False,+ weight_dtype=torch.bfloat16,+ **kwargs,):super().__init__()@@ -66,7 +66,7 @@ class QuantLinear(nn.Module):self.outfeatures = outfeaturesself.bits = bitsself.group_size = group_size if group_size != -1 else infeatures- self.maxq = 2 ** self.bits - 1+ self.maxq = 2**self.bits - 1self.weight_dtype = weight_dtypeself.asym = True@@ -121,20 +121,22 @@ class QuantLinear(nn.Module):from packaging import versionfrom auto_round.utils import get_library_version+ipex_version = get_library_version("intel_extension_for_pytorch")if version.parse(ipex_version) >= version.parse("2.5"):- self.ipex_linear = ipex.llm.quantization.IPEXWeightOnlyQuantizedLinear.from_weight(self.qweight,- self.scales,- self.qzeros, \- self.infeatures,- self.outfeatures,- None,- self.bias, \- self.group_size,- None,- ipex.llm.quantization.QuantMethod.GPTQ_GEMM,- ipex.llm.quantization.QuantDtype.INT4- )+ self.ipex_linear = ipex.llm.quantization.IPEXWeightOnlyQuantizedLinear.from_weight(+ self.qweight,+ self.scales,+ self.qzeros,+ self.infeatures,+ self.outfeatures,+ None,+ self.bias,+ self.group_size,+ None,+ ipex.llm.quantization.QuantMethod.GPTQ_GEMM,+ ipex.llm.quantization.QuantDtype.INT4,+ )else:import intel_extension_for_pytorch as ipexfrom intel_extension_for_pytorch.nn.modules import WeightOnlyQuantizedLinear as ipex_linear@@ -179,7 +181,7 @@ class QuantLinear(nn.Module):zero_points=qzeros,bias=(self.bias.float() if hasattr(self, "bias") and self.bias is not None else None),group_size=self.group_size,- g_idx=(self.g_idx if hasattr(self, "g_idx") else None)+ g_idx=(self.g_idx if hasattr(self, "g_idx") else None),)def pack(self, linear, scales, zeros, g_idx=None):@@ -201,8 +203,10 @@ class QuantLinear(nn.Module):intweight = []for idx in range(self.infeatures):intweight.append(- torch.round(- (W[:, idx] + scale_zeros[self.g_idx[idx]]) / self.scales[self.g_idx[idx]]).to(torch.int)[:, None])+ torch.round((W[:, idx] + scale_zeros[self.g_idx[idx]]) / self.scales[self.g_idx[idx]]).to(torch.int)[+ :, None+ ]+ )intweight = torch.cat(intweight, dim=1)intweight = intweight.t().contiguous()intweight = intweight.numpy().astype(np.uint32)@@ -288,7 +292,7 @@ class QuantLinear(nn.Module):torch.unsqueeze(self.qzeros, 2).expand(-1, -1, 32 // self.bits),self.wf.unsqueeze(0),).to(torch.int16)- zeros = torch.bitwise_and(zeros, (2 ** self.bits) - 1)+ zeros = torch.bitwise_and(zeros, (2**self.bits) - 1)zeros = zeros + 1zeros = zeros.reshape(self.scales.shape)@@ -297,7 +301,7 @@ class QuantLinear(nn.Module):torch.unsqueeze(self.qweight, 1).expand(-1, 32 // self.bits, -1),self.wf.unsqueeze(-1),).to(torch.int16)- weight = torch.bitwise_and(weight, (2 ** self.bits) - 1)+ weight = torch.bitwise_and(weight, (2**self.bits) - 1)weight = weight.reshape(weight.shape[0] * weight.shape[1], weight.shape[2])num_itr = self.g_idx.shape[0] // x.shape[-1]@@ -307,10 +311,10 @@ class QuantLinear(nn.Module):num_dim = self.g_idx.shape[0] // num_itrweights = []for i in range(num_itr):- scale_i = self.scales[:, i * num_dim: (i + 1) * num_dim]- weight_i = weight[:, i * num_dim: (i + 1) * num_dim]- zeros_i = zeros[:, i * num_dim: (i + 1) * num_dim]- g_idx_i = self.g_idx[i * num_dim: (i + 1) * num_dim]+ scale_i = self.scales[:, i * num_dim : (i + 1) * num_dim]+ weight_i = weight[:, i * num_dim : (i + 1) * num_dim]+ zeros_i = zeros[:, i * num_dim : (i + 1) * num_dim]+ g_idx_i = self.g_idx[i * num_dim : (i + 1) * num_dim]weights.append(scale_i[g_idx_i.long()] * (weight_i - zeros_i[g_idx_i.long()]))weights = torch.cat(weights, dim=1)out = torch.matmul(x, weights)@@ -329,10 +333,10 @@ def unpack_to_8bit_signed(qweight, qzeros, bits, g_idx=None):zp_shape = list(qzeros.shape)zp_shape[1] = zp_shape[1] * (32 // bits)- zeros = torch.bitwise_right_shift(- torch.unsqueeze(qzeros, 2).expand(-1, -1, 32 // bits), wf.unsqueeze(0)- ).to(torch.int16 if bits == 8 else torch.int8)- torch.bitwise_and(zeros, (2 ** bits) - 1, out=zeros)+ zeros = torch.bitwise_right_shift(torch.unsqueeze(qzeros, 2).expand(-1, -1, 32 // bits), wf.unsqueeze(0)).to(+ torch.int16 if bits == 8 else torch.int8+ )+ torch.bitwise_and(zeros, (2**bits) - 1, out=zeros)if bits == 8:zeros = zeros.to(torch.uint8)zeros = zeros + 1@@ -344,10 +348,10 @@ def unpack_to_8bit_signed(qweight, qzeros, bits, g_idx=None):zeros = zeros[zeros != 1]zeros = zeros.reshape(zp_shape)- weight = torch.bitwise_right_shift(- torch.unsqueeze(qweight, 1).expand(-1, 32 // bits, -1), wf.unsqueeze(-1)- ).to(torch.int16 if bits == 8 else torch.int8)- weight.bitwise_and_((2 ** bits) - 1)+ weight = torch.bitwise_right_shift(torch.unsqueeze(qweight, 1).expand(-1, 32 // bits, -1), wf.unsqueeze(-1)).to(+ torch.int16 if bits == 8 else torch.int8+ )+ weight.bitwise_and_((2**bits) - 1)weight = weight.view(-1, weight.shape[-1])if g_idx is not None:diff --git a/auto_round_extension/qbits/qbits_awq.py b/auto_round_extension/qbits/qbits_awq.pyindex 07b3012ba..d4b57505f 100644--- a/auto_round_extension/qbits/qbits_awq.py+++ b/auto_round_extension/qbits/qbits_awq.py@@ -16,6 +16,8 @@ import torchimport torch.nn as nnAWQ_REVERSE_ORDER = [0, 4, 1, 5, 2, 6, 3, 7]++def unpack_awq(qweight: torch.Tensor, qzeros: torch.Tensor, bits: int):shifts = torch.arange(0, 32, bits, device="cpu")@@ -53,7 +55,6 @@ def reverse_awq_order(iweights: torch.Tensor, izeros: torch.Tensor, bits: int):return iweights, izeros-try:from intel_extension_for_transformers import qbits # with QBits kernels ()@@ -87,8 +88,9 @@ class QuantLinear(nn.Module):def __init__(self, w_bit, group_size, in_features, out_features, bias, zero_point, dev):super().__init__()- assert QBITS_INSTALLED, \- "Please install ITREX qbits package with `pip install intel-extension-for-transformers`."+ assert (+ QBITS_INSTALLED+ ), "Please install ITREX qbits package with `pip install intel-extension-for-transformers`."self.use_bf16 = qbits.check_isa_supported("AMX")@@ -112,7 +114,7 @@ class QuantLinear(nn.Module):(in_features // self.group_size, out_features // self.pack_num),dtype=torch.int8,device=dev,- )+ ),)self.register_buffer("scales",@@ -120,7 +122,8 @@ class QuantLinear(nn.Module):(in_features // self.group_size, out_features),dtype=torch.bfloat16 if self.use_bf16 else torch.float32,device=dev,- ))+ ),+ )if bias:self.register_buffer("bias",@@ -139,20 +142,24 @@ class QuantLinear(nn.Module):intweight, zeros = unpack_awq(self.qweight, self.qzeros, self.w_bit) # weight: k x n zeros: k / group_size x nintweight, zeros = reverse_awq_order(intweight, zeros, self.w_bit) # weight: k x n zeros: k / group_size x n- if self.zero_point: ## asym has accuracy issue, have not root caused yet- intweight = torch.bitwise_and(intweight, (2 ** self.w_bit) - 1) - (2 ** (self.w_bit - 1))- zeros = torch.bitwise_and(zeros, (2 ** self.w_bit) - 1) - (2 ** (self.w_bit - 1))+ if self.zero_point: ## asym has accuracy issue, have not root caused yet+ intweight = torch.bitwise_and(intweight, (2**self.w_bit) - 1) - (2 ** (self.w_bit - 1))+ zeros = torch.bitwise_and(zeros, (2**self.w_bit) - 1) - (2 ** (self.w_bit - 1))else:##symmetric, our default zp is 8- intweight = torch.bitwise_and(intweight, (2 ** self.w_bit) - 1) - (2 ** (self.w_bit - 1))+ intweight = torch.bitwise_and(intweight, (2**self.w_bit) - 1) - (2 ** (self.w_bit - 1))g_idx = torch.empty(0, dtype=torch.int32)- self.qweight = qbits.repack_quantized_weight(intweight, self.scales.float(), zeros, g_idx,- BITS_DTYPE_MAPPING[self.w_bit],- convert_dtype_torch2str(self.scale_dtype),- convert_dtype_torch2str(self.scales.dtype), self.zero_point,- self.group_size)--+ self.qweight = qbits.repack_quantized_weight(+ intweight,+ self.scales.float(),+ zeros,+ g_idx,+ BITS_DTYPE_MAPPING[self.w_bit],+ convert_dtype_torch2str(self.scale_dtype),+ convert_dtype_torch2str(self.scales.dtype),+ self.zero_point,+ self.group_size,+ )@classmethoddef from_linear(cls, linear, w_bit, group_size, init_only=False, scales=None, zeros=None, has_zero_points=False):@@ -175,7 +182,8 @@ class QuantLinear(nn.Module):assert QBITS_INSTALLED, ("QBits kernels could not be loaded. ""Please install with `pip install intel-extension-for-transformers` and "- "refer to the detail https://github.com/intel/intel-extension-for-transformers/blob/main/docs/qbits.md")+ "refer to the detail https://github.com/intel/intel-extension-for-transformers/blob/main/docs/qbits.md"+ )input_dtype = x.dtypeout_shape = x.shape[:-1] + (self.out_features,)@@ -183,22 +191,33 @@ class QuantLinear(nn.Module):out_2d_shape = x.shape[:-1] + (self.out_features,)outputs = torch.zeros(out_2d_shape, dtype=input_dtype)- bias = self.bias if self.bias is not None else torch.empty(- 0, dtype=torch.bfloat16 if self.use_bf16 else torch.float32)+ bias = (+ self.bias+ if self.bias is not None+ else torch.empty(0, dtype=torch.bfloat16 if self.use_bf16 else torch.float32)+ )- qbits.woq_linear(x, self.qweight, bias, outputs, convert_dtype_torch2str(input_dtype),- BITS_DTYPE_MAPPING[self.w_bit], convert_dtype_torch2str(self.scale_dtype), True)+ qbits.woq_linear(+ x,+ self.qweight,+ bias,+ outputs,+ convert_dtype_torch2str(input_dtype),+ BITS_DTYPE_MAPPING[self.w_bit],+ convert_dtype_torch2str(self.scale_dtype),+ True,+ )return outputs.view(out_shape)def extra_repr(self) -> str:- return ("in_features={}, out_features={}, bias={}, w_bit={}, group_size={}".format(+ return "in_features={}, out_features={}, bias={}, w_bit={}, group_size={}".format(self.in_features,self.out_features,self.bias is not None,self.w_bit,self.group_size,- ))+ )def qbits_post_init(model):@@ -206,4 +225,4 @@ def qbits_post_init(model):if isinstance(submodule, QuantLinear):submodule.post_init()- return model\ No newline at end of file+ return modeldiff --git a/auto_round_extension/qbits/qlinear_qbits.py b/auto_round_extension/qbits/qlinear_qbits.pyindex 2dee2d506..202f1eb19 100644--- a/auto_round_extension/qbits/qlinear_qbits.py+++ b/auto_round_extension/qbits/qlinear_qbits.py@@ -1,4 +1,3 @@-# Copyright (c) 2024 Intel Corporation## Licensed under the Apache License, Version 2.0 (the "License");@@ -48,8 +47,7 @@ class QuantLinear(nn.Module):super().__init__()if bits not in [2, 4, 8]:- raise NotImplementedError(- "Only 2, 4,8 bits are supported for QBits.")+ raise NotImplementedError("Only 2, 4,8 bits are supported for QBits.")self.infeatures = infeaturesself.outfeatures = outfeatures@@ -62,8 +60,7 @@ class QuantLinear(nn.Module):self.register_buffer("qweight",- torch.zeros((infeatures // 32 * self.bits,- outfeatures), dtype=torch.int32),+ torch.zeros((infeatures // 32 * self.bits, outfeatures), dtype=torch.int32),)self.register_buffer("qzeros",@@ -83,8 +80,7 @@ class QuantLinear(nn.Module):),)if bias:- self.register_buffer("bias", torch.zeros(- (outfeatures), dtype=torch.float))+ self.register_buffer("bias", torch.zeros((outfeatures), dtype=torch.float))else:self.bias = None@@ -106,52 +102,57 @@ class QuantLinear(nn.Module):# f"Please install torch {version_match_map[itrex_version]} by command 'pip install torch=={version_match_map[itrex_version]} --extra-index-url https://download.pytorch.org/whl/cpu' as Intel Extension for Transformers {itrex_version} is not compatible with current torch.")else:logger.error(- "Please install Intel Extension for Transformers by running 'pip install intel-extension-for-transformers' as qbits linear requirements checking fail. ")+ "Please install Intel Extension for Transformers by running 'pip install intel-extension-for-transformers' as qbits linear requirements checking fail. "+ )exit(1)def post_init(self):import intel_extension_for_transformers+self.qbits = intel_extension_for_transformers.qbitsassert self.qweight.device.type == "cpu"if self.bias is not None:self.bias = self.bias.to(dtype=torch.float32)# intweight: k x n, zeros: k / group_size x n- intweight, zeros = unpack_to_8bit_signed(- self.qweight, self.qzeros, self.bits)+ intweight, zeros = unpack_to_8bit_signed(self.qweight, self.qzeros, self.bits)if zeros is None:zeros = torch.empty(0, dtype=torch.int8)self.asym = Falseelse:# change it to int8 with offset 128if self.bits == 8:- zeros = (zeros.to(torch.int32) -- (2 ** (self.bits - 1))).to(torch.int8)+ zeros = (zeros.to(torch.int32) - (2 ** (self.bits - 1))).to(torch.int8)else:- zeros -= (2**(self.bits - 1))+ zeros -= 2 ** (self.bits - 1)if not self.asym:- intweight -= (2**(self.bits - 1))+ intweight -= 2 ** (self.bits - 1)intweight = intweight.to(torch.uint8 if self.asym else torch.int8)# due to asym return torch.uint8 but backend request int8,# change it to int8 with offset 128if self.asym:- intweight = (intweight.to(torch.int32) -- (2 ** (self.bits - 1))).to(torch.int8)+ intweight = (intweight.to(torch.int32) - (2 ** (self.bits - 1))).to(torch.int8)scales = self.scaleslogger.debug(- f"QBits repack quantized weight: K:{intweight.shape[0]}, N:{intweight.shape[1]}, weight_dtype:{BITS_DTYPE_MAPPING[self.bits]}, scale_dtype:fp32, compute_dtype:fp32, group_size:{self.group_size}")- self.qweight = self.qbits.repack_quantized_weight(intweight.contiguous(), scales.float().contiguous(), zeros.contiguous(), torch.empty(0),- # weight_dtype- BITS_DTYPE_MAPPING[self.bits],- # scale_dtype- "fp32",- # TODO(zhe): consider dynamic-set cmpt for better perf?- "fp32",- self.asym,- self.group_size)+ f"QBits repack quantized weight: K:{intweight.shape[0]}, N:{intweight.shape[1]}, weight_dtype:{BITS_DTYPE_MAPPING[self.bits]}, scale_dtype:fp32, compute_dtype:fp32, group_size:{self.group_size}"+ )+ self.qweight = self.qbits.repack_quantized_weight(+ intweight.contiguous(),+ scales.float().contiguous(),+ zeros.contiguous(),+ torch.empty(0),+ # weight_dtype+ BITS_DTYPE_MAPPING[self.bits],+ # scale_dtype+ "fp32",+ # TODO(zhe): consider dynamic-set cmpt for better perf?+ "fp32",+ self.asym,+ self.group_size,+ )# free memself.qzeros = torch.empty(0)self.scales = torch.empty(0)@@ -165,14 +166,18 @@ class QuantLinear(nn.Module):out_2d_shape = x.shape[:-1] + (self.outfeatures,)outputs = torch.zeros(out_2d_shape, device=x.device, dtype=torch.float)- bias = self.bias if self.bias is not None else torch.empty(- 0, dtype=torch.float)-- self.qbits.woq_linear(x, self.qweight, bias, outputs,- convert_dtype_torch2str(torch.float), # compute_dtype- BITS_DTYPE_MAPPING[self.bits], # weight_dtype- "fp32", # scale_dtype- self.asym)+ bias = self.bias if self.bias is not None else torch.empty(0, dtype=torch.float)++ self.qbits.woq_linear(+ x,+ self.qweight,+ bias,+ outputs,+ convert_dtype_torch2str(torch.float), # compute_dtype+ BITS_DTYPE_MAPPING[self.bits], # weight_dtype+ "fp32", # scale_dtype+ self.asym,+ )return outputs.to(raw_input_dtype).view(out_shape)@@ -184,10 +189,9 @@ def unpack_to_8bit_signed(qweight, qzeros, bits):zp_shape = list(qzeros.shape)zp_shape[1] = zp_shape[1] * (32 // bits)- zeros = torch.bitwise_right_shift(- torch.unsqueeze(qzeros, 2).expand(-1, -1,- 32 // bits), wf.unsqueeze(0)- ).to(torch.int16 if bits == 8 else torch.int8)+ zeros = torch.bitwise_right_shift(torch.unsqueeze(qzeros, 2).expand(-1, -1, 32 // bits), wf.unsqueeze(0)).to(+ torch.int16 if bits == 8 else torch.int8+ )torch.bitwise_and(zeros, (2**bits) - 1, out=zeros)if bits == 8:zeros = zeros.to(torch.uint8)@@ -199,10 +203,9 @@ def unpack_to_8bit_signed(qweight, qzeros, bits):zeros = zeros[zeros != 1]zeros = zeros.reshape(zp_shape)- weight = torch.bitwise_right_shift(- torch.unsqueeze(qweight, 1).expand(-1, 32 //- bits, -1), wf.unsqueeze(-1)- ).to(torch.int16 if bits == 8 else torch.int8)+ weight = torch.bitwise_right_shift(torch.unsqueeze(qweight, 1).expand(-1, 32 // bits, -1), wf.unsqueeze(-1)).to(+ torch.int16 if bits == 8 else torch.int8+ )weight.bitwise_and_((2**bits) - 1)weight = weight.view(-1, weight.shape[-1])@@ -212,15 +215,13 @@ def unpack_to_8bit_signed(qweight, qzeros, bits):# Copied from qlinear_marlin.py@torch.no_grad()def dequantize_weight(qweight, qzeros, scales, bits):- unpacked_qweight, unpacked_qzeros = unpack_to_8bit_signed(- qweight, qzeros, bits)+ unpacked_qweight, unpacked_qzeros = unpack_to_8bit_signed(qweight, qzeros, bits)group_size = unpacked_qweight.shape[0] // scales.shape[0]scales = scales.repeat_interleave(group_size, dim=0)if unpacked_qzeros is not None:unpacked_qzeros = unpacked_qzeros.repeat_interleave(group_size, dim=0)else:- unpacked_qzeros = torch.full_like(- scales, 8 if bits == 4 else 128, dtype=torch.int32)+ unpacked_qzeros = torch.full_like(scales, 8 if bits == 4 else 128, dtype=torch.int32)unpacked_qweight = (unpacked_qweight - unpacked_qzeros) * scalesreturn unpacked_qweight, unpacked_qzerosdiff --git a/auto_round_extension/qbits/qlinear_qbits_gptq.py b/auto_round_extension/qbits/qlinear_qbits_gptq.pyindex e67bf6ef5..c9c9e1fea 100644--- a/auto_round_extension/qbits/qlinear_qbits_gptq.py+++ b/auto_round_extension/qbits/qlinear_qbits_gptq.py@@ -1,4 +1,3 @@-# Copyright (c) 2024 Intel Corporation## Licensed under the Apache License, Version 2.0 (the "License");@@ -48,8 +47,7 @@ class QuantLinear(nn.Module):super().__init__()if bits not in [2, 4, 8]:- raise NotImplementedError(- "Only 2, 4,8 bits are supported for QBits.")+ raise NotImplementedError("Only 2, 4,8 bits are supported for QBits.")self.infeatures = infeaturesself.outfeatures = outfeatures@@ -62,8 +60,7 @@ class QuantLinear(nn.Module):self.register_buffer("qweight",- torch.zeros((infeatures // 32 * self.bits,- outfeatures), dtype=torch.int32),+ torch.zeros((infeatures // 32 * self.bits, outfeatures), dtype=torch.int32),)self.register_buffer("qzeros",@@ -83,8 +80,7 @@ class QuantLinear(nn.Module):),)if bias:- self.register_buffer("bias", torch.zeros(- (outfeatures), dtype=torch.float))+ self.register_buffer("bias", torch.zeros((outfeatures), dtype=torch.float))else:self.bias = None@@ -106,52 +102,57 @@ class QuantLinear(nn.Module):# f"Please install torch {version_match_map[itrex_version]} by command 'pip install torch=={version_match_map[itrex_version]} --extra-index-url https://download.pytorch.org/whl/cpu' as Intel Extension for Transformers {itrex_version} is not compatible with current torch.")else:logger.error(- "Please install Intel Extension for Transformers by running 'pip install intel-extension-for-transformers' as qbits linear requirements checking fail. ")+ "Please install Intel Extension for Transformers by running 'pip install intel-extension-for-transformers' as qbits linear requirements checking fail. "+ )exit(1)def post_init(self):import intel_extension_for_transformers+self.qbits = intel_extension_for_transformers.qbitsassert self.qweight.device.type == "cpu"if self.bias is not None:self.bias = self.bias.to(dtype=torch.float32)# intweight: k x n, zeros: k / group_size x n- intweight, zeros = unpack_to_8bit_signed(- self.qweight, self.qzeros, self.bits)+ intweight, zeros = unpack_to_8bit_signed(self.qweight, self.qzeros, self.bits)if zeros is None:zeros = torch.empty(0, dtype=torch.int8)self.asym = Falseelse:# change it to int8 with offset 128if self.bits == 8:- zeros = (zeros.to(torch.int32) -- (2 ** (self.bits - 1))).to(torch.int8)+ zeros = (zeros.to(torch.int32) - (2 ** (self.bits - 1))).to(torch.int8)else:- zeros -= (2**(self.bits - 1))+ zeros -= 2 ** (self.bits - 1)if not self.asym:- intweight -= (2**(self.bits - 1))+ intweight -= 2 ** (self.bits - 1)intweight = intweight.to(torch.uint8 if self.asym else torch.int8)# due to asym return torch.uint8 but backend request int8,# change it to int8 with offset 128if self.asym:- intweight = (intweight.to(torch.int32) -- (2 ** (self.bits - 1))).to(torch.int8)+ intweight = (intweight.to(torch.int32) - (2 ** (self.bits - 1))).to(torch.int8)scales = self.scaleslogger.debug(- f"QBits repack quantized weight: K:{intweight.shape[0]}, N:{intweight.shape[1]}, weight_dtype:{BITS_DTYPE_MAPPING[self.bits]}, scale_dtype:fp32, compute_dtype:fp32, group_size:{self.group_size}")- self.qweight = self.qbits.repack_quantized_weight(intweight.contiguous(), scales.float().contiguous(), zeros.contiguous(), torch.empty(0),- # weight_dtype- BITS_DTYPE_MAPPING[self.bits],- # scale_dtype- "fp32",- # TODO(zhe): consider dynamic-set cmpt for better perf?- "fp32",- self.asym,- self.group_size)+ f"QBits repack quantized weight: K:{intweight.shape[0]}, N:{intweight.shape[1]}, weight_dtype:{BITS_DTYPE_MAPPING[self.bits]}, scale_dtype:fp32, compute_dtype:fp32, group_size:{self.group_size}"+ )+ self.qweight = self.qbits.repack_quantized_weight(+ intweight.contiguous(),+ scales.float().contiguous(),+ zeros.contiguous(),+ torch.empty(0),+ # weight_dtype+ BITS_DTYPE_MAPPING[self.bits],+ # scale_dtype+ "fp32",+ # TODO(zhe): consider dynamic-set cmpt for better perf?+ "fp32",+ self.asym,+ self.group_size,+ )# free memself.qzeros = torch.empty(0)self.scales = torch.empty(0)@@ -165,14 +166,18 @@ class QuantLinear(nn.Module):out_2d_shape = x.shape[:-1] + (self.outfeatures,)outputs = torch.zeros(out_2d_shape, device=x.device, dtype=torch.float)- bias = self.bias if self.bias is not None else torch.empty(- 0, dtype=torch.float)-- self.qbits.woq_linear(x, self.qweight, bias, outputs,- convert_dtype_torch2str(torch.float), # compute_dtype- BITS_DTYPE_MAPPING[self.bits], # weight_dtype- "fp32", # scale_dtype- self.asym)+ bias = self.bias if self.bias is not None else torch.empty(0, dtype=torch.float)++ self.qbits.woq_linear(+ x,+ self.qweight,+ bias,+ outputs,+ convert_dtype_torch2str(torch.float), # compute_dtype+ BITS_DTYPE_MAPPING[self.bits], # weight_dtype+ "fp32", # scale_dtype+ self.asym,+ )return outputs.to(raw_input_dtype).view(out_shape)@@ -184,14 +189,13 @@ def unpack_to_8bit_signed(qweight, qzeros, bits):zp_shape = list(qzeros.shape)zp_shape[1] = zp_shape[1] * (32 // bits)- zeros = torch.bitwise_right_shift(- torch.unsqueeze(qzeros, 2).expand(-1, -1,- 32 // bits), wf.unsqueeze(0)- ).to(torch.int16 if bits == 8 else torch.int8)+ zeros = torch.bitwise_right_shift(torch.unsqueeze(qzeros, 2).expand(-1, -1, 32 // bits), wf.unsqueeze(0)).to(+ torch.int16 if bits == 8 else torch.int8+ )torch.bitwise_and(zeros, (2**bits) - 1, out=zeros)if bits == 8:zeros = zeros.to(torch.uint8)- zeros+=1+ zeros += 1try:zeros = zeros.reshape(zp_shape)except:@@ -200,10 +204,9 @@ def unpack_to_8bit_signed(qweight, qzeros, bits):zeros = zeros[zeros != 1]zeros = zeros.reshape(zp_shape)- weight = torch.bitwise_right_shift(- torch.unsqueeze(qweight, 1).expand(-1, 32 //- bits, -1), wf.unsqueeze(-1)- ).to(torch.int16 if bits == 8 else torch.int8)+ weight = torch.bitwise_right_shift(torch.unsqueeze(qweight, 1).expand(-1, 32 // bits, -1), wf.unsqueeze(-1)).to(+ torch.int16 if bits == 8 else torch.int8+ )weight.bitwise_and_((2**bits) - 1)weight = weight.view(-1, weight.shape[-1])@@ -213,15 +216,13 @@ def unpack_to_8bit_signed(qweight, qzeros, bits):# Copied from qlinear_marlin.py@torch.no_grad()def dequantize_weight(qweight, qzeros, scales, bits):- unpacked_qweight, unpacked_qzeros = unpack_to_8bit_signed(- qweight, qzeros, bits)+ unpacked_qweight, unpacked_qzeros = unpack_to_8bit_signed(qweight, qzeros, bits)group_size = unpacked_qweight.shape[0] // scales.shape[0]scales = scales.repeat_interleave(group_size, dim=0)if unpacked_qzeros is not None:unpacked_qzeros = unpacked_qzeros.repeat_interleave(group_size, dim=0)else:- unpacked_qzeros = torch.full_like(- scales, 8 if bits == 4 else 128, dtype=torch.int32)+ unpacked_qzeros = torch.full_like(scales, 8 if bits == 4 else 128, dtype=torch.int32)unpacked_qweight = (unpacked_qweight - unpacked_qzeros) * scalesreturn unpacked_qweight, unpacked_qzerosdiff --git a/auto_round_extension/torch/__init__.py b/auto_round_extension/torch/__init__.pyindex 2b414eb72..0d2740cdb 100644--- a/auto_round_extension/torch/__init__.py+++ b/auto_round_extension/torch/__init__.py@@ -11,4 +11,3 @@# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.# See the License for the specific language governing permissions and# limitations under the License.-diff --git a/auto_round_extension/torch/qlinear_torch.py b/auto_round_extension/torch/qlinear_torch.pyindex 401ce149e..c45e183df 100644--- a/auto_round_extension/torch/qlinear_torch.py+++ b/auto_round_extension/torch/qlinear_torch.py@@ -30,9 +30,7 @@ class QuantLinear(nn.Module):QUANT_TYPE = "torch"- def __init__(- self, bits, group_size, infeatures, outfeatures, bias, trainable=False, **kwargs- ):+ def __init__(self, bits, group_size, infeatures, outfeatures, bias, trainable=False, **kwargs):super().__init__()if bits not in [2, 3, 4, 8]:raise NotImplementedError("Only 2,3,4,8 bits are supported.")@@ -64,14 +62,12 @@ class QuantLinear(nn.Module):),)if bias:- self.register_buffer(- "bias", torch.zeros((outfeatures), dtype=torch.float16)- )+ self.register_buffer("bias", torch.zeros((outfeatures), dtype=torch.float16))else:self.bias = Noneself.trainable = trainable-+# is performed by unpacking the weights and using torch.matmulif self.bits in [2, 4, 8]:self.wf = torch.tensor(list(range(0, 32, self.bits)), dtype=torch.int32).unsqueeze(0)@@ -86,7 +82,6 @@ class QuantLinear(nn.Module):).reshape(1, 3, 12)self.dequant_dtype = torch.int16 if self.bits == 8 else torch.int8-def post_init(self):pass@@ -111,15 +106,15 @@ class QuantLinear(nn.Module):repeat_scales = scales.to(device).repeat_interleave(self.group_size, 1)if isinstance(zeros, torch.Tensor):repeat_zeros = zeros.to(device).repeat_interleave(self.group_size, 1)- intweight = torch.round(W.to(device) / repeat_scales[:, :W.shape[1]] + repeat_zeros[:, :W.shape[1]]).to(- torch.int32)+ intweight = torch.round(W.to(device) / repeat_scales[:, : W.shape[1]] + repeat_zeros[:, : W.shape[1]]).to(+ torch.int32+ )else:repeat_zeros = zeros- intweight = torch.round(W.to(device) / repeat_scales[:, :W.shape[1]] + repeat_zeros).to(- torch.int32)+ intweight = torch.round(W.to(device) / repeat_scales[:, : W.shape[1]] + repeat_zeros).to(torch.int32)del repeat_scales-+if self.bits in [2, 4, 8]:intweight = intweight.reshape(-1, intweight.shape[1] // 32 * self.bits, 32 // self.bits)order_map = torch.arange(0, 32 // self.bits, device=device) * self.bits@@ -138,7 +133,7 @@ class QuantLinear(nn.Module):while row < qweight.shape[0]:packed_weight = torch.tensor(intweight[i : i + 10]).to(dtype=torch.int32).t()shifts = torch.arange(0, 10) * self.bits- shifted = (packed_weight << shifts)+ shifted = packed_weight << shiftsqweight[row] |= shifted.sum(dim=-1)i += 10qweight[row] |= intweight[i] << 30@@ -146,8 +141,8 @@ class QuantLinear(nn.Module):qweight[row] |= (intweight[i] >> 2) & 1i += 1packed_weight = torch.tensor(intweight[i : i + 10]).to(dtype=torch.int32).t()- shifts = torch.arange(0, 10) * self.bits + 1- shifted = packed_weight << shifts+ shifts = torch.arange(0, 10) * self.bits + 1+ shifted = packed_weight << shiftsqweight[row] |= shifted.sum(dim=-1)i += 10qweight[row] |= intweight[i] << 31@@ -156,13 +151,13 @@ class QuantLinear(nn.Module):i += 1packed_weight = torch.tensor(intweight[i : i + 10]).to(dtype=torch.int32).t()shifts = torch.arange(0, 10) * self.bits + 2- shifted = packed_weight << shifts+ shifted = packed_weight << shiftsqweight[row] |= shifted.sum(dim=-1)i += 10row += 1-+self.qweight = qweight.cpu()-+if isinstance(zeros, torch.Tensor):zeros = zeros.t().contiguous()zeros = zeros.numpy().astype(np.uint32)@@ -171,7 +166,7 @@ class QuantLinear(nn.Module):col = 0while col < qzeros.shape[1]:if self.bits in [2, 4, 8]:- packed_zeros = torch.tensor(zeros[:, i: i + (32 // self.bits)]).to(dtype=torch.int32)+ packed_zeros = torch.tensor(zeros[:, i : i + (32 // self.bits)]).to(dtype=torch.int32)shifts = torch.arange(0, (32 // self.bits)) * self.bitsshifted = packed_zeros << shiftsqzeros[:, col] |= shifted.sum(dim=-1)@@ -180,7 +175,7 @@ class QuantLinear(nn.Module):elif self.bits == 3:packed_zeros = torch.tensor(zeros[:, i : i + 10]).to(dtype=torch.int32)shifts = torch.arange(0, 10) * self.bits- shifted = packed_zeros << shifts+ shifted = packed_zeros << shiftsqzeros[:, col] = shifted.sum(dim=-1)i += 10qzeros[:, col] |= zeros[:, i] << 30@@ -188,8 +183,8 @@ class QuantLinear(nn.Module):qzeros[:, col] |= (zeros[:, i] >> 2) & 1i += 1packed_zeros = torch.tensor(zeros[:, i : i + 10]).to(dtype=torch.int32)- shifts = torch.arange(0, 10) * self.bits + 1- shifted = packed_zeros << shifts+ shifts = torch.arange(0, 10) * self.bits + 1+ shifted = packed_zeros << shiftsqzeros[:, col] |= shifted.sum(dim=-1)i += 10qzeros[:, col] |= zeros[:, i] << 31@@ -217,7 +212,7 @@ class QuantLinear(nn.Module):out_shape = x.shape[:-1] + (self.outfeatures,)x = x.reshape(-1, x.shape[-1])x_dtype = x.dtype-+if self.bits in [2, 4, 8]:if self.wf.device != self.qzeros.device:self.wf = self.wf.to(self.qzeros.device)@@ -232,14 +227,12 @@ class QuantLinear(nn.Module):torch.unsqueeze(self.qweight, 1).expand(-1, 32 // self.bits, -1),self.wf.unsqueeze(-1),).to(self.dequant_dtype),- self.maxq+ self.maxq,)elif self.bits == 3:if self.wf.device != self.qzeros.device:self.wf = self.wf.to(self.qzeros.device)- zeros = self.qzeros.reshape(self.qzeros.shape[0], self.qzeros.shape[1] // 3, 3, 1).expand(- -1, -1, -1, 12- )+ zeros = self.qzeros.reshape(self.qzeros.shape[0], self.qzeros.shape[1] // 3, 3, 1).expand(-1, -1, -1, 12)zeros = zeros >> self.wf.unsqueeze(0)zeros[:, :, 0, 10] = (zeros[:, :, 0, 10] & 0x3) | ((zeros[:, :, 1, 0] << 2) & 0x4)zeros[:, :, 1, 11] = (zeros[:, :, 1, 11] & 0x1) | ((zeros[:, :, 2, 0] << 1) & 0x6)@@ -259,7 +252,7 @@ class QuantLinear(nn.Module):weight[:, 1, 11] = (weight[:, 1, 11] & 0x1) | ((weight[:, 2, 0] << 1) & 0x6)weight = weight & 0x7weight = torch.cat([weight[:, 0, :11], weight[:, 1, 1:12], weight[:, 2, 1:11]], dim=1)-+weight = weight.reshape(weight.shape[0] * weight.shape[1], weight.shape[2])if hasattr(self, "g_idx"):num_itr = self.g_idx.shape[0] // x.shape[-1]@@ -276,7 +269,7 @@ class QuantLinear(nn.Module):repeat_scales = self.scales.repeat_interleave(self.group_size, dim=0)repeat_zeros = zeros.repeat_interleave(self.group_size, dim=0)weights = repeat_scales * (weight - repeat_zeros)-+weights = weights.to(x_dtype)out = torch.matmul(x, weights)out = out.to(x_dtype)@@ -286,4 +279,3 @@ class QuantLinear(nn.Module):__all__ = ["QuantLinear"]-diff --git a/auto_round_extension/torch/qlinear_torch_zp.py b/auto_round_extension/torch/qlinear_torch_zp.pyindex 503e40e49..32747de0d 100644--- a/auto_round_extension/torch/qlinear_torch_zp.py+++ b/auto_round_extension/torch/qlinear_torch_zp.py@@ -31,9 +31,7 @@ class QuantLinear(nn.Module):QUANT_TYPE = "torch"- def __init__(- self, bits, group_size, infeatures, outfeatures, bias, trainable=False, **kwargs- ):+ def __init__(self, bits, group_size, infeatures, outfeatures, bias, trainable=False, **kwargs):super().__init__()if bits not in [2, 3, 4, 8]:raise NotImplementedError("Only 2,3,4,8 bits are supported.")@@ -65,14 +63,12 @@ class QuantLinear(nn.Module):),)if bias:- self.register_buffer(- "bias", torch.zeros((outfeatures), dtype=torch.float16)- )+ self.register_buffer("bias", torch.zeros((outfeatures), dtype=torch.float16))else:self.bias = Noneself.trainable = trainable-+# is performed by unpacking the weights and using torch.matmulif self.bits in [2, 4, 8]:self.wf = torch.tensor(list(range(0, 32, self.bits)), dtype=torch.int32).unsqueeze(0)@@ -87,7 +83,6 @@ class QuantLinear(nn.Module):).reshape(1, 3, 12)self.dequant_dtype = torch.int16 if self.bits == 8 else torch.int8-def post_init(self):pass@@ -112,15 +107,15 @@ class QuantLinear(nn.Module):repeat_scales = scales.to(device).repeat_interleave(self.group_size, 1)if isinstance(zeros, torch.Tensor):repeat_zeros = zeros.to(device).repeat_interleave(self.group_size, 1)- intweight = torch.round(W.to(device) / repeat_scales[:, :W.shape[1]] + repeat_zeros[:, :W.shape[1]]).to(- torch.int32)+ intweight = torch.round(W.to(device) / repeat_scales[:, : W.shape[1]] + repeat_zeros[:, : W.shape[1]]).to(+ torch.int32+ )else:repeat_zeros = zeros- intweight = torch.round(W.to(device) / repeat_scales[:, :W.shape[1]] + repeat_zeros).to(- torch.int32)+ intweight = torch.round(W.to(device) / repeat_scales[:, : W.shape[1]] + repeat_zeros).to(torch.int32)del repeat_scales-+if self.bits in [2, 4, 8]:intweight = intweight.reshape(-1, intweight.shape[1] // 32 * self.bits, 32 // self.bits)order_map = torch.arange(0, 32 // self.bits, device=device) * self.bits@@ -139,7 +134,7 @@ class QuantLinear(nn.Module):while row < qweight.shape[0]:packed_weight = torch.tensor(intweight[i : i + 10]).to(dtype=torch.int32).t()shifts = torch.arange(0, 10) * self.bits- shifted = (packed_weight << shifts)+ shifted = packed_weight << shiftsqweight[row] |= shifted.sum(dim=-1)i += 10qweight[row] |= intweight[i] << 30@@ -147,8 +142,8 @@ class QuantLinear(nn.Module):qweight[row] |= (intweight[i] >> 2) & 1i += 1packed_weight = torch.tensor(intweight[i : i + 10]).to(dtype=torch.int32).t()- shifts = torch.arange(0, 10) * self.bits + 1- shifted = packed_weight << shifts+ shifts = torch.arange(0, 10) * self.bits + 1+ shifted = packed_weight << shiftsqweight[row] |= shifted.sum(dim=-1)i += 10qweight[row] |= intweight[i] << 31@@ -157,13 +152,13 @@ class QuantLinear(nn.Module):i += 1packed_weight = torch.tensor(intweight[i : i + 10]).to(dtype=torch.int32).t()shifts = torch.arange(0, 10) * self.bits + 2- shifted = packed_weight << shifts+ shifted = packed_weight << shiftsqweight[row] |= shifted.sum(dim=-1)i += 10row += 1-+self.qweight = qweight.cpu()-+zeros = zeros.t().contiguous()zeros -= 1zeros = zeros.numpy().astype(np.uint32)@@ -172,7 +167,7 @@ class QuantLinear(nn.Module):col = 0while col < qzeros.shape[1]:if self.bits in [2, 4, 8]:- packed_zeros = torch.tensor(zeros[:, i: i + (32 // self.bits)]).to(dtype=torch.int32)+ packed_zeros = torch.tensor(zeros[:, i : i + (32 // self.bits)]).to(dtype=torch.int32)shifts = torch.arange(0, (32 // self.bits)) * self.bitsshifted = packed_zeros << shiftsqzeros[:, col] |= shifted.sum(dim=-1)@@ -181,7 +176,7 @@ class QuantLinear(nn.Module):elif self.bits == 3:packed_zeros = torch.tensor(zeros[:, i : i + 10]).to(dtype=torch.int32)shifts = torch.arange(0, 10) * self.bits- shifted = packed_zeros << shifts+ shifted = packed_zeros << shiftsqzeros[:, col] = shifted.sum(dim=-1)i += 10qzeros[:, col] |= zeros[:, i] << 30@@ -189,28 +184,27 @@ class QuantLinear(nn.Module):qzeros[:, col] |= (zeros[:, i] >> 2) & 1i += 1packed_zeros = torch.tensor(zeros[:, i : i + 10]).to(dtype=torch.int32)- shifts = torch.arange(0, 10) * self.bits + 1- shifted = packed_zeros << shifts+ shifts = torch.arange(0, 10) * self.bits + 1+ shifted = packed_zeros << shiftsqzeros[:, col] |= shifted.sum(dim=-1)i += 10qzeros[:, col] |= zeros[:, i] << 31col += 1qzeros[:, col] |= (zeros[:, i] >> 1) & 0x3i += 1- packed_zeros = torch.tensor(zeros[:, i : i + 10]).to(dtype=torch.int32)+ packed_zeros = torch.tensor(zeros[:, i : i + 10]).to(dtype=torch.int32)shifts = torch.arange(0, 10) * self.bits + 2- shifted = packed_zeros << shifts+ shifted = packed_zeros << shiftsqzeros[:, col] |= shifted.sum(dim=-1)i += 10col += 1self.qzeros = qzeros.cpu()-def forward(self, x):out_shape = x.shape[:-1] + (self.outfeatures,)x = x.reshape(-1, x.shape[-1])x_dtype = x.dtype-+if self.bits in [2, 4, 8]:if self.wf.device != self.qzeros.device:self.wf = self.wf.to(self.qzeros.device)@@ -225,14 +219,12 @@ class QuantLinear(nn.Module):torch.unsqueeze(self.qweight, 1).expand(-1, 32 // self.bits, -1),self.wf.unsqueeze(-1),).to(self.dequant_dtype),- self.maxq+ self.maxq,)elif self.bits == 3:if self.wf.device != self.qzeros.device:self.wf = self.wf.to(self.qzeros.device)- zeros = self.qzeros.reshape(self.qzeros.shape[0], self.qzeros.shape[1] // 3, 3, 1).expand(- -1, -1, -1, 12- )+ zeros = self.qzeros.reshape(self.qzeros.shape[0], self.qzeros.shape[1] // 3, 3, 1).expand(-1, -1, -1, 12)zeros = zeros >> self.wf.unsqueeze(0)zeros[:, :, 0, 10] = (zeros[:, :, 0, 10] & 0x3) | ((zeros[:, :, 1, 0] << 2) & 0x4)zeros[:, :, 1, 11] = (zeros[:, :, 1, 11] & 0x1) | ((zeros[:, :, 2, 0] << 1) & 0x6)@@ -252,7 +244,7 @@ class QuantLinear(nn.Module):weight[:, 1, 11] = (weight[:, 1, 11] & 0x1) | ((weight[:, 2, 0] << 1) & 0x6)weight = weight & 0x7weight = torch.cat([weight[:, 0, :11], weight[:, 1, 1:12], weight[:, 2, 1:11]], dim=1)-+weight = weight.reshape(weight.shape[0] * weight.shape[1], weight.shape[2])if hasattr(self, "g_idx"):num_itr = self.g_idx.shape[0] // x.shape[-1]@@ -269,7 +261,7 @@ class QuantLinear(nn.Module):repeat_scales = self.scales.repeat_interleave(self.group_size, dim=0)repeat_zeros = zeros.repeat_interleave(self.group_size, dim=0)weights = repeat_scales * (weight - repeat_zeros)-+weights = weights.to(x_dtype)out = torch.matmul(x, weights)out = out.to(x_dtype)@@ -279,4 +271,3 @@ class QuantLinear(nn.Module):__all__ = ["QuantLinear"]-diff --git a/auto_round_extension/triton/__init__.py b/auto_round_extension/triton/__init__.pyindex 2b414eb72..0d2740cdb 100644--- a/auto_round_extension/triton/__init__.py+++ b/auto_round_extension/triton/__init__.py@@ -11,4 +11,3 @@# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.# See the License for the specific language governing permissions and# limitations under the License.-diff --git a/auto_round_extension/triton/qlinear_tritonv2.py b/auto_round_extension/triton/qlinear_tritonv2.pyindex 7320ba03f..aabf90a1c 100644--- a/auto_round_extension/triton/qlinear_tritonv2.py+++ b/auto_round_extension/triton/qlinear_tritonv2.py@@ -59,16 +59,12 @@ class QuantLinear(nn.Module, TritonModuleMixin):QUANT_TYPE = "tritonv2"- def __init__(- self, bits, group_size, infeatures, outfeatures, bias, trainable=False, **kwargs- ):+ def __init__(self, bits, group_size, infeatures, outfeatures, bias, trainable=False, **kwargs):super().__init__()if bits not in [2, 4, 8]:raise NotImplementedError("Only 2,4,8 bits are supported.")if infeatures % 32 != 0 or outfeatures % 32 != 0:- raise NotImplementedError(- "in_feature and out_feature must be divisible by 32."- )+ raise NotImplementedError("in_feature and out_feature must be divisible by 32.")self.infeatures = infeaturesself.outfeatures = outfeaturesself.bits = bits@@ -97,9 +93,7 @@ class QuantLinear(nn.Module, TritonModuleMixin):),)if bias:- self.register_buffer(- "bias", torch.zeros((outfeatures), dtype=torch.float16)- )+ self.register_buffer("bias", torch.zeros((outfeatures), dtype=torch.float16))else:self.bias = None@@ -128,12 +122,12 @@ class QuantLinear(nn.Module, TritonModuleMixin):repeat_scales = scales.to(device).repeat_interleave(self.group_size, 1)if isinstance(zeros, torch.Tensor):repeat_zeros = zeros.to(device).repeat_interleave(self.group_size, 1)- intweight = torch.round(W.to(device) / repeat_scales[:, :W.shape[1]] + repeat_zeros[:, :W.shape[1]]).to(- torch.int32)+ intweight = torch.round(W.to(device) / repeat_scales[:, : W.shape[1]] + repeat_zeros[:, : W.shape[1]]).to(+ torch.int32+ )else:repeat_zeros = zeros- intweight = torch.round(W.to(device) / repeat_scales[:, :W.shape[1]] + repeat_zeros).to(- torch.int32)+ intweight = torch.round(W.to(device) / repeat_scales[:, : W.shape[1]] + repeat_zeros).to(torch.int32)del repeat_scalesintweight = intweight.reshape(-1, intweight.shape[1] // 32 * self.bits, 32 // self.bits)@@ -171,10 +165,10 @@ class QuantLinear(nn.Module, TritonModuleMixin):def forward(self, x):out_shape = x.shape[:-1] + (self.outfeatures,)quant_linear_fn = QuantLinearFunction- if not hasattr(self,"g_idx"):- self.g_idx = torch.tensor(- [i // self.group_size for i in range(self.infeatures)], dtype=torch.int32- ).to(self.qweight.device)+ if not hasattr(self, "g_idx"):+ self.g_idx = torch.tensor([i // self.group_size for i in range(self.infeatures)], dtype=torch.int32).to(+ self.qweight.device+ )out = quant_linear_fn.apply(x.reshape(-1, x.shape[-1]),@@ -233,4 +227,4 @@ class QuantLinear(nn.Module, TritonModuleMixin):del kn_values-__all__ = ["QuantLinear"]\ No newline at end of file+__all__ = ["QuantLinear"]diff --git a/auto_round_extension/triton/qlinear_tritonv2_zp.py b/auto_round_extension/triton/qlinear_tritonv2_zp.pyindex af18f27b0..222e1d013 100644--- a/auto_round_extension/triton/qlinear_tritonv2_zp.py+++ b/auto_round_extension/triton/qlinear_tritonv2_zp.py@@ -59,16 +59,12 @@ class QuantLinear(nn.Module, TritonModuleMixin):QUANT_TYPE = "tritonv2"- def __init__(- self, bits, group_size, infeatures, outfeatures, bias, trainable=False, **kwargs- ):+ def __init__(self, bits, group_size, infeatures, outfeatures, bias, trainable=False, **kwargs):super().__init__()if bits not in [2, 4, 8]:raise NotImplementedError("Only 2,4,8 bits are supported.")if infeatures % 32 != 0 or outfeatures % 32 != 0:- raise NotImplementedError(- "in_feature and out_feature must be divisible by 32."- )+ raise NotImplementedError("in_feature and out_feature must be divisible by 32.")self.infeatures = infeaturesself.outfeatures = outfeaturesself.bits = bits@@ -103,9 +99,7 @@ class QuantLinear(nn.Module, TritonModuleMixin):# ),# )if bias:- self.register_buffer(- "bias", torch.zeros((outfeatures), dtype=torch.float16)- )+ self.register_buffer("bias", torch.zeros((outfeatures), dtype=torch.float16))else:self.bias = None@@ -113,6 +107,7 @@ class QuantLinear(nn.Module, TritonModuleMixin):def post_init(self):pass+## def pack(self, linear, scales, zeros, g_idx=None):# W = linear.weight.data.clone()@@ -182,9 +177,9 @@ class QuantLinear(nn.Module, TritonModuleMixin):out_shape = x.shape[:-1] + (self.outfeatures,)quant_linear_fn = QuantLinearFunctionif not hasattr(self, "g_idx"):- self.g_idx = torch.tensor(- [i // self.group_size for i in range(self.infeatures)], dtype=torch.int32- ).to(self.qweight.device)+ self.g_idx = torch.tensor([i // self.group_size for i in range(self.infeatures)], dtype=torch.int32).to(+ self.qweight.device+ )out = quant_linear_fn.apply(x.reshape(-1, x.shape[-1]),@@ -243,4 +238,4 @@ class QuantLinear(nn.Module, TritonModuleMixin):del kn_values-__all__ = ["QuantLinear"]\ No newline at end of file+__all__ = ["QuantLinear"]diff --git a/auto_round_extension/triton/triton_utils/__init__.py b/auto_round_extension/triton/triton_utils/__init__.pyindex 2045808ac..28f108cb6 100644--- a/auto_round_extension/triton/triton_utils/__init__.py+++ b/auto_round_extension/triton/triton_utils/__init__.py@@ -11,4 +11,3 @@# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.# See the License for the specific language governing permissions and# limitations under the License.-diff --git a/auto_round_extension/triton/triton_utils/dequant.py b/auto_round_extension/triton/triton_utils/dequant.pyindex 43905629d..33eb8d8e7 100644--- a/auto_round_extension/triton/triton_utils/dequant.py+++ b/auto_round_extension/triton/triton_utils/dequant.py@@ -54,17 +54,17 @@ DEFAULT_DEQUANT_CONFIGS = make_dequant_configs([128, 256, 512, 1024], [4, 8])@triton.autotune(DEFAULT_DEQUANT_CONFIGS, key=["numels"])@triton.jitdef dequant_kernel_248(- g_idx_ptr,- scales_ptr,- qweight_ptr,- qzeros_ptr,- out_ptr,- numels,- maxq: tl.constexpr,- bits: tl.constexpr,- outfeatures: tl.constexpr,- num_groups: tl.constexpr,- X_BLOCK: tl.constexpr,+ g_idx_ptr,+ scales_ptr,+ qweight_ptr,+ qzeros_ptr,+ out_ptr,+ numels,+ maxq: tl.constexpr,+ bits: tl.constexpr,+ outfeatures: tl.constexpr,+ num_groups: tl.constexpr,+ X_BLOCK: tl.constexpr,):# Block indexingxoffset = tl.program_id(0) * X_BLOCK@@ -91,9 +91,7 @@ def dequant_kernel_248(tl.device_assert(g_idx >= 0, "index out of bounds: 0 <= tmp0 < 0")groups = tl.where(tmp2, tmp1, g_idx) # tmp3 are g_idx- scales = tl.load(scales_ptr + (col_idx + (outfeatures * groups)), None).to(- tl.float32- )+ scales = tl.load(scales_ptr + (col_idx + (outfeatures * groups)), None).to(tl.float32)# Unpack weightsweights = qweights >> wf_weights # bit shift qweight@@ -129,7 +127,7 @@ def dequant248_core(qweight, scales, qzeros, g_idx, bits, maxq=None, input_dtypeout = torch.empty((infeatures, outfeatures), device=qweight.device, dtype=input_dtype)numels = out.numel()- maxq = 2 ** bits - 1 if maxq is None else maxq+ maxq = 2**bits - 1 if maxq is None else maxqgrid = lambda meta: (triton.cdiv(numels, meta["X_BLOCK"]),) # noqa: E731dequant_kernel_248[grid](@@ -158,9 +156,8 @@ def dequant248(qweight, scales, qzeros, g_idx, bits, maxq=None, input_dtype=torcelse:raise ValueError(f"Unsupported device type: {device_type}")-def quant_matmul_248(- input, qweight, scales, qzeros, g_idx, bits, maxq=None, transpose=False-):++def quant_matmul_248(input, qweight, scales, qzeros, g_idx, bits, maxq=None, transpose=False):input_dtype = input.dtypeW = dequant248(qweight, scales, qzeros, g_idx, bits, maxq=maxq, input_dtype=input_dtype)orig_device = input.device@@ -185,7 +182,5 @@ class QuantLinearFunction(torch.autograd.Function):grad_input = Noneif ctx.needs_input_grad[0]:- grad_input = quant_matmul_248(- grad_output, qweight, scales, qzeros, g_idx, bits, maxq, transpose=True- )+ grad_input = quant_matmul_248(grad_output, qweight, scales, qzeros, g_idx, bits, maxq, transpose=True)return grad_input, None, None, None, None, None, Nonediff --git a/auto_round_extension/triton/triton_utils/kernels.py b/auto_round_extension/triton/triton_utils/kernels.pyindex 0fb42ac45..a2756d1d9 100644--- a/auto_round_extension/triton/triton_utils/kernels.py+++ b/auto_round_extension/triton/triton_utils/kernels.py@@ -456,7 +456,6 @@ def transpose_quant_matmul_248(input, qweight, scales, qzeros, g_idx, bits, maxqraise ValueError(f"Unsupported device type: {device_type}")-class QuantLinearFunction(torch.autograd.Function):@staticmethod@custom_fwd@@ -516,7 +515,6 @@ def quant_matmul_inference_only_248(input, qweight, scales, qzeros, g_idx, bits,raise ValueError(f"Unsupported device type: {device_type}")-class QuantLinearInferenceOnlyFunction(torch.autograd.Function):@staticmethod@custom_fwd(cast_inputs=torch.float16)diff --git a/auto_round_extension/triton/triton_utils/mixin.py b/auto_round_extension/triton/triton_utils/mixin.pyindex 557d3b480..9fc4d9fec 100644--- a/auto_round_extension/triton/triton_utils/mixin.py+++ b/auto_round_extension/triton/triton_utils/mixin.py@@ -12,6 +12,7 @@# See the License for the specific language governing permissions and# limitations under the License.+# MIT License## Copyright (c) 2023 潘其威(William)diff --git a/auto_round_extension/triton/triton_utils_zp/__init__.py b/auto_round_extension/triton/triton_utils_zp/__init__.pyindex 2045808ac..28f108cb6 100644--- a/auto_round_extension/triton/triton_utils_zp/__init__.py+++ b/auto_round_extension/triton/triton_utils_zp/__init__.py@@ -11,4 +11,3 @@# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.# See the License for the specific language governing permissions and# limitations under the License.-diff --git a/auto_round_extension/triton/triton_utils_zp/dequant.py b/auto_round_extension/triton/triton_utils_zp/dequant.pyindex c6f47c1a1..7d1e63091 100644--- a/auto_round_extension/triton/triton_utils_zp/dequant.py+++ b/auto_round_extension/triton/triton_utils_zp/dequant.py@@ -53,17 +53,17 @@ DEFAULT_DEQUANT_CONFIGS = make_dequant_configs([128, 256, 512, 1024], [4, 8])@triton.autotune(DEFAULT_DEQUANT_CONFIGS, key=["numels"])@triton.jitdef dequant_kernel_248(- g_idx_ptr,- scales_ptr,- qweight_ptr,- qzeros_ptr,- out_ptr,- numels,- maxq: tl.constexpr,- bits: tl.constexpr,- outfeatures: tl.constexpr,- num_groups: tl.constexpr,- X_BLOCK: tl.constexpr,+ g_idx_ptr,+ scales_ptr,+ qweight_ptr,+ qzeros_ptr,+ out_ptr,+ numels,+ maxq: tl.constexpr,+ bits: tl.constexpr,+ outfeatures: tl.constexpr,+ num_groups: tl.constexpr,+ X_BLOCK: tl.constexpr,):# Block indexingxoffset = tl.program_id(0) * X_BLOCK@@ -90,9 +90,7 @@ def dequant_kernel_248(tl.device_assert(g_idx >= 0, "index out of bounds: 0 <= tmp0 < 0")groups = tl.where(tmp2, tmp1, g_idx) # tmp3 are g_idx- scales = tl.load(scales_ptr + (col_idx + (outfeatures * groups)), None).to(- tl.float32- )+ scales = tl.load(scales_ptr + (col_idx + (outfeatures * groups)), None).to(tl.float32)# Unpack weightsweights = qweights >> wf_weights # bit shift qweight@@ -128,7 +126,7 @@ def dequant248_core(qweight, scales, qzeros, g_idx, bits, maxq=None, input_dtypeout = torch.empty((infeatures, outfeatures), device=qweight.device, dtype=input_dtype)numels = out.numel()- maxq = 2 ** bits - 1 if maxq is None else maxq+ maxq = 2**bits - 1 if maxq is None else maxqgrid = lambda meta: (triton.cdiv(numels, meta["X_BLOCK"]),) # noqa: E731dequant_kernel_248[grid](@@ -158,9 +156,7 @@ def dequant248(qweight, scales, qzeros, g_idx, bits, maxq=None, input_dtype=torcraise ValueError(f"Unsupported device type: {device_type}")-def quant_matmul_248(- input, qweight, scales, qzeros, g_idx, bits, maxq=None, transpose=False-):+def quant_matmul_248(input, qweight, scales, qzeros, g_idx, bits, maxq=None, transpose=False):input_dtype = input.dtypeW = dequant248(qweight, scales, qzeros, g_idx, bits, maxq=maxq, input_dtype=input_dtype)orig_device = input.device@@ -185,7 +181,5 @@ class QuantLinearFunction(torch.autograd.Function):grad_input = Noneif ctx.needs_input_grad[0]:- grad_input = quant_matmul_248(- grad_output, qweight, scales, qzeros, g_idx, bits, maxq, transpose=True- )+ grad_input = quant_matmul_248(grad_output, qweight, scales, qzeros, g_idx, bits, maxq, transpose=True)return grad_input, None, None, None, None, None, Nonediff --git a/auto_round_extension/triton/triton_utils_zp/kernels.py b/auto_round_extension/triton/triton_utils_zp/kernels.pyindex 2d4a358aa..b8715519b 100644--- a/auto_round_extension/triton/triton_utils_zp/kernels.py+++ b/auto_round_extension/triton/triton_utils_zp/kernels.py@@ -377,6 +377,7 @@ def transpose_quant_matmul_248_kernel(def silu(x):return x * tl.sigmoid(x)+def quant_matmul_248_core(input, qweight, scales, qzeros, g_idx, bits, maxq):output = torch.empty((input.shape[0], qweight.shape[1]), device=input.device, dtype=input.dtype)grid = lambda META: ( # noqa: E731diff --git a/auto_round_extension/triton/triton_utils_zp/mixin.py b/auto_round_extension/triton/triton_utils_zp/mixin.pyindex 557d3b480..9fc4d9fec 100644--- a/auto_round_extension/triton/triton_utils_zp/mixin.py+++ b/auto_round_extension/triton/triton_utils_zp/mixin.py@@ -12,6 +12,7 @@# See the License for the specific language governing permissions and# limitations under the License.+# MIT License## Copyright (c) 2023 潘其威(William)diff --git a/docs/Llama-3.2-11B-Vision-Instruct-sym.md b/docs/Llama-3.2-11B-Vision-Instruct-sym.mdindex a027d98fd..9ef5ab302 100644--- a/docs/Llama-3.2-11B-Vision-Instruct-sym.md+++ b/docs/Llama-3.2-11B-Vision-Instruct-sym.md@@ -11,13 +11,13 @@ AutoRound version >= 0.4.1### INT4 Inference```python-from auto_round import AutoRoundConfig ## must import for auto-round format+from auto_round import AutoRoundConfig ## must import for auto-round formatimport requestsimport torchfrom PIL import Imagefrom transformers import MllamaForConditionalGeneration, AutoProcessor-quantized_model_path="Intel/Llama-3.2-11B-Vision-Instruct-inc-private"+quantized_model_path = "Intel/Llama-3.2-11B-Vision-Instruct-inc-private"model = MllamaForConditionalGeneration.from_pretrained(quantized_model_path,@@ -28,26 +28,21 @@ model = MllamaForConditionalGeneration.from_pretrained(processor = AutoProcessor.from_pretrained(quantized_model_path)image_url = "https://huggingface.co/datasets/huggingface/documentation-images/resolve/0052a70beed5bf71b92610a43a52df6d286cd5f3/diffusers/rabbit.jpg"messages = [- {"role": "user", "content": [- {"type": "image"},- {"type": "text", "text": "Please write a haiku for this one, it would be: "}- ]}+ {+ "role": "user",+ "content": [{"type": "image"}, {"type": "text", "text": "Please write a haiku for this one, it would be: "}],+ }]# Preparation for inferenceimage = Image.open(requests.get(image_url, stream=True).raw)input_text = processor.apply_chat_template(messages, add_generation_prompt=True)-inputs = processor(- image,- input_text,- add_special_tokens=False,- return_tensors="pt"-).to(model.device)+inputs = processor(image, input_text, add_special_tokens=False, return_tensors="pt").to(model.device)output = model.generate(**inputs, max_new_tokens=50)print(processor.decode(output[0]))-##INT4:+##INT4:## Here is a haiku for the rabbit:## Whiskers twitching bright@@ -55,7 +50,7 @@ print(processor.decode(output[0]))## Spring's gentle delight<|eot_id|>-##BF16:+##BF16:## Here is a haiku for the rabbit:## Whiskers twitching fast@@ -64,30 +59,32 @@ print(processor.decode(output[0]))image_url = "http://images.cocodataset.org/train2017/000000411975.jpg"messages = [- {"role": "user", "content": [- {"type": "image"},- {"type": "text", "text": "How many people are on the baseball field in the picture?"}- ]}+ {+ "role": "user",+ "content": [+ {"type": "image"},+ {"type": "text", "text": "How many people are on the baseball field in the picture?"},+ ],+ }]##INT4: There are five people on the baseball field in the picture.-##+####BF16: There are five people on the baseball field in the picture.-##+##image_url = "https://intelcorp.scene7.com/is/image/intelcorp/processor-overview-framed-badge:1920-1080?wid=480&hei=270"messages = [- {"role": "user", "content": [- {"type": "image"},- {"type": "text", "text": "Which company does this picture represent?"}- ]}+ {+ "role": "user",+ "content": [{"type": "image"}, {"type": "text", "text": "Which company does this picture represent?"}],+ }]##INT4: This picture represents Intel.-##+####BF16: This image represents Intel, a multinational semiconductor corporation headquartered in Santa Clara, California.-##-+##```## Evaluation the modeldiff --git a/docs/Phi-3.5-vision-instruct-sym.md b/docs/Phi-3.5-vision-instruct-sym.mdindex ec2001c67..17c465f97 100644--- a/docs/Phi-3.5-vision-instruct-sym.md+++ b/docs/Phi-3.5-vision-instruct-sym.md@@ -24,56 +24,43 @@ accelerate==0.30.0### INT4 Inference```python-from auto_round import AutoRoundConfig ##must import for auto-round format+from auto_round import AutoRoundConfig ##must import for auto-round formatimport requestsfrom PIL import Imagefrom transformers import AutoModelForCausalLM, AutoTokenizer, AutoProcessor-model_id = "Intel/Phi-3.5-vision-instruct-inc-private"+model_id = "Intel/Phi-3.5-vision-instruct-inc-private"model = AutoModelForCausalLM.from_pretrained(- model_id,- device_map="auto",- trust_remote_code=True,- torch_dtype="auto",- ##revision="13b4c3d" ##AutoGPTQ format+ model_id,+ device_map="auto",+ trust_remote_code=True,+ torch_dtype="auto",+ ##revision="13b4c3d" ##AutoGPTQ format)-processor = AutoProcessor.from_pretrained(model_id,- trust_remote_code=True,- num_crops=4-)+processor = AutoProcessor.from_pretrained(model_id, trust_remote_code=True, num_crops=4)image_url = "https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen-VL/assets/demo.jpeg"content = "Describe this image."messages = [- {"role": "user",- "content": "<|image_1|>\n"+content},+ {"role": "user", "content": "<|image_1|>\n" + content},]-prompt = processor.tokenizer.apply_chat_template(- messages,- tokenize=False,- add_generation_prompt=True-)+prompt = processor.tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)image_inputs = Image.open(requests.get(image_url, stream=True).raw)-inputs = processor(prompt, image_inputs, return_tensors="pt").to(model.device)+inputs = processor(prompt, image_inputs, return_tensors="pt").to(model.device)-generation_args = {- "max_new_tokens": 1000,- "temperature": 0.0,- "do_sample": False,-}+generation_args = {+ "max_new_tokens": 1000,+ "temperature": 0.0,+ "do_sample": False,+}-generate_ids = model.generate(**inputs,- eos_token_id=processor.tokenizer.eos_token_id,- **generation_args-)+generate_ids = model.generate(**inputs, eos_token_id=processor.tokenizer.eos_token_id, **generation_args)-# remove input tokens-generate_ids = generate_ids[:, inputs['input_ids'].shape[1]:]-response = processor.batch_decode(generate_ids,- skip_special_tokens=True,- clean_up_tokenization_spaces=False)[0]+# remove input tokens+generate_ids = generate_ids[:, inputs["input_ids"].shape[1] :]+response = processor.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]print(response)##INT4:diff --git a/docs/Qwen2.5-14B-Instruct-sym.md b/docs/Qwen2.5-14B-Instruct-sym.mdindex 6b6c3db99..a376b0c85 100644--- a/docs/Qwen2.5-14B-Instruct-sym.md+++ b/docs/Qwen2.5-14B-Instruct-sym.md@@ -9,14 +9,15 @@ This model is an int4 model with group_size 128 and symmetric quantization of [QCPU requires auto-round version>0.3.1```python-from auto_round import AutoRoundConfig ##must import for auto-round format-from transformers import AutoModelForCausalLM,AutoTokenizer+from auto_round import AutoRoundConfig ##must import for auto-round format+from transformers import AutoModelForCausalLM, AutoTokenizer+quantized_model_dir = "Intel/Qwen2.5-14B-Instruct-int4-inc"tokenizer = AutoTokenizer.from_pretrained(quantized_model_dir)model = AutoModelForCausalLM.from_pretrained(quantized_model_dir,- torch_dtype='auto',+ torch_dtype="auto",device_map="auto",##revision="f86a564" ##AutoGPTQ format)@@ -26,26 +27,17 @@ model = AutoModelForCausalLM.from_pretrained(##model = model.to(torch.bfloat16).to("hpu") ## uncommnet it for HPUprompt = "There is a girl who likes adventure,"-messages = [- {"role": "system", "content": "You are a helpful assistant."},- {"role": "user", "content": prompt}-]--text = tokenizer.apply_chat_template(- messages,- tokenize=False,- add_generation_prompt=True-)+messages = [{"role": "system", "content": "You are a helpful assistant."}, {"role": "user", "content": prompt}]++text = tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)model_inputs = tokenizer([text], return_tensors="pt").to(model.device)generated_ids = model.generate(model_inputs.input_ids,max_new_tokens=200, ##change this to align with the official usage- do_sample=False ##change this to align with the official usage+ do_sample=False, ##change this to align with the official usage)-generated_ids = [-output_ids[len(input_ids):] for input_ids, output_ids in zip(model_inputs.input_ids, generated_ids)-]+generated_ids = [output_ids[len(input_ids) :] for input_ids, output_ids in zip(model_inputs.input_ids, generated_ids)]response = tokenizer.batch_decode(generated_ids, skip_special_tokens=True)[0]print(response)@@ -76,8 +68,8 @@ The girl has a backpack that can carry up to 10 kilograms of rocks. She also hasGiven the following information:- Mountain: 5 unique rock types"""-prompt = "9.11和9.8哪个数字大"-#INT4:+prompt = "9.11和9.8哪个数字大"+# INT4:"""? 9.11 比 9.8 大。为了比较这两个数,我们可以从它们的小数部分开始:@@ -93,7 +85,7 @@ prompt = "9.11和9.8哪个数字大"(注意:在"""-##BF16:+##BF16:"""? 9.11 比 9.8 大。在比较两个小数时,我们从左到右逐位进行比较。首先比较整数部分,如果相同,则比较小数部分。对于9.11 和 9.8:@@ -117,7 +109,7 @@ prompt = "9.11和9.8哪个数字大"prompt = "Once upon a time,"-##INT4:+##INT4:""" there was a young man named John who had a passion for music. He loved playing the guitar and would spend hours every day practicing and perfecting his skills. However, he struggled to find an audience for his music and felt discouraged."""diff --git a/docs/Qwen2.5-32B-Instruct-sym.md b/docs/Qwen2.5-32B-Instruct-sym.mdindex 277b2ab29..ac4e78eb6 100644--- a/docs/Qwen2.5-32B-Instruct-sym.md+++ b/docs/Qwen2.5-32B-Instruct-sym.md@@ -9,14 +9,15 @@ This model is an int4 model with group_size 128 and symmetric quantization of [QCPU/ CUDA requires auto-round version>0.3.1```python-from auto_round import AutoRoundConfig ##must import for auto-round format-from transformers import AutoModelForCausalLM,AutoTokenizer+from auto_round import AutoRoundConfig ##must import for auto-round format+from transformers import AutoModelForCausalLM, AutoTokenizer+quantized_model_dir = "Intel/Qwen2.5-32B-Instruct-int4-inc"tokenizer = AutoTokenizer.from_pretrained(quantized_model_dir)model = AutoModelForCausalLM.from_pretrained(quantized_model_dir,- torch_dtype='auto',+ torch_dtype="auto",device_map="auto",)@@ -25,26 +26,17 @@ model = AutoModelForCausalLM.from_pretrained(##model = model.to(torch.bfloat16).to("hpu") ## uncommnet it for HPUprompt = "There is a girl who likes adventure,"-messages = [- {"role": "system", "content": "You are a helpful assistant."},- {"role": "user", "content": prompt}-]--text = tokenizer.apply_chat_template(- messages,- tokenize=False,- add_generation_prompt=True-)+messages = [{"role": "system", "content": "You are a helpful assistant."}, {"role": "user", "content": prompt}]++text = tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)model_inputs = tokenizer([text], return_tensors="pt").to(model.device)generated_ids = model.generate(model_inputs.input_ids,max_new_tokens=200, ##change this to align with the official usage- do_sample=False ##change this to align with the official usage+ do_sample=False, ##change this to align with the official usage)-generated_ids = [-output_ids[len(input_ids):] for input_ids, output_ids in zip(model_inputs.input_ids, generated_ids)-]+generated_ids = [output_ids[len(input_ids) :] for input_ids, output_ids in zip(model_inputs.input_ids, generated_ids)]response = tokenizer.batch_decode(generated_ids, skip_special_tokens=True)[0]print(response)@@ -60,11 +52,11 @@ Is there anything specific you had in mind for this character or story? I'd be h##BF16:"""That sounds exciting! What would you like to know or do regarding this girl who loves adventure? Perhaps you're looking for ideas on activities she might enjoy or ways to support her adventurous spirit. Let me know how I can assist you further!"""-prompt = "9.11和9.8哪个数字大"-#INT4:+prompt = "9.11和9.8哪个数字大"+# INT4:-##BF16:+##BF16:"""要比较两个数字的大小,我们可以从左到右逐位进行比较。首先,我们看整数部分:@@ -85,7 +77,7 @@ prompt = "9.11和9.8哪个数字大"prompt = "Once upon a time,"-##INT4:+##INT4:##BF16:diff --git a/docs/Qwen2.5-72B-Instruct-sym.md b/docs/Qwen2.5-72B-Instruct-sym.mdindex 30bca2040..d3ed45676 100644--- a/docs/Qwen2.5-72B-Instruct-sym.md+++ b/docs/Qwen2.5-72B-Instruct-sym.md@@ -8,14 +8,15 @@ This model is an int4 model with group_size 128 and and symmetric quantization oCPU requires auto-round version>0.3.1```python-from auto_round import AutoRoundConfig ##must import for auto-round format-from transformers import AutoModelForCausalLM,AutoTokenizer+from auto_round import AutoRoundConfig ##must import for auto-round format+from transformers import AutoModelForCausalLM, AutoTokenizer+quantized_model_dir = "Intel/Qwen2.5-72B-Instruct-int4-inc"tokenizer = AutoTokenizer.from_pretrained(quantized_model_dir)model = AutoModelForCausalLM.from_pretrained(quantized_model_dir,- torch_dtype='auto',+ torch_dtype="auto",device_map="auto",##revision="b7ea233" ##AutoGPTQ format)@@ -25,26 +26,17 @@ model = AutoModelForCausalLM.from_pretrained(##model = model.to(torch.bfloat16).to("hpu") ## uncommnet it for HPUprompt = "There is a girl who likes adventure,"-messages = [- {"role": "system", "content": "You are a helpful assistant."},- {"role": "user", "content": prompt}-]--text = tokenizer.apply_chat_template(- messages,- tokenize=False,- add_generation_prompt=True-)+messages = [{"role": "system", "content": "You are a helpful assistant."}, {"role": "user", "content": prompt}]++text = tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)model_inputs = tokenizer([text], return_tensors="pt").to(model.device)generated_ids = model.generate(model_inputs.input_ids,max_new_tokens=200, ##change this to align with the official usage- do_sample=False ##change this to align with the official usage+ do_sample=False, ##change this to align with the official usage)-generated_ids = [-output_ids[len(input_ids):] for input_ids, output_ids in zip(model_inputs.input_ids, generated_ids)-]+generated_ids = [output_ids[len(input_ids) :] for input_ids, output_ids in zip(model_inputs.input_ids, generated_ids)]response = tokenizer.batch_decode(generated_ids, skip_special_tokens=True)[0]print(response)@@ -58,8 +50,8 @@ prompt = "There is a girl who likes adventure,""""That sounds like a great quality to have! A love for adventure can lead to exciting experiences and personal growth. What kind of adventures does she enjoy? Does she like exploring new places, trying new activities, or seeking out thrilling experiences? Knowing more about her interests can help suggest specific ideas or activities that might appeal to her adventurous spirit."""-prompt = "9.11和9.8哪个数字大"-#INT4:+prompt = "9.11和9.8哪个数字大"+# INT4:"""要比较9.11和9.8的大小,可以按照以下步骤进行:1. **比较整数部分**:两个数字的整数部分都是9,所以需要进一步比较小数部分。@@ -78,7 +70,7 @@ prompt = "9.11和9.8哪个数字大"结论:9.8比9.11大。"""-##BF16:+##BF16:"""比较两个数字 9.11 和 9.8,可以按照以下步骤进行:1. **整数部分**:两个数字的整数部分都是 9,所以需要比较小数部分。@@ -93,7 +85,7 @@ prompt = "9.11和9.8哪个数字大"prompt = "Once upon a time,"-##INT4:+##INT4:"""Once upon a time, in a far-off land, there was a kingdom filled with wonder and magic. The kingdom was ruled by a wise and just king who loved his people dearly. In the heart of the kingdom stood a magnificent castle, surrounded by lush forests and rolling hills.The people of the kingdom lived happily, tending to their farms, crafting beautiful goods, and enjoying the simple pleasures of life. However, one day, a great darkness began to spread across the land. A wicked sorcerer had risen from the shadows, seeking to claim the throne for himself and plunge the kingdom into chaos.diff --git a/docs/Qwen2.5-7B-Instruct-sym.md b/docs/Qwen2.5-7B-Instruct-sym.mdindex f20e652a1..82fdf0c5b 100644--- a/docs/Qwen2.5-7B-Instruct-sym.md+++ b/docs/Qwen2.5-7B-Instruct-sym.md@@ -9,18 +9,18 @@ This model is an int4 model with group_size 128 and symmetric quantization of [QCPU requires auto-round version>0.3.1```python-from auto_round import AutoRoundConfig ##must import for auto-round format-from transformers import AutoModelForCausalLM,AutoTokenizer+from auto_round import AutoRoundConfig ##must import for auto-round format+from transformers import AutoModelForCausalLM, AutoTokenizer+quantized_model_dir = "Intel/Qwen2.5-7B-Instruct-int4-inc"tokenizer = AutoTokenizer.from_pretrained(quantized_model_dir)model = AutoModelForCausalLM.from_pretrained(quantized_model_dir,- torch_dtype='auto',+ torch_dtype="auto",device_map="auto",##revision="0b70f95" ##AutoGPTQ format##revision="5a6d912" ##Quantized lm-head version-)##import habana_frameworks.torch.core as htcore ## uncommnet it for HPU@@ -28,26 +28,17 @@ model = AutoModelForCausalLM.from_pretrained(##model = model.to(torch.bfloat16).to("hpu") ## uncommnet it for HPUprompt = "There is a girl who likes adventure,"-messages = [- {"role": "system", "content": "You are a helpful assistant."},- {"role": "user", "content": prompt}-]--text = tokenizer.apply_chat_template(- messages,- tokenize=False,- add_generation_prompt=True-)+messages = [{"role": "system", "content": "You are a helpful assistant."}, {"role": "user", "content": prompt}]++text = tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)model_inputs = tokenizer([text], return_tensors="pt").to(model.device)generated_ids = model.generate(model_inputs.input_ids,max_new_tokens=200, ##change this to align with the official usage- do_sample=False ##change this to align with the official usage+ do_sample=False, ##change this to align with the official usage)-generated_ids = [-output_ids[len(input_ids):] for input_ids, output_ids in zip(model_inputs.input_ids, generated_ids)-]+generated_ids = [output_ids[len(input_ids) :] for input_ids, output_ids in zip(model_inputs.input_ids, generated_ids)]response = tokenizer.batch_decode(generated_ids, skip_special_tokens=True)[0]print(response)@@ -61,8 +52,8 @@ prompt = "There is a girl who likes adventure,""""That sounds exciting! What kind of adventures does she enjoy? Is there something specific you'd like to plan or discuss related to her love for adventure?"""-prompt = "9.11和9.8哪个数字大"-#INT4:+prompt = "9.11和9.8哪个数字大"+# INT4:"""在比较9.11和9.8时,我们从左到右逐位进行比较:1. 首先比较整数部分:两个数的整数部分都是9,相等。@@ -75,7 +66,7 @@ prompt = "9.11和9.8哪个数字大"因此,9.8比9.11大。"""-##BF16:+##BF16:"""在比较9.11和9.8这两个数字时,我们可以直接进行比较:- 9.11 是九点一一@@ -87,7 +78,7 @@ prompt = "9.11和9.8哪个数字大"prompt = "Once upon a time,"-##INT4:+##INT4:"""Once upon a time, in a land filled with wonder and magic, there lived a young girl named Elara. She had bright eyes that sparkled like the stars on a clear night and hair as golden as the sun-kissed fields of wheat. Elara's home was a cozy cottage nestled at the edge of a vast, enchanted forest, where ancient trees whispered secrets to one another and mystical creatures roamed freely.Every day, Elara would venture into the forest, exploring its hidden paths and marveling at the wonders it held. One sunny morning, as she wandered deeper into the woods than ever before, she stumbled upon a glade bathed in a soft, ethereal light. In the center of this glade stood an enormous tree, its trunk wider than any building Elara had ever seen, and its branches stretching high into the sky.diff --git a/docs/Qwen3-8B-sym-recipe.md b/docs/Qwen3-8B-sym-recipe.mdindex 51ef6d659..74c96fa97 100644--- a/docs/Qwen3-8B-sym-recipe.md+++ b/docs/Qwen3-8B-sym-recipe.md@@ -10,28 +10,23 @@ This model is an int4 model with group_size 128 and symmetric quantization of [Q### INT4 Inference(CPU/CUDA/INTEL GPU)```python-from auto_round import AutoRoundConfig ##must import for auto-round format if transformers <= 4.51.3-from transformers import AutoModelForCausalLM,AutoTokenizer+from auto_round import AutoRoundConfig ##must import for auto-round format if transformers <= 4.51.3+from transformers import AutoModelForCausalLM, AutoTokenizer+quantized_model_dir = "Intel/Qwen3-8B-250424-int4-sym-AutoRound"# load the tokenizer and the modeltokenizer = AutoTokenizer.from_pretrained(quantized_model_dir)-model = AutoModelForCausalLM.from_pretrained(- quantized_model_dir,- torch_dtype="auto",- device_map="auto"-)+model = AutoModelForCausalLM.from_pretrained(quantized_model_dir, torch_dtype="auto", device_map="auto")# prepare the model inputprompt = "Give me a short introduction to large language model."-messages = [- {"role": "user", "content": prompt}-]+messages = [{"role": "user", "content": prompt}]text = tokenizer.apply_chat_template(messages,tokenize=False,add_generation_prompt=True,- enable_thinking=True # Switches between thinking and non-thinking modes. Default is True.+ enable_thinking=True, # Switches between thinking and non-thinking modes. Default is True.)model_inputs = tokenizer([text], return_tensors="pt").to(model.device)@@ -41,7 +36,7 @@ generated_ids = model.generate(max_new_tokens=512, ##change this to align with the official usagedo_sample=False ##change this to align with the official usage)-output_ids = generated_ids[0][len(model_inputs.input_ids[0]):].tolist()+output_ids = generated_ids[0][len(model_inputs.input_ids[0]) :].tolist()# parsing thinking contenttry:@@ -70,7 +65,7 @@ print("content:", content)prompt = "9.11和9.8哪个数字大"##INT4:-# thinking content:+# thinking content:# content: <think># 好的,我现在需要比较9.11和9.8哪个数字更大。首先,我应该回忆一下小数比较的方法。通常,比较小数的时候,我们会从左到右逐位比较,先看整数部分,如果整数部分相同,再比较小数部分。# 首先,这两个数的整数部分都是9,所以整数部分相同。接下来比较小数部分。9.11的小数部分是0.11,而9.8的小数部分是0.8。这时候我需要比较0.11和0.8的大小。@@ -83,7 +78,7 @@ prompt = "9.11和9.8哪个数字大"# 或者,有没有可能把9.8看成9.80,而9.11是9.11,所以##BF16:-# thinking content:+# thinking content:# content: <think># 嗯,用户问的是9.11和9.8哪个数字大。首先,我需要确认这两个数字的结构。9.11是一个小数,而9.8也是一个小数。看起来都是以9开头,但后面的小数部分不同。# 首先,我应该比较整数部分。两个数的整数部分都是9,所以这时候需要比较小数部分。小数部分的话,9.11的小数部分是0.11,而9.8的小数部分是0.8。这时候可能需要将它们转换成相同的位数来比较,或者直接比较小数点后的数字。@@ -96,7 +91,7 @@ prompt = "9.11和9.8哪个数字大"prompt = "How many r in word strawberry"##INT4:-# thinking content:+# thinking content:# content: <think># Okay, let's see. The user is asking how many times the letter 'r' appears in the word "strawberry". Hmm, first I need to make sure I spell the word correctly. Let me write it out: S-T-R-A-W-B-E-R-R-Y. Wait, is that right? Let me check again. S-T-R-A-W-B-E-R-R-Y. Yes, that's strawberry. Now, I need to count the number of 'r's in there.# Starting from the beginning: S is the first letter, then T, then R. So that's one 'r'. Then the next letters are A, W, B, E, R, R, Y. Wait, after the first R, there's another R later. Let me break it down letter by letter.@@ -138,7 +133,7 @@ prompt = "How many r in word strawberry"# Yes, three R's. So the answer is 3. I think that's correct. I don't see any other R's. Maybe the user is testing if I can count correctly, so I need to make sure. Let me check an online source or##BF16:-# thinking content:+# thinking content:# content: <think># Okay, let's see. The user is asking how many times the letter 'r' appears in the word "strawberry". Hmm, first I need to make sure I have the correct spelling of the word. Let me write it out: S-T-R-A-W-B-E-R-R-Y. Wait, is that right? Let me check again. Strawberry... yes, that's correct. Now, I need to count the number of 'r's in that spelling.@@ -183,8 +178,6 @@ prompt = "请简短介绍一下阿里巴巴公司"# 最后检查有没有遗漏的重要信息,比如是否提到支付宝和阿里云,这些是阿里巴巴的重要组成部分。同时,可能需要提到其对中国经济和全球电商的影响,但保持简洁。确保没有错误,比如成立年份、上市地点等。这样用户就能得到一个全面但简短的介绍。# </think># content: 阿里巴巴集团(Alibaba Group)成立于1999年,总部位于中国杭州,由马云创立。它是全球领先的互联网科技公司,核心业务涵盖电子商务(淘宝、天猫)、云计算(阿里云)、金融科技(支付宝)、物流(菜鸟网络)及创新业务(如盒马鲜生、阿里健康等)。阿里巴巴致力于通过数字化技术赋能企业与消费者,推动全球商业变革,旗下拥有--```### Evaluate the modeldiff --git a/docs/cogvlm2-llama3-chat-19B-sym.md b/docs/cogvlm2-llama3-chat-19B-sym.mdindex 257be35f0..61dc7d33d 100644--- a/docs/cogvlm2-llama3-chat-19B-sym.md+++ b/docs/cogvlm2-llama3-chat-19B-sym.md@@ -7,23 +7,19 @@ This model is an int4 model with group_size 128 and symmetric quantization of [T```pythonimport torchfrom PIL import Image-from auto_round import AutoRoundConfig ##must import for auto-round format+from auto_round import AutoRoundConfig ##must import for auto-round formatfrom transformers import AutoModelForCausalLM, AutoTokenizerimport requestsMODEL_PATH = "Intel/cogvlm2-llama3-chat-19B-inc-private"-DEVICE = 'cuda' if torch.cuda.is_available() else 'cpu'+DEVICE = "cuda" if torch.cuda.is_available() else "cpu"-tokenizer = AutoTokenizer.from_pretrained(- MODEL_PATH,- trust_remote_code=True+tokenizer = AutoTokenizer.from_pretrained(MODEL_PATH, trust_remote_code=True)+model = (+ AutoModelForCausalLM.from_pretrained(MODEL_PATH, torch_dtype="auto", trust_remote_code=True, device_map=DEVICE)+ .to(DEVICE)+ .eval())-model = AutoModelForCausalLM.from_pretrained(- MODEL_PATH,- torch_dtype="auto",- trust_remote_code=True,- device_map=DEVICE-).to(DEVICE).eval()image_url = "https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen-VL/assets/demo.jpeg"content = "Describe this image."@@ -32,29 +28,24 @@ text_only_template = "A chat between a curious user and an artificial intelligenquery = text_only_template.format(content)image = Image.open(requests.get(image_url, stream=True).raw)-input_by_model = model.build_conversation_input_ids(- tokenizer,- query=query,- images=[image],- template_version='chat'-)+input_by_model = model.build_conversation_input_ids(tokenizer, query=query, images=[image], template_version="chat")inputs = {- 'input_ids': input_by_model['input_ids'].unsqueeze(0).to(DEVICE),- 'token_type_ids': input_by_model['token_type_ids'].unsqueeze(0).to(DEVICE),- 'attention_mask': input_by_model['attention_mask'].unsqueeze(0).to(DEVICE),- 'images': [[input_by_model['images'][0].to(DEVICE).to(model.dtype)]] if image is not None else None,+ "input_ids": input_by_model["input_ids"].unsqueeze(0).to(DEVICE),+ "token_type_ids": input_by_model["token_type_ids"].unsqueeze(0).to(DEVICE),+ "attention_mask": input_by_model["attention_mask"].unsqueeze(0).to(DEVICE),+ "images": [[input_by_model["images"][0].to(DEVICE).to(model.dtype)]] if image is not None else None,}gen_kwargs = {"max_new_tokens": 2048,- "pad_token_id": 128002,+ "pad_token_id": 128002,}with torch.no_grad():outputs = model.generate(**inputs, **gen_kwargs)- outputs = outputs[:, inputs['input_ids'].shape[1]:]+ outputs = outputs[:, inputs["input_ids"].shape[1] :]response = tokenizer.decode(outputs[0])response = response.split("<|end_of_text|>")[0]- print(response)+ print(response)##INT4:## The image depicts a serene beach scene during what appears to be the golden hour, just before sunset. A woman is seated on the sandy shore, facing the vast expanse of the ocean. The waves are gently crashing on the beach, and the sky is painted with hues of orange and blue. The woman seems to be in a relaxed posture, possibly enjoying the tranquility of the moment. Beside her is a large, golden-colored dog, possibly a Labrador, wearing a colorful harness. The dog is sitting upright, looking at the woman, and they seem to share a bond of trust and affection. The overall mood of the image is peaceful, reflective, and heartwarming.diff --git a/docs/glm-4-9b-chat-recipe.md b/docs/glm-4-9b-chat-recipe.mdindex abadb9868..3a3a6fbb1 100644--- a/docs/glm-4-9b-chat-recipe.md+++ b/docs/glm-4-9b-chat-recipe.md@@ -7,41 +7,41 @@ This is an int4 recipe with group_size 128 of [THUDM/glm-4-9b-chat](https://hu### INT4 Inference on CPU/CUDA/HPU```python-from auto_round import AutoRoundConfig ## must import for auto-round format+from auto_round import AutoRoundConfig ## must import for auto-round formatfrom transformers import AutoModelForCausalLM, AutoTokenizerimport torchquantized_model_dir = "Intel/glm-4-9b-chat-int4-inc"backend = "auto" ##cuda, hpu, cpu(supported in auto_round>0.3.1),cuda:marlin(supported in auto_round>0.3.1 'pip install -v gptqmodel --no-build-isolation')-quantization_config = AutoRoundConfig(- backend=backend-)-model = AutoModelForCausalLM.from_pretrained(quantized_model_dir,- device_map=backend.split(':')[0],- torch_dtype=torch.float16,- quantization_config=quantization_config,- trust_remote_code=True- ).eval()+quantization_config = AutoRoundConfig(backend=backend)+model = AutoModelForCausalLM.from_pretrained(+ quantized_model_dir,+ device_map=backend.split(":")[0],+ torch_dtype=torch.float16,+ quantization_config=quantization_config,+ trust_remote_code=True,+).eval()tokenizer = AutoTokenizer.from_pretrained(quantized_model_dir, trust_remote_code=True)query = "请介绍一下智谱华章科技有限公司"-inputs = tokenizer.apply_chat_template([{"role": "user", "content": query}],- add_generation_prompt=True,- tokenize=True,- return_tensors="pt",- return_dict=True- )+inputs = tokenizer.apply_chat_template(+ [{"role": "user", "content": query}],+ add_generation_prompt=True,+ tokenize=True,+ return_tensors="pt",+ return_dict=True,+)inputs = inputs.to(model.device)-gen_kwargs = {"max_length": 50, "do_sample": False, "top_k": 1}##change this to follow official usage+gen_kwargs = {"max_length": 50, "do_sample": False, "top_k": 1} ##change this to follow official usagewith torch.no_grad():outputs = model.generate(**inputs, **gen_kwargs)- outputs = outputs[:, inputs['input_ids'].shape[1]:]+ outputs = outputs[:, inputs["input_ids"].shape[1] :]print(tokenizer.decode(outputs[0], skip_special_tokens=True))##请介绍一下智谱华章科技有限公司-#智谱华章科技有限公司是一家专注于人工智能、大数据、云计算等前沿技术领域的创新型企业。公司成立于2016年,总部位于中国北京,是一家集技术研发、产品开发、+# 智谱华章科技有限公司是一家专注于人工智能、大数据、云计算等前沿技术领域的创新型企业。公司成立于2016年,总部位于中国北京,是一家集技术研发、产品开发、##9.8大还是9.11大@@ -50,7 +50,7 @@ with torch.no_grad():##Once upon a time,-#In a land where the sun kissed the horizon with a golden glow and the stars whispered secrets to the night, there was a village nestled among rolling hills and whispering forests. This was a place+# In a land where the sun kissed the horizon with a golden glow and the stars whispered secrets to the night, there was a village nestled among rolling hills and whispering forests. This was a place##There is a girl who likes adventure,diff --git a/docs/llava-v1.5-7b-sym.md b/docs/llava-v1.5-7b-sym.mdindex b5c6058d3..065e460ee 100644--- a/docs/llava-v1.5-7b-sym.md+++ b/docs/llava-v1.5-7b-sym.md@@ -26,17 +26,20 @@ pip install -e .### INT4 Inference```python-from auto_round import AutoRoundConfig ## must import for auto-round format+from auto_round import AutoRoundConfig ## must import for auto-round formatimport requestsimport torchfrom PIL import Imagefrom llava.model.builder import load_pretrained_modelfrom llava.train.train import preprocess, preprocess_multimodal, DataCollatorForSupervisedDataset++class DataArgs:is_multimodal = Truemm_use_im_start_end = False-quantized_model_path="Intel/llava-v1.5-7b-inc-private"++quantized_model_path = "Intel/llava-v1.5-7b-inc-private"tokenizer, model, image_processor, _ = load_pretrained_model(quantized_model_path,@@ -50,12 +53,12 @@ image_url = "http://images.cocodataset.org/train2017/000000116003.jpg"messages = [{"from": "human", "value": "What is the tennis player doing in the image?\n<image>"}]# Preparation for inference-image = Image.open(requests.get(image_url, stream=True).raw).convert('RGB')-image_input = image_processor.preprocess(image, return_tensors='pt')['pixel_values'][0].to(model.device)+image = Image.open(requests.get(image_url, stream=True).raw).convert("RGB")+image_input = image_processor.preprocess(image, return_tensors="pt")["pixel_values"][0].to(model.device)input_data = preprocess_multimodal([messages], DataArgs())inputs = preprocess(input_data, tokenizer, has_image=(image_input is not None))-output = model.generate(inputs['input_ids'].to(model.device), images=image_input.unsqueeze(0).half(), max_new_tokens=50)+output = model.generate(inputs["input_ids"].to(model.device), images=image_input.unsqueeze(0).half(), max_new_tokens=50)print(tokenizer.batch_decode(output))##INT4: The tennis player is celebrating a victory, raising his arms in the air, and holding his tennis racket.@@ -76,7 +79,6 @@ messages = [{"from": "human", "value": "How many people and animals are there in##INT4: There are two people and one animal in the image.##BF16: There are two people and one animal in the image.-```## Evaluation the modeldiff --git a/docs/step_by_step.md b/docs/step_by_step.mdindex 12081356a..8fe57336c 100644--- a/docs/step_by_step.md+++ b/docs/step_by_step.md@@ -136,7 +136,7 @@ autoround = AutoRound(output_dir = "./tmp_autoround"# format= 'auto_round'(default), 'auto_gptq', 'auto_awq'-autoround.quantize_and_save(output_dir, format='auto_gptq,auto_awq,auto_round')+autoround.quantize_and_save(output_dir, format="auto_gptq,auto_awq,auto_round")```#### Mixed bits Usage@@ -152,10 +152,10 @@ model_name = "facebook/opt-125m"model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto")tokenizer = AutoTokenizer.from_pretrained(model_name)bits, group_size, sym = 4, 128, True-layer_config = {# Supports both full layer names and fuzzy (partial) matching- "model.decoder.layers.6.self_attn.out_proj": {"bits": 8, "group_size": 32},- "model.decoder.layers.*k_proj": {"bits": 2, "group_size": 32}- }+layer_config = { # Supports both full layer names and fuzzy (partial) matching+ "model.decoder.layers.6.self_attn.out_proj": {"bits": 8, "group_size": 32},+ "model.decoder.layers.*k_proj": {"bits": 2, "group_size": 32},+}autoround = AutoRound(model,tokenizer,@@ -166,7 +166,7 @@ autoround = AutoRound()output_dir = "./tmp_autoround"-autoround.quantize_and_save(output_dir, format='auto_round')+autoround.quantize_and_save(output_dir, format="auto_round")```#### AutoRoundBest recipe@@ -180,18 +180,11 @@ model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto")tokenizer = AutoTokenizer.from_pretrained(model_name)bits, group_size, sym = 4, 128, Trueautoround = AutoRound(- model,- tokenizer,- bits=bits,- group_size=group_size,- sym=sym,- nsamples=512,- iters=1000,- low_gpu_mem_usage=True+ model, tokenizer, bits=bits, group_size=group_size, sym=sym, nsamples=512, iters=1000, low_gpu_mem_usage=True)output_dir = "./tmp_autoround"-autoround.quantize_and_save(output_dir, format='auto_round')+autoround.quantize_and_save(output_dir, format="auto_round")```#### AutoRoundLight recipeThis setting offers the best speed (2 - 3X faster than AutoRound), but it may cause a significant accuracy drop for small models and 2-bit quantization. It is recommended for 4-bit settings and models larger than 3B.@@ -215,7 +208,7 @@ autoround = AutoRound()output_dir = "./tmp_autoround"-autoround.quantize_and_save(output_dir, format='auto_round')+autoround.quantize_and_save(output_dir, format="auto_round")```### RTN mode@@ -239,7 +232,7 @@ autoround = AutoRound()output_dir = "./tmp_autoround"-autoround.quantize_and_save(output_dir, format='auto_round')+autoround.quantize_and_save(output_dir, format="auto_round")```### GGUF format@@ -264,7 +257,7 @@ autoround = AutoRound()output_dir = "./tmp_autoround"-autoround.quantize_and_save(output_dir, format='gguf:q4_0') # gguf:q4_1, gguf:q*_k_s+autoround.quantize_and_save(output_dir, format="gguf:q4_0") # gguf:q4_1, gguf:q*_k_s```### Device/Multi-GPU setting in Quantization@@ -446,12 +439,12 @@ in [Gaudi Guide](https://docs.habana.ai/en/latest/).```pythonimport habana_frameworks.torch.core as htcoreimport habana_frameworks.torch.hpu as hthpu-from transformers import AutoModelForCausalLM,AutoTokenizer+from transformers import AutoModelForCausalLM, AutoTokenizerimport torchmodel_name = "Intel/Qwen2-7B-int4-inc"tokenizer = AutoTokenizer.from_pretrained(model_name)-model = AutoModelForCausalLM.from_pretrained(model_name).to('hpu').to(torch.bfloat16)+model = AutoModelForCausalLM.from_pretrained(model_name).to("hpu").to(torch.bfloat16)text = "There is a girl who likes adventure,"inputs = tokenizer(text, return_tensors="pt").to(model.device)print(tokenizer.decode(model.generate(**inputs, max_new_tokens=50, do_sample=False)[0]))@@ -471,7 +464,9 @@ from auto_round import AutoRoundConfigmodel_name = "OPEA/Qwen2.5-1.5B-Instruct-int4-sym-inc"quantization_config = AutoRoundConfig(backend="ipex")-model = AutoModelForCausalLM.from_pretrained(model_name, device_map="cpu", quantization_config=quantization_config, torch_dtype="auto")+model = AutoModelForCausalLM.from_pretrained(+ model_name, device_map="cpu", quantization_config=quantization_config, torch_dtype="auto"+)tokenizer = AutoTokenizer.from_pretrained(model_name)text = "There is a girl who likes adventure,"inputs = tokenizer(text, return_tensors="pt").to(model.device)@@ -501,7 +496,9 @@ from auto_round import AutoRoundConfigmodel_name = "ybelkada/opt-125m-gptq-4bit"quantization_config = AutoRoundConfig()-model = AutoModelForCausalLM.from_pretrained(model_name, device_map="cpu", quantization_config=quantization_config, torch_dtype="auto")+model = AutoModelForCausalLM.from_pretrained(+ model_name, device_map="cpu", quantization_config=quantization_config, torch_dtype="auto"+)tokenizer = AutoTokenizer.from_pretrained(model_name)text = "There is a girl who likes adventure,"inputs = tokenizer(text, return_tensors="pt").to(model.device)diff --git a/pyproject.toml b/pyproject.tomlindex 5fe81084a..c44a1b42e 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -4,9 +4,12 @@ ignore-words = ".azure-pipelines/scripts/codeScan/codespell/autoround_dict.txt"[tool.isort]profile = "black"line_length = 120-known_first_party = ["auto_round"]+known_first_party = ["auto_round", "auto_round_extension"]extend_skip_glob = ["**/__init__.py"]+[tool.black]+line-length = 120+[tool.ruff]# Exclude a variety of commonly ignored directories.exclude = [diff --git a/setup.py b/setup.pyindex 1d21ef8d5..840e611d4 100644--- a/setup.py+++ b/setup.py@@ -39,6 +39,7 @@ def is_habana_framework_installed():def is_hpu_available():try:import habana_frameworks.torch.core as htcore # pylint: disable=E0401+return Trueexcept ImportError:return False@@ -54,12 +55,14 @@ def is_cpu_env():import torchexcept Exception as e:print(- f"Building extension requires PyTorch being installed, please install PyTorch first: {e}.\n NOTE: This issue may be raised due to pip build isolation system (ignoring local packages). Please use `--no-build-isolation` when installing with pip, and refer to https://github.com/intel/auto-round for more details.")+ f"Building extension requires PyTorch being installed, please install PyTorch first: {e}.\n NOTE: This issue may be raised due to pip build isolation system (ignoring local packages). Please use `--no-build-isolation` when installing with pip, and refer to https://github.com/intel/auto-round for more details."+ )sys.exit(1)if torch.cuda.is_available():return Falsetry:import habana_frameworks.torch.core as htcore+return Falseexcept:return Truediff --git a/test/test_cpu/_test_helpers.py b/test/test_cpu/_test_helpers.pyindex 7d848c728..48e8398d7 100644--- a/test/test_cpu/_test_helpers.py+++ b/test/test_cpu/_test_helpers.py@@ -19,14 +19,8 @@ def model_infer(model, tokenizer, apply_chat_template=False):if apply_chat_template:texts = []for prompt in prompts:- messages = [- {"role": "user", "content": prompt}- ]- text = tokenizer.apply_chat_template(- messages,- tokenize=False,- add_generation_prompt=True- )+ messages = [{"role": "user", "content": prompt}]+ text = tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)texts.append(text)prompts = texts@@ -36,11 +30,9 @@ def model_infer(model, tokenizer, apply_chat_template=False):input_ids=inputs["input_ids"].to(model.device),attention_mask=inputs["attention_mask"].to(model.device),do_sample=False, ## change this to follow official usage- max_new_tokens=5+ max_new_tokens=5,)- generated_ids = [- output_ids[len(input_ids):] for input_ids, output_ids in zip(inputs["input_ids"], outputs)- ]+ generated_ids = [output_ids[len(input_ids) :] for input_ids, output_ids in zip(inputs["input_ids"], outputs)]decoded_outputs = tokenizer.batch_decode(generated_ids, skip_special_tokens=True)diff --git a/test/test_cpu/test_act_quantization.py b/test/test_cpu/test_act_quantization.pyindex f65c6d690..2ff5c8501 100644--- a/test/test_cpu/test_act_quantization.py+++ b/test/test_cpu/test_act_quantization.py@@ -47,7 +47,7 @@ class TestAutoRoundAct(unittest.TestCase):seqlen=2,dataset=self.llm_dataloader,act_bits=4,- data_type="mx_fp"+ data_type="mx_fp",)autoround.quantize()@@ -84,9 +84,10 @@ class TestAutoRoundAct(unittest.TestCase):act_bits=8,data_type="fp8_to_int_sym",act_dynamic=False,- act_data_type="fp8"+ act_data_type="fp8",)autoround.quantize()+if __name__ == "__main__":unittest.main()diff --git a/test/test_cpu/test_autoopt.py b/test/test_cpu/test_autoopt.pyindex 536906e8f..fc6446dea 100644--- a/test/test_cpu/test_autoopt.py+++ b/test/test_cpu/test_autoopt.py@@ -44,14 +44,14 @@ class TestAutoRound(unittest.TestCase):iters=2,seqlen=10,dataset=self.llm_dataloader,- to_quant_block_names=None+ to_quant_block_names=None,)autoround.quantize()--+def test_Adam(self):bits, group_size, sym = 4, 128, Falsefrom auto_round.utils import get_block_names+llm_block_names = get_block_names(self.model, quant_vision=True)bits, group_size, sym, batch_size = 4, 128, False, 20adamround = AutoRoundAdam(@@ -64,12 +64,10 @@ class TestAutoRound(unittest.TestCase):seqlen=2,batch_size=batch_size,dataset=self.llm_dataloader,- to_quant_block_names=llm_block_names+ to_quant_block_names=llm_block_names,)adamround.quantize()if __name__ == "__main__":unittest.main()--diff --git a/test/test_cpu/test_autoround.py b/test/test_cpu/test_autoround.pyindex fc577e319..94097d01b 100644--- a/test/test_cpu/test_autoround.py+++ b/test/test_cpu/test_autoround.py@@ -37,13 +37,14 @@ class TestAutoRound(unittest.TestCase):shutil.rmtree("runs", ignore_errors=True)def test_remove_whole_block(self):- layer_config = {"model.decoder.layers.0.self_attn.k_proj": {"bits": 32},- "model.decoder.layers.0.self_attn.v_proj": {"bits": 32},- "model.decoder.layers.0.self_attn.q_proj": {"bits": 32},- "model.decoder.layers.0.self_attn.out_proj": {"bits": 32},- "model.decoder.layers.0.fc1": {"bits": 32},- "model.decoder.layers.0.fc2": {"bits": 32},- }+ layer_config = {+ "model.decoder.layers.0.self_attn.k_proj": {"bits": 32},+ "model.decoder.layers.0.self_attn.v_proj": {"bits": 32},+ "model.decoder.layers.0.self_attn.q_proj": {"bits": 32},+ "model.decoder.layers.0.self_attn.out_proj": {"bits": 32},+ "model.decoder.layers.0.fc1": {"bits": 32},+ "model.decoder.layers.0.fc2": {"bits": 32},+ }bits, group_size, sym = 4, 128, Falseautoround = AutoRound(self.model,@@ -54,7 +55,7 @@ class TestAutoRound(unittest.TestCase):iters=2,seqlen=2,dataset=self.llm_dataloader,- layer_config=layer_config+ layer_config=layer_config,)autoround.quantize()@@ -97,7 +98,7 @@ class TestAutoRound(unittest.TestCase):iters=2,seqlen=2,dataset=self.llm_dataloader,- data_type="mx_fp4"+ data_type="mx_fp4",)autoround.quantize()@@ -112,7 +113,8 @@ class TestAutoRound(unittest.TestCase):batch_size=3,iters=2,dataset=self.llm_dataloader,- gradient_accumulate_steps=4)+ gradient_accumulate_steps=4,+ )autoround.quantize()def test_default(self):@@ -224,7 +226,6 @@ class TestAutoRound(unittest.TestCase):)autoround.quantize()-def test_enable_norm_bias_tuning_qwen3(self):bits, group_size, sym = 4, 128, Truemodel_name = "Qwen/Qwen3-0.6B"@@ -311,8 +312,9 @@ class TestAutoRound(unittest.TestCase):def test_auto_device_map(self):bits, group_size, sym = 4, 128, Falsemodel_name = "facebook/opt-125m"- model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True,- device_map='auto')+ model = AutoModelForCausalLM.from_pretrained(+ model_name, torch_dtype="auto", trust_remote_code=True, device_map="auto"+ )autoround = AutoRound(model,self.tokenizer,@@ -328,8 +330,9 @@ class TestAutoRound(unittest.TestCase):def test_fp32(self):bits, group_size, sym = 4, 128, Falsemodel_name = "facebook/opt-125m"- model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype=torch.float32, trust_remote_code=True,- device_map='auto')+ model = AutoModelForCausalLM.from_pretrained(+ model_name, torch_dtype=torch.float32, trust_remote_code=True, device_map="auto"+ )autoround = AutoRound(model,self.tokenizer,@@ -339,7 +342,7 @@ class TestAutoRound(unittest.TestCase):iters=2,seqlen=2,dataset=self.llm_dataloader,- amp=False+ amp=False,)autoround.quantize()@@ -364,16 +367,7 @@ class TestAutoRound(unittest.TestCase):tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)bits, group_size, sym = 4, 128, True- autoround = AutoRound(- model,- tokenizer,- bits=bits,- group_size=group_size,- sym=sym,- iters=1,- nsamples=1-- )+ autoround = AutoRound(model, tokenizer, bits=bits, group_size=group_size, sym=sym, iters=1, nsamples=1)quantized_model_path = self.save_folderautoround.quantize_and_save(output_dir=quantized_model_path, format="auto_round")model = AutoModelForCausalLM.from_pretrained(@@ -389,10 +383,13 @@ class TestAutoRound(unittest.TestCase):def test_fallback_layers(self):bits, group_size, sym = 4, 128, Truemodel_name = "facebook/opt-125m"- model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype=torch.float32, trust_remote_code=True,- device_map='auto')- layer_config = {"model.decoder.layers.0.self_attn.q_proj": {"bits": 16},- "model.decoder.layers.1.self_attn.k_proj": {"bits": 16}}+ model = AutoModelForCausalLM.from_pretrained(+ model_name, torch_dtype=torch.float32, trust_remote_code=True, device_map="auto"+ )+ layer_config = {+ "model.decoder.layers.0.self_attn.q_proj": {"bits": 16},+ "model.decoder.layers.1.self_attn.k_proj": {"bits": 16},+ }autoround = AutoRound(model,self.tokenizer,@@ -402,18 +399,17 @@ class TestAutoRound(unittest.TestCase):iters=2,seqlen=2,dataset=self.llm_dataloader,- layer_config=layer_config+ layer_config=layer_config,)autoround.quantize()quantized_model_path = self.save_folderautoround.save_quantized(output_dir=quantized_model_path, format="auto_round", inplace=True)- quantization_config = AutoRoundConfig(- backend="ipex"- )+ quantization_config = AutoRoundConfig(backend="ipex")- model = AutoModelForCausalLM.from_pretrained(quantized_model_path,- device_map='cpu', quantization_config=quantization_config)+ model = AutoModelForCausalLM.from_pretrained(+ quantized_model_path, device_map="cpu", quantization_config=quantization_config+ )tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)text = "There is a girl who likes adventure,"inputs = tokenizer(text, return_tensors="pt").to(model.device)@@ -426,15 +422,17 @@ class TestAutoRound(unittest.TestCase):from transformers import AutoProcessor, Qwen2VLForConditionalGenerationfrom auto_round_extension.ipex.qlinear_ipex_awq import QuantLinear+model_name = "Qwen/Qwen2-VL-2B-Instruct-AWQ"quantization_config = AutoRoundConfig()model = Qwen2VLForConditionalGeneration.from_pretrained(- model_name, quantization_config=quantization_config, device_map="cpu", torch_dtype=torch.float16)+ model_name, quantization_config=quantization_config, device_map="cpu", torch_dtype=torch.float16+ )self.assertTrue(isinstance(model.visual.blocks[0].attn.qkv, torch.nn.Linear))self.assertFalse(isinstance(model.visual.merger.mlp[0], QuantLinear))if hasattr(model.model, "language_model"):self.assertTrue(isinstance(model.model.language_model.layers[0].self_attn.v_proj, QuantLinear))- else:+ else:self.assertTrue(isinstance(model.model.layers[0].self_attn.v_proj, QuantLinear))processor = AutoProcessor.from_pretrained(model_name, size=None)@@ -453,9 +451,7 @@ class TestAutoRound(unittest.TestCase):]# Preparation for inference- text = processor.apply_chat_template(- messages, tokenize=False, add_generation_prompt=True- )+ text = processor.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)image_inputs = Image.open(requests.get(image_url, stream=True).raw)inputs = processor(text=[text],@@ -466,9 +462,7 @@ class TestAutoRound(unittest.TestCase):# Inference: Generation of the outputgenerated_ids = model.generate(**inputs, max_new_tokens=1)- generated_ids_trimmed = [- out_ids[len(in_ids):] for in_ids, out_ids in zip(inputs.input_ids, generated_ids)- ]+ generated_ids_trimmed = [out_ids[len(in_ids) :] for in_ids, out_ids in zip(inputs.input_ids, generated_ids)]output_text = processor.batch_decode(generated_ids_trimmed, skip_special_tokens=True, clean_up_tokenization_spaces=False)@@ -479,8 +473,10 @@ class TestAutoRound(unittest.TestCase):bits, group_size, sym = 4, 128, Truemodel = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True)tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)- layer_config = {"model\.decoder\.layers\.(?:[0-9]|1[0-1])\.self_attn\.q_proj": {"bits": 16},- "model.decoder.layers.1.self_attn.k_proj": {"bits": 16}}+ layer_config = {+ "model\.decoder\.layers\.(?:[0-9]|1[0-1])\.self_attn\.q_proj": {"bits": 16},+ "model.decoder.layers.1.self_attn.k_proj": {"bits": 16},+ }autoround = AutoRound(model,tokenizer=tokenizer,@@ -490,7 +486,7 @@ class TestAutoRound(unittest.TestCase):iters=2,seqlen=2,dataset=self.llm_dataloader,- layer_config=layer_config+ layer_config=layer_config,)autoround.quantize()quantized_model_path = self.save_folder@@ -498,8 +494,9 @@ class TestAutoRound(unittest.TestCase):autoround.save_quantized(output_dir=quantized_model_path, format="auto_awq", inplace=True)quantization_config = AutoRoundConfig()- model = AutoModelForCausalLM.from_pretrained(quantized_model_path,- device_map='auto', quantization_config=quantization_config)+ model = AutoModelForCausalLM.from_pretrained(+ quantized_model_path, device_map="auto", quantization_config=quantization_config+ )tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)text = "There is a girl who likes adventure,"inputs = tokenizer(text, return_tensors="pt").to(model.device)@@ -512,9 +509,10 @@ class TestAutoRound(unittest.TestCase):bits, group_size, sym = 4, 128, Truemodel = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True)tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)- layer_config = {"model\.decoder\.layers\.(?:[0-9]|1[0-1])\.self_attn\.q_proj": {"bits": 16},- ##"model.decoder.layers.1.self_attn.k_proj": {"bits": 16}- }+ layer_config = {+ "model\.decoder\.layers\.(?:[0-9]|1[0-1])\.self_attn\.q_proj": {"bits": 16},+ ##"model.decoder.layers.1.self_attn.k_proj": {"bits": 16}+ }autoround = AutoRound(model,tokenizer=tokenizer,@@ -524,7 +522,7 @@ class TestAutoRound(unittest.TestCase):iters=2,seqlen=2,dataset=self.llm_dataloader,- layer_config=layer_config+ layer_config=layer_config,)autoround.quantize()quantized_model_path = self.save_folder@@ -532,8 +530,9 @@ class TestAutoRound(unittest.TestCase):autoround.save_quantized(output_dir=quantized_model_path, format="auto_gptq", inplace=True)quantization_config = AutoRoundConfig()- model = AutoModelForCausalLM.from_pretrained(quantized_model_path,- device_map='auto', quantization_config=quantization_config)+ model = AutoModelForCausalLM.from_pretrained(+ quantized_model_path, device_map="auto", quantization_config=quantization_config+ )tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)text = "There is a girl who likes adventure,"inputs = tokenizer(text, return_tensors="pt").to(model.device)@@ -546,9 +545,10 @@ class TestAutoRound(unittest.TestCase):bits, group_size, sym = 4, 128, Truemodel = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True)tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)- layer_config = {"model\.decoder\.layers\.(?:[0-9]|1[0-1])\.self_attn\.q_proj": {"bits": 16},- "model.decoder.layers.1.self_attn.k_proj": {"bits": 16}- }+ layer_config = {+ "model\.decoder\.layers\.(?:[0-9]|1[0-1])\.self_attn\.q_proj": {"bits": 16},+ "model.decoder.layers.1.self_attn.k_proj": {"bits": 16},+ }autoround = AutoRound(model,tokenizer=tokenizer,@@ -558,7 +558,7 @@ class TestAutoRound(unittest.TestCase):iters=2,seqlen=2,dataset=self.llm_dataloader,- layer_config=layer_config+ layer_config=layer_config,)autoround.quantize()quantized_model_path = self.save_folder@@ -566,8 +566,9 @@ class TestAutoRound(unittest.TestCase):autoround.save_quantized(output_dir=quantized_model_path, format="auto_round", inplace=True)quantization_config = AutoRoundConfig()- model = AutoModelForCausalLM.from_pretrained(quantized_model_path,- device_map='auto', quantization_config=quantization_config)+ model = AutoModelForCausalLM.from_pretrained(+ quantized_model_path, device_map="auto", quantization_config=quantization_config+ )tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)text = "There is a girl who likes adventure,"inputs = tokenizer(text, return_tensors="pt").to(model.device)@@ -580,9 +581,7 @@ class TestAutoRound(unittest.TestCase):bits, group_size, sym = 4, 128, Truemodel = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True)tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)- layer_config = {- "model.decoder.layers.12.self_attn.k_proj": {"bits": 16}- }+ layer_config = {"model.decoder.layers.12.self_attn.k_proj": {"bits": 16}}with self.assertRaises(ValueError):autoround = AutoRound(model,@@ -593,7 +592,7 @@ class TestAutoRound(unittest.TestCase):iters=2,seqlen=2,dataset=self.llm_dataloader,- layer_config=layer_config+ layer_config=layer_config,)autoround.quantize()diff --git a/test/test_cpu/test_autoround_acc.py b/test/test_cpu/test_autoround_acc.pyindex 656064907..1a3ca5018 100644--- a/test/test_cpu/test_autoround_acc.py+++ b/test/test_cpu/test_autoround_acc.py@@ -1,4 +1,3 @@-import copyimport shutilimport sys@@ -22,13 +21,15 @@ class LLMDataLoader:def __iter__(self):for i in range(2):yield torch.ones([1, 10], dtype=torch.long)-+class TestAutoRound(unittest.TestCase):@classmethoddef setUpClass(self):self.model_name = "hf-internal-testing/tiny-random-GPTJForCausalLM"- self.model = AutoModelForCausalLM.from_pretrained(self.model_name, torch_dtype=torch.float32, trust_remote_code=True)+ self.model = AutoModelForCausalLM.from_pretrained(+ self.model_name, torch_dtype=torch.float32, trust_remote_code=True+ )self.tokenizer = AutoTokenizer.from_pretrained(self.model_name, trust_remote_code=True)self.llm_dataloader = LLMDataLoader()@@ -49,13 +50,15 @@ class TestAutoRound(unittest.TestCase):sym=sym,iters=2,seqlen=10,- dataset=self.llm_dataloader+ dataset=self.llm_dataloader,)autoround.quantize()out0 = self.model(inp)print(f"out0 = {float(out0[0][0][0][0])}")-- model_tmp = AutoModelForCausalLM.from_pretrained(self.model_name, torch_dtype=torch.float32, trust_remote_code=True)++ model_tmp = AutoModelForCausalLM.from_pretrained(+ self.model_name, torch_dtype=torch.float32, trust_remote_code=True+ )autoround_1 = AutoRound(model_tmp,self.tokenizer,@@ -65,15 +68,14 @@ class TestAutoRound(unittest.TestCase):device="cpu",iters=2,seqlen=10,- dataset=self.llm_dataloader+ dataset=self.llm_dataloader,)autoround_1.quantize()out1 = model_tmp(inp)-+assert out0[0].equal(out1[0])self.assertTrue(isclose(float(out0[0][0][0][0]), -0.021002087742090225, rel_tol=5e-04))if __name__ == "__main__":unittest.main()-diff --git a/test/test_cpu/test_autoround_export_to_itrex.py b/test/test_cpu/test_autoround_export_to_itrex.pyindex 7e894b633..90415f240 100644--- a/test/test_cpu/test_autoround_export_to_itrex.py+++ b/test/test_cpu/test_autoround_export_to_itrex.py@@ -57,7 +57,7 @@ class TestAutoroundExport(unittest.TestCase):out1 = model(self.lm_input)round = AutoRoundoptq_1 = round(model, self.tokenizer, nsamples=20, amp=False, seqlen=10, iters=10, enable_torch_compile=False)- q_model, layer_config1 = optq_1.quantize() ##compile model+ q_model, layer_config1 = optq_1.quantize() ##compile modelfrom auto_round.export.export_to_itrex import pack_modelcompressed_model = pack_model(model=q_model, layer_config=layer_config1)@@ -115,4 +115,3 @@ class TestAutoroundExport(unittest.TestCase):if __name__ == "__main__":unittest.main()-diff --git a/test/test_cpu/test_basic_usage.py b/test/test_cpu/test_basic_usage.pyindex eeedbcd65..7de1204c2 100644--- a/test/test_cpu/test_basic_usage.py+++ b/test/test_cpu/test_basic_usage.py@@ -3,7 +3,7 @@ import shutilimport sysimport unittest-sys.path.insert(0, '../..')+sys.path.insert(0, "../..")class TestAutoRoundCmd(unittest.TestCase):@@ -15,7 +15,7 @@ class TestAutoRoundCmd(unittest.TestCase):def tearDownClass(self):shutil.rmtree("./saved", ignore_errors=True)shutil.rmtree("runs", ignore_errors=True)-+def test_auto_round_cmd(self):python_path = sys.executable@@ -26,10 +26,11 @@ class TestAutoRoundCmd(unittest.TestCase):# assert False, "cmd line test fail, please have a check"#res = os.system(- f"cd ../.. && {python_path} -m auto_round --model 'facebook/opt-125m' --seqlen 32 --iter 2 --nsamples 1 --format auto_gptq,auto_round --output_dir ./saved --tasks piqa")+ f"cd ../.. && {python_path} -m auto_round --model 'facebook/opt-125m' --seqlen 32 --iter 2 --nsamples 1 --format auto_gptq,auto_round --output_dir ./saved --tasks piqa"+ )if res > 0 or res == -1:assert False, "cmd line test fail, please have a check"-+res = os.system(f"cd ../.. && {python_path} -m auto_round --model 'facebook/opt-125m' --seqlen 8 --iter 1 --nsamples 1 --eval_task_by_task --tasks openbookqa --bs 32")@@ -37,37 +38,37 @@ class TestAutoRoundCmd(unittest.TestCase):assert False, "cmd line test fail, please have a check"res = os.system(- f"cd ../.. && {python_path} -c 'from auto_round.__main__ import run_light; run_light()' --seqlen 8 --iter 2 --nsamples 8 --output_dir ./saved --tasks lambada_openai")+ f"cd ../.. && {python_path} -c 'from auto_round.__main__ import run_light; run_light()' --seqlen 8 --iter 2 --nsamples 8 --output_dir ./saved --tasks lambada_openai"+ )if res > 0 or res == -1:assert False, "cmd line test fail, please have a check"# test mllm script# test auto_round_mllm help- res = os.system(- f"cd ../.. && {python_path} -m auto_round --mllm -h")+ res = os.system(f"cd ../.. && {python_path} -m auto_round --mllm -h")if res > 0 or res == -1:assert False, "cmd line test fail, please have a check"# test auto_round_mllm --eval help- res = os.system(- f"cd ../.. && {python_path} -m auto_round --mllm --eval -h")+ res = os.system(f"cd ../.. && {python_path} -m auto_round --mllm --eval -h")if res > 0 or res == -1:assert False, "cmd line test fail, please have a check"# test auto_round_mllm --lmms help- res = os.system(- f"cd ../.. && {python_path} -m auto_round --mllm --lmms -h")+ res = os.system(f"cd ../.. && {python_path} -m auto_round --mllm --lmms -h")if res > 0 or res == -1:assert False, "cmd line test fail, please have a check"res = os.system(- f"cd ../.. && {python_path} -m auto_round --mllm --iter 2 --nsamples 10 --seqlen 32 --format auto_round --output_dir ./saved")+ f"cd ../.. && {python_path} -m auto_round --mllm --iter 2 --nsamples 10 --seqlen 32 --format auto_round --output_dir ./saved"+ )if res > 0 or res == -1:assert False, "cmd line test fail, please have a check"res = os.system(f"cd ../.. && {python_path} -m auto_round --mllm --iter 2 --nsamples 10 --seqlen 256 --format auto_round"- " --quant_nontext_module --output_dir ./saved ")+ " --quant_nontext_module --output_dir ./saved "+ )if res > 0 or res == -1:assert False, "cmd line test fail, please have a check"diff --git a/test/test_cpu/test_block_names.py b/test/test_cpu/test_block_names.pyindex f7555a944..403382580 100644--- a/test/test_cpu/test_block_names.py+++ b/test/test_cpu/test_block_names.py@@ -16,10 +16,10 @@ class LLMDataLoader:def __init__(self, input_size=10):self.batch_size = 1self.input_size = input_size+def __iter__(self):for i in range(2):yield torch.ones([1, self.input_size], dtype=torch.long)-# ================= simple multimodal model =================@@ -27,39 +27,47 @@ class TextEncoder(nn.Module):def __init__(self, input_size, hidden_size):super(TextEncoder, self).__init__()self.fc = nn.Linear(input_size, hidden_size)-+def forward(self, x):return self.fc(x)+class VisionEncoder(nn.Module):def __init__(self, input_size, hidden_size):super(VisionEncoder, self).__init__()self.fc = nn.Linear(input_size, hidden_size)-+def forward(self, x):return self.fc(x)+class TextEncoderModuleList(nn.ModuleList):pass+class VisionEncoderModuleList(nn.ModuleList):pass+class SimpleMultimodalModel(nn.Module):def __init__(self, text_input_size, image_input_size, hidden_size, num_text_encoders, num_image_encoders):super(SimpleMultimodalModel, self).__init__()- self.text_encoders = TextEncoderModuleList([TextEncoder(text_input_size, hidden_size) for _ in range(num_text_encoders)])- self.vision_encoders = VisionEncoderModuleList([VisionEncoder(image_input_size, hidden_size) for _ in range(num_image_encoders)])+ self.text_encoders = TextEncoderModuleList(+ [TextEncoder(text_input_size, hidden_size) for _ in range(num_text_encoders)]+ )+ self.vision_encoders = VisionEncoderModuleList(+ [VisionEncoder(image_input_size, hidden_size) for _ in range(num_image_encoders)]+ )self.classifier = nn.Linear(hidden_size * 2, 1) # Binary classification taskdef forward(self, text_input, image_input):text_features = [encoder(text_input) for encoder in self.text_encoders]image_features = [encoder(image_input) for encoder in self.vision_encoders]-+# Summing the outputs of all text encoders and image encoderscombined_text_features = torch.stack(text_features, dim=1).sum(dim=1)combined_image_features = torch.stack(image_features, dim=1).sum(dim=1)-+combined_features = torch.cat((combined_text_features, combined_image_features), dim=1)output = self.classifier(combined_features)return output@@ -70,23 +78,28 @@ class Expert(nn.Module):def __init__(self, input_size, hidden_size):super(Expert, self).__init__()self.fc = nn.Linear(input_size, hidden_size)-+def forward(self, x):return self.fc(x)+class ExpertGroupModuleList(nn.ModuleList):pass+class ExpertModuleList(nn.ModuleList):pass+class NestedMoEModel(nn.Module):def __init__(self, input_size, hidden_size, num_groups, experts_per_group):super(NestedMoEModel, self).__init__()- self.expert_groups = ExpertModuleList([- ExpertGroupModuleList([Expert(input_size, hidden_size) for _ in range(experts_per_group)])- for _ in range(num_groups)- ])+ self.expert_groups = ExpertModuleList(+ [+ ExpertGroupModuleList([Expert(input_size, hidden_size) for _ in range(experts_per_group)])+ for _ in range(num_groups)+ ]+ )self.gate = nn.Linear(input_size, num_groups * experts_per_group)def forward(self, x):@@ -115,7 +128,6 @@ class TestQuantizationBlocks(unittest.TestCase):shutil.rmtree("./saved", ignore_errors=True)shutil.rmtree("runs", ignore_errors=True)-def test_moe_quant(self):input_size = 10hidden_size = 10@@ -124,11 +136,11 @@ class TestQuantizationBlocks(unittest.TestCase):self.llm_dataloader = LLMDataLoader(input_size)self.model = NestedMoEModel(input_size, hidden_size, num_groups, experts_per_group)from auto_round.utils import get_block_names+llm_block_names = get_block_names(self.model)all_block_names = []all_block_names = get_block_names(self.model, quant_vision=True)assert len(llm_block_names) == len(all_block_names)-def test_multimodal_quant(self):num_text_encoders = 1@@ -136,18 +148,21 @@ class TestQuantizationBlocks(unittest.TestCase):image_input_size = 10text_input_size = 10hidden_size = 10- self.model = SimpleMultimodalModel(text_input_size, image_input_size, hidden_size, num_text_encoders, num_image_encoders)+ self.model = SimpleMultimodalModel(+ text_input_size, image_input_size, hidden_size, num_text_encoders, num_image_encoders+ )from auto_round.utils import get_block_names+llm_block_names = get_block_names(self.model)block_names_wo_vision = get_block_names(self.model, quant_vision=False)block_names_with_vision = get_block_names(self.model, quant_vision=True)assert block_names_wo_vision == llm_block_namesassert len(block_names_wo_vision) != (block_names_with_vision)--+def test_block_name_quant(self):self.model = AutoModelForCausalLM.from_pretrained(self.model_name, torch_dtype="auto", trust_remote_code=True)from auto_round.utils import get_block_names+llm_block_names = get_block_names(self.model)bits, group_size, sym, batch_size = 4, 128, True, 20autoround = AutoRound(@@ -160,7 +175,7 @@ class TestQuantizationBlocks(unittest.TestCase):seqlen=2,batch_size=batch_size,dataset=self.llm_dataloader,- to_quant_block_names=llm_block_names+ to_quant_block_names=llm_block_names,)autoround.quantize()@@ -175,34 +190,33 @@ class TestQuantizationBlocks(unittest.TestCase):quant_config = model.config.quantization_configassert quant_config.block_name_to_quantize is not Noneshutil.rmtree("./saved", ignore_errors=True)-def test_mm_block_name(self):from transformers import Qwen2VLForConditionalGenerationfrom auto_round.utils import get_block_names+model_name = "Qwen/Qwen2-VL-2B-Instruct"- model = Qwen2VLForConditionalGeneration.from_pretrained(- model_name, trust_remote_code=True, device_map="auto")+ model = Qwen2VLForConditionalGeneration.from_pretrained(model_name, trust_remote_code=True, device_map="auto")block_name = get_block_names(model, quant_vision=True)self.assertTrue(len(block_name) == 2)self.assertTrue(all(["visual.merger.mlp" not in n for n in block_name]))block_name = get_block_names(model, quant_vision=False)self.assertTrue(len(block_name) == 1)- self.assertTrue(block_name == get_block_names(model))+ self.assertTrue(block_name == get_block_names(model))def test_moe(self):from auto_round.utils import get_block_names+model_name = "Qwen/Qwen1.5-MoE-A2.7B"# config = AutoConfig.from_pretrained(model_name)model = AutoModelForCausalLM.from_pretrained(model_name)-block_name = get_block_names(model)block_name_2 = get_block_names(model, quant_vision=True)self.assertTrue(block_name == block_name_2)self.assertTrue(len(block_name_2) == 1)- self.assertTrue('model.layers.23' == block_name_2[0][-1])+ self.assertTrue("model.layers.23" == block_name_2[0][-1])##tokenizer = AutoTokenizer.from_pretrained(model_name)# python_path = sys.executable# res = os.system(@@ -223,12 +237,5 @@ class TestQuantizationBlocks(unittest.TestCase):# assert quant_config.to_quant_block_names is not None--if __name__ == "__main__":unittest.main()-----diff --git a/test/test_cpu/test_calib_dataset.py b/test/test_cpu/test_calib_dataset.pyindex 5dea5bf05..ce201048d 100644--- a/test/test_cpu/test_calib_dataset.py+++ b/test/test_cpu/test_calib_dataset.py@@ -36,7 +36,7 @@ class TestLocalCalibDataset(unittest.TestCase):with open(self.jsonl_file, "w") as jsonl_file:for item in jsonl_data:json.dump(item, jsonl_file, ensure_ascii=False)- jsonl_file.write('\n')+ jsonl_file.write("\n")model_name = "facebook/opt-125m"self.model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True)@@ -113,5 +113,3 @@ class TestLocalCalibDataset(unittest.TestCase):if __name__ == "__main__":unittest.main()--diff --git a/test/test_cpu/test_conv1d.py b/test/test_cpu/test_conv1d.pyindex 0009bcedf..25e695b40 100644--- a/test/test_cpu/test_conv1d.py+++ b/test/test_cpu/test_conv1d.py@@ -32,7 +32,6 @@ class TestQuantizationConv1d(unittest.TestCase):shutil.rmtree("./saved", ignore_errors=True)shutil.rmtree("runs", ignore_errors=True)-def test_quant(self):self.model = AutoModelForCausalLM.from_pretrained(self.model_name, torch_dtype="auto", trust_remote_code=True)bits, group_size, sym = 4, 128, True@@ -45,7 +44,6 @@ class TestQuantizationConv1d(unittest.TestCase):iters=2,seqlen=2,dataset=self.llm_dataloader,-)autoround.quantize()@@ -55,6 +53,5 @@ class TestQuantizationConv1d(unittest.TestCase):model_infer(model, self.tokenizer)-if __name__ == "__main__":unittest.main()diff --git a/test/test_cpu/test_export.py b/test/test_cpu/test_export.pyindex bfd54ba31..9d363e921 100644--- a/test/test_cpu/test_export.py+++ b/test/test_cpu/test_export.py@@ -52,11 +52,10 @@ class TestAutoRound(unittest.TestCase):if group_size == -1:shutil.rmtree("./saved", ignore_errors=True)continue- quantization_config = AutoRoundConfig(+ quantization_config = AutoRoundConfig()+ model = AutoModelForCausalLM.from_pretrained(+ quantized_model_path, device_map="cpu", trust_remote_code=True, quantization_config=quantization_config)- model = AutoModelForCausalLM.from_pretrained(quantized_model_path, device_map="cpu",- trust_remote_code=True,- quantization_config=quantization_config)tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)text = "There is a girl who likes adventure,"inputs = tokenizer(text, return_tensors="pt").to(model.device)@@ -90,7 +89,6 @@ class TestAutoRound(unittest.TestCase):print(tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0]))shutil.rmtree("./saved", ignore_errors=True)-def test_autoround_awq_format(self):for group_size in [-1, 32, 128]:bits, sym = 4, False@@ -138,21 +136,20 @@ class TestAutoRound(unittest.TestCase):autoround.quantize()quantized_model_path = "./saved"- autoround.save_quantized(output_dir=quantized_model_path, inplace=False, \- format="auto_awq")+ autoround.save_quantized(output_dir=quantized_model_path, inplace=False, format="auto_awq")if group_size == -1:shutil.rmtree("./saved", ignore_errors=True)continuequantization_config = AutoRoundConfig()- model = AutoModelForCausalLM.from_pretrained(quantized_model_path, device_map="cpu",- quantization_config=quantization_config)+ model = AutoModelForCausalLM.from_pretrained(+ quantized_model_path, device_map="cpu", quantization_config=quantization_config+ )tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)text = "There is a girl who likes adventure,"inputs = tokenizer(text, return_tensors="pt").to(model.device)print(tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0]))shutil.rmtree("./saved", ignore_errors=True)-def test_autoround_3bit_asym_format(self):bits, group_size, sym = 3, 128, False@@ -168,7 +165,7 @@ class TestAutoRound(unittest.TestCase):)autoround.quantize()quantized_model_path = self.save_dir-+autoround.save_quantized(output_dir=quantized_model_path, inplace=False, format="auto_round")device = "cpu" ##cpu, hpu, cudamodel = AutoModelForCausalLM.from_pretrained(quantized_model_path, device_map=device)@@ -178,7 +175,6 @@ class TestAutoRound(unittest.TestCase):print(tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0]))shutil.rmtree(quantized_model_path, ignore_errors=True)-def test_autoround_3bit_sym_format(self):bits, group_size, sym = 3, 128, Trueautoround = AutoRound(@@ -193,7 +189,7 @@ class TestAutoRound(unittest.TestCase):)autoround.quantize()quantized_model_path = self.save_dir-+autoround.save_quantized(output_dir=quantized_model_path, inplace=False, format="auto_round")device = "cpu" ##cpu, hpu, cudamodel = AutoModelForCausalLM.from_pretrained(quantized_model_path, device_map=device)@@ -202,8 +198,7 @@ class TestAutoRound(unittest.TestCase):inputs = tokenizer(text, return_tensors="pt").to(model.device)print(tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0]))shutil.rmtree(quantized_model_path, ignore_errors=True)--+def test_static_afp8_export(self):import os@@ -228,7 +223,7 @@ class TestAutoRound(unittest.TestCase):f = safe_open(os.path.join(quantized_model_path, "model.safetensors"), framework="pt")self.assertIn("model.decoder.layers.8.self_attn.k_proj.act_scale", f.keys())self.assertIn("model.decoder.layers.8.self_attn.k_proj.weight_scale", f.keys())- self.assertEqual(f.get_tensor("model.decoder.layers.5.self_attn.v_proj.act_scale").shape, torch.Size([1,1]))+ self.assertEqual(f.get_tensor("model.decoder.layers.5.self_attn.v_proj.act_scale").shape, torch.Size([1, 1]))self.assertEqual(f.get_tensor("model.decoder.layers.5.self_attn.v_proj.weight").dtype, torch.float8_e4m3fn)shutil.rmtree(quantized_model_path, ignore_errors=True)@@ -248,14 +243,13 @@ class TestAutoRound(unittest.TestCase):quantized_model_path = "./saved"autoround.quantize_and_save(output_dir=quantized_model_path, format="auto_round")-f = safe_open(os.path.join(quantized_model_path, "model.safetensors"), framework="pt")self.assertIn("model.decoder.layers.8.self_attn.k_proj.act_scale", f.keys())self.assertIn("model.decoder.layers.8.self_attn.k_proj.weight_scale", f.keys())- self.assertEqual(f.get_tensor("model.decoder.layers.5.self_attn.v_proj.act_scale").shape, torch.Size([1,1]))+ self.assertEqual(f.get_tensor("model.decoder.layers.5.self_attn.v_proj.act_scale").shape, torch.Size([1, 1]))self.assertEqual(f.get_tensor("model.decoder.layers.5.self_attn.v_proj.weight").dtype, torch.float8_e4m3fn)shutil.rmtree(quantized_model_path, ignore_errors=True)+if __name__ == "__main__":unittest.main()-diff --git a/test/test_cpu/test_generation.py b/test/test_cpu/test_generation.pyindex 11c9ca1bb..d1e51f722 100644--- a/test/test_cpu/test_generation.py+++ b/test/test_cpu/test_generation.py@@ -51,27 +51,26 @@ class TestAutoRoundFormatGeneration(unittest.TestCase):autoround.quantize_and_save(output_dir=quantized_model_path, format="auto_round", inplace=False)- quantization_config = AutoRoundConfig(- backend="ipex"+ quantization_config = AutoRoundConfig(backend="ipex")+ model = AutoModelForCausalLM.from_pretrained(+ quantized_model_path, device_map="cpu", quantization_config=quantization_config)- model = AutoModelForCausalLM.from_pretrained(quantized_model_path,- device_map="cpu", quantization_config=quantization_config)tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)text = "My name is "inputs = tokenizer(text, return_tensors="pt").to(model.device)res = tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0])print(res)- assert ("!!!" not in res)+ assert "!!!" not in res- model = AutoModelForCausalLM.from_pretrained(quantized_model_path,- device_map="cpu", quantization_config=quantization_config,- torch_dtype=torch.float16)+ model = AutoModelForCausalLM.from_pretrained(+ quantized_model_path, device_map="cpu", quantization_config=quantization_config, torch_dtype=torch.float16+ )tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)text = "There is a girl who likes adventure,"inputs = tokenizer(text, return_tensors="pt").to(model.device)res = tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0])print(res)- assert ("!!!" not in res)+ assert "!!!" not in resdef test_autoround_sym(self):for bits in [4]:@@ -92,13 +91,13 @@ class TestAutoRoundFormatGeneration(unittest.TestCase):autoround.quantize_and_save(output_dir=quantized_model_path, format="auto_round")- model = AutoModelForCausalLM.from_pretrained(quantized_model_path, device_map="auto",- trust_remote_code=True)+ model = AutoModelForCausalLM.from_pretrained(+ quantized_model_path, device_map="auto", trust_remote_code=True+ )tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)text = "There is a girl who likes adventure,"inputs = tokenizer(text, return_tensors="pt").to(model.device)res = tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0])print(res)- assert ("!!!" not in res)+ assert "!!!" not in resshutil.rmtree(self.save_folder, ignore_errors=True)-diff --git a/test/test_cpu/test_gguf_format.py b/test/test_cpu/test_gguf_format.pyindex cb7691a1b..cacaca71e 100644--- a/test/test_cpu/test_gguf_format.py+++ b/test/test_cpu/test_gguf_format.py@@ -27,7 +27,7 @@ class TestGGUF(unittest.TestCase):self.model = AutoModelForCausalLM.from_pretrained(self.model_name, torch_dtype="auto", trust_remote_code=True)self.tokenizer = AutoTokenizer.from_pretrained(self.model_name, trust_remote_code=True)self.llm_dataloader = LLMDataLoader()-+@classmethoddef tearDownClass(self):shutil.rmtree("./saved", ignore_errors=True)@@ -51,17 +51,10 @@ class TestGGUF(unittest.TestCase):assert False, "cmd line test fail, please have a check"shutil.rmtree("./saved", ignore_errors=True)-def test_q4_0(self):bits, group_size, sym = 4, 32, Trueautoround = AutoRound(- self.model,- self.tokenizer,- bits=bits,- group_size=group_size,- sym=sym,- iters=1,- data_type="int"+ self.model, self.tokenizer, bits=bits, group_size=group_size, sym=sym, iters=1, data_type="int")quantized_model_path = "./saved"@@ -81,13 +74,7 @@ class TestGGUF(unittest.TestCase):def test_q4_1(self):bits, group_size, sym = 4, 32, Falseautoround = AutoRound(- self.model,- self.tokenizer,- bits=bits,- group_size=group_size,- sym=sym,- iters=1,- data_type="int"+ self.model, self.tokenizer, bits=bits, group_size=group_size, sym=sym, iters=1, data_type="int")quantized_model_path = "./saved"@@ -137,7 +124,7 @@ class TestGGUF(unittest.TestCase):iters=1,data_type="int_sym_dq",super_group_size=16,- super_bits=6+ super_bits=6,)quantized_model_path = "./saved"# autoround.quantize_and_save(output_dir=quantized_model_path, inplace=False, format="gguf:q*_k_s")@@ -160,7 +147,7 @@ class TestGGUF(unittest.TestCase):iters=1,data_type="int_asym_dq",super_group_size=8,- super_bits=6+ super_bits=6,)quantized_model_path = "./saved"autoround.quantize_and_save(output_dir=quantized_model_path, inplace=False, format="gguf:q*_k_s")@@ -183,7 +170,7 @@ class TestGGUF(unittest.TestCase):iters=1,data_type="int_sym_dq",super_group_size=16,- super_bits=8+ super_bits=8,)quantized_model_path = "./saved"autoround.quantize_and_save(output_dir=quantized_model_path, inplace=False, format="gguf:q*_k")@@ -237,16 +224,24 @@ class TestGGUF(unittest.TestCase):print(self.tokenizer.decode(model.generate(**inputs, max_new_tokens=10)[0]))shutil.rmtree("./saved", ignore_errors=True)-def test_q4_k_m(self):model_name = "Qwen/Qwen2.5-1.5B-Instruct"model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True)tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)layer_config = {- "lm_head": {'bits': 4, 'group_size': 32, 'sym': False, 'data_type': 'int_asym_dq', 'scale_dtype': torch.float32, 'super_bits': 6, 'super_group_size': 8, 'in_blocks': False},- "model.embed_tokens": {'bits': 6, 'group_size': 32, 'super_bits': 6, 'super_group_size': 8},- "model.layers.12.mlp.gate_proj": {'bits': 3},- "model.layers.10.mlp.gate_proj": {'bits': 8},+ "lm_head": {+ "bits": 4,+ "group_size": 32,+ "sym": False,+ "data_type": "int_asym_dq",+ "scale_dtype": torch.float32,+ "super_bits": 6,+ "super_group_size": 8,+ "in_blocks": False,+ },+ "model.embed_tokens": {"bits": 6, "group_size": 32, "super_bits": 6, "super_group_size": 8},+ "model.layers.12.mlp.gate_proj": {"bits": 3},+ "model.layers.10.mlp.gate_proj": {"bits": 8},}autoround = AutoRound(model,@@ -255,7 +250,7 @@ class TestGGUF(unittest.TestCase):iters=0,seqlen=1,dataset=self.llm_dataloader,- disable_opt_rtn=True+ disable_opt_rtn=True,)quantized_model_path = "./saved"autoround.quantize_and_save(output_dir=quantized_model_path, format="gguf:q4_k_m,fake")@@ -268,21 +263,14 @@ class TestGGUF(unittest.TestCase):self.assertEqual(autoround.model.model.embed_tokens.group_size, 16)self.assertEqual(autoround.model.model.layers[12].mlp.gate_proj.bits, 3)self.assertEqual(autoround.model.model.layers[10].mlp.gate_proj.bits, 8)- self.assertEqual(autoround.layer_config['model.layers.10.mlp.gate_proj']['mostly'], "gguf:q8_0")+ self.assertEqual(autoround.layer_config["model.layers.10.mlp.gate_proj"]["mostly"], "gguf:q8_0")shutil.rmtree("./saved", ignore_errors=True)- autoround = AutoRound(- model,- tokenizer,- iters=0,- nsamples=1,- seqlen=128,- disable_opt_rtn=False- )+ autoround = AutoRound(model, tokenizer, iters=0, nsamples=1, seqlen=128, disable_opt_rtn=False)quantized_model_path = "./saved"autoround.quantize_and_save(output_dir=quantized_model_path, format="gguf:q4_k_m,fake")shutil.rmtree("./saved", ignore_errors=True)-+def test_all_format(self):model_name = "Qwen/Qwen2.5-1.5B-Instruct"python_path = sys.executable@@ -302,11 +290,12 @@ class TestGGUF(unittest.TestCase):if res > 0 or res == -1:assert False, "cmd line test fail, please have a check"shutil.rmtree("../../tmp_autoround", ignore_errors=True)-+def test_vlm_gguf(self):model_name = "Qwen/Qwen2-VL-2B-Instruct"from auto_round.mllm.autoround_mllm import AutoRoundMLLMfrom auto_round.utils import mllm_load_model+model, processor, tokenizer, image_processor = mllm_load_model(model_name)autoround = AutoRoundMLLM(model,@@ -318,10 +307,10 @@ class TestGGUF(unittest.TestCase):quantized_model_path = "./saved"autoround.quantize_and_save(output_dir=quantized_model_path, format="gguf:q4_0")self.assertTrue("mmproj-model.gguf" in os.listdir("./saved"))- file_size = os.path.getsize("./saved/Qwen2-VL-2B-Instruct-1.5B-Q4_0.gguf") / 1024 ** 2+ file_size = os.path.getsize("./saved/Qwen2-VL-2B-Instruct-1.5B-Q4_0.gguf") / 1024**2self.assertAlmostEqual(file_size, 892, delta=1.0)shutil.rmtree("./saved", ignore_errors=True)+if __name__ == "__main__":unittest.main()-diff --git a/test/test_cpu/test_hpu.py b/test/test_cpu/test_hpu.pyindex 035060817..290dcfbca 100644--- a/test/test_cpu/test_hpu.py+++ b/test/test_cpu/test_hpu.py@@ -17,6 +17,7 @@ class LLMDataLoader:for i in range(2):yield torch.ones([1, 10], dtype=torch.long)+def is_hpu_supported():try:import habana_frameworks.torch.core as htcore # pylint: disable=E0401@@ -60,15 +61,17 @@ class TestAutoRound(unittest.TestCase):quantized_model_path = "./saved"autoround.save_quantized(output_dir=quantized_model_path, inplace=False, format="auto_gptq")- model = AutoModelForCausalLM.from_pretrained(quantized_model_path, device_map="auto", \- trust_remote_code=True).to('hpu').to(torch.float32)+ model = (+ AutoModelForCausalLM.from_pretrained(quantized_model_path, device_map="auto", trust_remote_code=True)+ .to("hpu")+ .to(torch.float32)+ )tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)text = "There is a girl who likes adventure,"inputs = tokenizer(text, return_tensors="pt").to(model.device)print(tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0]))shutil.rmtree("./saved", ignore_errors=True)-def test_autoround_format_hpu_inference(self):if not is_hpu_supported():return@@ -88,8 +91,9 @@ class TestAutoRound(unittest.TestCase):autoround.save_quantized(output_dir=quantized_model_path, inplace=False, format="auto_round")- model = AutoModelForCausalLM.from_pretrained(quantized_model_path, \- device_map="auto").to('hpu').to(torch.float32)+ model = (+ AutoModelForCausalLM.from_pretrained(quantized_model_path, device_map="auto").to("hpu").to(torch.float32)+ )tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)text = "There is a girl who likes adventure,"inputs = tokenizer(text, return_tensors="pt").to(model.device)diff --git a/test/test_cpu/test_llmcompressor_w8a8.py b/test/test_cpu/test_llmcompressor_w8a8.pyindex 978a8ed5c..149d5e33f 100644--- a/test/test_cpu/test_llmcompressor_w8a8.py+++ b/test/test_cpu/test_llmcompressor_w8a8.py@@ -17,23 +17,26 @@ class TestLLMC(unittest.TestCase):self.model_name = "stas/tiny-random-llama-2"self.model = AutoModelForCausalLM.from_pretrained(self.model_name, torch_dtype="auto", trust_remote_code=True)self.tokenizer = AutoTokenizer.from_pretrained(self.model_name, trust_remote_code=True)-+@classmethoddef tearDownClass(self):shutil.rmtree("./saved", ignore_errors=True)shutil.rmtree("runs", ignore_errors=True)-def test_llmcompressor_w8a8(self):bits, group_size, sym, act_bits = 8, -1, True, 8## quantize the modelautoround = AutoRound(- self.model, self.tokenizer,- bits=bits, group_size=group_size, sym=sym, act_bits=act_bits,+ self.model,+ self.tokenizer,+ bits=bits,+ group_size=group_size,+ sym=sym,+ act_bits=act_bits,iters=0,)autoround.quantize()- autoround.save_quantized("./saved", format='llmcompressor', inplace=True)+ autoround.save_quantized("./saved", format="llmcompressor", inplace=True)if __name__ == "__main__":diff --git a/test/test_cpu/test_load_awq_gptq.py b/test/test_cpu/test_load_awq_gptq.pyindex 88e9730bc..bda3d4447 100644--- a/test/test_cpu/test_load_awq_gptq.py+++ b/test/test_cpu/test_load_awq_gptq.py@@ -22,11 +22,9 @@ class TestAutoRound(unittest.TestCase):input_ids=inputs["input_ids"].to(model.device),attention_mask=inputs["attention_mask"].to(model.device),do_sample=False, ## change this to follow official usage- max_new_tokens=5+ max_new_tokens=5,)- generated_ids = [- output_ids[len(input_ids):] for input_ids, output_ids in zip(inputs["input_ids"], outputs)- ]+ generated_ids = [output_ids[len(input_ids) :] for input_ids, output_ids in zip(inputs["input_ids"], outputs)]decoded_outputs = tokenizer.batch_decode(generated_ids, skip_special_tokens=True)@@ -43,15 +41,23 @@ class TestAutoRound(unittest.TestCase):model_name = "ModelCloud/Llama-3.2-1B-Instruct-gptqmodel-4bit-vortex-v1"quantization_config = AutoRoundConfig()with self.assertRaises(NotImplementedError) as cm:- model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True,- device_map="cpu",- quantization_config=quantization_config)+ model = AutoModelForCausalLM.from_pretrained(+ model_name,+ torch_dtype="auto",+ trust_remote_code=True,+ device_map="cpu",+ quantization_config=quantization_config,+ )def test_load_awq(self):model_name = "casperhansen/opt-125m-awq"quantization_config = AutoRoundConfig()- model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True,- device_map="cpu",- quantization_config=quantization_config)+ model = AutoModelForCausalLM.from_pretrained(+ model_name,+ torch_dtype="auto",+ trust_remote_code=True,+ device_map="cpu",+ quantization_config=quantization_config,+ )tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)self.model_infer(model, tokenizer)diff --git a/test/test_cpu/test_low_cpu_mem.py b/test/test_cpu/test_low_cpu_mem.pyindex 5ad8f5659..85869dde4 100644--- a/test/test_cpu/test_low_cpu_mem.py+++ b/test/test_cpu/test_low_cpu_mem.py@@ -26,13 +26,16 @@ class LLMDataLoader:for i in range(2):yield torch.ones([1, 10], dtype=torch.long)+class TestLowCPUMem(unittest.TestCase):@classmethoddef setUpClass(self):self.model_name = "facebook/opt-125m"- self.saved_path = './test_tmp_saved'+ self.saved_path = "./test_tmp_saved"self.ori_model = AutoModelForCausalLM.from_pretrained(self.model_name, trust_remote_code=True)- self.model = load_model_with_hooks(self.model_name, AutoModelForCausalLM, saved_path=self.saved_path, device='cpu')+ self.model = load_model_with_hooks(+ self.model_name, AutoModelForCausalLM, saved_path=self.saved_path, device="cpu"+ )self.tokenizer = AutoTokenizer.from_pretrained(self.model_name, trust_remote_code=True)self.llm_dataloader = LLMDataLoader()@@ -41,20 +44,24 @@ class TestLowCPUMem(unittest.TestCase):shutil.rmtree(self.saved_path, ignore_errors=True)def test_default(self):- self.assertTrue(self.model.device.type, 'meta')+ self.assertTrue(self.model.device.type, "meta")layers = get_layers_before_block(self.model)- self.assertEqual(layers[0][0], 'model.decoder.embed_tokens')+ self.assertEqual(layers[0][0], "model.decoder.embed_tokens")# test get_weight bias- self.assertTrue(torch.equal(- self.model.model.decoder.layers[0].self_attn.k_proj.get_weight(),- self.ori_model.model.decoder.layers[0].self_attn.k_proj.weight,- ))- self.assertTrue(torch.equal(- self.model.model.decoder.layers[0].self_attn.k_proj.get_bias(),- self.ori_model.model.decoder.layers[0].self_attn.k_proj.bias,- ))+ self.assertTrue(+ torch.equal(+ self.model.model.decoder.layers[0].self_attn.k_proj.get_weight(),+ self.ori_model.model.decoder.layers[0].self_attn.k_proj.weight,+ )+ )+ self.assertTrue(+ torch.equal(+ self.model.model.decoder.layers[0].self_attn.k_proj.get_bias(),+ self.ori_model.model.decoder.layers[0].self_attn.k_proj.bias,+ )+ )# test hookstext = ["Hello, my dog is cute"]@@ -68,43 +75,41 @@ class TestLowCPUMem(unittest.TestCase):# test save and loadlayer_wise_save(self.model, self.saved_path)state_dict = layer_wise_load(self.saved_path)- self.assertTrue(torch.equal(- state_dict['lm_head.weight'],- self.ori_model.lm_head.weight- ))+ self.assertTrue(torch.equal(state_dict["lm_head.weight"], self.ori_model.lm_head.weight))# test layer-wise auto_roundbits, group_size, sym = 4, 128, Falseautoround = AutoRound(self.model,self.tokenizer,- device='cpu',+ device="cpu",bits=bits,group_size=group_size,sym=sym,iters=2,seqlen=2,dataset=self.llm_dataloader,- enable_torch_compile=False+ enable_torch_compile=False,)autoround.quantize()# test block-wise auto_round- self.model = load_empty_model(self.model_name, AutoModelForCausalLM, saved_path=self.saved_path, device='cpu')+ self.model = load_empty_model(self.model_name, AutoModelForCausalLM, saved_path=self.saved_path, device="cpu")bits, group_size, sym = 4, 128, Falseautoround = AutoRound(self.model,self.tokenizer,- device='cpu',+ device="cpu",bits=bits,group_size=group_size,sym=sym,iters=2,seqlen=2,dataset=self.llm_dataloader,- low_cpu_mem_usage=True+ low_cpu_mem_usage=True,)autoround.quantize()+if __name__ == "__main__":- unittest.main()\ No newline at end of file+ unittest.main()diff --git a/test/test_cpu/test_mllm.py b/test/test_cpu/test_mllm.pyindex 4206847f0..afeb8a570 100644--- a/test/test_cpu/test_mllm.py+++ b/test/test_cpu/test_mllm.py@@ -15,14 +15,11 @@ class FakeDataLoader:self.batch_size = 1self.data = {- "text": [{- 'role': 'user',- 'content': '<image>\nWhat are the colors of the bus in the image?'- }, {- 'role': 'assistant',- 'content': 'The bus in the image is white and red.'- }],- "image": "http://images.cocodataset.org/train2017/000000033471.jpg"+ "text": [+ {"role": "user", "content": "<image>\nWhat are the colors of the bus in the image?"},+ {"role": "assistant", "content": "The bus in the image is white and red."},+ ],+ "image": "http://images.cocodataset.org/train2017/000000033471.jpg",}def __iter__(self):@@ -47,72 +44,110 @@ class TestAutoRoundMLLM(unittest.TestCase):tokenizer = AutoTokenizer.from_pretrained(self.model_name)processor = AutoProcessor.from_pretrained(self.model_name, trust_remote_code=True)model = Qwen2VLForConditionalGeneration.from_pretrained(- self.model_name, trust_remote_code=True, device_map="auto")+ self.model_name, trust_remote_code=True, device_map="auto"+ )bits, group_size = 4, 128autoround = AutoRoundMLLM(- model, tokenizer, processor=processor,- bits=bits, group_size=group_size,+ model,+ tokenizer,+ processor=processor,+ bits=bits,+ group_size=group_size,nsamples=1,- batch_size=1, iters=2, dataset=self.dataset,seqlen=10)+ batch_size=1,+ iters=2,+ dataset=self.dataset,+ seqlen=10,+ )autoround.quantize()autoround.save_quantized("./saved/", format="auto_gptq", inplace=False)autoround.save_quantized("./saved/", format="auto_round", inplace=False)- def test_quant_vision(self): ## bug need to fix+ def test_quant_vision(self): ## bug need to fixtokenizer = AutoTokenizer.from_pretrained(self.model_name)processor = AutoProcessor.from_pretrained(self.model_name, trust_remote_code=True)model = Qwen2VLForConditionalGeneration.from_pretrained(- self.model_name, trust_remote_code=True, device_map="auto")+ self.model_name, trust_remote_code=True, device_map="auto"+ )bits, group_size = 4, 128autoround = AutoRoundMLLM(- model, tokenizer, processor=processor,- bits=bits, group_size=group_size,+ model,+ tokenizer,+ processor=processor,+ bits=bits,+ group_size=group_size,nsamples=5,- batch_size=3, iters=2, dataset=self.dataset, quant_nontext_module=True, seqlen=10)+ batch_size=3,+ iters=2,+ dataset=self.dataset,+ quant_nontext_module=True,+ seqlen=10,+ )autoround.quantize()autoround.save_quantized("./saved/", format="auto_round", inplace=True)def test_quant_block_names(self):from auto_round.utils import find_matching_blocks, get_block_names+tokenizer = AutoTokenizer.from_pretrained(self.model_name)processor = AutoProcessor.from_pretrained(self.model_name, trust_remote_code=True)model = Qwen2VLForConditionalGeneration.from_pretrained(- self.model_name, trust_remote_code=True, device_map="auto")- to_quant_block_names = 'visual.*12,layers.0,model.layers.*9'- target_blocks = [['model.visual.blocks.12'], ['model.language_model.layers.0', 'model.language_model.layers.9', 'model.language_model.layers.19']]+ self.model_name, trust_remote_code=True, device_map="auto"+ )+ to_quant_block_names = "visual.*12,layers.0,model.layers.*9"+ target_blocks = [+ ["model.visual.blocks.12"],+ ["model.language_model.layers.0", "model.language_model.layers.9", "model.language_model.layers.19"],+ ]all_blocks = get_block_names(model, quant_vision=True)blocks = find_matching_blocks(model, all_blocks, to_quant_block_names)assert target_blocks == blocksdef test_dataset_check(self):from auto_round.mllm.mllm_dataset import MLLM_DATASET+class Myclass:- model_type=None- dataset = MLLM_DATASET['liuhaotian/llava'](template=Myclass(), model=None, tokenzier=None, dataset_path="liuhaotian/llava", seqlen=32, nsamples=32)+ model_type = None++ dataset = MLLM_DATASET["liuhaotian/llava"](+ template=Myclass(), model=None, tokenzier=None, dataset_path="liuhaotian/llava", seqlen=32, nsamples=32+ )self.assertEqual(len(dataset.questions), 32)- dataset = MLLM_DATASET['liuhaotian/llava'](template=Myclass(), model=None, tokenzier=None, dataset_path="liuhaotian/llava", seqlen=2048, nsamples=512)+ dataset = MLLM_DATASET["liuhaotian/llava"](+ template=Myclass(), model=None, tokenzier=None, dataset_path="liuhaotian/llava", seqlen=2048, nsamples=512+ )self.assertEqual(len(dataset.questions), 512)def test_diff_dataset(self):tokenizer = AutoTokenizer.from_pretrained(self.model_name)processor = AutoProcessor.from_pretrained(self.model_name, trust_remote_code=True)model = Qwen2VLForConditionalGeneration.from_pretrained(- self.model_name, trust_remote_code=True, device_map="auto")+ self.model_name, trust_remote_code=True, device_map="auto"+ )bits, group_size = 4, 128dataset = ["dataset test", "list test"]autoround = AutoRoundMLLM(- model, tokenizer, processor=processor,- bits=bits, group_size=group_size,+ model,+ tokenizer,+ processor=processor,+ bits=bits,+ group_size=group_size,nsamples=2,- batch_size=1, iters=2, dataset=dataset, seqlen=1)+ batch_size=1,+ iters=2,+ dataset=dataset,+ seqlen=1,+ )autoround.quantize()-+def test_pure_text_model_check(self):from transformers import AutoModelForCausalLMfrom auto_round.utils import is_pure_text_model+model = Qwen2VLForConditionalGeneration.from_pretrained(- self.model_name, trust_remote_code=True, device_map="auto")+ self.model_name, trust_remote_code=True, device_map="auto"+ )self.assertFalse(is_pure_text_model(model))model = AutoModelForCausalLM.from_pretrained("facebook/opt-125m", trust_remote_code=True)self.assertTrue(is_pure_text_model(model))@@ -121,14 +156,22 @@ class TestAutoRoundMLLM(unittest.TestCase):tokenizer = AutoTokenizer.from_pretrained(self.model_name)processor = AutoProcessor.from_pretrained(self.model_name, trust_remote_code=True)model = Qwen2VLForConditionalGeneration.from_pretrained(- self.model_name, trust_remote_code=True, device_map="auto")+ self.model_name, trust_remote_code=True, device_map="auto"+ )bits, group_size = 4, 128dataset = ["test pure text", "input for mllm"]autoround = AutoRoundMLLM(- model, tokenizer, processor=processor,- bits=bits, group_size=group_size,+ model,+ tokenizer,+ processor=processor,+ bits=bits,+ group_size=group_size,nsamples=2,- batch_size=1, iters=2, dataset=dataset, seqlen=1)+ batch_size=1,+ iters=2,+ dataset=dataset,+ seqlen=1,+ )autoround.quantize()quantized_model_path = "./saved"autoround.save_quantized(quantized_model_path, format="auto_round", inplace=False)@@ -156,9 +199,7 @@ class TestAutoRoundMLLM(unittest.TestCase):]# Preparation for inference- text = processor.apply_chat_template(- messages, tokenize=False, add_generation_prompt=True- )+ text = processor.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)image_inputs = Image.open(requests.get(image_url, stream=True).raw)inputs = processor(text=[text],@@ -169,9 +210,7 @@ class TestAutoRoundMLLM(unittest.TestCase):inputs = inputs.to(model.device)generated_ids = model.generate(**inputs, max_new_tokens=128)- generated_ids_trimmed = [- out_ids[len(in_ids) :] for in_ids, out_ids in zip(inputs.input_ids, generated_ids)- ]+ generated_ids_trimmed = [out_ids[len(in_ids) :] for in_ids, out_ids in zip(inputs.input_ids, generated_ids)]output_text = processor.batch_decode(generated_ids_trimmed, skip_special_tokens=True, clean_up_tokenization_spaces=False)diff --git a/test/test_cpu/test_script.py b/test/test_cpu/test_script.pyindex 069a59ece..01bbba644 100644--- a/test/test_cpu/test_script.py+++ b/test/test_cpu/test_script.py@@ -7,12 +7,15 @@ sys.path.insert(0, "../..")class TestScript(unittest.TestCase):def test_default(self):- os.system('''+ os.system(+ """cd ../.. &&python -m auto_round--iters 2--deployment_device fake- --output_dir ./tmp_script_test''')+ --output_dir ./tmp_script_test"""+ )+if __name__ == "__main__":- unittest.main()\ No newline at end of file+ unittest.main()diff --git a/test/test_cpu/test_woq_linear.py b/test/test_cpu/test_woq_linear.pyindex 673b5d06d..f12400777 100644--- a/test/test_cpu/test_woq_linear.py+++ b/test/test_cpu/test_woq_linear.py@@ -31,17 +31,11 @@ class TestWeightOnlyLinear:group_size = 32origin_shape = weight.shapefrom auto_round.data_type.int import quant_tensor_sym+origin_shape = weight.shapeweight = weight.reshape(-1, group_size)- qdq, scale, zp = quant_tensor_sym( weight, -1- )- int_weight = (- qdq.div(scale)- .add(zp)- .clamp(0, 2 ** (bits) - 1)- .to(torch.int32)- .reshape(origin_shape)- )+ qdq, scale, zp = quant_tensor_sym(weight, -1)+ int_weight = qdq.div(scale).add(zp).clamp(0, 2 ** (bits) - 1).to(torch.int32).reshape(origin_shape)scale = scale.reshape(origin_shape[0], -1)zp = zp.reshape(origin_shape[0], -1).to(torch.int32).clamp(0, 2 ** (bits) - 1)module_with_legacy_pack = WeightOnlyLinear(@@ -56,9 +50,7 @@ class TestWeightOnlyLinear:compression_dtype=compression_dtype,use_legacy_pack=True,)- module_with_legacy_pack.pack(- int_weight.clone(), scale.clone(), zp.clone(), m.bias- )+ module_with_legacy_pack.pack(int_weight.clone(), scale.clone(), zp.clone(), m.bias)module_with_new_pack = WeightOnlyLinear(in_features=m.in_features,out_features=m.out_features,@@ -73,13 +65,9 @@ class TestWeightOnlyLinear:)module_with_new_pack.pack(int_weight.clone(), scale.clone(), zp.clone(), m.bias)- assert torch.equal(- module_with_new_pack.qweight, module_with_legacy_pack.qweight- )+ assert torch.equal(module_with_new_pack.qweight, module_with_legacy_pack.qweight)assert torch.equal(module_with_new_pack.qzeros, module_with_legacy_pack.qzeros)assert torch.equal(module_with_new_pack.scales, module_with_legacy_pack.scales)- unpacked_int_weight = module_with_new_pack.unpack_tensor(- module_with_legacy_pack.qweight- )+ unpacked_int_weight = module_with_new_pack.unpack_tensor(module_with_legacy_pack.qweight)assert torch.equal(unpacked_int_weight, int_weight)diff --git a/test/test_cuda/_test_helpers.py b/test/test_cuda/_test_helpers.pyindex 3c499c499..b4b8a5955 100644--- a/test/test_cuda/_test_helpers.py+++ b/test/test_cuda/_test_helpers.py@@ -1,4 +1,3 @@-def model_infer(model, tokenizer, apply_chat_template=False):prompts = ["Hello,my name is",@@ -9,14 +8,8 @@ def model_infer(model, tokenizer, apply_chat_template=False):if apply_chat_template:texts = []for prompt in prompts:- messages = [- {"role": "user", "content": prompt}- ]- text = tokenizer.apply_chat_template(- messages,- tokenize=False,- add_generation_prompt=True- )+ messages = [{"role": "user", "content": prompt}]+ text = tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)texts.append(text)prompts = texts@@ -26,11 +19,9 @@ def model_infer(model, tokenizer, apply_chat_template=False):input_ids=inputs["input_ids"].to(model.device),attention_mask=inputs["attention_mask"].to(model.device),do_sample=False, ## change this to follow official usage- max_new_tokens=5+ max_new_tokens=5,)- generated_ids = [- output_ids[len(input_ids):] for input_ids, output_ids in zip(inputs["input_ids"], outputs)- ]+ generated_ids = [output_ids[len(input_ids) :] for input_ids, output_ids in zip(inputs["input_ids"], outputs)]decoded_outputs = tokenizer.batch_decode(generated_ids, skip_special_tokens=True)diff --git a/test/test_cuda/test_2_3bits.py b/test/test_cuda/test_2_3bits.pyindex 519d4e6de..51982854a 100644--- a/test/test_cuda/test_2_3bits.py+++ b/test/test_cuda/test_2_3bits.py@@ -16,7 +16,7 @@ from auto_round.testing_utils import require_autogptq, require_greater_than_050,def get_accuracy(data):- match = re.search(r'\|acc\s+\|[↑↓]\s+\|\s+([\d.]+)\|', data)+ match = re.search(r"\|acc\s+\|[↑↓]\s+\|\s+([\d.]+)\|", data)if match:accuracy = float(match.group(1))@@ -44,13 +44,16 @@ class TestAutoRound(unittest.TestCase):autoround = AutoRound(model, tokenizer, bits=3)autoround.quantize_and_save(self.save_dir, format="auto_round", inplace=False)model_args = f"pretrained={self.save_dir}"- res = simple_evaluate(model="hf", model_args=model_args,- # tasks="arc_easy",- tasks=self.tasks,- batch_size="auto")+ res = simple_evaluate(+ model="hf",+ model_args=model_args,+ # tasks="arc_easy",+ tasks=self.tasks,+ batch_size="auto",+ )## 0.3130- accuracy = res['results']['lambada_openai']['acc,none']+ accuracy = res["results"]["lambada_openai"]["acc,none"]assert accuracy > 0.3shutil.rmtree("./saved", ignore_errors=True)@@ -63,13 +66,16 @@ class TestAutoRound(unittest.TestCase):autoround = AutoRound(model, tokenizer, bits=bits, sym=sym)autoround.quantize_and_save(self.save_dir, format="auto_round", inplace=False)model_args = f"pretrained={self.save_dir}"- res = simple_evaluate(model="hf", model_args=model_args,- # tasks="arc_easy",- tasks=self.tasks,- batch_size="auto")+ res = simple_evaluate(+ model="hf",+ model_args=model_args,+ # tasks="arc_easy",+ tasks=self.tasks,+ batch_size="auto",+ )## 0.3423- accuracy = res['results']['lambada_openai']['acc,none']+ accuracy = res["results"]["lambada_openai"]["acc,none"]assert accuracy > 0.32shutil.rmtree("./saved", ignore_errors=True)@@ -84,9 +90,7 @@ class TestAutoRound(unittest.TestCase):##test auto_round formatautoround.save_quantized(self.save_dir, format="auto_round", inplace=False)model_args = f"pretrained={self.save_dir}"- res = simple_evaluate(model="hf", model_args=model_args,- tasks=self.tasks,- batch_size="auto")+ res = simple_evaluate(model="hf", model_args=model_args, tasks=self.tasks, batch_size="auto")res = make_table(res) ##0.2212 0.1844accuracy = get_accuracy(res)assert accuracy > 0.18@@ -103,27 +107,20 @@ class TestAutoRound(unittest.TestCase):##test auto_round formatautoround.save_quantized(self.save_dir, format="auto_round", inplace=False)model_args = f"pretrained={self.save_dir}"- res = simple_evaluate(model="hf", model_args=model_args,- tasks=self.tasks,- batch_size="auto")- res = make_table(res) ##0.1985+ res = simple_evaluate(model="hf", model_args=model_args, tasks=self.tasks, batch_size="auto")+ res = make_table(res) ##0.1985accuracy = get_accuracy(res)assert accuracy > 0.18shutil.rmtree("./saved", ignore_errors=True)-autoround.save_quantized(self.save_dir, format="auto_gptq", inplace=False)model_args = f"pretrained={self.save_dir}"- res = simple_evaluate(model="hf", model_args=model_args,- tasks=self.tasks,- batch_size="auto")- res = make_table(res) ##0.1985+ res = simple_evaluate(model="hf", model_args=model_args, tasks=self.tasks, batch_size="auto")+ res = make_table(res) ##0.1985accuracy = get_accuracy(res)assert accuracy > 0.18shutil.rmtree("./saved", ignore_errors=True)+if __name__ == "__main__":unittest.main()---diff --git a/test/test_cuda/test_auto_round_format.py b/test/test_cuda/test_auto_round_format.pyindex 22c963a48..f5c6ae5df 100644--- a/test/test_cuda/test_auto_round_format.py+++ b/test/test_cuda/test_auto_round_format.py@@ -56,11 +56,9 @@ class TestAutoRound(unittest.TestCase):input_ids=inputs["input_ids"].to(model.device),attention_mask=inputs["attention_mask"].to(model.device),do_sample=False, ## change this to follow official usage- max_new_tokens=5+ max_new_tokens=5,)- generated_ids = [- output_ids[len(input_ids):] for input_ids, output_ids in zip(inputs["input_ids"], outputs)- ]+ generated_ids = [output_ids[len(input_ids) :] for input_ids, output_ids in zip(inputs["input_ids"], outputs)]decoded_outputs = tokenizer.batch_decode(generated_ids, skip_special_tokens=True)@@ -94,14 +92,15 @@ class TestAutoRound(unittest.TestCase):autoround.quantize_and_save(output_dir=quantized_model_path, format="auto_round")- model = AutoModelForCausalLM.from_pretrained(quantized_model_path, device_map="cuda:0",- trust_remote_code=True)+ model = AutoModelForCausalLM.from_pretrained(+ quantized_model_path, device_map="cuda:0", trust_remote_code=True+ )tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)text = "There is a girl who likes adventure,"inputs = tokenizer(text, return_tensors="pt").to(model.device)res = tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0])print(res)- assert ("!!!" not in res)+ assert "!!!" not in resshutil.rmtree(self.save_folder, ignore_errors=True)@require_autogptq@@ -111,33 +110,25 @@ class TestAutoRound(unittest.TestCase):layer_config = {}layer_config["model.decoder.layers.0.self_attn.k_proj"] = {"bits": 8}- layer_config["model.decoder.layers.2.self_attn.q_proj"] = {"bits": 3,- "group_size": 64} ## 3bits when using asym will have some issue+ layer_config["model.decoder.layers.2.self_attn.q_proj"] = {+ "bits": 3,+ "group_size": 64,+ } ## 3bits when using asym will have some issuelayer_config["model.decoder.layers.6.self_attn.out_proj"] = {"bits": 2, "group_size": 32}bits, group_size, sym = 4, 128, True- autoround = AutoRound(- model,- tokenizer,- bits=bits,- group_size=group_size,- sym=sym,- layer_config=layer_config- )+ autoround = AutoRound(model, tokenizer, bits=bits, group_size=group_size, sym=sym, layer_config=layer_config)quantized_model_path = self.save_folderautoround.quantize_and_save(output_dir=quantized_model_path, format="auto_round")quantization_config = AutoRoundConfig(backend="auto")model = AutoModelForCausalLM.from_pretrained(- self.save_folder,- torch_dtype=torch.float16,- device_map="auto",- quantization_config=quantization_config+ self.save_folder, torch_dtype=torch.float16, device_map="auto", quantization_config=quantization_config)tokenizer = AutoTokenizer.from_pretrained(self.save_folder)self.model_infer(model, tokenizer)result = simple_evaluate_user_model(model, tokenizer, batch_size=16, tasks="lambada_openai")- print(result['results']['lambada_openai']['acc,none'])- self.assertGreater(result['results']['lambada_openai']['acc,none'], 0.32)+ print(result["results"]["lambada_openai"]["acc,none"])+ self.assertGreater(result["results"]["lambada_openai"]["acc,none"], 0.32)@require_awqdef test_awq_backend(self):@@ -158,24 +149,18 @@ class TestAutoRound(unittest.TestCase):quantization_config = AutoRoundConfig(backend="auto")model = AutoModelForCausalLM.from_pretrained(- self.save_folder,- torch_dtype=torch.float16,- device_map="auto",- quantization_config=quantization_config+ self.save_folder, torch_dtype=torch.float16, device_map="auto", quantization_config=quantization_config)tokenizer = AutoTokenizer.from_pretrained(self.save_folder)self.model_infer(model, tokenizer)result = simple_evaluate_user_model(model, tokenizer, batch_size=16, tasks="lambada_openai")- print(result['results']['lambada_openai']['acc,none'])- self.assertGreater(result['results']['lambada_openai']['acc,none'], 0.18)+ print(result["results"]["lambada_openai"]["acc,none"])+ self.assertGreater(result["results"]["lambada_openai"]["acc,none"], 0.18)torch.cuda.empty_cache()model = AutoModelForCausalLM.from_pretrained(- self.save_folder,- torch_dtype=torch.bfloat16,- device_map="auto",- quantization_config=quantization_config+ self.save_folder, torch_dtype=torch.bfloat16, device_map="auto", quantization_config=quantization_config)tokenizer = AutoTokenizer.from_pretrained(self.save_folder)@@ -187,10 +172,7 @@ class TestAutoRound(unittest.TestCase):model_name = "OPEA/Meta-Llama-3.1-8B-Instruct-int4-sym-inc"quantization_config = AutoRoundConfig(backend="tritonv2")model = AutoModelForCausalLM.from_pretrained(- model_name,- torch_dtype=torch.bfloat16,- device_map="auto",- quantization_config=quantization_config+ model_name, torch_dtype=torch.bfloat16, device_map="auto", quantization_config=quantization_config)tokenizer = AutoTokenizer.from_pretrained(model_name)@@ -218,16 +200,18 @@ class TestAutoRound(unittest.TestCase):autoround.quantize_and_save(output_dir=quantized_model_path)from auto_round import AutoRoundConfig+quantization_config = AutoRoundConfig(backend="ipex_gptq")- model = AutoModelForCausalLM.from_pretrained(quantized_model_path, device_map="cpu", trust_remote_code=True,- quantization_config=quantization_config)+ model = AutoModelForCausalLM.from_pretrained(+ quantized_model_path, device_map="cpu", trust_remote_code=True, quantization_config=quantization_config+ )tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)text = "There is a girl who likes adventure,"inputs = tokenizer(text, return_tensors="pt").to(model.device)res = tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0])print(res)- assert ("!!!" not in res)+ assert "!!!" not in resmodel = AutoModelForCausalLM.from_pretrained(quantized_model_path, device_map="auto", trust_remote_code=True)tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)@@ -235,16 +219,17 @@ class TestAutoRound(unittest.TestCase):inputs = tokenizer(text, return_tensors="pt").to(model.device)res = tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0])print(res)- assert ("!!!" not in res)+ assert "!!!" not in res- model = AutoModelForCausalLM.from_pretrained(quantized_model_path, device_map="cpu", trust_remote_code=True,- quantization_config=quantization_config)+ model = AutoModelForCausalLM.from_pretrained(+ quantized_model_path, device_map="cpu", trust_remote_code=True, quantization_config=quantization_config+ )tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)text = "There is a girl who likes adventure,"inputs = tokenizer(text, return_tensors="pt").to(model.device)res = tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0])print(res)- assert ("!!!" not in res)+ assert "!!!" not in resshutil.rmtree("./saved", ignore_errors=True)@@ -274,16 +259,17 @@ class TestAutoRound(unittest.TestCase):inputs = tokenizer(text, return_tensors="pt").to(model.device)res = tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0])print(res)- assert ("!!!" not in res)+ assert "!!!" not in res- model = AutoModelForCausalLM.from_pretrained(quantized_model_path, device_map="cpu", trust_remote_code=True,- torch_dtype=torch.bfloat16)+ model = AutoModelForCausalLM.from_pretrained(+ quantized_model_path, device_map="cpu", trust_remote_code=True, torch_dtype=torch.bfloat16+ )tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)text = "There is a girl who likes adventure,"inputs = tokenizer(text, return_tensors="pt").to(model.device)res = tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0])print(res)- assert ("!!!" not in res)+ assert "!!!" not in resshutil.rmtree("./saved", ignore_errors=True)@@ -307,23 +293,28 @@ class TestAutoRound(unittest.TestCase):autoround.quantize_and_save(output_dir=quantized_model_path, format="auto_round")- model = AutoModelForCausalLM.from_pretrained(quantized_model_path, device_map="auto",- trust_remote_code=True)+ model = AutoModelForCausalLM.from_pretrained(+ quantized_model_path, device_map="auto", trust_remote_code=True+ )tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)text = "There is a girl who likes adventure,"inputs = tokenizer(text, return_tensors="pt").to(model.device)res = tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0])print(res)- assert ("!!!" not in res)+ assert "!!!" not in resshutil.rmtree(self.save_folder, ignore_errors=True)@require_greater_than_050def test_load_gptq_model_3bits(self):model_name = "LucasSantiago257/gemma-2b-2bits-gptq"quantization_config = AutoRoundConfig()- model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True,- device_map="auto",- quantization_config=quantization_config)+ model = AutoModelForCausalLM.from_pretrained(+ model_name,+ torch_dtype="auto",+ trust_remote_code=True,+ device_map="auto",+ quantization_config=quantization_config,+ )tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)self.model_infer(model, tokenizer)diff --git a/test/test_cuda/test_calib_dataset.py b/test/test_cuda/test_calib_dataset.pyindex 47da6ab94..b66f60127 100644--- a/test/test_cuda/test_calib_dataset.py+++ b/test/test_cuda/test_calib_dataset.py@@ -27,15 +27,14 @@ class TestLocalCalibDataset(unittest.TestCase):with open(self.jsonl_file, "w") as jsonl_file:for item in jsonl_data:json.dump(item, jsonl_file, ensure_ascii=False)- jsonl_file.write('\n')+ jsonl_file.write("\n")model_name = "facebook/opt-125m"self.model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True)self.tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)def test_combine_dataset(self):- dataset = (- "NeelNanda/pile-10k" + ",BAAI/CCI3-HQ" + ",madao33/new-title-chinese")+ dataset = "NeelNanda/pile-10k" + ",BAAI/CCI3-HQ" + ",madao33/new-title-chinese"bits, group_size, sym = 4, 128, Trueautoround = AutoRound(self.model, self.tokenizer, bits=bits, group_size=group_size, sym=sym, iters=2, seqlen=128, dataset=dataset@@ -45,4 +44,3 @@ class TestLocalCalibDataset(unittest.TestCase):if __name__ == "__main__":unittest.main()-diff --git a/test/test_cuda/test_conv1d.py b/test/test_cuda/test_conv1d.pyindex b8fa0aa43..e617bf55e 100644--- a/test/test_cuda/test_conv1d.py+++ b/test/test_cuda/test_conv1d.py@@ -38,6 +38,7 @@ class TestQuantizationConv1d(unittest.TestCase):self.model = AutoModelForCausalLM.from_pretrained(self.model_name, torch_dtype="auto", trust_remote_code=True)bits, group_size, sym = 4, 128, Truefrom auto_round import AutoRoundConfig+autoround = AutoRound(self.model,self.tokenizer,@@ -47,7 +48,6 @@ class TestQuantizationConv1d(unittest.TestCase):iters=2,seqlen=2,dataset=self.llm_dataloader,-)autoround.quantize()@@ -57,6 +57,5 @@ class TestQuantizationConv1d(unittest.TestCase):model_infer(model, self.tokenizer)-if __name__ == "__main__":unittest.main()diff --git a/test/test_cuda/test_exllamav2_backend.py b/test/test_cuda/test_exllamav2_backend.pyindex 4279d6e57..5c12e0557 100644--- a/test/test_cuda/test_exllamav2_backend.py+++ b/test/test_cuda/test_exllamav2_backend.py@@ -46,11 +46,9 @@ class TestAutoRoundMarlinBackend(unittest.TestCase):input_ids=inputs["input_ids"].to(model.device),attention_mask=inputs["attention_mask"].to(model.device),do_sample=False, ## change this to follow official usage- max_new_tokens=5+ max_new_tokens=5,)- generated_ids = [- output_ids[len(input_ids):] for input_ids, output_ids in zip(inputs["input_ids"], outputs)- ]+ generated_ids = [output_ids[len(input_ids) :] for input_ids, output_ids in zip(inputs["input_ids"], outputs)]decoded_outputs = tokenizer.batch_decode(generated_ids, skip_special_tokens=True)@@ -71,45 +69,32 @@ class TestAutoRoundMarlinBackend(unittest.TestCase):tokenizer = AutoTokenizer.from_pretrained(self.model_name, trust_remote_code=True)bits, group_size, sym = 4, 128, Falseautoround = AutoRound(- model,- tokenizer,- bits=bits,- group_size=group_size,- sym=sym,- iters=1,- seqlen=2,- dataset=self.llm_dataloader+ model, tokenizer, bits=bits, group_size=group_size, sym=sym, iters=1, seqlen=2, dataset=self.llm_dataloader)quantized_model_path = self.save_folderautoround.quantize_and_save(output_dir=quantized_model_path, format="auto_round:gptqmodel")quantization_config = AutoRoundConfig(backend="gptqmodel:exllamav2")model = AutoModelForCausalLM.from_pretrained(- self.save_folder,- torch_dtype=torch.float16,- device_map="auto",- quantization_config=quantization_config+ self.save_folder, torch_dtype=torch.float16, device_map="auto", quantization_config=quantization_config)tokenizer = AutoTokenizer.from_pretrained(self.save_folder)self.model_infer(model, tokenizer)result = simple_evaluate_user_model(model, tokenizer, batch_size=16, tasks="lambada_openai")- print(result['results']['lambada_openai']['acc,none'])- self.assertGreater(result['results']['lambada_openai']['acc,none'], 0.35)+ print(result["results"]["lambada_openai"]["acc,none"])+ self.assertGreater(result["results"]["lambada_openai"]["acc,none"], 0.35)torch.cuda.empty_cache()model = AutoModelForCausalLM.from_pretrained(- self.save_folder,- torch_dtype=torch.bfloat16,- device_map="auto",- quantization_config=quantization_config+ self.save_folder, torch_dtype=torch.bfloat16, device_map="auto", quantization_config=quantization_config)tokenizer = AutoTokenizer.from_pretrained(self.save_folder)self.model_infer(model, tokenizer)result = simple_evaluate_user_model(model, tokenizer, batch_size=16, tasks="lambada_openai")- print(result['results']['lambada_openai']['acc,none'])- self.assertGreater(result['results']['lambada_openai']['acc,none'], 0.35)+ print(result["results"]["lambada_openai"]["acc,none"])+ self.assertGreater(result["results"]["lambada_openai"]["acc,none"], 0.35)torch.cuda.empty_cache()shutil.rmtree("./saved", ignore_errors=True)@@ -133,17 +118,14 @@ class TestAutoRoundMarlinBackend(unittest.TestCase):quantization_config = AutoRoundConfig(backend="gptq:exllamav2") ## or exllamav2model = AutoModelForCausalLM.from_pretrained(- self.save_folder,- torch_dtype=torch.float16,- device_map="auto",- quantization_config=quantization_config+ self.save_folder, torch_dtype=torch.float16, device_map="auto", quantization_config=quantization_config)tokenizer = AutoTokenizer.from_pretrained(self.save_folder)self.model_infer(model, tokenizer)result = simple_evaluate_user_model(model, tokenizer, batch_size=16, tasks="lambada_openai")- print(result['results']['lambada_openai']['acc,none'])- self.assertGreater(result['results']['lambada_openai']['acc,none'], 0.27)+ print(result["results"]["lambada_openai"]["acc,none"])+ self.assertGreater(result["results"]["lambada_openai"]["acc,none"], 0.27)torch.cuda.empty_cache()shutil.rmtree(self.save_folder, ignore_errors=True)@@ -164,22 +146,20 @@ class TestAutoRoundMarlinBackend(unittest.TestCase):sym=sym,)quantized_model_path = self.save_folder- autoround.quantize_and_save(output_dir=quantized_model_path,- format="auto_round") ##will convert to gptq model+ autoround.quantize_and_save(+ output_dir=quantized_model_path, format="auto_round"+ ) ##will convert to gptq modelquantization_config = AutoRoundConfig(backend="gptq:exllamav2") ## or exllamav2model = AutoModelForCausalLM.from_pretrained(- self.save_folder,- torch_dtype=torch.float16,- device_map="auto",- quantization_config=quantization_config+ self.save_folder, torch_dtype=torch.float16, device_map="auto", quantization_config=quantization_config)tokenizer = AutoTokenizer.from_pretrained(self.save_folder)self.model_infer(model, tokenizer)result = simple_evaluate_user_model(model, tokenizer, batch_size=64, tasks="lambada_openai")- print(result['results']['lambada_openai']['acc,none'])- self.assertGreater(result['results']['lambada_openai']['acc,none'], 0.15)+ print(result["results"]["lambada_openai"]["acc,none"])+ self.assertGreater(result["results"]["lambada_openai"]["acc,none"], 0.15)torch.cuda.empty_cache()shutil.rmtree(self.save_folder, ignore_errors=True)diff --git a/test/test_cuda/test_export.py b/test/test_cuda/test_export.pyindex 6dfeb2f1d..36021d3d5 100644--- a/test/test_cuda/test_export.py+++ b/test/test_cuda/test_export.py@@ -57,9 +57,11 @@ class TestAutoRound(unittest.TestCase):text = "There is a girl who likes adventure,"inputs = tokenizer(text, return_tensors="pt").to(model.device)res = tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0])- assert (res == "</s>There is a girl who likes adventure, she is a good friend of mine, she is a good friend of"- " mine, she is a good friend of mine, she is a good friend of mine, she is a good friend of mine, "- "she is a good friend of mine, she is")+ assert (+ res == "</s>There is a girl who likes adventure, she is a good friend of mine, she is a good friend of"+ " mine, she is a good friend of mine, she is a good friend of mine, she is a good friend of mine, "+ "she is a good friend of mine, she is"+ )shutil.rmtree("./saved", ignore_errors=True)@require_optimum@@ -81,7 +83,7 @@ class TestAutoRound(unittest.TestCase):iters=1,seqlen=2,dataset=self.llm_dataloader,- layer_config=layer_config+ layer_config=layer_config,)autoround.quantize()quantized_model_path = "./saved"@@ -118,7 +120,7 @@ class TestAutoRound(unittest.TestCase):iters=1,seqlen=2,dataset=self.llm_dataloader,- layer_config=layer_config+ layer_config=layer_config,)quantized_model_path = "./saved"autoround.quantize_and_save(output_dir=quantized_model_path, format="auto_gptq")@@ -129,22 +131,25 @@ class TestAutoRound(unittest.TestCase):inputs = tokenizer(text, return_tensors="pt").to(model.device)res = tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0])assert (- res == "</s>There is a girl who likes adventure, she is a great artist, she is a great artist,"- " she is a great artist, she is a great artist, she is a great artist, "- "she is a great artist, she is a great artist, she is a great artist, she is")+ res == "</s>There is a girl who likes adventure, she is a great artist, she is a great artist,"+ " she is a great artist, she is a great artist, she is a great artist, "+ "she is a great artist, she is a great artist, she is a great artist, she is"+ )from auto_round import AutoRoundConfig- model = AutoModelForCausalLM.from_pretrained(quantized_model_path, device_map="auto", trust_remote_code=True,- quantization_config=AutoRoundConfig())+ model = AutoModelForCausalLM.from_pretrained(+ quantized_model_path, device_map="auto", trust_remote_code=True, quantization_config=AutoRoundConfig()+ )tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)text = "There is a girl who likes adventure,"inputs = tokenizer(text, return_tensors="pt").to(model.device)g_tokens = model.generate(**inputs, max_new_tokens=50)[0]res = tokenizer.decode(g_tokens)assert (- res == "</s>There is a girl who likes adventure, she is a great artist, she is a great artist,"- " she is a great artist, she is a great artist, she is a great artist, she is a great "- "artist, she is a great artist, she is a great artist, she is")+ res == "</s>There is a girl who likes adventure, she is a great artist, she is a great artist,"+ " she is a great artist, she is a great artist, she is a great artist, she is a great "+ "artist, she is a great artist, she is a great artist, she is"+ )##print(res)shutil.rmtree("./saved", ignore_errors=True)@@ -165,8 +170,7 @@ class TestAutoRound(unittest.TestCase):autoround.quantize()quantized_model_path = "./saved"- autoround.save_quantized(output_dir=quantized_model_path, inplace=False, \- format="auto_round")+ autoround.save_quantized(output_dir=quantized_model_path, inplace=False, format="auto_round")model = AutoModelForCausalLM.from_pretrained(quantized_model_path, device_map="auto")tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)@@ -198,25 +202,28 @@ class TestAutoRound(unittest.TestCase):autoround.quantize()quantized_model_path = "./saved"- autoround.save_quantized(output_dir=quantized_model_path, inplace=False, \- format="auto_awq")+ autoround.save_quantized(output_dir=quantized_model_path, inplace=False, format="auto_awq")model = AutoModelForCausalLM.from_pretrained(quantized_model_path, device_map="auto")tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)text = "There is a girl who likes adventure,"inputs = tokenizer(text, return_tensors="pt").to(model.device)res = tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0])- assert res == ("</s>There is a girl who likes adventure, but she's not a very good one.\nI don't"- " know why you're getting downvoted. I think you're just being a dick.\nI'm not a dick, "- "I just think it's funny that people are downvoting")+ assert res == (+ "</s>There is a girl who likes adventure, but she's not a very good one.\nI don't"+ " know why you're getting downvoted. I think you're just being a dick.\nI'm not a dick, "+ "I just think it's funny that people are downvoting"+ )shutil.rmtree("./saved", ignore_errors=True)@require_optimum@require_awqdef test_autoawq_format_fp_qsave_layers(self):model = AutoModelForCausalLM.from_pretrained(self.model_name, torch_dtype="auto", trust_remote_code=True)- layer_config = {"model.decoder.layers.0.self_attn.k_proj": {"bits": 16},- "model.decoder.layers.9.self_attn.v_proj": {"bits": 16}, }+ layer_config = {+ "model.decoder.layers.0.self_attn.k_proj": {"bits": 16},+ "model.decoder.layers.9.self_attn.v_proj": {"bits": 16},+ }tokenizer = AutoTokenizer.from_pretrained(self.model_name, trust_remote_code=True)bits, group_size, sym = 4, 128, Falseautoround = AutoRound(@@ -228,14 +235,15 @@ class TestAutoRound(unittest.TestCase):iters=1,seqlen=2,dataset=self.llm_dataloader,- layer_config=layer_config+ layer_config=layer_config,)quantized_model_path = "./saved/test_export"- autoround.quantize_and_save(output_dir=quantized_model_path,- format="auto_awq")+ autoround.quantize_and_save(output_dir=quantized_model_path, format="auto_awq")from auto_round import AutoRoundConfig- model = AutoModelForCausalLM.from_pretrained(quantized_model_path, device_map="auto",- quantization_config=AutoRoundConfig())++ model = AutoModelForCausalLM.from_pretrained(+ quantized_model_path, device_map="auto", quantization_config=AutoRoundConfig()+ )tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)text = "There is a girl who likes adventure,"inputs = tokenizer(text, return_tensors="pt").to(model.device)@@ -251,7 +259,6 @@ class TestAutoRound(unittest.TestCase):shutil.rmtree("./saved", ignore_errors=True)-def test_autoround_3bit_asym_torch_format(self):model = AutoModelForCausalLM.from_pretrained(self.model_name, torch_dtype="auto", trust_remote_code=True)tokenizer = AutoTokenizer.from_pretrained(self.model_name, trust_remote_code=True)@@ -271,17 +278,16 @@ class TestAutoRound(unittest.TestCase):autoround.save_quantized(output_dir=quantized_model_path, inplace=False, format="auto_round:gptqmodel")-device = "auto" ##cpu, hpu, cudafrom auto_round import AutoRoundConfig+model = AutoModelForCausalLM.from_pretrained(quantized_model_path, device_map=device)tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)text = "There is a girl who likes adventure,"inputs = tokenizer(text, return_tensors="pt").to(model.device)print(tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0]))shutil.rmtree("./saved", ignore_errors=True)--+def test_autoround_3bit_sym_torch_format(self):model = AutoModelForCausalLM.from_pretrained(self.model_name, torch_dtype="auto", trust_remote_code=True)tokenizer = AutoTokenizer.from_pretrained(self.model_name, trust_remote_code=True)@@ -301,14 +307,13 @@ class TestAutoRound(unittest.TestCase):autoround.save_quantized(output_dir=quantized_model_path, inplace=False, format="auto_round")-device = "auto" ##cpu, hpu, cudafrom auto_round import AutoRoundConfig- quantization_config = AutoRoundConfig(- backend=device++ quantization_config = AutoRoundConfig(backend=device)+ model = AutoModelForCausalLM.from_pretrained(+ quantized_model_path, device_map=device, quantization_config=quantization_config)- model = AutoModelForCausalLM.from_pretrained(quantized_model_path, device_map=device,- quantization_config=quantization_config)tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)text = "There is a girl who likes adventure,"inputs = tokenizer(text, return_tensors="pt").to(model.device)@@ -317,4 +322,4 @@ class TestAutoRound(unittest.TestCase):if __name__ == "__main__":- unittest.main()\ No newline at end of file+ unittest.main()diff --git a/test/test_cuda/test_get_block_name.py b/test/test_cuda/test_get_block_name.pyindex f796970d3..8c26ea51b 100644--- a/test/test_cuda/test_get_block_name.py+++ b/test/test_cuda/test_get_block_name.py@@ -116,9 +116,15 @@ class TestAutoRound(unittest.TestCase):self.check_block_names(block_names, ["language_model.model.layers"], [40])block_names = get_block_names(model, quant_vision=True)- self.check_block_names(block_names,- ["vision_model.transformer.layers", "vision_model.global_transformer.layers",- "language_model.model.layers"], [32, 8, 40])+ self.check_block_names(+ block_names,+ [+ "vision_model.transformer.layers",+ "vision_model.global_transformer.layers",+ "language_model.model.layers",+ ],+ [32, 8, 40],+ )assert not is_pure_text_model(model)@@ -139,8 +145,9 @@ class TestAutoRound(unittest.TestCase):self.check_block_names(block_names, ["transformer.encoder.layers"], [40])block_names = get_block_names(model, quant_vision=True)- self.check_block_names(block_names, ["transformer.encoder.layers", "transformer.vision.transformer.layers"],- [40, 63])+ self.check_block_names(+ block_names, ["transformer.encoder.layers", "transformer.vision.transformer.layers"], [40, 63]+ )assert not is_pure_text_model(model)def test_gemma3(self):@@ -150,8 +157,9 @@ class TestAutoRound(unittest.TestCase):self.check_block_names(block_names, ["language_model.model.layers"], [48])block_names = get_block_names(model, quant_vision=True)- self.check_block_names(block_names, ["vision_tower.vision_model.encoder.layers", "language_model.model.layers"],- [27, 48])+ self.check_block_names(+ block_names, ["vision_tower.vision_model.encoder.layers", "language_model.model.layers"], [27, 48]+ )assert not is_pure_text_model(model)def test_Mistral3(self):@@ -161,8 +169,9 @@ class TestAutoRound(unittest.TestCase):self.check_block_names(block_names, ["language_model.model.layers"], [40])block_names = get_block_names(model, quant_vision=True)- self.check_block_names(block_names, ["vision_tower.transformer.layers", "language_model.model.layers"],- [24, 40])+ self.check_block_names(+ block_names, ["vision_tower.transformer.layers", "language_model.model.layers"], [24, 40]+ )assert not is_pure_text_model(model)def test_Molmo(self):@@ -172,9 +181,9 @@ class TestAutoRound(unittest.TestCase):self.check_block_names(block_names, ["model.transformer.blocks"], [28])block_names = get_block_names(model, quant_vision=True)- self.check_block_names(block_names,- ["model.transformer.blocks", "model.vision_backbone.image_vit.transformer.resblocks"],- [28, 23])+ self.check_block_names(+ block_names, ["model.transformer.blocks", "model.vision_backbone.image_vit.transformer.resblocks"], [28, 23]+ )assert not is_pure_text_model(model)diff --git a/test/test_cuda/test_gguf.py b/test/test_cuda/test_gguf.pyindex 6faf31eb4..b81f9f764 100644--- a/test/test_cuda/test_gguf.py+++ b/test/test_cuda/test_gguf.py@@ -49,16 +49,17 @@ class TestAutoRound(unittest.TestCase):dataset=self.llm_dataloader,)autoround.quantize()- quantized_model_path = "./saved"+ quantized_model_path = "./saved"autoround.save_quantized(output_dir=quantized_model_path, format="gguf:q4_1")from llama_cpp import Llama+gguf_file = os.listdir("saved")[0]llm = Llama(f"saved/{gguf_file}", n_gpu_layers=-1)output = llm("There is a girl who likes adventure,", max_tokens=32)print(output)shutil.rmtree("./saved", ignore_errors=True)-+save_dir = os.path.join(os.path.dirname(__file__), "saved")model_path = "Qwen/Qwen2.5-0.5B-Instruct"res = os.system(@@ -67,14 +68,15 @@ class TestAutoRound(unittest.TestCase):)print(save_dir)self.assertFalse(res > 0 or res == -1, msg="qwen2 tuning fail")-+from llama_cpp import Llama+gguf_file = os.listdir("saved/Qwen2.5-0.5B-Instruct-gguf")[0]llm = Llama(f"saved/Qwen2.5-0.5B-Instruct-gguf/{gguf_file}", n_gpu_layers=-1)output = llm("There is a girl who likes adventure,", max_tokens=32)print(output)shutil.rmtree("./saved", ignore_errors=True)-+@require_ggufdef test_q2_k_export(self):bits, group_size, sym = 2, 16, False@@ -90,7 +92,7 @@ class TestAutoRound(unittest.TestCase):iters=1,seqlen=1,dataset=self.llm_dataloader,- data_type="int_asym_dq"+ data_type="int_asym_dq",)autoround.quantize()quantized_model_path = "./saved"@@ -104,9 +106,10 @@ class TestAutoRound(unittest.TestCase):print(result)from auto_round.eval.evaluation import simple_evaluate_user_model+result = simple_evaluate_user_model(model, self.tokenizer, batch_size=16, tasks="piqa")- self.assertGreater(result['results']['piqa']['acc,none'], 0.45)-+ self.assertGreater(result["results"]["piqa"]["acc,none"], 0.45)+shutil.rmtree("./saved", ignore_errors=True)@require_gguf@@ -124,13 +127,7 @@ class TestAutoRound(unittest.TestCase):def test_q4_0(self):bits, group_size, sym = 4, 32, Trueautoround = AutoRound(- self.model,- self.tokenizer,- bits=bits,- group_size=group_size,- sym=sym,- iters=1,- data_type="int"+ self.model, self.tokenizer, bits=bits, group_size=group_size, sym=sym, iters=1, data_type="int")autoround.quantize()quantized_model_path = "./saved"@@ -143,21 +140,16 @@ class TestAutoRound(unittest.TestCase):print(self.tokenizer.decode(model.generate(**inputs, max_new_tokens=10)[0]))from auto_round.eval.evaluation import simple_evaluate_user_model+result = simple_evaluate_user_model(model, self.tokenizer, batch_size=16, tasks="piqa")- self.assertAlmostEqual(result['results']['piqa']['acc,none'], 0.55, delta=0.01)+ self.assertAlmostEqual(result["results"]["piqa"]["acc,none"], 0.55, delta=0.01)shutil.rmtree("./saved", ignore_errors=True)-+@require_ggufdef test_q4_1(self):bits, group_size, sym = 4, 32, Falseautoround = AutoRound(- self.model,- self.tokenizer,- bits=bits,- group_size=group_size,- sym=sym,- iters=1,- data_type="int"+ self.model, self.tokenizer, bits=bits, group_size=group_size, sym=sym, iters=1, data_type="int")autoround.quantize()quantized_model_path = "./saved"@@ -170,13 +162,15 @@ class TestAutoRound(unittest.TestCase):print(self.tokenizer.decode(model.generate(**inputs, max_new_tokens=10)[0]))from auto_round.eval.evaluation import simple_evaluate_user_model+result = simple_evaluate_user_model(model, self.tokenizer, batch_size=16, tasks="piqa")- self.assertAlmostEqual(result['results']['piqa']['acc,none'], 0.55, delta=0.01)+ self.assertAlmostEqual(result["results"]["piqa"]["acc,none"], 0.55, delta=0.01)shutil.rmtree("./saved", ignore_errors=True)-+@require_ggufdef test_all_format(self):from auto_round.export.export_to_gguf.config import GGUF_CONFIG+python_path = sys.executablefor model_name in ["/models/Qwen3-8B/", "/models/Llama-3.2-3B/", "/models/Meta-Llama-3.1-8B-Instruct"]:for gguf_format in GGUF_CONFIG.keys():@@ -196,12 +190,13 @@ class TestAutoRound(unittest.TestCase):if res > 0 or res == -1:assert False, "cmd line test fail, please have a check"shutil.rmtree("../../tmp_autoround", ignore_errors=True)-+@require_ggufdef test_vlm_gguf(self):model_name = "/models/Qwen2.5-VL-7B-Instruct"from auto_round.mllm.autoround_mllm import AutoRoundMLLMfrom auto_round.utils import mllm_load_model+model, processor, tokenizer, image_processor = mllm_load_model(model_name)autoround = AutoRoundMLLM(model,@@ -214,15 +209,16 @@ class TestAutoRound(unittest.TestCase):quantized_model_path = "./saved"autoround.quantize_and_save(output_dir=quantized_model_path, format="gguf:q4_0")self.assertTrue("mmproj-model.gguf" in os.listdir("./saved"))- file_size = os.path.getsize("./saved/Qwen2.5-VL-7B-Instruct-7.6B-Q4_0.gguf") / 1024 ** 2+ file_size = os.path.getsize("./saved/Qwen2.5-VL-7B-Instruct-7.6B-Q4_0.gguf") / 1024**2self.assertAlmostEqual(file_size, 4226, delta=1.0)- file_size = os.path.getsize("./saved/mmproj-model.gguf") / 1024 ** 2+ file_size = os.path.getsize("./saved/mmproj-model.gguf") / 1024**2self.assertAlmostEqual(file_size, 2578, delta=1.0)shutil.rmtree("./saved", ignore_errors=True)-+model_name = "/models/gemma-3-12b-it"from auto_round.mllm.autoround_mllm import AutoRoundMLLMfrom auto_round.utils import mllm_load_model+model, processor, tokenizer, image_processor = mllm_load_model(model_name)autoround = AutoRoundMLLM(model,@@ -236,17 +232,18 @@ class TestAutoRound(unittest.TestCase):quantized_model_path = "./saved"autoround.quantize_and_save(output_dir=quantized_model_path, format="gguf:q4_k_m")self.assertTrue("mmproj-model.gguf" in os.listdir("./saved"))- file_size = os.path.getsize("./saved/gemma-3-12b-it-12B-Q4_K_M.gguf") / 1024 ** 2+ file_size = os.path.getsize("./saved/gemma-3-12b-it-12B-Q4_K_M.gguf") / 1024**2self.assertAlmostEqual(file_size, 6568, delta=1.0)- file_size = os.path.getsize("./saved/mmproj-model.gguf") / 1024 ** 2+ file_size = os.path.getsize("./saved/mmproj-model.gguf") / 1024**2self.assertAlmostEqual(file_size, 1599, delta=1.0)shutil.rmtree("./saved", ignore_errors=True)-+@require_ggufdef test_llama_4(self):model_name = "/dataset/Llama-4-Scout-17B-16E-Instruct/"from auto_round.mllm.autoround_mllm import AutoRoundMLLMfrom auto_round.utils import mllm_load_model+model, processor, tokenizer, image_processor = mllm_load_model(model_name, use_auto_mapping=False)autoround = AutoRoundMLLM(model,@@ -259,11 +256,15 @@ class TestAutoRound(unittest.TestCase):quantized_model_path = "/dataset/Llam-4-test"autoround.quantize_and_save(output_dir=quantized_model_path, format="gguf:q4_0")self.assertTrue("mmproj-model.gguf" in os.listdir(quantized_model_path))- file_size = os.path.getsize(os.path.join(quantized_model_path, "Llama-4-Scout-17B-16E-Instruct-16x17B-Q4_0.gguf")) / 1024 ** 2+ file_size = (+ os.path.getsize(os.path.join(quantized_model_path, "Llama-4-Scout-17B-16E-Instruct-16x17B-Q4_0.gguf"))+ / 1024**2+ )self.assertAlmostEqual(file_size, 58093.62, delta=1.0)- file_size = os.path.getsize(os.path.join(quantized_model_path, "mmproj-model.gguf")) / 1024 ** 2+ file_size = os.path.getsize(os.path.join(quantized_model_path, "mmproj-model.gguf")) / 1024**2self.assertAlmostEqual(file_size, 3323.52, delta=1.0)shutil.rmtree(quantized_model_path, ignore_errors=True)-++if __name__ == "__main__":- unittest.main()\ No newline at end of file+ unittest.main()diff --git a/test/test_cuda/test_main_func.py b/test/test_cuda/test_main_func.pyindex d32faf92a..6983dbe91 100644--- a/test/test_cuda/test_main_func.py+++ b/test/test_cuda/test_main_func.py@@ -16,7 +16,7 @@ from auto_round.testing_utils import require_awq, require_gptqmodel, require_optdef get_accuracy(data):- match = re.search(r'\|acc\s+\|[↑↓]\s+\|\s+([\d.]+)\|', data)+ match = re.search(r"\|acc\s+\|[↑↓]\s+\|\s+([\d.]+)\|", data)if match:accuracy = float(match.group(1))@@ -49,9 +49,7 @@ class TestMainFunc(unittest.TestCase):##test auto_round formatautoround.save_quantized(self.save_dir, format="auto_round", inplace=False)model_args = f"pretrained={self.save_dir}"- res = simple_evaluate(model="hf", model_args=model_args,- tasks=self.tasks,- batch_size="auto")+ res = simple_evaluate(model="hf", model_args=model_args, tasks=self.tasks, batch_size="auto")res = make_table(res)accuracy = get_accuracy(res)assert accuracy > 0.35@@ -60,9 +58,7 @@ class TestMainFunc(unittest.TestCase):##test auto_gptq formatautoround.save_quantized(self.save_dir, format="auto_gptq", inplace=False)model_args = f"pretrained={self.save_dir}"- res = simple_evaluate(model="hf", model_args=model_args,- tasks=self.tasks,- batch_size="auto")+ res = simple_evaluate(model="hf", model_args=model_args, tasks=self.tasks, batch_size="auto")res = make_table(res)accuracy = get_accuracy(res)assert accuracy > 0.35@@ -71,9 +67,7 @@ class TestMainFunc(unittest.TestCase):##test auto_awq formatautoround.save_quantized(self.save_dir, format="auto_awq", inplace=False)model_args = f"pretrained={self.save_dir}"- res = simple_evaluate(model="hf", model_args=model_args,- tasks=self.tasks,- batch_size="auto")+ res = simple_evaluate(model="hf", model_args=model_args, tasks=self.tasks, batch_size="auto")res = make_table(res)accuracy = get_accuracy(res)assert accuracy > 0.35@@ -87,6 +81,7 @@ class TestMainFunc(unittest.TestCase):model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype=torch.float16, device_map="auto")tokenizer = AutoTokenizer.from_pretrained(model_name)from auto_round.utils import get_fp_layer_names+layer_names = get_fp_layer_names(model, "model.decoder.layers.0,model.decoder.layers.1")layer_configs = {}for name in layer_names:@@ -97,9 +92,7 @@ class TestMainFunc(unittest.TestCase):##test auto_round formatautoround.save_quantized(self.save_dir, format="auto_round", inplace=False)model_args = f"pretrained={self.save_dir}"- res = simple_evaluate(model="hf", model_args=model_args,- tasks=self.tasks,- batch_size="auto")+ res = simple_evaluate(model="hf", model_args=model_args, tasks=self.tasks, batch_size="auto")res = make_table(res)accuracy = get_accuracy(res)assert accuracy > 0.35@@ -108,9 +101,7 @@ class TestMainFunc(unittest.TestCase):##test auto_awq formatautoround.save_quantized(self.save_dir, format="auto_awq", inplace=False)model_args = f"pretrained={self.save_dir}"- res = simple_evaluate(model="hf", model_args=model_args,- tasks=self.tasks,- batch_size="auto")+ res = simple_evaluate(model="hf", model_args=model_args, tasks=self.tasks, batch_size="auto")res = make_table(res)accuracy = get_accuracy(res)assert accuracy > 0.35@@ -136,15 +127,13 @@ class TestMainFunc(unittest.TestCase):##test auto_round formatautoround.save_quantized(self.save_dir, format="auto_round", inplace=False)model_args = f"pretrained={self.save_dir}"- res = simple_evaluate(model="hf", model_args=model_args,- tasks=self.tasks,- batch_size="auto")+ res = simple_evaluate(model="hf", model_args=model_args, tasks=self.tasks, batch_size="auto")res = make_table(res)accuracy = get_accuracy(res)assert accuracy > 0.34shutil.rmtree("./saved", ignore_errors=True)- def test_autoround_asym(self): ##need to install false+ def test_autoround_asym(self): ##need to install falsetry:from autoround_exllamav2_kernels import gemm_half_q_half, make_q_matrixexcept ImportError as e:@@ -159,16 +148,12 @@ class TestMainFunc(unittest.TestCase):##test auto_round formatautoround.save_quantized(self.save_dir, format="auto_round", inplace=False)model_args = f"pretrained={self.save_dir}"- res = simple_evaluate(model="hf", model_args=model_args,- tasks=self.tasks,- batch_size="auto")+ res = simple_evaluate(model="hf", model_args=model_args, tasks=self.tasks, batch_size="auto")res = make_table(res)accuracy = get_accuracy(res)assert accuracy > 0.35shutil.rmtree("./saved", ignore_errors=True)--if __name__ == "__main__":- unittest.main()\ No newline at end of file+ unittest.main()diff --git a/test/test_cuda/test_marlin_backend.py b/test/test_cuda/test_marlin_backend.pyindex f398029c2..26d3ddca2 100644--- a/test/test_cuda/test_marlin_backend.py+++ b/test/test_cuda/test_marlin_backend.py@@ -44,17 +44,14 @@ class TestAutoRoundMarlinBackend(unittest.TestCase):quantization_config = AutoRoundConfig(backend="marlin")model = AutoModelForCausalLM.from_pretrained(- self.save_folder,- torch_dtype=torch.float16,- device_map="auto",- quantization_config=quantization_config+ self.save_folder, torch_dtype=torch.float16, device_map="auto", quantization_config=quantization_config)tokenizer = AutoTokenizer.from_pretrained(self.save_folder)self.model_infer(model, tokenizer)result = simple_evaluate_user_model(model, tokenizer, batch_size=16, tasks="lambada_openai")- print(result['results']['lambada_openai']['acc,none'])- self.assertGreater(result['results']['lambada_openai']['acc,none'], 0.14)+ print(result["results"]["lambada_openai"]["acc,none"])+ self.assertGreater(result["results"]["lambada_openai"]["acc,none"], 0.14)for group_size in [32, 128]:print(f"{group_size}!!!!!!!!!!!!!!!!!")@@ -76,17 +73,14 @@ class TestAutoRoundMarlinBackend(unittest.TestCase):quantization_config = AutoRoundConfig(backend="marlin")model = AutoModelForCausalLM.from_pretrained(- self.save_folder,- torch_dtype=torch.float16,- device_map="auto",- quantization_config=quantization_config+ self.save_folder, torch_dtype=torch.float16, device_map="auto", quantization_config=quantization_config)tokenizer = AutoTokenizer.from_pretrained(self.save_folder)self.model_infer(model, tokenizer)result = simple_evaluate_user_model(model, tokenizer, batch_size=16, tasks="lambada_openai")- print(result['results']['lambada_openai']['acc,none'])- self.assertGreater(result['results']['lambada_openai']['acc,none'], 0.14)+ print(result["results"]["lambada_openai"]["acc,none"])+ self.assertGreater(result["results"]["lambada_openai"]["acc,none"], 0.14)@classmethoddef setUpClass(self):@@ -108,11 +102,9 @@ class TestAutoRoundMarlinBackend(unittest.TestCase):input_ids=inputs["input_ids"].to(model.device),attention_mask=inputs["attention_mask"].to(model.device),do_sample=False, ## change this to follow official usage- max_new_tokens=5+ max_new_tokens=5,)- generated_ids = [- output_ids[len(input_ids):] for input_ids, output_ids in zip(inputs["input_ids"], outputs)- ]+ generated_ids = [output_ids[len(input_ids) :] for input_ids, output_ids in zip(inputs["input_ids"], outputs)]decoded_outputs = tokenizer.batch_decode(generated_ids, skip_special_tokens=True)@@ -146,31 +138,25 @@ class TestAutoRoundMarlinBackend(unittest.TestCase):quantization_config = AutoRoundConfig(backend="marlin")model = AutoModelForCausalLM.from_pretrained(- self.save_folder,- torch_dtype=torch.float16,- device_map="auto",- quantization_config=quantization_config+ self.save_folder, torch_dtype=torch.float16, device_map="auto", quantization_config=quantization_config)tokenizer = AutoTokenizer.from_pretrained(self.save_folder)self.model_infer(model, tokenizer)result = simple_evaluate_user_model(model, tokenizer, batch_size=16, tasks="lambada_openai")- print(result['results']['lambada_openai']['acc,none'])- self.assertGreater(result['results']['lambada_openai']['acc,none'], 0.27)+ print(result["results"]["lambada_openai"]["acc,none"])+ self.assertGreater(result["results"]["lambada_openai"]["acc,none"], 0.27)torch.cuda.empty_cache()model = AutoModelForCausalLM.from_pretrained(- self.save_folder,- torch_dtype=torch.bfloat16,- device_map="auto",- quantization_config=quantization_config+ self.save_folder, torch_dtype=torch.bfloat16, device_map="auto", quantization_config=quantization_config)tokenizer = AutoTokenizer.from_pretrained(self.save_folder)self.model_infer(model, tokenizer)result = simple_evaluate_user_model(model, tokenizer, batch_size=16, tasks="lambada_openai")- print(result['results']['lambada_openai']['acc,none'])- self.assertGreater(result['results']['lambada_openai']['acc,none'], 0.27)+ print(result["results"]["lambada_openai"]["acc,none"])+ self.assertGreater(result["results"]["lambada_openai"]["acc,none"], 0.27)torch.cuda.empty_cache()shutil.rmtree("./saved", ignore_errors=True)diff --git a/test/test_cuda/test_multiple_card.py b/test/test_cuda/test_multiple_card.pyindex 1710c1660..db31acbea 100644--- a/test/test_cuda/test_multiple_card.py+++ b/test/test_cuda/test_multiple_card.py@@ -16,7 +16,7 @@ from auto_round.testing_utils import multi_card, require_gptqmodel, require_greadef get_accuracy(data):- match = re.search(r'\|acc\s+\|[↑↓]\s+\|\s+([\d.]+)\|', data)+ match = re.search(r"\|acc\s+\|[↑↓]\s+\|\s+([\d.]+)\|", data)if match:accuracy = float(match.group(1))@@ -44,18 +44,16 @@ class TestAutoRound(unittest.TestCase):model_name = "/models/Qwen2-0.5B-Instruct"model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype=torch.float16)tokenizer = AutoTokenizer.from_pretrained(model_name)- device_map = '.*q_proj:0,.*k_proj:cuda:0,v_proj:1,.*up_proj:1'- autoround = AutoRound(model, tokenizer,device_map=device_map)+ device_map = ".*q_proj:0,.*k_proj:cuda:0,v_proj:1,.*up_proj:1"+ autoround = AutoRound(model, tokenizer, device_map=device_map)autoround.quantize()autoround.save_quantized(self.save_dir, format="auto_round", inplace=False)model_args = f"pretrained={self.save_dir}"- res = simple_evaluate(model="hf", model_args=model_args,- tasks=self.tasks,- batch_size="auto")+ res = simple_evaluate(model="hf", model_args=model_args, tasks=self.tasks, batch_size="auto")res = make_table(res)accuracy = get_accuracy(res)print(accuracy)- assert accuracy > 0.45 ##0.4786+ assert accuracy > 0.45 ##0.4786shutil.rmtree("./saved", ignore_errors=True)@multi_card@@ -64,8 +62,9 @@ class TestAutoRound(unittest.TestCase):model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype=torch.float16)tokenizer = AutoTokenizer.from_pretrained(model_name)device_map = {"norm": "cuda:1"}- autoround = AutoRound(model, tokenizer, iters=2, device_map=device_map, nsamples=7, seqlen=32,- enable_norm_bias_tuning=True)+ autoround = AutoRound(+ model, tokenizer, iters=2, device_map=device_map, nsamples=7, seqlen=32, enable_norm_bias_tuning=True+ )autoround.quantize()@multi_card@@ -74,8 +73,9 @@ class TestAutoRound(unittest.TestCase):model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype=torch.float16)tokenizer = AutoTokenizer.from_pretrained(model_name)device_map = {"norm": "cuda:1"}- autoround = AutoRound(model, tokenizer, iters=2, device_map=device_map, nsamples=7, seqlen=32,- enable_norm_bias_tuning=True)+ autoround = AutoRound(+ model, tokenizer, iters=2, device_map=device_map, nsamples=7, seqlen=32, enable_norm_bias_tuning=True+ )autoround.quantize()@multi_card@@ -83,8 +83,10 @@ class TestAutoRound(unittest.TestCase):model_name = "/models/Qwen2-0.5B-Instruct"model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype=torch.float16)tokenizer = AutoTokenizer.from_pretrained(model_name)- device_map = {".*q_proj": '0', ".*k_proj": "cuda:1", "v_proj": 1, ".*up_proj": "1"}- autoround = AutoRound(model, tokenizer, iters=2, device_map=device_map, nsamples=7,seqlen=32,act_bits=4,act_dynamic=False)+ device_map = {".*q_proj": "0", ".*k_proj": "cuda:1", "v_proj": 1, ".*up_proj": "1"}+ autoround = AutoRound(+ model, tokenizer, iters=2, device_map=device_map, nsamples=7, seqlen=32, act_bits=4, act_dynamic=False+ )autoround.quantize()@multi_card@@ -92,10 +94,18 @@ class TestAutoRound(unittest.TestCase):model_name = "/models/Qwen2.5-7B-Instruct"model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype=torch.float16)tokenizer = AutoTokenizer.from_pretrained(model_name)- device_map = {".*q_proj": '0', ".*k_proj": "cuda:1", "v_proj": 1, ".*up_proj": "1","lm_head":1}- layer_config={"lm_head": {"bits": 4}}- autoround = AutoRound(model, tokenizer, iters=2, device_map=device_map, nsamples=7, seqlen=32,- enable_norm_bias_tuning=True,layer_config=layer_config)+ device_map = {".*q_proj": "0", ".*k_proj": "cuda:1", "v_proj": 1, ".*up_proj": "1", "lm_head": 1}+ layer_config = {"lm_head": {"bits": 4}}+ autoround = AutoRound(+ model,+ tokenizer,+ iters=2,+ device_map=device_map,+ nsamples=7,+ seqlen=32,+ enable_norm_bias_tuning=True,+ layer_config=layer_config,+ )autoround.quantize()@multi_card@@ -103,8 +113,8 @@ class TestAutoRound(unittest.TestCase):model_name = "/models/Qwen2-0.5B-Instruct"model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype=torch.float16)tokenizer = AutoTokenizer.from_pretrained(model_name)- device_map = {".*q_proj": '0', ".*k_proj": "cuda:1", "v_proj": 1, ".*up_proj": "cpu"}- autoround = AutoRound(model, tokenizer, iters=2, device_map=device_map, nsamples=7,seqlen=32)+ device_map = {".*q_proj": "0", ".*k_proj": "cuda:1", "v_proj": 1, ".*up_proj": "cpu"}+ autoround = AutoRound(model, tokenizer, iters=2, device_map=device_map, nsamples=7, seqlen=32)autoround.quantize()model_name = "OPEA/Meta-Llama-3.1-8B-Instruct-int4-sym-inc"@@ -147,14 +157,20 @@ class TestAutoRound(unittest.TestCase):device_map1["model.rotary_emb"] = "cuda"device_map1["model.embed_tokens"] = "cuda"- for tmp_device_map in [device_map1, device_map, None, 0, "balanced", "balanced_low_0", "sequential", "cpu",- "cuda:0", "cuda", "auto",- ]:- model = AutoModelForCausalLM.from_pretrained(- model_name,- torch_dtype="auto",- device_map=tmp_device_map- )+ for tmp_device_map in [+ device_map1,+ device_map,+ None,+ 0,+ "balanced",+ "balanced_low_0",+ "sequential",+ "cpu",+ "cuda:0",+ "cuda",+ "auto",+ ]:+ model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto", device_map=tmp_device_map)tokenizer = AutoTokenizer.from_pretrained(model_name)@@ -167,14 +183,8 @@ class TestAutoRound(unittest.TestCase):texts = []for prompt in prompts:- messages = [- {"role": "user", "content": prompt}- ]- text = tokenizer.apply_chat_template(- messages,- tokenize=False,- add_generation_prompt=True- )+ messages = [{"role": "user", "content": prompt}]+ text = tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)texts.append(text)inputs = tokenizer(texts, return_tensors="pt", padding=False, truncation=True)@@ -183,10 +193,10 @@ class TestAutoRound(unittest.TestCase):input_ids=inputs["input_ids"].to(model.device),attention_mask=inputs["attention_mask"].to(model.device),do_sample=False, ## change this to follow official usage- max_new_tokens=5+ max_new_tokens=5,)generated_ids = [- output_ids[len(input_ids):] for input_ids, output_ids in zip(inputs["input_ids"], outputs)+ output_ids[len(input_ids) :] for input_ids, output_ids in zip(inputs["input_ids"], outputs)]decoded_outputs = tokenizer.batch_decode(generated_ids, skip_special_tokens=True)@@ -203,6 +213,7 @@ class TestAutoRound(unittest.TestCase):@require_greater_than_050def test_device_map_for_triton(self):from transformers import AutoModelForCausalLM, AutoTokenizer+model_name = "OPEA/Meta-Llama-3.1-8B-Instruct-int4-sym-inc"device_map = {}@@ -243,16 +254,21 @@ class TestAutoRound(unittest.TestCase):device_map1["model.rotary_emb"] = "cuda"device_map1["model.embed_tokens"] = "cuda"from auto_round import AutoRoundConfig+quantization_config = AutoRoundConfig(backend="tritonv2")- for tmp_device_map in [device_map1, 0, "balanced", "balanced_low_0", "sequential",- "cuda:0", "cuda", "auto",- ]:+ for tmp_device_map in [+ device_map1,+ 0,+ "balanced",+ "balanced_low_0",+ "sequential",+ "cuda:0",+ "cuda",+ "auto",+ ]:model = AutoModelForCausalLM.from_pretrained(- model_name,- torch_dtype="auto",- device_map=tmp_device_map,- quantization_config=quantization_config+ model_name, torch_dtype="auto", device_map=tmp_device_map, quantization_config=quantization_config)tokenizer = AutoTokenizer.from_pretrained(model_name)@@ -266,14 +282,8 @@ class TestAutoRound(unittest.TestCase):texts = []for prompt in prompts:- messages = [- {"role": "user", "content": prompt}- ]- text = tokenizer.apply_chat_template(- messages,- tokenize=False,- add_generation_prompt=True- )+ messages = [{"role": "user", "content": prompt}]+ text = tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)texts.append(text)inputs = tokenizer(texts, return_tensors="pt", padding=False, truncation=True)@@ -282,10 +292,10 @@ class TestAutoRound(unittest.TestCase):input_ids=inputs["input_ids"].to(model.device),attention_mask=inputs["attention_mask"].to(model.device),do_sample=False, ## change this to follow official usage- max_new_tokens=5+ max_new_tokens=5,)generated_ids = [- output_ids[len(input_ids):] for input_ids, output_ids in zip(inputs["input_ids"], outputs)+ output_ids[len(input_ids) :] for input_ids, output_ids in zip(inputs["input_ids"], outputs)]decoded_outputs = tokenizer.batch_decode(generated_ids, skip_special_tokens=True)@@ -298,5 +308,6 @@ class TestAutoRound(unittest.TestCase):del modeltorch.cuda.empty_cache()+if __name__ == "__main__":unittest.main()diff --git a/test/test_cuda/test_multiple_card_calib.py b/test/test_cuda/test_multiple_card_calib.pyindex 8f3b584bc..69caf6a8b 100644--- a/test/test_cuda/test_multiple_card_calib.py+++ b/test/test_cuda/test_multiple_card_calib.py@@ -10,7 +10,7 @@ from auto_round.testing_utils import multi_carddef get_accuracy(data):- match = re.search(r'\|acc\s+\|[↑↓]\s+\|\s+([\d.]+)\|', data)+ match = re.search(r"\|acc\s+\|[↑↓]\s+\|\s+([\d.]+)\|", data)if match:accuracy = float(match.group(1))@@ -36,12 +36,11 @@ class TestAutoRound(unittest.TestCase):##test llm scriptres = os.system(- f"cd ../.. && {python_path} -m auto_round --model /models/Meta-Llama-3.1-8B-Instruct --devices '0,1' --quant_lm_head --disable_eval --iters 1 --nsamples 1 --output_dir None")+ f"cd ../.. && {python_path} -m auto_round --model /models/Meta-Llama-3.1-8B-Instruct --devices '0,1' --quant_lm_head --disable_eval --iters 1 --nsamples 1 --output_dir None"+ )if res > 0 or res == -1:assert False, "cmd line test fail, please have a check"if __name__ == "__main__":unittest.main()--diff --git a/test/test_cuda/test_qbits.py b/test/test_cuda/test_qbits.pyindex 4af9fb358..d73d474d6 100644--- a/test/test_cuda/test_qbits.py+++ b/test/test_cuda/test_qbits.py@@ -30,11 +30,9 @@ class TestAutoRound(unittest.TestCase):input_ids=inputs["input_ids"].to(model.device),attention_mask=inputs["attention_mask"].to(model.device),do_sample=False, ## change this to follow official usage- max_new_tokens=5+ max_new_tokens=5,)- generated_ids = [- output_ids[len(input_ids):] for input_ids, output_ids in zip(inputs["input_ids"], outputs)- ]+ generated_ids = [output_ids[len(input_ids) :] for input_ids, output_ids in zip(inputs["input_ids"], outputs)]decoded_outputs = tokenizer.batch_decode(generated_ids, skip_special_tokens=True)@@ -47,15 +45,18 @@ class TestAutoRound(unittest.TestCase):def tearDownClass(self):shutil.rmtree("runs", ignore_errors=True)-## require torch 2.6@require_itrexdef test_load_gptq_model_8bits(self):model_name = "acloudfan/opt-125m-gptq-8bit"quantization_config = AutoRoundConfig()- model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True,- device_map="cpu",- quantization_config=quantization_config)+ model = AutoModelForCausalLM.from_pretrained(+ model_name,+ torch_dtype="auto",+ trust_remote_code=True,+ device_map="cpu",+ quantization_config=quantization_config,+ )tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)self.model_infer(model, tokenizer)@@ -63,9 +64,13 @@ class TestAutoRound(unittest.TestCase):def test_load_gptq_model_2bits(self):model_name = "LucasSantiago257/gemma-2b-2bits-gptq"quantization_config = AutoRoundConfig()- model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True,- device_map="cpu",- quantization_config=quantization_config)+ model = AutoModelForCausalLM.from_pretrained(+ model_name,+ torch_dtype="auto",+ trust_remote_code=True,+ device_map="cpu",+ quantization_config=quantization_config,+ )tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)self.model_infer(model, tokenizer)@@ -81,15 +86,9 @@ class TestAutoRound(unittest.TestCase):import torchfrom auto_round import AutoRound+autoround = AutoRound(- model,- tokenizer,- bits=bits,- group_size=group_size,- iters=1,- nsamples=1,- sym=sym,- layer_config=layer_config+ model, tokenizer, bits=bits, group_size=group_size, iters=1, nsamples=1, sym=sym, layer_config=layer_config)quantized_model_path = self.save_folderautoround.quantize_and_save(output_dir=quantized_model_path, format="auto_round")@@ -104,7 +103,7 @@ class TestAutoRound(unittest.TestCase):inputs = tokenizer(text, return_tensors="pt").to(model.device)res = tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0])print(res)- assert ("!!!" not in res)+ assert "!!!" not in resshutil.rmtree(self.save_folder, ignore_errors=True)@require_gptqmodel@@ -113,27 +112,18 @@ class TestAutoRound(unittest.TestCase):model = AutoModelForCausalLM.from_pretrained(self.model_name, torch_dtype="auto", trust_remote_code=True)tokenizer = AutoTokenizer.from_pretrained(self.model_name, trust_remote_code=True)bits, group_size, sym = bits, 128, True- autoround = AutoRound(- model,- tokenizer,- bits=bits,- group_size=group_size,- sym=sym,- iters=2,- seqlen=2- )+ autoround = AutoRound(model, tokenizer, bits=bits, group_size=group_size, sym=sym, iters=2, seqlen=2)quantized_model_path = "./saved"autoround.quantize_and_save(output_dir=quantized_model_path, format="auto_round")- model = AutoModelForCausalLM.from_pretrained(quantized_model_path, device_map="auto",- trust_remote_code=True)+ model = AutoModelForCausalLM.from_pretrained(+ quantized_model_path, device_map="auto", trust_remote_code=True+ )tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)text = "There is a girl who likes adventure,"inputs = tokenizer(text, return_tensors="pt").to(model.device)res = tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0])print(res)- assert ("!!!" not in res)+ assert "!!!" not in resshutil.rmtree(self.save_folder, ignore_errors=True)--diff --git a/test/test_cuda/test_support_vlms.py b/test/test_cuda/test_support_vlms.pyindex 8208ae255..4fac40c2c 100644--- a/test/test_cuda/test_support_vlms.py+++ b/test/test_cuda/test_support_vlms.py@@ -3,7 +3,7 @@ import shutilimport sysimport unittest-sys.path.insert(0, '../..')+sys.path.insert(0, "../..")import requestsfrom PIL import Image@@ -23,19 +23,19 @@ class TestSupportVLMS(unittest.TestCase):@classmethoddef tearDownClass(self):shutil.rmtree(self.save_dir, ignore_errors=True)-+@require_gptqmodel# def test_qwen2(self):# model_path = "/models/Qwen2-VL-2B-Instruct/"# # test tune# res = os.system(# f"cd ../.. && {self.python_path} -m auto_round --mllm "- # f"--model {model_path} --iter 2 --output_dir {self.save_dir} --device {self.device}")+ # f"--model {model_path} --iter 2 --output_dir {self.save_dir} --device {self.device}")# self.assertFalse(res > 0 or res == -1, msg="qwen2 tuning fail")# # test infer# quantized_model_path = os.path.join(self.save_dir, "Qwen2-VL-2B-Instruct-w4g128")-+# from transformers import Qwen2VLForConditionalGeneration, AutoProcessor# model = Qwen2VLForConditionalGeneration.from_pretrained(# quantized_model_path,@@ -86,55 +86,44 @@ class TestSupportVLMS(unittest.TestCase):## test tuneres = os.system(f"cd ../.. && {self.python_path} -m auto_round --mllm "- f"--model {model_path} --iter 2 --output_dir {self.save_dir} --device {self.device}")+ f"--model {model_path} --iter 2 --output_dir {self.save_dir} --device {self.device}"+ )self.assertFalse(res > 0 or res == -1, msg="Phi-3.5 tuning fail")## test inferfrom transformers import AutoModelForCausalLM, AutoProcessor+quantized_model_path = os.path.join(self.save_dir, "Phi-3.5-vision-instruct-w4g128")res = os.system(f"cp /models/Phi-3.5-vision-instruct/*.py {quantized_model_path}")model = AutoModelForCausalLM.from_pretrained(- quantized_model_path,- device_map=f"cuda:{self.device}",- trust_remote_code=True,+ quantized_model_path,+ device_map=f"cuda:{self.device}",+ trust_remote_code=True,torch_dtype="float16",- )- processor = AutoProcessor.from_pretrained(quantized_model_path,- trust_remote_code=True,- num_crops=4)+ processor = AutoProcessor.from_pretrained(quantized_model_path, trust_remote_code=True, num_crops=4)image_url = "https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen-VL/assets/demo.jpeg"content = "Describe this image."messages = [- {"role": "user",- "content": "<|image_1|>\n"+content},+ {"role": "user", "content": "<|image_1|>\n" + content},]- prompt = processor.tokenizer.apply_chat_template(- messages,- tokenize=False,- add_generation_prompt=True- )+ prompt = processor.tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)image_inputs = Image.open(requests.get(image_url, stream=True).raw)- inputs = processor(prompt, image_inputs, return_tensors="pt").to(model.device)+ inputs = processor(prompt, image_inputs, return_tensors="pt").to(model.device)generation_args = {- "max_new_tokens": 1000,- "temperature": 0.0,- "do_sample": False,+ "max_new_tokens": 1000,+ "temperature": 0.0,+ "do_sample": False,}- generate_ids = model.generate(**inputs,- eos_token_id=processor.tokenizer.eos_token_id,- **generation_args- )+ generate_ids = model.generate(**inputs, eos_token_id=processor.tokenizer.eos_token_id, **generation_args)- # remove input tokens- generate_ids = generate_ids[:, inputs['input_ids'].shape[1]:]- response = processor.batch_decode(generate_ids,- skip_special_tokens=True,- clean_up_tokenization_spaces=False)[0]+ # remove input tokens+ generate_ids = generate_ids[:, inputs["input_ids"].shape[1] :]+ response = processor.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]print(response)shutil.rmtree(quantized_model_path, ignore_errors=True)@@ -146,58 +135,44 @@ class TestSupportVLMS(unittest.TestCase):f"cd ../.. && {self.python_path} -m auto_round --mllm "f"--model {model_path} --iter 2 --quant_nontext_module "f"--nsample 64 --seqlen 32 "- f"--format auto_awq --output_dir {self.save_dir} --device {self.device}")+ f"--format auto_awq --output_dir {self.save_dir} --device {self.device}"+ )self.assertFalse(res > 0 or res == -1, msg="Phi-3.5 tuning fail")## test inferfrom transformers import AutoModelForCausalLM, AutoProcessor+quantized_model_path = os.path.join(self.save_dir, "Phi-3.5-vision-instruct-w4g128")res = os.system(f"cp /models/Phi-3.5-vision-instruct/*.py {quantized_model_path}")model = AutoModelForCausalLM.from_pretrained(- quantized_model_path,- device_map=f"cuda:{self.device}",- trust_remote_code=True,- torch_dtype="auto"+ quantized_model_path, device_map=f"cuda:{self.device}", trust_remote_code=True, torch_dtype="auto")assert "WQLinear_GEMM" in str(- type(model.model.vision_embed_tokens.img_processor.vision_model.encoder.layers[0].mlp.fc1)), \- "model quantization failed."- processor = AutoProcessor.from_pretrained(quantized_model_path,- trust_remote_code=True,- num_crops=4- )+ type(model.model.vision_embed_tokens.img_processor.vision_model.encoder.layers[0].mlp.fc1)+ ), "model quantization failed."+ processor = AutoProcessor.from_pretrained(quantized_model_path, trust_remote_code=True, num_crops=4)image_url = "https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen-VL/assets/demo.jpeg"content = "Describe this image."messages = [- {"role": "user",- "content": "<|image_1|>\n"+content},+ {"role": "user", "content": "<|image_1|>\n" + content},]- prompt = processor.tokenizer.apply_chat_template(- messages,- tokenize=False,- add_generation_prompt=True- )+ prompt = processor.tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)image_inputs = Image.open(requests.get(image_url, stream=True).raw)- inputs = processor(prompt, image_inputs, return_tensors="pt").to(model.device)+ inputs = processor(prompt, image_inputs, return_tensors="pt").to(model.device)generation_args = {- "max_new_tokens": 1000,- "temperature": 0.0,- "do_sample": False,+ "max_new_tokens": 1000,+ "temperature": 0.0,+ "do_sample": False,}- generate_ids = model.generate(**inputs,- eos_token_id=processor.tokenizer.eos_token_id,- **generation_args- )+ generate_ids = model.generate(**inputs, eos_token_id=processor.tokenizer.eos_token_id, **generation_args)- # remove input tokens- generate_ids = generate_ids[:, inputs['input_ids'].shape[1]:]- response = processor.batch_decode(generate_ids,- skip_special_tokens=True,- clean_up_tokenization_spaces=False)[0]+ # remove input tokens+ generate_ids = generate_ids[:, inputs["input_ids"].shape[1] :]+ response = processor.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]print(response)shutil.rmtree(quantized_model_path, ignore_errors=True)@@ -207,12 +182,14 @@ class TestSupportVLMS(unittest.TestCase):## test tuneres = os.system(f"cd ../.. && {self.python_path} -m auto_round --mllm "- f"--model {model_path} --iter 2 --output_dir {self.save_dir} --device {self.device}")+ f"--model {model_path} --iter 2 --output_dir {self.save_dir} --device {self.device}"+ )self.assertFalse(res > 0 or res == -1, msg="llava-v1.5-7b tuning fail")-+## test inferfrom llava.model.builder import load_pretrained_modelfrom llava.train.train import preprocess, preprocess_multimodal+class DataArgs:is_multimodal = Truemm_use_im_start_end = False@@ -229,22 +206,24 @@ class TestSupportVLMS(unittest.TestCase):messages = [{"from": "human", "value": "What is the tennis player doing in the image?\n<image>"}]# Preparation for inference- image = Image.open(requests.get(image_url, stream=True).raw).convert('RGB')- image_input = image_processor.preprocess(image, return_tensors='pt')['pixel_values'][0].to(model.device)+ image = Image.open(requests.get(image_url, stream=True).raw).convert("RGB")+ image_input = image_processor.preprocess(image, return_tensors="pt")["pixel_values"][0].to(model.device)input_data = preprocess_multimodal([messages], DataArgs())inputs = preprocess(input_data, tokenizer, has_image=(image_input is not None))- output = model.generate(inputs['input_ids'].to(model.device), images=image_input.unsqueeze(0).half(), max_new_tokens=50)+ output = model.generate(+ inputs["input_ids"].to(model.device), images=image_input.unsqueeze(0).half(), max_new_tokens=50+ )print(tokenizer.batch_decode(output))shutil.rmtree(quantized_model_path, ignore_errors=True)-+@require_gptqmodel# def test_llama(self):# model_path = "/models/Llama-3.2-11B-Vision-Instruct/"# ## test tune# res = os.system(# f"cd ../.. && {self.python_path} -m auto_round --mllm "- # f"--model {model_path} --iter 2 --output_dir {self.save_dir} --device {self.device}")+ # f"--model {model_path} --iter 2 --output_dir {self.save_dir} --device {self.device}")# self.assertFalse(res > 0 or res == -1, msg="llama-3.2 tuning fail")# ## test infer@@ -277,30 +256,33 @@ class TestSupportVLMS(unittest.TestCase):# output = model.generate(**inputs, max_new_tokens=50)# print(processor.decode(output[0]))# shutil.rmtree(quantized_model_path, ignore_errors=True)-+@require_vlm_envdef test_cogvlm(self):model_path = "/models/cogvlm2-llama3-chat-19B/"## test tuneres = os.system(f"cd ../.. && {self.python_path} -m auto_round --mllm "- f"--model {model_path} --iter 2 --output_dir {self.save_dir} --device {self.device}")+ f"--model {model_path} --iter 2 --output_dir {self.save_dir} --device {self.device}"+ )self.assertFalse(res > 0 or res == -1, msg="cogvlm2 tuning fail")-+## test inferDEVICE = f"cuda:{self.device}"from transformers import AutoModelForCausalLM, AutoTokenizer+quantized_model_path = os.path.join(self.save_dir, "cogvlm2-llama3-chat-19B-w4g128")- tokenizer = AutoTokenizer.from_pretrained(- quantized_model_path,- trust_remote_code=True+ tokenizer = AutoTokenizer.from_pretrained(quantized_model_path, trust_remote_code=True)+ model = (+ AutoModelForCausalLM.from_pretrained(+ quantized_model_path,+ torch_dtype="float16",+ trust_remote_code=True,+ device_map=DEVICE,+ )+ .to(DEVICE)+ .eval())- model = AutoModelForCausalLM.from_pretrained(- quantized_model_path,- torch_dtype="float16",- trust_remote_code=True,- device_map=DEVICE,- ).to(DEVICE).eval()image_url = "https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen-VL/assets/demo.jpeg"content = "Describe this image."@@ -310,29 +292,26 @@ class TestSupportVLMS(unittest.TestCase):image = Image.open(requests.get(image_url, stream=True).raw)input_by_model = model.build_conversation_input_ids(- tokenizer,- query=query,- images=[image],- template_version='chat'+ tokenizer, query=query, images=[image], template_version="chat")inputs = {- 'input_ids': input_by_model['input_ids'].unsqueeze(0).to(DEVICE),- 'token_type_ids': input_by_model['token_type_ids'].unsqueeze(0).to(DEVICE),- 'attention_mask': input_by_model['attention_mask'].unsqueeze(0).to(DEVICE),- 'images': [[input_by_model['images'][0].to(DEVICE).to(model.dtype)]] if image is not None else None,+ "input_ids": input_by_model["input_ids"].unsqueeze(0).to(DEVICE),+ "token_type_ids": input_by_model["token_type_ids"].unsqueeze(0).to(DEVICE),+ "attention_mask": input_by_model["attention_mask"].unsqueeze(0).to(DEVICE),+ "images": [[input_by_model["images"][0].to(DEVICE).to(model.dtype)]] if image is not None else None,}gen_kwargs = {"max_new_tokens": 2048,- "pad_token_id": 128002,+ "pad_token_id": 128002,}outputs = model.generate(**inputs, **gen_kwargs)- outputs = outputs[:, inputs['input_ids'].shape[1]:]+ outputs = outputs[:, inputs["input_ids"].shape[1] :]response = tokenizer.decode(outputs[0])response = response.split("<|end_of_text|>")[0]- print(response)+ print(response)shutil.rmtree(quantized_model_path, ignore_errors=True)-+# def test_72b(self):# model_path = "/models/Qwen2-VL-72B-Instruct/"# res = os.system(@@ -341,7 +320,7 @@ class TestSupportVLMS(unittest.TestCase):# )# self.assertFalse(res > 0 or res == -1, msg="qwen2-72b tuning fail")# shutil.rmtree(self.save_dir, ignore_errors=True)-+@require_vlm_envdef test_deepseek_vl2(self):model_path = "/models/deepseek-vl2-tiny"@@ -349,12 +328,13 @@ class TestSupportVLMS(unittest.TestCase):f"cd ../.. && {self.python_path} -m auto_round --mllm "f"--model {model_path} --iter 3 --nsamples 10 --bs 4 --output_dir {self.save_dir} --device auto --group_size 32 "f"--fp_layers language.model.layers.4,language.model.layers.6"- )+ )self.assertFalse(res > 0 or res == -1, msg="deepseek vl2 tuning fail")quantized_model_path = os.path.join(self.save_dir, "deepseek-vl2-tiny-w4g32")from deepseek_vl2.models import DeepseekVLV2ForCausalLM, DeepseekVLV2Processorfrom transformers import AutoModelForCausalLM+vl_chat_processor: DeepseekVLV2Processor = DeepseekVLV2Processor.from_pretrained(quantized_model_path)tokenizer = vl_chat_processor.tokenizer@@ -381,10 +361,7 @@ class TestSupportVLMS(unittest.TestCase):# load images and prepare for inputspil_images = Image.open(requests.get(image_url, stream=True).raw)prepare_inputs = vl_chat_processor(- conversations=conversation,- images=[pil_images],- force_batchify=True,- system_prompt=""+ conversations=conversation, images=[pil_images], force_batchify=True, system_prompt="")prepare_inputs = prepare_inputs.to(vl_gpt.device)@@ -393,21 +370,20 @@ class TestSupportVLMS(unittest.TestCase):# run the model to get the responseoutputs = vl_gpt.language.generate(- input_ids = prepare_inputs["input_ids"],+ input_ids=prepare_inputs["input_ids"],inputs_embeds=inputs_embeds,attention_mask=prepare_inputs.attention_mask,pad_token_id=tokenizer.eos_token_id,bos_token_id=tokenizer.bos_token_id,eos_token_id=tokenizer.eos_token_id,- max_new_tokens=512,- do_sample=False,- use_cache=True+ max_new_tokens=512,+ do_sample=False,+ use_cache=True,)answer = tokenizer.decode(outputs[0].cpu().tolist(), skip_special_tokens=True)print(f"{prepare_inputs['sft_format'][0]}", answer)+if __name__ == "__main__":unittest.main()--diff --git a/test/test_cuda/test_transformers.py b/test/test_cuda/test_transformers.pyindex e429f14d4..712a2bd5b 100644--- a/test/test_cuda/test_transformers.py+++ b/test/test_cuda/test_transformers.py@@ -205,4 +205,4 @@ class AutoRoundTest(unittest.TestCase):if __name__ == "__main__":- unittest.main()\ No newline at end of file+ unittest.main()diff --git a/test/test_cuda/test_triton_backend.py b/test/test_cuda/test_triton_backend.pyindex e31b659dd..7cbc8719d 100644--- a/test/test_cuda/test_triton_backend.py+++ b/test/test_cuda/test_triton_backend.py@@ -1,4 +1,3 @@-import shutilimport sysimport unittest@@ -42,11 +41,9 @@ class TestAutoRoundTritonBackend(unittest.TestCase):input_ids=inputs["input_ids"].to(model.device),attention_mask=inputs["attention_mask"].to(model.device),do_sample=False, ## change this to follow official usage- max_new_tokens=5+ max_new_tokens=5,)- generated_ids = [- output_ids[len(input_ids):] for input_ids, output_ids in zip(inputs["input_ids"], outputs)- ]+ generated_ids = [output_ids[len(input_ids) :] for input_ids, output_ids in zip(inputs["input_ids"], outputs)]decoded_outputs = tokenizer.batch_decode(generated_ids, skip_special_tokens=True)@@ -77,35 +74,29 @@ class TestAutoRoundTritonBackend(unittest.TestCase):dataset=self.llm_dataloader,)quantized_model_path = self.save_folder- autoround.quantize_and_save(output_dir=quantized_model_path,format="auto_round:gptqmodel")+ autoround.quantize_and_save(output_dir=quantized_model_path, format="auto_round:gptqmodel")quantization_config = AutoRoundConfig(backend="tritonv2")model = AutoModelForCausalLM.from_pretrained(- self.save_folder,- torch_dtype=torch.float16,- device_map="auto",- quantization_config=quantization_config+ self.save_folder, torch_dtype=torch.float16, device_map="auto", quantization_config=quantization_config)tokenizer = AutoTokenizer.from_pretrained(self.save_folder)self.model_infer(model, tokenizer)result = simple_evaluate_user_model(model, tokenizer, batch_size=16, tasks="lambada_openai")- print(result['results']['lambada_openai']['acc,none'])- self.assertGreater(result['results']['lambada_openai']['acc,none'], 0.34)+ print(result["results"]["lambada_openai"]["acc,none"])+ self.assertGreater(result["results"]["lambada_openai"]["acc,none"], 0.34)torch.cuda.empty_cache()model = AutoModelForCausalLM.from_pretrained(- self.save_folder,- torch_dtype=torch.bfloat16,- device_map="auto",- quantization_config=quantization_config+ self.save_folder, torch_dtype=torch.bfloat16, device_map="auto", quantization_config=quantization_config)tokenizer = AutoTokenizer.from_pretrained(self.save_folder)self.model_infer(model, tokenizer)result = simple_evaluate_user_model(model, tokenizer, batch_size=16, tasks="lambada_openai")- print(result['results']['lambada_openai']['acc,none'])- self.assertGreater(result['results']['lambada_openai']['acc,none'], 0.34)+ print(result["results"]["lambada_openai"]["acc,none"])+ self.assertGreater(result["results"]["lambada_openai"]["acc,none"], 0.34)torch.cuda.empty_cache()shutil.rmtree("./saved", ignore_errors=True)@@ -114,43 +105,31 @@ class TestAutoRoundTritonBackend(unittest.TestCase):model = AutoModelForCausalLM.from_pretrained(self.model_name, torch_dtype="auto", trust_remote_code=True)tokenizer = AutoTokenizer.from_pretrained(self.model_name, trust_remote_code=True)bits, group_size, sym = 2, 32, False- autoround = AutoRound(- model,- tokenizer,- bits=bits,- group_size=group_size,- sym=sym- )+ autoround = AutoRound(model, tokenizer, bits=bits, group_size=group_size, sym=sym)quantized_model_path = self.save_folderautoround.quantize_and_save(output_dir=quantized_model_path)quantization_config = AutoRoundConfig(backend="tritonv2")model = AutoModelForCausalLM.from_pretrained(- self.save_folder,- torch_dtype=torch.float16,- device_map="auto",- quantization_config=quantization_config+ self.save_folder, torch_dtype=torch.float16, device_map="auto", quantization_config=quantization_config)tokenizer = AutoTokenizer.from_pretrained(self.save_folder)self.model_infer(model, tokenizer)result = simple_evaluate_user_model(model, tokenizer, batch_size=16, tasks="lambada_openai")- print(result['results']['lambada_openai']['acc,none'])- self.assertGreater(result['results']['lambada_openai']['acc,none'], 0.19)+ print(result["results"]["lambada_openai"]["acc,none"])+ self.assertGreater(result["results"]["lambada_openai"]["acc,none"], 0.19)torch.cuda.empty_cache()model = AutoModelForCausalLM.from_pretrained(- self.save_folder,- torch_dtype=torch.bfloat16,- device_map="auto",- quantization_config=quantization_config+ self.save_folder, torch_dtype=torch.bfloat16, device_map="auto", quantization_config=quantization_config)tokenizer = AutoTokenizer.from_pretrained(self.save_folder)self.model_infer(model, tokenizer)result = simple_evaluate_user_model(model, tokenizer, batch_size=16, tasks="lambada_openai")- print(result['results']['lambada_openai']['acc,none'])- self.assertGreater(result['results']['lambada_openai']['acc,none'], 0.19)+ print(result["results"]["lambada_openai"]["acc,none"])+ self.assertGreater(result["results"]["lambada_openai"]["acc,none"], 0.19)torch.cuda.empty_cache()shutil.rmtree("./saved", ignore_errors=True)@@ -174,31 +153,25 @@ class TestAutoRoundTritonBackend(unittest.TestCase):quantization_config = AutoRoundConfig(backend="tritonv2")model = AutoModelForCausalLM.from_pretrained(- self.save_folder,- torch_dtype=torch.float16,- device_map="auto",- quantization_config=quantization_config+ self.save_folder, torch_dtype=torch.float16, device_map="auto", quantization_config=quantization_config)tokenizer = AutoTokenizer.from_pretrained(self.save_folder)self.model_infer(model, tokenizer)result = simple_evaluate_user_model(model, tokenizer, batch_size=16, tasks="lambada_openai")# print(result['results']['lambada_openai']['acc,none'])- self.assertGreater(result['results']['lambada_openai']['acc,none'], 0.26)+ self.assertGreater(result["results"]["lambada_openai"]["acc,none"], 0.26)torch.cuda.empty_cache()model = AutoModelForCausalLM.from_pretrained(- self.save_folder,- torch_dtype=torch.bfloat16,- device_map="auto",- quantization_config=quantization_config+ self.save_folder, torch_dtype=torch.bfloat16, device_map="auto", quantization_config=quantization_config)tokenizer = AutoTokenizer.from_pretrained(self.save_folder)self.model_infer(model, tokenizer)result = simple_evaluate_user_model(model, tokenizer, batch_size=16, tasks="lambada_openai")# print(result['results']['lambada_openai']['acc,none'])- self.assertGreater(result['results']['lambada_openai']['acc,none'], 0.26)+ self.assertGreater(result["results"]["lambada_openai"]["acc,none"], 0.26)torch.cuda.empty_cache()shutil.rmtree("./saved", ignore_errors=True)@@ -208,45 +181,31 @@ class TestAutoRoundTritonBackend(unittest.TestCase):model = AutoModelForCausalLM.from_pretrained(self.model_name, torch_dtype="auto", trust_remote_code=True)tokenizer = AutoTokenizer.from_pretrained(self.model_name, trust_remote_code=True)bits, group_size, sym = 4, 256, True- autoround = AutoRound(- model,- tokenizer,- bits=bits,- group_size=group_size,- sym=sym,- nsamples=1,- iters=1- )+ autoround = AutoRound(model, tokenizer, bits=bits, group_size=group_size, sym=sym, nsamples=1, iters=1)quantized_model_path = self.save_folderautoround.quantize_and_save(output_dir=quantized_model_path)quantization_config = AutoRoundConfig(backend="tritonv2")model = AutoModelForCausalLM.from_pretrained(- self.save_folder,- torch_dtype=torch.float16,- device_map="auto",- quantization_config=quantization_config+ self.save_folder, torch_dtype=torch.float16, device_map="auto", quantization_config=quantization_config)tokenizer = AutoTokenizer.from_pretrained(self.save_folder)self.model_infer(model, tokenizer)result = simple_evaluate_user_model(model, tokenizer, batch_size=16, tasks="lambada_openai")- print(result['results']['lambada_openai']['acc,none'])- self.assertGreater(result['results']['lambada_openai']['acc,none'], 0.27)+ print(result["results"]["lambada_openai"]["acc,none"])+ self.assertGreater(result["results"]["lambada_openai"]["acc,none"], 0.27)torch.cuda.empty_cache()model = AutoModelForCausalLM.from_pretrained(- self.save_folder,- torch_dtype=torch.bfloat16,- device_map="auto",- quantization_config=quantization_config+ self.save_folder, torch_dtype=torch.bfloat16, device_map="auto", quantization_config=quantization_config)tokenizer = AutoTokenizer.from_pretrained(self.save_folder)self.model_infer(model, tokenizer)result = simple_evaluate_user_model(model, tokenizer, batch_size=16, tasks="lambada_openai")# print(result['results']['lambada_openai']['acc,none'])- self.assertGreater(result['results']['lambada_openai']['acc,none'], 0.27)+ self.assertGreater(result["results"]["lambada_openai"]["acc,none"], 0.27)torch.cuda.empty_cache()shutil.rmtree("./saved", ignore_errors=True)@@ -267,31 +226,25 @@ class TestAutoRoundTritonBackend(unittest.TestCase):quantization_config = AutoRoundConfig(backend="tritonv2")model = AutoModelForCausalLM.from_pretrained(- self.save_folder,- torch_dtype=torch.float16,- device_map="auto",- quantization_config=quantization_config+ self.save_folder, torch_dtype=torch.float16, device_map="auto", quantization_config=quantization_config)tokenizer = AutoTokenizer.from_pretrained(self.save_folder)self.model_infer(model, tokenizer)result = simple_evaluate_user_model(model, tokenizer, batch_size=16, tasks="lambada_openai")- print(result['results']['lambada_openai']['acc,none'])- self.assertGreater(result['results']['lambada_openai']['acc,none'], 0.18)+ print(result["results"]["lambada_openai"]["acc,none"])+ self.assertGreater(result["results"]["lambada_openai"]["acc,none"], 0.18)torch.cuda.empty_cache()model = AutoModelForCausalLM.from_pretrained(- self.save_folder,- torch_dtype=torch.bfloat16,- device_map="auto",- quantization_config=quantization_config+ self.save_folder, torch_dtype=torch.bfloat16, device_map="auto", quantization_config=quantization_config)tokenizer = AutoTokenizer.from_pretrained(self.save_folder)self.model_infer(model, tokenizer)result = simple_evaluate_user_model(model, tokenizer, batch_size=16, tasks="lambada_openai")# print(result['results']['lambada_openai']['acc,none'])- self.assertGreater(result['results']['lambada_openai']['acc,none'], 0.18)+ self.assertGreater(result["results"]["lambada_openai"]["acc,none"], 0.18)torch.cuda.empty_cache()shutil.rmtree("./saved", ignore_errors=True)diff --git a/test/test_cuda/test_vllm.py b/test/test_cuda/test_vllm.pyindex 6ad99d7a2..157457215 100644--- a/test/test_cuda/test_vllm.py+++ b/test/test_cuda/test_vllm.py@@ -1,10 +1,10 @@# SPDX-License-Identifier: Apache-2.0"""Test model set-up and inference for quantized HF models supported- on the AutoRound.+on the AutoRound.- Validating the configuration and printing results for manual checking.+Validating the configuration and printing results for manual checking.- Run `pytest test/test_cuda/test_vllm.py`.+Run `pytest test/test_cuda/test_vllm.py`."""import pytest@@ -13,14 +13,14 @@ from vllm.platforms import current_platformMODELS = ["OPEA/Qwen2.5-0.5B-Instruct-int4-sym-inc", ##auto_round:auto_gptq- "Intel/Qwen2-0.5B-Instruct-int4-sym-AutoRound" ##auto_round:auto_awq+ "Intel/Qwen2-0.5B-Instruct-int4-sym-AutoRound", ##auto_round:auto_awq][email protected](not current_platform.is_cpu()- and not current_platform.is_xpu()- and not current_platform.is_cuda(),- reason="only supports CPU/XPU/CUDA backend.")+ not current_platform.is_cpu() and not current_platform.is_xpu() and not current_platform.is_cuda(),+ reason="only supports CPU/XPU/CUDA backend.",+)@pytest.mark.parametrize("model", MODELS)def test_auto_round(model):# Sample prompts.diff --git a/test/test_cuda/test_vlms.py b/test/test_cuda/test_vlms.pyindex 62444d786..d18c53a66 100644--- a/test/test_cuda/test_vlms.py+++ b/test/test_cuda/test_vlms.py@@ -25,7 +25,6 @@ class TestAutoRound(unittest.TestCase):shutil.rmtree(self.save_dir, ignore_errors=True)shutil.rmtree("runs", ignore_errors=True)-# def test_vision_generation(self):# quantized_model_path = "OPEA/Phi-3.5-vision-instruct-qvision-int4-sym-inc"# from auto_round import AutoRoundConfig@@ -45,6 +44,7 @@ class TestAutoRound(unittest.TestCase):def qwen_inference(self, quantized_model_dir):from transformers import AutoProcessor, AutoTokenizer, Qwen2VLForConditionalGeneration+tokenizer = AutoTokenizer.from_pretrained(quantized_model_dir)processor = AutoProcessor.from_pretrained(quantized_model_dir, trust_remote_code=True)model = Qwen2VLForConditionalGeneration.from_pretrained(@@ -69,9 +69,7 @@ class TestAutoRound(unittest.TestCase):]# Preparation for inference- text = processor.apply_chat_template(- messages, tokenize=False, add_generation_prompt=True- )+ text = processor.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)image_inputs = Image.open(requests.get(image_url, stream=True).raw)inputs = processor(text=[text],@@ -82,9 +80,7 @@ class TestAutoRound(unittest.TestCase):inputs = inputs.to(model.device)generated_ids = model.generate(**inputs, max_new_tokens=128)- generated_ids_trimmed = [- out_ids[len(in_ids):] for in_ids, out_ids in zip(inputs.input_ids, generated_ids)- ]+ generated_ids_trimmed = [out_ids[len(in_ids) :] for in_ids, out_ids in zip(inputs.input_ids, generated_ids)]output_text = processor.batch_decode(generated_ids_trimmed, skip_special_tokens=True, clean_up_tokenization_spaces=False)@@ -99,22 +95,22 @@ class TestAutoRound(unittest.TestCase):## load the modelmodel_name = "/models/Qwen2-VL-2B-Instruct"- model = Qwen2VLForConditionalGeneration.from_pretrained(- model_name, trust_remote_code=True, device_map="auto")+ model = Qwen2VLForConditionalGeneration.from_pretrained(model_name, trust_remote_code=True, device_map="auto")tokenizer = AutoTokenizer.from_pretrained(model_name)processor = AutoProcessor.from_pretrained(model_name, trust_remote_code=True)## quantize the modelbits, group_size, sym = 4, 128, True- autoround = AutoRoundMLLM(model, tokenizer, processor,- bits=bits, group_size=group_size, sym=sym, iters=1, nsamples=1)+ autoround = AutoRoundMLLM(+ model, tokenizer, processor, bits=bits, group_size=group_size, sym=sym, iters=1, nsamples=1+ )autoround.quantize()quantized_model_path = self.save_dir- autoround.save_quantized(quantized_model_path, format='auto_round', inplace=False)+ autoround.save_quantized(quantized_model_path, format="auto_round", inplace=False)self.qwen_inference(quantized_model_path)shutil.rmtree(self.save_dir, ignore_errors=True)- autoround.save_quantized(quantized_model_path, format='auto_gptq', inplace=False)+ autoround.save_quantized(quantized_model_path, format="auto_gptq", inplace=False)self.qwen_inference(quantized_model_path)shutil.rmtree(self.save_dir, ignore_errors=True)@@ -124,8 +120,8 @@ class TestAutoRound(unittest.TestCase):model_name = "/models/Llama-3.2-11B-Vision-Instruct/"from transformers import MllamaForConditionalGeneration- model = MllamaForConditionalGeneration.from_pretrained(- model_name, trust_remote_code=True, device_map="auto")++ model = MllamaForConditionalGeneration.from_pretrained(model_name, trust_remote_code=True, device_map="auto")block_name = get_block_names(model, quant_vision=True)self.assertTrue(len(block_name) == 3)self.assertTrue(any(["vision_model.global_transformer.layers.0" not in n for n in block_name]))@@ -137,6 +133,3 @@ class TestAutoRound(unittest.TestCase):if __name__ == "__main__":unittest.main()---diff --git a/test/test_xpu/test_autoround.py b/test/test_xpu/test_autoround.pyindex 0c547b236..8052a8af0 100644--- a/test/test_xpu/test_autoround.py+++ b/test/test_xpu/test_autoround.py@@ -34,8 +34,9 @@ class TestAutoRoundXPU(unittest.TestCase):def test_gptq_format(self):model_name = "facebook/opt-125m"- model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True,- device_map="auto")+ model = AutoModelForCausalLM.from_pretrained(+ model_name, torch_dtype="auto", trust_remote_code=True, device_map="auto"+ )tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)bits, group_size, sym = 4, 128, True@@ -52,25 +53,23 @@ class TestAutoRoundXPU(unittest.TestCase):quantized_model_path = "./saved"autoround.quantize_and_save(output_dir=quantized_model_path)- quantization_config = AutoRoundConfig(- backend="auto"- )+ quantization_config = AutoRoundConfig(backend="auto")- model = AutoModelForCausalLM.from_pretrained(quantized_model_path,- device_map="auto", quantization_config=quantization_config)+ model = AutoModelForCausalLM.from_pretrained(+ quantized_model_path, device_map="auto", quantization_config=quantization_config+ )tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)text = "There is a girl who likes adventure,"inputs = tokenizer(text, return_tensors="pt").to(model.device)res = tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0])print(res)- assert ("!!!" not in res)--+ assert "!!!" not in resdef test_awq_format(self):model_name = "facebook/opt-125m"- model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True,- device_map="xpu")+ model = AutoModelForCausalLM.from_pretrained(+ model_name, torch_dtype="auto", trust_remote_code=True, device_map="xpu"+ )tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)bits, group_size, sym = 4, 128, Trueautoround = AutoRound(@@ -87,18 +86,18 @@ class TestAutoRoundXPU(unittest.TestCase):autoround.quantize_and_save(output_dir=quantized_model_path, format="auto_round:auto_awq")quantized_model_path = "./saved"- quantization_config = AutoRoundConfig(- backend="auto"- )+ quantization_config = AutoRoundConfig(backend="auto")- model = AutoModelForCausalLM.from_pretrained(quantized_model_path,- device_map="auto", quantization_config=quantization_config)+ model = AutoModelForCausalLM.from_pretrained(+ quantized_model_path, device_map="auto", quantization_config=quantization_config+ )tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)text = "There is a girl who likes adventure,"inputs = tokenizer(text, return_tensors="pt").to(model.device)res = tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0])print(res)- assert ("!!!" not in res)+ assert "!!!" not in res+if __name__ == "__main__":- unittest.main()\ No newline at end of file+ unittest.main()