PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
1 fix a bug in autoround format with the latest transformers 2 rename n_samples n_blocks to nsamples nblocks (#163)
wenhuach21f9e7d79e9b64
diff --git a/README.md b/README.mdindex d9027c20d..377560986 100644--- a/README.md+++ b/README.md@@ -11,12 +11,9 @@ AutoRound<div align="left">AutoRound is an advanced weight-only quantization algorithm for low-bits LLM inference. It's tailored for a wide range-of models and consistently delivers noticeable improvements, often significantly outperforming SignRound with the cost-of more tuning time for quantization.--Our method adopts sign gradient descent to fine-tune rounding values and minmax values of weights in just 200 steps,+of models. Our method adopts sign gradient descent to fine-tune rounding values and minmax values of weights in just 200 steps,which competes impressively against recent methods without introducing any additional inference overhead. The below-image presents an overview of AutoRound.+image presents an overview of AutoRound. Check out our updated paper on [arxiv](https://arxiv.org/pdf/2309.05516v4)<div align="center">@@ -26,7 +23,6 @@ image presents an overview of AutoRound.## What's New* [2024/06] AutoRound format supports mixed bit-widths and group sizes for inference, resolving the significant performance drop issue with the asymmetric kernel-* [2024/05] Check out our updated paper on [arxiv](https://arxiv.org/pdf/2309.05516v4)* [2024/05] AutoRound supports lm-head quantization, saving 0.7G for LLaMA3-8B at W4G128.* [2024/05] AutoRound performs wellin [low_bit_open_llm_leaderboard](https://huggingface.co/spaces/Intel/low_bit_open_llm_leaderboard)@@ -102,7 +98,7 @@ autoround.save_quantized(output_dir) ##save_quantized(output_dir,format=="auto_r- `minmax_lr (float)`: The learning rate for min-max tuning (default is None, it will be set to lr automatically).-- `n_samples (int)`: Number of samples for tuning (default is 512).+- `nsamples (int)`: Number of samples for tuning (default is 512).- `seqlen (int)`: Data length of the sequence for tuning (default is 2048).@@ -113,7 +109,7 @@ autoround.save_quantized(output_dir) ##save_quantized(output_dir,format=="auto_r- `amp (bool)`: Whether to use automatic mixed precision (default is True).-- `n_blocks (int)`: Packing several blocks as one for tuning together (default is 1).+- `nblocks (int)`: Packing several blocks as one for tuning together (default is 1).- `gradient_accumulate_steps (int)`: Number of gradient accumulation steps (default is 1).@@ -137,13 +133,13 @@ Please run the quantization code first.### CPU```python-##Install the latest https://github.com/intel/intel-extension-for-transformers from source first.+##pip install intel-extension-for-transformersfrom intel_extension_for_transformers.transformers import AutoModelForCausalLMfrom transformers import AutoTokenizerquantized_model_path = "./tmp_autoround"-model = AutoModelForCausalLM.from_pretrained(quantized_model_path, device_map="auto", trust_remote_code=True)-tokenizer = AutoTokenizer.from_pretrained(quantized_model_path, use_fast=True)+model = AutoModelForCausalLM.from_pretrained(quantized_model_path, device_map="auto")+tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)text = "There is a girl who likes adventure,"inputs = tokenizer(text, return_tensors="pt").to(model.device)print(tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0]))@@ -156,8 +152,8 @@ from transformers import AutoModelForCausalLM, AutoTokenizer##from auto_round.auto_quantizer import AutoHfQuantizer ## uncomment it for models with auto_round formatquantized_model_path = "./tmp_autoround"-model = AutoModelForCausalLM.from_pretrained(quantized_model_path, device_map="auto", trust_remote_code=True)-tokenizer = AutoTokenizer.from_pretrained(quantized_model_path, use_fast=True)+model = AutoModelForCausalLM.from_pretrained(quantized_model_path, device_map="auto")+tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)text = "There is a girl who likes adventure,"inputs = tokenizer(text, return_tensors="pt").to(model.device)print(tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0]))@@ -174,10 +170,10 @@ print(tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0]))| google/gemma-2b | [HF-int4-model](https://huggingface.co/Intel/gemma-2b-int4-inc), [accuracy](./docs/gemma-2b-acc.md), [recipe](./examples/language-modeling/scripts/gemma-2b.sh), [example](./examples/language-modeling/)| tiiuae/falcon-7b | [HF-int4-model-G64](https://huggingface.co/Intel/falcon-7b-int4-inc), [accuracy](./docs/falcon-7b-acc.md), [recipe](./examples/language-modeling/scripts/falcon-7b.sh), [example](./examples/language-modeling/) || mistralai/Mistral-7B-Instruct-v0.2 | [HF-int4-model](https://huggingface.co/Intel/Mistral-7B-Instruct-v0.2-int4-inc) (under review), [accuracy](./docs/Mistral-7B-Instruct-v0.2-acc.md), [recipe](./examples/language-modeling/scripts/Mistral-7B-Instruct-v0.2.sh), [example](./examples/language-modeling/) |-| google/gemma-7b | [HF-int4-model](https://huggingface.co/Intel/gemma-7b-int4-inc) (under review), [accuracy](./docs/gemma-7b-acc.md), [recipe](./examples/language-modeling/scripts/gemma-7b.sh), [example](./examples/language-modeling/) || mistralai/Mixtral-8x7B-Instruct-v0.1 | [HF-int4-model](https://huggingface.co/Intel/Mixtral-8x7B-Instruct-v0.1-int4-inc) (under review), [accuracy](./docs/Mixtral-8x7B-Instruct-v0.1-acc.md), [recipe](./examples/language-modeling/scripts/Mixtral-8x7B-Instruct-v0.1.sh), [example](./examples/language-modeling/) || mistralai/Mixtral-8x7B-v0.1 | [HF-int4-model](https://huggingface.co/Intel/Mixtral-8x7B-v0.1-int4-inc) (under review), [accuracy](./docs/Mixtral-8x7B-v0.1-acc.md), [recipe](./examples/language-modeling/scripts/Mixtral-8x7B-v0.1.sh), [example](./examples/language-modeling/) || meta-llama/Meta-Llama-3-8B-Instruct | [accuracy](./docs/Meta-Llama-3-8B-Instruct-acc.md), [recipe](./examples/language-modeling/scripts/Meta-Llama-3-8B-Instruct.sh), [example](./examples/language-modeling/) |+| google/gemma-7b | [accuracy](./docs/gemma-7b-acc.md), [recipe](./examples/language-modeling/scripts/gemma-7b.sh), [example](./examples/language-modeling/) || meta-llama/Llama-2-7b-chat-hf | [accuracy](./docs/Llama-2-7b-chat-hf-acc.md), [recipe](./examples/language-modeling/scripts/Llama-2-7b-chat-hf.sh), [example](./examples/language-modeling/) || Qwen/Qwen1.5-7B-Chat | [accuracy](./docs/Qwen1.5-7B-Chat-acc.md), [sym recipe](./examples/language-modeling/scripts/Qwen1.5-7B-Chat-sym.sh), [asym recipe ](./examples/language-modeling/scripts/Qwen1.5-7B-Chat-asym.sh), [example](./examples/language-modeling/) || baichuan-inc/Baichuan2-7B-Chat | [accuracy](./docs/baichuan2-7b-chat-acc.md), [recipe](./examples/language-modeling/scripts/baichuan2-7b-chat.sh), [example](./examples/language-modeling/) |@@ -205,9 +201,7 @@ average accuracies of 11 zero-shot tasks.1 Consider increasing tuning steps to achieve better results, albeit with increased tuning time.-2 Setting 'enable_quanted_input' to False has been observed to occasionally yield improved results.--3 Setting 'minmax_lr' to 2.0/iters has been observed to occasionally yield improved results.+2 Setting 'minmax_lr' to 2.0/iters has been observed to occasionally yield improved results.## Referencediff --git a/auto_round/auto_quantizer.py b/auto_round/auto_quantizer.pyindex a0e08db41..045a64572 100644--- a/auto_round/auto_quantizer.py+++ b/auto_round/auto_quantizer.py@@ -43,7 +43,7 @@ from transformers.utils.quantization_config import AwqConfig, GPTQConfig, Quantifrom auto_round.utils import get_module, set_module, dynamic_import_inference_linearimport auto_round_extension.qbits.qlinear_qbits as qlinear_qbits-+from enum import Enumlogger = getLogger(__name__)import sys@@ -194,6 +194,9 @@ class AutoHfQuantizer:return quantization_config+class AutoRoundQuantizationMethod(str, Enum):+ AutoRound = "intel/auto-round"+@dataclassclass AutoRoundConfig(QuantizationConfigMixin):@@ -222,6 +225,7 @@ class AutoRoundConfig(QuantizationConfigMixin):weight_config: dict = None,**kwargs,):+self.bits = bitsself.tokenizer = tokenizerself.dataset = dataset@@ -232,7 +236,7 @@ class AutoRoundConfig(QuantizationConfigMixin):if kwargs is not None:for key in kwargs.keys():setattr(self, key, kwargs[key])-+ self.quant_method = AutoRoundQuantizationMethod.AutoRoundself.post_init()def get_loading_attributes(self):@@ -378,11 +382,6 @@ class AutoRoundQuantizer(HfQuantizer):The input model"""- #- # if self.bits == 4: if get_device(model) == torch.device("cpu") or ( hasattr(model, "hf_device_map") and- # any(d in model.hf_device_map for d in ["cpu", "disk"]) ): raise ValueError( "Found modules on cpu/disk.- # Using Exllamav2 backend requires all the modules to be on GPU." "You can deactivate exllama backend by- # setting `disable_exllama=True` in the quantization config object" )class StoreAttr(object):pass@@ -406,11 +405,7 @@ class AutoRoundQuantizer(HfQuantizer):model = self.post_init_model(model)else:raise NotImplementedError- # if self.quantization_config.tokenizer is None:- # self.quantization_config.tokenizer = model.name_or_path- #- # self.optimum_quantizer.quantize_model(model, self.quantization_config.tokenizer)- # model.config.quantization_config = GPTQConfig.from_dict(self.optimum_quantizer.to_dict())+@propertydef is_trainable(self, model: Optional["PreTrainedModel"] = None):diff --git a/auto_round/autoround.py b/auto_round/autoround.pyindex c5df1ad1a..ff3889110 100644--- a/auto_round/autoround.py+++ b/auto_round/autoround.py@@ -79,15 +79,15 @@ class AutoRound(object):low_gpu_mem_usage (bool): Whether to use low GPU memory (default is True).iters (int): Number of iterations (default is 200).seqlen (int): Data length of the sequence for tuning (default is 2048).- n_samples (int): Number of samples (default is 512).+ nsamples (int): Number of samples (default is 512).sampler (str): The sampling method (default is "rand").seed (int): The random seed (default is 42).- n_blocks (int): Number of blocks (default is 1).+ nblocks (int): Number of blocks (default is 1).gradient_accumulate_steps (int): Number of gradient accumulation steps (default is 1).not_use_best_mse (bool): Whether to use mean squared error (default is False).dynamic_max_gap (int): The dynamic maximum gap (default is -1).data_type (str): The data type to be used (default is "int").- scale_dtype (str): The data type of quantization scale to be used (default is "float32"), different kernels+ scale_dtype (str): The data type of quantization scale to be used (default is "float16"), different kernelshave different choices.Returns:@@ -115,10 +115,10 @@ class AutoRound(object):low_gpu_mem_usage: bool = True,iters: int = 200,seqlen: int = 2048,- n_samples: int = 512,+ nsamples: int = 512,sampler: str = "rand",seed: int = 42,- n_blocks: int = 1,+ nblocks: int = 1,gradient_accumulate_steps: int = 1,not_use_best_mse: bool = False,dynamic_max_gap: int = -1,@@ -132,8 +132,8 @@ class AutoRound(object):self.amp = ampself.enable_quanted_input = enable_quanted_inputself.enable_minmax_tuning = enable_minmax_tuning- self.n_samples = n_samples- self.n_blocks = n_blocks+ self.nsamples = nsamples+ self.nblocks = nblocksself.bits = bitsself.group_size = group_sizeself.sym = sym@@ -145,7 +145,7 @@ class AutoRound(object):self.tokenizer = tokenizerself.seqlen = seqlenself.train_bs = batch_size- self.n_blocks = n_blocks+ self.nblocks = nblocksself.device = detect_device(device)self.scale_dtype = convert_dtype_str2torch(scale_dtype)self.set_amp_dtype()@@ -156,12 +156,9 @@ class AutoRound(object):if self.iters <= 0:logger.warning("iters must be positive, reset it to 200")self.iters = 200- self.lr = lr- if self.lr is None:- self.lr = 1.0 / self.iters- self.minmax_lr = minmax_lr- if self.minmax_lr is None:- self.minmax_lr = self.lr++ self.lr = lr or (1.0 / self.iters)+ self.minmax_lr = minmax_lr or self.lrself.sampler = samplerself.gradient_accumulate_steps = gradient_accumulate_steps@@ -176,35 +173,6 @@ class AutoRound(object):torch.set_printoptions(precision=3, sci_mode=True)self.check_configs()- serialization_keys = [- "bits",- "group_size",- "sym",- "data_type",- "enable_quanted_input",- "enable_minmax_tuning",- "data_type",- "seqlen",- "train_bs",- "scale_dtype",- "lr",- "minmax_lr",- "gradient_accumulate_steps",- "iters",- "amp",- "n_samples",- "low_gpu_mem_usage",- ]- if isinstance(dataset, str):- serialization_keys.append("dataset")- self.serialization_dict = {}- for key in serialization_keys:- self.serialization_dict[key] = getattr(self, key)- from .version import __version__-- self.serialization_dict["autoround_version"] = __version__- if "scale_dtype" in self.serialization_dict.keys():- self.serialization_dict["scale_dtype"] = str(self.serialization_dict["scale_dtype"])if is_optimum_habana_available():logger.info("Optimum Habana is available, import htcore explicitly.")import habana_frameworks.torch.core as htcore # pylint: disable=E0401@@ -222,7 +190,7 @@ class AutoRound(object):assert self.train_bs > 0, "batch size must be positive"assert self.iters > 0, "iters must be positive"assert self.seqlen > 0, "seqlen must be positive"- assert self.n_blocks > 0, "n_blocks must be positive"+ assert self.nblocks > 0, "nblocks must be positive"assert self.gradient_accumulate_steps > 0, "gradient accumulate step must be positive"assert self.enable_full_range is False, "only support enable_full_range=False currently"# assert self.tokenizer != None or self.dataloader != None@@ -245,7 +213,7 @@ class AutoRound(object):layer_names = self.get_quantized_layer_names_outside_blocks()self.start_time = time.time()- all_inputs = self.try_cache_inter_data_gpucpu([block_names[0]], self.n_samples, layer_names=layer_names)+ all_inputs = self.try_cache_inter_data_gpucpu([block_names[0]], self.nsamples, layer_names=layer_names)del self.inputsinputs = all_inputs[block_names[0]]@@ -254,7 +222,7 @@ class AutoRound(object):del self.inputsif "input_ids" in inputs.keys():total_samples = len(inputs["input_ids"])- self.n_samples = total_samples+ self.nsamples = total_samplesif total_samples < self.train_bs:self.train_bs = total_sampleslogger.warning(f"force the train batch size to {total_samples} ")@@ -265,7 +233,7 @@ class AutoRound(object):self.model,inputs,block_names,- n_blocks=self.n_blocks,+ nblocks=self.nblocks,device=self.device,)@@ -344,7 +312,7 @@ class AutoRound(object):returnq_layer_inputs = Noneif self.enable_quanted_input:- q_layer_inputs = self.try_cache_inter_data_gpucpu([], self.n_samples, layer_names=layer_names)+ q_layer_inputs = self.try_cache_inter_data_gpucpu([], self.nsamples, layer_names=layer_names)self.model = self.model.to("cpu")torch.cuda.empty_cache()@@ -370,11 +338,11 @@ class AutoRound(object):Returns:None"""- layers_in_blocks = get_layer_names_in_block(self.model, self.supported_types)+ layers_inblocks = get_layer_names_in_block(self.model, self.supported_types)for n, m in self.model.named_modules():if not isinstance(m, tuple(self.supported_types)):continue- if n not in weight_config.keys() and n in layers_in_blocks:+ if n not in weight_config.keys() and n in layers_inblocks:weight_config[n] = {}weight_config[n]["data_type"] = self.data_typeweight_config[n]["bits"] = self.bits@@ -424,8 +392,8 @@ class AutoRound(object):"""output = []- for i in range(0, self.n_samples, bs):- end_index = min(self.n_samples, i + bs)+ for i in range(0, self.nsamples, bs):+ end_index = min(self.nsamples, i + bs)indices = torch.arange(i, end_index).to(torch.long)tmp_input_ids, tmp_input_others = sampling_inputs(input_ids, input_others, indices, self.seqlen, self.share_attention_mask_flag, self.input_dim@@ -439,7 +407,7 @@ class AutoRound(object):return output@torch.no_grad()- def calib(self, n_samples, bs):+ def calib(self, nsamples, bs):"""Perform calibration for quantization.This method calibrates the model for quantization by processing a specified@@ -448,7 +416,7 @@ class AutoRound(object):is less than the specified number, it logs a warning. If no samples are processed,it logs an error and exits.Args:- n_samples (int): The number of samples to use for calibration.+ nsamples (int): The number of samples to use for calibration.bs (int): The number of samples to use for calibration"""@@ -460,7 +428,7 @@ class AutoRound(object):dataset,self.seed,bs,- self.n_samples,+ self.nsamples,)else:self.dataloader = self.dataset@@ -499,7 +467,7 @@ class AutoRound(object):except Exception as error:logger.error(error)total_cnt += input_ids.shape[0]- if total_cnt >= n_samples:+ if total_cnt >= nsamples:breakif total_cnt == 0:logger.error(@@ -507,19 +475,19 @@ class AutoRound(object):f"dataset or decease the sequence length")exit()- elif total_cnt < n_samples:+ elif total_cnt < nsamples:logger.warning(f"Insufficient number of samples collected may affect the quantification. "- f"Valid samples size:{total_cnt}, Target sample size:{n_samples}"+ f"Valid samples size:{total_cnt}, Target sample size:{nsamples}")@torch.no_grad()- def try_cache_inter_data_gpucpu(self, block_names, n_samples, layer_names=[], last_cache_name=None):+ def try_cache_inter_data_gpucpu(self, block_names, nsamples, layer_names=[], last_cache_name=None):"""Attempts to cache intermediate data on GPU,if failed, then using CPU.Args:block_names (list): List of block names to cache data for.- n_samples (int): Number of samples to use for caching.+ nsamples (int): Number of samples to use for caching.layer_names (list, optional): List of layer names to cache data for. Defaults to [].last_cache_name (str, optional): Name of the last cache. Defaults to None.@@ -532,7 +500,7 @@ class AutoRound(object):try:self.model = self.model.to(self.device)all_inputs = self.cache_inter_data(- block_names, n_samples, layer_names=layer_names, last_cache_name=last_cache_name+ block_names, nsamples, layer_names=layer_names, last_cache_name=last_cache_name)self.model = self.model.to("cpu")torch.cuda.empty_cache()@@ -541,12 +509,12 @@ class AutoRound(object):self.model = self.model.to("cpu")torch.cuda.empty_cache()all_inputs = self.cache_inter_data(- block_names, n_samples, layer_names=layer_names, last_cache_name=last_cache_name+ block_names, nsamples, layer_names=layer_names, last_cache_name=last_cache_name)return all_inputs@torch.no_grad()- def cache_inter_data(self, block_names, n_samples, layer_names=[], last_cache_name=None):+ def cache_inter_data(self, block_names, nsamples, layer_names=[], last_cache_name=None):"""Save the inputs of block_name for calibration. For layers, we cache both of inputs and output.This method temporarily replaces the forward method of the model to capture@@ -556,7 +524,7 @@ class AutoRound(object):Args:block_names (list): The names of the blocks for which inputs are to be saved.layer_names (list):The names of the layers for which inputs are to be saved.- n_samples (int): The number of samples to use for calibration.+ nsamples (int): The number of samples to use for calibration.last_cache_name (str, optional): The name of the last layer to be cached,we could break the forward in this layer to save time@@ -577,7 +545,7 @@ class AutoRound(object):calib_bs = self.train_bsself.hook_handles = []self._replace_forward()- self.calib(n_samples, calib_bs)+ self.calib(nsamples, calib_bs)self._recover_forward()res = self.inputsdel self.last_cache_name@@ -732,7 +700,7 @@ class AutoRound(object):)else:lr_schedule = copy.deepcopy(self.lr_scheduler)- n_samples = len(inputs)+ nsamples = len(inputs)last_best_iter = 0best_loss = torch.finfo(torch.float).maxmse_loss = torch.nn.MSELoss().to(device)@@ -744,13 +712,12 @@ class AutoRound(object):pick_samples = train_bs * gradient_accumulate_stepsif self.sampler != "rand":- whole_indices = torch.randperm(n_samples)[:pick_samples]+ whole_indices = torch.randperm(nsamples)[:pick_samples]for i in range(self.iters):total_loss = 0if self.sampler == "rand":- whole_indices = torch.randperm(n_samples)[:pick_samples]+ whole_indices = torch.randperm(nsamples)[:pick_samples]for tmp_step in range(gradient_accumulate_steps):- org_input = Noneindices = whole_indices[tmp_step * train_bs: (tmp_step + 1) * train_bs]if q_inputs is not None:current_input = [q_inputs[i] for i in indices]@@ -859,9 +826,9 @@ class AutoRound(object):lr_schedule = copy.deepcopy(self.lr_scheduler)pick_samples = self.train_bs * self.gradient_accumulate_steps- n_samples = len(input_ids)+ nsamples = len(input_ids)if self.sampler != "rand":- whole_indices = torch.randperm(n_samples)[:pick_samples]+ whole_indices = torch.randperm(nsamples)[:pick_samples]last_best_iter = 0best_loss = torch.finfo(torch.float).maxmse_loss = torch.nn.MSELoss().to(device)@@ -871,7 +838,7 @@ class AutoRound(object):for i in range(self.iters):total_loss = 0if self.sampler == "rand":- whole_indices = torch.randperm(n_samples)[:pick_samples]+ whole_indices = torch.randperm(nsamples)[:pick_samples]for tmp_step in range(self.gradient_accumulate_steps):indices = whole_indices[tmp_step * self.train_bs: (tmp_step + 1) * self.train_bs]current_input_ids, current_input_others = sampling_inputs(@@ -956,7 +923,7 @@ class AutoRound(object):model: torch.nn.Module,inputs,block_names,- n_blocks=1,+ nblocks=1,device=torch.device("cpu"),):"""Quantize and dequantize the weights of the specified blocks in the model.@@ -965,7 +932,7 @@ class AutoRound(object):model: The PyTorch model to be quantized.inputs: The input data for quantization.block_names: The names of the blocks to be quantized and dequantized.- n_blocks: The number of blocks to quantize and dequantize.+ nblocks: The number of blocks to quantize and dequantize.device: The device for quantization and dequantization.Returns:@@ -995,13 +962,13 @@ class AutoRound(object):for i in range(len(input_others[key])):input_others[key][i].to(tmp_dtype)- for i in range(0, len(block_names), n_blocks):- if n_blocks == 1:+ for i in range(0, len(block_names), nblocks):+ if nblocks == 1:n = block_names[i]logger.info(f"quantizing {i + 1}/{len(block_names)}, {n}")m = get_module(model, n)else:- names = block_names[i: i + n_blocks]+ names = block_names[i: i + nblocks]logger.info(names)modules = [get_module(model, n) for n in names]m = WrapperMultiblock(modules)@@ -1046,6 +1013,36 @@ class AutoRound(object):logger.error(f"export format only supports {EXPORT_FORMAT.keys()}")exit()save_quantized_as_format = EXPORT_FORMAT.get(format)+ serialization_keys = [+ "bits",+ "group_size",+ "sym",+ "data_type",+ "enable_quanted_input",+ "enable_minmax_tuning",+ "data_type",+ "seqlen",+ "train_bs",+ "scale_dtype",+ "lr",+ "minmax_lr",+ "gradient_accumulate_steps",+ "iters",+ "amp",+ "nsamples",+ "low_gpu_mem_usage",+ ]+ if isinstance(self.dataset, str):+ serialization_keys.append("dataset")+ serialization_dict = {}+ for key in serialization_keys:+ serialization_dict[key] = getattr(self, key)+ from .version import __version__++ serialization_dict["autoround_version"] = __version__+ if "scale_dtype" in serialization_dict.keys():+ serialization_dict["scale_dtype"] = str(serialization_dict["scale_dtype"])+compressed_model = save_quantized_as_format( ##TODO refine the codeoutput_dir,model=self.model,@@ -1063,7 +1060,7 @@ class AutoRound(object):tokenizer=self.tokenizer,supported_types=self.supported_types,data_type=self.data_type,- serialization_dict=self.serialization_dict,+ serialization_dict=serialization_dict,**kwargs,)return compressed_model@@ -1187,15 +1184,15 @@ class AutoOPTRound(AutoRound):low_gpu_mem_usage (bool): Whether to use low GPU memory (default is True).iters (int): Number of iterations (default is 200).seqlen (int): Length of the sequence.- n_samples (int): Number of samples (default is 512).+ nsamples (int): Number of samples (default is 512).sampler (str): The sampling method (default is "rand").seed (int): The random seed (default is 42).- n_blocks (int): Number of blocks (default is 1).+ nblocks (int): Number of blocks (default is 1).gradient_accumulate_steps (int): Number of gradient accumulation steps (default is 1).not_use_best_mse (bool): Whether to use mean squared error (default is False).dynamic_max_gap (int): The dynamic maximum gap (default is -1).data_type (str): The data type to be used (default is "int").- scale_dtype (str): The data type of quantization scale to be used (default is "float32"), different kernels+ scale_dtype (str): The data type of quantization scale to be used (default is "float16"), different kernelshave different choices.**kwargs: Additional keyword arguments.@@ -1224,10 +1221,10 @@ class AutoOPTRound(AutoRound):low_gpu_mem_usage: bool = True,iters: int = 200,seqlen: int = 2048,- n_samples: int = 512,+ nsamples: int = 512,sampler: str = "rand",seed: int = 42,- n_blocks: int = 1,+ nblocks: int = 1,gradient_accumulate_steps: int = 1,not_use_best_mse: bool = False,dynamic_max_gap: int = -1,@@ -1256,10 +1253,10 @@ class AutoOPTRound(AutoRound):low_gpu_mem_usage,iters,seqlen,- n_samples,+ nsamples,sampler,seed,- n_blocks,+ nblocks,gradient_accumulate_steps,not_use_best_mse,dynamic_max_gap,@@ -1335,10 +1332,10 @@ class AutoAdamRound(AutoOPTRound):low_gpu_mem_usage (bool): Whether to use low GPU memory (default is True).iters (int): Number of iterations (default is 200).seqlen (int): Length of the sequence.- n_samples (int): Number of samples (default is 512).+ nsamples (int): Number of samples (default is 512).sampler (str): The sampling method (default is "rand").seed (int): The random seed (default is 42).- n_blocks (int): Number of blocks (default is 1).+ nblocks (int): Number of blocks (default is 1).gradient_accumulate_steps (int): Number of gradient accumulation steps (default is 1).not_use_best_mse (bool): Whether to use mean squared error (default is False).dynamic_max_gap (int): The dynamic maximum gap (default is -1).@@ -1372,10 +1369,10 @@ class AutoAdamRound(AutoOPTRound):low_gpu_mem_usage: bool = True,iters: int = 200,seqlen: int = 2048,- n_samples: int = 512,+ nsamples: int = 512,sampler: str = "rand",seed: int = 42,- n_blocks: int = 1,+ nblocks: int = 1,gradient_accumulate_steps: int = 1,not_use_best_mse: bool = False,dynamic_max_gap: int = -1,@@ -1404,10 +1401,10 @@ class AutoAdamRound(AutoOPTRound):low_gpu_mem_usage,iters,seqlen,- n_samples,+ nsamples,sampler,seed,- n_blocks,+ nblocks,gradient_accumulate_steps,not_use_best_mse,dynamic_max_gap,diff --git a/auto_round/calib_dataset.py b/auto_round/calib_dataset.pyindex d4b4d649a..fe8a1be6e 100644--- a/auto_round/calib_dataset.py+++ b/auto_round/calib_dataset.py@@ -227,7 +227,7 @@ def get_local_dataset(tokenizer, seqlen, dataset_name="./tmp.json", split=None,return calib_dataset-def get_dataloader(tokenizer, seqlen, dataset_name="NeelNanda/pile-10k", seed=42, bs=8, n_samples=512):+def get_dataloader(tokenizer, seqlen, dataset_name="NeelNanda/pile-10k", seed=42, bs=8, nsamples=512):"""Generate a DataLoader for calibration using specified parameters.Args:@@ -239,7 +239,7 @@ def get_dataloader(tokenizer, seqlen, dataset_name="NeelNanda/pile-10k", seed=42split (str, optional): The data split to use. Defaults to None.seed (int, optional): The random seed for reproducibility. Defaults to 42.bs (int, optional): The batch size. Defaults to 4.- n_samples (int, optional): The total number of samples to include. Defaults to 512.+ nsamples (int, optional): The total number of samples to include. Defaults to 512.Returns:DataLoader: The DataLoader for the calibrated dataset.@@ -340,14 +340,14 @@ def get_dataloader(tokenizer, seqlen, dataset_name="NeelNanda/pile-10k", seed=42dataset_names = [dataset_names[index] for index in indices]cnt = 0 if not data_lens else sum(data_lens.values())dataset_cnt_info = {}- if cnt > n_samples:+ if cnt > nsamples:cnt = 0for i in range(len(datasets)):name = dataset_names[i].split(':')[0]if name not in data_lens:- target_cnt = (n_samples - cnt) // (len(datasets) - len(data_lens)) if data_lens \- else (n_samples - cnt) // (len(datasets) - i)+ target_cnt = (nsamples - cnt) // (len(datasets) - len(data_lens)) if data_lens \+ else (nsamples - cnt) // (len(datasets) - i)target_cnt = min(target_cnt, len(datasets[i]))cnt += target_cntelse:diff --git a/auto_round/export/export_to_autogptq.py b/auto_round/export/export_to_autogptq.pyindex e83324592..67f7b6765 100644--- a/auto_round/export/export_to_autogptq.py+++ b/auto_round/export/export_to_autogptq.py@@ -50,7 +50,8 @@ from ..utils import convert_dtype_torch2str_hf@register_format("auto_gptq")-def save_quantized_as_autogptq(output_dir, use_triton=True, inplace=True, **kwargs):##TODO align with autoround format+def save_quantized_as_autogptq(output_dir, use_triton=True, inplace=True,+ **kwargs):"""Export the model to autogptq format to easily leverage cuda kernel."""try:import auto_gptq@@ -58,15 +59,8 @@ def save_quantized_as_autogptq(output_dir, use_triton=True, inplace=True, **kwarraise ImportError("export to autogptq requires autogptq library. Please run 'pip install auto-gptq'")model = kwargs["model"]weight_config = kwargs["weight_config"]- sym = kwargs["sym"]bits = kwargs["bits"]group_size = kwargs["group_size"]- iters = kwargs["iters"]- lr = kwargs["lr"]- minmax_lr = kwargs["minmax_lr"]- enable_minmax_tuning = kwargs["enable_minmax_tuning"]- enable_quanted_input = kwargs["enable_quanted_input"]- scale_dtype = kwargs["scale_dtype"]tokenizer = kwargs["tokenizer"]supported_types = kwargs["supported_types"]@@ -102,172 +96,61 @@ def save_quantized_as_autogptq(output_dir, use_triton=True, inplace=True, **kwarfrom auto_gptq.modeling._utils import pack_model # pylint: disable=E0401if bits == 3 or use_triton is False:- if bits == 3 and use_triton is True:- logger.warning("triton does not support 3 bits, reset it to False")- quantizers = {}- for key in weight_config:- if key == "lm_head": ##TODO remove this after pr 87 is merged- continue- info = weight_config[key]- if not check_to_quantized(info):- continue- ##force to float32 to be compatible with torch 2.0- quantizers[key] = (None, info["scale"], info["zp"].to(torch.float32), info["g_idx"])- pack_model(- compressed_model,- quantizers,- bits,- group_size,- use_cuda_fp16=True,- desc_act=False,- force_layer_back_to_cpu=True,- use_triton=False,- )- else:- quantizers = {}- for key in weight_config:- if key == "lm_head": ##TODO remove this after pr 87 is merged- continue- info = weight_config[key]- if not check_to_quantized(info):- continue- quantizers[key] = (None, info["scale"], info["zp"].to(torch.float32), info["g_idx"])- pack_model(- compressed_model,- quantizers,- bits,- group_size,- use_cuda_fp16=True,- desc_act=False,- force_layer_back_to_cpu=True,- use_triton=True,- )- if output_dir is None:- return compressed_model-- _save_quantized_to_autogptq(+ use_triton = False+ quantizers = {}+ for key in weight_config:+ if key == "lm_head": ##TODO remove this after pr 87 is merged+ continue+ info = weight_config[key]+ if not check_to_quantized(info):+ continue+ quantizers[key] = (None, info["scale"], info["zp"].to(torch.float32), info["g_idx"])+ pack_model(compressed_model,- output_dir,- bits=bits,- group_size=group_size,- sym=sym,- iters=iters,- lr=lr,- minmax_lr=minmax_lr,- enable_minmax_tuning=enable_minmax_tuning,- enable_quanted_input=enable_quanted_input,- scale_dtype=scale_dtype,- use_safetensors=True,- modules_in_block_to_quantize=modules_in_block_to_quantize,- )---def _save_quantized_to_autogptq(- model,- save_dir: str,- bits=4,- group_size=128,- sym=False,- iters=200,- lr=5e-3,- minmax_lr=5e-3,- enable_minmax_tuning=True,- enable_quanted_input=True,- use_safetensors: bool = True,- scale_dtype=torch.float32,- safetensors_metadata: Optional[Dict[str, str]] = None,- modules_in_block_to_quantize=None,-):- """Save quantized model and configs to local disk for cuda."""- os.makedirs(save_dir, exist_ok=True)- model.to("cpu")-- model_base_name = "model"- if use_safetensors:- model_save_name = model_base_name + ".safetensors"- state_dict = model.state_dict()- state_dict = {k: v.clone().contiguous() for k, v in state_dict.items()}- if safetensors_metadata is None:- safetensors_metadata = {}- elif not isinstance(safetensors_metadata, dict):- raise TypeError("safetensors_metadata must be a dictionary.")- else:- logger.debug(f"Received safetensors_metadata: {safetensors_metadata}")- new_safetensors_metadata = {}- converted_keys = False- for key, value in safetensors_metadata.items():- if not isinstance(key, str) or not isinstance(value, str):- converted_keys = True- try:- new_key = str(key)- new_value = str(value)- except Exception as e:- raise TypeError(- "safetensors_metadata: both keys and values must be strings"- / f" and an error occurred when trying to convert them: {e}"- )- if new_key in new_safetensors_metadata:- logger.warning(- f"After converting safetensors_metadata keys to strings, the key '{new_key}' "- / "is duplicated. Ensure that all your metadata keys are strings to avoid overwriting."- )- new_safetensors_metadata[new_key] = new_value- safetensors_metadata = new_safetensors_metadata- if converted_keys:- logger.debug(- "One or more safetensors_metadata keys or values had to be converted to str()."- / f" Final safetensors_metadata: {safetensors_metadata}"- )-- # Format is required to enable Accelerate to load the metadata- # otherwise it raises an OSError- safetensors_metadata["format"] = "pt"-- # Store the quantization configuration as safetensors metadata- from auto_round import __version__-- safetensors_metadata["autoround_version"] = str(__version__)- safetensors_metadata["bits"] = str(bits)- safetensors_metadata["group_size"] = str(group_size)- safetensors_metadata["iters"] = str(iters)- safetensors_metadata["lr"] = str(lr)- safetensors_metadata["minmax_lr"] = str(minmax_lr)- safetensors_metadata["enable_minmax_tuning"] = str(enable_minmax_tuning)- safetensors_metadata["enable_quanted_input"] = str(enable_quanted_input)- safetensors_metadata["scale_dtype"] = convert_dtype_torch2str_hf(scale_dtype)- safe_save(state_dict, join(save_dir, model_save_name), safetensors_metadata)- else:- model_save_name = model_base_name + ".bin"- torch.save(model.state_dict(), join(save_dir, model_save_name))-- from auto_gptq.modeling._base import BaseQuantizeConfig # pylint: disable=E0401-- quantization_config = BaseQuantizeConfig(- bits=bits,- group_size=group_size,+ quantizers,+ bits,+ group_size,+ use_cuda_fp16=True,desc_act=False,- sym=sym,- true_sequential=False,- static_groups=False,- model_file_base_name=model_base_name,+ force_layer_back_to_cpu=True,+ use_triton=use_triton)- quantization_config.model_file_base_name = model_base_name-- config_dict = quantization_config.to_dict()- config_dict["quant_method"] = "intel/auto-round"- config_dict["autoround_version"] = __version__- config_dict["iters"] = iters- config_dict["lr"] = lr- config_dict["minmax_lr"] = minmax_lr- config_dict["enable_minmax_tuning"] = enable_minmax_tuning- config_dict["enable_quanted_input"] = enable_quanted_input- config_dict["scale_dtype"] = convert_dtype_torch2str_hf(scale_dtype)+ if output_dir is None:+ return compressed_model+ quantization_config = kwargs["serialization_dict"]+ quantization_config["quant_method"] = "gptq"+ quantization_config.pop("dataset", None) ## pile-10k is not supported in gptqif modules_in_block_to_quantize is not None:- config_dict["modules_in_block_to_quantize"] = modules_in_block_to_quantize-- with open(join(save_dir, "quantize_config.json"), "w", encoding="utf-8") as f:- json.dump(config_dict, f, indent=2)-- config_dict["quant_method"] = "gptq" ##hf transformers could only recognize this value- model.config.quantization_config = config_dict- model.config.save_pretrained(save_dir)+ quantization_config["modules_in_block_to_quantize"] = modules_in_block_to_quantize+ if hasattr(model, "config"):+ model.config.quantization_config = quantization_config+ save(compressed_model, output_dir)+++def save(model: torch.nn.Module, save_dir: str, max_shard_size: str = "5GB", safe_serialization: bool = True):+ """Save model state dict and configs.++ Args:+ model (`nn.Module`):+ Model to be saved. The model can be wrapped or unwrapped.+ save_dir (`str`):+ Directory to which to save. Will be created if it doesn't exist.+ max_shard_size (`str`, defaults to `"10GB"`):+ The maximum size for a checkpoint before being sharded. Checkpoints shard will then be each of size+ lower than this size. If expressed as a string, needs to be digits followed by a unit (like `"5MB"`).+ <Tip warning={true}>++ If a single weight of the model is bigger than `max_shard_size`, it will be in its own checkpoint shard+ which will be bigger than `max_shard_size`.++ </Tip>+ safe_serialization (`bool`, defaults to `True`):+ Whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`).+ """+ os.makedirs(save_dir, exist_ok=True)+ model.save_pretrained(save_dir, max_shard_size=max_shard_size, safe_serialization=safe_serialization)+ config_file = "quantization_config.json"+ if hasattr(model, "config") and hasattr(model.config, "quantization_config"):+ model.config.quantization_config["quant_method"] = "intel/auto-round"+ with open(os.path.join(save_dir, config_file), "w", encoding="utf-8") as f:+ json.dump(model.config.quantization_config, f, indent=2)diff --git a/examples/language-modeling/README.md b/examples/language-modeling/README.mdindex da08449af..6a633d51a 100644--- a/examples/language-modeling/README.md+++ b/examples/language-modeling/README.md@@ -56,6 +56,11 @@ CUDA_VISIBLE_DEVICES=0 python3 main.py --model_name facebook/opt-125m --bits 4set "--train_bs 1 --gradient_accumulate_steps 8" (more tuning cost)reduce the train bs to 4(potential accuracy drop)++reduce the nsamples to 128 (potential accuracy drop)++or combine them+- **Speedup the tuning:**disable_low_gpu_mem_usage(more gpu memory)@@ -64,11 +69,13 @@ reduce the train bs to 4(little accuracy drop)reduce the seqlen to 512(potential large accuracy drop)+reduce the nsamples to 128 (potential accuracy drop)+or combine them- **Enable quantized lm-head:**-Currently only support in Intel xpu,however, we found the fake tuning could improve the accuracy is some scenarios. --disable_low_gpu_mem_usage is strongly recommended if the whole model could be loaded to the device, otherwise it will be quite slow to cache the inputs of lm-head. Another way is reducing n_samples,e.g. 128, to alleviate the issue.+Currently only support in Intel xpu and AutoRound format,however, we found the fake tuning could improve the accuracy is some scenarios. --disable_low_gpu_mem_usage is strongly recommended if the whole model could be loaded to the device, otherwise it will be quite slow to cache the inputs of lm-head. Another way is reducing nsamples,e.g. 128, to alleviate the issue.```bashCUDA_VISIBLE_DEVICES=0 python3 main.py --model_name facebook/opt-125m --bits 4 --group_size 128 --quant_lm_head --disable_low_gpu_mem_usage```@@ -77,10 +84,6 @@ CUDA_VISIBLE_DEVICES=0 python3 main.py --model_name facebook/opt-125m --bits 4Include the flag `--adam`. Note that AdamW is less effective than sign gradient descent in many scenarios we tested.-- **Running the Original SignRound:**-```bash-CUDA_VISIBLE_DEVICES=0 python3 main.py --model_name facebook/opt-125m --bits 4 --group_size 128 --iters 400 --lr 0.0025 --disable_minmax_tuning --disable_quanted_input-```- **Code generation LLM:**@@ -117,7 +120,7 @@ CUDA_VISIBLE_DEVICES=1,2 python3 eval/evaluation.py --model_name /save_model_patYou can also utilize the official lm_eval [link](https://github.com/EleutherAI/lm-evaluation-harness/tree/main?tab=readme-ov-file#basic-usage).## 4. Known Issues-* Random issues in tuning Qwen models+* Random issues in tuning some models* ChatGlm-V1 is not supporteddiff --git a/examples/language-modeling/main.py b/examples/language-modeling/main.pyindex 3569ce671..690f5bdc2 100644--- a/examples/language-modeling/main.py+++ b/examples/language-modeling/main.py@@ -74,9 +74,9 @@ if __name__ == '__main__':parser.add_argument("--gradient_accumulate_steps", default=1, type=int, help="gradient accumulate steps")- parser.add_argument("--n_blocks", default=1, type=int, help="num of blocks to tune together")+ parser.add_argument("--nblocks", default=1, type=int, help="num of blocks to tune together")- parser.add_argument("--n_samples", default=512, type=int,+ parser.add_argument("--nsamples", default=512, type=int,help="number of samples")parser.add_argument("--low_gpu_mem_usage", action='store_true',@@ -298,9 +298,9 @@ if __name__ == '__main__':gpu_format = "auto_round"autoround = round(model, tokenizer, args.bits, args.group_size, sym=args.sym, batch_size=args.train_bs,- dataset=args.dataset, seqlen=seqlen, n_blocks=args.n_blocks, iters=args.iters, lr=args.lr,+ dataset=args.dataset, seqlen=seqlen, nblocks=args.nblocks, iters=args.iters, lr=args.lr,minmax_lr=args.minmax_lr, enable_quanted_input=not args.disable_quanted_input, device=device_str,- amp=not args.disable_amp, n_samples=args.n_samples,+ amp=not args.disable_amp, nsamples=args.nsamples,low_gpu_mem_usage=not args.disable_low_gpu_mem_usage,seed=args.seed, gradient_accumulate_steps=args.gradient_accumulate_steps,scale_dtype=args.scale_dtype, weight_config=weight_config,@@ -344,8 +344,10 @@ if __name__ == '__main__':if 'gpu' in deployment_device or "auto_round" in gpu_format or "auto-round" in gpu_format:model_args = f"pretrained={export_dir}-gpu"- else:+ elif "fake" in deployment_device:model_args = f"pretrained={output_dir}"+ else:+ exit() ## does not support cpu,xpu model evalres = simple_evaluate(model="hf", model_args=model_args,tasks=tasks,diff --git a/test/test_autoround_export.py b/test/test_autoround_export.pyindex c49395e8d..591064594 100644--- a/test/test_autoround_export.py+++ b/test/test_autoround_export.py@@ -56,7 +56,7 @@ class TestAutoroundExport(unittest.TestCase):model = copy.deepcopy(self.gptj)out1 = model(self.lm_input)round = AutoRound- optq_1 = round(model, self.tokenizer, n_samples=20, amp=False, seqlen=10, iters=10)+ optq_1 = round(model, self.tokenizer, nsamples=20, amp=False, seqlen=10, iters=10)q_model, weight_config1 = optq_1.quantize()from auto_round.export.export_to_itrex import pack_model@@ -72,7 +72,7 @@ class TestAutoroundExport(unittest.TestCase):model = copy.deepcopy(self.gptj)out6 = model(self.lm_input)- optq_2 = round(model, self.tokenizer, device="cpu", n_samples=20, seqlen=10)+ optq_2 = round(model, self.tokenizer, device="cpu", nsamples=20, seqlen=10)q_model, weight_config2 = optq_2.quantize()compressed_model = pack_model(model=q_model, weight_config=weight_config2, inplace=False)out4 = q_model(self.lm_input)