Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

fix glm4-9b batch dim issue (#304)

wenhuach213376edd365ee
diff --git a/auto_round/__init__.py b/auto_round/__init__.py
index c8de3ea93..f031c758f 100644
--- a/auto_round/__init__.py
+++ b/auto_round/__init__.py
@@ -13,5 +13,7 @@
# limitations under the License.
from .autoround import AutoRound, AutoRoundAdam, AutoRoundOPT
from .mllm import AutoRoundMLLM
+from auto_round.utils import LazyImport
+
from .auto_quantizer import AutoHfQuantizer,AutoRoundConfig
-from .version import __version__
\ No newline at end of file
+from .version import __version__
diff --git a/auto_round/autoround.py b/auto_round/autoround.py
index 579248876..564ba6fa3 100644
--- a/auto_round/autoround.py
+++ b/auto_round/autoround.py
@@ -25,12 +25,12 @@ import accelerate
from packaging import version
from .quantizer import WrapperMultiblock, wrapper_block, unwrapper_block, WrapperLinear, unwrapper_layer, \
WrapperTransformerConv1d
-from .special_model_handler import (check_hidden_state_dim,
- shareable_keywords,
- special_model_init,
- reset_params,
- check_skippable_keywords
- )
+from .special_model_handler import (
+ shareable_keywords,
+ init_cache_for_special_model,
+ reset_params,
+ check_skippable_keywords
+)
from .utils import (
CpuInfo,
block_forward,
@@ -206,7 +206,7 @@ class AutoRound(object):
self.dynamic_max_gap = dynamic_max_gap
self.lr_scheduler = lr_scheduler
self.optimizer = self.get_optimizer(None)
- self.input_dim = None
+ self.batch_dim = None
self.infer_bs_coeff = 1
self.set_layerwise_config(self.layer_config) ##better place in the end
@@ -250,6 +250,12 @@ class AutoRound(object):
if "mx_fp" in self.data_type and self.group_size != 32:
logger.warning("mx_fp should only support group_size of 32 in real deployment")
+ if self.batch_size > self.nsamples:
+ logger.warning(
+ f"reset batch_size to nsamples({self.nsamples}) "
+ f"as batch_size({self.batch_size}) must be smaller than nsamples")
+ self.batch_size = self.nsamples
+
def quantize(self):
"""Quantize the model and return the quantized model along with layer configurations.
the entry of AutoRound.
@@ -283,8 +289,8 @@ class AutoRound(object):
keys = inputs.keys()
input_id_str = [key for key in keys if key.startswith('hidden_state')]
if len(input_id_str) != 1:
- raise RuntimeError("hidden_states arg mismatch error," \
- " please check the input kwargs of block forward for more details.")
+ raise RuntimeError(f"hidden_states arg mismatch error,"
+ "please raise an issue in https://github.com/intel/auto-round/issues")
inputs["input_ids"] = inputs.pop(input_id_str[0], None)
clear_memory(self.inputs)
@@ -462,12 +468,12 @@ class AutoRound(object):
input_others,
indices,
self.seqlen,
- self.input_dim
+ self.batch_dim
)
tmp_output = block_forward(block, tmp_input_ids, tmp_input_others, self.amp, self.amp_dtype, device).to(
cache_device
)
- output.extend(list(torch.split(tmp_output, 1, dim=self.input_dim)))
+ output.extend(list(torch.split(tmp_output, 1, dim=self.batch_dim)))
if self.low_gpu_mem_usage:
clear_memory()
@@ -547,7 +553,7 @@ class AutoRound(object):
pass
except RuntimeError as error:
logger.warning("When quantization encounters tensor" \
- " shape mismatch error, you can try to avoid it with batch_size=1")
+ " shape mismatch error, you can try to avoid it with batch_size=1")
logger.error(error)
pass
except Exception as error:
@@ -672,7 +678,7 @@ class AutoRound(object):
Returns:
function: The forward function.
"""
-
+
def post_process_cache_data(batch_size, data, data_name):
"""
Processes store data for batch handling, reshaping if necessary.
@@ -710,18 +716,28 @@ class AutoRound(object):
"""
if name not in self.inputs:
self.inputs[name] = {}
- special_model_init(self.model, positional_inputs, self.inputs[name])
-
- if self.input_dim is None:
- self.input_dim = check_hidden_state_dim(self.model, positional_inputs)
-
+ init_cache_for_special_model(self.model, positional_inputs, self.inputs[name])
+
+ if self.batch_dim is None :
+ self.batch_dim = 0
+ if hidden_states is not None and self.batch_size>1:
+ if hidden_states.shape[0] > self.batch_size:
+ self.batch_dim = 1
+ if len(hidden_states.shape) > 1 and hidden_states.shape[1] > self.batch_size:
+ logger.error(
+ f"this model has not been supported, "
+ f"please raise an issue in https://github.com/intel/auto-round/issues"
+ f" or try to set the `batch_size` to 1 and "
+ f"`gradient_accumulate_steps` to your current batch size.")
+ exit(-1)
+
if hidden_states is not None:
kwargs['hidden_states'] = hidden_states
for key in kwargs.keys():
if isinstance(kwargs[key], torch.Tensor) or isinstance(kwargs[key], list) \
or isinstance(kwargs[key], tuple):
- if key not in self.inputs[name].keys(): # initialization
+ if key not in self.inputs[name].keys(): # initialization
data = to_device(kwargs[key], device=torch.device("cpu"))
if data is None or (self.batch_size > 1 and key in shareable_keywords):
self.inputs[name][key] = data
@@ -730,16 +746,16 @@ class AutoRound(object):
self.inputs[name][key] = [data]
else:
data = post_process_cache_data(self.batch_size, data, key)
- self.inputs[name][key] = list(torch.split(data, 1, dim=self.input_dim))
- else: # append cache inputs
+ self.inputs[name][key] = list(torch.split(data, 1, dim=self.batch_dim))
+ else: # append cache inputs
new_data = post_process_cache_data(self.batch_size, kwargs[key], key)
- if new_data is None: # shareable args or NoneType
+ if new_data is None: # shareable args or NoneType
continue
new_data = to_device(new_data, device=torch.device("cpu"))
if self.batch_size <= 1:
self.inputs[name][key].append(new_data)
else:
- self.inputs[name][key].extend(list(torch.split(new_data, 1, dim=self.input_dim)))
+ self.inputs[name][key].extend(list(torch.split(new_data, 1, dim=self.batch_dim)))
elif isinstance(kwargs[key], (str, bool, type(None))):
if key not in self.inputs[name].keys():
self.inputs[name][key] = kwargs[key]
@@ -756,7 +772,7 @@ class AutoRound(object):
kwargs.pop('hidden_states')
return m.orig_forward(hidden_states, *positional_inputs, **kwargs)
else:
- #Currently only for Llama-3.2-Vision-Instruct Series
+ # Currently only for Llama-3.2-Vision-Instruct Series
return m.orig_forward(*positional_inputs, **kwargs)
return forward
@@ -996,11 +1012,11 @@ class AutoRound(object):
input_others,
indices,
seqlen=self.seqlen,
- input_dim=self.input_dim,
+ batch_dim=self.batch_dim,
)
current_output = [output[x] for x in indices]
- current_output = torch.cat(current_output, dim=self.input_dim)
+ current_output = torch.cat(current_output, dim=self.batch_dim)
current_output = to_device(current_output, device)
@@ -1627,4 +1643,3 @@ class AutoRoundAdam(AutoRoundOPT):
optimizer=optimizer,
**kwargs,
)
-
diff --git a/auto_round/script/llm.py b/auto_round/script/llm.py
index 871e0e410..181fd53ef 100644
--- a/auto_round/script/llm.py
+++ b/auto_round/script/llm.py
@@ -43,116 +43,122 @@ from auto_round.eval.evaluation import simple_evaluate
from auto_round.utils import detect_device, get_library_version, detect_device_count
from auto_round.utils import logger
+
class BasicArgumentParser(argparse.ArgumentParser):
def __init__(self, *args, **kwargs):
- super().__init__( *args, **kwargs)
- self.add_argument("--model", default="facebook/opt-125m", help="model name or path")
+ super().__init__(*args, **kwargs)
+ self.add_argument("--model", "--model_name", "--model_name_or_path", default="facebook/opt-125m",
+ help="model name or path")
self.add_argument('--eval', action='store_true',
- help="whether to use eval only mode.")
+ help="whether to use eval only mode")
self.add_argument("--bits", default=4, type=int,
- help="number of bits")
+ help="number of weight bits")
self.add_argument("--eval_bs", default=None, type=int,
- help="eval batch size")
+ help="batch size in evaluation")
self.add_argument("--device", default="auto", type=str,
- help="The device to be used for tuning. The default is set to auto/None,"
- "allowing for automatic detection."
- " Currently, device settings support CPU, GPU, and HPU.")
-
+ help="the device to be used for tuning. The default is set to auto,"
+ "allowing for automatic detection."
+ "Currently, device settings support CPU, GPU, and HPU.")
+
self.add_argument("--asym", action='store_true',
- help=" asym quantization")
+ help="whether to use asym quantization")
self.add_argument("--dataset", default="NeelNanda/pile-10k", type=str,
- help="The dataset for quantization training. It can be a custom one.")
+ help="the dataset for quantization training")
self.add_argument("--lr", default=None, type=float,
- help="learning rate, if None, it will be set to 1.0/iters automatically")
+ help="learning rate, if None, it will be set to 1.0/iters automatically")
self.add_argument("--minmax_lr", default=None, type=float,
- help="minmax learning rate, if None,it will beset to be the same with lr")
+ help="minmax learning rate, if None, it will beset to be the same with lr")
self.add_argument("--seed", default=42, type=int,
- help="seed")
+ help="random seed")
self.add_argument("--adam", action='store_true',
- help="adam")
-
- self.add_argument("--gradient_accumulate_steps", default=1, type=int, help="gradient accumulate steps")
+ help="whether to use adam optimizer instead of SignSGD")
+
+ self.add_argument("--gradient_accumulate_steps", default=1, type=int,
+ help="gradient accumulate steps")
- self.add_argument("--nblocks", default=1, type=int, help="num of blocks to tune together")
+ self.add_argument("--nblocks", default=1, type=int,
+ help="how many blocks to tune together")
self.add_argument("--low_gpu_mem_usage", action='store_true',
- help="lower gpu memory usage but 50-100% slower")
-
+ help="offload intermediate features to cpu")
+
self.add_argument("--format", default=None, type=str,
- help="The format in which to save the model. "
- "The options are 'auto_round', 'auto_round:gptq','auto_round:awq',"
- " 'auto_gptq', 'auto_awq', 'itrex', 'itrex_xpu' and 'fake'."
- "default to 'auto_round."
- )
+ choices=["auto_round", "auto_gptq", "auto_awq", "auto_round:gptq", "auto_round:auto_gptq",
+ "auto_round:auto_gptq:marlin", "auto_round:gptq:marlin", "auto_round:auto_awq",
+ "auto_round:awq", "auto_awq", "itrex", "iterx_xpu", "fake"],
+ help="the format to save the model"
+ )
- self.add_argument("--data_type", default='int',
- help="data type for tuning, 'int', 'mx_fp' and etc.")
+ self.add_argument("--data_type", "--dtype", default='int',
+ help="data type for tuning, 'int', 'mx_fp' and etc")
- self.add_argument("--scale_dtype", default='fp16',
- help="which scale data type to use for quantization, 'fp16', 'fp32' or 'bf16'.")
+ self.add_argument("--scale_dtype", default='fp16', choices=["fp16", "float16",
+ "bf16", "bfloat16", "fp32", "float32"],
+ help="scale data type to use for quantization")
self.add_argument("--tasks",
- default="lambada_openai,hellaswag,winogrande,piqa,mmlu,wikitext,truthfulqa_mc1," \
- "truthfulqa_mc2,openbookqa,boolq,rte,arc_easy,arc_challenge",
- help="lm-eval tasks for lm_eval version 0.4")
+ default="lambada_openai,hellaswag,winogrande,piqa,mmlu,wikitext,truthfulqa_mc1," \
+ "truthfulqa_mc2,openbookqa,boolq,rte,arc_easy,arc_challenge",
+ help="lm-eval tasks")
self.add_argument("--output_dir", default="./tmp_autoround", type=str,
- help="Where to store the final model.")
+ help="the directory to save quantized model")
self.add_argument("--disable_eval", action='store_true',
- help="Whether to do lm-eval evaluation after tuning.")
+ help="whether to do lm-eval evaluation after tuning")
self.add_argument("--disable_amp", action='store_true',
- help="disable amp")
+ help="disable amp")
self.add_argument("--disable_minmax_tuning", action='store_true',
- help="whether disable enable weight minmax tuning")
+ help="whether to disable enable weight minmax tuning")
self.add_argument("--enable_norm_bias_tuning", action='store_true',
- help="whether enable norm bias tuning")
+ help="whether to enable norm bias tuning")
self.add_argument("--disable_trust_remote_code", action='store_true',
- help="Whether to disable trust_remote_code")
+ help="whether to disable trust_remote_code")
self.add_argument("--disable_quanted_input", action='store_true',
- help="whether to disuse the output of quantized block to tune the next block")
+ help="whether to disuse the output of quantized block to tune the next block")
self.add_argument("--quant_lm_head", action='store_true',
- help="quant_lm_head")
+ help="whether to quant lm_head")
- self.add_argument("--low_cpu_mem_mode", default=0, type=int,
- help="Choose which low cpu memory mode to use. "
- "Can significantly reduce cpu memory footprint but cost more time."
- "1 means choose block-wise mode, load the weights of each block"
- " from disk when tuning and release the memory of the block after tuning."
- "2 means choose layer-wise mode, load the weights of each layer from disk when tuning,"
- " minimum memory consumption and also slowest running speed."
- "others means not use low cpu memory. Default to 0, not use low cpu memory.")
+ self.add_argument("--low_cpu_mem_mode", default=0, type=int, choices=[0, 1, 2],
+ help="choose which low cpu memory mode to use. "
+ "Can significantly reduce cpu memory footprint but cost more time."
+ "1 means choose block-wise mode, load the weights of each block"
+ " from disk when tuning and release the memory of the block after tuning."
+ "2 means choose layer-wise mode, load the weights of each layer from disk when tuning,"
+ " minimum memory consumption and also slowest running speed."
+ "others means not use low cpu memory. Default to 0, not use low cpu memory.")
self.add_argument("--low_cpu_mem_tmp_dir", default=None, type=str,
- help="temp work space to store the temporary files "
- "when using low cpu memory mode. Will remove after tuning.")
+ help="temporary work space to store the temporary files "
+ "when using low cpu memory mode. Will remove after tuning.")
- self.add_argument("--model_dtype", default=None, type=str,
- help="force to convert the dtype, some backends supports fp16 dtype better")
+ self.add_argument("--model_dtype", default=None, type=str, choices=["fp16", "float16",
+ "bf16", "bfloat16", "fp32", "float32"],
+ help="force to convert the dtype, some backends supports fp16 dtype better")
self.add_argument("--act_bits", default=16, type=int,
- help="activation bits")
+ help="activation bits")
+
+ self.add_argument("--fp_layers", default="", type=str,
+ help="list of Layer names to maintain original data type")
- self.add_argument("--fp_layers_list", default="", type=str,
- help="List of Layers to maintain original data type")
-
self.add_argument("--not_use_best_mse", action='store_true',
- help="To determine whether the quantization should handle vision component.")
+ help="whether to use the iter of best mes loss in the tuning phase")
def setup_parser():
@@ -161,14 +167,14 @@ def setup_parser():
parser.add_argument("--group_size", default=128, type=int,
help="group size")
- parser.add_argument("--batch_size", default=8, type=int,
+ parser.add_argument("--batch_size", "--train_bs", default=8, type=int,
help="train batch size")
- parser.add_argument("--iters", default=200, type=int,
- help=" iters")
+ parser.add_argument("--iters", "--iter", default=200, type=int,
+ help="iteration to tune each block")
- parser.add_argument("--seqlen", default=2048, type=int,
- help="sequence length")
+ parser.add_argument("--seqlen", "--seq_len", default=2048, type=int,
+ help="sequence length of the calibration samples")
parser.add_argument("--nsamples", default=128, type=int,
help="number of samples")
@@ -176,20 +182,21 @@ def setup_parser():
args = parser.parse_args()
return args
+
def setup_best_parser():
parser = BasicArgumentParser()
parser.add_argument("--group_size", default=128, type=int,
help="group size")
- parser.add_argument("--batch_size", default=8, type=int,
+ parser.add_argument("--batch_size", "--train_bs", default=8, type=int,
help="train batch size")
- parser.add_argument("--iters", default=1000, type=int,
- help=" iters")
+ parser.add_argument("--iters","--iter", default=1000, type=int,
+ help="iterations to tune each block")
- parser.add_argument("--seqlen", default=2048, type=int,
- help="sequence length")
+ parser.add_argument("--seqlen", "--seq_len", default=2048, type=int,
+ help="sequence length of the calibration samples")
parser.add_argument("--nsamples", default=512, type=int,
help="number of samples")
@@ -199,20 +206,21 @@ def setup_best_parser():
return args
+
def setup_fast_parser():
parser = BasicArgumentParser()
parser.add_argument("--group_size", default=128, type=int,
help="group size")
- parser.add_argument("--batch_size", default=4, type=int,
+ parser.add_argument("--batch_size", "--train_bs", default=4, type=int,
help="train batch size")
parser.add_argument("--iters", default=200, type=int,
- help=" iters")
+ help="iterations to tune each block")
- parser.add_argument("--seqlen", default=512, type=int,
- help="sequence length")
+ parser.add_argument("--seqlen", "--seq_len", default=512, type=int,
+ help="sequence length of the calibration samples")
parser.add_argument("--nsamples", default=128, type=int,
help="number of samples")
@@ -221,6 +229,7 @@ def setup_fast_parser():
return args
+
def tune(args):
tasks = args.tasks
if args.format is None:
@@ -303,7 +312,6 @@ def tune(args):
if args.model_dtype == "bfloat16" or args.model_dtype == "bfp16":
model = model.to(torch.bfloat16)
-
if hasattr(tokenizer, "model_max_length"):
if tokenizer.model_max_length < seqlen:
logger.info(
@@ -326,12 +334,12 @@ def tune(args):
logger.info(
f"{n} will not be quantized due to its shape not being divisible by 32,"
" resulting in an exporting issue to autogptq")
- fp_layers_list = args.fp_layers_list.split(",")
- if bool(fp_layers_list):
+ fp_layers = args.fp_layers.split(",")
+ if bool(fp_layers):
for n, m in model.named_modules():
if isinstance(m, torch.nn.Linear) or isinstance(m, transformers.modeling_utils.Conv1D):
name = n.split('.')[-1]
- if n in fp_layers_list or name in fp_layers_list:
+ if n in fp_layers or name in fp_layers:
layer_config[n] = {"bits": 16}
logger.info(
f"{n} will not be quantized.")
@@ -376,12 +384,11 @@ def tune(args):
if "cpu" not in device_str:
torch.cuda.empty_cache()
- if model_name.split('/')[-1] == ".":
- export_dir = os.path.join(args.output_dir, f"w{args.bits}g{args.group_size}")
+ if model_name.split('/')[-1].strip('.') == "":
+ export_dir = os.path.join(args.output_dir, f"w{args.bits}g{args.group_size}")
else:
export_dir = os.path.join(args.output_dir, model_name.split('/')[-1] + f"-w{args.bits}g{args.group_size}")
-
format_list = args.format.replace(' ', '').split(',')
inplace = False if len(format_list) > 1 else True
for format_ in format_list:
@@ -432,4 +439,3 @@ def eval(args):
from lm_eval.utils import make_table # pylint: disable=E0401
print(make_table(res))
-
diff --git a/auto_round/script/mllm.py b/auto_round/script/mllm.py
index 3ec7404db..aba755584 100644
--- a/auto_round/script/mllm.py
+++ b/auto_round/script/mllm.py
@@ -25,138 +25,146 @@ from transformers import AutoModelForCausalLM, AutoTokenizer, AutoConfig, AutoPr
from auto_round.utils import detect_device
from auto_round.utils import logger
+
class BasicArgumentParser(argparse.ArgumentParser):
def __init__(self, *args, **kwargs):
- super().__init__( *args, **kwargs)
- self.add_argument("--model", default="facebook/opt-125m", help="model name or path")
+ super().__init__(*args, **kwargs)
+ self.add_argument("--model", "--model_name", "--model_name_or_path",
+ default="facebook/opt-125m",
+ help="model name or path")
self.add_argument('--eval', action='store_true',
- help="whether to use eval only mode.")
+ help="whether to use eval only mode.")
self.add_argument("--bits", default=4, type=int,
- help="number of bits")
+ help="weight bits")
self.add_argument("--eval_bs", default=None, type=int,
- help="eval batch size")
+ help="batch size in evaluation")
self.add_argument("--device", default="auto", type=str,
- help="The device to be used for tuning. The default is set to auto/None,"
- "allowing for automatic detection."
- " Currently, device settings support CPU, GPU, and HPU.")
-
+ help="the device to be used for tuning. The default is set to auto,"
+ "allowing for automatic detection."
+ "Currently, device settings support CPU, GPU, and HPU.")
+
self.add_argument("--asym", action='store_true',
- help=" asym quantization")
+ help="whether to use asym quantization")
self.add_argument("--dataset", type=str, default=None,
- help="The dataset for quantization training. It can be a custom one.")
+ help="the dataset for quantization training. It can be a custom one.")
self.add_argument("--lr", default=None, type=float,
- help="learning rate, if None, it will be set to 1.0/iters automatically")
+ help="learning rate, if None, it will be set to 1.0/iters automatically")
self.add_argument("--minmax_lr", default=None, type=float,
- help="minmax learning rate, if None,it will beset to be the same with lr")
+ help="minmax learning rate, if None,it will beset to be the same with lr")
self.add_argument("--seed", default=42, type=int,
- help="seed")
+ help="random seed")
self.add_argument("--adam", action='store_true',
- help="adam")
-
- self.add_argument("--gradient_accumulate_steps", default=1, type=int, help="gradient accumulate steps")
+ help="whether to use adam optimizer instead of SignSGD")
- self.add_argument("--nblocks", default=1, type=int, help="num of blocks to tune together")
+ self.add_argument("--gradient_accumulate_steps", default=1, type=int,
+ help="gradient accumulate steps")
+
+ self.add_argument("--nblocks", default=1, type=int,
+ help="how many blocks to tune together")
self.add_argument("--low_gpu_mem_usage", action='store_true',
- help="lower gpu memory usage but 50-100% slower")
-
- self.add_argument("--format", default=None, type=str,
- help="The format in which to save the model. "
- "The options are 'auto_round', 'auto_round:gptq','auto_round:awq',"
- " and 'fake'.default to 'auto_round."
- )
+ help="offload intermediate features to cpu")
+
+ self.add_argument("--format", default="auto_round", type=str,
+ choices=["auto_round", "auto_round:gptq", "auto_round:auto_gptq", "auto_round:awq",
+ "auto_round:auto_awq", "fake"],
+ help="the format to save the model"
+ )
- self.add_argument("--data_type", default='int',
- help="data type for tuning, 'int', 'mx_fp' and etc.")
+ self.add_argument("--data_type", "--dtype", default='int',
+ help="data type for tuning, 'int', 'mx_fp' and etc")
- self.add_argument("--scale_dtype", default='fp16',
- help="which scale data type to use for quantization, 'fp16', 'fp32' or 'bf16'.")
+ self.add_argument("--scale_dtype", default='fp16', choices=["fp16", "float16",
+ "bf16", "bfloat16", "fp32", "float32"],
+ help="scale data type to use for quantization")
self.add_argument("--output_dir", default="./tmp_autoround", type=str,
- help="Where to store the final model.")
+ help="the directory to save quantized model")
self.add_argument("--disable_amp", action='store_true',
- help="disable amp")
+ help="disable amp")
self.add_argument("--disable_minmax_tuning", action='store_true',
- help="whether disable enable weight minmax tuning")
+ help="whether disable enable weight minmax tuning")
self.add_argument("--enable_norm_bias_tuning", action='store_true',
- help="whether enable norm bias tuning")
+ help="whether enable norm bias tuning")
self.add_argument("--disable_trust_remote_code", action='store_true',
- help="Whether to disable trust_remote_code")
+ help="whether to disable trust_remote_code")
self.add_argument("--disable_quanted_input", action='store_true',
- help="whether to disuse the output of quantized block to tune the next block")
+ help="whether to disuse the output of quantized block to tune the next block")
self.add_argument("--quant_lm_head", action='store_true',
- help="quant_lm_head")
+ help="whether to quant lm_head")
- self.add_argument("--low_cpu_mem_mode", default=0, type=int,
- help="Choose which low cpu memory mode to use. "
- "Can significantly reduce cpu memory footprint but cost more time."
- "1 means choose block-wise mode, load the weights of each block"
- " from disk when tuning and release the memory of the block after tuning."
- "2 means choose layer-wise mode, load the weights of each layer from disk when tuning,"
- " minimum memory consumption and also slowest running speed."
- "others means not use low cpu memory. Default to 0, not use low cpu memory.")
+ self.add_argument("--low_cpu_mem_mode", default=0, type=int, choices=[0, 1, 2],
+ help="choose which low cpu memory mode to use. "
+ "Can significantly reduce cpu memory footprint but cost more time."
+ "1 means choose block-wise mode, load the weights of each block"
+ " from disk when tuning and release the memory of the block after tuning."
+ "2 means choose layer-wise mode, load the weights of each layer from disk when tuning,"
+ " minimum memory consumption and also slowest running speed."
+ "others means not use low cpu memory. Default to 0, not use low cpu memory.")
self.add_argument("--low_cpu_mem_tmp_dir", default=None, type=str,
- help="temp work space to store the temporary files "
- "when using low cpu memory mode. Will remove after tuning.")
+ help="temporary work space to store the temporary files "
+ "when using low cpu memory mode. Will remove after tuning.")
- self.add_argument("--model_dtype", default=None, type=str,
- help="force to convert the dtype, some backends supports fp16 dtype better")
+ self.add_argument("--model_dtype", default=None, type=str, choices=["fp16", "float16",
+ "bf16", "bfloat16", "fp32", "float32"],
+ help="force to convert the dtype, some backends supports fp16 dtype better")
self.add_argument("--act_bits", default=32, type=int,
- help="activation bits")
+ help="activation bits")
+
+ self.add_argument("--fp_layers", default="", type=str,
+ help="layers to maintain original data type")
- self.add_argument("--fp_layers_list", default="", type=str,
- help="List of Layers to maintain original data type")
-
self.add_argument("--not_use_best_mse", action='store_true',
- help="To determine whether the quantization should handle vision component.")
+ help="whether to use the iter of best mes loss in the tuning phase")
+
## ======================= VLM =======================
self.add_argument("--quant_nontext_module", action='store_true',
- help="To determine whether the quantization should handle vision component.")
+ help="whether to quantize non-text module, e.g. vision component")
self.add_argument("--extra_data_dir", default="", type=str,
- help="Dataset dir for storing images/audio/videos. "
- "Can be a dir path or multiple dir path with format as "
- "'image=path_to_image,video=path_to_video,audio=path_to_audio'"
- "By default, it will search in the relative path.")
-
+ help="dataset dir for storing images/audio/videos. "
+ "Can be a dir path or multiple dir path with format as "
+ "'image=path_to_image,video=path_to_video,audio=path_to_audio'"
+ "By default, it will search in the relative path.")
+
self.add_argument("--template", default=None, type=str,
- help="The template for building training dataset. It can be a custom one.")
-
+ help="the template for building training dataset. It can be a custom one.")
+
## ======================= VLM eval=======================
self.add_argument("--tasks", type=str, default="COCO_VAL",
help="eval tasks for VLMEvalKit.")
# Args that only apply to Video Dataset
self.add_argument("--nframe", type=int, default=8,
- help="The number of frames to sample from a video,"
- " only applicable to the evaluation of video benchmarks.")
- self.add_argument("--pack", action='store_true',
- help="A video may associate with multiple questions, if pack==True,"
- " will ask all questions for a video in a single")
+ help="the number of frames to sample from a video,"
+ " only applicable to the evaluation of video benchmarks.")
+ self.add_argument("--pack", action='store_true',
+ help="a video may associate with multiple questions, if pack==True,"
+ " will ask all questions for a video in a single")
self.add_argument("--use-subtitle", action='store_true')
self.add_argument("--fps", type=float, default=-1)
# Work Dir
# Infer + Eval or Infer Only
self.add_argument("--mode", type=str, default='all', choices=['all', 'infer'],
- help="When mode set to 'all', will perform both inference and evaluation;"
- " when set to 'infer' will only perform the inference.")
+ help="when mode set to 'all', will perform both inference and evaluation;"
+ " when set to 'infer' will only perform the inference.")
self.add_argument('--eval_data_dir', type=str, default=None,
help='path for VLMEvalKit to store the eval data. Default will store in ~/LMUData')
# API Kwargs, Apply to API VLMs and Judge API LLMs
@@ -167,23 +175,24 @@ class BasicArgumentParser(argparse.ArgumentParser):
self.add_argument('--verbose', action='store_true')
# Configuration for Resume
# Ignore: will not rerun failed VLM inference
- self.add_argument('--ignore', action='store_true', help='Ignore failed indices. ')
+ self.add_argument('--ignore', action='store_true', help='ignore failed indices. ')
# Rerun: will remove all evaluation temp files
self.add_argument('--rerun', action='store_true')
+
def setup_parser():
parser = BasicArgumentParser()
parser.add_argument("--group_size", default=128, type=int,
help="group size")
- parser.add_argument("--batch_size", default=1, type=int,
+ parser.add_argument("--batch_size", "--train_bs", default=1, type=int,
help="train batch size")
- parser.add_argument("--iters", default=200, type=int,
+ parser.add_argument("--iters", "--iter", default=200, type=int,
help=" iters")
- parser.add_argument("--seqlen", default=2048, type=int,
+ parser.add_argument("--seqlen", "--seq_len", default=2048, type=int,
help="sequence length")
parser.add_argument("--nsamples", default=128, type=int,
@@ -192,10 +201,11 @@ def setup_parser():
args = parser.parse_args()
return args
+
def tune(args):
if args.format is None:
args.format = "auto_round"
-
+
model_name = args.model
if model_name[-1] == "/":
model_name = model_name[:-1]
@@ -207,7 +217,7 @@ def tune(args):
torch_dtype = "auto"
if "hpu" in device_str:
torch_dtype = torch.bfloat16
-
+
# load_model
config = AutoConfig.from_pretrained(model_name, trust_remote_code=not args.disable_trust_remote_code)
tokenizer = AutoTokenizer.from_pretrained(model_name)
@@ -223,7 +233,7 @@ def tune(args):
else:
cls = AutoModelForCausalLM
model = cls.from_pretrained(
- model_name,trust_remote_code=not args.disable_trust_remote_code, torch_dtype=torch_dtype)
+ model_name, trust_remote_code=not args.disable_trust_remote_code, torch_dtype=torch_dtype)
if "cogvlm2" in model_name:
model.config.model_type = "cogvlm2"
@@ -240,7 +250,7 @@ def tune(args):
if m.weight.shape[0] % 32 != 0 or m.weight.shape[1] % 32 != 0:
layer_config[n] = {"bits": 32}
logger.info(
- f"{n} will not be quantized due to its shape not being divisible by 32,"
+ f"{n} will not be quantized due to its shape not being divisible by 32,"
" resulting in an exporting issue to autogptq")
lm_head_layer_name = "lm_head"
for n, _ in model.named_modules():
@@ -266,7 +276,7 @@ def tune(args):
if args.quant_lm_head and args.low_gpu_mem_usage:
print(f"warning, low_gpu_mem_usage=False is strongly recommended if the whole model could be loaded to "
f"gpu")
-
+
autoround = round(model, tokenizer, dataset=args.dataset, extra_data_dir=args.extra_data_dir,
bits=args.bits, group_size=args.group_size, sym=not args.asym,
batch_size=args.batch_size, seqlen=seqlen, nblocks=args.nblocks, iters=args.iters,
@@ -281,9 +291,9 @@ def tune(args):
model.eval()
if args.device != "cpu":
torch.cuda.empty_cache()
-
- if model_name.split('/')[-1] == ".":
- export_dir = os.path.join(args.output_dir, f"w{args.bits}g{args.group_size}")
+
+ if model_name.split('/')[-1].strip('.') == "":
+ export_dir = os.path.join(args.output_dir, f"w{args.bits}g{args.group_size}")
else:
export_dir = os.path.join(args.output_dir, model_name.split('/')[-1] + f"-w{args.bits}g{args.group_size}")
@@ -304,12 +314,12 @@ def eval(args):
data_store_dir=args.eval_data_dir,
dataset=args.tasks,
pack=args.pack,
- use_subtitle = args.use_subtitle,
- fps = args.fps,
- nframe = args.nframe,
- rerun = args.rerun,
- judge = args.judge,
- verbose = args.verbose,
- mode = args.mode,
- ignore = args.ignore
- )
\ No newline at end of file
+ use_subtitle=args.use_subtitle,
+ fps=args.fps,
+ nframe=args.nframe,
+ rerun=args.rerun,
+ judge=args.judge,
+ verbose=args.verbose,
+ mode=args.mode,
+ ignore=args.ignore
+ )
diff --git a/auto_round/special_model_handler.py b/auto_round/special_model_handler.py
index d0f07c3f4..c8d1fdcbd 100644
--- a/auto_round/special_model_handler.py
+++ b/auto_round/special_model_handler.py
@@ -14,7 +14,6 @@
import torch
from collections import UserDict
-special_states_dim_tuple = ("chatglm",) # input_dim is not the default dimension 0
shareable_keywords = ("position_ids", "cache_position", "position_embeddings")
mllms_with_limited_bs = ("llava", "qwen2-vl", "phi3_v", "mllama") # Limitations on batch_size
skippable_cache_keys = ("past_key_value",)
@@ -50,24 +49,7 @@ def to_device(input, device=torch.device("cpu")):
return input
-def check_hidden_state_dim(model, positional_inputs):
- """Check the concatenable dimension of hidden states.
-
- Args:
- positional_inputs: The positional arguments.
-
- Returns:
- int: 1 if the model type is 'chatglm' and positional arguments are not None, 0 otherwise.
- """
- is_special = False
- for key in special_states_dim_tuple:
- if hasattr(model, "config") and key in model.config.model_type:
- is_special = True
- break
- return int(is_special and positional_inputs is not None)
-
-
-def special_model_init(model, positional_inputs, inputs):
+def init_cache_for_special_model(model, positional_inputs, inputs):
"""
Initializes special model inputs by adding positional inputs if missing.
diff --git a/auto_round/utils.py b/auto_round/utils.py
index 9b21a15cf..d12d2164e 100644
--- a/auto_round/utils.py
+++ b/auto_round/utils.py
@@ -348,7 +348,7 @@ def collect_best_params(block):
@torch.no_grad()
def sampling_inputs(input_ids, input_others, indices, seqlen,
- input_dim=0):
+ batch_dim=0):
"""Samples inputs based on the given indices and sequence length.
Args:
@@ -362,7 +362,7 @@ def sampling_inputs(input_ids, input_others, indices, seqlen,
current_input_others: The sampled other input data.
"""
current_input_ids = [input_ids[i] for i in indices]
- current_input_ids = torch.cat(current_input_ids, dim=input_dim)
+ current_input_ids = torch.cat(current_input_ids, dim=batch_dim)
current_input_others = {"positional_inputs": input_others["positional_inputs"]}
for key in input_others.keys():
if "positional_inputs" in key:
diff --git a/examples/language-modeling/main.py b/examples/language-modeling/main.py
index eab5069a1..f55551d33 100644
--- a/examples/language-modeling/main.py
+++ b/examples/language-modeling/main.py
@@ -31,7 +31,7 @@ os.environ["TOKENIZERS_PARALLELISM"] = "false"
if __name__ == '__main__':
parser.add_argument(
- "--model_name", default="facebook/opt-125m"
+ "--model_name", "--model", "--model_name_or_path", default="facebook/opt-125m"
)
parser.add_argument("--bits", default=4, type=int,
@@ -40,7 +40,7 @@ if __name__ == '__main__':
parser.add_argument("--group_size", default=128, type=int,
help="group size")
- parser.add_argument("--batch_size", default=8, type=int,
+ parser.add_argument("--batch_size", "--train_bs", default=8, type=int,
help="train batch size")
parser.add_argument("--eval_bs", default=None, type=int,
@@ -53,7 +53,7 @@ if __name__ == '__main__':
parser.add_argument("--asym", action='store_true',
help=" asym quantization")
- parser.add_argument("--iters", default=200, type=int,
+ parser.add_argument("--iters", "--iter", default=200, type=int,
help=" iters")
parser.add_argument("--dataset", default="NeelNanda/pile-10k", type=str,
@@ -77,7 +77,7 @@ if __name__ == '__main__':
parser.add_argument("--adam", action='store_true',
help="adam")
- parser.add_argument("--seqlen", default=2048, type=int,
+ parser.add_argument("--seqlen", "--seq_len", default=2048, type=int,
help="sequence length")
parser.add_argument("--gradient_accumulate_steps", default=1, type=int, help="gradient accumulate steps")
@@ -88,22 +88,20 @@ if __name__ == '__main__':
help="number of samples")
parser.add_argument("--low_gpu_mem_usage", action='store_true',
- help="lower gpu memory but 50%-100% slower")
+ help="offload intermediate features to cpu")
parser.add_argument("--enable_minmax_tuning", action='store_true',
help="enable_minmax_tuning is deprecated")
- parser.add_argument("--deployment_device", default=None, type=str,
- help="targeted inference acceleration platform,The options are 'fake', 'cpu', 'xpu', 'gpu(auto_gptq)' and 'auto_round'."
- "default to 'auto_round', 'fake' indicating that it only performs fake quantization and won't be exported to any device.")
parser.add_argument("--format", default=None, type=str,
+ choices=["auto_round", "auto_gptq", "auto_awq", "auto_round:gptq", "auto_round:auto_gptq",
+ "auto_round:auto_gptq:marlin", "auto_round:gptq:marlin", "auto_round:auto_awq",
+ "auto_round:awq", "auto_awq", "itrex", "iterx_xpu", "fake"],
help="The format in which to save the model. "
- "The options are 'auto_round', 'auto_gptq', 'auto_awq', 'itrex', 'itrex_xpu' and 'fake'."
- "default to 'auto_round."
)
- parser.add_argument("--data_type", default='int',
+ parser.add_argument("--data_type", "--dtype", default='int',
help="data type for tuning, 'int', 'mx_fp' and etc.")
parser.add_argument("--scale_dtype", default='fp16',
@@ -115,10 +113,10 @@ if __name__ == '__main__':
help="lm-eval tasks for lm_eval version 0.4")
parser.add_argument("--output_dir", default="./tmp_autoround", type=str,
- help="Where to store the final model.")
+ help="the directory to save quantized model")
parser.add_argument("--disable_eval", action='store_true',
- help="Whether to do lmeval evaluation.")
+ help="whether to do lmeval evaluation.")
parser.add_argument("--disable_amp", action='store_true',
help="disable amp")
@@ -175,22 +173,6 @@ if __name__ == '__main__':
tasks = args.tasks
use_eval_legacy = False
- if args.format and args.deployment_device:
- assert False, "please only specify one of format and deployment_device"
-
- if args.deployment_device is None and args.format is None:
- args.format = "auto_round"
-
- if args.deployment_device:
- warnings.warn("The deployment_device is deprecated and will be removed in future version."
- "Please use format instead", DeprecationWarning)
- if "gpu" in args.deployment_device and args.asym is True:
- print(
- "warning: The auto_gptq kernel has issues with asymmetric quantization. It is recommended to use sym quantization or --format='auto_round'")
-
- if "marlin" in args.deployment_device and args.asym is True:
- assert False, "marlin backend only supports sym quantization, please remove --asym"
-
model_name = args.model_name
if model_name[-1] == "/":
model_name = model_name[:-1]
@@ -318,10 +300,6 @@ if __name__ == '__main__':
break
if args.quant_lm_head:
layer_config[lm_head_layer_name] = {"bits": args.bits}
- transformers_version = [int(item) for item in transformers.__version__.split('.')[:2]]
- if transformers_version[0] == 4 and transformers_version[1] < 38:
- error_message = "Please upgrade transformers>=4.38.0 to support lm-head quantization."
- raise EnvironmentError(error_message)
autoround = round(model, tokenizer, args.bits, args.group_size, sym=not args.asym, batch_size=args.batch_size,
dataset=args.dataset, seqlen=seqlen, nblocks=args.nblocks, iters=args.iters, lr=args.lr,
@@ -344,56 +322,20 @@ if __name__ == '__main__':
if "cpu" not in device_str:
torch.cuda.empty_cache()
- if model_name.split('/')[-1] == ".":
+ if model_name.split('/')[-1].strip(".") == "":
export_dir = os.path.join(args.output_dir, f"w{args.bits}g{args.group_size}")
else:
export_dir = os.path.join(args.output_dir, model_name.split('/')[-1] + f"-w{args.bits}g{args.group_size}")
eval_folder = None
- if args.format:
- format_list = args.format.replace(' ', '').split(',')
- inplace = False if len(format_list) > 1 else True
- for format_ in format_list:
- save_format_ = format_.replace(":", "-")
- save_format_ = save_format_.replace("_", "-")
- eval_folder = f'{export_dir}-{save_format_}'
- autoround.save_quantized(eval_folder, format=format_, inplace=inplace)
- else:
- deployment_device = args.deployment_device.split(',')
- gpu_formats = []
- for item in deployment_device:
- if item in ["gpu", "auto_gptq", "auto_round", "auto_awq"]:
- if item == "gpu":
- if lm_head_layer_name in layer_config.keys() and layer_config[lm_head_layer_name][
- "data_type"] == "int":
- gpu_formats.append("auto_round")
- else:
- gpu_formats.append("auto_gptq")
- else:
- gpu_formats.append(item)
-
- gpu_formats = list(set(gpu_formats))
-
- inplace = True if len(deployment_device) < 2 else False
- for gpu_format in gpu_formats:
- eval_folder = f'{export_dir}-{gpu_format}'
- autoround.save_quantized(eval_folder, format=gpu_format, inplace=inplace)
-
- if 'xpu' in deployment_device:
- autoround.save_quantized(f'{export_dir}-xpu', format="itrex_xpu", use_triton=True, inplace=inplace)
- if "cpu" in deployment_device:
- autoround.save_quantized(output_dir=f'{export_dir}-cpu', format='itrex', inplace=inplace)
- if "fake" in deployment_device:
- output_dir = f'{export_dir}-qdq'
- model = model.to("cpu")
- model.save_pretrained(output_dir)
- tokenizer.save_pretrained(output_dir)
- if eval_folder is None:
- eval_folder = output_dir
-
- if (not ('gpu' in deployment_device or len(gpu_formats) > 0)) and 'fake' not in deployment_device:
- print('does not support cpu, xpu model evaluation.')
- exit() ## does not support cpu,xpu model eval
+
+ format_list = args.format.replace(' ', '').split(',')
+ inplace = False if len(format_list) > 1 else True
+ for format_ in format_list:
+ save_format_ = format_.replace(":", "-")
+ save_format_ = save_format_.replace("_", "-")
+ eval_folder = f'{export_dir}-{save_format_}'
+ autoround.save_quantized(eval_folder, format=format_, inplace=inplace)
if args.disable_eval:
exit()
@@ -437,8 +379,9 @@ if __name__ == '__main__':
user_model = model.to(device_str)
if args.eval_bs == "auto":
args.eval_bs = 16
- from auto_round.eval.evaluation import simple_evaluate_user_model
- res = simple_evaluate_user_model(user_model, tokenizer,tasks=tasks,batch_size=args.eval_bs)
+ from auto_round.eval.evaluation import simple_evaluate_user_model
+
+ res = simple_evaluate_user_model(user_model, tokenizer, tasks=tasks, batch_size=args.eval_bs)
else:
res = simple_evaluate(model="hf", model_args=model_args,
tasks=tasks,
@@ -446,4 +389,3 @@ if __name__ == '__main__':
from lm_eval.utils import make_table
print(make_table(res))
-
diff --git a/requirements.txt b/requirements.txt
index d010ea536..698d8c34b 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -3,7 +3,7 @@ datasets
py-cpuinfo
sentencepiece
torch
-transformers
+transformers>=4.38
triton
numpy < 2.0
threadpoolctl