Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

Fix packing hang, torch compile and force to fp16 at exporting (#430)

Wenhua Cheng40ab98254d2b
diff --git a/auto_round/autoround.py b/auto_round/autoround.py
index 813856eb8..b0955f15b 100644
--- a/auto_round/autoround.py
+++ b/auto_round/autoround.py
@@ -52,7 +52,7 @@ from .utils import (
mv_module_from_gpu,
unsupport_meta_device, clear_memory,
compile_func,
- find_matching_blocks
+ find_matching_blocks, is_debug_mode
)
from .low_cpu_mem.utils import get_layers_before_block
@@ -231,7 +231,24 @@ class AutoRound(object):
self.model = self.model.to(torch.bfloat16)
else:
logger.info(f"using {self.model.dtype} for quantization tuning")
+
self.enable_torch_compile = enable_torch_compile
+ if self.act_bits <= 8 and self.enable_torch_compile != False:
+ self.enable_torch_compile = False
+ logger.warning("reset enable_torch_compile to `False` as activation quantization is enabled")
+
+ if self.low_cpu_mem_usage == True and self.enable_torch_compile != False:
+ self.enable_torch_compile = False
+ logger.warning("reset enable_torch_compile to `False` as low_cpu_mem_usage is enabled")
+
+ if is_debug_mode() and self.enable_torch_compile != False:
+ self.enable_torch_compile = False
+ logger.warning("reset enable_torch_compile to `False` as debug mode is enabled")
+
+ if ("fp8" in self.data_type or "fp8" in self.act_data_type) and self.enable_torch_compile != False:
+ self.enable_torch_compile = False
+ logger.warning("reset enable_torch_compile to `False` as fp8 is enabled")
+
if is_optimum_habana_available():
logger.info("Optimum Habana is available, import htcore explicitly.")
import habana_frameworks.torch.core as htcore # pylint: disable=E0401
@@ -477,7 +494,15 @@ class AutoRound(object):
self.model = mv_module_from_gpu(self.model, self.low_cpu_mem_usage)
clear_memory()
device = next(self.model.parameters()).device
- quant_layer = compile_func(self.quant_layer, device, self.enable_torch_compile)
+ if self.enable_torch_compile != False:
+ try:
+ quant_layer = compile_func(self.quant_layer, device, self.enable_torch_compile)
+ except:
+ logger.warning("torch compile failed, reset it to `False`")
+ self.enable_torch_compile = False
+ quant_layer = self.quant_layer
+ else:
+ quant_layer = self.quant_layer
for layer_name in layer_names:
layer_input = layer_inputs[layer_name]
layer_input = to_device(layer_input, self.cache_device)
@@ -654,8 +679,8 @@ class AutoRound(object):
exit(-1)
elif total_cnt < nsamples:
logger.warning(
- f"Insufficient number of samples collected may affect the quantification. "
- f"target samples count is {nsamples}, while valid samples count is {total_cnt}"
+ f"An insufficient number of samples likely reduces the accuracy of the quantized model."
+ f"Target samples count is {nsamples}, while valid samples count is {total_cnt}"
)
# clean embed weight to save memory
@@ -1287,7 +1312,15 @@ class AutoRound(object):
elif isinstance(input_others[key], list):
for i in range(len(input_others[key])):
to_dtype(input_others[key][i], tmp_dtype)
- quant_block = compile_func(self.quant_block, device, self.enable_torch_compile)
+ if self.enable_torch_compile != False:
+ try:
+ quant_block = compile_func(self.quant_block, device, self.enable_torch_compile)
+ except:
+ logger.warning("torch compile failed, reset it to `False`")
+ self.enable_torch_compile = False
+ quant_block = self.quant_block
+ else:
+ quant_block = self.quant_block
if pbar is None:
pbar = tqdm(range(0, len(block_names), nblocks))
diff --git a/auto_round/calib_dataset.py b/auto_round/calib_dataset.py
index 1fd8a5cd0..ad4f3f0dc 100644
--- a/auto_round/calib_dataset.py
+++ b/auto_round/calib_dataset.py
@@ -100,14 +100,14 @@ def get_pile_dataset(tokenizer, seqlen, dataset_name="NeelNanda/pile-10k", split
from datasets import load_dataset
split = "train"
-
+
tokenizer_function = get_tokenizer_function(tokenizer, seqlen, apply_chat_template=apply_chat_template)
try:
calib_dataset = load_dataset(dataset_name, split=split)
except Exception as e:
logger.error(f"Failed to load the dataset: {e}." \
- "Consider using a backup dataset by `pip install modelscope`" \
- " and set '--dataset swift/pile-val-backup' in AutoRound API.")
+ "Consider using a backup dataset by `pip install modelscope`" \
+ " and set '--dataset swift/pile-val-backup' in AutoRound API.")
sys.exit(1)
calib_dataset = calib_dataset.shuffle(seed=seed)
calib_dataset = calib_dataset.map(tokenizer_function, batched=True)
@@ -117,7 +117,7 @@ def get_pile_dataset(tokenizer, seqlen, dataset_name="NeelNanda/pile-10k", split
@register_dataset("swift/pile-val-backup")
def get_pile_val_dataset(tokenizer, seqlen, dataset_name="swift/pile-val-backup", split=None, seed=42,
- apply_chat_template=False):
+ apply_chat_template=False):
"""Returns a dataloader for the specified dataset and split.
Args:
@@ -134,14 +134,14 @@ def get_pile_val_dataset(tokenizer, seqlen, dataset_name="swift/pile-val-backup"
from datasets import load_dataset
split = "validation"
-
+
tokenizer_function = get_tokenizer_function(tokenizer, seqlen, apply_chat_template=apply_chat_template)
from transformers.utils.versions import require_version
require_version("modelscope", "Loading 'swift/pile-val-backup' dataset requires modelscope to be installed, " \
- "`pip install modelscope`")
- from modelscope import MsDataset # pylint: disable=E0401
+ "`pip install modelscope`")
+ from modelscope import MsDataset # pylint: disable=E0401
calib_dataset = MsDataset.load('swift/pile-val-backup',
- 'default', split=split).to_iterable_dataset() #, use_streaming=True
+ 'default', split=split).to_iterable_dataset() # , use_streaming=True
calib_dataset = calib_dataset.take(10000)
calib_dataset = calib_dataset.shuffle(seed=seed)
calib_dataset = calib_dataset.map(tokenizer_function, batched=True)
@@ -480,7 +480,7 @@ def get_dataloader(
return False
input_ids = example["input_ids"][:seqlen]
input_ids_list = input_ids.tolist()
- if len(input_ids_list) > 1 and input_ids_list.count(input_ids_list[-1]) > seqlen // 2:
+ if len(input_ids_list) > 1 and seqlen > 2 and input_ids_list.count(input_ids_list[-1]) > seqlen // 2:
return False
return True
@@ -639,4 +639,3 @@ def get_dataloader(
calib_dataloader = DataLoader(dataset_final, batch_size=bs, shuffle=False, collate_fn=collate_batch)
return calib_dataloader
-
diff --git a/auto_round/export/export_to_autogptq/export.py b/auto_round/export/export_to_autogptq/export.py
index fe4707983..7fb7d465a 100644
--- a/auto_round/export/export_to_autogptq/export.py
+++ b/auto_round/export/export_to_autogptq/export.py
@@ -169,7 +169,11 @@ def save_quantized_as_autogptq(output_dir, inplace=True, backend="auto_gptq:exll
layer_config = kwargs["layer_config"]
names = list(layer_config.keys())
- with ThreadPoolExecutor(max_workers=2) as executor:
+ max_workers = 1
+ if not torch.cuda.is_available():
+ max_workers = 2 ## 2 with cuda packing will cause hang occasionally
+
+ with ThreadPoolExecutor(max_workers=max_workers) as executor:
with tqdm(total=len(names), leave=True) as pbar:
def wrapper(name):
pack_layer(name, model, layer_config, backend, pbar)
@@ -188,11 +192,14 @@ def save_quantized_as_autogptq(output_dir, inplace=True, backend="auto_gptq:exll
quantization_config["modules_in_block_to_quantize"] = modules_in_block_to_quantize
if hasattr(model, "config"):
model.config.quantization_config = quantization_config
- save(model, output_dir, safe_serialization=safe_serialization)
+
+ dtype = torch.float16 ##force dtype to fp16
+ save(model, output_dir, safe_serialization=safe_serialization, dtype=dtype)
return model
-def save(model: torch.nn.Module, save_dir: str, max_shard_size: str = "5GB", safe_serialization: bool = True):
+def save(model: torch.nn.Module, save_dir: str, max_shard_size: str = "5GB", safe_serialization: bool = True,
+ dtype=None):
"""Save model state dict and configs.
Args:
@@ -215,6 +222,14 @@ def save(model: torch.nn.Module, save_dir: str, max_shard_size: str = "5GB", saf
##max_shard_size = "10000GB" ## API of auto-gptq with marlin does not support shard size
os.makedirs(save_dir, exist_ok=True)
model.save_pretrained(save_dir, max_shard_size=max_shard_size, safe_serialization=safe_serialization)
+ config_path = os.path.join(save_dir, "config.json")
+ if dtype is not None and dtype != model.dtype and os.path.exists(os.path.join(save_dir, "config.json")):
+ with open(config_path, 'r') as file:
+ data = json.load(file)
+ data["torch_dtype"] = str(dtype).split(".")[-1]
+ with open(config_path, 'w') as file:
+ json.dump(data, file, indent=2)
+
config_file = "quantize_config.json"
if hasattr(model, "config") and hasattr(model.config, "quantization_config"):
with open(os.path.join(save_dir, config_file), "w", encoding="utf-8") as f:
diff --git a/auto_round/export/export_to_autoround/export.py b/auto_round/export/export_to_autoround/export.py
index ee36b4ebd..48a8c96a3 100644
--- a/auto_round/export/export_to_autoround/export.py
+++ b/auto_round/export/export_to_autoround/export.py
@@ -226,7 +226,10 @@ def save_quantized_as_autoround(output_dir, inplace=True, backend="auto_round:ex
if len(extra_config) > 0:
quantization_config["extra_config"] = extra_config
names = list(layer_config.keys())
- with ThreadPoolExecutor(max_workers=2) as executor:
+ max_workers = 1
+ if not torch.cuda.is_available():
+ max_workers = 2 ## 2 with cuda packing will cause hang occasionally
+ with ThreadPoolExecutor(max_workers=max_workers) as executor:
with tqdm(total=len(names), leave=True) as pbar:
def wrapper(name):
pack_layer(name, model, layer_config, backend, pbar)
@@ -238,7 +241,7 @@ def save_quantized_as_autoround(output_dir, inplace=True, backend="auto_round:ex
model.config.quantization_config = quantization_config
if output_dir is None:
return model
-
+
if output_dir is None:
model.tokenizer = tokenizer
return model
@@ -247,12 +250,13 @@ def save_quantized_as_autoround(output_dir, inplace=True, backend="auto_round:ex
if processor is not None:
processor.save_pretrained(output_dir)
- save(model, output_dir, safe_serialization=safe_serialization)
+ dtype = torch.float16 ##force dtype to fp16
+ save(model, output_dir, safe_serialization=safe_serialization, dtype=dtype)
return model
-def save(model: nn.Module, save_dir: str, max_shard_size: str = "5GB", safe_serialization: bool = True):
+def save(model: nn.Module, save_dir: str, max_shard_size: str = "5GB", safe_serialization: bool = True, dtype=None):
"""Save model state dict and configs.
Args:
@@ -274,10 +278,14 @@ def save(model: nn.Module, save_dir: str, max_shard_size: str = "5GB", safe_seri
"""
os.makedirs(save_dir, exist_ok=True)
model.save_pretrained(save_dir, max_shard_size=max_shard_size, safe_serialization=safe_serialization)
+ config_path = os.path.join(save_dir, "config.json")
+ if dtype is not None and dtype != model.dtype and os.path.exists(os.path.join(save_dir, "config.json")):
+ with open(config_path, 'r') as file:
+ data = json.load(file)
+ data["torch_dtype"] = str(dtype).split(".")[-1]
+ with open(config_path, 'w') as file:
+ json.dump(data, file, indent=2)
config_file = "quantization_config.json"
if hasattr(model, "config") and hasattr(model.config, "quantization_config"):
with open(os.path.join(save_dir, config_file), "w", encoding="utf-8") as f:
json.dump(model.config.quantization_config, f, indent=2)
-
-
-
diff --git a/auto_round/export/export_to_awq/export.py b/auto_round/export/export_to_awq/export.py
index edcd61773..eae25c1bf 100644
--- a/auto_round/export/export_to_awq/export.py
+++ b/auto_round/export/export_to_awq/export.py
@@ -54,8 +54,8 @@ def pack_layer(name, model, layer_config, backend, pbar):
bits = config["bits"]
group_size = config["group_size"]
linear_layer = get_module(model, name)
- if config["sym"] :
- zp = 2** (config["bits"]-1)
+ if config["sym"]:
+ zp = 2 ** (config["bits"] - 1)
q_linear = WQLinear_GEMM.from_linear(
linear=linear_layer,
w_bit=bits,
@@ -76,7 +76,7 @@ def save_quantized_as_autoawq(output_dir, inplace=True, **kwargs):
tokenizer = kwargs.get("tokenizer", None)
processor = kwargs.get("processor", None)
modules_to_not_convert = []
-
+
logger.info("Saving quantized model to auto_awq format")
if tokenizer is not None:
tokenizer.save_pretrained(output_dir)
@@ -97,7 +97,10 @@ def save_quantized_as_autoawq(output_dir, inplace=True, **kwargs):
names = list(layer_config.keys())
backend = None
- with ThreadPoolExecutor(max_workers=2) as executor:
+ max_workers = 1
+ if not torch.cuda.is_available():
+ max_workers = 2 ## 2 with cuda packing will cause hang occasionally
+ with ThreadPoolExecutor(max_workers=max_workers) as executor:
with tqdm(total=len(names), leave=True) as pbar:
def wrapper(name):
pack_layer(name, compressed_model, layer_config, backend, pbar)
@@ -111,13 +114,13 @@ def save_quantized_as_autoawq(output_dir, inplace=True, **kwargs):
if output_dir is None:
return compressed_model
-
+
layer_config = kwargs["layer_config"]
for key in layer_config.keys():
if not check_to_quantized(layer_config[key]) and \
not any(name in key for name in modules_to_not_convert):
modules_to_not_convert.append(key)
-
+
quantization_config["quant_method"] = "awq"
quantization_config["zero_point"] = not quantization_config["sym"]
quantization_config["version"] = "gemm"
@@ -128,12 +131,13 @@ def save_quantized_as_autoawq(output_dir, inplace=True, **kwargs):
if hasattr(compressed_model, "config"):
compressed_model.config.quantization_config = quantization_config
safe_serialization = kwargs.get('safe_serialization', True)
- save(compressed_model, output_dir, safe_serialization=safe_serialization)
+ dtype = torch.float16 ##force dtype to fp16
+ save(compressed_model, output_dir, safe_serialization=safe_serialization, dtype=dtype)
return compressed_model
-def save(model: nn.Module, save_dir: str, max_shard_size: str = "5GB", safe_serialization: bool = True):
+def save(model: nn.Module, save_dir: str, max_shard_size: str = "5GB", safe_serialization: bool = True, dtype=None):
"""Save model state dict and configs.
Args:
@@ -155,8 +159,14 @@ def save(model: nn.Module, save_dir: str, max_shard_size: str = "5GB", safe_seri
"""
os.makedirs(save_dir, exist_ok=True)
model.save_pretrained(save_dir, max_shard_size=max_shard_size, safe_serialization=safe_serialization)
+ config_path = os.path.join(save_dir, "config.json")
+ if dtype is not None and dtype != model.dtype and os.path.exists(os.path.join(save_dir, "config.json")):
+ with open(config_path, 'r') as file:
+ data = json.load(file)
+ data["torch_dtype"] = str(dtype).split(".")[-1]
+ with open(config_path, 'w') as file:
+ json.dump(data, file, indent=2)
config_file = "quantization_config.json"
if hasattr(model, "config") and hasattr(model.config, "quantization_config"):
with open(os.path.join(save_dir, config_file), "w", encoding="utf-8") as f:
json.dump(model.config.quantization_config, f, indent=2)
-
diff --git a/auto_round/script/llm.py b/auto_round/script/llm.py
index f6b6e1b4d..e44cd9f0d 100644
--- a/auto_round/script/llm.py
+++ b/auto_round/script/llm.py
@@ -28,10 +28,13 @@
import os
import re
import argparse
-from auto_round.utils import get_fp_layer_names, set_cuda_visible_devices, clear_memory
+import sys
+
+from auto_round.utils import get_fp_layer_names, set_cuda_visible_devices, clear_memory, is_debug_mode
os.environ["CUBLAS_WORKSPACE_CONFIG"] = ":4096:8"
+
class BasicArgumentParser(argparse.ArgumentParser):
def __init__(self, *args, **kwargs):
@@ -51,10 +54,10 @@ class BasicArgumentParser(argparse.ArgumentParser):
default="0",
type=str,
help="the device to be used for tuning. "
- "Currently, device settings support CPU, GPU, and HPU."
- "The default is set to cuda:0,"
- "allowing for automatic detection and switch to HPU or CPU."
- "set --device 0,1,2 to use multiple cards.")
+ "Currently, device settings support CPU, GPU, and HPU."
+ "The default is set to cuda:0,"
+ "allowing for automatic detection and switch to HPU or CPU."
+ "set --device 0,1,2 to use multiple cards.")
self.add_argument("--asym", action='store_true', help="whether to use asym quantization")
@@ -123,19 +126,19 @@ class BasicArgumentParser(argparse.ArgumentParser):
type=int,
choices=[0, 1, 2],
help="choose which low cpu memory mode to use. "
- "Can significantly reduce cpu memory footprint but cost more time."
- "1 means choose block-wise mode, load the weights of each block"
- " from disk when tuning and release the memory of the block after tuning."
- "2 means choose layer-wise mode, load the weights of each layer from disk when tuning,"
- " minimum memory consumption and also slowest running speed."
- "others means not use low cpu memory. Default to 0, not use low cpu memory.")
+ "Can significantly reduce cpu memory footprint but cost more time."
+ "1 means choose block-wise mode, load the weights of each block"
+ " from disk when tuning and release the memory of the block after tuning."
+ "2 means choose layer-wise mode, load the weights of each layer from disk when tuning,"
+ " minimum memory consumption and also slowest running speed."
+ "others means not use low cpu memory. Default to 0, not use low cpu memory.")
self.add_argument(
"--low_cpu_mem_tmp_dir",
default=None,
type=str,
help="temporary work space to store the temporary files "
- "when using low cpu memory mode. Will remove after tuning.")
+ "when using low cpu memory mode. Will remove after tuning.")
self.add_argument(
"--model_dtype",
@@ -160,13 +163,14 @@ class BasicArgumentParser(argparse.ArgumentParser):
type=str,
help="Names of quantitative blocks, please use commas to separate them.")
- self.add_argument("--enable_torch_compile", default=None, type=bool, help="whether to enable torch compile")
+ self.add_argument("--disable_torch_compile", action='store_true',
+ help="whether to disable torch compile")
self.add_argument("--act_data_type", default=None, type=str, help="activation data type")
self.add_argument("--disable_act_dynamic", action='store_true', help="activation static quantization")
-
- self.add_argument("--disable_deterministic_algorithms", action='store_true',
+
+ self.add_argument("--disable_deterministic_algorithms", action='store_true',
help="disable torch deterministic algorithms.")
self.add_argument("--device_map", default=None, type=str, help="device_map for block in tuning phase")
@@ -184,10 +188,10 @@ class EvalArgumentParser(argparse.ArgumentParser):
default="0",
type=str,
help="the device to be used for tuning. "
- "Currently, device settings support CPU, GPU, and HPU."
- "The default is set to cuda:0,"
- "allowing for automatic detection and switch to HPU or CPU."
- "set --device 0,1,2 to use multiple cards.")
+ "Currently, device settings support CPU, GPU, and HPU."
+ "The default is set to cuda:0,"
+ "allowing for automatic detection and switch to HPU or CPU."
+ "set --device 0,1,2 to use multiple cards.")
self.add_argument("--tasks",
default="lambada_openai,hellaswag,winogrande,piqa,mmlu,wikitext,truthfulqa_mc1," \
"openbookqa,boolq,arc_easy,arc_challenge",
@@ -327,9 +331,8 @@ def tune(args):
args = _gguf_args_check(args)
if "auto_gptq" in args.format and args.asym is True:
- print(
- "warning: The auto_gptq kernel has issues with asymmetric quantization. "
- "It is recommended to use sym quantization or --format='auto_round'")
+ logger.warning("The auto_gptq kernel has issues with asymmetric quantization. "
+ "It is recommended to use sym quantization or --format='auto_round'")
if "marlin" in args.format and args.asym is True:
assert False, "marlin backend only supports sym quantization, please remove --asym"
@@ -340,9 +343,12 @@ def tune(args):
import torch
if not args.disable_deterministic_algorithms:
torch.use_deterministic_algorithms(True, warn_only=True)
- print("'torch.use_deterministic_algorithms' is turned on by default for reproducibility, "\
- "and can be turned off by setting the '--disable_deterministic_algorithms' parameter.")
-
+ # logger.info("`torch.use_deterministic_algorithms` is enabled by default for reproducibility "
+ # "and can be disabled using the `--disable_deterministic_algorithms` argument.")
+
+ if not args.disable_torch_compile:
+ logger.info("`torch.compile` is enabled by default to reduce tuning costs. "
+ "If it causes issues, you can disable it using the `--disable_torch_compile` argument.")
model_name = args.model
if model_name[-1] == "/":
@@ -469,6 +475,7 @@ def tune(args):
if not awq_supported:
logger.warning(f"The AutoAWQ format may not be supported due to {info}")
+ enable_torch_compile = False if "--disable_torch_compile" in sys.argv else None
autoround = round(
model,
@@ -499,7 +506,7 @@ def tune(args):
enable_norm_bias_tuning=args.enable_norm_bias_tuning,
not_use_best_mse=args.not_use_best_mse,
to_quant_block_names=args.to_quant_block_names,
- enable_torch_compile=args.enable_torch_compile,
+ enable_torch_compile=enable_torch_compile,
act_data_type=args.act_data_type,
act_dynamic=not args.disable_act_dynamic,
device_map=args.device_map)
@@ -551,7 +558,7 @@ def tune(args):
dispatch_model(model, model.hf_device_map)
user_model = model
else:
- user_model = model #.to(device_str)
+ user_model = model # .to(device_str)
if args.eval_bs is None or args.eval_bs == "auto":
args.eval_bs = 16
@@ -567,7 +574,8 @@ def tune(args):
def _eval_init(tasks, model_path, device, disable_trust_remote_code=False):
device_str, parallelism = set_cuda_visible_devices(device)
- model_args = f"pretrained={model_path},trust_remote_code={not disable_trust_remote_code}"
+ ##model_args = f'pretrained={model_path},trust_remote_code={not disable_trust_remote_code},add_bos_token=True'
+ model_args = f'pretrained={model_path},trust_remote_code={not disable_trust_remote_code}'
if parallelism:
model_args += ",parallelize=True"
if isinstance(tasks, str):
diff --git a/auto_round/script/mllm.py b/auto_round/script/mllm.py
index a7c597f4a..884ac9201 100644
--- a/auto_round/script/mllm.py
+++ b/auto_round/script/mllm.py
@@ -16,7 +16,7 @@ import os
import sys
import argparse
-from auto_round.utils import clear_memory, get_fp_layer_names, set_cuda_visible_devices, logger
+from auto_round.utils import clear_memory, get_fp_layer_names, set_cuda_visible_devices, logger, is_debug_mode
os.environ["CUBLAS_WORKSPACE_CONFIG"] = ":4096:8"
@@ -44,10 +44,10 @@ class BasicArgumentParser(argparse.ArgumentParser):
default="0",
type=str,
help="the device to be used for tuning. "
- "Currently, device settings support CPU, GPU, and HPU."
- "The default is set to cuda:0,"
- "allowing for automatic detection and switch to HPU or CPU."
- "set --device 0,1,2 to use multiple cards.")
+ "Currently, device settings support CPU, GPU, and HPU."
+ "The default is set to cuda:0,"
+ "allowing for automatic detection and switch to HPU or CPU."
+ "set --device 0,1,2 to use multiple cards.")
self.add_argument("--asym", action='store_true', help="whether to use asym quantization")
@@ -56,9 +56,9 @@ class BasicArgumentParser(argparse.ArgumentParser):
type=str,
default=None,
help="the dataset for quantization training."
- " current support NeelNanda/pile-10k,liuhaotian/llava_conv_58k,"
- "liuhaotian/llava_instruct_80k,liuhaotian/llava_instruct_150k"
- "It can be a custom one. Default is NeelNanda/pile-10k")
+ " current support NeelNanda/pile-10k,liuhaotian/llava_conv_58k,"
+ "liuhaotian/llava_instruct_80k,liuhaotian/llava_instruct_150k"
+ "It can be a custom one. Default is NeelNanda/pile-10k")
self.add_argument(
"--lr", default=None, type=float, help="learning rate, if None, it will be set to 1.0/iters automatically")
@@ -115,19 +115,19 @@ class BasicArgumentParser(argparse.ArgumentParser):
type=int,
choices=[0, 1, 2],
help="choose which low cpu memory mode to use. "
- "Can significantly reduce cpu memory footprint but cost more time."
- "1 means choose block-wise mode, load the weights of each block"
- " from disk when tuning and release the memory of the block after tuning."
- "2 means choose layer-wise mode, load the weights of each layer from disk when tuning,"
- " minimum memory consumption and also slowest running speed."
- "others means not use low cpu memory. Default to 0, not use low cpu memory.")
+ "Can significantly reduce cpu memory footprint but cost more time."
+ "1 means choose block-wise mode, load the weights of each block"
+ " from disk when tuning and release the memory of the block after tuning."
+ "2 means choose layer-wise mode, load the weights of each layer from disk when tuning,"
+ " minimum memory consumption and also slowest running speed."
+ "others means not use low cpu memory. Default to 0, not use low cpu memory.")
self.add_argument(
"--low_cpu_mem_tmp_dir",
default=None,
type=str,
help="temporary work space to store the temporary files "
- "when using low cpu memory mode. Will remove after tuning.")
+ "when using low cpu memory mode. Will remove after tuning.")
self.add_argument(
"--model_dtype",
@@ -145,9 +145,10 @@ class BasicArgumentParser(argparse.ArgumentParser):
action='store_true',
help="whether to use the iter of best mes loss in the tuning phase")
- self.add_argument("--enable_torch_compile", default=None, type=bool, help="whether to enable torch compile")
-
- self.add_argument("--disable_deterministic_algorithms", action='store_true',
+ self.add_argument("--disable_torch_compile", action='store_true',
+ help="whether to disable torch compile")
+
+ self.add_argument("--disable_deterministic_algorithms", action='store_true',
help="disable torch deterministic algorithms.")
## ======================= VLM =======================
@@ -161,10 +162,10 @@ class BasicArgumentParser(argparse.ArgumentParser):
default=None,
type=str,
help="dataset dir for storing images/audio/videos. "
- "Can be a dir path or multiple dir path with format as "
- "'image=path_to_image,video=path_to_video,audio=path_to_audio'"
- "By default, it will search in the relative path, "
- "and if not find, will automatic download.")
+ "Can be a dir path or multiple dir path with format as "
+ "'image=path_to_image,video=path_to_video,audio=path_to_audio'"
+ "By default, it will search in the relative path, "
+ "and if not find, will automatic download.")
self.add_argument(
"--template",
@@ -176,7 +177,7 @@ class BasicArgumentParser(argparse.ArgumentParser):
"--truncation",
action="store_true",
help="whether to truncate sequences at the maximum length."
- " Default True for pile and False for llava dataset.")
+ " Default True for pile and False for llava dataset.")
self.add_argument(
"--to_quant_block_names",
@@ -218,10 +219,10 @@ def setup_lmeval_parser():
default="0",
type=str,
help="the device to be used for tuning. "
- "Currently, device settings support CPU, GPU, and HPU."
- "The default is set to cuda:0,"
- "allowing for automatic detection and switch to HPU or CPU."
- "set --device 0,1,2 to use multiple cards.")
+ "Currently, device settings support CPU, GPU, and HPU."
+ "The default is set to cuda:0,"
+ "allowing for automatic detection and switch to HPU or CPU."
+ "set --device 0,1,2 to use multiple cards.")
parser.add_argument(
"--tasks",
type=str,
@@ -233,12 +234,12 @@ def setup_lmeval_parser():
type=int,
default=8,
help="the number of frames to sample from a video,"
- " only applicable to the evaluation of video benchmarks.")
+ " only applicable to the evaluation of video benchmarks.")
parser.add_argument(
"--pack",
action='store_true',
help="a video may associate with multiple questions, if pack==True,"
- " will ask all questions for a video in a single")
+ " will ask all questions for a video in a single")
parser.add_argument("--fps", type=float, default=-1, help="set the fps for a video.")
# Work Dir
# Infer + Eval or Infer Only
@@ -248,7 +249,7 @@ def setup_lmeval_parser():
default='all',
choices=['all', 'infer'],
help="when mode set to 'all', will perform both inference and evaluation;"
- " when set to 'infer' will only perform the inference.")
+ " when set to 'infer' will only perform the inference.")
parser.add_argument(
'--eval_data_dir',
type=str,
@@ -294,8 +295,8 @@ def tune(args):
import torch
if not args.disable_deterministic_algorithms:
torch.use_deterministic_algorithms(True, warn_only=True)
- print("'torch.use_deterministic_algorithms' is turned on by default for reproducibility, "\
- "and can be turned off by setting the '--disable_deterministic_algorithms' parameter.")
+ print("'torch.use_deterministic_algorithms' is turned on by default for reproducibility, " \
+ "and can be turned off by setting the '--disable_deterministic_algorithms' parameter.")
model_name = args.model
if model_name[-1] == "/":
@@ -437,6 +438,8 @@ def tune(args):
if not awq_supported:
logger.warning(f"The AutoAWQ format may not be supported due to {info}")
+ enable_torch_compile = False if "--disable_torch_compile" in sys.argv else None
+
autoround = round(
model,
tokenizer,
@@ -469,7 +472,7 @@ def tune(args):
quant_nontext_module=args.quant_nontext_module,
not_use_best_mse=args.not_use_best_mse,
to_quant_block_names=args.to_quant_block_names,
- enable_torch_compile=args.enable_torch_compile,
+ enable_torch_compile=enable_torch_compile,
device_map=args.device_map)
model, _ = autoround.quantize()
@@ -494,7 +497,7 @@ def tune(args):
def eval(args):
device_str, parallelism = set_cuda_visible_devices(args.device)
if parallelism:
- os.environ['AUTO_SPLIT'] = '1'
+ os.environ['AUTO_SPLIT'] = '1'
if isinstance(args.tasks, str):
args.tasks = args.tasks.replace(' ', '').split(',')
from auto_round.mllm import mllm_eval
@@ -550,17 +553,17 @@ def setup_lmms_parser():
default="0",
type=str,
help="the device to be used for tuning. "
- "Currently, device settings support CPU, GPU, and HPU."
- "The default is set to cuda:0,"
- "allowing for automatic detection and switch to HPU or CPU."
- "set --device 0,1,2 to use multiple cards.")
+ "Currently, device settings support CPU, GPU, and HPU."
+ "The default is set to cuda:0,"
+ "allowing for automatic detection and switch to HPU or CPU."
+ "set --device 0,1,2 to use multiple cards.")
parser.add_argument(
"--limit",
type=float,
default=None,
help="Limit the number of examples per task. "
- "If <1, limit is a percentage of the total"
- " number of examples.",
+ "If <1, limit is a percentage of the total"
+ " number of examples.",
)
args = parser.parse_args()
return args
@@ -584,4 +587,3 @@ def lmms_eval(args):
apply_chat_template=False,
)
return results
-
diff --git a/auto_round/utils.py b/auto_round/utils.py
index 1511c81e0..a9861ee8c 100644
--- a/auto_round/utils.py
+++ b/auto_round/utils.py
@@ -1182,7 +1182,7 @@ def set_cuda_visible_devices(device):
os.environ["CUDA_VISIBLE_DEVICES"] = device
device = ",".join(map(str, range(len(devices))))
devices = device.replace(" ", "").split(',')
- if len(devices) > 1: ##for 70B model on single card, use auto will cause some layer offload to cpu
+ if len(devices) > 1: ##for 70B model on single card, use auto will cause some layer offload to cpu
parallelism = True
device_str = None
elif device == "auto":
@@ -1191,4 +1191,12 @@ def set_cuda_visible_devices(device):
else:
device_str = detect_device(device.replace(" ", ""))
return device_str, parallelism
-
\ No newline at end of file
+
+
+def is_debug_mode():
+ """Checks if the Python interpreter is running in debug mode.
+
+ Returns:
+ bool: True if debugging is enabled, False otherwise.
+ """
+ return sys.gettrace() is not None or sys.flags.debug == 1
diff --git a/docs/step_by_step.md b/docs/step_by_step.md
index 1d8fbd874..b2b60502a 100644
--- a/docs/step_by_step.md
+++ b/docs/step_by_step.md
@@ -23,7 +23,7 @@ See more about loading [huggingface dataset](https://huggingface.co/docs/dataset
### Customized Dataset
- Option 1: Pass a local json file path to dataset argument
-- Option 2: Register your dataset following the [code](../../auto_round/calib_dataset.py) and pass the new dataset and
+- Option 2: Register your dataset following the [code](../auto_round/calib_dataset.py) and pass the new dataset and
split args to initialize AutoRound object, e.g. autoround=Autoround(dataset="NeelNanda/pile-10k:train", ...)
- Option 3: pass list of string or list of input_ids to dataset.
@@ -82,7 +82,7 @@ See more about loading [huggingface dataset](https://huggingface.co/docs/dataset
- or combine them
-- **Reduced CPU Memory Usage:**
+- **Reduced CPU Memory Usage (only available for .bin file currently):**
- set "--low_cpu_mem_mode 1" to use block-wise mode, load the weights from disk of each block when tuning and
release the memory of the block after tuning. (more tuning cost)
@@ -184,6 +184,8 @@ See more about loading [huggingface dataset](https://huggingface.co/docs/dataset
~~~
## Inference
+
+### AutoRound format
**CPU**: **auto_round version >0.3.1**, pip install intel-extension-for-pytorch(much higher speed on Intel CPU) or pip
install intel-extension-for-transformers,
@@ -191,7 +193,8 @@ install intel-extension-for-transformers,
in [Gaudi Guide](https://docs.habana.ai/en/latest/).
**CUDA**: no extra operations for sym quantization, for asym quantization, need to install auto-round from source
-### AutoRound format
+
+
- The following code will automatically detect device, and typically some error message will remind you to install some extra libraries
```python
from transformers import AutoModelForCausalLM, AutoTokenizer
@@ -211,7 +214,7 @@ in [Gaudi Guide](https://docs.habana.ai/en/latest/).
from transformers import AutoModelForCausalLM, AutoTokenizer
from auto_round import AutoRoundConfig
- backend = "auto" ##cpu, hpu, cuda, cuda:marlin(supported in auto_round>0.3.1 and 'pip install -v gptqmodel --no-build-isolation')
+ backend = "auto" ##cpu, hpu, cuda
quantization_config = AutoRoundConfig(
backend=backend
)
diff --git a/test/test_autoround.py b/test/test_autoround.py
index 2b842505f..b2c796cdd 100644
--- a/test/test_autoround.py
+++ b/test/test_autoround.py
@@ -342,9 +342,13 @@ class TestAutoRound(unittest.TestCase):
quantized_model_path = "./saved"
autoround.save_quantized(output_dir=quantized_model_path, format="auto_round", inplace=True)
+ from auto_round import AutoRoundConfig
+ quantization_config = AutoRoundConfig(
+ backend="cpu:auto_round:qbits_zp"
+ )
model = AutoModelForCausalLM.from_pretrained(quantized_model_path,
- device_map='auto')
+ device_map='cpu',quantization_config=quantization_config)
tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)
text = "There is a girl who likes adventure,"
inputs = tokenizer(text, return_tensors="pt").to(model.device)
diff --git a/test/test_autoround_export_to_itrex.py b/test/test_autoround_export_to_itrex.py
index 502b4d2c6..7d6ee77cf 100644
--- a/test/test_autoround_export_to_itrex.py
+++ b/test/test_autoround_export_to_itrex.py
@@ -56,8 +56,8 @@ class TestAutoroundExport(unittest.TestCase):
model = copy.deepcopy(self.gptj)
out1 = model(self.lm_input)
round = AutoRound
- optq_1 = round(model, self.tokenizer, nsamples=20, amp=False, seqlen=10, iters=10)
- q_model, layer_config1 = optq_1.quantize()
+ optq_1 = round(model, self.tokenizer, nsamples=20, amp=False, seqlen=10, iters=10, enable_torch_compile=False)
+ q_model, layer_config1 = optq_1.quantize() ##compile model
from auto_round.export.export_to_itrex import pack_model
compressed_model = pack_model(model=q_model, layer_config=layer_config1)
@@ -95,7 +95,7 @@ class TestAutoroundExport(unittest.TestCase):
model = copy.deepcopy(self.gptj)
out1 = model(self.lm_input)
round = AutoRound
- optq_1 = round(model, self.tokenizer, nsamples=20, amp=False, seqlen=10, iters=10)
+ optq_1 = round(model, self.tokenizer, nsamples=20, amp=False, seqlen=10, iters=10, enable_torch_compile=False)
q_model, layer_config1 = optq_1.quantize()
from auto_round.export.export_to_itrex import pack_model
diff --git a/test/test_generation.py b/test/test_generation.py
index 6133bdfe3..c2f2a333e 100644
--- a/test/test_generation.py
+++ b/test/test_generation.py
@@ -63,42 +63,8 @@ class TestAutoRoundFormatGeneration(unittest.TestCase):
text = "There is a girl who likes adventure,"
inputs = tokenizer(text, return_tensors="pt").to(model.device)
res = tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0])
- assert (
- res == """</s>There is a girl who likes adventure, and I'm not sure if she's into it, but I'm sure she's into it.\nI'm not sure if she's into adventure, but I'm sure she's into it.\nI'm not sure if she's into adventure""")
-
- #
- # @unittest.skipIf(torch.cuda.is_available() is False, "Skipping because no cuda")
- # def test_llm_generation_sym_gpu_gptq_marlin(self): ##need auto_gptq >0.7.1
- # bits = 4
- # group_size = 128
- # autoround = AutoRound(
- # self.model,
- # self.tokenizer,
- # bits=bits,
- # group_size=group_size,
- # sym=True,
- # iters=1,
- # seqlen=2,
- # dataset=self.llm_dataloader,
- # )
- # autoround.quantize()
- # quantized_model_path = "./saved"
- #
- # autoround.save_quantized(output_dir=quantized_model_path, format="auto_round:marlin")
- # device = "auto" ##cpu, hpu, cuda
- # from auto_round import AutoRoundConfig
- # quantization_config = AutoRoundConfig(
- # backend=device
- # )
- #
- # model = AutoModelForCausalLM.from_pretrained(quantized_model_path,
- # device_map=device, quantization_config=quantization_config)
- # tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)
- # text = "There is a girl who likes adventure,"
- # inputs = tokenizer(text, return_tensors="pt").to(model.device)
- # res = tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0])
- # assert (
- # res == """</s>There is a girl who likes adventure, and I'm not sure if she's into it, but I'm sure she's into it.\nI'm not sure if she's into adventure, but I'm sure she's into it.\nI'm not sure if she's into adventure""")
+ print(res)
+ assert ("!!!" not in res)
@unittest.skipIf(torch.cuda.is_available() is False, "Skipping because no cuda")
def test_llm_generation_asym_gpu_awq(self):
@@ -130,9 +96,8 @@ class TestAutoRoundFormatGeneration(unittest.TestCase):
text = "There is a girl who likes adventure,"
inputs = tokenizer(text, return_tensors="pt").to(model.device)
res = tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0])
- assert (
- res == """</s>There is a girl who likes adventure, and I'm not sure if she's into it, but I'm sure she's into it.\nI'm not sure if she's into adventure, but I'm sure she's into it.\nI'm not sure if she's into adventure""")
-
+ print(res)
+ assert ("!!!" not in res)
def test_llm_generation_asym_qbits(self):
try:
import intel_extension_for_transformers
@@ -157,7 +122,7 @@ class TestAutoRoundFormatGeneration(unittest.TestCase):
device = "cpu" ##cpu, hpu, cuda
from auto_round import AutoRoundConfig
quantization_config = AutoRoundConfig(
- backend=device
+ backend="cpu:auto_round:qbits_zp"
)
model = AutoModelForCausalLM.from_pretrained(quantized_model_path,
@@ -166,6 +131,7 @@ class TestAutoRoundFormatGeneration(unittest.TestCase):
text = "There is a girl who likes adventure,"
inputs = tokenizer(text, return_tensors="pt").to(model.device)
res = tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0])
+ print(res)
assert ("!!!" not in res)
diff --git a/test/test_low_cpu_mem.py b/test/test_low_cpu_mem.py
index 49cdfc5c7..737fc1642 100644
--- a/test/test_low_cpu_mem.py
+++ b/test/test_low_cpu_mem.py
@@ -84,6 +84,7 @@ class TestLowCPUMem(unittest.TestCase):
iters=2,
seqlen=2,
dataset=self.llm_dataloader,
+ enable_torch_compile=False
)
autoround.quantize()