Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

Refine quntization config (#17)

WeiweiZhang12b821bcae998
diff --git a/auto_round/autoround.py b/auto_round/autoround.py
index 1167d7512..cb46acfde 100644
--- a/auto_round/autoround.py
+++ b/auto_round/autoround.py
@@ -26,14 +26,9 @@ logger.addHandler(fh)
import copy
import time
-from torch.amp import autocast
-from functools import partial
-from torch.functional import F
from .utils import (quant_weight, set_module, get_module, get_block_names, block_forward, sampling_inputs,
get_scale_shape, move_input_to_device, check_is_cpu, collect_round_v,
- collect_minmax_scale, get_batch_dim, is_hpu_available, check_to_quantized)
-from .calib_dataset import CALIB_DATASETS
-
+ collect_minmax_scale, get_batch_dim, htcore, is_hpu_available, check_to_quantized)
class SaveInputs:
"""Cache the inputs of the first block."""
@@ -171,6 +166,7 @@ class SaveInputs:
def _replace_forward(self):
"""Replaces the forward function."""
+ from functools import partial
for n, m in self.model.named_modules():
if n == self.block_name:
m.orig_forward = m.forward
@@ -258,6 +254,7 @@ class WrapperLinear(torch.nn.Module):
Returns:
- torch.Tensor: The output tensor after applying the linear transformation with quantized weights.
"""
+ from torch.functional import F
weight = self.orig_layer.weight
self.min_scale.data.copy_(torch.clamp(self.min_scale.data, -1, 0))
self.max_scale.data.copy_(torch.clamp(self.max_scale.data, -1, 0))
@@ -266,6 +263,7 @@ class WrapperLinear(torch.nn.Module):
self.scale_dtype
)
weight_q = weight_q.to(weight.dtype)
+ # pylint: disable=not-callable
return F.linear(x, weight_q, self.orig_layer.bias)
@@ -533,6 +531,7 @@ class AutoRound(object):
scale_dtype="fp16",
**kwargs,
):
+ from .calib_dataset import CALIB_DATASETS
self.model = model
self.model = self.model.to("cpu")
self.amp = amp
@@ -778,6 +777,7 @@ class AutoRound(object):
Returns:
Tuple: (q_outputs, output) if self.use_quant_input is True, else (None, output)
"""
+ from torch.amp import autocast
batch_dim = get_batch_dim(input_others)
if not self.low_gpu_mem_usage and input_ids.device != device:
input_ids = move_input_to_device(input_ids, device)
@@ -853,8 +853,10 @@ class AutoRound(object):
)
if self.amp and not check_is_cpu(device):
with autocast(device_type="cuda", dtype=self.amp_dtype):
+ # pylint: disable=not-callable
loss = mse_loss(output_q, current_output)
else:
+ # pylint: disable=not-callable
loss = mse_loss(output_q.to(torch.float32), current_output.to(torch.float32))
total_loss += loss.item() / self.gradient_accumulate_steps
@@ -945,14 +947,16 @@ class AutoRound(object):
torch.cuda.empty_cache()
def save_quantized(self, output_dir, format="itrex", **kwargs):
+ compress_model = None
if format == "itrex":
- self.save_quantized_as_itrex(output_dir)
+ compress_model = self.save_quantized_as_itrex(output_dir, **kwargs)
elif format == "auto_gptq":
self.save_quantized_as_autogptq(output_dir, **kwargs)
else:
logger.error("export only supports itrex and auto_gptq now")
+ return compress_model
- def save_quantized_as_autogptq(self, output_dir, use_triton=False):
+ def save_quantized_as_autogptq(self, output_dir, use_triton=False, inplace=True):
"""
Export the model to autogptq format to easily leverage cuda kernel
"""
@@ -982,10 +986,14 @@ class AutoRound(object):
modules_in_block_to_quantize = [modules_in_block_to_quantize] ##align with autogptq
if all_to_quantized:
modules_in_block_to_quantize = None
-
- model = copy.deepcopy(self.model.to("cpu")) ##TODO avoid this deepcopy
+
+ if inplace:
+ model = self.model.to("cpu")
+ else:
+ model = copy.deepcopy(self.model.to("cpu"))
from auto_gptq.modeling._utils import pack_model
+ sym = self.scheme == 'sym'
if self.bits == 3 or use_triton is False:
if self.bits == 3 and use_triton is True:
logger.warning("triton does not support 3 bits, reset it to False")
@@ -1003,22 +1011,26 @@ class AutoRound(object):
info = self.weight_config[key]
if not check_to_quantized(info):
continue
- quantizers[key] = (None, info['scale'].to(torch.float32), info['zp'].to(torch.float32), info['g_idx'])
-
+ info['zp'] = None if sym else info['zp'].to(torch.float32)
+ quantizers[key] = (None, info['scale'].to(torch.float32), info['zp'], info['g_idx'])
pack_model(model, quantizers, self.bits, self.group_size, use_cuda_fp16=True, desc_act=False,
force_layer_back_to_cpu=True, use_triton=True)
from auto_round import save_quantized_to_autogptq
- sym = self.scheme == "sym"
save_quantized_to_autogptq(model, output_dir, bits=self.bits, group_size=self.group_size, sym=sym,
iters=self.iters, lr=self.lr, minmax_lr=self.minmax_lr,
enable_minmax_tuning=self.enable_minmax_tuning, use_quant_input=self.use_quant_input,
scale_dtype=self.scale_dtype,
use_safetensors=True, modules_in_block_to_quantize=modules_in_block_to_quantize)
- def save_quantized_as_itrex(self, output_dir):
+ def save_quantized_as_itrex(self, output_dir, inplace=True):
"""Save configure file and weights for CPU backend inference."""
- from auto_round.export.export_to_itrex import compress_model
- compressed_model, quantize_config = compress_model(self.model, self.weight_config)
+ from auto_round.export.export_to_itrex import compress_model, QuantConfig
+ compressed_model = compress_model(self.model, self.weight_config, inplace=inplace)
+ sym = self.scheme == "sym"
+ quantize_config = QuantConfig(bits=self.bits, group_size=self.group_size, sym=sym,
+ iters=self.iters, lr=self.lr, minmax_lr=self.minmax_lr,
+ enable_minmax_tuning=self.enable_minmax_tuning, use_quant_input=self.use_quant_input,
+ scale_dtype=str(self.scale_dtype))
if quantize_config is not None:
config = compressed_model.config
setattr(config, "quantization_config", quantize_config.to_dict())
@@ -1032,6 +1044,7 @@ class AutoRound(object):
logger.info("Saved config file and weights of quantized model to {}.".format(output_dir))
except IOError as e: # pragma: no cover
logger.error("Fail to save configure file and weights due to {}.".format(e))
+ return compressed_model
def quantize(self):
"""Quantize the model and return the quantized model along with weight configurations.
@@ -1354,3 +1367,4 @@ class AutoAdamRound(AutoOPTRound):
optimizer,
**kwargs,
)
+
diff --git a/auto_round/calib_dataset.py b/auto_round/calib_dataset.py
index 617494b69..7935112db 100644
--- a/auto_round/calib_dataset.py
+++ b/auto_round/calib_dataset.py
@@ -113,8 +113,8 @@ def get_mbpp_dataloader(tokenizer, seqlen, dataset_name="mbpp", split=['train',
tokenizer: The tokenizer to be used for tokenization.
seqlen: The maximum sequence length.
- Returns: A default tokenizer function that applies the provided tokenizer with truncation and a maximum length of
- seqlen to the "text" field of examples.
+ Returns: A default tokenizer function that applies the provided tokenizer with truncation and
+ a maximum length of seqlen to the "text" field of examples.
"""
def default_tokenizer_function(examples):
@@ -159,3 +159,4 @@ def get_mbpp_dataloader(tokenizer, seqlen, dataset_name="mbpp", split=['train',
calib_dataloader = DataLoader(samples, batch_size=bs, shuffle=False, collate_fn=collate_batch)
return calib_dataloader
+
diff --git a/auto_round/export/export_to_autogptq.py b/auto_round/export/export_to_autogptq.py
index a81c8b2b8..aa393fc3a 100644
--- a/auto_round/export/export_to_autogptq.py
+++ b/auto_round/export/export_to_autogptq.py
@@ -76,15 +76,18 @@ def save_quantized_to_autogptq(model, save_dir: str, bits=4, group_size=128, sym
new_value = str(value)
except Exception as e:
raise TypeError(
- f"safetensors_metadata: both keys and values must be strings and an error occured when trying to convert them: {e}")
+ f"safetensors_metadata: both keys and values must be strings" /
+ f" and an error occurred when trying to convert them: {e}")
if new_key in new_safetensors_metadata:
logger.warning(
- f"After converting safetensors_metadata keys to strings, the key '{new_key}' is duplicated. Ensure that all your metadata keys are strings to avoid overwriting.")
+ f"After converting safetensors_metadata keys to strings, the key '{new_key}' " /
+ f"is duplicated. Ensure that all your metadata keys are strings to avoid overwriting.")
new_safetensors_metadata[new_key] = new_value
safetensors_metadata = new_safetensors_metadata
if converted_keys:
logger.debug(
- f"One or more safetensors_metadata keys or values had to be converted to str(). Final safetensors_metadata: {safetensors_metadata}")
+ f"One or more safetensors_metadata keys or values had to be converted to str()." /
+ f" Final safetensors_metadata: {safetensors_metadata}")
# Format is required to enable Accelerate to load the metadata
# otherwise it raises an OSError
@@ -131,3 +134,4 @@ def save_quantized_to_autogptq(model, save_dir: str, bits=4, group_size=128, sym
config_dict["quant_method"] = "gptq" ##hf transformers could only recognize this value
model.config.quantization_config = config_dict
model.config.save_pretrained(save_dir)
+
diff --git a/auto_round/export/export_to_itrex/config.py b/auto_round/export/export_to_itrex/config.py
index 62e87ef6c..25ac08e6a 100644
--- a/auto_round/export/export_to_itrex/config.py
+++ b/auto_round/export/export_to_itrex/config.py
@@ -33,21 +33,31 @@ class QuantConfig(PretrainedConfig):
def __init__(
self,
bits=4,
- scale_dtype="fp32",
+ scale_dtype='torch.float32',
group_size=128,
sym=False,
- quant_method="Autoround",
+ quant_method="autoround",
model_name_or_path=None,
model_file_base_name='model',
+ enable_minmax_tuning=True,
+ iters=1000,
+ lr=0.001,
+ minmax_lr=0.001,
+ use_quant_input=True,
**kwargs,
):
self.bits = bits
self.group_size = group_size
- self.scale_dtype = scale_dtype ## reserved parameter
+ self.scale_dtype = scale_dtype
self.sym = sym
self.quant_method = quant_method
self.model_name_or_path = model_name_or_path
self.model_file_base_name = model_file_base_name
+ self.enable_minmax_tuning=enable_minmax_tuning
+ self.iters=iters
+ self.lr=lr
+ self.minmax_lr=minmax_lr
+ self.use_quant_input=use_quant_input
### Redundant parameters, will be removed later. ###
self.damp_percent = 0.01
@@ -61,8 +71,8 @@ class QuantConfig(PretrainedConfig):
Safety checker that arguments are correct - also replaces some NoneType arguments with their default values.
"""
- if self.scale_dtype not in ["fp32", "fp16"]:
- raise ValueError("scale_dtype must be 'fp32', 'fp16'.")
+ if self.scale_dtype not in ['torch.float32', 'torch.float16', 'torch.bfloat16']:
+ raise ValueError("scale_dtype must be 'fp32', 'fp16' or 'bf16'.")
if self.group_size not in [-1, 32, 128]:
raise ValueError("group_size must be an integer in [-1, 32, 128]")
@@ -78,7 +88,7 @@ class QuantConfig(PretrainedConfig):
@classmethod
def from_dict(cls, config_dict, return_unused_kwargs=False, **kwargs):
"""
- Instantiates a [`AutoroundQuantConfig`] from a Python dictionary of parameters.
+ Instantiates a [`QuantConfig`] from a Python dictionary of parameters.
Args:
config_dict (`Dict[str, Any]`):
Dictionary that will be used to instantiate the configuration object.
@@ -88,7 +98,7 @@ class QuantConfig(PretrainedConfig):
kwargs (`Dict[str, Any]`):
Additional parameters from which to initialize the configuration object.
Returns:
- [`AutoroundQuantConfig`]: The configuration object instantiated from those parameters.
+ [`QuantConfig`]: The configuration object instantiated from those parameters.
"""
config = cls(**config_dict)
@@ -143,7 +153,7 @@ class QuantConfig(PretrainedConfig):
Args:
use_diff (`bool`, *optional*, defaults to `True`):
If set to `True`, only the difference between the config instance and the default
- `AutoroundQuantConfig()`
+ `QuantConfig()`
is serialized to JSON string.
Returns:
`str`: String containing all the attributes that make up this configuration instance in JSON format.
@@ -165,7 +175,7 @@ class QuantConfig(PretrainedConfig):
config_dict = self.to_dict()
# get the default config dict
- default_config_dict = AutoroundQuantConfig().to_dict()
+ default_config_dict = QuantConfig().to_dict()
serializable_config_dict = {}
@@ -222,4 +232,4 @@ class QuantConfig(PretrainedConfig):
def get_config_dict(
cls, pretrained_model_name_or_path: Union[str, os.PathLike], **kwargs
) -> Tuple[Dict[str, Any], Dict[str, Any]]:
- return super().get_config_dict(pretrained_model_name_or_path, _configuration_file=QUANT_CONFIG, **kwargs)
\ No newline at end of file
+ return super().get_config_dict(pretrained_model_name_or_path, _configuration_file=QUANT_CONFIG, **kwargs)
diff --git a/auto_round/export/export_to_itrex/export.py b/auto_round/export/export_to_itrex/export.py
index 60d529a01..9db61f692 100644
--- a/auto_round/export/export_to_itrex/export.py
+++ b/auto_round/export/export_to_itrex/export.py
@@ -35,6 +35,7 @@ def compress_model(
compression_dim=1,
device="cpu",
use_optimum_format=True,
+ inplace=False
):
"""Convert Linear to WeightOnlyLinear for low memory inference.
@@ -53,8 +54,12 @@ def compress_model(
3: g_idx: use same number for one group instead of recording the channel order.
4. parameter name changed, such as 'packed_weight' -> 'qweight'.
5. zeros is always needed even for sym.
+ inplace (bool, optional): Compress the model in place, or copy the model and compress it.
"""
- compressed_model = copy.deepcopy(model)
+ if inplace:
+ compressed_model = model
+ else:
+ compressed_model = copy.deepcopy(model)
if isinstance(weight_config, str):
with open(weight_config, "r") as f:
q_config = json.load(f)
@@ -71,13 +76,15 @@ def compress_model(
scale_dtype = v["scale_dtype"]
m = get_module(compressed_model, k)
fp_weight = m.weight.data
- # scale = torch.tensor(v["scale"], dtype=scale_dtype)
scale, zp = v["scale"], v["zp"]
convert_dtype = torch.float32 if fp_weight.device.type == "cpu" else scale_dtype
if not isinstance(scale, torch.Tensor):
scale = torch.tensor(scale, dtype=convert_dtype)
zp = None if scheme == "sym" else torch.tensor(zp, dtype=torch.int32)
else:
+ if not inplace:
+ scale = scale.clone()
+ zp = None if scheme == "sym" else zp.clone()
scale = scale.to(dtype=convert_dtype)
zp = None if scheme == "sym" else zp.to(dtype=torch.int32)
int_weight = quant_weight_w_scale(fp_weight, scale, zp, group_size, fp_weight.device)
@@ -97,5 +104,5 @@ def compress_model(
new_module.pack(int_weight, scale, zp, m.bias)
set_module(compressed_model, k, new_module)
- quantize_config = QuantConfig(bits=num_bits, sym=(scheme == "sym"), group_size=group_size)
- return compressed_model, quantize_config
+ return compressed_model
+
diff --git a/auto_round/utils.py b/auto_round/utils.py
index c3347842b..45ecc1dc3 100644
--- a/auto_round/utils.py
+++ b/auto_round/utils.py
@@ -35,6 +35,7 @@ try:
is_hpu_available = False
except ImportError:
is_hpu_available = False
+ htcore = None
def round_ste(x: torch.Tensor):
"""Straight-Through Estimator for rounding.
@@ -138,7 +139,15 @@ def quant_weight_actor(weight, num_bits, scheme, v, min_scale, max_scale, scale_
return quant_weight_asym(weight, num_bits, v, min_scale, max_scale, scale_dtype)
-def quant_weight(weight, num_bits=4, group_size=-1, scheme="asym", v=0, min_scale=0, max_scale=0, scale_dtype=torch.float16):
+def quant_weight(
+ weight,
+ num_bits=4,
+ group_size=-1,
+ scheme="asym",
+ v=0,
+ min_scale=0,
+ max_scale=0,
+ scale_dtype=torch.float16):
"""Quantizes and dequantizes weight, handing the group size issue .
Args:
@@ -223,14 +232,13 @@ def quant_weight_w_scale(weight, scale, zp, group_size=-1, device="cpu"):
return int_weight
-def get_module(model, key):
+def get_module(module, key):
"""Get module from model by key name.
Args:
- model (torch.nn.Module): original model
+ module (torch.nn.Module): original model
key (str): module name to be replaced
"""
- module = model
name_list = key.split(".")
for name in name_list:
if hasattr(module, name):
@@ -474,4 +482,4 @@ def check_to_quantized(config):
else:
if config.bits > 8 or "fp" in config.data_type or "float" in config.data_type:
return False
- return True
\ No newline at end of file
+ return True
diff --git a/auto_round/version.py b/auto_round/version.py
index 1a1828052..952b12d4e 100644
--- a/auto_round/version.py
+++ b/auto_round/version.py
@@ -11,5 +11,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-"""Intel® auto-round: An open-source Python library supporting popular model weight only compression based on signround."""
-__version__ = "0.1"
\ No newline at end of file
+"""Intel® auto-round: An open-source Python library
+supporting popular model weight only compression based on signround."""
+
+__version__ = "0.1"
diff --git a/docs/acc.md b/docs/acc.md
index a3e286235..a7c3925b1 100644
--- a/docs/acc.md
+++ b/docs/acc.md
@@ -1,5 +1,5 @@
## Comparison with other methods
-To ensure a fair comparison as much as possible and alleviate overfitting in perplexity evaluation on Wikitext or C4, we utilized 512 samples from NeelNanda/pile-10k for all methods during calibration. Due to memory constraints, we maintained the original sequence length of 512 for AWQ, while for GPTQ and our approach, a sequence length of 2048 is used. We have enalbed act-order and true-seqential in GPTQ, and also activated static group in scenarios where group_size!=-1. The notation GPTQ* indicates that we adjusted the random seed or data preprocessing to address issues related to the non-positive definite Hessian matrix or other issues. With these configurations, the tuning costs for GPTQ, AWQ, and ours are comparable.
+To ensure a fair comparison as much as possible and alleviate overfitting in perplexity evaluation on Wikitext or C4, we utilized 512 samples from NeelNanda/pile-10k for all methods during calibration. Due to memory constraints, we maintained the original sequence length of 512 for AWQ, while for GPTQ and our approach, a sequence length of 2048 is used. We have enabled act-order and true-seqential in GPTQ, and also activated static group in scenarios where group_size!=-1. The notation GPTQ* indicates that we adjusted the random seed or data preprocessing to address issues related to the non-positive definite Hessian matrix or other issues. With these configurations, the tuning costs for GPTQ, AWQ, and ours are comparable.
For wikitext2/ptb-new/c4-new ppl, we follow the code of gptq and set the sequence length to 2048. For lm-eval wikitext ppl, we adopt lm-eval. The lm-eval-harness git id we used in the following is 008fc2a23245c40384f2312718433eeb1e0f87a9 and we evaluated on qdq fake model.
@@ -2737,3 +2737,4 @@ For wikitext2/ptb-new/c4-new ppl, we follow the code of gptq and set the sequenc
<td>11.35</td> <! lm-eval wikitext ppl -->
</tr>
</table>
+
diff --git a/examples/code-generation/main.py b/examples/code-generation/main.py
index e11520cc5..c721abfbd 100644
--- a/examples/code-generation/main.py
+++ b/examples/code-generation/main.py
@@ -53,7 +53,7 @@ if __name__ == '__main__':
help="whether to use the output of quantized block to tune the next block")
parser.add_argument("--lr", default=None, type=float,
- help="learning rate, if None, it will be set to 1.0/iters automatially")
+ help="learning rate, if None, it will be set to 1.0/iters automatically")
parser.add_argument("--minmax_lr", default=None, type=float,
help="minmax learning rate, if None,it will beset to be the same with lr")
@@ -196,3 +196,4 @@ if __name__ == '__main__':
model = model.to("cpu")
model.save_pretrained(output_dir)
tokenizer.save_pretrained(output_dir)
+
diff --git a/examples/language-modeling/eval/__init__.py b/examples/language-modeling/eval/__init__.py
index e69de29bb..07f1ae76d 100644
--- a/examples/language-modeling/eval/__init__.py
+++ b/examples/language-modeling/eval/__init__.py
@@ -0,0 +1,5 @@
+if __name__ == "__main__":
+ import sys
+ sys.path.insert(0, './')
+from .evaluation import eval_model
+
diff --git a/examples/language-modeling/eval/evaluation.py b/examples/language-modeling/eval/evaluation.py
index 3fef08105..86c68fadd 100644
--- a/examples/language-modeling/eval/evaluation.py
+++ b/examples/language-modeling/eval/evaluation.py
@@ -3,15 +3,22 @@ import os
os.environ["TOKENIZERS_PARALLELISM"] = "false"
import os.path
import torch
-
+import logging
import pprint
import re
import shutil
import transformers
+from typing import TYPE_CHECKING, Optional, Union
+import time
+try:
+ from lm_eval.tasks import TaskManager
+except ImportError:
+ TaskManager = None
+
if __name__ == "__main__":
import sys
sys.path.insert(0, './')
-import time
+
EXT_TASKS = ['wikitext2', 'ptb', 'c4', 'ptb-new', 'c4-new']
fewshots_dict = {}
fewshots_dict['paper'] = {
@@ -60,30 +67,39 @@ fewshots_dict['all'] = {
def simple_evaluate(
model,
- model_args=None,
- tasks=[],
- num_fewshot=None,
- batch_size=None,
- max_batch_size=None,
- device=None,
- use_cache=None,
- limit=None,
+ model_args: Optional[Union[str, dict, None]] = None,
+ tasks=None,
+ num_fewshot: Optional[int] = None,
+ batch_size: Optional[int] = None,
+ max_batch_size: Optional[int] = None,
+ device: Optional[str] = None,
+ use_cache: Optional[str] = None,
+ cache_requests: bool = False,
+ rewrite_requests_cache: bool = False,
+ delete_requests_cache: bool = False,
+ limit: Optional[Union[int, float]] = None,
bootstrap_iters: int = 100000,
check_integrity: bool = False,
decontamination_ngrams_path=None,
write_out: bool = False,
log_samples: bool = True,
gen_kwargs: str = None,
+ task_manager: TaskManager = None,
+ verbosity: str = "INFO",
+ predict_only: bool = False,
+ random_seed: int = 1234,
+ numpy_random_seed: int = 1234,
+ torch_random_seed: int = 1234,
lm=None
):
"""Instantiate and evaluate a model on a list of tasks.
:param model: Union[str, LM]
Name of model, transformers.PreTrainedModel object or LM object, see lm_eval.models.get_model
- :param model_args: Optional[str]
- String arguments for each model class, see LM.create_from_arg_string.
+ :param model_args: Optional[str, dict]
+ String or dict arguments for each model class, see LM.create_from_arg_string and LM.create_from_arg_object.
Ignored if `model` argument is a LM object.
- :param tasks: list[Union[str, Task]]
+ :param tasks: list[Union[str, dict, Task]]
List of task names or Task objects. Task objects will be taken to have name task.EVAL_HARNESS_NAME if defined and type(task).__name__ otherwise.
:param num_fewshot: int
Number of examples in few-shot context
@@ -95,6 +111,12 @@ def simple_evaluate(
PyTorch device (e.g. "cpu" or "cuda:0") for running models
:param use_cache: str, optional
A path to a sqlite db file for caching model responses. `None` if not caching.
+ :param cache_requests: bool, optional
+ Speed up evaluation by caching the building of dataset requests. `None` if not caching.
+ :param rewrite_requests_cache: bool, optional
+ Rewrites all of the request cache if set to `True`. `None` if not desired.
+ :param delete_requests_cache: bool, optional
+ Deletes all of the request cache if set to `True`. `None` if not desired.
:param limit: int or float, optional
Limit the number of examples per task (only use this for testing), If <1, limit is a percentage of the total number of examples.
:param bootstrap_iters:
@@ -108,6 +130,15 @@ def simple_evaluate(
:param gen_kwargs: str
String arguments for model generation
Ignored for all tasks with loglikelihood output_type
+ :param predict_only: bool
+ If true only model outputs will be generated and returned. Metrics will not be evaluated
+ :param random_seed: int
+ Random seed for python's random module. If set to None, the seed will not be set.
+ :param numpy_random_seed: int
+ Random seed for numpy. If set to None, the seed will not be set.
+ :param torch_random_seed: int
+ Random seed for torch. If set to None, the seed will not be set.
+
:return
Dictionary of results
"""
@@ -120,32 +151,46 @@ def simple_evaluate(
import lm_eval.api.metrics
import lm_eval.api.registry
from lm_eval.evaluator import evaluate
- lm_eval.tasks.initialize_tasks()
+ from lm_eval.logging_utils import add_env_info, get_git_commit_hash
+ from lm_eval.tasks import get_task_dict
from lm_eval.utils import (
- positional_deprecated,
- run_task_tests,
- make_table,
- get_git_commit_hash,
- simple_parse_args_string,
- eval_logger,
+ eval_logger,
+ positional_deprecated,
+ run_task_tests,
+ simple_parse_args_string,
)
- random.seed(1234)
- np.random.seed(1234)
- torch.manual_seed(
- 1234
- ) # TODO: this may affect training runs that are run with evaluation mid-run.
+ eval_logger.setLevel(getattr(logging, f"{verbosity}"))
+
+ if delete_requests_cache:
+ eval_logger.info("Deleting requests cache...")
+ delete_cache()
+
+ if random_seed is not None:
+ # See https://github.com/EleutherAI/lm-evaluation-harness/pull/1412
+ eval_logger.info(f"Setting random seed to {random_seed}")
+ random.seed(random_seed)
+
+ if numpy_random_seed is not None:
+ eval_logger.info(f"Setting numpy seed to {numpy_random_seed}")
+ np.random.seed(numpy_random_seed)
+
+ if torch_random_seed is not None:
+ eval_logger.info(f"Setting torch manual seed to {torch_random_seed}")
+ torch.manual_seed(torch_random_seed)
- assert (
- tasks != []
- ), "No tasks specified, or no tasks found. Please verify the task names."
+ if tasks is None:
+ tasks = []
+ assert (
+ tasks != []
+ ), "No tasks specified, or no tasks found. Please verify the task names."
if lm == None:
if gen_kwargs is not None:
gen_kwargs = simple_parse_args_string(gen_kwargs)
eval_logger.warning(
- f"generation_kwargs specified through cli, these settings will be used over set parameters in yaml tasks."
+ "generation_kwargs specified through cli, these settings will update set parameters in yaml tasks. Ensure 'do_sample=True' for non-greedy decoding!"
)
if gen_kwargs == "":
gen_kwargs = None
@@ -153,16 +198,28 @@ def simple_evaluate(
if isinstance(model, str):
if model_args is None:
model_args = ""
- lm = lm_eval.api.registry.get_model(model).create_from_arg_string(
- model_args,
- {
- "batch_size": batch_size,
- "max_batch_size": max_batch_size,
- "device": device,
- },
- )
+
+ elif isinstance(model_args, dict):
+ lm = lm_eval.api.registry.get_model(model).create_from_arg_obj(
+ model_args,
+ {
+ "batch_size": batch_size,
+ "max_batch_size": max_batch_size,
+ "device": device,
+ },
+ )
+
+ else:
+ lm = lm_eval.api.registry.get_model(model).create_from_arg_string(
+ model_args,
+ {
+ "batch_size": batch_size,
+ "max_batch_size": max_batch_size,
+ "device": device,
+ },
+ )
elif isinstance(model, transformers.PreTrainedModel):
- lm = lm_eval.api.registry.get_model("hf-causal")(
+ lm = lm_eval.api.registry.get_model("hf")(
pretrained=model,
batch_size=batch_size,
max_batch_size=max_batch_size,
@@ -182,33 +239,46 @@ def simple_evaluate(
+ model_args.replace("=", "-").replace(",", "_").replace("/", "-")
+ ".db",
)
+
+ if task_manager is None:
+ task_manager = TaskManager(verbosity)
- task_dict = lm_eval.tasks.get_task_dict(tasks)
+ eval_logger.info(
+ "get_task_dict has been updated to accept an optional argument, `task_manager`"
+ "Read more here:https://github.com/EleutherAI/lm-evaluation-harness/blob/main/docs/interface.md#external-library-usage"
+ )
+ task_dict = get_task_dict(tasks, task_manager)
for task_name in task_dict.keys():
task_obj = task_dict[task_name]
- if type(task_obj) == tuple:
- group, task_obj = task_obj
+ if isinstance(task_obj, tuple):
+ _, task_obj = task_obj
if task_obj is None:
continue
- config = task_obj._config
- if config["output_type"] == "generate_until" and gen_kwargs is not None:
- config["generation_kwargs"].update(gen_kwargs)
+ if task_obj.get_config("output_type") == "generate_until":
+ if gen_kwargs is not None:
+ task_obj.set_config(
+ key="generation_kwargs", value=gen_kwargs, update=True
+ )
+
+ if predict_only:
+ log_samples = True
+ eval_logger.info(
+ f"Processing {task_name} in output-only mode. Metrics will not be calculated!"
+ )
+ # we have to change the class properties post-hoc. This is pretty hacky.
+ task_obj.override_metric(metric_name="bypass")
if num_fewshot is not None:
- if config["num_fewshot"] == 0:
+ if (default_num_fewshot := task_obj.get_config("num_fewshot")) == 0:
eval_logger.info(
f"num_fewshot has been set to 0 for {task_name} in its config. Manual configuration will be ignored."
)
else:
- # default_num_fewshot = config["num_fewshot"]
- # eval_logger.warning(
- # f"Overwriting default num_fewshot of {task_name} from {default_num_fewshot} to {num_fewshot}"
- # )
-
- task_obj._config["num_fewshot"] = num_fewshot
- else:
- num_fewshot = config["num_fewshot"]
+ eval_logger.warning(
+ f"Overwriting default num_fewshot of {task_name} from {default_num_fewshot} to {num_fewshot}"
+ )
+ task_obj.set_config(key="num_fewshot", value=num_fewshot)
if check_integrity:
run_task_tests(task_list=tasks)
@@ -217,40 +287,47 @@ def simple_evaluate(
lm=lm,
task_dict=task_dict,
limit=limit,
+ cache_requests=cache_requests,
+ rewrite_requests_cache=rewrite_requests_cache,
bootstrap_iters=bootstrap_iters,
decontamination_ngrams_path=decontamination_ngrams_path,
write_out=write_out,
log_samples=log_samples,
+ verbosity=verbosity,
)
if lm.rank == 0:
+ if isinstance(model, str):
+ model_name = model
+ elif hasattr(model, "config") and hasattr(model.config, "_name_or_path"):
+ model_name = model.config._name_or_path
+ else:
+ model_name = type(model).__name__
+
# add info about the model and few shot config
results["config"] = {
- "model": model
- if isinstance(model, str)
- else model.model.config._name_or_path,
+ "model": model_name,
"model_args": model_args,
"batch_size": batch_size,
- "batch_sizes": list(lm.batch_sizes.values())
- if hasattr(lm, "batch_sizes")
- else [],
+ "batch_sizes": (
+ list(lm.batch_sizes.values()) if hasattr(lm, "batch_sizes") else []
+ ),
"device": device,
"use_cache": use_cache,
"limit": limit,
- "num_fewshot": num_fewshot,
"bootstrap_iters": bootstrap_iters,
"gen_kwargs": gen_kwargs,
}
results["git_hash"] = get_git_commit_hash()
+ add_env_info(results) # additional environment info to results
return results, lm
else:
return None
-def eval_model(output_dir=None, model=None, tokenizer=None,
- tasks=["lambada_openai", "hellaswag", "winogrande", "piqa"],
+def eval_model(model_path, tasks=["lambada_openai", "hellaswag", "winogrande", "piqa"],
eval_bs=32, use_accelerate=True, dtype="float16", limit=None,
- device="cuda:0", seed=0, nsamples=128, eval_orig_float=False, mark="paper", excel_file="tmp.xlsx"):
+ device="cuda:0", seed=0, nsamples=128, mark="paper", excel_file="tmp.xlsx"):
print("evaluation with official lm-eval", flush=True)
try:
import lm_eval.api
@@ -264,29 +341,6 @@ def eval_model(output_dir=None, model=None, tokenizer=None,
raise ImportError("""follow requirements to install dependencies.""")
import collections
org_s = time.time()
- ##save model
- if output_dir is None:
- output_dir = "./tmp_signround"
-
- if os.path.exists(output_dir) and not eval_orig_float:
- shutil.rmtree(output_dir)
-
- if (hasattr(model, 'config') and (model.dtype is torch.bfloat16 or model.config.torch_dtype is torch.bfloat16)):
- dtype = 'bfloat16'
- pt_dtype = torch.bfloat16
- else:
- if str(device) != "cpu":
- pt_dtype = torch.float16
- dtype = 'float16'
- else:
- pt_dtype = torch.float32
- dtype = 'float32'
-
- if not eval_orig_float:
- model = model.to(pt_dtype)
- model = model.to("cpu")
- model.save_pretrained(output_dir)
- tokenizer.save_pretrained(output_dir)
external_tasks = []
for each in EXT_TASKS:
@@ -304,8 +358,8 @@ def eval_model(output_dir=None, model=None, tokenizer=None,
task_s = time.time()
for shot in num_fewshot:
model_type = "hf"
- model_args = f'pretrained={output_dir},tokenizer={output_dir},dtype={dtype},trust_remote_code=True'
- if use_accelerate: # bool(re.search("chatglm", output_dir.lower()))
+ model_args = f'pretrained={model_path},tokenizer={model_path},dtype={dtype},trust_remote_code=True'
+ if use_accelerate: # bool(re.search("chatglm", model_path.lower()))
model_args += f',parallelize=True'
if "wikitext" in tmp_tasks:
@@ -331,7 +385,7 @@ def eval_model(output_dir=None, model=None, tokenizer=None,
print(str(e))
continue
- tokenizer = transformers.AutoTokenizer.from_pretrained(output_dir, use_fast=False, trust_remote_code=True)
+ tokenizer = transformers.AutoTokenizer.from_pretrained(model_path, use_fast=False, trust_remote_code=True)
model = lm.model
# for external tasks
# maybe adjust for specific model
@@ -420,10 +474,11 @@ if __name__ == "__main__":
"arc_easy", "arc_challenge"]
model_name = args.model_name.rstrip('/')
excel_name = model_name.split('/')[-1] + ".xlsx"
- eval_model(output_dir=args.model_name,
+ eval_model(model_path=args.model_name,
tasks=test_tasks,
- eval_bs=args.eval_bs, eval_orig_float=True, limit=None, excel_file=excel_name)
+ eval_bs=args.eval_bs, limit=None, excel_file=excel_name)
print("cost time: ", time.time() - s)
+
diff --git a/examples/language-modeling/eval_legacy/__init__.py b/examples/language-modeling/eval_legacy/__init__.py
new file mode 100644
index 000000000..07f1ae76d
--- /dev/null
+++ b/examples/language-modeling/eval_legacy/__init__.py
@@ -0,0 +1,5 @@
+if __name__ == "__main__":
+ import sys
+ sys.path.insert(0, './')
+from .evaluation import eval_model
+
diff --git a/examples/language-modeling/eval_legacy/evaluation.py b/examples/language-modeling/eval_legacy/evaluation.py
index b0a423993..2ba6a29f9 100644
--- a/examples/language-modeling/eval_legacy/evaluation.py
+++ b/examples/language-modeling/eval_legacy/evaluation.py
@@ -11,7 +11,7 @@ import transformers
if __name__ == "__main__":
import sys
sys.path.insert(0, './')
-from eval.parse_results import result_parser
+from .parse_results import result_parser
import time
EXT_TASKS = ['wikitext2', 'ptb', 'c4', 'ptb-new', 'c4-new']
fewshots_dict = {}
@@ -201,10 +201,9 @@ def simple_evaluate(
return results, lm
-def eval_model(output_dir=None, model=None, tokenizer=None,
- tasks=["lambada_openai", "hellaswag", "winogrande", "piqa"],
+def eval_model(model_path=None, tasks=["lambada_openai", "hellaswag", "winogrande", "piqa"],
eval_bs=32, use_accelerate=True, dtype="float16", limit=None,
- device="cuda:0", seed=0, nsamples=128, eval_orig_float=False, mark="paper", excel_file="tmp.xlsx"):
+ device="cuda:0", seed=0, nsamples=128, mark="paper", excel_file="tmp.xlsx"):
print("evaluation with official lm-eval", flush=True)
try:
import lm_eval
@@ -214,31 +213,6 @@ def eval_model(output_dir=None, model=None, tokenizer=None,
raise ImportError("""follow requirements to install dependencies.""")
org_s = time.time()
- ##save model
- if output_dir is None:
- output_dir = "./tmp_signround"
-
- if os.path.exists(output_dir) and not eval_orig_float:
- shutil.rmtree(output_dir)
-
- if (hasattr(model, 'config') and (model.dtype is torch.bfloat16 or model.config.torch_dtype is torch.bfloat16)):
- dtype = 'bfloat16'
- pt_dtype = torch.bfloat16
- else:
- if str(device) != "cpu":
- pt_dtype = torch.float16
- dtype = 'float16'
- else:
- pt_dtype = torch.float32
- dtype = 'float32'
-
- if not eval_orig_float:
- model = model.to(pt_dtype)
- model = model.to("cpu")
- model.save_pretrained(output_dir)
- tokenizer.save_pretrained(output_dir)
-
-
external_tasks = []
for each in EXT_TASKS:
@@ -256,11 +230,11 @@ def eval_model(output_dir=None, model=None, tokenizer=None,
print(f'********* {tmp_tasks} evaluate ************')
task_s = time.time()
for shot in num_fewshot:
- if bool(re.search("chatglm", output_dir.lower())):
- model_args = f'pretrained={output_dir},tokenizer={output_dir},dtype={dtype},trust_remote_code=True'
+ if bool(re.search("chatglm", model_path.lower())):
+ model_args = f'pretrained={model_path},tokenizer={model_path},dtype={dtype},trust_remote_code=True'
model_type = "hf-causal"
else:
- model_args = f'pretrained={output_dir},tokenizer={output_dir},dtype={dtype},use_accelerate={use_accelerate},trust_remote_code=True'
+ model_args = f'pretrained={model_path},tokenizer={model_path},dtype={dtype},use_accelerate={use_accelerate},trust_remote_code=True'
model_type = "hf-causal-experimental"
if "wikitext" in task_names:
@@ -280,7 +254,7 @@ def eval_model(output_dir=None, model=None, tokenizer=None,
print(str(e))
continue
- tokenizer = transformers.AutoTokenizer.from_pretrained(output_dir, use_fast=False, trust_remote_code=True)
+ tokenizer = transformers.AutoTokenizer.from_pretrained(model_path, use_fast=False, trust_remote_code=True)
model = lm.model
# for external tasks
# maybe adjust for specific model
@@ -371,10 +345,11 @@ if __name__ == "__main__":
"arc_challenge"]
model_name = args.model_name.rstrip('/')
excel_name = model_name.split('/')[-1] + ".xlsx"
- eval_model(output_dir=args.model_name,
+ eval_model(model_path=args.model_name,
tasks=test_tasks,
- eval_bs=args.eval_bs, eval_orig_float=True, limit=None, excel_file=excel_name)
+ eval_bs=args.eval_bs, limit=None, excel_file=excel_name)
print("cost time: ", time.time() - s)
+
diff --git a/examples/language-modeling/main.py b/examples/language-modeling/main.py
index 47b51ffe6..365ddfeaf 100644
--- a/examples/language-modeling/main.py
+++ b/examples/language-modeling/main.py
@@ -1,6 +1,7 @@
import argparse
import sys
sys.path.insert(0, '../..')
+
from auto_round import (AutoRound,
AutoAdamRound)
@@ -85,7 +86,7 @@ if __name__ == '__main__':
help="whether enable weight minmax tuning")
parser.add_argument("--deployment_device", default='fake', type=str,
- help="targeted inference acceleration platform,The options are 'fake', 'cpu' and 'gpu',"
+ help="targeted inference acceleration platform,The options are 'fake', 'cpu' and 'gpu'."
"default to 'fake', indicating that it only performs fake quantization and won't be exported to any device.")
parser.add_argument("--scale_dtype", default='fp32',
@@ -96,30 +97,43 @@ if __name__ == '__main__':
"mmlu", "wikitext", "truthfulqa_mc1", "truthfulqa_mc2", "openbookqa", "boolq", "rte",
"arc_easy", "arc_challenge"],
help="lm-eval tasks for lm_eval version 0.4")
-
- # parser.add_argument("--tasks",
- # default=['wikitext2', 'ptb-new', 'c4-new', 'lambada_openai', 'hellaswag', 'winogrande', 'piqa',
- # "hendrycksTest-*", "wikitext", "truthfulqa_mc", "openbookqa", "boolq", "rte",
- # "arc_easy", "arc_challenge"],
- # help="lm-eval tasks for lm_eval version 0.3")
parser.add_argument("--output_dir", default="./tmp_autoround", type=str,
help="Where to store the final model.")
parser.add_argument("--eval_legacy", action='store_true',
help="Whether to evaluate with a old lm_eval version(e.g. 0.3.0).")
+
+ parser.add_argument("--disable_lmeval", action='store_true',
+ help="Whether to do lmeval evaluation.")
args = parser.parse_args()
set_seed(args.seed)
+
+ tasks = args.tasks
+ if not args.eval_legacy:
+ from eval import eval_model
+ else:
+ from eval_legacy import eval_model
+ if isinstance(tasks, str):
+ tasks = tasks.split(',')
+ if isinstance(tasks, list):
+ if "mmlu" in tasks:
+ tmp_tasks = tasks
+ tasks = ["hendrycksTest-*" if x == "mmlu" else x for x in tmp_tasks]
+ if "truthfulqa_mc1" in tasks or "truthfulqa_mc2" in tasks:
+ tmp_tasks = tasks
+ tasks = ["truthfulqa_mc" if "truthfulqa_mc" in x else x for x in tmp_tasks]
+ tasks = list(set(tasks))
+ if isinstance(args.tasks, str):
+ tasks = ','.join(tasks)
model_name = args.model_name
if model_name[-1] == "/":
model_name = model_name[:-1]
print(model_name, flush=True)
- tasks = args.tasks
-
if args.device == "cpu":
device_str = "cpu"
else:
@@ -154,6 +168,7 @@ if __name__ == '__main__':
tokenizer.add_special_tokens({'pad_token': '[PAD]'})
else:
tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
+
if hasattr(tokenizer, "model_max_length"):
if tokenizer.model_max_length < seqlen:
print(f"change sequence length to {tokenizer.model_max_length} due to the limitation of model_max_length",
@@ -161,14 +176,26 @@ if __name__ == '__main__':
seqlen = min(seqlen, tokenizer.model_max_length)
args.seqlen = seqlen
+ pt_dtype = torch.float16
+ if (hasattr(model, 'config') and (model.dtype is torch.bfloat16 or model.config.torch_dtype is torch.bfloat16)):
+ dtype = 'bfloat16'
+ pt_dtype = torch.bfloat16
+ else:
+ if str(args.device) != "cpu":
+ pt_dtype = torch.float16
+ dtype = 'float16'
+ else:
+ pt_dtype = torch.float32
+ dtype = 'float32'
+
excel_name = f"{model_name}_{args.bits}_{args.group_size}"
if args.eval_fp16_baseline:
if not args.low_gpu_mem_usage:
model = model.to(torch_device)
excel_name += "_fp16.xlsx"
- eval_model(output_dir=model_name, model=model, tokenizer=tokenizer, tasks=args.tasks, \
- eval_bs=args.eval_bs, use_accelerate=args.low_gpu_mem_usage, device=torch_device,
- eval_orig_float=True, excel_file=excel_name)
+ eval_model(model_path=model_name, tasks=tasks, dtype=dtype, \
+ eval_bs=args.eval_bs, use_accelerate=args.low_gpu_mem_usage,
+ device=torch_device, excel_file=excel_name)
exit()
if not args.low_gpu_mem_usage:
@@ -196,44 +223,34 @@ if __name__ == '__main__':
amp=args.amp, n_samples=args.n_samples, low_gpu_mem_usage=args.low_gpu_mem_usage,
seed=args.seed, gradient_accumulate_steps=args.gradient_accumulate_steps,
scale_dtype=args.scale_dtype, weight_config=weight_config) ##TODO args pass
- model, q_config = autoround.quantize()
+ model, _ = autoround.quantize()
model_name = args.model_name.rstrip("/")
- export_dir = args.output_dir + "/" + model_name.split('/')[-1] + f"-autoround-w{args.bits}g{args.group_size}"
-
- if args.deployment_device == 'cpu':
- export_dir += "-cpu"
- autoround.save_quantized(output_dir=export_dir)
- del q_config
- elif args.deployment_device == 'gpu':
- export_dir += "-gpu"
- autoround.save_quantized(export_dir, format="auto_gptq", use_triton=True)
- model = model.eval()
-
+
+ model.eval()
if args.device != "cpu":
torch.cuda.empty_cache()
- model.eval()
+
+ export_dir = args.output_dir + "/" + model_name.split('/')[-1] + f"-autoround-w{args.bits}g{args.group_size}"
output_dir = args.output_dir + "/" + model_name.split('/')[-1] + f"-autoround-w{args.bits}g{args.group_size}-qdq"
-
- pt_dtype = torch.float16
- if (hasattr(model, 'config') and (model.dtype is torch.bfloat16 or model.config.torch_dtype is torch.bfloat16)):
- dtype = 'bfloat16'
- pt_dtype = torch.bfloat16
- else:
- if str(args.device) != "cpu":
- pt_dtype = torch.float16
- dtype = 'float16'
+ deployment_device = args.deployment_device.split(',')
+ for dev in deployment_device:
+ if dev == 'gpu':
+ autoround.save_quantized(f'{export_dir}-gpu', format="auto_gptq", use_triton=True, inplace=False)
+ elif dev == 'cpu':
+ autoround.save_quantized(output_dir=f'{export_dir}-cpu', inplace=False)
+ elif dev == 'fake':
+ model = model.to("cpu")
+ model.save_pretrained(output_dir)
+ tokenizer.save_pretrained(output_dir)
else:
- pt_dtype = torch.float32
- dtype = 'float32'
+ print(f"The '{dev}' deployment type is not enabled yet.")
+
+ if not args.disable_lmeval:
+ excel_name = f"{output_dir}_result.xlsx"
+ output_dir += "/"
+ print(excel_name, flush=True)
+ eval_model(model_path=output_dir, tasks=tasks, dtype=dtype, limit=None,
+ eval_bs=args.eval_bs, use_accelerate=args.low_gpu_mem_usage,
+ device=torch_device, excel_file=excel_name)
- excel_name = f"{output_dir}_result.xlsx"
- output_dir += "/"
- print(excel_name, flush=True)
- if not args.eval_legacy:
- from eval.evaluation import eval_model
- else:
- from eval_legacy.evaluation import eval_model
- eval_model(output_dir=output_dir, model=model, tokenizer=tokenizer, tasks=args.tasks, \
- eval_bs=args.eval_bs, use_accelerate=args.low_gpu_mem_usage, device=torch_device, excel_file=excel_name,
- limit=None)
diff --git a/examples/language-modeling/requirements.txt b/examples/language-modeling/requirements.txt
index 61b731f0a..4cc53c9ce 100644
--- a/examples/language-modeling/requirements.txt
+++ b/examples/language-modeling/requirements.txt
@@ -1,6 +1,8 @@
transformers
torch
-git+https://github.com/EleutherAI/lm-evaluation-harness.git@008fc2a23245c40384f2312718433eeb1e0f87a9
+git+https://github.com/EleutherAI/lm-evaluation-harness.git@96d185fa6232a5ab685ba7c43e45d1dbb3bb906d
+# For the paper results use the old lm_eval (0.3.0)
+# git+https://github.com/EleutherAI/lm-evaluation-harness.git@008fc2a23245c40384f2312718433eeb1e0f87a9s
tiktoken
transformers_stream_generator
peft
@@ -11,3 +13,5 @@ datasets
protobuf
auto-gptq
openpyxl
+wandb
+
diff --git a/examples/language-modeling/run_autoround.sh b/examples/language-modeling/run_autoround.sh
index c6574049a..60c47f4ab 100644
--- a/examples/language-modeling/run_autoround.sh
+++ b/examples/language-modeling/run_autoround.sh
@@ -12,7 +12,7 @@ python3 main.py \
--use_quant_input \
--amp \
--iters 200 \
---deployment_device 'cpu' \
+--deployment_device 'fake,cpu' \
--scale_dtype 'fp32' \
--eval_bs 32 \
--output_dir "./tmp_autoround"