Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

set autoround format as default to unify CPU/HPU/CUDA (#205)

wenhuach211e75afd1df2f
diff --git a/README.md b/README.md
index f14a08a79..b9f614aba 100644
--- a/README.md
+++ b/README.md
@@ -55,9 +55,6 @@ pip install auto-round
### Gaudi2/ CPU/ GPU
-We found a significant accuracy discrepancy with the qdq model using the AutoGPTQ GPU backend with asymmetric
-quantization in some scenarios, especially at lower bits,like 2. Please save quantized model to AuoRound format to fix this issue.
-
```python
from transformers import AutoModelForCausalLM, AutoTokenizer
@@ -71,7 +68,7 @@ bits, group_size, sym = 4, 128, False
autoround = AutoRound(model, tokenizer, bits=bits, group_size=group_size, sym=sym)
autoround.quantize()
output_dir = "./tmp_autoround"
-autoround.save_quantized(output_dir) ##save_quantized(output_dir,format="auto_round")
+autoround.save_quantized(output_dir) ##save_quantized(output_dir,format="auto_gptq)
```
<details>
@@ -135,49 +132,31 @@ autoround.save_quantized(output_dir) ##save_quantized(output_dir,format="auto_ro
3 Setting 'minmax_lr' to 2.0/iters has been observed to occasionally yield improved results.
## Model inference
+Please run the quantization code first
-Please run the quantization code first.
+### Setup env for AutoRound format
-### CPU
-```python
-##pip install intel-extension-for-transformers
-from intel_extension_for_transformers.transformers import AutoModelForCausalLM
-from transformers import AutoTokenizer
+2 Set up the environment
-quantized_model_path = "./tmp_autoround"
-model = AutoModelForCausalLM.from_pretrained(quantized_model_path)
-tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)
-text = "There is a girl who likes adventure,"
-inputs = tokenizer(text, return_tensors="pt").to(model.device)
-print(tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0]))
-```
+**cuda**: git clone https://github.com/intel/auto-round.git && cd auto-round && pip install -vvv --no-build-isolation -e .
-### GPU
+cpu: pip install intel-extension-for-transformers
-```python
-##pip install auto-gptq
-from transformers import AutoModelForCausalLM, AutoTokenizer
-##from auto_round.auto_quantizer import AutoHfQuantizer ## uncomment it for models with auto_round format
+hpu: docker image with Gaudi Software Stack is recommended, please refer to following script for environment setup. More details can be found in Gaudi Guide.
-quantized_model_path = "./tmp_autoround"
-model = AutoModelForCausalLM.from_pretrained(quantized_model_path, device_map="auto")
-tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)
-text = "There is a girl who likes adventure,"
-inputs = tokenizer(text, return_tensors="pt").to(model.device)
-print(tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0]))
-```
-### Intel Gaudi-2
+### Gaudi2/ CPU/ GPU
```python
-import torch
-from transformers import AutoModelForCausalLM, AutoTokenizer
-from auto_round.auto_quantizer import AutoHfQuantizer
-import habana_frameworks.torch.core as htcore
-import habana_frameworks.torch.hpu as hthpu
+from transformers import AutoModelForCausalLM,AutoTokenizer
+from auto_round import AutoRoundConfig
+device = "auto" ##cpu, hpu, cuda
+quantization_config = AutoRoundConfig(
+ backend=device
+)
quantized_model_path = "./tmp_autoround"
-model = AutoModelForCausalLM.from_pretrained(quantized_model_path).to('hpu').to(torch.float32)
+model = AutoModelForCausalLM.from_pretrained(quantized_model_path, device_map=device,quantization_config=quantization_config)
tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)
text = "There is a girl who likes adventure,"
inputs = tokenizer(text, return_tensors="pt").to(model.device)
diff --git a/auto_round/__init__.py b/auto_round/__init__.py
index 46f42e4a2..0f00cf352 100644
--- a/auto_round/__init__.py
+++ b/auto_round/__init__.py
@@ -12,4 +12,5 @@
# See the License for the specific language governing permissions and
# limitations under the License.
from .autoround import AutoRound, AutoAdamRound, AutoOPTRound
+from .auto_quantizer import AutoHfQuantizer,AutoRoundConfig
from .version import __version__
\ No newline at end of file
diff --git a/auto_round/auto_quantizer.py b/auto_round/auto_quantizer.py
index 4bc89710b..3aab37894 100644
--- a/auto_round/auto_quantizer.py
+++ b/auto_round/auto_quantizer.py
@@ -44,6 +44,7 @@ from transformers.utils.quantization_config import AwqConfig, GPTQConfig, Quanti
from auto_round.utils import get_module, set_module, dynamic_import_inference_linear
import auto_round_extension.qbits.qlinear_qbits as qlinear_qbits
from enum import Enum
+import copy
logger = getLogger(__name__)
import sys
@@ -106,17 +107,19 @@ def is_autoround_exllamav2_available():
res = False
return res
-def is_hpu_supported(): # pragma: no cover
+
+def is_hpu_supported(): # pragma: no cover
try:
import subprocess
- import habana_frameworks.torch.core as htcore # pylint: disable=E0401
+ import habana_frameworks.torch.core as htcore # pylint: disable=E0401
hqt_version = subprocess.check_output(['pip', 'show', \
- 'habana_quantization_toolkit']).decode().split('\n')[1].split(': ')[1]
- assert(hqt_version >= "1.17")
+ 'habana_quantization_toolkit']).decode().split('\n')[1].split(': ')[1]
+ assert (hqt_version >= "1.17")
except ImportError as e:
return False
return True
+
if is_auto_round_available():
from auto_round_extension.cuda.post_init import autoround_post_init
@@ -155,7 +158,7 @@ class AutoHfQuantizer:
f"Unknown quantization type, got {quant_method} - supported types are:"
f" {list(AUTO_QUANTIZER_MAPPING.keys())}"
)
- if "auto-round" in quant_method or is_hpu_supported(): # pragma: no cover
+ if "auto-round" in quant_method or is_hpu_supported(): # pragma: no cover
target_cls = AutoRoundQuantizer
else:
target_cls = AUTO_QUANTIZER_MAPPING[quant_method]
@@ -190,7 +193,8 @@ class AutoHfQuantizer:
else:
quantization_config = AutoQuantizationConfig.from_dict(quantization_config) # pylint: disable=E1101
- if isinstance(quantization_config, (GPTQConfig, AwqConfig)) and quantization_config_from_args is not None:
+ if isinstance(quantization_config,
+ (GPTQConfig, AwqConfig, AutoRoundConfig)) and quantization_config_from_args is not None:
# special case for GPTQ / AWQ config collision
loading_attr_dict = quantization_config_from_args.get_loading_attributes()
for attr, val in loading_attr_dict.items():
@@ -228,12 +232,12 @@ class AutoRoundConfig(QuantizationConfigMixin):
def __init__(
self,
- bits: int,
+ bits: int = 4,
tokenizer: Any = None,
dataset: str = None,
group_size: int = 128,
sym: bool = False,
- backend="auto_round:exllamav2",
+ backend="auto",
layer_config: dict = None,
**kwargs,
):
@@ -243,6 +247,13 @@ class AutoRoundConfig(QuantizationConfigMixin):
self.dataset = dataset
self.group_size = group_size
self.sym = sym
+ if "auto" == backend:
+ if torch.cuda.is_available():
+ backend = "auto_round:exllamav2"
+ elif is_hpu_supported():
+ backend = "hpu"
+ else:
+ backend = "cpu"
self.backend = backend
self.layer_config = layer_config
if kwargs is not None:
@@ -251,8 +262,6 @@ class AutoRoundConfig(QuantizationConfigMixin):
self.quant_method = AutoRoundQuantizationMethod.AutoRound
self.post_init()
- def get_loading_attributes(self):
- return {}
def post_init(self):
r"""Safety checker that arguments are correct."""
@@ -262,6 +271,12 @@ class AutoRoundConfig(QuantizationConfigMixin):
raise ValueError("group_size must be greater than 0 or equal to -1")
##TODO add more check
+ def get_loading_attributes(self):
+ attributes_dict = copy.deepcopy(self.__dict__)
+ loading_attributes = ["backend"]
+ loading_attibutes_dict = {i: j for i, j in attributes_dict.items() if i in loading_attributes}
+ return loading_attibutes_dict
+
def to_dict(self):
config_dict = super().to_dict()
config_dict.pop("disable_exllama", None)
@@ -296,7 +311,7 @@ class AutoRoundQuantizer(HfQuantizer):
def update_torch_dtype(self, torch_dtype: "torch.dtype") -> "torch.dtype":
if torch_dtype is None:
torch_dtype = torch.float16
- elif torch_dtype != torch.float16:
+ elif torch_dtype != torch.float16 and not is_hpu_supported():
logger.info("We suggest you to set `torch_dtype=torch.float16` for better efficiency with AutoRound.")
return torch_dtype
@@ -311,10 +326,18 @@ class AutoRoundQuantizer(HfQuantizer):
layer_names = get_layer_names_in_block(model)
quantization_config = model.config.quantization_config
+ if hasattr(quantization_config, "backend"): # pragma: no cover
+ backend = quantization_config.backend
+ if "hpu" in backend and model.dtype != torch.bfloat16:
+ logger.info("We suggest you to set `torch_dtype=torch.bfloat16` for better efficiency with AutoRound.")
+ model = model.to(torch.bfloat16)
+ elif model.dtype != torch.float16:
+ logger.info("We suggest you to set `torch_dtype=torch.float16` for better efficiency with AutoRound.")
+ model = model.to(torch.float16)
bits = quantization_config.bits
group_size = quantization_config.group_size
data_type = quantization_config.data_type if hasattr(quantization_config, "data_type") \
- else "int" # pragma: no cover
+ else "int" # pragma: no cover
sym = quantization_config.sym
extra_config = {}
if hasattr(quantization_config, "extra_config"):
@@ -334,11 +357,11 @@ class AutoRoundQuantizer(HfQuantizer):
layer_configs[layer_name]["group_size"] = extra_config[layer_name].get("group_size", group_size)
layer_configs[layer_name]["data_type"] = extra_config[layer_name].get("data_type", data_type)
layer_configs[layer_name]["sym"] = extra_config[layer_name].get("sym", sym)
- if hasattr(quantization_config, "backend"): # pragma: no cover
+ if hasattr(quantization_config, "backend"): # pragma: no cover
backend = quantization_config.backend
- elif 'gptq' in quantization_config.quant_method: # pragma: no cover
+ elif 'gptq' in quantization_config.quant_method: # pragma: no cover
backend = 'gptq'
- else: # pragma: no cover
+ else: # pragma: no cover
logger.error("Please specify quantization backend")
self._replace_by_quant_layers(model, layer_configs, backend)
diff --git a/auto_round/autoround.py b/auto_round/autoround.py
index aefc0d7f4..72ce8a235 100644
--- a/auto_round/autoround.py
+++ b/auto_round/autoround.py
@@ -49,6 +49,7 @@ from .utils import (
from .low_cpu_mem.utils import get_layers_before_block
+
class AutoRound(object):
"""This is Signround+ which is an advanced version of Signround. For more information,
please refer to Cheng, Wenhua, et al. "Optimize weight rounding via signed gradient descent
@@ -151,8 +152,8 @@ class AutoRound(object):
self.quantized = False
self.model_orig_dtype = model.dtype
self.low_cpu_mem_usage = low_cpu_mem_usage
- assert not unsupport_meta_device(model), (
- "autoround does not support for params on meta device by transformers` interfaces,"
+ assert not unsupport_meta_device(model), (
+ "autoround does not support for params on meta device by transformers` interfaces,"
"please do not using device_map='auto' in model loading, "
"or follow examples/language-modeling/main.py to enable low_cpu_mem_usage")
self.model = model.eval()
@@ -222,7 +223,7 @@ class AutoRound(object):
assert self.bits > 0, "bits must be positive"
assert self.act_bits > 0, "bits must be positive"
assert self.group_size == -1 or self.group_size >= 1, "only supports positive group_size or -1(per channel)"
- assert self.act_group_size == -1 or self.act_group_size >= 1,\
+ assert self.act_group_size == -1 or self.act_group_size >= 1, \
"only supports positive group_size or -1(per channel)"
assert self.train_bs > 0, "batch size must be positive"
assert self.iters > 0, "iters must be positive"
@@ -245,7 +246,7 @@ class AutoRound(object):
all_blocks = self.quant_block_list
else:
all_blocks = get_block_names(self.model)
-
+
if len(all_blocks) == 0:
logger.warning("could not find blocks, exit with original model")
return self.model, self.layer_config
@@ -400,7 +401,6 @@ class AutoRound(object):
for key in keys:
setattr(m, key, layer_config[n][key])
-
@torch.no_grad()
def get_block_outputs(self, block, input_ids, input_others, bs, device, cache_device):
"""Compute the output of a given block of the model for a given input.
@@ -473,7 +473,7 @@ class AutoRound(object):
embed_layers = get_layers_before_block(self.model)
for n, m in embed_layers:
m = m.to(self.device)
-
+
for data in self.dataloader:
if data is None:
continue
@@ -490,8 +490,8 @@ class AutoRound(object):
data_new[key] = data[key].to(self.device)
input_ids = data_new["input_ids"]
elif isinstance(data, tuple) or isinstance(data, list):
- data_new = data
- input_ids = data_new[0]
+ data_new = data
+ input_ids = data_new[0]
else:
data_new = {}
for key in data.keys():
@@ -501,7 +501,7 @@ class AutoRound(object):
input_ids = data_new["input_ids"]
if input_ids.shape[-1] < self.seqlen:
continue
-
+
try:
if isinstance(data_new, torch.Tensor):
self.model(data_new)
@@ -527,7 +527,7 @@ class AutoRound(object):
f"Insufficient number of samples collected may affect the quantification. "
f"Valid samples size:{total_cnt}, Target sample size:{nsamples}"
)
-
+
# clean embed weight to save memory
if self.low_cpu_mem_usage:
for n, m in embed_layers:
@@ -671,8 +671,8 @@ class AutoRound(object):
elif "position_ids" in key:
if key not in self.inputs[name].keys():
self.inputs[name][key] = list(torch.split(kwargs[key].to("cpu"), 1, dim=0)) \
- if self.not_share_position_ids_flag \
- else to_device(kwargs[key], device=torch.device("cpu"))
+ if self.not_share_position_ids_flag \
+ else to_device(kwargs[key], device=torch.device("cpu"))
elif kwargs[key] is not None and self.not_share_position_ids_flag:
self.inputs[name][key].extend(list(torch.split(kwargs[key].to("cpu"), 1, dim=0)))
elif key not in self.inputs[name].keys():
@@ -1064,7 +1064,7 @@ class AutoRound(object):
torch.cuda.empty_cache()
- def save_quantized(self, output_dir=None, format="auto_gptq", inplace=True, **kwargs):
+ def save_quantized(self, output_dir=None, format="auto_round", inplace=True, **kwargs):
"""Save the quantized model to the specified output directory in the specified format.
Args:
@@ -1529,7 +1529,3 @@ class AutoAdamRound(AutoOPTRound):
optimizer,
**kwargs,
)
-
-
-
-
diff --git a/auto_round/data_type/mxfp.py b/auto_round/data_type/mxfp.py
index efadf0c2b..c95db837c 100644
--- a/auto_round/data_type/mxfp.py
+++ b/auto_round/data_type/mxfp.py
@@ -22,6 +22,7 @@ MXFP_FORMAT_CACHE = {
"mx_int4": (0, 4, 0, 1.75, 0),
"mx_int2": (0, 2, 0, 1.0, 0),
"mx_fp8e5m2": (5, 4, 15, 57344.0, 6.103515625e-05),
+ "mx_fp8": (4, 5, 8, 448.0, 0.015625),
"mx_fp8e4m3": (4, 5, 8, 448.0, 0.015625),
"mx_fp6e3m2": (3, 4, 4, 28.0, 0.25),
"mx_fp6e2m3": (2, 5, 2, 7.5, 1.0),
@@ -73,7 +74,8 @@ def quant_mx(tensor, bits, data_type, v, max_scale, **kwargs):
shared_exp[shared_exp > scale_emax] = scale_emax ##changed Nan
shared_exp[shared_exp < -scale_emax] = -scale_emax
tensor = tensor / (2 ** shared_exp)
- multiply = 2 if data_type == "mx_fp4" else 1 ## 2 is a tricky setting
+ is_mx_fp4 = data_type == "mx_fp4" or ("mx_fp" in data_type and bits==4)
+ multiply = 2 if is_mx_fp4 else 1 ## 2 is a tricky setting
tensor = tensor + v * multiply
if ebits != 0:
private_exp = floor_ste(torch.log2(torch.abs(tensor) + (tensor == 0).type(tensor.dtype)))
diff --git a/auto_round/utils.py b/auto_round/utils.py
index a3f2a5e18..ff2d473a4 100644
--- a/auto_round/utils.py
+++ b/auto_round/utils.py
@@ -804,7 +804,7 @@ def dynamic_import_inference_linear(backend, bits, group_size, sym):
else:
from auto_round_extension.hpu.qlinear_hpu_gptq import QuantLinear
return QuantLinear
- if bits == 4 and is_optimum_habana_available(): # pragma: no cover
+ if (bits == 4 and is_optimum_habana_available()) or "hpu" in backend: # pragma: no cover
try:
import habana_frameworks.torch.hpu # noqa: F401 # pylint: disable=E0401
except Exception as e:
diff --git a/auto_round/version.py b/auto_round/version.py
index c7c0b0e2a..b2bcf8e76 100644
--- a/auto_round/version.py
+++ b/auto_round/version.py
@@ -14,4 +14,4 @@
"""Intel® auto-round: An open-source Python library
supporting popular model weight only compression based on signround."""
-__version__ = "0.3.0.dev"
+__version__ = "0.3.1.dev"
diff --git a/examples/language-modeling/main.py b/examples/language-modeling/main.py
index 733447600..0b0548332 100644
--- a/examples/language-modeling/main.py
+++ b/examples/language-modeling/main.py
@@ -81,14 +81,13 @@ if __name__ == '__main__':
parser.add_argument("--enable_minmax_tuning", action='store_true',
help="enable_minmax_tuning is deprecated")
- parser.add_argument("--deployment_device", default='fake', type=str,
+ parser.add_argument("--deployment_device", default=None, type=str,
help="targeted inference acceleration platform,The options are 'fake', 'cpu', 'gpu' and 'xpu'."
"default to 'fake', indicating that it only performs fake quantization and won't be exported to any device.")
parser.add_argument("--data_type", default='int',
help="data type for tuning, 'int', 'mx_fp' and etc.")
-
parser.add_argument("--scale_dtype", default='fp16',
help="which scale data type to use for quantization, 'fp16', 'fp32' or 'bf16'.")
@@ -120,10 +119,10 @@ if __name__ == '__main__':
parser.add_argument("--low_cpu_mem_mode", default=0, type=int,
help="Choose which low cpu memory mode to use. Can significantly reduce cpu memory footprint but cost more time."
- "1 means choose block-wise mode, load the weights of each block from disk when tuning and release the memory of the block after tuning."
- "2 means choose layer-wise mode, load the weights of each layer from disk when tuning, minimum memory consumption and also slowest running speed."
- "others means not use low cpu memory. Default to 0, not use low cpu memory.")
-
+ "1 means choose block-wise mode, load the weights of each block from disk when tuning and release the memory of the block after tuning."
+ "2 means choose layer-wise mode, load the weights of each layer from disk when tuning, minimum memory consumption and also slowest running speed."
+ "others means not use low cpu memory. Default to 0, not use low cpu memory.")
+
parser.add_argument("--low_cpu_mem_tmp_dir", default=None, type=str,
help="temp work space to store the temporary files when using low cpu memory mode. Will remove after tuning.")
@@ -151,9 +150,12 @@ if __name__ == '__main__':
tasks = args.tasks
use_eval_legacy = False
+ if args.deployment_device is None:
+ args.deployment_device = "auto_round"
if "gpu" in args.deployment_device and args.sym is False:
- print("warning: The auto_gptq kernel has issues with asymmetric quantization. It is recommended to use --deployment_device='auto_round'")
+ print(
+ "warning: The auto_gptq kernel has issues with asymmetric quantization. It is recommended to use --deployment_device='auto_round'")
if "marlin" in args.deployment_device and args.sym is False:
assert False, "marlin backend only supports sym quantization, please set --sym"
@@ -187,6 +189,7 @@ if __name__ == '__main__':
args.low_cpu_mem_tmp_dir = os.path.join(args.output_dir, "low_cpu_mem_tmp")
if args.low_cpu_mem_mode == 2:
from auto_round.low_cpu_mem.utils import load_model_with_hooks
+
model = load_model_with_hooks(
model_name,
model_cls,
@@ -198,6 +201,7 @@ if __name__ == '__main__':
)
elif args.low_cpu_mem_mode == 1:
from auto_round.low_cpu_mem.utils import load_empty_model
+
low_cpu_mem_usage = True
model = load_empty_model(
model_name,
@@ -206,7 +210,7 @@ if __name__ == '__main__':
saved_path=args.low_cpu_mem_tmp_dir,
torch_dtype=torch_dtype,
trust_remote_code=not args.disable_trust_remote_code
- )
+ )
else:
model = model_cls.from_pretrained(
model_name, low_cpu_mem_usage=True, torch_dtype=torch_dtype,
@@ -305,6 +309,7 @@ if __name__ == '__main__':
model_name = args.model_name.rstrip("/")
if args.low_cpu_mem_mode == 1 or args.low_cpu_mem_mode == 2:
import shutil
+
shutil.rmtree(args.low_cpu_mem_tmp_dir, ignore_errors=True)
model.eval()
@@ -336,7 +341,7 @@ if __name__ == '__main__':
elif "gptq" in gpu_format:
eval_folder = f'{export_dir}-gpu'
autoround.save_quantized(eval_folder, format=gpu_format, use_triton=False, inplace=inplace)
- elif "auto_awq" in gpu_format: # pragma: no cover
+ elif "auto_awq" in gpu_format: # pragma: no cover
eval_folder = f'{export_dir}-awq'
autoround.save_quantized(eval_folder, format=gpu_format, inplace=inplace, model_path=model_name)
@@ -419,6 +424,3 @@ if __name__ == '__main__':
from lm_eval.utils import make_table
print(make_table(res))
-
-
-