Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

force to fp32 tuning if amp is disabled and align sym quantization (#18)

wenhuach213938d4ad1a7e
diff --git a/README.md b/README.md
index 77893faa1..93553a103 100644
--- a/README.md
+++ b/README.md
@@ -38,7 +38,7 @@ autoround.quantize()
output_dir = "./tmp_autoround"
deployment_device = "cpu" ## or gpu
-if deployment_device=="cpu":
+if deployment_device == "cpu":
autoround.save_quantized(output_dir, format="itrex") ## export to itrex format
else:
autoround.save_quantized(output_dir, format="auto_gptq", use_triton=True) ##export to autogptq format
diff --git a/auto_round/autoround.py b/auto_round/autoround.py
index cb46acfde..eb3c2c1c5 100644
--- a/auto_round/autoround.py
+++ b/auto_round/autoround.py
@@ -13,22 +13,13 @@
# limitations under the License.
-import logging
import torch
-logger = logging.getLogger("autoround")
-logger.setLevel(logging.INFO)
-fh = logging.StreamHandler()
-fh_formatter = logging.Formatter('%(asctime)s %(levelname)s %(filename)s L%(lineno)d: %(message)s',
- "%Y-%m-%d %H:%M:%S")
-fh.setFormatter(fh_formatter)
-logger.addHandler(fh)
-
import copy
import time
from .utils import (quant_weight, set_module, get_module, get_block_names, block_forward, sampling_inputs,
get_scale_shape, move_input_to_device, check_is_cpu, collect_round_v,
- collect_minmax_scale, get_batch_dim, htcore, is_hpu_available, check_to_quantized)
+ collect_minmax_scale, get_batch_dim, htcore, is_hpu_available,check_to_quantized,logger)
class SaveInputs:
"""Cache the inputs of the first block."""
@@ -533,6 +524,7 @@ class AutoRound(object):
):
from .calib_dataset import CALIB_DATASETS
self.model = model
+ self.model_orig_dtype = model.dtype
self.model = self.model.to("cpu")
self.amp = amp
self.use_quant_input = use_quant_input
@@ -574,10 +566,10 @@ class AutoRound(object):
self.amp_dtype = torch.bfloat16
if self.amp:
self.model = self.model.to(self.amp_dtype)
- logger.info(f"using {self.amp_dtype}")
- elif self.device == "cpu" and self.model.dtype == torch.float16:
+ logger.info(f"using {self.amp_dtype} for quantization tuning")
+ else:
self.model = self.model.to(torch.float32)
- logger.info(f"using {torch.float32} for cpu quantization")
+ logger.info(f"using {torch.float32} for quantization tuning")
self.dataset_name = dataset_name
if dataloader is None:
@@ -1011,7 +1003,7 @@ class AutoRound(object):
info = self.weight_config[key]
if not check_to_quantized(info):
continue
- info['zp'] = None if sym else info['zp'].to(torch.float32)
+ info['zp'] = info['zp'].to(torch.float32)
quantizers[key] = (None, info['scale'].to(torch.float32), info['zp'], info['g_idx'])
pack_model(model, quantizers, self.bits, self.group_size, use_cuda_fp16=True, desc_act=False,
force_layer_back_to_cpu=True, use_triton=True)
@@ -1107,6 +1099,7 @@ class AutoRound(object):
cost_time = end_time - start_time
logger.info(f"quantization tuning time {cost_time}")
self.quantized = True
+ self.model = self.model.to(self.model_orig_dtype)
return self.model, self.weight_config
diff --git a/auto_round/export/export_to_autogptq.py b/auto_round/export/export_to_autogptq.py
index aa393fc3a..cc46ce3fb 100644
--- a/auto_round/export/export_to_autogptq.py
+++ b/auto_round/export/export_to_autogptq.py
@@ -35,9 +35,7 @@
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
import torch
-from logging import getLogger
-
-logger = getLogger(__name__)
+from auto_round.utils import logger
import os
from typing import Dict, List, Optional, Union
from safetensors.torch import save_file as safe_save
diff --git a/auto_round/export/export_to_itrex/config.py b/auto_round/export/export_to_itrex/config.py
index 25ac08e6a..4b2f861ba 100644
--- a/auto_round/export/export_to_itrex/config.py
+++ b/auto_round/export/export_to_itrex/config.py
@@ -16,15 +16,14 @@
# limitations under the License.
"""Configs for Autoround quantization."""
-import logging
import torch
-logger = logging.getLogger()
import copy
import json
import os
from dataclasses import dataclass, field
from typing import Any, Dict, Tuple, Union
from transformers import PretrainedConfig
+from auto_round.utils import logger
QUANT_CONFIG = "quantize_config.json"
diff --git a/auto_round/export/export_to_itrex/export.py b/auto_round/export/export_to_itrex/export.py
index 9db61f692..e1b3afce2 100644
--- a/auto_round/export/export_to_itrex/export.py
+++ b/auto_round/export/export_to_itrex/export.py
@@ -13,9 +13,7 @@
# limitations under the License.
import torch
-from logging import getLogger
-
-logger = getLogger(__name__)
+from auto_round.utils import logger
import os
from typing import Dict, List, Optional, Union
from safetensors.torch import save_file as safe_save
diff --git a/auto_round/export/export_to_itrex/model_wrapper.py b/auto_round/export/export_to_itrex/model_wrapper.py
index 256903091..39270a84e 100644
--- a/auto_round/export/export_to_itrex/model_wrapper.py
+++ b/auto_round/export/export_to_itrex/model_wrapper.py
@@ -23,8 +23,7 @@ import torch
from packaging.version import Version
from torch.autograd import Function
from torch.nn import functional as F
-import logging
-logger = logging.getLogger()
+from auto_round.utils import logger
NF4 = [
diff --git a/auto_round/utils.py b/auto_round/utils.py
index 45ecc1dc3..614b5674a 100644
--- a/auto_round/utils.py
+++ b/auto_round/utils.py
@@ -17,6 +17,14 @@ import torch
import copy
from torch.amp import autocast
from collections import UserDict
+import logging
+logger = logging.getLogger("autoround")
+logger.setLevel(logging.INFO)
+fh = logging.StreamHandler()
+fh_formatter = logging.Formatter('%(asctime)s %(levelname)s %(filename)s L%(lineno)d: %(message)s',
+ "%Y-%m-%d %H:%M:%S")
+fh.setFormatter(fh_formatter)
+logger.addHandler(fh)
def is_optimum_habana_available():
@@ -102,20 +110,35 @@ def quant_weight_sym(weight, num_bits=4, v=0, min_scale=0, max_scale=0, scale_dt
Returns:
Quantized and dequantized weight, scale, zero-point
"""
- maxq = torch.tensor(2 ** (num_bits - 1) - 1).to(weight.device)
- minq = torch.tensor(-(2 ** (num_bits - 1))).to(weight.device)
- if num_bits == 1:
- maxq = torch.tensor(2 ** (num_bits - 1))
- minq = torch.tensor(2 ** (num_bits - 1) - 1)
-
- wmax = torch.abs(weight).max(1)[0]
- wmax *= 1 + max_scale
- tmp = wmax == 0
- wmax[tmp] = +1
- scale = (wmax / ((maxq - minq) / 2)).to(scale_dtype)
- scale.unsqueeze_(dim=-1)
- q = torch.clamp(round_ste(weight / scale + v), minq, maxq)
- return scale * q, scale, None
+ maxq = torch.tensor(2 ** num_bits - 1)
+ zeros = torch.zeros(weight.shape[0], device=weight.device, dtype=scale_dtype)
+ if isinstance(min_scale, torch.Tensor):
+ wmin_tmp = torch.minimum(weight.min(1)[0], zeros)
+ wmax_tmp = torch.maximum(weight.max(1)[0], zeros)
+ wmin_tmp *= min_scale + 1.0
+ wmax_tmp *= max_scale + 1.0
+ wmax = torch.maximum(wmax_tmp, wmin_tmp)
+ wmin = torch.minimum(wmax_tmp, wmin_tmp)
+ else:
+ wmin = torch.minimum(weight.min(1)[0], zeros)
+ wmax = torch.maximum(weight.max(1)[0], zeros)
+ wmax_new = torch.max(wmin.abs(), wmax)
+ tmp = wmin < 0
+ wmin_new = wmin.clone() ##must clone, otherwise inplace backward will occur
+ if torch.any(tmp):
+ wmin_new[tmp] = -wmax_new[tmp]
+
+ tmp = (wmin_new == 0) & (wmax_new == 0)
+ wmin_new[tmp] = -1
+ wmax_new[tmp] = +1
+ scale = ((wmax_new - wmin_new) / maxq).to(scale_dtype)
+
+ scale = scale.unsqueeze(dim=-1)
+ zp = torch.full_like(scale, (maxq + 1) / 2)
+
+ int_w = round_ste(weight / scale + v)
+ q = torch.clamp(int_w + zp, 0, maxq)
+ return scale * (q - zp), scale, zp
def quant_weight_actor(weight, num_bits, scheme, v, min_scale, max_scale, scale_dtype=torch.float16):
diff --git a/examples/language-modeling/main.py b/examples/language-modeling/main.py
index 365ddfeaf..5815a44e1 100644
--- a/examples/language-modeling/main.py
+++ b/examples/language-modeling/main.py
@@ -151,7 +151,7 @@ if __name__ == '__main__':
model_name, low_cpu_mem_usage=True, torch_dtype="auto", trust_remote_code=True
)
model = model.eval()
- # align wigh GPTQ to eval ppl
+ # align with GPTQ to eval ppl
if "opt" in model_name:
seqlen = model.config.max_position_embeddings
model.seqlen = model.config.max_position_embeddings
@@ -233,19 +233,16 @@ if __name__ == '__main__':
export_dir = args.output_dir + "/" + model_name.split('/')[-1] + f"-autoround-w{args.bits}g{args.group_size}"
output_dir = args.output_dir + "/" + model_name.split('/')[-1] + f"-autoround-w{args.bits}g{args.group_size}-qdq"
deployment_device = args.deployment_device.split(',')
- for dev in deployment_device:
- if dev == 'gpu':
- autoround.save_quantized(f'{export_dir}-gpu', format="auto_gptq", use_triton=True, inplace=False)
- elif dev == 'cpu':
- autoround.save_quantized(output_dir=f'{export_dir}-cpu', inplace=False)
- elif dev == 'fake':
- model = model.to("cpu")
- model.save_pretrained(output_dir)
- tokenizer.save_pretrained(output_dir)
- else:
- print(f"The '{dev}' deployment type is not enabled yet.")
-
- if not args.disable_lmeval:
+ if 'gpu' in deployment_device:
+ autoround.save_quantized(f'{export_dir}-gpu', format="auto_gptq", use_triton=True, inplace=False)
+ if "cpu" in deployment_device:
+ autoround.save_quantized(output_dir=f'{export_dir}-cpu', inplace=False)
+ if "fake" in deployment_device:
+ model = model.to("cpu")
+ model.save_pretrained(output_dir)
+ tokenizer.save_pretrained(output_dir)
+
+ if not args.disable_lmeval and "fake" in deployment_device: ##support autogptq real eval later
excel_name = f"{output_dir}_result.xlsx"
output_dir += "/"
print(excel_name, flush=True)