Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

move some settings from example to main (#241)

wenhuach21ec20b5923846
diff --git a/auto_round/autoround.py b/auto_round/autoround.py
index 00cc0e308..318d708a9 100644
--- a/auto_round/autoround.py
+++ b/auto_round/autoround.py
@@ -12,13 +12,16 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+import os
+import torch
+import transformers
+
+torch.use_deterministic_algorithms(True, warn_only=True)
import copy
import time
from typing import Optional, Union
-import torch
-import transformers
from transformers import set_seed
from torch import autocast
from tqdm import tqdm
@@ -240,6 +243,18 @@ class AutoRound(object):
assert self.enable_full_range is False, "only support enable_full_range=False currently"
assert self.act_dynamic is True, "only support dynamic quantization for activation currently"
# assert self.tokenizer != None or self.dataloader != None
+ if self.act_bits <= 8:
+ logger.warning(
+ "please save the quantized model to fake format "
+ "as real deployment is not supported for activation quantization currently")
+
+ if "mx_fp" in self.data_type:
+ logger.warning(
+ "please save the quantized model to fake format"
+ "as real deployment is not supported for mx_fp datatype currently")
+
+ if "mx_fp" in self.data_type and self.group_size != 32:
+ logger.warning("mx_fp should only support group_size of 32 in real deployment")
def quantize(self):
"""Quantize the model and return the quantized model along with layer configurations.
diff --git a/examples/language-modeling/main.py b/examples/language-modeling/main.py
index ad45c8542..9dbf2ea67 100644
--- a/examples/language-modeling/main.py
+++ b/examples/language-modeling/main.py
@@ -164,14 +164,6 @@ if __name__ == '__main__':
if "marlin" in args.deployment_device and args.sym is False:
assert False, "marlin backend only supports sym quantization, please set --sym"
- if args.act_bits <= 8 and args.deployment_device != "fake":
- assert False, "only support fake mode for activation quantization currently"
-
- if "mx_fp" in args.data_type and args.deployment_device != "fake":
- assert False, "only support fake mode for mx_fp data type currently"
-
- if "mx_fp" in args.data_type and args.group_size != 32:
- print("warning, mx_fp should only support group_size of 32 in real deployment")
model_name = args.model_name
if model_name[-1] == "/":