Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

bugfix (#160)

WeiweiZhang131c566ccbb27
diff --git a/auto_round/autoround.py b/auto_round/autoround.py
index 878eda467..c5df1ad1a 100644
--- a/auto_round/autoround.py
+++ b/auto_round/autoround.py
@@ -891,7 +891,7 @@ class AutoRound(object):
output_q = block_forward(
block, current_input_ids, current_input_others, self.amp, self.amp_dtype, device
)
- if self.amp and not check_is_cpu(device):
+ if self.amp:
with autocast(device_type=device.split(":")[0], dtype=self.amp_dtype):
loss = mse_loss(output_q, current_output) # pylint: disable=not-callable
else:
@@ -1416,3 +1416,4 @@ class AutoAdamRound(AutoOPTRound):
optimizer,
**kwargs,
)
+
diff --git a/examples/language-modeling/eval_042/evaluation.py b/examples/language-modeling/eval_042/evaluation.py
index ea19c8fae..b14a67c82 100644
--- a/examples/language-modeling/eval_042/evaluation.py
+++ b/examples/language-modeling/eval_042/evaluation.py
@@ -573,6 +573,10 @@ if __name__ == "__main__":
parser.add_argument(
"--eval_bs", default=1,
)
+ parser.add_argument(
+ "--trust_remote_code", action='store_true',
+ help="Whether to enable trust_remote_code"
+ )
parser.add_argument("--tasks",
default="lambada_openai,hellaswag,winogrande,piqa,mmlu,truthfulqa_mc1," \
"openbookqa,boolq,rte,arc_easy,arc_challenge",
@@ -582,7 +586,7 @@ if __name__ == "__main__":
s = time.time()
from transformers import AutoConfig
- config = AutoConfig.from_pretrained(args.model_name)
+ config = AutoConfig.from_pretrained(args.model_name, trust_remote_code=args.trust_remote_code)
if hasattr(config, "quantization_config"):
quantization_config = config.quantization_config
@@ -599,6 +603,8 @@ if __name__ == "__main__":
if config.torch_dtype == torch.float32:
model_args += ",dtype=float16"
# model_args += ",dtype=float16"
+ if args.trust_remote_code:
+ model_args += f",trust_remote_code=True"
result = simple_evaluate(model="hf",
model_args=model_args,
tasks=test_tasks,
@@ -606,3 +612,4 @@ if __name__ == "__main__":
print(make_table(result))
print("cost time: ", time.time() - s)
+
diff --git a/examples/language-modeling/main.py b/examples/language-modeling/main.py
index d5171fe55..3569ce671 100644
--- a/examples/language-modeling/main.py
+++ b/examples/language-modeling/main.py
@@ -231,16 +231,12 @@ if __name__ == '__main__':
args.seqlen = seqlen
excel_name = f"{model_name}_{args.bits}_{args.group_size}"
- pt_dtype = torch.float16
if (hasattr(model, 'config') and (model.dtype is torch.bfloat16 or model.config.torch_dtype is torch.bfloat16)):
dtype = 'bfloat16'
- pt_dtype = torch.bfloat16
else:
- if str(args.device) != "cpu":
- pt_dtype = torch.float16
+ if "cpu" not in device_str:
dtype = 'float16'
else:
- pt_dtype = torch.float32
dtype = 'float32'
excel_name = f"{model_name}_{args.bits}_{args.group_size}"
@@ -313,7 +309,7 @@ if __name__ == '__main__':
model_name = args.model_name.rstrip("/")
model.eval()
- if args.device != "cpu":
+ if "cpu" not in device_str:
torch.cuda.empty_cache()
export_dir = args.output_dir + "/" + model_name.split('/')[-1] + f"-autoround-w{args.bits}g{args.group_size}"
@@ -357,3 +353,4 @@ if __name__ == '__main__':
from lm_eval.utils import make_table
print(make_table(res))
+