PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
fix bug of block-wise quant imatrix (#663)
Heng Guo5d68aeb75cfb
diff --git a/auto_round/autoround.py b/auto_round/autoround.pyindex 8acb21d00..a0c675490 100644--- a/auto_round/autoround.py+++ b/auto_round/autoround.py@@ -1183,15 +1183,16 @@ class AutoRound(object):hook = AlignDevicesHook(m.tuning_device, io_same_device=True)add_hook_to_module(m, hook, True)- input_ids = self.get_block_outputs(- block,- input_ids,- input_others,- self.batch_size * self.infer_bs_coeff,- self.device,- self.cache_device,- )+ input_ids = self.get_block_outputs(+ block,+ input_ids,+ input_others,+ self.batch_size * self.infer_bs_coeff,+ self.device,+ self.cache_device,+ )+ if self.device_map is not None:accelerate.hooks.remove_hook_from_submodules(block)# Normalize imatrix and quantize layers