Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

fix bug of block-wise quant imatrix (#663)

Heng Guo5d68aeb75cfb
diff --git a/auto_round/autoround.py b/auto_round/autoround.py
index 8acb21d00..a0c675490 100644
--- a/auto_round/autoround.py
+++ b/auto_round/autoround.py
@@ -1183,15 +1183,16 @@ class AutoRound(object):
hook = AlignDevicesHook(m.tuning_device, io_same_device=True)
add_hook_to_module(m, hook, True)
- input_ids = self.get_block_outputs(
- block,
- input_ids,
- input_others,
- self.batch_size * self.infer_bs_coeff,
- self.device,
- self.cache_device,
- )
+ input_ids = self.get_block_outputs(
+ block,
+ input_ids,
+ input_others,
+ self.batch_size * self.infer_bs_coeff,
+ self.device,
+ self.cache_device,
+ )
+ if self.device_map is not None:
accelerate.hooks.remove_hook_from_submodules(block)
# Normalize imatrix and quantize layers