Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

Fix reload issue for lm_head quantization and update CI to cover it (#1563)

Xin He13fbffee25f3
diff --git a/auto_round/compressors/base.py b/auto_round/compressors/base.py
index e15f918aa..0a2352e2a 100644
--- a/auto_round/compressors/base.py
+++ b/auto_round/compressors/base.py
@@ -1859,15 +1859,14 @@ class BaseCompressor(object):
)
pbar.set_description("Quantizing done")
pbar.close()
+ if self.low_cpu_mem_usage:
+ self._offloader.reload(self.model)
self._quantize_layers(layer_names, all_inputs)
convert_module_to_hp_if_necessary(self.model, self.amp_dtype, self.device, to_cpu=True)
if self.is_immediate_saving:
shard_writer(self, is_finalize=True)
- if self.low_cpu_mem_usage:
- self._offloader.reload(self.model)
-
end_time = time.time()
cost_time = end_time - start_time
logger.info(f"quantization tuning time {cost_time}")
diff --git a/test/test_cpu/core/test_autoround.py b/test/test_cpu/core/test_autoround.py
index 0ed63dda4..701e02ea1 100644
--- a/test/test_cpu/core/test_autoround.py
+++ b/test/test_cpu/core/test_autoround.py
@@ -253,19 +253,16 @@ class TestAutoRound:
)
autoround.quantize()
- def test_lm_head_layer_config_way(self, dataloader):
+ def test_lm_head_layer_config_way(self, tiny_untied_qwen_model_path, dataloader):
bits, group_size, sym = 4, -1, False
layer_config = {"lm_head": {"data_type": "int"}}
autoround = AutoRound(
- self.model,
- self.tokenizer,
+ tiny_untied_qwen_model_path,
bits=bits,
group_size=group_size,
sym=sym,
- iters=2,
- seqlen=10,
- enable_minmax_tuning=False,
- enable_quanted_input=False,
+ iters=1,
+ seqlen=1,
dataset=dataloader,
layer_config=layer_config,
)
@@ -637,21 +634,13 @@ class TestAutoRound:
def test_quant_lm_head(self, tiny_untied_qwen_model_path):
model_name = tiny_untied_qwen_model_path
- ar = AutoRound(model_name, quant_lm_head=True, iters=0, seqlen=8, nsamples=1, disable_opt_rtn=True)
- ar.quantize_and_save(output_dir=self.save_folder, format="auto_round")
- model = AutoModelForCausalLM.from_pretrained(self.save_folder, device_map="cpu")
- assert "lm_head" in model.config.quantization_config.extra_config
- assert model.config.quantization_config.extra_config["lm_head"]["bits"] == 4
-
- layer_config = {"lm_head": {"bits": 4}}
ar = AutoRound(
model_name,
- quant_lm_head=False,
+ quant_lm_head=True,
iters=0,
seqlen=8,
nsamples=1,
disable_opt_rtn=True,
- layer_config=layer_config,
)
ar.quantize_and_save(output_dir=self.save_folder, format="auto_round")
model = AutoModelForCausalLM.from_pretrained(self.save_folder, device_map="cpu")