PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
fix dynamic int8 w8a8 export issue with tuning (#1525)
Huang, Taia42fd85d894b
diff --git a/auto_round/export/export_to_llmcompressor/export.py b/auto_round/export/export_to_llmcompressor/export.pyindex 6ea8eca1d..8df22b93d 100644--- a/auto_round/export/export_to_llmcompressor/export.py+++ b/auto_round/export/export_to_llmcompressor/export.py@@ -68,15 +68,23 @@ def pack_layer(name, model, device=None):if type(layer) not in SUPPORTED_LAYER_TYPES and not isinstance(layer, WrapperWALayer): ##already packedreturn+ if hasattr(layer, "orig_layer"): # revert WrapperWALayer for offline usage+ wp_layer = layer+ layer = wp_layer.orig_layer+ set_module(model, name, layer)+if not check_to_quantized(layer):returnif hasattr(layer, "quantization_status") and layer.quantization_status == QuantizationStatus.COMPRESSED:return+ # explicitly obtain the underlying device to prevent RuntimeError mismatched tensors+ weight_device = layer.weight.device+scheme = construct_ct_scheme(layer)setattr(layer, "quantization_scheme", scheme)- setattr(layer, "weight_scale", torch.nn.Parameter(layer.scale.to(device)))+ setattr(layer, "weight_scale", torch.nn.Parameter(layer.scale.to(weight_device)))if not isinstance(layer.zp, torch.Tensor):if layer.sym:zp = torch.full_like(layer.weight_scale, 0).to(torch.int8)@@ -85,7 +93,7 @@ def pack_layer(name, model, device=None):else:zp = layer.zp- setattr(layer, "weight_zero_point", torch.nn.Parameter(zp.to(device), requires_grad=False))+ setattr(layer, "weight_zero_point", torch.nn.Parameter(zp.to(weight_device), requires_grad=False))delattr(layer, "scale")compressor = IntQuantizationCompressor()diff --git a/test/test_cpu/export/test_export.py b/test/test_cpu/export/test_export.pyindex 35f9adb77..f0535ddd3 100644--- a/test/test_cpu/export/test_export.py+++ b/test/test_cpu/export/test_export.py@@ -502,7 +502,18 @@ class TestAutoRound:)quantized_model_path = "./saved"autoround.quantize_and_save(output_dir=quantized_model_path, format="llm_compressor")- f = safe_open(os.path.join(quantized_model_path, "model.safetensors"), framework="pt")- assert "model.decoder.layers.8.self_attn.k_proj.weight_scale" in f.keys()- assert f.get_tensor("model.decoder.layers.5.self_attn.v_proj.weight").dtype == torch.int8+ with safe_open(os.path.join(quantized_model_path, "model.safetensors"), framework="pt") as f:+ assert "model.decoder.layers.8.self_attn.k_proj.weight_scale" in f.keys()+ assert f.get_tensor("model.decoder.layers.5.self_attn.v_proj.weight").dtype == torch.int8+ shutil.rmtree(quantized_model_path, ignore_errors=True)++ def test_llmc_dynamic_wint8aint8_export_with_tuning(self, dataloader):+ from safetensors import safe_open++ autoround = AutoRound(self.model_name, iters=1, nsamples=2, seqlen=2, dataset=dataloader, scheme="INT8_W8A8")+ quantized_model_path = "./saved"+ autoround.quantize_and_save(output_dir=quantized_model_path, format="llm_compressor")+ with safe_open(os.path.join(quantized_model_path, "model.safetensors"), framework="pt") as f:+ assert "model.decoder.layers.8.self_attn.k_proj.weight_scale" in f.keys()+ assert f.get_tensor("model.decoder.layers.5.self_attn.v_proj.weight").dtype == torch.int8shutil.rmtree(quantized_model_path, ignore_errors=True)