Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

fix dynamic int8 w8a8 export issue with tuning (#1525)

Huang, Taia42fd85d894b
diff --git a/auto_round/export/export_to_llmcompressor/export.py b/auto_round/export/export_to_llmcompressor/export.py
index 6ea8eca1d..8df22b93d 100644
--- a/auto_round/export/export_to_llmcompressor/export.py
+++ b/auto_round/export/export_to_llmcompressor/export.py
@@ -68,15 +68,23 @@ def pack_layer(name, model, device=None):
if type(layer) not in SUPPORTED_LAYER_TYPES and not isinstance(layer, WrapperWALayer): ##already packed
return
+ if hasattr(layer, "orig_layer"): # revert WrapperWALayer for offline usage
+ wp_layer = layer
+ layer = wp_layer.orig_layer
+ set_module(model, name, layer)
+
if not check_to_quantized(layer):
return
if hasattr(layer, "quantization_status") and layer.quantization_status == QuantizationStatus.COMPRESSED:
return
+ # explicitly obtain the underlying device to prevent RuntimeError mismatched tensors
+ weight_device = layer.weight.device
+
scheme = construct_ct_scheme(layer)
setattr(layer, "quantization_scheme", scheme)
- setattr(layer, "weight_scale", torch.nn.Parameter(layer.scale.to(device)))
+ setattr(layer, "weight_scale", torch.nn.Parameter(layer.scale.to(weight_device)))
if not isinstance(layer.zp, torch.Tensor):
if layer.sym:
zp = torch.full_like(layer.weight_scale, 0).to(torch.int8)
@@ -85,7 +93,7 @@ def pack_layer(name, model, device=None):
else:
zp = layer.zp
- setattr(layer, "weight_zero_point", torch.nn.Parameter(zp.to(device), requires_grad=False))
+ setattr(layer, "weight_zero_point", torch.nn.Parameter(zp.to(weight_device), requires_grad=False))
delattr(layer, "scale")
compressor = IntQuantizationCompressor()
diff --git a/test/test_cpu/export/test_export.py b/test/test_cpu/export/test_export.py
index 35f9adb77..f0535ddd3 100644
--- a/test/test_cpu/export/test_export.py
+++ b/test/test_cpu/export/test_export.py
@@ -502,7 +502,18 @@ class TestAutoRound:
)
quantized_model_path = "./saved"
autoround.quantize_and_save(output_dir=quantized_model_path, format="llm_compressor")
- f = safe_open(os.path.join(quantized_model_path, "model.safetensors"), framework="pt")
- assert "model.decoder.layers.8.self_attn.k_proj.weight_scale" in f.keys()
- assert f.get_tensor("model.decoder.layers.5.self_attn.v_proj.weight").dtype == torch.int8
+ with safe_open(os.path.join(quantized_model_path, "model.safetensors"), framework="pt") as f:
+ assert "model.decoder.layers.8.self_attn.k_proj.weight_scale" in f.keys()
+ assert f.get_tensor("model.decoder.layers.5.self_attn.v_proj.weight").dtype == torch.int8
+ shutil.rmtree(quantized_model_path, ignore_errors=True)
+
+ def test_llmc_dynamic_wint8aint8_export_with_tuning(self, dataloader):
+ from safetensors import safe_open
+
+ autoround = AutoRound(self.model_name, iters=1, nsamples=2, seqlen=2, dataset=dataloader, scheme="INT8_W8A8")
+ quantized_model_path = "./saved"
+ autoround.quantize_and_save(output_dir=quantized_model_path, format="llm_compressor")
+ with safe_open(os.path.join(quantized_model_path, "model.safetensors"), framework="pt") as f:
+ assert "model.decoder.layers.8.self_attn.k_proj.weight_scale" in f.keys()
+ assert f.get_tensor("model.decoder.layers.5.self_attn.v_proj.weight").dtype == torch.int8
shutil.rmtree(quantized_model_path, ignore_errors=True)