Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

remove force fp16 dtype export (#192)

WeiweiZhang1d9f7ceeee54d
diff --git a/auto_round/export/export_to_autogptq/export.py b/auto_round/export/export_to_autogptq/export.py
index ffe106b86..3e18bf9b7 100644
--- a/auto_round/export/export_to_autogptq/export.py
+++ b/auto_round/export/export_to_autogptq/export.py
@@ -137,7 +137,6 @@ def save_quantized_as_autogptq(output_dir, inplace=True, backend="auto_gptq:exll
if all_to_quantized:
modules_in_block_to_quantize = None
- model = model.to(torch.float16) ##force to fp16
if not inplace:
model = copy.deepcopy(model.to("cpu"))
@@ -238,3 +237,4 @@ def save(model: torch.nn.Module, save_dir: str, max_shard_size: str = "5GB", saf
json.dump(model.config.quantization_config, f, indent=2)
+
diff --git a/auto_round/export/export_to_autoround/export.py b/auto_round/export/export_to_autoround/export.py
index 309aba660..bc1cee2d2 100644
--- a/auto_round/export/export_to_autoround/export.py
+++ b/auto_round/export/export_to_autoround/export.py
@@ -167,7 +167,6 @@ def save_quantized_as_autoround(output_dir, inplace=True, backend="auto_round:ex
model = kwargs["model"]
quant_block_list = kwargs["quant_block_list"]
safe_serialization = True if 'safe_serialization' not in kwargs.keys() else kwargs["safe_serialization"]
- model = model.to(torch.float16) ##force to fp16
if not inplace:
model = copy.deepcopy(model.to("cpu"))
layer_names_in_block = get_layer_names_in_block(model, quant_block_list=quant_block_list)
@@ -282,3 +281,4 @@ def save(model: nn.Module, save_dir: str, max_shard_size: str = "5GB", safe_seri
json.dump(model.config.quantization_config, f, indent=2)
+