Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

fix gguf acc and oom bug when iters > 0 (#1098)

Heng Guo09d9962aca0d
diff --git a/auto_round/compressors/base.py b/auto_round/compressors/base.py
index fb2681183..a3309bc41 100644
--- a/auto_round/compressors/base.py
+++ b/auto_round/compressors/base.py
@@ -2478,16 +2478,18 @@ class BaseCompressor(object):
if q_inputs is not None:
q_inputs[i] = q_inputs[i].to(layer.weight.dtype)
- if q_inputs is None:
+ if self.act_bits <= 8 and check_need_act_calibration(
+ self.act_dynamic,
+ self.act_data_type,
+ self.act_bits,
+ self.static_kv_dtype,
+ self.static_attention_dtype,
+ ):
+ tmp_inputs = q_inputs if q_inputs is not None else inputs
hook_handles = self._register_act_max_hook(layer)
with torch.no_grad():
- layer(torch.cat(inputs, dim=0))
- for handle in hook_handles:
- handle.remove()
- else:
- hook_handles = self._register_act_max_hook(layer)
- if hook_handles:
- layer(torch.cat(q_inputs, dim=0))
+ for input in tmp_inputs:
+ layer(input)
for handle in hook_handles:
handle.remove()
diff --git a/auto_round/export/export_to_gguf/export.py b/auto_round/export/export_to_gguf/export.py
index 0a5bfc461..a01f660f1 100644
--- a/auto_round/export/export_to_gguf/export.py
+++ b/auto_round/export/export_to_gguf/export.py
@@ -186,8 +186,6 @@ def pack_gguf_layer(
for n, m in block.named_modules():
if check_to_quantized(m):
names_in_blocks.append(m.tmp_name)
- names_outside_blocks = list(set(layer_config.keys()) - set(names_in_blocks))
- model.names_outside_blocks = names_outside_blocks
if name in model.last_layer_name_to_block_name:
# Packing block
@@ -205,21 +203,6 @@ def pack_gguf_layer(
if len(model.last_layer_name_to_block_name) == 0:
for gguf_model in gguf_model_instance_global:
gguf_model.current_packing_block = None
- if name in model.names_outside_blocks:
- # Packing block
- for gguf_model in gguf_model_instance_global:
- gguf_model.current_packing_block = name
- gguf_model.prepare_tensors()
-
- layer = get_module(model, name)
- if hasattr(layer, "weight"):
- layer.weight = None
- if hasattr(layer, "bias"):
- layer.bias = None
- model.names_outside_blocks.remove(name)
- if len(model.names_outside_blocks) == 0:
- for gguf_model in gguf_model_instance_global:
- gguf_model.current_packing_block = None
@torch.inference_mode()