PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
fix gguf acc and oom bug when iters > 0 (#1098)
Heng Guo09d9962aca0d
diff --git a/auto_round/compressors/base.py b/auto_round/compressors/base.pyindex fb2681183..a3309bc41 100644--- a/auto_round/compressors/base.py+++ b/auto_round/compressors/base.py@@ -2478,16 +2478,18 @@ class BaseCompressor(object):if q_inputs is not None:q_inputs[i] = q_inputs[i].to(layer.weight.dtype)- if q_inputs is None:+ if self.act_bits <= 8 and check_need_act_calibration(+ self.act_dynamic,+ self.act_data_type,+ self.act_bits,+ self.static_kv_dtype,+ self.static_attention_dtype,+ ):+ tmp_inputs = q_inputs if q_inputs is not None else inputshook_handles = self._register_act_max_hook(layer)with torch.no_grad():- layer(torch.cat(inputs, dim=0))- for handle in hook_handles:- handle.remove()- else:- hook_handles = self._register_act_max_hook(layer)- if hook_handles:- layer(torch.cat(q_inputs, dim=0))+ for input in tmp_inputs:+ layer(input)for handle in hook_handles:handle.remove()diff --git a/auto_round/export/export_to_gguf/export.py b/auto_round/export/export_to_gguf/export.pyindex 0a5bfc461..a01f660f1 100644--- a/auto_round/export/export_to_gguf/export.py+++ b/auto_round/export/export_to_gguf/export.py@@ -186,8 +186,6 @@ def pack_gguf_layer(for n, m in block.named_modules():if check_to_quantized(m):names_in_blocks.append(m.tmp_name)- names_outside_blocks = list(set(layer_config.keys()) - set(names_in_blocks))- model.names_outside_blocks = names_outside_blocksif name in model.last_layer_name_to_block_name:# Packing block@@ -205,21 +203,6 @@ def pack_gguf_layer(if len(model.last_layer_name_to_block_name) == 0:for gguf_model in gguf_model_instance_global:gguf_model.current_packing_block = None- if name in model.names_outside_blocks:- # Packing block- for gguf_model in gguf_model_instance_global:- gguf_model.current_packing_block = name- gguf_model.prepare_tensors()-- layer = get_module(model, name)- if hasattr(layer, "weight"):- layer.weight = None- if hasattr(layer, "bias"):- layer.bias = None- model.names_outside_blocks.remove(name)- if len(model.names_outside_blocks) == 0:- for gguf_model in gguf_model_instance_global:- gguf_model.current_packing_block = None@torch.inference_mode()