Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

fix falcon quant issue with disable_trust_remote_code (#126)

WeiweiZhang1e2985fdfa170
diff --git a/auto_round/autoround.py b/auto_round/autoround.py
index 52ce1a7ed..aa54e99d7 100644
--- a/auto_round/autoround.py
+++ b/auto_round/autoround.py
@@ -664,8 +664,8 @@ class AutoRound(object):
layer_input = to_device(layer_input, self.cache_device)
q_layer_input = q_layer_inputs[layer_name] if self.enable_quanted_input else None
q_layer_input = to_device(q_layer_input, self.cache_device)
- self.quant_layer(layer_name, layer_inputs[layer_name], q_layer_input, device=self.device)
- for i in range(len(layer_inputs)):
+ self.quant_layer(layer_name, layer_input, q_layer_input, device=self.device)
+ for i in range(len(layer_input)):
layer_input[i] = None
if q_layer_input is not None:
q_layer_input[i] = None
@@ -934,14 +934,14 @@ class AutoRound(object):
self.inputs[name]["positional_inputs"] = to_device(positional_args)
for key in kwargs.keys():
- if isinstance(kwargs[key], torch.Tensor) or isinstance(kwargs[key], list) or (key == "alibi"):
+ if isinstance(kwargs[key], torch.Tensor) or isinstance(kwargs[key], list) \
+ or (key == "alibi") or (key == "attention_mask"):
if "attention_mask" in key:
if key not in self.inputs[name].keys():
self.inputs[name][key] = None
if kwargs[key] is not None:
if (not self.share_attention_mask_flag) and self.inputs[name][key] is not None:
self.inputs[name][key].extend(list(torch.split(kwargs[key].to("cpu"), 1, dim=0)))
-
else:
self.inputs[name][key] = list(torch.split(kwargs[key].to("cpu"), 1, dim=0))
elif "alibi" in key:
@@ -954,9 +954,7 @@ class AutoRound(object):
if (not self.share_attention_mask_flag) and self.inputs[name][key] is not None:
self.inputs[name][key].extend(list(torch.split(alibi.to("cpu"), 1, dim=0)))
else:
-
self.inputs[name][key] = list(torch.split(alibi.to("cpu"), 1, dim=0))
-
elif key not in self.inputs[name].keys():
self.inputs[name][key] = to_device(kwargs[key], device=torch.device("cpu"))
if name == self.last_cache_name:
diff --git a/auto_round/utils.py b/auto_round/utils.py
index c830b8d8b..d8f4bf59b 100644
--- a/auto_round/utils.py
+++ b/auto_round/utils.py
@@ -475,25 +475,24 @@ def block_forward(block, input_ids, input_others, amp=False, amp_dtype=torch.flo
# input_ids, input_others = move_to_device(input_ids, input_others, device)
input_ids = to_device(input_ids, device)
input_others = to_device(input_others, device)
+ input_tuple = input_others.pop("positional_inputs", None)
if "alibi" in input_others.keys():
- attention_mask = input_others["attention_mask"]
- alibi = input_others["alibi"]
+ alibi = input_others.pop("alibi")
if alibi is not None:
alibi = alibi.reshape(-1, alibi.shape[2], alibi.shape[3])
if amp:
with autocast(device_type=device.split(":")[0], dtype=amp_dtype): # pragma: no cover
output = block(
- input_ids, attention_mask=attention_mask, alibi=alibi
+ input_ids, alibi=alibi, *input_tuple, **input_others
) ##TODO is this correct for all models with alibi?
else:
- output = block(input_ids, attention_mask=attention_mask, alibi=alibi)
+ output = block(input_ids, alibi=alibi, *input_tuple, **input_others)
else:
- input_tuple = input_others.pop("positional_inputs", None)
if amp:
with autocast(device_type=device.split(":")[0], dtype=amp_dtype): # pragma: no cover
- output = block.forward(input_ids, *input_tuple, **input_others)
+ output = block(input_ids, *input_tuple, **input_others)
else:
- output = block.forward(input_ids, *input_tuple, **input_others)
+ output = block(input_ids, *input_tuple, **input_others)
if isinstance(output, list) or isinstance(output, tuple):
output = output[0]
return output