PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
fix falcon quant issue with disable_trust_remote_code (#126)
WeiweiZhang1e2985fdfa170
diff --git a/auto_round/autoround.py b/auto_round/autoround.pyindex 52ce1a7ed..aa54e99d7 100644--- a/auto_round/autoround.py+++ b/auto_round/autoround.py@@ -664,8 +664,8 @@ class AutoRound(object):layer_input = to_device(layer_input, self.cache_device)q_layer_input = q_layer_inputs[layer_name] if self.enable_quanted_input else Noneq_layer_input = to_device(q_layer_input, self.cache_device)- self.quant_layer(layer_name, layer_inputs[layer_name], q_layer_input, device=self.device)- for i in range(len(layer_inputs)):+ self.quant_layer(layer_name, layer_input, q_layer_input, device=self.device)+ for i in range(len(layer_input)):layer_input[i] = Noneif q_layer_input is not None:q_layer_input[i] = None@@ -934,14 +934,14 @@ class AutoRound(object):self.inputs[name]["positional_inputs"] = to_device(positional_args)for key in kwargs.keys():- if isinstance(kwargs[key], torch.Tensor) or isinstance(kwargs[key], list) or (key == "alibi"):+ if isinstance(kwargs[key], torch.Tensor) or isinstance(kwargs[key], list) \+ or (key == "alibi") or (key == "attention_mask"):if "attention_mask" in key:if key not in self.inputs[name].keys():self.inputs[name][key] = Noneif kwargs[key] is not None:if (not self.share_attention_mask_flag) and self.inputs[name][key] is not None:self.inputs[name][key].extend(list(torch.split(kwargs[key].to("cpu"), 1, dim=0)))-else:self.inputs[name][key] = list(torch.split(kwargs[key].to("cpu"), 1, dim=0))elif "alibi" in key:@@ -954,9 +954,7 @@ class AutoRound(object):if (not self.share_attention_mask_flag) and self.inputs[name][key] is not None:self.inputs[name][key].extend(list(torch.split(alibi.to("cpu"), 1, dim=0)))else:-self.inputs[name][key] = list(torch.split(alibi.to("cpu"), 1, dim=0))-elif key not in self.inputs[name].keys():self.inputs[name][key] = to_device(kwargs[key], device=torch.device("cpu"))if name == self.last_cache_name:diff --git a/auto_round/utils.py b/auto_round/utils.pyindex c830b8d8b..d8f4bf59b 100644--- a/auto_round/utils.py+++ b/auto_round/utils.py@@ -475,25 +475,24 @@ def block_forward(block, input_ids, input_others, amp=False, amp_dtype=torch.flo# input_ids, input_others = move_to_device(input_ids, input_others, device)input_ids = to_device(input_ids, device)input_others = to_device(input_others, device)+ input_tuple = input_others.pop("positional_inputs", None)if "alibi" in input_others.keys():- attention_mask = input_others["attention_mask"]- alibi = input_others["alibi"]+ alibi = input_others.pop("alibi")if alibi is not None:alibi = alibi.reshape(-1, alibi.shape[2], alibi.shape[3])if amp:with autocast(device_type=device.split(":")[0], dtype=amp_dtype): # pragma: no coveroutput = block(- input_ids, attention_mask=attention_mask, alibi=alibi+ input_ids, alibi=alibi, *input_tuple, **input_others) ##TODO is this correct for all models with alibi?else:- output = block(input_ids, attention_mask=attention_mask, alibi=alibi)+ output = block(input_ids, alibi=alibi, *input_tuple, **input_others)else:- input_tuple = input_others.pop("positional_inputs", None)if amp:with autocast(device_type=device.split(":")[0], dtype=amp_dtype): # pragma: no cover- output = block.forward(input_ids, *input_tuple, **input_others)+ output = block(input_ids, *input_tuple, **input_others)else:- output = block.forward(input_ids, *input_tuple, **input_others)+ output = block(input_ids, *input_tuple, **input_others)if isinstance(output, list) or isinstance(output, tuple):output = output[0]return output