Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

Tensor reshape bugfix (#364)

WeiweiZhang13ec66bd9e8fd
diff --git a/auto_round/data_type/utils.py b/auto_round/data_type/utils.py
index 4d774c88b..95538b7e7 100644
--- a/auto_round/data_type/utils.py
+++ b/auto_round/data_type/utils.py
@@ -67,7 +67,11 @@ def revert_tensor_by_pad(data: torch.Tensor, orig_shape: tuple, pad_len: int):
if pad_len == 0:
return data.reshape(orig_shape)
else:
- data_new = data.reshape(data.shape[0], -1)
+ if len(orig_shape) > 2:
+ tmp_shape = torch.prod(torch.tensor(orig_shape[:-1])).item()
+ else:
+ tmp_shape = orig_shape[0]
+ data_new = data.reshape(tmp_shape, -1)
data_new = data_new[:, :-pad_len]
data_new = data_new.reshape(orig_shape)
return data_new
@@ -150,3 +154,4 @@ def floor_ste(x: torch.Tensor):
torch.Tensor
"""
return (x.floor() - x).detach() + x
+
diff --git a/test/test_autoround.py b/test/test_autoround.py
index 5e831aca4..2b842505f 100644
--- a/test/test_autoround.py
+++ b/test/test_autoround.py
@@ -350,8 +350,24 @@ class TestAutoRound(unittest.TestCase):
inputs = tokenizer(text, return_tensors="pt").to(model.device)
res = tokenizer.decode(model.generate(**inputs, max_new_tokens=1)[0])
+ def test_tensor_reshape(self):
+ model_name = "facebook/opt-125m"
+ model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype=torch.float32, trust_remote_code=True, device_map='auto')
+ bits, group_size, sym = 4, 100, False
+ autoround = AutoRound(
+ self.model,
+ self.tokenizer,
+ bits=bits,
+ group_size=group_size,
+ sym=sym,
+ iters=2,
+ seqlen=2,
+ dataset=self.llm_dataloader,
+ )
+ autoround.quantize()
if __name__ == "__main__":
unittest.main()
+