Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

fix 3bits asym accuracy and calib dataset issues (#674)

Weiweibc14beafd322
diff --git a/auto_round/autoround.py b/auto_round/autoround.py
index 7169e5c56..594f82900 100644
--- a/auto_round/autoround.py
+++ b/auto_round/autoround.py
@@ -559,7 +559,7 @@ class AutoRound(object):
for index in range(len(formats)):
format = formats[index]
if format == "auto_round":
- if (self.sym or self.bits == 3) and "int" in self.data_type:
+ if self.sym and "int" in self.data_type:
format = format.replace('auto_round', 'auto_round:auto_gptq')
formats[index] = format
if self.bits == 4 and not self.sym and "int" in self.data_type:
@@ -2934,3 +2934,4 @@ class AutoRoundAdam(AutoRoundOPT):
**kwargs,
)
+
diff --git a/auto_round/calib_dataset.py b/auto_round/calib_dataset.py
index 9225dbaec..6fdd2b32b 100644
--- a/auto_round/calib_dataset.py
+++ b/auto_round/calib_dataset.py
@@ -249,11 +249,19 @@ def get_github_code_clean_dataset(tokenizer, seqlen, dataset_name="codeparrot/gi
return default_tokenizer_function
tokenizer_function = get_default_tokenizer_function()
-
- dataset_mit = load_dataset("codeparrot/github-code-clean", "all-mit", split='train',
- streaming=True, trust_remote_code=True).shuffle(seed=seed)
- dataset_apache = load_dataset("codeparrot/github-code-clean", "all-apache-2.0", split='train',
- streaming=True, trust_remote_code=True).shuffle(seed=seed)
+ try:
+ dataset_mit = load_dataset("codeparrot/github-code-clean", "all-mit", split='train',
+ trust_remote_code=True, streaming=True).shuffle(seed=seed)
+ dataset_apache = load_dataset("codeparrot/github-code-clean", "all-apache-2.0", split='train',
+ trust_remote_code=True, streaming=True).shuffle(seed=seed)
+ except RuntimeError as error:
+ if "Dataset scripts are no longer supported" in str(error):
+ raise RuntimeError(
+ f"Failed to load dataset `{dataset_name}`\n"
+ "💡 This dataset uses an old script-based format. To load it, please install `datasets<=3.6.0`:\n\n"
+ )
+ else:
+ raise error
calib_dataset = concatenate_datasets([dataset_mit, dataset_apache])
calib_dataset = calib_dataset.shuffle(seed=seed).take(10000) ##TODO concat data'shuffle may have bugs
calib_dataset = calib_dataset.map(tokenizer_function, batched=True)
@@ -798,3 +806,4 @@ def get_dataloader(
calib_dataloader = DataLoader(dataset_final, batch_size=bs, shuffle=False, collate_fn=collate_batch)
return calib_dataloader
+
diff --git a/auto_round/export/export_to_autoround/export_to_fp8_woq.py b/auto_round/export/export_to_autoround/export_to_fp8_woq.py
index 598a5a8fe..7c6862b5c 100644
--- a/auto_round/export/export_to_autoround/export_to_fp8_woq.py
+++ b/auto_round/export/export_to_autoround/export_to_fp8_woq.py
@@ -179,7 +179,7 @@ def save_quantized_as_autoround(output_dir, inplace=True, backend="auto_round",
quantization_config["extra_config"] = extra_config
names = list(layer_config.keys())
max_workers = 1
- if not torch.cuda.is_available() or not torch.xpu.is_available():
+ if not torch.cuda.is_available() and not torch.xpu.is_available():
max_workers = 2 ## 2 with cuda packing will cause hang occasionally
packing_device = "cpu"
if torch.cuda.is_available():
@@ -257,3 +257,4 @@ def save(model: torch.nn.Module, save_dir: str, max_shard_size: str = "5GB", saf
if hasattr(model, "config") and hasattr(model.config, "quantization_config"):
with open(os.path.join(save_dir, config_file), "w", encoding="utf-8") as f:
json.dump(model.config.quantization_config, f, indent=2)
+
diff --git a/auto_round/testing_utils.py b/auto_round/testing_utils.py
index b1ace9a40..7a656ee3a 100644
--- a/auto_round/testing_utils.py
+++ b/auto_round/testing_utils.py
@@ -53,7 +53,15 @@ def is_gptqmodel_available():
def greater_than_050():
try:
- require_version("auto-round>=0.5.0")
+ require_version("auto-round>=0.5.0")
+ return True
+ except ImportError:
+ return False
+
+
+def greater_than_051():
+ try:
+ require_version("auto-round>0.5.1")
return True
except ImportError:
return False
@@ -138,6 +146,16 @@ def require_greater_than_050(test_case):
return unittest.skipUnless(greater_than_050(), "test requires auto-round>=0.5.0")(test_case)
+def require_greater_than_051(test_case):
+ """
+ Decorator marking a test that requires auto-round>0.5.1
+
+ These tests are skipped when auto-round<=0.5.1.
+
+ """
+ return unittest.skipUnless(greater_than_051(), "test requires auto-round>0.5.1")(test_case)
+
+
def multi_card(test_case):
"""
Decorator marking a test that requires multi cards.
@@ -190,4 +208,4 @@ def require_vlm_env(test_case):
-
\ No newline at end of file
+
diff --git a/test/test_cuda/test_2_3bits.py b/test/test_cuda/test_2_3bits.py
index 7b1f3d31b..2b5fe543c 100644
--- a/test/test_cuda/test_2_3bits.py
+++ b/test/test_cuda/test_2_3bits.py
@@ -36,27 +36,25 @@ class TestAutoRound(unittest.TestCase):
shutil.rmtree("./saved", ignore_errors=True)
shutil.rmtree("runs", ignore_errors=True)
- @require_autogptq
+ @require_greater_than_051
def test_3bits_autoround(self):
model_name = "/models/opt-125m"
model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype=torch.float16, device_map="auto")
tokenizer = AutoTokenizer.from_pretrained(model_name)
autoround = AutoRound(model, tokenizer, bits=3)
- autoround.quantize()
-
- # autoround.save_quantized(self.save_dir, format="auto_gptq", inplace=False)
- autoround.save_quantized(self.save_dir, format="auto_round", inplace=False)
+ autoround.quantize_and_save(self.save_dir, format="auto_round", inplace=False)
model_args = f"pretrained={self.save_dir}"
res = simple_evaluate(model="hf", model_args=model_args,
# tasks="arc_easy",
tasks=self.tasks,
batch_size="auto")
- ## 0.2529
+ ## 0.3130
accuracy = res['results']['lambada_openai']['acc,none']
assert accuracy > 0.3
shutil.rmtree("./saved", ignore_errors=True)
-
+
+ @require_greater_than_051
def test_3bits_asym_autoround(self):
model_name = "/models/opt-125m"
model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype=torch.float16, device_map="auto")
@@ -127,3 +125,4 @@ class TestAutoRound(unittest.TestCase):
if __name__ == "__main__":
unittest.main()
+
diff --git a/test/test_cuda/test_calib_dataset.py b/test/test_cuda/test_calib_dataset.py
index 69479da17..47da6ab94 100644
--- a/test/test_cuda/test_calib_dataset.py
+++ b/test/test_cuda/test_calib_dataset.py
@@ -35,7 +35,7 @@ class TestLocalCalibDataset(unittest.TestCase):
def test_combine_dataset(self):
dataset = (
- "NeelNanda/pile-10k" + ",codeparrot/github-code-clean" + ",BAAI/CCI3-HQ" + ",madao33/new-title-chinese")
+ "NeelNanda/pile-10k" + ",BAAI/CCI3-HQ" + ",madao33/new-title-chinese")
bits, group_size, sym = 4, 128, True
autoround = AutoRound(
self.model, self.tokenizer, bits=bits, group_size=group_size, sym=sym, iters=2, seqlen=128, dataset=dataset
@@ -45,3 +45,4 @@ class TestLocalCalibDataset(unittest.TestCase):
if __name__ == "__main__":
unittest.main()
+