PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
fix 3bits asym accuracy and calib dataset issues (#674)
Weiweibc14beafd322
diff --git a/auto_round/autoround.py b/auto_round/autoround.pyindex 7169e5c56..594f82900 100644--- a/auto_round/autoround.py+++ b/auto_round/autoround.py@@ -559,7 +559,7 @@ class AutoRound(object):for index in range(len(formats)):format = formats[index]if format == "auto_round":- if (self.sym or self.bits == 3) and "int" in self.data_type:+ if self.sym and "int" in self.data_type:format = format.replace('auto_round', 'auto_round:auto_gptq')formats[index] = formatif self.bits == 4 and not self.sym and "int" in self.data_type:@@ -2934,3 +2934,4 @@ class AutoRoundAdam(AutoRoundOPT):**kwargs,)+diff --git a/auto_round/calib_dataset.py b/auto_round/calib_dataset.pyindex 9225dbaec..6fdd2b32b 100644--- a/auto_round/calib_dataset.py+++ b/auto_round/calib_dataset.py@@ -249,11 +249,19 @@ def get_github_code_clean_dataset(tokenizer, seqlen, dataset_name="codeparrot/gireturn default_tokenizer_functiontokenizer_function = get_default_tokenizer_function()-- dataset_mit = load_dataset("codeparrot/github-code-clean", "all-mit", split='train',- streaming=True, trust_remote_code=True).shuffle(seed=seed)- dataset_apache = load_dataset("codeparrot/github-code-clean", "all-apache-2.0", split='train',- streaming=True, trust_remote_code=True).shuffle(seed=seed)+ try:+ dataset_mit = load_dataset("codeparrot/github-code-clean", "all-mit", split='train',+ trust_remote_code=True, streaming=True).shuffle(seed=seed)+ dataset_apache = load_dataset("codeparrot/github-code-clean", "all-apache-2.0", split='train',+ trust_remote_code=True, streaming=True).shuffle(seed=seed)+ except RuntimeError as error:+ if "Dataset scripts are no longer supported" in str(error):+ raise RuntimeError(+ f"Failed to load dataset `{dataset_name}`\n"+ "💡 This dataset uses an old script-based format. To load it, please install `datasets<=3.6.0`:\n\n"+ )+ else:+ raise errorcalib_dataset = concatenate_datasets([dataset_mit, dataset_apache])calib_dataset = calib_dataset.shuffle(seed=seed).take(10000) ##TODO concat data'shuffle may have bugscalib_dataset = calib_dataset.map(tokenizer_function, batched=True)@@ -798,3 +806,4 @@ def get_dataloader(calib_dataloader = DataLoader(dataset_final, batch_size=bs, shuffle=False, collate_fn=collate_batch)return calib_dataloader+diff --git a/auto_round/export/export_to_autoround/export_to_fp8_woq.py b/auto_round/export/export_to_autoround/export_to_fp8_woq.pyindex 598a5a8fe..7c6862b5c 100644--- a/auto_round/export/export_to_autoround/export_to_fp8_woq.py+++ b/auto_round/export/export_to_autoround/export_to_fp8_woq.py@@ -179,7 +179,7 @@ def save_quantized_as_autoround(output_dir, inplace=True, backend="auto_round",quantization_config["extra_config"] = extra_confignames = list(layer_config.keys())max_workers = 1- if not torch.cuda.is_available() or not torch.xpu.is_available():+ if not torch.cuda.is_available() and not torch.xpu.is_available():max_workers = 2 ## 2 with cuda packing will cause hang occasionallypacking_device = "cpu"if torch.cuda.is_available():@@ -257,3 +257,4 @@ def save(model: torch.nn.Module, save_dir: str, max_shard_size: str = "5GB", safif hasattr(model, "config") and hasattr(model.config, "quantization_config"):with open(os.path.join(save_dir, config_file), "w", encoding="utf-8") as f:json.dump(model.config.quantization_config, f, indent=2)+diff --git a/auto_round/testing_utils.py b/auto_round/testing_utils.pyindex b1ace9a40..7a656ee3a 100644--- a/auto_round/testing_utils.py+++ b/auto_round/testing_utils.py@@ -53,7 +53,15 @@ def is_gptqmodel_available():def greater_than_050():try:- require_version("auto-round>=0.5.0")+ require_version("auto-round>=0.5.0")+ return True+ except ImportError:+ return False+++def greater_than_051():+ try:+ require_version("auto-round>0.5.1")return Trueexcept ImportError:return False@@ -138,6 +146,16 @@ def require_greater_than_050(test_case):return unittest.skipUnless(greater_than_050(), "test requires auto-round>=0.5.0")(test_case)+def require_greater_than_051(test_case):+ """+ Decorator marking a test that requires auto-round>0.5.1++ These tests are skipped when auto-round<=0.5.1.++ """+ return unittest.skipUnless(greater_than_051(), "test requires auto-round>0.5.1")(test_case)++def multi_card(test_case):"""Decorator marking a test that requires multi cards.@@ -190,4 +208,4 @@ def require_vlm_env(test_case):-\ No newline at end of file+diff --git a/test/test_cuda/test_2_3bits.py b/test/test_cuda/test_2_3bits.pyindex 7b1f3d31b..2b5fe543c 100644--- a/test/test_cuda/test_2_3bits.py+++ b/test/test_cuda/test_2_3bits.py@@ -36,27 +36,25 @@ class TestAutoRound(unittest.TestCase):shutil.rmtree("./saved", ignore_errors=True)shutil.rmtree("runs", ignore_errors=True)- @require_autogptq+ @require_greater_than_051def test_3bits_autoround(self):model_name = "/models/opt-125m"model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype=torch.float16, device_map="auto")tokenizer = AutoTokenizer.from_pretrained(model_name)autoround = AutoRound(model, tokenizer, bits=3)- autoround.quantize()-- # autoround.save_quantized(self.save_dir, format="auto_gptq", inplace=False)- autoround.save_quantized(self.save_dir, format="auto_round", inplace=False)+ autoround.quantize_and_save(self.save_dir, format="auto_round", inplace=False)model_args = f"pretrained={self.save_dir}"res = simple_evaluate(model="hf", model_args=model_args,# tasks="arc_easy",tasks=self.tasks,batch_size="auto")- ## 0.2529+ ## 0.3130accuracy = res['results']['lambada_openai']['acc,none']assert accuracy > 0.3shutil.rmtree("./saved", ignore_errors=True)-++ @require_greater_than_051def test_3bits_asym_autoround(self):model_name = "/models/opt-125m"model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype=torch.float16, device_map="auto")@@ -127,3 +125,4 @@ class TestAutoRound(unittest.TestCase):if __name__ == "__main__":unittest.main()+diff --git a/test/test_cuda/test_calib_dataset.py b/test/test_cuda/test_calib_dataset.pyindex 69479da17..47da6ab94 100644--- a/test/test_cuda/test_calib_dataset.py+++ b/test/test_cuda/test_calib_dataset.py@@ -35,7 +35,7 @@ class TestLocalCalibDataset(unittest.TestCase):def test_combine_dataset(self):dataset = (- "NeelNanda/pile-10k" + ",codeparrot/github-code-clean" + ",BAAI/CCI3-HQ" + ",madao33/new-title-chinese")+ "NeelNanda/pile-10k" + ",BAAI/CCI3-HQ" + ",madao33/new-title-chinese")bits, group_size, sym = 4, 128, Trueautoround = AutoRound(self.model, self.tokenizer, bits=bits, group_size=group_size, sym=sym, iters=2, seqlen=128, dataset=dataset@@ -45,3 +45,4 @@ class TestLocalCalibDataset(unittest.TestCase):if __name__ == "__main__":unittest.main()+