PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
fix bug of lm_head and dispatch model,gguf eval (#1025)
Heng Guo2fb3ee1bc1e8
diff --git a/auto_round/compressors/base.py b/auto_round/compressors/base.pyindex c098320f9..2634769a1 100644--- a/auto_round/compressors/base.py+++ b/auto_round/compressors/base.py@@ -1994,6 +1994,8 @@ class BaseCompressor(object):devices = parse_available_devices(self.device_map)max_memory = get_max_memory()new_max_memory = {}+ if "cpu" not in devices:+ devices.append("cpu")for device in devices:if ":" in device:device = int(device.split(":")[-1])@@ -2005,8 +2007,25 @@ class BaseCompressor(object):device_map = infer_auto_device_map(self.model, max_memory=new_max_memory, no_split_module_classes=no_split_modules)-- self.model = dispatch_model(self.model, device_map=device_map)+ if len(devices) > 1 and "cpu" in device_map.values():+ logger.warning(+ "Not enough vram cause the ram to be used, which may severely impact speed."+ " Please consider using more cards."+ )++ try:+ self.model = dispatch_model(self.model, device_map=device_map)+ except ValueError as e:+ if "offload_dir" in e.__str__():+ logger.warning(+ f"Due to insufficient resources, disk is used to store the model."+ f" `offload_dir={envs.AR_WORK_SPACE}`"+ )+ self.model = dispatch_model(+ self.model, device_map=device_map, offload_dir=envs.AR_WORK_SPACE+ )+ else:+ raiseelse:self.model = self.model.to(self.device)diff --git a/auto_round/compressors/utils.py b/auto_round/compressors/utils.pyindex cf8d90098..2514c9f34 100644--- a/auto_round/compressors/utils.py+++ b/auto_round/compressors/utils.py@@ -380,6 +380,9 @@ def set_layer_config(if hasattr(model, "config") and hasattr(model.config, "tie_word_embeddings"):tie_word_embeddings = model.config.tie_word_embeddings+ if lm_head_name in layer_config:+ quant_lm_head = True+if quant_lm_head and tie_word_embeddings and not gguf_name:quant_lm_head = Falselogger.warning(diff --git a/auto_round/envs.py b/auto_round/envs.pyindex 1a1f51de0..5ff90d170 100644--- a/auto_round/envs.py+++ b/auto_round/envs.py@@ -25,6 +25,7 @@ environment_variables: dict[str, Callable[[], Any]] = {"AR_LOG_LEVEL": lambda: os.getenv("AR_LOG_LEVEL", "INFO").upper(),"AR_ENABLE_COMPILE_PACKING": lambda: os.getenv("AR_ENABLE_COMPILE_PACKING", "0").lower() in ("1", "true", "yes"),"AR_USE_MODELSCOPE": lambda: os.getenv("AR_USE_MODELSCOPE", "False").lower() in ["1", "true"],+ "AR_WORK_SPACE": lambda: os.getenv("AR_WORK_SPACE", "ar_work_space").lower(),}diff --git a/auto_round/eval/eval_cli.py b/auto_round/eval/eval_cli.pyindex 7235e9e4c..0c96b4bd8 100644--- a/auto_round/eval/eval_cli.py+++ b/auto_round/eval/eval_cli.py@@ -166,6 +166,7 @@ def eval(args):if file.endswith(".gguf"):is_gguf_file = Truegguf_file = file+ model = args.modeleval_model_dtype = get_model_dtype(args.eval_model_dtype)if is_gguf_file:import torchdiff --git a/test/test_cpu/test_autoround.py b/test/test_cpu/test_autoround.pyindex dd188e6ad..87b9e5a96 100644--- a/test/test_cpu/test_autoround.py+++ b/test/test_cpu/test_autoround.py@@ -731,6 +731,13 @@ class TestAutoRound(unittest.TestCase):assert "lm_head" in model.config.quantization_config.extra_configassert model.config.quantization_config.extra_config["lm_head"]["bits"] == 4+ layer_config = {"lm_head": {"bits": 4}}+ ar = AutoRound(model_name, quant_lm_head=False, iters=0, disable_opt_rtn=True, layer_config=layer_config)+ ar.quantize_and_save(output_dir=self.save_folder, format="auto_round")+ model = AutoModelForCausalLM.from_pretrained(self.save_folder, device_map="cpu")+ assert "lm_head" in model.config.quantization_config.extra_config+ assert model.config.quantization_config.extra_config["lm_head"]["bits"] == 4+def test_quant_lm_head_layer_config(self):model_name = "/tf_dataset/auto_round/models/Qwen/Qwen3-8B"layer_config = {"lm_head": {"bits": 4}}diff --git a/test/test_cuda/test_gguf.py b/test/test_cuda/test_gguf.pyindex 299c45668..312e561cf 100644--- a/test/test_cuda/test_gguf.py+++ b/test/test_cuda/test_gguf.py@@ -220,7 +220,7 @@ class TestAutoRound(unittest.TestCase):autoround.quantize_and_save(output_dir=quantized_model_path, format="gguf:q4_k_m")self.assertTrue("mmproj-model.gguf" in os.listdir("./saved"))file_size = os.path.getsize("./saved/gemma-3-12B-it-Q4_K_M.gguf") / 1024**2- self.assertAlmostEqual(file_size, 6962, delta=5.0)+ self.assertAlmostEqual(file_size, 6568, delta=5.0)file_size = os.path.getsize("./saved/mmproj-model.gguf") / 1024**2self.assertAlmostEqual(file_size, 1599, delta=5.0)shutil.rmtree(quantized_model_path, ignore_errors=True)