PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
fix gguf mistral_common import (#736)
Heng Guo92d31fa24d4b
diff --git a/auto_round/export/export_to_gguf/convert_hf_to_gguf.py b/auto_round/export/export_to_gguf/convert_hf_to_gguf.pyindex 0b6ffeccc..2a468bd42 100644--- a/auto_round/export/export_to_gguf/convert_hf_to_gguf.py+++ b/auto_round/export/export_to_gguf/convert_hf_to_gguf.py@@ -46,13 +46,8 @@ from auto_round.utils import LazyImportgguf = LazyImport("gguf")MistralTokenizerType = LazyImport("gguf.vocab.MistralTokenizerType")MistralVocab = LazyImport("gguf.vocab.MistralVocab")-from mistral_common.tokens.tokenizers.base import TokenizerVersion # pylint: disable=E0401-from mistral_common.tokens.tokenizers.multimodal import DATASET_MEAN, DATASET_STD # pylint: disable=E0401-from mistral_common.tokens.tokenizers.sentencepiece import ( # pylint: disable=E0401- SentencePieceTokenizer,-)-from mistral_common.tokens.tokenizers.tekken import Tekkenizer # pylint: disable=E0401-+DATASET_MEAN = LazyImport("mistral_common.tokens.tokenizers.multimodal.DATASET_MEAN")+DATASET_STD = LazyImport("mistral_common.tokens.tokenizers.multimodal.DATASET_STD")logger = logging.getLogger("hf-to-gguf")###### MODEL DEFINITIONS ######@@ -1365,6 +1360,7 @@ class MmprojModel(ModelBase):self.gguf_writer.add_type(gguf.GGUFType.MMPROJ)def set_gguf_parameters(self):+self.gguf_writer.add_file_type(self.ftype)if self.has_vision_encoder:@@ -8628,7 +8624,11 @@ class MistralModel(LlamaModel):undo_permute = False@staticmethod- def get_community_chat_template(vocab: MistralVocab, templates_dir: Path):+ def get_community_chat_template(vocab, templates_dir: Path):+ from mistral_common.tokens.tokenizers.base import TokenizerVersion # pylint: disable=E0401+ from mistral_common.tokens.tokenizers.sentencepiece import SentencePieceTokenizer # pylint: disable=E0401+ from mistral_common.tokens.tokenizers.tekken import Tekkenizer # pylint: disable=E0401+assert TokenizerVersion is not None, "mistral_common is not installed"assert isinstance(vocab.tokenizer, (Tekkenizer, SentencePieceTokenizer)diff --git a/test/test_cpu/requirements.txt b/test/test_cpu/requirements.txtindex 0edadd323..bc6615906 100644--- a/test/test_cpu/requirements.txt+++ b/test/test_cpu/requirements.txt@@ -1,5 +1,4 @@addictmodelscopegguf-mistral_commontorchvisiondiff --git a/test/test_cpu/test_gguf_format.py b/test/test_cpu/test_gguf_format.pyindex a937c9f50..028a6fedd 100644--- a/test/test_cpu/test_gguf_format.py+++ b/test/test_cpu/test_gguf_format.py@@ -59,7 +59,7 @@ class TestGGUF(unittest.TestCase):quantized_model_path = "./saved"autoround.quantize_and_save(output_dir=quantized_model_path, inplace=False, format="gguf:q4_0")- gguf_file = "Qwen2.5-0.5B-Instruct-494M-Q4_0.gguf"+ gguf_file = os.listdir(quantized_model_path)[0]model = AutoModelForCausalLM.from_pretrained(quantized_model_path, gguf_file=gguf_file, device_map="auto")text = "There is a girl who likes adventure,"inputs = self.tokenizer(text, return_tensors="pt").to(model.device)@@ -79,7 +79,7 @@ class TestGGUF(unittest.TestCase):quantized_model_path = "./saved"autoround.quantize_and_save(output_dir=quantized_model_path, inplace=False, format="gguf:q4_1")- gguf_file = "Qwen2.5-0.5B-Instruct-494M-Q4_1.gguf"+ gguf_file = os.listdir(quantized_model_path)[0]model = AutoModelForCausalLM.from_pretrained(quantized_model_path, gguf_file=gguf_file, device_map="auto")text = "There is a girl who likes adventure,"inputs = self.tokenizer(text, return_tensors="pt").to(model.device)@@ -317,7 +317,8 @@ class TestGGUF(unittest.TestCase):quantized_model_path = "./saved"autoround.quantize_and_save(output_dir=quantized_model_path, format="gguf:q4_0")self.assertTrue("mmproj-model.gguf" in os.listdir("./saved"))- file_size = os.path.getsize("./saved/Qwen2-VL-2B-Instruct-1.5B-Q4_0.gguf") / 1024**2+ file_name = os.listdir(quantized_model_path)[0]+ file_size = os.path.getsize(os.path.join(quantized_model_path, file_name)) / 1024**2self.assertAlmostEqual(file_size, 892, delta=1.0)shutil.rmtree("./saved", ignore_errors=True)