Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

fix gguf mistral_common import (#736)

Heng Guo92d31fa24d4b
diff --git a/auto_round/export/export_to_gguf/convert_hf_to_gguf.py b/auto_round/export/export_to_gguf/convert_hf_to_gguf.py
index 0b6ffeccc..2a468bd42 100644
--- a/auto_round/export/export_to_gguf/convert_hf_to_gguf.py
+++ b/auto_round/export/export_to_gguf/convert_hf_to_gguf.py
@@ -46,13 +46,8 @@ from auto_round.utils import LazyImport
gguf = LazyImport("gguf")
MistralTokenizerType = LazyImport("gguf.vocab.MistralTokenizerType")
MistralVocab = LazyImport("gguf.vocab.MistralVocab")
-from mistral_common.tokens.tokenizers.base import TokenizerVersion # pylint: disable=E0401
-from mistral_common.tokens.tokenizers.multimodal import DATASET_MEAN, DATASET_STD # pylint: disable=E0401
-from mistral_common.tokens.tokenizers.sentencepiece import ( # pylint: disable=E0401
- SentencePieceTokenizer,
-)
-from mistral_common.tokens.tokenizers.tekken import Tekkenizer # pylint: disable=E0401
-
+DATASET_MEAN = LazyImport("mistral_common.tokens.tokenizers.multimodal.DATASET_MEAN")
+DATASET_STD = LazyImport("mistral_common.tokens.tokenizers.multimodal.DATASET_STD")
logger = logging.getLogger("hf-to-gguf")
###### MODEL DEFINITIONS ######
@@ -1365,6 +1360,7 @@ class MmprojModel(ModelBase):
self.gguf_writer.add_type(gguf.GGUFType.MMPROJ)
def set_gguf_parameters(self):
+
self.gguf_writer.add_file_type(self.ftype)
if self.has_vision_encoder:
@@ -8628,7 +8624,11 @@ class MistralModel(LlamaModel):
undo_permute = False
@staticmethod
- def get_community_chat_template(vocab: MistralVocab, templates_dir: Path):
+ def get_community_chat_template(vocab, templates_dir: Path):
+ from mistral_common.tokens.tokenizers.base import TokenizerVersion # pylint: disable=E0401
+ from mistral_common.tokens.tokenizers.sentencepiece import SentencePieceTokenizer # pylint: disable=E0401
+ from mistral_common.tokens.tokenizers.tekken import Tekkenizer # pylint: disable=E0401
+
assert TokenizerVersion is not None, "mistral_common is not installed"
assert isinstance(
vocab.tokenizer, (Tekkenizer, SentencePieceTokenizer)
diff --git a/test/test_cpu/requirements.txt b/test/test_cpu/requirements.txt
index 0edadd323..bc6615906 100644
--- a/test/test_cpu/requirements.txt
+++ b/test/test_cpu/requirements.txt
@@ -1,5 +1,4 @@
addict
modelscope
gguf
-mistral_common
torchvision
diff --git a/test/test_cpu/test_gguf_format.py b/test/test_cpu/test_gguf_format.py
index a937c9f50..028a6fedd 100644
--- a/test/test_cpu/test_gguf_format.py
+++ b/test/test_cpu/test_gguf_format.py
@@ -59,7 +59,7 @@ class TestGGUF(unittest.TestCase):
quantized_model_path = "./saved"
autoround.quantize_and_save(output_dir=quantized_model_path, inplace=False, format="gguf:q4_0")
- gguf_file = "Qwen2.5-0.5B-Instruct-494M-Q4_0.gguf"
+ gguf_file = os.listdir(quantized_model_path)[0]
model = AutoModelForCausalLM.from_pretrained(quantized_model_path, gguf_file=gguf_file, device_map="auto")
text = "There is a girl who likes adventure,"
inputs = self.tokenizer(text, return_tensors="pt").to(model.device)
@@ -79,7 +79,7 @@ class TestGGUF(unittest.TestCase):
quantized_model_path = "./saved"
autoround.quantize_and_save(output_dir=quantized_model_path, inplace=False, format="gguf:q4_1")
- gguf_file = "Qwen2.5-0.5B-Instruct-494M-Q4_1.gguf"
+ gguf_file = os.listdir(quantized_model_path)[0]
model = AutoModelForCausalLM.from_pretrained(quantized_model_path, gguf_file=gguf_file, device_map="auto")
text = "There is a girl who likes adventure,"
inputs = self.tokenizer(text, return_tensors="pt").to(model.device)
@@ -317,7 +317,8 @@ class TestGGUF(unittest.TestCase):
quantized_model_path = "./saved"
autoround.quantize_and_save(output_dir=quantized_model_path, format="gguf:q4_0")
self.assertTrue("mmproj-model.gguf" in os.listdir("./saved"))
- file_size = os.path.getsize("./saved/Qwen2-VL-2B-Instruct-1.5B-Q4_0.gguf") / 1024**2
+ file_name = os.listdir(quantized_model_path)[0]
+ file_size = os.path.getsize(os.path.join(quantized_model_path, file_name)) / 1024**2
self.assertAlmostEqual(file_size, 892, delta=1.0)
shutil.rmtree("./saved", ignore_errors=True)