Skip to content

LLM inference in C/C++

C++ ggml

convert : import bytes_to_unicode from convert_slow_tokenizer (#26217)

Caleb DeLeeuwb06fbc968b98
diff --git a/conversion/chatglm.py b/conversion/chatglm.py
index 801913075d..d638550387 100644
--- a/conversion/chatglm.py
+++ b/conversion/chatglm.py
@@ -81,7 +81,7 @@ class ChatGLMModel(TextModel):
@staticmethod
def token_bytes_to_string(b):
- from transformers.models.gpt2.tokenization_gpt2 import bytes_to_unicode # ty: ignore[unresolved-import]
+ from transformers.convert_slow_tokenizer import bytes_to_unicode
byte_encoder = bytes_to_unicode()
return ''.join([byte_encoder[ord(char)] for char in b.decode('latin-1')])
diff --git a/conversion/qwen.py b/conversion/qwen.py
index 7e3d8c0d12..b4ae528bf2 100644
--- a/conversion/qwen.py
+++ b/conversion/qwen.py
@@ -18,7 +18,7 @@ class QwenModel(TextModel):
@staticmethod
def token_bytes_to_string(b):
- from transformers.models.gpt2.tokenization_gpt2 import bytes_to_unicode # ty: ignore[unresolved-import]
+ from transformers.convert_slow_tokenizer import bytes_to_unicode
byte_encoder = bytes_to_unicode()
return ''.join([byte_encoder[ord(char)] for char in b.decode('latin-1')])