LLM inference in C/C++
C++ ggml
convert : import bytes_to_unicode from convert_slow_tokenizer (#26217)
Caleb DeLeeuwb06fbc968b98
diff --git a/conversion/chatglm.py b/conversion/chatglm.pyindex 801913075d..d638550387 100644--- a/conversion/chatglm.py+++ b/conversion/chatglm.py@@ -81,7 +81,7 @@ class ChatGLMModel(TextModel):@staticmethoddef token_bytes_to_string(b):- from transformers.models.gpt2.tokenization_gpt2 import bytes_to_unicode # ty: ignore[unresolved-import]+ from transformers.convert_slow_tokenizer import bytes_to_unicodebyte_encoder = bytes_to_unicode()return ''.join([byte_encoder[ord(char)] for char in b.decode('latin-1')])diff --git a/conversion/qwen.py b/conversion/qwen.pyindex 7e3d8c0d12..b4ae528bf2 100644--- a/conversion/qwen.py+++ b/conversion/qwen.py@@ -18,7 +18,7 @@ class QwenModel(TextModel):@staticmethoddef token_bytes_to_string(b):- from transformers.models.gpt2.tokenization_gpt2 import bytes_to_unicode # ty: ignore[unresolved-import]+ from transformers.convert_slow_tokenizer import bytes_to_unicodebyte_encoder = bytes_to_unicode()return ''.join([byte_encoder[ord(char)] for char in b.decode('latin-1')])