The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.
Allow SPieceTokenizer to load model from a byte string.
comfyanonymous88ed89303463
diff --git a/comfy/text_encoders/spiece_tokenizer.py b/comfy/text_encoders/spiece_tokenizer.pyindex d611d5bb7..fa303da0b 100644--- a/comfy/text_encoders/spiece_tokenizer.py+++ b/comfy/text_encoders/spiece_tokenizer.py@@ -1,14 +1,18 @@import osclass SPieceTokenizer:+ add_eos = True+@staticmethoddef from_pretrained(path):return SPieceTokenizer(path)def __init__(self, tokenizer_path):import sentencepiece- self.tokenizer = sentencepiece.SentencePieceProcessor(model_file=tokenizer_path)- self.end = self.tokenizer.eos_id()+ if isinstance(tokenizer_path, bytes):+ self.tokenizer = sentencepiece.SentencePieceProcessor(model_proto=tokenizer_path, add_eos=self.add_eos)+ else:+ self.tokenizer = sentencepiece.SentencePieceProcessor(model_file=tokenizer_path, add_eos=self.add_eos)def get_vocab(self):out = {}@@ -18,5 +22,4 @@ class SPieceTokenizer:def __call__(self, string):out = self.tokenizer.encode(string)- out += [self.end]return {"input_ids": out}