The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.
Better tokenizing code for AuraFlow.
comfyanonymous29c2e26724d4
diff --git a/comfy/text_encoders/aura_t5.py b/comfy/text_encoders/aura_t5.pyindex 0e84189aa..95f942ef5 100644--- a/comfy/text_encoders/aura_t5.py+++ b/comfy/text_encoders/aura_t5.py@@ -1,5 +1,5 @@from comfy import sd1_clip-from transformers import LlamaTokenizerFast+from .llama_tokenizer import LLAMATokenizerimport comfy.t5import os@@ -10,8 +10,8 @@ class PT5XlModel(sd1_clip.SDClipModel):class PT5XlTokenizer(sd1_clip.SDTokenizer):def __init__(self, embedding_directory=None):- tokenizer_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), "t5_pile_tokenizer")- super().__init__(tokenizer_path, pad_with_end=False, embedding_size=2048, embedding_key='pile_t5xl', tokenizer_class=LlamaTokenizerFast, has_start_token=False, pad_to_max_length=False, max_length=99999999, min_length=256, pad_token=1)+ tokenizer_path = os.path.join(os.path.join(os.path.dirname(os.path.realpath(__file__)), "t5_pile_tokenizer"), "tokenizer.model")+ super().__init__(tokenizer_path, pad_with_end=False, embedding_size=2048, embedding_key='pile_t5xl', tokenizer_class=LLAMATokenizer, has_start_token=False, pad_to_max_length=False, max_length=99999999, min_length=256, pad_token=1)class AuraT5Tokenizer(sd1_clip.SD1Tokenizer):def __init__(self, embedding_directory=None):diff --git a/comfy/text_encoders/llama_tokenizer.py b/comfy/text_encoders/llama_tokenizer.pynew file mode 100644index 000000000..a6db1da62--- /dev/null+++ b/comfy/text_encoders/llama_tokenizer.py@@ -0,0 +1,22 @@+import os++class LLAMATokenizer:+ @staticmethod+ def from_pretrained(path):+ return LLAMATokenizer(path)++ def __init__(self, tokenizer_path):+ import sentencepiece+ self.tokenizer = sentencepiece.SentencePieceProcessor(model_file=tokenizer_path)+ self.end = self.tokenizer.eos_id()++ def get_vocab(self):+ out = {}+ for i in range(self.tokenizer.get_piece_size()):+ out[self.tokenizer.id_to_piece(i)] = i+ return out++ def __call__(self, string):+ out = self.tokenizer.encode(string)+ out += [self.end]+ return {"input_ids": out}diff --git a/comfy/text_encoders/t5_pile_tokenizer/added_tokens.json b/comfy/text_encoders/t5_pile_tokenizer/added_tokens.jsondeleted file mode 100644index 3f5132007..000000000--- a/comfy/text_encoders/t5_pile_tokenizer/added_tokens.json+++ /dev/null@@ -1,102 +0,0 @@-{- "<extra_id_0>": 32099,- "<extra_id_10>": 32089,- "<extra_id_11>": 32088,- "<extra_id_12>": 32087,- "<extra_id_13>": 32086,- "<extra_id_14>": 32085,- "<extra_id_15>": 32084,- "<extra_id_16>": 32083,- "<extra_id_17>": 32082,- "<extra_id_18>": 32081,- "<extra_id_19>": 32080,- "<extra_id_1>": 32098,- "<extra_id_20>": 32079,- "<extra_id_21>": 32078,- "<extra_id_22>": 32077,- "<extra_id_23>": 32076,- "<extra_id_24>": 32075,- "<extra_id_25>": 32074,- "<extra_id_26>": 32073,- "<extra_id_27>": 32072,- "<extra_id_28>": 32071,- "<extra_id_29>": 32070,- "<extra_id_2>": 32097,- "<extra_id_30>": 32069,- "<extra_id_31>": 32068,- "<extra_id_32>": 32067,- "<extra_id_33>": 32066,- "<extra_id_34>": 32065,- "<extra_id_35>": 32064,- "<extra_id_36>": 32063,- "<extra_id_37>": 32062,- "<extra_id_38>": 32061,- "<extra_id_39>": 32060,- "<extra_id_3>": 32096,- "<extra_id_40>": 32059,- "<extra_id_41>": 32058,- "<extra_id_42>": 32057,- "<extra_id_43>": 32056,- "<extra_id_44>": 32055,- "<extra_id_45>": 32054,- "<extra_id_46>": 32053,- "<extra_id_47>": 32052,- "<extra_id_48>": 32051,- "<extra_id_49>": 32050,- "<extra_id_4>": 32095,- "<extra_id_50>": 32049,- "<extra_id_51>": 32048,- "<extra_id_52>": 32047,- "<extra_id_53>": 32046,- "<extra_id_54>": 32045,- "<extra_id_55>": 32044,- "<extra_id_56>": 32043,- "<extra_id_57>": 32042,- "<extra_id_58>": 32041,- "<extra_id_59>": 32040,- "<extra_id_5>": 32094,- "<extra_id_60>": 32039,- "<extra_id_61>": 32038,- "<extra_id_62>": 32037,- "<extra_id_63>": 32036,- "<extra_id_64>": 32035,- "<extra_id_65>": 32034,- "<extra_id_66>": 32033,- "<extra_id_67>": 32032,- "<extra_id_68>": 32031,- "<extra_id_69>": 32030,- "<extra_id_6>": 32093,- "<extra_id_70>": 32029,- "<extra_id_71>": 32028,- "<extra_id_72>": 32027,- "<extra_id_73>": 32026,- "<extra_id_74>": 32025,- "<extra_id_75>": 32024,- "<extra_id_76>": 32023,- "<extra_id_77>": 32022,- "<extra_id_78>": 32021,- "<extra_id_79>": 32020,- "<extra_id_7>": 32092,- "<extra_id_80>": 32019,- "<extra_id_81>": 32018,- "<extra_id_82>": 32017,- "<extra_id_83>": 32016,- "<extra_id_84>": 32015,- "<extra_id_85>": 32014,- "<extra_id_86>": 32013,- "<extra_id_87>": 32012,- "<extra_id_88>": 32011,- "<extra_id_89>": 32010,- "<extra_id_8>": 32091,- "<extra_id_90>": 32009,- "<extra_id_91>": 32008,- "<extra_id_92>": 32007,- "<extra_id_93>": 32006,- "<extra_id_94>": 32005,- "<extra_id_95>": 32004,- "<extra_id_96>": 32003,- "<extra_id_97>": 32002,- "<extra_id_98>": 32001,- "<extra_id_99>": 32000,- "<extra_id_9>": 32090-}diff --git a/comfy/text_encoders/t5_pile_tokenizer/special_tokens_map.json b/comfy/text_encoders/t5_pile_tokenizer/special_tokens_map.jsondeleted file mode 100644index 19fb1d5f4..000000000--- a/comfy/text_encoders/t5_pile_tokenizer/special_tokens_map.json+++ /dev/null@@ -1,125 +0,0 @@-{- "additional_special_tokens": [- "<extra_id_99>",- "<extra_id_98>",- "<extra_id_97>",- "<extra_id_96>",- "<extra_id_95>",- "<extra_id_94>",- "<extra_id_93>",- "<extra_id_92>",- "<extra_id_91>",- "<extra_id_90>",- "<extra_id_89>",- "<extra_id_88>",- "<extra_id_87>",- "<extra_id_86>",- "<extra_id_85>",- "<extra_id_84>",- "<extra_id_83>",- "<extra_id_82>",- "<extra_id_81>",- "<extra_id_80>",- "<extra_id_79>",- "<extra_id_78>",- "<extra_id_77>",- "<extra_id_76>",- "<extra_id_75>",- "<extra_id_74>",- "<extra_id_73>",- "<extra_id_72>",- "<extra_id_71>",- "<extra_id_70>",- "<extra_id_69>",- "<extra_id_68>",- "<extra_id_67>",- "<extra_id_66>",- "<extra_id_65>",- "<extra_id_64>",- "<extra_id_63>",- "<extra_id_62>",- "<extra_id_61>",- "<extra_id_60>",- "<extra_id_59>",- "<extra_id_58>",- "<extra_id_57>",- "<extra_id_56>",- "<extra_id_55>",- "<extra_id_54>",- "<extra_id_53>",- "<extra_id_52>",- "<extra_id_51>",- "<extra_id_50>",- "<extra_id_49>",- "<extra_id_48>",- "<extra_id_47>",- "<extra_id_46>",- "<extra_id_45>",- "<extra_id_44>",- "<extra_id_43>",- "<extra_id_42>",- "<extra_id_41>",- "<extra_id_40>",- "<extra_id_39>",- "<extra_id_38>",- "<extra_id_37>",- "<extra_id_36>",- "<extra_id_35>",- "<extra_id_34>",- "<extra_id_33>",- "<extra_id_32>",- "<extra_id_31>",- "<extra_id_30>",- "<extra_id_29>",- "<extra_id_28>",- "<extra_id_27>",- "<extra_id_26>",- "<extra_id_25>",- "<extra_id_24>",- "<extra_id_23>",- "<extra_id_22>",- "<extra_id_21>",- "<extra_id_20>",- "<extra_id_19>",- "<extra_id_18>",- "<extra_id_17>",- "<extra_id_16>",- "<extra_id_15>",- "<extra_id_14>",- "<extra_id_13>",- "<extra_id_12>",- "<extra_id_11>",- "<extra_id_10>",- "<extra_id_9>",- "<extra_id_8>",- "<extra_id_7>",- "<extra_id_6>",- "<extra_id_5>",- "<extra_id_4>",- "<extra_id_3>",- "<extra_id_2>",- "<extra_id_1>",- "<extra_id_0>"- ],- "bos_token": {- "content": "<s>",- "lstrip": false,- "normalized": false,- "rstrip": false,- "single_word": false- },- "eos_token": {- "content": "</s>",- "lstrip": false,- "normalized": false,- "rstrip": false,- "single_word": false- },- "unk_token": {- "content": "<unk>",- "lstrip": false,- "normalized": false,- "rstrip": false,- "single_word": false- }-}diff --git a/comfy/text_encoders/t5_pile_tokenizer/tokenizer_config.json b/comfy/text_encoders/t5_pile_tokenizer/tokenizer_config.jsondeleted file mode 100644index 81f8e11e9..000000000--- a/comfy/text_encoders/t5_pile_tokenizer/tokenizer_config.json+++ /dev/null@@ -1,945 +0,0 @@-{- "add_bos_token": false,- "add_eos_token": true,- "add_prefix_space": true,- "added_tokens_decoder": {- "0": {- "content": "<unk>",- "lstrip": false,- "normalized": false,- "rstrip": false,- "single_word": false,- "special": true- },- "1": {- "content": "<s>",- "lstrip": false,- "normalized": false,- "rstrip": false,- "single_word": false,- "special": true- },- "2": {- "content": "</s>",- "lstrip": false,- "normalized": false,- "rstrip": false,- "single_word": false,- "special": true- },- "32000": {- "content": "<extra_id_99>",- "lstrip": false,- "normalized": false,- "rstrip": false,- "single_word": false,- "special": true- },- "32001": {- "content": "<extra_id_98>",- "lstrip": false,- "normalized": false,- "rstrip": false,- "single_word": false,- "special": true- },- "32002": {- "content": "<extra_id_97>",- "lstrip": false,- "normalized": false,- "rstrip": false,- "single_word": false,- "special": true- },- "32003": {- "content": "<extra_id_96>",- "lstrip": false,- "normalized": false,- "rstrip": false,- "single_word": false,- "special": true- },- "32004": {- "content": "<extra_id_95>",- "lstrip": false,- "normalized": false,- "rstrip": false,- "single_word": false,- "special": true- },- "32005": {- "content": "<extra_id_94>",- "lstrip": false,- "normalized": false,- "rstrip": false,- "single_word": false,- "special": true- },- "32006": {- "content": "<extra_id_93>",- "lstrip": false,- "normalized": false,- "rstrip": false,- "single_word": false,- "special": true- },- "32007": {- "content": "<extra_id_92>",- "lstrip": false,- "normalized": false,- "rstrip": false,- "single_word": false,- "special": true- },- "32008": {- "content": "<extra_id_91>",- "lstrip": false,- "normalized": false,- "rstrip": false,- "single_word": false,- "special": true- },- "32009": {- "content": "<extra_id_90>",- "lstrip": false,- "normalized": false,- "rstrip": false,- "single_word": false,- "special": true- },- "32010": {- "content": "<extra_id_89>",- "lstrip": false,- "normalized": false,- "rstrip": false,- "single_word": false,- "special": true- },- "32011": {- "content": "<extra_id_88>",- "lstrip": false,- "normalized": false,- "rstrip": false,- "single_word": false,- "special": true- },- "32012": {- "content": "<extra_id_87>",- "lstrip": false,- "normalized": false,- "rstrip": false,- "single_word": false,- "special": true- },- "32013": {- "content": "<extra_id_86>",- "lstrip": false,- "normalized": false,- "rstrip": false,- "single_word": false,- "special": true- },- "32014": {- "content": "<extra_id_85>",- "lstrip": false,- "normalized": false,- "rstrip": false,- "single_word": false,- "special": true- },- "32015": {- "content": "<extra_id_84>",- "lstrip": false,- "normalized": false,- "rstrip": false,- "single_word": false,- "special": true- },- "32016": {- "content": "<extra_id_83>",- "lstrip": false,- "normalized": false,- "rstrip": false,- "single_word": false,- "special": true- },- "32017": {- "content": "<extra_id_82>",- "lstrip": false,- "normalized": false,- "rstrip": false,- "single_word": false,- "special": true- },- "32018": {- "content": "<extra_id_81>",- "lstrip": false,- "normalized": false,- "rstrip": false,- "single_word": false,- "special": true- },- "32019": {- "content": "<extra_id_80>",- "lstrip": false,- "normalized": false,- "rstrip": false,- "single_word": false,- "special": true- },- "32020": {- "content": "<extra_id_79>",- "lstrip": false,- "normalized": false,- "rstrip": false,- "single_word": false,- "special": true- },- "32021": {- "content": "<extra_id_78>",- "lstrip": false,- "normalized": false,- "rstrip": false,- "single_word": false,- "special": true- },- "32022": {- "content": "<extra_id_77>",- "lstrip": false,- "normalized": false,- "rstrip": false,- "single_word": false,- "special": true- },- "32023": {- "content": "<extra_id_76>",- "lstrip": false,- "normalized": false,- "rstrip": false,- "single_word": false,- "special": true- },- "32024": {- "content": "<extra_id_75>",- "lstrip": false,- "normalized": false,- "rstrip": false,- "single_word": false,- "special": true- },- "32025": {- "content": "<extra_id_74>",- "lstrip": false,- "normalized": false,- "rstrip": false,- "single_word": false,- "special": true- },- "32026": {- "content": "<extra_id_73>",- "lstrip": false,- "normalized": false,- "rstrip": false,- "single_word": false,- "special": true- },- "32027": {- "content": "<extra_id_72>",- "lstrip": false,- "normalized": false,- "rstrip": false,- "single_word": false,- "special": true- },- "32028": {- "content": "<extra_id_71>",- "lstrip": false,- "normalized": false,- "rstrip": false,- "single_word": false,- "special": true- },- "32029": {- "content": "<extra_id_70>",- "lstrip": false,- "normalized": false,- "rstrip": false,- "single_word": false,- "special": true- },- "32030": {- "content": "<extra_id_69>",- "lstrip": false,- "normalized": false,- "rstrip": false,- "single_word": false,- "special": true- },- "32031": {- "content": "<extra_id_68>",- "lstrip": false,- "normalized": false,- "rstrip": false,- "single_word": false,- "special": true- },- "32032": {- "content": "<extra_id_67>",- "lstrip": false,- "normalized": false,- "rstrip": false,- "single_word": false,- "special": true- },- "32033": {- "content": "<extra_id_66>",- "lstrip": false,- "normalized": false,- "rstrip": false,- "single_word": false,- "special": true- },- "32034": {- "content": "<extra_id_65>",- "lstrip": false,- "normalized": false,- "rstrip": false,- "single_word": false,- "special": true- },- "32035": {- "content": "<extra_id_64>",- "lstrip": false,- "normalized": false,- "rstrip": false,- "single_word": false,- "special": true- },- "32036": {- "content": "<extra_id_63>",- "lstrip": false,- "normalized": false,- "rstrip": false,- "single_word": false,- "special": true- },- "32037": {- "content": "<extra_id_62>",- "lstrip": false,- "normalized": false,- "rstrip": false,- "single_word": false,- "special": true- },- "32038": {- "content": "<extra_id_61>",- "lstrip": false,- "normalized": false,- "rstrip": false,- "single_word": false,- "special": true- },- "32039": {- "content": "<extra_id_60>",- "lstrip": false,- "normalized": false,- "rstrip": false,- "single_word": false,- "special": true- },- "32040": {- "content": "<extra_id_59>",- "lstrip": false,- "normalized": false,- "rstrip": false,- "single_word": false,- "special": true- },- "32041": {- "content": "<extra_id_58>",- "lstrip": false,- "normalized": false,- "rstrip": false,- "single_word": false,- "special": true- },- "32042": {- "content": "<extra_id_57>",- "lstrip": false,- "normalized": false,- "rstrip": false,- "single_word": false,- "special": true- },- "32043": {- "content": "<extra_id_56>",- "lstrip": false,- "normalized": false,- "rstrip": false,- "single_word": false,- "special": true- },- "32044": {- "content": "<extra_id_55>",- "lstrip": false,- "normalized": false,- "rstrip": false,- "single_word": false,- "special": true- },- "32045": {- "content": "<extra_id_54>",- "lstrip": false,- "normalized": false,- "rstrip": false,- "single_word": false,- "special": true- },- "32046": {- "content": "<extra_id_53>",- "lstrip": false,- "normalized": false,- "rstrip": false,- "single_word": false,- "special": true- },- "32047": {- "content": "<extra_id_52>",- "lstrip": false,- "normalized": false,- "rstrip": false,- "single_word": false,- "special": true- },- "32048": {- "content": "<extra_id_51>",- "lstrip": false,- "normalized": false,- "rstrip": false,- "single_word": false,- "special": true- },- "32049": {- "content": "<extra_id_50>",- "lstrip": false,- "normalized": false,- "rstrip": false,- "single_word": false,- "special": true- },- "32050": {- "content": "<extra_id_49>",- "lstrip": false,- "normalized": false,- "rstrip": false,- "single_word": false,- "special": true- },- "32051": {- "content": "<extra_id_48>",- "lstrip": false,- "normalized": false,- "rstrip": false,- "single_word": false,- "special": true- },- "32052": {- "content": "<extra_id_47>",- "lstrip": false,- "normalized": false,- "rstrip": false,- "single_word": false,- "special": true- },- "32053": {- "content": "<extra_id_46>",- "lstrip": false,- "normalized": false,- "rstrip": false,- "single_word": false,- "special": true- },- "32054": {- "content": "<extra_id_45>",- "lstrip": false,- "normalized": false,- "rstrip": false,- "single_word": false,- "special": true- },- "32055": {- "content": "<extra_id_44>",- "lstrip": false,- "normalized": false,- "rstrip": false,- "single_word": false,- "special": true- },- "32056": {- "content": "<extra_id_43>",- "lstrip": false,- "normalized": false,- "rstrip": false,- "single_word": false,- "special": true- },- "32057": {- "content": "<extra_id_42>",- "lstrip": false,- "normalized": false,- "rstrip": false,- "single_word": false,- "special": true- },- "32058": {- "content": "<extra_id_41>",- "lstrip": false,- "normalized": false,- "rstrip": false,- "single_word": false,- "special": true- },- "32059": {- "content": "<extra_id_40>",- "lstrip": false,- "normalized": false,- "rstrip": false,- "single_word": false,- "special": true- },- "32060": {- "content": "<extra_id_39>",- "lstrip": false,- "normalized": false,- "rstrip": false,- "single_word": false,- "special": true- },- "32061": {- "content": "<extra_id_38>",- "lstrip": false,- "normalized": false,- "rstrip": false,- "single_word": false,- "special": true- },- "32062": {- "content": "<extra_id_37>",- "lstrip": false,- "normalized": false,- "rstrip": false,- "single_word": false,- "special": true- },- "32063": {- "content": "<extra_id_36>",- "lstrip": false,- "normalized": false,- "rstrip": false,- "single_word": false,- "special": true- },- "32064": {- "content": "<extra_id_35>",- "lstrip": false,- "normalized": false,- "rstrip": false,- "single_word": false,- "special": true- },- "32065": {- "content": "<extra_id_34>",- "lstrip": false,- "normalized": false,- "rstrip": false,- "single_word": false,- "special": true- },- "32066": {- "content": "<extra_id_33>",- "lstrip": false,- "normalized": false,- "rstrip": false,- "single_word": false,- "special": true- },- "32067": {- "content": "<extra_id_32>",- "lstrip": false,- "normalized": false,- "rstrip": false,- "single_word": false,- "special": true- },- "32068": {- "content": "<extra_id_31>",- "lstrip": false,- "normalized": false,- "rstrip": false,- "single_word": false,- "special": true- },- "32069": {- "content": "<extra_id_30>",- "lstrip": false,- "normalized": false,- "rstrip": false,- "single_word": false,- "special": true- },- "32070": {- "content": "<extra_id_29>",- "lstrip": false,- "normalized": false,- "rstrip": false,- "single_word": false,- "special": true- },- "32071": {- "content": "<extra_id_28>",- "lstrip": false,- "normalized": false,- "rstrip": false,- "single_word": false,- "special": true- },- "32072": {- "content": "<extra_id_27>",- "lstrip": false,- "normalized": false,- "rstrip": false,- "single_word": false,- "special": true- },- "32073": {- "content": "<extra_id_26>",- "lstrip": false,- "normalized": false,- "rstrip": false,- "single_word": false,- "special": true- },- "32074": {- "content": "<extra_id_25>",- "lstrip": false,- "normalized": false,- "rstrip": false,- "single_word": false,- "special": true- },- "32075": {- "content": "<extra_id_24>",- "lstrip": false,- "normalized": false,- "rstrip": false,- "single_word": false,- "special": true- },- "32076": {- "content": "<extra_id_23>",- "lstrip": false,- "normalized": false,- "rstrip": false,- "single_word": false,- "special": true- },- "32077": {- "content": "<extra_id_22>",- "lstrip": false,- "normalized": false,- "rstrip": false,- "single_word": false,- "special": true- },- "32078": {- "content": "<extra_id_21>",- "lstrip": false,- "normalized": false,- "rstrip": false,- "single_word": false,- "special": true- },- "32079": {- "content": "<extra_id_20>",- "lstrip": false,- "normalized": false,- "rstrip": false,- "single_word": false,- "special": true- },- "32080": {- "content": "<extra_id_19>",- "lstrip": false,- "normalized": false,- "rstrip": false,- "single_word": false,- "special": true- },- "32081": {- "content": "<extra_id_18>",- "lstrip": false,- "normalized": false,- "rstrip": false,- "single_word": false,- "special": true- },- "32082": {- "content": "<extra_id_17>",- "lstrip": false,- "normalized": false,- "rstrip": false,- "single_word": false,- "special": true- },- "32083": {- "content": "<extra_id_16>",- "lstrip": false,- "normalized": false,- "rstrip": false,- "single_word": false,- "special": true- },- "32084": {- "content": "<extra_id_15>",- "lstrip": false,- "normalized": false,- "rstrip": false,- "single_word": false,- "special": true- },- "32085": {- "content": "<extra_id_14>",- "lstrip": false,- "normalized": false,- "rstrip": false,- "single_word": false,- "special": true- },- "32086": {- "content": "<extra_id_13>",- "lstrip": false,- "normalized": false,- "rstrip": false,- "single_word": false,- "special": true- },- "32087": {- "content": "<extra_id_12>",- "lstrip": false,- "normalized": false,- "rstrip": false,- "single_word": false,- "special": true- },- "32088": {- "content": "<extra_id_11>",- "lstrip": false,- "normalized": false,- "rstrip": false,- "single_word": false,- "special": true- },- "32089": {- "content": "<extra_id_10>",- "lstrip": false,- "normalized": false,- "rstrip": false,- "single_word": false,- "special": true- },- "32090": {- "content": "<extra_id_9>",- "lstrip": false,- "normalized": false,- "rstrip": false,- "single_word": false,- "special": true- },- "32091": {- "content": "<extra_id_8>",- "lstrip": false,- "normalized": false,- "rstrip": false,- "single_word": false,- "special": true- },- "32092": {- "content": "<extra_id_7>",- "lstrip": false,- "normalized": false,- "rstrip": false,- "single_word": false,- "special": true- },- "32093": {- "content": "<extra_id_6>",- "lstrip": false,- "normalized": false,- "rstrip": false,- "single_word": false,- "special": true- },- "32094": {- "content": "<extra_id_5>",- "lstrip": false,- "normalized": false,- "rstrip": false,- "single_word": false,- "special": true- },- "32095": {- "content": "<extra_id_4>",- "lstrip": false,- "normalized": false,- "rstrip": false,- "single_word": false,- "special": true- },- "32096": {- "content": "<extra_id_3>",- "lstrip": false,- "normalized": false,- "rstrip": false,- "single_word": false,- "special": true- },- "32097": {- "content": "<extra_id_2>",- "lstrip": false,- "normalized": false,- "rstrip": false,- "single_word": false,- "special": true- },- "32098": {- "content": "<extra_id_1>",- "lstrip": false,- "normalized": false,- "rstrip": false,- "single_word": false,- "special": true- },- "32099": {- "content": "<extra_id_0>",- "lstrip": false,- "normalized": false,- "rstrip": false,- "single_word": false,- "special": true- }- },- "additional_special_tokens": [- "<extra_id_99>",- "<extra_id_98>",- "<extra_id_97>",- "<extra_id_96>",- "<extra_id_95>",- "<extra_id_94>",- "<extra_id_93>",- "<extra_id_92>",- "<extra_id_91>",- "<extra_id_90>",- "<extra_id_89>",- "<extra_id_88>",- "<extra_id_87>",- "<extra_id_86>",- "<extra_id_85>",- "<extra_id_84>",- "<extra_id_83>",- "<extra_id_82>",- "<extra_id_81>",- "<extra_id_80>",- "<extra_id_79>",- "<extra_id_78>",- "<extra_id_77>",- "<extra_id_76>",- "<extra_id_75>",- "<extra_id_74>",- "<extra_id_73>",- "<extra_id_72>",- "<extra_id_71>",- "<extra_id_70>",- "<extra_id_69>",- "<extra_id_68>",- "<extra_id_67>",- "<extra_id_66>",- "<extra_id_65>",- "<extra_id_64>",- "<extra_id_63>",- "<extra_id_62>",- "<extra_id_61>",- "<extra_id_60>",- "<extra_id_59>",- "<extra_id_58>",- "<extra_id_57>",- "<extra_id_56>",- "<extra_id_55>",- "<extra_id_54>",- "<extra_id_53>",- "<extra_id_52>",- "<extra_id_51>",- "<extra_id_50>",- "<extra_id_49>",- "<extra_id_48>",- "<extra_id_47>",- "<extra_id_46>",- "<extra_id_45>",- "<extra_id_44>",- "<extra_id_43>",- "<extra_id_42>",- "<extra_id_41>",- "<extra_id_40>",- "<extra_id_39>",- "<extra_id_38>",- "<extra_id_37>",- "<extra_id_36>",- "<extra_id_35>",- "<extra_id_34>",- "<extra_id_33>",- "<extra_id_32>",- "<extra_id_31>",- "<extra_id_30>",- "<extra_id_29>",- "<extra_id_28>",- "<extra_id_27>",- "<extra_id_26>",- "<extra_id_25>",- "<extra_id_24>",- "<extra_id_23>",- "<extra_id_22>",- "<extra_id_21>",- "<extra_id_20>",- "<extra_id_19>",- "<extra_id_18>",- "<extra_id_17>",- "<extra_id_16>",- "<extra_id_15>",- "<extra_id_14>",- "<extra_id_13>",- "<extra_id_12>",- "<extra_id_11>",- "<extra_id_10>",- "<extra_id_9>",- "<extra_id_8>",- "<extra_id_7>",- "<extra_id_6>",- "<extra_id_5>",- "<extra_id_4>",- "<extra_id_3>",- "<extra_id_2>",- "<extra_id_1>",- "<extra_id_0>"- ],- "bos_token": "<s>",- "clean_up_tokenization_spaces": false,- "eos_token": "</s>",- "legacy": false,- "model_max_length": 512,- "pad_token": null,- "padding_side": "right",- "sp_model_kwargs": {},- "spaces_between_special_tokens": false,- "tokenizer_class": "LlamaTokenizer",- "unk_token": "<unk>",- "use_default_system_prompt": false-}