Skip to content

The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.

Better tokenizing code for AuraFlow.

comfyanonymous29c2e26724d4
diff --git a/comfy/text_encoders/aura_t5.py b/comfy/text_encoders/aura_t5.py
index 0e84189aa..95f942ef5 100644
--- a/comfy/text_encoders/aura_t5.py
+++ b/comfy/text_encoders/aura_t5.py
@@ -1,5 +1,5 @@
from comfy import sd1_clip
-from transformers import LlamaTokenizerFast
+from .llama_tokenizer import LLAMATokenizer
import comfy.t5
import os
@@ -10,8 +10,8 @@ class PT5XlModel(sd1_clip.SDClipModel):
class PT5XlTokenizer(sd1_clip.SDTokenizer):
def __init__(self, embedding_directory=None):
- tokenizer_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), "t5_pile_tokenizer")
- super().__init__(tokenizer_path, pad_with_end=False, embedding_size=2048, embedding_key='pile_t5xl', tokenizer_class=LlamaTokenizerFast, has_start_token=False, pad_to_max_length=False, max_length=99999999, min_length=256, pad_token=1)
+ tokenizer_path = os.path.join(os.path.join(os.path.dirname(os.path.realpath(__file__)), "t5_pile_tokenizer"), "tokenizer.model")
+ super().__init__(tokenizer_path, pad_with_end=False, embedding_size=2048, embedding_key='pile_t5xl', tokenizer_class=LLAMATokenizer, has_start_token=False, pad_to_max_length=False, max_length=99999999, min_length=256, pad_token=1)
class AuraT5Tokenizer(sd1_clip.SD1Tokenizer):
def __init__(self, embedding_directory=None):
diff --git a/comfy/text_encoders/llama_tokenizer.py b/comfy/text_encoders/llama_tokenizer.py
new file mode 100644
index 000000000..a6db1da62
--- /dev/null
+++ b/comfy/text_encoders/llama_tokenizer.py
@@ -0,0 +1,22 @@
+import os
+
+class LLAMATokenizer:
+ @staticmethod
+ def from_pretrained(path):
+ return LLAMATokenizer(path)
+
+ def __init__(self, tokenizer_path):
+ import sentencepiece
+ self.tokenizer = sentencepiece.SentencePieceProcessor(model_file=tokenizer_path)
+ self.end = self.tokenizer.eos_id()
+
+ def get_vocab(self):
+ out = {}
+ for i in range(self.tokenizer.get_piece_size()):
+ out[self.tokenizer.id_to_piece(i)] = i
+ return out
+
+ def __call__(self, string):
+ out = self.tokenizer.encode(string)
+ out += [self.end]
+ return {"input_ids": out}
diff --git a/comfy/text_encoders/t5_pile_tokenizer/added_tokens.json b/comfy/text_encoders/t5_pile_tokenizer/added_tokens.json
deleted file mode 100644
index 3f5132007..000000000
--- a/comfy/text_encoders/t5_pile_tokenizer/added_tokens.json
+++ /dev/null
@@ -1,102 +0,0 @@
-{
- "<extra_id_0>": 32099,
- "<extra_id_10>": 32089,
- "<extra_id_11>": 32088,
- "<extra_id_12>": 32087,
- "<extra_id_13>": 32086,
- "<extra_id_14>": 32085,
- "<extra_id_15>": 32084,
- "<extra_id_16>": 32083,
- "<extra_id_17>": 32082,
- "<extra_id_18>": 32081,
- "<extra_id_19>": 32080,
- "<extra_id_1>": 32098,
- "<extra_id_20>": 32079,
- "<extra_id_21>": 32078,
- "<extra_id_22>": 32077,
- "<extra_id_23>": 32076,
- "<extra_id_24>": 32075,
- "<extra_id_25>": 32074,
- "<extra_id_26>": 32073,
- "<extra_id_27>": 32072,
- "<extra_id_28>": 32071,
- "<extra_id_29>": 32070,
- "<extra_id_2>": 32097,
- "<extra_id_30>": 32069,
- "<extra_id_31>": 32068,
- "<extra_id_32>": 32067,
- "<extra_id_33>": 32066,
- "<extra_id_34>": 32065,
- "<extra_id_35>": 32064,
- "<extra_id_36>": 32063,
- "<extra_id_37>": 32062,
- "<extra_id_38>": 32061,
- "<extra_id_39>": 32060,
- "<extra_id_3>": 32096,
- "<extra_id_40>": 32059,
- "<extra_id_41>": 32058,
- "<extra_id_42>": 32057,
- "<extra_id_43>": 32056,
- "<extra_id_44>": 32055,
- "<extra_id_45>": 32054,
- "<extra_id_46>": 32053,
- "<extra_id_47>": 32052,
- "<extra_id_48>": 32051,
- "<extra_id_49>": 32050,
- "<extra_id_4>": 32095,
- "<extra_id_50>": 32049,
- "<extra_id_51>": 32048,
- "<extra_id_52>": 32047,
- "<extra_id_53>": 32046,
- "<extra_id_54>": 32045,
- "<extra_id_55>": 32044,
- "<extra_id_56>": 32043,
- "<extra_id_57>": 32042,
- "<extra_id_58>": 32041,
- "<extra_id_59>": 32040,
- "<extra_id_5>": 32094,
- "<extra_id_60>": 32039,
- "<extra_id_61>": 32038,
- "<extra_id_62>": 32037,
- "<extra_id_63>": 32036,
- "<extra_id_64>": 32035,
- "<extra_id_65>": 32034,
- "<extra_id_66>": 32033,
- "<extra_id_67>": 32032,
- "<extra_id_68>": 32031,
- "<extra_id_69>": 32030,
- "<extra_id_6>": 32093,
- "<extra_id_70>": 32029,
- "<extra_id_71>": 32028,
- "<extra_id_72>": 32027,
- "<extra_id_73>": 32026,
- "<extra_id_74>": 32025,
- "<extra_id_75>": 32024,
- "<extra_id_76>": 32023,
- "<extra_id_77>": 32022,
- "<extra_id_78>": 32021,
- "<extra_id_79>": 32020,
- "<extra_id_7>": 32092,
- "<extra_id_80>": 32019,
- "<extra_id_81>": 32018,
- "<extra_id_82>": 32017,
- "<extra_id_83>": 32016,
- "<extra_id_84>": 32015,
- "<extra_id_85>": 32014,
- "<extra_id_86>": 32013,
- "<extra_id_87>": 32012,
- "<extra_id_88>": 32011,
- "<extra_id_89>": 32010,
- "<extra_id_8>": 32091,
- "<extra_id_90>": 32009,
- "<extra_id_91>": 32008,
- "<extra_id_92>": 32007,
- "<extra_id_93>": 32006,
- "<extra_id_94>": 32005,
- "<extra_id_95>": 32004,
- "<extra_id_96>": 32003,
- "<extra_id_97>": 32002,
- "<extra_id_98>": 32001,
- "<extra_id_99>": 32000,
- "<extra_id_9>": 32090
-}
diff --git a/comfy/text_encoders/t5_pile_tokenizer/special_tokens_map.json b/comfy/text_encoders/t5_pile_tokenizer/special_tokens_map.json
deleted file mode 100644
index 19fb1d5f4..000000000
--- a/comfy/text_encoders/t5_pile_tokenizer/special_tokens_map.json
+++ /dev/null
@@ -1,125 +0,0 @@
-{
- "additional_special_tokens": [
- "<extra_id_99>",
- "<extra_id_98>",
- "<extra_id_97>",
- "<extra_id_96>",
- "<extra_id_95>",
- "<extra_id_94>",
- "<extra_id_93>",
- "<extra_id_92>",
- "<extra_id_91>",
- "<extra_id_90>",
- "<extra_id_89>",
- "<extra_id_88>",
- "<extra_id_87>",
- "<extra_id_86>",
- "<extra_id_85>",
- "<extra_id_84>",
- "<extra_id_83>",
- "<extra_id_82>",
- "<extra_id_81>",
- "<extra_id_80>",
- "<extra_id_79>",
- "<extra_id_78>",
- "<extra_id_77>",
- "<extra_id_76>",
- "<extra_id_75>",
- "<extra_id_74>",
- "<extra_id_73>",
- "<extra_id_72>",
- "<extra_id_71>",
- "<extra_id_70>",
- "<extra_id_69>",
- "<extra_id_68>",
- "<extra_id_67>",
- "<extra_id_66>",
- "<extra_id_65>",
- "<extra_id_64>",
- "<extra_id_63>",
- "<extra_id_62>",
- "<extra_id_61>",
- "<extra_id_60>",
- "<extra_id_59>",
- "<extra_id_58>",
- "<extra_id_57>",
- "<extra_id_56>",
- "<extra_id_55>",
- "<extra_id_54>",
- "<extra_id_53>",
- "<extra_id_52>",
- "<extra_id_51>",
- "<extra_id_50>",
- "<extra_id_49>",
- "<extra_id_48>",
- "<extra_id_47>",
- "<extra_id_46>",
- "<extra_id_45>",
- "<extra_id_44>",
- "<extra_id_43>",
- "<extra_id_42>",
- "<extra_id_41>",
- "<extra_id_40>",
- "<extra_id_39>",
- "<extra_id_38>",
- "<extra_id_37>",
- "<extra_id_36>",
- "<extra_id_35>",
- "<extra_id_34>",
- "<extra_id_33>",
- "<extra_id_32>",
- "<extra_id_31>",
- "<extra_id_30>",
- "<extra_id_29>",
- "<extra_id_28>",
- "<extra_id_27>",
- "<extra_id_26>",
- "<extra_id_25>",
- "<extra_id_24>",
- "<extra_id_23>",
- "<extra_id_22>",
- "<extra_id_21>",
- "<extra_id_20>",
- "<extra_id_19>",
- "<extra_id_18>",
- "<extra_id_17>",
- "<extra_id_16>",
- "<extra_id_15>",
- "<extra_id_14>",
- "<extra_id_13>",
- "<extra_id_12>",
- "<extra_id_11>",
- "<extra_id_10>",
- "<extra_id_9>",
- "<extra_id_8>",
- "<extra_id_7>",
- "<extra_id_6>",
- "<extra_id_5>",
- "<extra_id_4>",
- "<extra_id_3>",
- "<extra_id_2>",
- "<extra_id_1>",
- "<extra_id_0>"
- ],
- "bos_token": {
- "content": "<s>",
- "lstrip": false,
- "normalized": false,
- "rstrip": false,
- "single_word": false
- },
- "eos_token": {
- "content": "</s>",
- "lstrip": false,
- "normalized": false,
- "rstrip": false,
- "single_word": false
- },
- "unk_token": {
- "content": "<unk>",
- "lstrip": false,
- "normalized": false,
- "rstrip": false,
- "single_word": false
- }
-}
diff --git a/comfy/text_encoders/t5_pile_tokenizer/tokenizer_config.json b/comfy/text_encoders/t5_pile_tokenizer/tokenizer_config.json
deleted file mode 100644
index 81f8e11e9..000000000
--- a/comfy/text_encoders/t5_pile_tokenizer/tokenizer_config.json
+++ /dev/null
@@ -1,945 +0,0 @@
-{
- "add_bos_token": false,
- "add_eos_token": true,
- "add_prefix_space": true,
- "added_tokens_decoder": {
- "0": {
- "content": "<unk>",
- "lstrip": false,
- "normalized": false,
- "rstrip": false,
- "single_word": false,
- "special": true
- },
- "1": {
- "content": "<s>",
- "lstrip": false,
- "normalized": false,
- "rstrip": false,
- "single_word": false,
- "special": true
- },
- "2": {
- "content": "</s>",
- "lstrip": false,
- "normalized": false,
- "rstrip": false,
- "single_word": false,
- "special": true
- },
- "32000": {
- "content": "<extra_id_99>",
- "lstrip": false,
- "normalized": false,
- "rstrip": false,
- "single_word": false,
- "special": true
- },
- "32001": {
- "content": "<extra_id_98>",
- "lstrip": false,
- "normalized": false,
- "rstrip": false,
- "single_word": false,
- "special": true
- },
- "32002": {
- "content": "<extra_id_97>",
- "lstrip": false,
- "normalized": false,
- "rstrip": false,
- "single_word": false,
- "special": true
- },
- "32003": {
- "content": "<extra_id_96>",
- "lstrip": false,
- "normalized": false,
- "rstrip": false,
- "single_word": false,
- "special": true
- },
- "32004": {
- "content": "<extra_id_95>",
- "lstrip": false,
- "normalized": false,
- "rstrip": false,
- "single_word": false,
- "special": true
- },
- "32005": {
- "content": "<extra_id_94>",
- "lstrip": false,
- "normalized": false,
- "rstrip": false,
- "single_word": false,
- "special": true
- },
- "32006": {
- "content": "<extra_id_93>",
- "lstrip": false,
- "normalized": false,
- "rstrip": false,
- "single_word": false,
- "special": true
- },
- "32007": {
- "content": "<extra_id_92>",
- "lstrip": false,
- "normalized": false,
- "rstrip": false,
- "single_word": false,
- "special": true
- },
- "32008": {
- "content": "<extra_id_91>",
- "lstrip": false,
- "normalized": false,
- "rstrip": false,
- "single_word": false,
- "special": true
- },
- "32009": {
- "content": "<extra_id_90>",
- "lstrip": false,
- "normalized": false,
- "rstrip": false,
- "single_word": false,
- "special": true
- },
- "32010": {
- "content": "<extra_id_89>",
- "lstrip": false,
- "normalized": false,
- "rstrip": false,
- "single_word": false,
- "special": true
- },
- "32011": {
- "content": "<extra_id_88>",
- "lstrip": false,
- "normalized": false,
- "rstrip": false,
- "single_word": false,
- "special": true
- },
- "32012": {
- "content": "<extra_id_87>",
- "lstrip": false,
- "normalized": false,
- "rstrip": false,
- "single_word": false,
- "special": true
- },
- "32013": {
- "content": "<extra_id_86>",
- "lstrip": false,
- "normalized": false,
- "rstrip": false,
- "single_word": false,
- "special": true
- },
- "32014": {
- "content": "<extra_id_85>",
- "lstrip": false,
- "normalized": false,
- "rstrip": false,
- "single_word": false,
- "special": true
- },
- "32015": {
- "content": "<extra_id_84>",
- "lstrip": false,
- "normalized": false,
- "rstrip": false,
- "single_word": false,
- "special": true
- },
- "32016": {
- "content": "<extra_id_83>",
- "lstrip": false,
- "normalized": false,
- "rstrip": false,
- "single_word": false,
- "special": true
- },
- "32017": {
- "content": "<extra_id_82>",
- "lstrip": false,
- "normalized": false,
- "rstrip": false,
- "single_word": false,
- "special": true
- },
- "32018": {
- "content": "<extra_id_81>",
- "lstrip": false,
- "normalized": false,
- "rstrip": false,
- "single_word": false,
- "special": true
- },
- "32019": {
- "content": "<extra_id_80>",
- "lstrip": false,
- "normalized": false,
- "rstrip": false,
- "single_word": false,
- "special": true
- },
- "32020": {
- "content": "<extra_id_79>",
- "lstrip": false,
- "normalized": false,
- "rstrip": false,
- "single_word": false,
- "special": true
- },
- "32021": {
- "content": "<extra_id_78>",
- "lstrip": false,
- "normalized": false,
- "rstrip": false,
- "single_word": false,
- "special": true
- },
- "32022": {
- "content": "<extra_id_77>",
- "lstrip": false,
- "normalized": false,
- "rstrip": false,
- "single_word": false,
- "special": true
- },
- "32023": {
- "content": "<extra_id_76>",
- "lstrip": false,
- "normalized": false,
- "rstrip": false,
- "single_word": false,
- "special": true
- },
- "32024": {
- "content": "<extra_id_75>",
- "lstrip": false,
- "normalized": false,
- "rstrip": false,
- "single_word": false,
- "special": true
- },
- "32025": {
- "content": "<extra_id_74>",
- "lstrip": false,
- "normalized": false,
- "rstrip": false,
- "single_word": false,
- "special": true
- },
- "32026": {
- "content": "<extra_id_73>",
- "lstrip": false,
- "normalized": false,
- "rstrip": false,
- "single_word": false,
- "special": true
- },
- "32027": {
- "content": "<extra_id_72>",
- "lstrip": false,
- "normalized": false,
- "rstrip": false,
- "single_word": false,
- "special": true
- },
- "32028": {
- "content": "<extra_id_71>",
- "lstrip": false,
- "normalized": false,
- "rstrip": false,
- "single_word": false,
- "special": true
- },
- "32029": {
- "content": "<extra_id_70>",
- "lstrip": false,
- "normalized": false,
- "rstrip": false,
- "single_word": false,
- "special": true
- },
- "32030": {
- "content": "<extra_id_69>",
- "lstrip": false,
- "normalized": false,
- "rstrip": false,
- "single_word": false,
- "special": true
- },
- "32031": {
- "content": "<extra_id_68>",
- "lstrip": false,
- "normalized": false,
- "rstrip": false,
- "single_word": false,
- "special": true
- },
- "32032": {
- "content": "<extra_id_67>",
- "lstrip": false,
- "normalized": false,
- "rstrip": false,
- "single_word": false,
- "special": true
- },
- "32033": {
- "content": "<extra_id_66>",
- "lstrip": false,
- "normalized": false,
- "rstrip": false,
- "single_word": false,
- "special": true
- },
- "32034": {
- "content": "<extra_id_65>",
- "lstrip": false,
- "normalized": false,
- "rstrip": false,
- "single_word": false,
- "special": true
- },
- "32035": {
- "content": "<extra_id_64>",
- "lstrip": false,
- "normalized": false,
- "rstrip": false,
- "single_word": false,
- "special": true
- },
- "32036": {
- "content": "<extra_id_63>",
- "lstrip": false,
- "normalized": false,
- "rstrip": false,
- "single_word": false,
- "special": true
- },
- "32037": {
- "content": "<extra_id_62>",
- "lstrip": false,
- "normalized": false,
- "rstrip": false,
- "single_word": false,
- "special": true
- },
- "32038": {
- "content": "<extra_id_61>",
- "lstrip": false,
- "normalized": false,
- "rstrip": false,
- "single_word": false,
- "special": true
- },
- "32039": {
- "content": "<extra_id_60>",
- "lstrip": false,
- "normalized": false,
- "rstrip": false,
- "single_word": false,
- "special": true
- },
- "32040": {
- "content": "<extra_id_59>",
- "lstrip": false,
- "normalized": false,
- "rstrip": false,
- "single_word": false,
- "special": true
- },
- "32041": {
- "content": "<extra_id_58>",
- "lstrip": false,
- "normalized": false,
- "rstrip": false,
- "single_word": false,
- "special": true
- },
- "32042": {
- "content": "<extra_id_57>",
- "lstrip": false,
- "normalized": false,
- "rstrip": false,
- "single_word": false,
- "special": true
- },
- "32043": {
- "content": "<extra_id_56>",
- "lstrip": false,
- "normalized": false,
- "rstrip": false,
- "single_word": false,
- "special": true
- },
- "32044": {
- "content": "<extra_id_55>",
- "lstrip": false,
- "normalized": false,
- "rstrip": false,
- "single_word": false,
- "special": true
- },
- "32045": {
- "content": "<extra_id_54>",
- "lstrip": false,
- "normalized": false,
- "rstrip": false,
- "single_word": false,
- "special": true
- },
- "32046": {
- "content": "<extra_id_53>",
- "lstrip": false,
- "normalized": false,
- "rstrip": false,
- "single_word": false,
- "special": true
- },
- "32047": {
- "content": "<extra_id_52>",
- "lstrip": false,
- "normalized": false,
- "rstrip": false,
- "single_word": false,
- "special": true
- },
- "32048": {
- "content": "<extra_id_51>",
- "lstrip": false,
- "normalized": false,
- "rstrip": false,
- "single_word": false,
- "special": true
- },
- "32049": {
- "content": "<extra_id_50>",
- "lstrip": false,
- "normalized": false,
- "rstrip": false,
- "single_word": false,
- "special": true
- },
- "32050": {
- "content": "<extra_id_49>",
- "lstrip": false,
- "normalized": false,
- "rstrip": false,
- "single_word": false,
- "special": true
- },
- "32051": {
- "content": "<extra_id_48>",
- "lstrip": false,
- "normalized": false,
- "rstrip": false,
- "single_word": false,
- "special": true
- },
- "32052": {
- "content": "<extra_id_47>",
- "lstrip": false,
- "normalized": false,
- "rstrip": false,
- "single_word": false,
- "special": true
- },
- "32053": {
- "content": "<extra_id_46>",
- "lstrip": false,
- "normalized": false,
- "rstrip": false,
- "single_word": false,
- "special": true
- },
- "32054": {
- "content": "<extra_id_45>",
- "lstrip": false,
- "normalized": false,
- "rstrip": false,
- "single_word": false,
- "special": true
- },
- "32055": {
- "content": "<extra_id_44>",
- "lstrip": false,
- "normalized": false,
- "rstrip": false,
- "single_word": false,
- "special": true
- },
- "32056": {
- "content": "<extra_id_43>",
- "lstrip": false,
- "normalized": false,
- "rstrip": false,
- "single_word": false,
- "special": true
- },
- "32057": {
- "content": "<extra_id_42>",
- "lstrip": false,
- "normalized": false,
- "rstrip": false,
- "single_word": false,
- "special": true
- },
- "32058": {
- "content": "<extra_id_41>",
- "lstrip": false,
- "normalized": false,
- "rstrip": false,
- "single_word": false,
- "special": true
- },
- "32059": {
- "content": "<extra_id_40>",
- "lstrip": false,
- "normalized": false,
- "rstrip": false,
- "single_word": false,
- "special": true
- },
- "32060": {
- "content": "<extra_id_39>",
- "lstrip": false,
- "normalized": false,
- "rstrip": false,
- "single_word": false,
- "special": true
- },
- "32061": {
- "content": "<extra_id_38>",
- "lstrip": false,
- "normalized": false,
- "rstrip": false,
- "single_word": false,
- "special": true
- },
- "32062": {
- "content": "<extra_id_37>",
- "lstrip": false,
- "normalized": false,
- "rstrip": false,
- "single_word": false,
- "special": true
- },
- "32063": {
- "content": "<extra_id_36>",
- "lstrip": false,
- "normalized": false,
- "rstrip": false,
- "single_word": false,
- "special": true
- },
- "32064": {
- "content": "<extra_id_35>",
- "lstrip": false,
- "normalized": false,
- "rstrip": false,
- "single_word": false,
- "special": true
- },
- "32065": {
- "content": "<extra_id_34>",
- "lstrip": false,
- "normalized": false,
- "rstrip": false,
- "single_word": false,
- "special": true
- },
- "32066": {
- "content": "<extra_id_33>",
- "lstrip": false,
- "normalized": false,
- "rstrip": false,
- "single_word": false,
- "special": true
- },
- "32067": {
- "content": "<extra_id_32>",
- "lstrip": false,
- "normalized": false,
- "rstrip": false,
- "single_word": false,
- "special": true
- },
- "32068": {
- "content": "<extra_id_31>",
- "lstrip": false,
- "normalized": false,
- "rstrip": false,
- "single_word": false,
- "special": true
- },
- "32069": {
- "content": "<extra_id_30>",
- "lstrip": false,
- "normalized": false,
- "rstrip": false,
- "single_word": false,
- "special": true
- },
- "32070": {
- "content": "<extra_id_29>",
- "lstrip": false,
- "normalized": false,
- "rstrip": false,
- "single_word": false,
- "special": true
- },
- "32071": {
- "content": "<extra_id_28>",
- "lstrip": false,
- "normalized": false,
- "rstrip": false,
- "single_word": false,
- "special": true
- },
- "32072": {
- "content": "<extra_id_27>",
- "lstrip": false,
- "normalized": false,
- "rstrip": false,
- "single_word": false,
- "special": true
- },
- "32073": {
- "content": "<extra_id_26>",
- "lstrip": false,
- "normalized": false,
- "rstrip": false,
- "single_word": false,
- "special": true
- },
- "32074": {
- "content": "<extra_id_25>",
- "lstrip": false,
- "normalized": false,
- "rstrip": false,
- "single_word": false,
- "special": true
- },
- "32075": {
- "content": "<extra_id_24>",
- "lstrip": false,
- "normalized": false,
- "rstrip": false,
- "single_word": false,
- "special": true
- },
- "32076": {
- "content": "<extra_id_23>",
- "lstrip": false,
- "normalized": false,
- "rstrip": false,
- "single_word": false,
- "special": true
- },
- "32077": {
- "content": "<extra_id_22>",
- "lstrip": false,
- "normalized": false,
- "rstrip": false,
- "single_word": false,
- "special": true
- },
- "32078": {
- "content": "<extra_id_21>",
- "lstrip": false,
- "normalized": false,
- "rstrip": false,
- "single_word": false,
- "special": true
- },
- "32079": {
- "content": "<extra_id_20>",
- "lstrip": false,
- "normalized": false,
- "rstrip": false,
- "single_word": false,
- "special": true
- },
- "32080": {
- "content": "<extra_id_19>",
- "lstrip": false,
- "normalized": false,
- "rstrip": false,
- "single_word": false,
- "special": true
- },
- "32081": {
- "content": "<extra_id_18>",
- "lstrip": false,
- "normalized": false,
- "rstrip": false,
- "single_word": false,
- "special": true
- },
- "32082": {
- "content": "<extra_id_17>",
- "lstrip": false,
- "normalized": false,
- "rstrip": false,
- "single_word": false,
- "special": true
- },
- "32083": {
- "content": "<extra_id_16>",
- "lstrip": false,
- "normalized": false,
- "rstrip": false,
- "single_word": false,
- "special": true
- },
- "32084": {
- "content": "<extra_id_15>",
- "lstrip": false,
- "normalized": false,
- "rstrip": false,
- "single_word": false,
- "special": true
- },
- "32085": {
- "content": "<extra_id_14>",
- "lstrip": false,
- "normalized": false,
- "rstrip": false,
- "single_word": false,
- "special": true
- },
- "32086": {
- "content": "<extra_id_13>",
- "lstrip": false,
- "normalized": false,
- "rstrip": false,
- "single_word": false,
- "special": true
- },
- "32087": {
- "content": "<extra_id_12>",
- "lstrip": false,
- "normalized": false,
- "rstrip": false,
- "single_word": false,
- "special": true
- },
- "32088": {
- "content": "<extra_id_11>",
- "lstrip": false,
- "normalized": false,
- "rstrip": false,
- "single_word": false,
- "special": true
- },
- "32089": {
- "content": "<extra_id_10>",
- "lstrip": false,
- "normalized": false,
- "rstrip": false,
- "single_word": false,
- "special": true
- },
- "32090": {
- "content": "<extra_id_9>",
- "lstrip": false,
- "normalized": false,
- "rstrip": false,
- "single_word": false,
- "special": true
- },
- "32091": {
- "content": "<extra_id_8>",
- "lstrip": false,
- "normalized": false,
- "rstrip": false,
- "single_word": false,
- "special": true
- },
- "32092": {
- "content": "<extra_id_7>",
- "lstrip": false,
- "normalized": false,
- "rstrip": false,
- "single_word": false,
- "special": true
- },
- "32093": {
- "content": "<extra_id_6>",
- "lstrip": false,
- "normalized": false,
- "rstrip": false,
- "single_word": false,
- "special": true
- },
- "32094": {
- "content": "<extra_id_5>",
- "lstrip": false,
- "normalized": false,
- "rstrip": false,
- "single_word": false,
- "special": true
- },
- "32095": {
- "content": "<extra_id_4>",
- "lstrip": false,
- "normalized": false,
- "rstrip": false,
- "single_word": false,
- "special": true
- },
- "32096": {
- "content": "<extra_id_3>",
- "lstrip": false,
- "normalized": false,
- "rstrip": false,
- "single_word": false,
- "special": true
- },
- "32097": {
- "content": "<extra_id_2>",
- "lstrip": false,
- "normalized": false,
- "rstrip": false,
- "single_word": false,
- "special": true
- },
- "32098": {
- "content": "<extra_id_1>",
- "lstrip": false,
- "normalized": false,
- "rstrip": false,
- "single_word": false,
- "special": true
- },
- "32099": {
- "content": "<extra_id_0>",
- "lstrip": false,
- "normalized": false,
- "rstrip": false,
- "single_word": false,
- "special": true
- }
- },
- "additional_special_tokens": [
- "<extra_id_99>",
- "<extra_id_98>",
- "<extra_id_97>",
- "<extra_id_96>",
- "<extra_id_95>",
- "<extra_id_94>",
- "<extra_id_93>",
- "<extra_id_92>",
- "<extra_id_91>",
- "<extra_id_90>",
- "<extra_id_89>",
- "<extra_id_88>",
- "<extra_id_87>",
- "<extra_id_86>",
- "<extra_id_85>",
- "<extra_id_84>",
- "<extra_id_83>",
- "<extra_id_82>",
- "<extra_id_81>",
- "<extra_id_80>",
- "<extra_id_79>",
- "<extra_id_78>",
- "<extra_id_77>",
- "<extra_id_76>",
- "<extra_id_75>",
- "<extra_id_74>",
- "<extra_id_73>",
- "<extra_id_72>",
- "<extra_id_71>",
- "<extra_id_70>",
- "<extra_id_69>",
- "<extra_id_68>",
- "<extra_id_67>",
- "<extra_id_66>",
- "<extra_id_65>",
- "<extra_id_64>",
- "<extra_id_63>",
- "<extra_id_62>",
- "<extra_id_61>",
- "<extra_id_60>",
- "<extra_id_59>",
- "<extra_id_58>",
- "<extra_id_57>",
- "<extra_id_56>",
- "<extra_id_55>",
- "<extra_id_54>",
- "<extra_id_53>",
- "<extra_id_52>",
- "<extra_id_51>",
- "<extra_id_50>",
- "<extra_id_49>",
- "<extra_id_48>",
- "<extra_id_47>",
- "<extra_id_46>",
- "<extra_id_45>",
- "<extra_id_44>",
- "<extra_id_43>",
- "<extra_id_42>",
- "<extra_id_41>",
- "<extra_id_40>",
- "<extra_id_39>",
- "<extra_id_38>",
- "<extra_id_37>",
- "<extra_id_36>",
- "<extra_id_35>",
- "<extra_id_34>",
- "<extra_id_33>",
- "<extra_id_32>",
- "<extra_id_31>",
- "<extra_id_30>",
- "<extra_id_29>",
- "<extra_id_28>",
- "<extra_id_27>",
- "<extra_id_26>",
- "<extra_id_25>",
- "<extra_id_24>",
- "<extra_id_23>",
- "<extra_id_22>",
- "<extra_id_21>",
- "<extra_id_20>",
- "<extra_id_19>",
- "<extra_id_18>",
- "<extra_id_17>",
- "<extra_id_16>",
- "<extra_id_15>",
- "<extra_id_14>",
- "<extra_id_13>",
- "<extra_id_12>",
- "<extra_id_11>",
- "<extra_id_10>",
- "<extra_id_9>",
- "<extra_id_8>",
- "<extra_id_7>",
- "<extra_id_6>",
- "<extra_id_5>",
- "<extra_id_4>",
- "<extra_id_3>",
- "<extra_id_2>",
- "<extra_id_1>",
- "<extra_id_0>"
- ],
- "bos_token": "<s>",
- "clean_up_tokenization_spaces": false,
- "eos_token": "</s>",
- "legacy": false,
- "model_max_length": 512,
- "pad_token": null,
- "padding_side": "right",
- "sp_model_kwargs": {},
- "spaces_between_special_tokens": false,
- "tokenizer_class": "LlamaTokenizer",
- "unk_token": "<unk>",
- "use_default_system_prompt": false
-}