Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

84.2 KB
Raw Blame
{
  "texts": [
    "Hello world",
    "",
    " ",
    "   ",
    "\t",
    "a\tb\nc",
    "café résumé",
    "注文をキャンセル",
    "مرحبا",
    "क्‌ष",
    "😀",
    "emoji 😀 here",
    "🇬🇧",
    "▁literal marker",
    "pre▁fix",
    " nbsp figure",
    "<0x41>",
    "a<0xFF>b",
    "\u0001\u0002",
    "",
    "󰀀",
    "I cannot log in and need a password reset.",
    "Esta mensagem e confidencial."
  ],
  "by": {
    "english": {
      "decoder": {
        "type": "ByteLevel"
      },
      "cases": [
        {
          "text": "Hello world",
          "ids": [
            12092,
            1533
          ],
          "decoded": "Hello world",
          "slices": [
            {
              "start": 0,
              "size": 1,
              "decoded": "Hello"
            },
            {
              "start": 1,
              "size": 1,
              "decoded": " world"
            },
            {
              "start": 0,
              "size": 2,
              "decoded": "Hello world"
            },
            {
              "start": 0,
              "size": 3,
              "decoded": "Hello world"
            },
            {
              "start": 0,
              "size": 5,
              "decoded": "Hello world"
            }
          ],
          "with_specials": {
            "ids": [
              50281,
              12092,
              1533,
              50282
            ],
            "skipping_specials": "Hello world",
            "kept": "[CLS]Hello world[SEP]"
          }
        },
        {
          "text": "",
          "ids": [],
          "decoded": "",
          "slices": [],
          "with_specials": {
            "ids": [
              50281,
              50282
            ],
            "skipping_specials": "",
            "kept": "[CLS][SEP]"
          }
        },
        {
          "text": " ",
          "ids": [
            209
          ],
          "decoded": " ",
          "slices": [
            {
              "start": 0,
              "size": 1,
              "decoded": " "
            },
            {
              "start": 0,
              "size": 2,
              "decoded": " "
            },
            {
              "start": 0,
              "size": 3,
              "decoded": " "
            },
            {
              "start": 0,
              "size": 5,
              "decoded": " "
            }
          ],
          "with_specials": {
            "ids": [
              50281,
              209,
              50282
            ],
            "skipping_specials": " ",
            "kept": "[CLS] [SEP]"
          }
        },
        {
          "text": "   ",
          "ids": [
            50275
          ],
          "decoded": "   ",
          "slices": [
            {
              "start": 0,
              "size": 1,
              "decoded": "   "
            },
            {
              "start": 0,
              "size": 2,
              "decoded": "   "
            },
            {
              "start": 0,
              "size": 3,
              "decoded": "   "
            },
            {
              "start": 0,
              "size": 5,
              "decoded": "   "
            }
          ],
          "with_specials": {
            "ids": [
              50281,
              50275,
              50282
            ],
            "skipping_specials": "   ",
            "kept": "[CLS]   [SEP]"
          }
        },
        {
          "text": "\t",
          "ids": [
            186
          ],
          "decoded": "\t",
          "slices": [
            {
              "start": 0,
              "size": 1,
              "decoded": "\t"
            },
            {
              "start": 0,
              "size": 2,
              "decoded": "\t"
            },
            {
              "start": 0,
              "size": 3,
              "decoded": "\t"
            },
            {
              "start": 0,
              "size": 5,
              "decoded": "\t"
            }
          ],
          "with_specials": {
            "ids": [
              50281,
              186,
              50282
            ],
            "skipping_specials": "\t",
            "kept": "[CLS]\t[SEP]"
          }
        },
        {
          "text": "a\tb\nc",
          "ids": [
            66,
            186,
            67,
            187,
            68
          ],
          "decoded": "a\tb\nc",
          "slices": [
            {
              "start": 0,
              "size": 1,
              "decoded": "a"
            },
            {
              "start": 1,
              "size": 1,
              "decoded": "\t"
            },
            {
              "start": 2,
              "size": 1,
              "decoded": "b"
            },
            {
              "start": 3,
              "size": 1,
              "decoded": "\n"
            },
            {
              "start": 4,
              "size": 1,
              "decoded": "c"
            },
            {
              "start": 0,
              "size": 2,
              "decoded": "a\t"
            },
            {
              "start": 2,
              "size": 2,
              "decoded": "b\n"
            },
            {
              "start": 4,
              "size": 2,
              "decoded": "c"
            },
            {
              "start": 0,
              "size": 3,
              "decoded": "a\tb"
            },
            {
              "start": 3,
              "size": 3,
              "decoded": "\nc"
            },
            {
              "start": 0,
              "size": 5,
              "decoded": "a\tb\nc"
            }
          ],
          "with_specials": {
            "ids": [
              50281,
              66,
              186,
              67,
              187,
              68,
              50282
            ],
            "skipping_specials": "a\tb\nc",
            "kept": "[CLS]a\tb\nc[SEP]"
          }
        },
        {
          "text": "café résumé",
          "ids": [
            68,
            2320,
            860,
            9568,
            2204,
            860
          ],
          "decoded": "café résumé",
          "slices": [
            {
              "start": 0,
              "size": 1,
              "decoded": "c"
            },
            {
              "start": 1,
              "size": 1,
              "decoded": "af"
            },
            {
              "start": 2,
              "size": 1,
              "decoded": "é"
            },
            {
              "start": 3,
              "size": 1,
              "decoded": " ré"
            },
            {
              "start": 4,
              "size": 1,
              "decoded": "sum"
            },
            {
              "start": 5,
              "size": 1,
              "decoded": "é"
            },
            {
              "start": 0,
              "size": 2,
              "decoded": "caf"
            },
            {
              "start": 2,
              "size": 2,
              "decoded": "é ré"
            },
            {
              "start": 4,
              "size": 2,
              "decoded": "sumé"
            },
            {
              "start": 0,
              "size": 3,
              "decoded": "café"
            },
            {
              "start": 3,
              "size": 3,
              "decoded": " résumé"
            },
            {
              "start": 0,
              "size": 5,
              "decoded": "café résum"
            },
            {
              "start": 5,
              "size": 5,
              "decoded": "é"
            }
          ],
          "with_specials": {
            "ids": [
              50281,
              68,
              2320,
              860,
              9568,
              2204,
              860,
              50282
            ],
            "skipping_specials": "café résumé",
            "kept": "[CLS]café résumé[SEP]"
          }
        },
        {
          "text": "注文をキャンセル",
          "ids": [
            44712,
            20855,
            6449,
            33548,
            41760,
            9467,
            41662,
            16509
          ],
          "decoded": "注文をキャンセル",
          "slices": [
            {
              "start": 0,
              "size": 1,
              "decoded": "注"
            },
            {
              "start": 1,
              "size": 1,
              "decoded": "文"
            },
            {
              "start": 2,
              "size": 1,
              "decoded": "を"
            },
            {
              "start": 3,
              "size": 1,
              "decoded": "キ"
            },
            {
              "start": 4,
              "size": 1,
              "decoded": "ャ"
            },
            {
              "start": 5,
              "size": 1,
              "decoded": "ン"
            },
            {
              "start": 6,
              "size": 1,
              "decoded": "セ"
            },
            {
              "start": 7,
              "size": 1,
              "decoded": "ル"
            },
            {
              "start": 0,
              "size": 2,
              "decoded": "注文"
            },
            {
              "start": 2,
              "size": 2,
              "decoded": "をキ"
            },
            {
              "start": 4,
              "size": 2,
              "decoded": "ャン"
            },
            {
              "start": 6,
              "size": 2,
              "decoded": "セル"
            },
            {
              "start": 0,
              "size": 3,
              "decoded": "注文を"
            },
            {
              "start": 3,
              "size": 3,
              "decoded": "キャン"
            },
            {
              "start": 6,
              "size": 3,
              "decoded": "セル"
            },
            {
              "start": 0,
              "size": 5,
              "decoded": "注文をキャ"
            },
            {
              "start": 5,
              "size": 5,
              "decoded": "ンセル"
            }
          ],
          "with_specials": {
            "ids": [
              50281,
              44712,
              20855,
              6449,
              33548,
              41760,
              9467,
              41662,
              16509,
              50282
            ],
            "skipping_specials": "注文をキャンセル",
            "kept": "[CLS]注文をキャンセル[SEP]"
          }
        },
        {
          "text": "مرحبا",
          "ids": [
            5843,
            6900,
            21931,
            13621,
            3142
          ],
          "decoded": "مرحبا",
          "slices": [
            {
              "start": 0,
              "size": 1,
              "decoded": "م"
            },
            {
              "start": 1,
              "size": 1,
              "decoded": "ر"
            },
            {
              "start": 2,
              "size": 1,
              "decoded": "ح"
            },
            {
              "start": 3,
              "size": 1,
              "decoded": "ب"
            },
            {
              "start": 4,
              "size": 1,
              "decoded": "ا"
            },
            {
              "start": 0,
              "size": 2,
              "decoded": "مر"
            },
            {
              "start": 2,
              "size": 2,
              "decoded": "حب"
            },
            {
              "start": 4,
              "size": 2,
              "decoded": "ا"
            },
            {
              "start": 0,
              "size": 3,
              "decoded": "مرح"
            },
            {
              "start": 3,
              "size": 3,
              "decoded": "با"
            },
            {
              "start": 0,
              "size": 5,
              "decoded": "مرحبا"
            }
          ],
          "with_specials": {
            "ids": [
              50281,
              5843,
              6900,
              21931,
              13621,
              3142,
              50282
            ],
            "skipping_specials": "مرحبا",
            "kept": "[CLS]مرحبا[SEP]"
          }
        },
        {
          "text": "क्‌ष",
          "ids": [
            24381,
            17665,
            49003,
            2483,
            117
          ],
          "decoded": "क्‌ष",
          "slices": [
            {
              "start": 0,
              "size": 1,
              "decoded": "क"
            },
            {
              "start": 1,
              "size": 1,
              "decoded": "्"
            },
            {
              "start": 2,
              "size": 1,
              "decoded": "‌"
            },
            {
              "start": 3,
              "size": 1,
              "decoded": "�"
            },
            {
              "start": 4,
              "size": 1,
              "decoded": "�"
            },
            {
              "start": 0,
              "size": 2,
              "decoded": "क्"
            },
            {
              "start": 2,
              "size": 2,
              "decoded": "‌�"
            },
            {
              "start": 4,
              "size": 2,
              "decoded": "�"
            },
            {
              "start": 0,
              "size": 3,
              "decoded": "क्‌"
            },
            {
              "start": 3,
              "size": 3,
              "decoded": "ष"
            },
            {
              "start": 0,
              "size": 5,
              "decoded": "क्‌ष"
            }
          ],
          "with_specials": {
            "ids": [
              50281,
              24381,
              17665,
              49003,
              2483,
              117,
              50282
            ],
            "skipping_specials": "क्‌ष",
            "kept": "[CLS]क्‌ष[SEP]"
          }
        },
        {
          "text": "😀",
          "ids": [
            14931,
            235,
            211
          ],
          "decoded": "😀",
          "slices": [
            {
              "start": 0,
              "size": 1,
              "decoded": "�"
            },
            {
              "start": 1,
              "size": 1,
              "decoded": "�"
            },
            {
              "start": 2,
              "size": 1,
              "decoded": "�"
            },
            {
              "start": 0,
              "size": 2,
              "decoded": "�"
            },
            {
              "start": 2,
              "size": 2,
              "decoded": "�"
            },
            {
              "start": 0,
              "size": 3,
              "decoded": "😀"
            },
            {
              "start": 0,
              "size": 5,
              "decoded": "😀"
            }
          ],
          "with_specials": {
            "ids": [
              50281,
              14931,
              235,
              211,
              50282
            ],
            "skipping_specials": "😀",
            "kept": "[CLS]😀[SEP]"
          }
        },
        {
          "text": "emoji 😀 here",
          "ids": [
            43208,
            8020,
            49042,
            211,
            1060
          ],
          "decoded": "emoji 😀 here",
          "slices": [
            {
              "start": 0,
              "size": 1,
              "decoded": "emo"
            },
            {
              "start": 1,
              "size": 1,
              "decoded": "ji"
            },
            {
              "start": 2,
              "size": 1,
              "decoded": " �"
            },
            {
              "start": 3,
              "size": 1,
              "decoded": "�"
            },
            {
              "start": 4,
              "size": 1,
              "decoded": " here"
            },
            {
              "start": 0,
              "size": 2,
              "decoded": "emoji"
            },
            {
              "start": 2,
              "size": 2,
              "decoded": " 😀"
            },
            {
              "start": 4,
              "size": 2,
              "decoded": " here"
            },
            {
              "start": 0,
              "size": 3,
              "decoded": "emoji �"
            },
            {
              "start": 3,
              "size": 3,
              "decoded": "� here"
            },
            {
              "start": 0,
              "size": 5,
              "decoded": "emoji 😀 here"
            }
          ],
          "with_specials": {
            "ids": [
              50281,
              43208,
              8020,
              49042,
              211,
              1060,
              50282
            ],
            "skipping_specials": "emoji 😀 here",
            "kept": "[CLS]emoji 😀 here[SEP]"
          }
        },
        {
          "text": "🇬🇧",
          "ids": [
            14931,
            218,
            107,
            14931,
            218,
            102
          ],
          "decoded": "🇬🇧",
          "slices": [
            {
              "start": 0,
              "size": 1,
              "decoded": "�"
            },
            {
              "start": 1,
              "size": 1,
              "decoded": "�"
            },
            {
              "start": 2,
              "size": 1,
              "decoded": "�"
            },
            {
              "start": 3,
              "size": 1,
              "decoded": "�"
            },
            {
              "start": 4,
              "size": 1,
              "decoded": "�"
            },
            {
              "start": 5,
              "size": 1,
              "decoded": "�"
            },
            {
              "start": 0,
              "size": 2,
              "decoded": "�"
            },
            {
              "start": 2,
              "size": 2,
              "decoded": "��"
            },
            {
              "start": 4,
              "size": 2,
              "decoded": "��"
            },
            {
              "start": 0,
              "size": 3,
              "decoded": "🇬"
            },
            {
              "start": 3,
              "size": 3,
              "decoded": "🇧"
            },
            {
              "start": 0,
              "size": 5,
              "decoded": "🇬�"
            },
            {
              "start": 5,
              "size": 5,
              "decoded": "�"
            }
          ],
          "with_specials": {
            "ids": [
              50281,
              14931,
              218,
              107,
              14931,
              218,
              102,
              50282
            ],
            "skipping_specials": "🇬🇧",
            "kept": "[CLS]🇬🇧[SEP]"
          }
        },
        {
          "text": "▁literal marker",
          "ids": [
            10897,
            212,
            34943,
            10705
          ],
          "decoded": "▁literal marker",
          "slices": [
            {
              "start": 0,
              "size": 1,
              "decoded": "�"
            },
            {
              "start": 1,
              "size": 1,
              "decoded": "�"
            },
            {
              "start": 2,
              "size": 1,
              "decoded": "literal"
            },
            {
              "start": 3,
              "size": 1,
              "decoded": " marker"
            },
            {
              "start": 0,
              "size": 2,
              "decoded": "▁"
            },
            {
              "start": 2,
              "size": 2,
              "decoded": "literal marker"
            },
            {
              "start": 0,
              "size": 3,
              "decoded": "▁literal"
            },
            {
              "start": 3,
              "size": 3,
              "decoded": " marker"
            },
            {
              "start": 0,
              "size": 5,
              "decoded": "▁literal marker"
            }
          ],
          "with_specials": {
            "ids": [
              50281,
              10897,
              212,
              34943,
              10705,
              50282
            ],
            "skipping_specials": "▁literal marker",
            "kept": "[CLS]▁literal marker[SEP]"
          }
        },
        {
          "text": "pre▁fix",
          "ids": [
            3456,
            10897,
            212,
            11097
          ],
          "decoded": "pre▁fix",
          "slices": [
            {
              "start": 0,
              "size": 1,
              "decoded": "pre"
            },
            {
              "start": 1,
              "size": 1,
              "decoded": "�"
            },
            {
              "start": 2,
              "size": 1,
              "decoded": "�"
            },
            {
              "start": 3,
              "size": 1,
              "decoded": "fix"
            },
            {
              "start": 0,
              "size": 2,
              "decoded": "pre�"
            },
            {
              "start": 2,
              "size": 2,
              "decoded": "�fix"
            },
            {
              "start": 0,
              "size": 3,
              "decoded": "pre▁"
            },
            {
              "start": 3,
              "size": 3,
              "decoded": "fix"
            },
            {
              "start": 0,
              "size": 5,
              "decoded": "pre▁fix"
            }
          ],
          "with_specials": {
            "ids": [
              50281,
              3456,
              10897,
              212,
              11097,
              50282
            ],
            "skipping_specials": "pre▁fix",
            "kept": "[CLS]pre▁fix[SEP]"
          }
        },
        {
          "text": " nbsp figure",
          "ids": [
            575,
            6732,
            325,
            218,
            13206
          ],
          "decoded": " nbsp figure",
          "slices": [
            {
              "start": 0,
              "size": 1,
              "decoded": " "
            },
            {
              "start": 1,
              "size": 1,
              "decoded": "nbsp"
            },
            {
              "start": 2,
              "size": 1,
              "decoded": "�"
            },
            {
              "start": 3,
              "size": 1,
              "decoded": "�"
            },
            {
              "start": 4,
              "size": 1,
              "decoded": "figure"
            },
            {
              "start": 0,
              "size": 2,
              "decoded": " nbsp"
            },
            {
              "start": 2,
              "size": 2,
              "decoded": " "
            },
            {
              "start": 4,
              "size": 2,
              "decoded": "figure"
            },
            {
              "start": 0,
              "size": 3,
              "decoded": " nbsp�"
            },
            {
              "start": 3,
              "size": 3,
              "decoded": "�figure"
            },
            {
              "start": 0,
              "size": 5,
              "decoded": " nbsp figure"
            }
          ],
          "with_specials": {
            "ids": [
              50281,
              575,
              6732,
              325,
              218,
              13206,
              50282
            ],
            "skipping_specials": " nbsp figure",
            "kept": "[CLS] nbsp figure[SEP]"
          }
        },
        {
          "text": "<0x41>",
          "ids": [
            29,
            17,
            89,
            3156,
            31
          ],
          "decoded": "<0x41>",
          "slices": [
            {
              "start": 0,
              "size": 1,
              "decoded": "<"
            },
            {
              "start": 1,
              "size": 1,
              "decoded": "0"
            },
            {
              "start": 2,
              "size": 1,
              "decoded": "x"
            },
            {
              "start": 3,
              "size": 1,
              "decoded": "41"
            },
            {
              "start": 4,
              "size": 1,
              "decoded": ">"
            },
            {
              "start": 0,
              "size": 2,
              "decoded": "<0"
            },
            {
              "start": 2,
              "size": 2,
              "decoded": "x41"
            },
            {
              "start": 4,
              "size": 2,
              "decoded": ">"
            },
            {
              "start": 0,
              "size": 3,
              "decoded": "<0x"
            },
            {
              "start": 3,
              "size": 3,
              "decoded": "41>"
            },
            {
              "start": 0,
              "size": 5,
              "decoded": "<0x41>"
            }
          ],
          "with_specials": {
            "ids": [
              50281,
              29,
              17,
              89,
              3156,
              31,
              50282
            ],
            "skipping_specials": "<0x41>",
            "kept": "[CLS]<0x41>[SEP]"
          }
        },
        {
          "text": "a<0xFF>b",
          "ids": [
            66,
            29,
            17,
            30962,
            31,
            67
          ],
          "decoded": "a<0xFF>b",
          "slices": [
            {
              "start": 0,
              "size": 1,
              "decoded": "a"
            },
            {
              "start": 1,
              "size": 1,
              "decoded": "<"
            },
            {
              "start": 2,
              "size": 1,
              "decoded": "0"
            },
            {
              "start": 3,
              "size": 1,
              "decoded": "xFF"
            },
            {
              "start": 4,
              "size": 1,
              "decoded": ">"
            },
            {
              "start": 5,
              "size": 1,
              "decoded": "b"
            },
            {
              "start": 0,
              "size": 2,
              "decoded": "a<"
            },
            {
              "start": 2,
              "size": 2,
              "decoded": "0xFF"
            },
            {
              "start": 4,
              "size": 2,
              "decoded": ">b"
            },
            {
              "start": 0,
              "size": 3,
              "decoded": "a<0"
            },
            {
              "start": 3,
              "size": 3,
              "decoded": "xFF>b"
            },
            {
              "start": 0,
              "size": 5,
              "decoded": "a<0xFF>"
            },
            {
              "start": 5,
              "size": 5,
              "decoded": "b"
            }
          ],
          "with_specials": {
            "ids": [
              50281,
              66,
              29,
              17,
              30962,
              31,
              67,
              50282
            ],
            "skipping_specials": "a<0xFF>b",
            "kept": "[CLS]a<0xFF>b[SEP]"
          }
        },
        {
          "text": "\u0001\u0002",
          "ids": [
            178,
            179
          ],
          "decoded": "\u0001\u0002",
          "slices": [
            {
              "start": 0,
              "size": 1,
              "decoded": "\u0001"
            },
            {
              "start": 1,
              "size": 1,
              "decoded": "\u0002"
            },
            {
              "start": 0,
              "size": 2,
              "decoded": "\u0001\u0002"
            },
            {
              "start": 0,
              "size": 3,
              "decoded": "\u0001\u0002"
            },
            {
              "start": 0,
              "size": 5,
              "decoded": "\u0001\u0002"
            }
          ],
          "with_specials": {
            "ids": [
              50281,
              178,
              179,
              50282
            ],
            "skipping_specials": "\u0001\u0002",
            "kept": "[CLS]\u0001\u0002[SEP]"
          }
        },
        {
          "text": "",
          "ids": [
            170,
            211,
            211
          ],
          "decoded": "",
          "slices": [
            {
              "start": 0,
              "size": 1,
              "decoded": "�"
            },
            {
              "start": 1,
              "size": 1,
              "decoded": "�"
            },
            {
              "start": 2,
              "size": 1,
              "decoded": "�"
            },
            {
              "start": 0,
              "size": 2,
              "decoded": "�"
            },
            {
              "start": 2,
              "size": 2,
              "decoded": "�"
            },
            {
              "start": 0,
              "size": 3,
              "decoded": ""
            },
            {
              "start": 0,
              "size": 5,
              "decoded": ""
            }
          ],
          "with_specials": {
            "ids": [
              50281,
              170,
              211,
              211,
              50282
            ],
            "skipping_specials": "",
            "kept": "[CLS][SEP]"
          }
        },
        {
          "text": "󰀀",
          "ids": [
            175,
            110,
            211,
            211
          ],
          "decoded": "󰀀",
          "slices": [
            {
              "start": 0,
              "size": 1,
              "decoded": "�"
            },
            {
              "start": 1,
              "size": 1,
              "decoded": "�"
            },
            {
              "start": 2,
              "size": 1,
              "decoded": "�"
            },
            {
              "start": 3,
              "size": 1,
              "decoded": "�"
            },
            {
              "start": 0,
              "size": 2,
              "decoded": "�"
            },
            {
              "start": 2,
              "size": 2,
              "decoded": "��"
            },
            {
              "start": 0,
              "size": 3,
              "decoded": "�"
            },
            {
              "start": 3,
              "size": 3,
              "decoded": "�"
            },
            {
              "start": 0,
              "size": 5,
              "decoded": "󰀀"
            }
          ],
          "with_specials": {
            "ids": [
              50281,
              175,
              110,
              211,
              211,
              50282
            ],
            "skipping_specials": "󰀀",
            "kept": "[CLS]󰀀[SEP]"
          }
        },
        {
          "text": "I cannot log in and need a password reset.",
          "ids": [
            42,
            2550,
            2412,
            275,
            285,
            878,
            247,
            9868,
            14932,
            15
          ],
          "decoded": "I cannot log in and need a password reset.",
          "slices": [
            {
              "start": 0,
              "size": 1,
              "decoded": "I"
            },
            {
              "start": 1,
              "size": 1,
              "decoded": " cannot"
            },
            {
              "start": 2,
              "size": 1,
              "decoded": " log"
            },
            {
              "start": 3,
              "size": 1,
              "decoded": " in"
            },
            {
              "start": 4,
              "size": 1,
              "decoded": " and"
            },
            {
              "start": 5,
              "size": 1,
              "decoded": " need"
            },
            {
              "start": 6,
              "size": 1,
              "decoded": " a"
            },
            {
              "start": 7,
              "size": 1,
              "decoded": " password"
            },
            {
              "start": 8,
              "size": 1,
              "decoded": " reset"
            },
            {
              "start": 9,
              "size": 1,
              "decoded": "."
            },
            {
              "start": 0,
              "size": 2,
              "decoded": "I cannot"
            },
            {
              "start": 2,
              "size": 2,
              "decoded": " log in"
            },
            {
              "start": 4,
              "size": 2,
              "decoded": " and need"
            },
            {
              "start": 6,
              "size": 2,
              "decoded": " a password"
            },
            {
              "start": 8,
              "size": 2,
              "decoded": " reset."
            },
            {
              "start": 0,
              "size": 3,
              "decoded": "I cannot log"
            },
            {
              "start": 3,
              "size": 3,
              "decoded": " in and need"
            },
            {
              "start": 6,
              "size": 3,
              "decoded": " a password reset"
            },
            {
              "start": 9,
              "size": 3,
              "decoded": "."
            },
            {
              "start": 0,
              "size": 5,
              "decoded": "I cannot log in and"
            },
            {
              "start": 5,
              "size": 5,
              "decoded": " need a password reset."
            }
          ],
          "with_specials": {
            "ids": [
              50281,
              42,
              2550,
              2412,
              275,
              285,
              878,
              247,
              9868,
              14932,
              15,
              50282
            ],
            "skipping_specials": "I cannot log in and need a password reset.",
            "kept": "[CLS]I cannot log in and need a password reset.[SEP]"
          }
        },
        {
          "text": "Esta mensagem e confidencial.",
          "ids": [
            38,
            18233,
            34939,
            31646,
            299,
            1461,
            301,
            49016,
            15
          ],
          "decoded": "Esta mensagem e confidencial.",
          "slices": [
            {
              "start": 0,
              "size": 1,
              "decoded": "E"
            },
            {
              "start": 1,
              "size": 1,
              "decoded": "sta"
            },
            {
              "start": 2,
              "size": 1,
              "decoded": " mens"
            },
            {
              "start": 3,
              "size": 1,
              "decoded": "agem"
            },
            {
              "start": 4,
              "size": 1,
              "decoded": " e"
            },
            {
              "start": 5,
              "size": 1,
              "decoded": " conf"
            },
            {
              "start": 6,
              "size": 1,
              "decoded": "id"
            },
            {
              "start": 7,
              "size": 1,
              "decoded": "encial"
            },
            {
              "start": 8,
              "size": 1,
              "decoded": "."
            },
            {
              "start": 0,
              "size": 2,
              "decoded": "Esta"
            },
            {
              "start": 2,
              "size": 2,
              "decoded": " mensagem"
            },
            {
              "start": 4,
              "size": 2,
              "decoded": " e conf"
            },
            {
              "start": 6,
              "size": 2,
              "decoded": "idencial"
            },
            {
              "start": 8,
              "size": 2,
              "decoded": "."
            },
            {
              "start": 0,
              "size": 3,
              "decoded": "Esta mens"
            },
            {
              "start": 3,
              "size": 3,
              "decoded": "agem e conf"
            },
            {
              "start": 6,
              "size": 3,
              "decoded": "idencial."
            },
            {
              "start": 0,
              "size": 5,
              "decoded": "Esta mensagem e"
            },
            {
              "start": 5,
              "size": 5,
              "decoded": " confidencial."
            }
          ],
          "with_specials": {
            "ids": [
              50281,
              38,
              18233,
              34939,
              31646,
              299,
              1461,
              301,
              49016,
              15,
              50282
            ],
            "skipping_specials": "Esta mensagem e confidencial.",
            "kept": "[CLS]Esta mensagem e confidencial.[SEP]"
          }
        }
      ]
    },
    "multilingual": {
      "decoder": {
        "type": "Sequence",
        "decoders": [
          {
            "type": "Replace"
          },
          {
            "type": "ByteFallback"
          },
          {
            "type": "Fuse"
          }
        ]
      },
      "cases": [
        {
          "text": "Hello world",
          "ids": [
            25957,
            2134
          ],
          "decoded": " Hello world",
          "slices": [
            {
              "start": 0,
              "size": 1,
              "decoded": " Hello"
            },
            {
              "start": 1,
              "size": 1,
              "decoded": " world"
            },
            {
              "start": 0,
              "size": 2,
              "decoded": " Hello world"
            },
            {
              "start": 0,
              "size": 3,
              "decoded": " Hello world"
            },
            {
              "start": 0,
              "size": 5,
              "decoded": " Hello world"
            }
          ],
          "with_specials": {
            "ids": [
              2,
              25957,
              2134,
              1
            ],
            "skipping_specials": " Hello world",
            "kept": "<bos> Hello world<eos>"
          }
        },
        {
          "text": "",
          "ids": [],
          "decoded": "",
          "slices": [],
          "with_specials": {
            "ids": [
              2,
              1
            ],
            "skipping_specials": "",
            "kept": "<bos><eos>"
          }
        },
        {
          "text": " ",
          "ids": [
            235248
          ],
          "decoded": " ",
          "slices": [
            {
              "start": 0,
              "size": 1,
              "decoded": " "
            },
            {
              "start": 0,
              "size": 2,
              "decoded": " "
            },
            {
              "start": 0,
              "size": 3,
              "decoded": " "
            },
            {
              "start": 0,
              "size": 5,
              "decoded": " "
            }
          ],
          "with_specials": {
            "ids": [
              2,
              235248,
              1
            ],
            "skipping_specials": " ",
            "kept": "<bos> <eos>"
          }
        },
        {
          "text": "   ",
          "ids": [
            235248,
            235248,
            235248
          ],
          "decoded": "   ",
          "slices": [
            {
              "start": 0,
              "size": 1,
              "decoded": " "
            },
            {
              "start": 1,
              "size": 1,
              "decoded": " "
            },
            {
              "start": 2,
              "size": 1,
              "decoded": " "
            },
            {
              "start": 0,
              "size": 2,
              "decoded": "  "
            },
            {
              "start": 2,
              "size": 2,
              "decoded": " "
            },
            {
              "start": 0,
              "size": 3,
              "decoded": "   "
            },
            {
              "start": 0,
              "size": 5,
              "decoded": "   "
            }
          ],
          "with_specials": {
            "ids": [
              2,
              235248,
              235248,
              235248,
              1
            ],
            "skipping_specials": "   ",
            "kept": "<bos>   <eos>"
          }
        },
        {
          "text": "\t",
          "ids": [
            235248,
            226
          ],
          "decoded": " \t",
          "slices": [
            {
              "start": 0,
              "size": 1,
              "decoded": " "
            },
            {
              "start": 1,
              "size": 1,
              "decoded": "\t"
            },
            {
              "start": 0,
              "size": 2,
              "decoded": " \t"
            },
            {
              "start": 0,
              "size": 3,
              "decoded": " \t"
            },
            {
              "start": 0,
              "size": 5,
              "decoded": " \t"
            }
          ],
          "with_specials": {
            "ids": [
              2,
              235248,
              226,
              1
            ],
            "skipping_specials": " \t",
            "kept": "<bos> \t<eos>"
          }
        },
        {
          "text": "a\tb\nc",
          "ids": [
            476,
            226,
            235268,
            108,
            498
          ],
          "decoded": " a\tb\n c",
          "slices": [
            {
              "start": 0,
              "size": 1,
              "decoded": " a"
            },
            {
              "start": 1,
              "size": 1,
              "decoded": "\t"
            },
            {
              "start": 2,
              "size": 1,
              "decoded": "b"
            },
            {
              "start": 3,
              "size": 1,
              "decoded": "\n"
            },
            {
              "start": 4,
              "size": 1,
              "decoded": " c"
            },
            {
              "start": 0,
              "size": 2,
              "decoded": " a\t"
            },
            {
              "start": 2,
              "size": 2,
              "decoded": "b\n"
            },
            {
              "start": 4,
              "size": 2,
              "decoded": " c"
            },
            {
              "start": 0,
              "size": 3,
              "decoded": " a\tb"
            },
            {
              "start": 3,
              "size": 3,
              "decoded": "\n c"
            },
            {
              "start": 0,
              "size": 5,
              "decoded": " a\tb\n c"
            }
          ],
          "with_specials": {
            "ids": [
              2,
              476,
              226,
              235268,
              108,
              498,
              1
            ],
            "skipping_specials": " a\tb\n c",
            "kept": "<bos> a\tb\n c<eos>"
          }
        },
        {
          "text": "café résumé",
          "ids": [
            18688,
            96537
          ],
          "decoded": " café résumé",
          "slices": [
            {
              "start": 0,
              "size": 1,
              "decoded": " café"
            },
            {
              "start": 1,
              "size": 1,
              "decoded": " résumé"
            },
            {
              "start": 0,
              "size": 2,
              "decoded": " café résumé"
            },
            {
              "start": 0,
              "size": 3,
              "decoded": " café résumé"
            },
            {
              "start": 0,
              "size": 5,
              "decoded": " café résumé"
            }
          ],
          "with_specials": {
            "ids": [
              2,
              18688,
              96537,
              1
            ],
            "skipping_specials": " café résumé",
            "kept": "<bos> café résumé<eos>"
          }
        },
        {
          "text": "注文をキャンセル",
          "ids": [
            231293,
            235432,
            182129
          ],
          "decoded": " 注文をキャンセル",
          "slices": [
            {
              "start": 0,
              "size": 1,
              "decoded": " 注文"
            },
            {
              "start": 1,
              "size": 1,
              "decoded": "を"
            },
            {
              "start": 2,
              "size": 1,
              "decoded": "キャンセル"
            },
            {
              "start": 0,
              "size": 2,
              "decoded": " 注文を"
            },
            {
              "start": 2,
              "size": 2,
              "decoded": "キャンセル"
            },
            {
              "start": 0,
              "size": 3,
              "decoded": " 注文をキャンセル"
            },
            {
              "start": 0,
              "size": 5,
              "decoded": " 注文をキャンセル"
            }
          ],
          "with_specials": {
            "ids": [
              2,
              231293,
              235432,
              182129,
              1
            ],
            "skipping_specials": " 注文をキャンセル",
            "kept": "<bos> 注文をキャンセル<eos>"
          }
        },
        {
          "text": "مرحبا",
          "ids": [
            89187,
            19833
          ],
          "decoded": " مرحبا",
          "slices": [
            {
              "start": 0,
              "size": 1,
              "decoded": " مرح"
            },
            {
              "start": 1,
              "size": 1,
              "decoded": "با"
            },
            {
              "start": 0,
              "size": 2,
              "decoded": " مرحبا"
            },
            {
              "start": 0,
              "size": 3,
              "decoded": " مرحبا"
            },
            {
              "start": 0,
              "size": 5,
              "decoded": " مرحبا"
            }
          ],
          "with_specials": {
            "ids": [
              2,
              89187,
              19833,
              1
            ],
            "skipping_specials": " مرحبا",
            "kept": "<bos> مرحبا<eos>"
          }
        },
        {
          "text": "क्‌ष",
          "ids": [
            2280,
            235485,
            235762,
            236355
          ],
          "decoded": " क्‌ष",
          "slices": [
            {
              "start": 0,
              "size": 1,
              "decoded": " क"
            },
            {
              "start": 1,
              "size": 1,
              "decoded": "्"
            },
            {
              "start": 2,
              "size": 1,
              "decoded": "‌"
            },
            {
              "start": 3,
              "size": 1,
              "decoded": "ष"
            },
            {
              "start": 0,
              "size": 2,
              "decoded": " क्"
            },
            {
              "start": 2,
              "size": 2,
              "decoded": "‌ष"
            },
            {
              "start": 0,
              "size": 3,
              "decoded": " क्‌"
            },
            {
              "start": 3,
              "size": 3,
              "decoded": "ष"
            },
            {
              "start": 0,
              "size": 5,
              "decoded": " क्‌ष"
            }
          ],
          "with_specials": {
            "ids": [
              2,
              2280,
              235485,
              235762,
              236355,
              1
            ],
            "skipping_specials": " क्‌ष",
            "kept": "<bos> क्‌ष<eos>"
          }
        },
        {
          "text": "😀",
          "ids": [
            79937
          ],
          "decoded": " 😀",
          "slices": [
            {
              "start": 0,
              "size": 1,
              "decoded": " 😀"
            },
            {
              "start": 0,
              "size": 2,
              "decoded": " 😀"
            },
            {
              "start": 0,
              "size": 3,
              "decoded": " 😀"
            },
            {
              "start": 0,
              "size": 5,
              "decoded": " 😀"
            }
          ],
          "with_specials": {
            "ids": [
              2,
              79937,
              1
            ],
            "skipping_specials": " 😀",
            "kept": "<bos> 😀<eos>"
          }
        },
        {
          "text": "emoji 😀 here",
          "ids": [
            52810,
            79937,
            1517
          ],
          "decoded": " emoji 😀 here",
          "slices": [
            {
              "start": 0,
              "size": 1,
              "decoded": " emoji"
            },
            {
              "start": 1,
              "size": 1,
              "decoded": " 😀"
            },
            {
              "start": 2,
              "size": 1,
              "decoded": " here"
            },
            {
              "start": 0,
              "size": 2,
              "decoded": " emoji 😀"
            },
            {
              "start": 2,
              "size": 2,
              "decoded": " here"
            },
            {
              "start": 0,
              "size": 3,
              "decoded": " emoji 😀 here"
            },
            {
              "start": 0,
              "size": 5,
              "decoded": " emoji 😀 here"
            }
          ],
          "with_specials": {
            "ids": [
              2,
              52810,
              79937,
              1517,
              1
            ],
            "skipping_specials": " emoji 😀 here",
            "kept": "<bos> emoji 😀 here<eos>"
          }
        },
        {
          "text": "🇬🇧",
          "ids": [
            235248,
            242330,
            241461
          ],
          "decoded": " 🇬🇧",
          "slices": [
            {
              "start": 0,
              "size": 1,
              "decoded": " "
            },
            {
              "start": 1,
              "size": 1,
              "decoded": "🇬"
            },
            {
              "start": 2,
              "size": 1,
              "decoded": "🇧"
            },
            {
              "start": 0,
              "size": 2,
              "decoded": " 🇬"
            },
            {
              "start": 2,
              "size": 2,
              "decoded": "🇧"
            },
            {
              "start": 0,
              "size": 3,
              "decoded": " 🇬🇧"
            },
            {
              "start": 0,
              "size": 5,
              "decoded": " 🇬🇧"
            }
          ],
          "with_specials": {
            "ids": [
              2,
              235248,
              242330,
              241461,
              1
            ],
            "skipping_specials": " 🇬🇧",
            "kept": "<bos> 🇬🇧<eos>"
          }
        },
        {
          "text": "▁literal marker",
          "ids": [
            36414,
            24416
          ],
          "decoded": " literal marker",
          "slices": [
            {
              "start": 0,
              "size": 1,
              "decoded": " literal"
            },
            {
              "start": 1,
              "size": 1,
              "decoded": " marker"
            },
            {
              "start": 0,
              "size": 2,
              "decoded": " literal marker"
            },
            {
              "start": 0,
              "size": 3,
              "decoded": " literal marker"
            },
            {
              "start": 0,
              "size": 5,
              "decoded": " literal marker"
            }
          ],
          "with_specials": {
            "ids": [
              2,
              36414,
              24416,
              1
            ],
            "skipping_specials": " literal marker",
            "kept": "<bos> literal marker<eos>"
          }
        },
        {
          "text": "pre▁fix",
          "ids": [
            953,
            6530
          ],
          "decoded": " pre fix",
          "slices": [
            {
              "start": 0,
              "size": 1,
              "decoded": " pre"
            },
            {
              "start": 1,
              "size": 1,
              "decoded": " fix"
            },
            {
              "start": 0,
              "size": 2,
              "decoded": " pre fix"
            },
            {
              "start": 0,
              "size": 3,
              "decoded": " pre fix"
            },
            {
              "start": 0,
              "size": 5,
              "decoded": " pre fix"
            }
          ],
          "with_specials": {
            "ids": [
              2,
              953,
              6530,
              1
            ],
            "skipping_specials": " pre fix",
            "kept": "<bos> pre fix<eos>"
          }
        },
        {
          "text": " nbsp figure",
          "ids": [
            84306,
            11450,
            243499,
            6270
          ],
          "decoded": "  nbsp figure",
          "slices": [
            {
              "start": 0,
              "size": 1,
              "decoded": "  "
            },
            {
              "start": 1,
              "size": 1,
              "decoded": "nbsp"
            },
            {
              "start": 2,
              "size": 1,
              "decoded": " "
            },
            {
              "start": 3,
              "size": 1,
              "decoded": "figure"
            },
            {
              "start": 0,
              "size": 2,
              "decoded": "  nbsp"
            },
            {
              "start": 2,
              "size": 2,
              "decoded": " figure"
            },
            {
              "start": 0,
              "size": 3,
              "decoded": "  nbsp "
            },
            {
              "start": 3,
              "size": 3,
              "decoded": "figure"
            },
            {
              "start": 0,
              "size": 5,
              "decoded": "  nbsp figure"
            }
          ],
          "with_specials": {
            "ids": [
              2,
              84306,
              11450,
              243499,
              6270,
              1
            ],
            "skipping_specials": "  nbsp figure",
            "kept": "<bos>  nbsp figure<eos>"
          }
        },
        {
          "text": "<0x41>",
          "ids": [
            968,
            235276,
            235297,
            235310,
            235274,
            235313
          ],
          "decoded": " <0x41>",
          "slices": [
            {
              "start": 0,
              "size": 1,
              "decoded": " <"
            },
            {
              "start": 1,
              "size": 1,
              "decoded": "0"
            },
            {
              "start": 2,
              "size": 1,
              "decoded": "x"
            },
            {
              "start": 3,
              "size": 1,
              "decoded": "4"
            },
            {
              "start": 4,
              "size": 1,
              "decoded": "1"
            },
            {
              "start": 5,
              "size": 1,
              "decoded": ">"
            },
            {
              "start": 0,
              "size": 2,
              "decoded": " <0"
            },
            {
              "start": 2,
              "size": 2,
              "decoded": "x4"
            },
            {
              "start": 4,
              "size": 2,
              "decoded": "1>"
            },
            {
              "start": 0,
              "size": 3,
              "decoded": " <0x"
            },
            {
              "start": 3,
              "size": 3,
              "decoded": "41>"
            },
            {
              "start": 0,
              "size": 5,
              "decoded": " <0x41"
            },
            {
              "start": 5,
              "size": 5,
              "decoded": ">"
            }
          ],
          "with_specials": {
            "ids": [
              2,
              968,
              235276,
              235297,
              235310,
              235274,
              235313,
              1
            ],
            "skipping_specials": " <0x41>",
            "kept": "<bos> <0x41><eos>"
          }
        },
        {
          "text": "a<0xFF>b",
          "ids": [
            476,
            235322,
            235276,
            16663,
            235313,
            235268
          ],
          "decoded": " a<0xFF>b",
          "slices": [
            {
              "start": 0,
              "size": 1,
              "decoded": " a"
            },
            {
              "start": 1,
              "size": 1,
              "decoded": "<"
            },
            {
              "start": 2,
              "size": 1,
              "decoded": "0"
            },
            {
              "start": 3,
              "size": 1,
              "decoded": "xFF"
            },
            {
              "start": 4,
              "size": 1,
              "decoded": ">"
            },
            {
              "start": 5,
              "size": 1,
              "decoded": "b"
            },
            {
              "start": 0,
              "size": 2,
              "decoded": " a<"
            },
            {
              "start": 2,
              "size": 2,
              "decoded": "0xFF"
            },
            {
              "start": 4,
              "size": 2,
              "decoded": ">b"
            },
            {
              "start": 0,
              "size": 3,
              "decoded": " a<0"
            },
            {
              "start": 3,
              "size": 3,
              "decoded": "xFF>b"
            },
            {
              "start": 0,
              "size": 5,
              "decoded": " a<0xFF>"
            },
            {
              "start": 5,
              "size": 5,
              "decoded": "b"
            }
          ],
          "with_specials": {
            "ids": [
              2,
              476,
              235322,
              235276,
              16663,
              235313,
              235268,
              1
            ],
            "skipping_specials": " a<0xFF>b",
            "kept": "<bos> a<0xFF>b<eos>"
          }
        },
        {
          "text": "\u0001\u0002",
          "ids": [
            235248,
            238213,
            247977
          ],
          "decoded": " \u0001\u0002",
          "slices": [
            {
              "start": 0,
              "size": 1,
              "decoded": " "
            },
            {
              "start": 1,
              "size": 1,
              "decoded": "\u0001"
            },
            {
              "start": 2,
              "size": 1,
              "decoded": "\u0002"
            },
            {
              "start": 0,
              "size": 2,
              "decoded": " \u0001"
            },
            {
              "start": 2,
              "size": 2,
              "decoded": "\u0002"
            },
            {
              "start": 0,
              "size": 3,
              "decoded": " \u0001\u0002"
            },
            {
              "start": 0,
              "size": 5,
              "decoded": " \u0001\u0002"
            }
          ],
          "with_specials": {
            "ids": [
              2,
              235248,
              238213,
              247977,
              1
            ],
            "skipping_specials": " \u0001\u0002",
            "kept": "<bos> \u0001\u0002<eos>"
          }
        },
        {
          "text": "",
          "ids": [
            235248,
            240539
          ],
          "decoded": " ",
          "slices": [
            {
              "start": 0,
              "size": 1,
              "decoded": " "
            },
            {
              "start": 1,
              "size": 1,
              "decoded": ""
            },
            {
              "start": 0,
              "size": 2,
              "decoded": " "
            },
            {
              "start": 0,
              "size": 3,
              "decoded": " "
            },
            {
              "start": 0,
              "size": 5,
              "decoded": " "
            }
          ],
          "with_specials": {
            "ids": [
              2,
              235248,
              240539,
              1
            ],
            "skipping_specials": " ",
            "kept": "<bos> <eos>"
          }
        },
        {
          "text": "󰀀",
          "ids": [
            235248,
            460,
            393,
            345,
            345
          ],
          "decoded": " 󰀀",
          "slices": [
            {
              "start": 0,
              "size": 1,
              "decoded": " "
            },
            {
              "start": 1,
              "size": 1,
              "decoded": "�"
            },
            {
              "start": 2,
              "size": 1,
              "decoded": "�"
            },
            {
              "start": 3,
              "size": 1,
              "decoded": "�"
            },
            {
              "start": 4,
              "size": 1,
              "decoded": "�"
            },
            {
              "start": 0,
              "size": 2,
              "decoded": " �"
            },
            {
              "start": 2,
              "size": 2,
              "decoded": "��"
            },
            {
              "start": 4,
              "size": 2,
              "decoded": "�"
            },
            {
              "start": 0,
              "size": 3,
              "decoded": " ��"
            },
            {
              "start": 3,
              "size": 3,
              "decoded": "��"
            },
            {
              "start": 0,
              "size": 5,
              "decoded": " 󰀀"
            }
          ],
          "with_specials": {
            "ids": [
              2,
              235248,
              460,
              393,
              345,
              345,
              1
            ],
            "skipping_specials": " 󰀀",
            "kept": "<bos> 󰀀<eos>"
          }
        },
        {
          "text": "I cannot log in and need a password reset.",
          "ids": [
            590,
            2952,
            3846,
            575,
            578,
            1476,
            476,
            8003,
            13846,
            235265
          ],
          "decoded": " I cannot log in and need a password reset.",
          "slices": [
            {
              "start": 0,
              "size": 1,
              "decoded": " I"
            },
            {
              "start": 1,
              "size": 1,
              "decoded": " cannot"
            },
            {
              "start": 2,
              "size": 1,
              "decoded": " log"
            },
            {
              "start": 3,
              "size": 1,
              "decoded": " in"
            },
            {
              "start": 4,
              "size": 1,
              "decoded": " and"
            },
            {
              "start": 5,
              "size": 1,
              "decoded": " need"
            },
            {
              "start": 6,
              "size": 1,
              "decoded": " a"
            },
            {
              "start": 7,
              "size": 1,
              "decoded": " password"
            },
            {
              "start": 8,
              "size": 1,
              "decoded": " reset"
            },
            {
              "start": 9,
              "size": 1,
              "decoded": "."
            },
            {
              "start": 0,
              "size": 2,
              "decoded": " I cannot"
            },
            {
              "start": 2,
              "size": 2,
              "decoded": " log in"
            },
            {
              "start": 4,
              "size": 2,
              "decoded": " and need"
            },
            {
              "start": 6,
              "size": 2,
              "decoded": " a password"
            },
            {
              "start": 8,
              "size": 2,
              "decoded": " reset."
            },
            {
              "start": 0,
              "size": 3,
              "decoded": " I cannot log"
            },
            {
              "start": 3,
              "size": 3,
              "decoded": " in and need"
            },
            {
              "start": 6,
              "size": 3,
              "decoded": " a password reset"
            },
            {
              "start": 9,
              "size": 3,
              "decoded": "."
            },
            {
              "start": 0,
              "size": 5,
              "decoded": " I cannot log in and"
            },
            {
              "start": 5,
              "size": 5,
              "decoded": " need a password reset."
            }
          ],
          "with_specials": {
            "ids": [
              2,
              590,
              2952,
              3846,
              575,
              578,
              1476,
              476,
              8003,
              13846,
              235265,
              1
            ],
            "skipping_specials": " I cannot log in and need a password reset.",
            "kept": "<bos> I cannot log in and need a password reset.<eos>"
          }
        },
        {
          "text": "Esta mensagem e confidencial.",
          "ids": [
            22974,
            70816,
            526,
            2103,
            45972,
            235265
          ],
          "decoded": " Esta mensagem e confidencial.",
          "slices": [
            {
              "start": 0,
              "size": 1,
              "decoded": " Esta"
            },
            {
              "start": 1,
              "size": 1,
              "decoded": " mensagem"
            },
            {
              "start": 2,
              "size": 1,
              "decoded": " e"
            },
            {
              "start": 3,
              "size": 1,
              "decoded": " conf"
            },
            {
              "start": 4,
              "size": 1,
              "decoded": "idencial"
            },
            {
              "start": 5,
              "size": 1,
              "decoded": "."
            },
            {
              "start": 0,
              "size": 2,
              "decoded": " Esta mensagem"
            },
            {
              "start": 2,
              "size": 2,
              "decoded": " e conf"
            },
            {
              "start": 4,
              "size": 2,
              "decoded": "idencial."
            },
            {
              "start": 0,
              "size": 3,
              "decoded": " Esta mensagem e"
            },
            {
              "start": 3,
              "size": 3,
              "decoded": " confidencial."
            },
            {
              "start": 0,
              "size": 5,
              "decoded": " Esta mensagem e confidencial"
            },
            {
              "start": 5,
              "size": 5,
              "decoded": "."
            }
          ],
          "with_specials": {
            "ids": [
              2,
              22974,
              70816,
              526,
              2103,
              45972,
              235265,
              1
            ],
            "skipping_specials": " Esta mensagem e confidencial.",
            "kept": "<bos> Esta mensagem e confidencial.<eos>"
          }
        }
      ]
    }
  },
  "mixed_token": {
    "spec": {
      "version": "1.0",
      "truncation": null,
      "padding": null,
      "added_tokens": [],
      "normalizer": null,
      "pre_tokenizer": {
        "type": "ByteLevel",
        "add_prefix_space": true,
        "trim_offsets": true,
        "use_regex": true
      },
      "post_processor": null,
      "decoder": {
        "type": "ByteLevel",
        "add_prefix_space": true,
        "trim_offsets": true,
        "use_regex": true
      },
      "model": {
        "type": "BPE",
        "dropout": null,
        "unk_token": null,
        "continuing_subword_prefix": null,
        "end_of_word_suffix": null,
        "fuse_unk": false,
        "byte_fallback": false,
        "ignore_merges": true,
        "vocab": {
          "!": 0,
          "\"": 1,
          "#": 2,
          "$": 3,
          "%": 4,
          "&": 5,
          "'": 6,
          "(": 7,
          ")": 8,
          "*": 9,
          "+": 10,
          ",": 11,
          "-": 12,
          ".": 13,
          "/": 14,
          "0": 15,
          "1": 16,
          "2": 17,
          "3": 18,
          "4": 19,
          "5": 20,
          "6": 21,
          "7": 22,
          "8": 23,
          "9": 24,
          ":": 25,
          ";": 26,
          "<": 27,
          "=": 28,
          ">": 29,
          "?": 30,
          "@": 31,
          "A": 32,
          "B": 33,
          "C": 34,
          "D": 35,
          "E": 36,
          "F": 37,
          "G": 38,
          "H": 39,
          "I": 40,
          "J": 41,
          "K": 42,
          "L": 43,
          "M": 44,
          "N": 45,
          "O": 46,
          "P": 47,
          "Q": 48,
          "R": 49,
          "S": 50,
          "T": 51,
          "U": 52,
          "V": 53,
          "W": 54,
          "X": 55,
          "Y": 56,
          "Z": 57,
          "[": 58,
          "\\": 59,
          "]": 60,
          "^": 61,
          "_": 62,
          "`": 63,
          "a": 64,
          "b": 65,
          "c": 66,
          "d": 67,
          "e": 68,
          "f": 69,
          "g": 70,
          "h": 71,
          "i": 72,
          "j": 73,
          "k": 74,
          "l": 75,
          "m": 76,
          "n": 77,
          "o": 78,
          "p": 79,
          "q": 80,
          "r": 81,
          "s": 82,
          "t": 83,
          "u": 84,
          "v": 85,
          "w": 86,
          "x": 87,
          "y": 88,
          "z": 89,
          "{": 90,
          "|": 91,
          "}": 92,
          "~": 93,
          "¡": 94,
          "¢": 95,
          "£": 96,
          "¤": 97,
          "¥": 98,
          "¦": 99,
          "§": 100,
          "¨": 101,
          "©": 102,
          "ª": 103,
          "«": 104,
          "¬": 105,
          "®": 106,
          "¯": 107,
          "°": 108,
          "±": 109,
          "²": 110,
          "³": 111,
          "´": 112,
          "µ": 113,
          "¶": 114,
          "·": 115,
          "¸": 116,
          "¹": 117,
          "º": 118,
          "»": 119,
          "¼": 120,
          "½": 121,
          "¾": 122,
          "¿": 123,
          "À": 124,
          "Á": 125,
          "Â": 126,
          "Ã": 127,
          "Ä": 128,
          "Å": 129,
          "Æ": 130,
          "Ç": 131,
          "È": 132,
          "É": 133,
          "Ê": 134,
          "Ë": 135,
          "Ì": 136,
          "Í": 137,
          "Î": 138,
          "Ï": 139,
          "Ð": 140,
          "Ñ": 141,
          "Ò": 142,
          "Ó": 143,
          "Ô": 144,
          "Õ": 145,
          "Ö": 146,
          "×": 147,
          "Ø": 148,
          "Ù": 149,
          "Ú": 150,
          "Û": 151,
          "Ü": 152,
          "Ý": 153,
          "Þ": 154,
          "ß": 155,
          "à": 156,
          "á": 157,
          "â": 158,
          "ã": 159,
          "ä": 160,
          "å": 161,
          "æ": 162,
          "ç": 163,
          "è": 164,
          "é": 165,
          "ê": 166,
          "ë": 167,
          "ì": 168,
          "í": 169,
          "î": 170,
          "ï": 171,
          "ð": 172,
          "ñ": 173,
          "ò": 174,
          "ó": 175,
          "ô": 176,
          "õ": 177,
          "ö": 178,
          "÷": 179,
          "ø": 180,
          "ù": 181,
          "ú": 182,
          "û": 183,
          "ü": 184,
          "ý": 185,
          "þ": 186,
          "ÿ": 187,
          "Ā": 188,
          "ā": 189,
          "Ă": 190,
          "ă": 191,
          "Ą": 192,
          "ą": 193,
          "Ć": 194,
          "ć": 195,
          "Ĉ": 196,
          "ĉ": 197,
          "Ċ": 198,
          "ċ": 199,
          "Č": 200,
          "č": 201,
          "Ď": 202,
          "ď": 203,
          "Đ": 204,
          "đ": 205,
          "Ē": 206,
          "ē": 207,
          "Ĕ": 208,
          "ĕ": 209,
          "Ė": 210,
          "ė": 211,
          "Ę": 212,
          "ę": 213,
          "Ě": 214,
          "ě": 215,
          "Ĝ": 216,
          "ĝ": 217,
          "Ğ": 218,
          "ğ": 219,
          "Ġ": 220,
          "ġ": 221,
          "Ģ": 222,
          "ģ": 223,
          "Ĥ": 224,
          "ĥ": 225,
          "Ħ": 226,
          "ħ": 227,
          "Ĩ": 228,
          "ĩ": 229,
          "Ī": 230,
          "ī": 231,
          "Ĭ": 232,
          "ĭ": 233,
          "Į": 234,
          "į": 235,
          "İ": 236,
          "ı": 237,
          "IJ": 238,
          "ij": 239,
          "Ĵ": 240,
          "ĵ": 241,
          "Ķ": 242,
          "ķ": 243,
          "ĸ": 244,
          "Ĺ": 245,
          "ĺ": 246,
          "Ļ": 247,
          "ļ": 248,
          "Ľ": 249,
          "ľ": 250,
          "Ŀ": 251,
          "ŀ": 252,
          "Ł": 253,
          "ł": 254,
          "Ń": 255,
          "Ġ ": 256,
          "Ġhi": 257,
          "   ": 258
        },
        "merges": []
      }
    },
    "separating_token": "Ġ ",
    "scope": "token",
    "cases": [
      {
        "why": "the mixed token alone: the whole point",
        "ids": [
          256
        ],
        "decoded": "Ġ "
      },
      {
        "why": "entirely in the alphabet",
        "ids": [
          257
        ],
        "decoded": " hi"
      },
      {
        "why": "entirely literal",
        "ids": [
          258
        ],
        "decoded": "   "
      },
      {
        "why": "a mixed token beside a plain one",
        "ids": [
          256,
          257
        ],
        "decoded": "Ġ  hi"
      }
    ]
  },
  "synthetic_decoders": {
    "strip": {
      "why": "Strip, which no shipped checkpoint declares",
      "spec": {
        "version": "1.0",
        "truncation": null,
        "padding": null,
        "added_tokens": [],
        "normalizer": null,
        "pre_tokenizer": null,
        "post_processor": null,
        "model": {
          "type": "BPE",
          "dropout": null,
          "unk_token": null,
          "continuing_subword_prefix": null,
          "end_of_word_suffix": null,
          "fuse_unk": false,
          "byte_fallback": true,
          "ignore_merges": true,
          "vocab": {
            "  hello ": 0,
            " x": 1,
            "  ": 2,
            "no": 3
          },
          "merges": []
        },
        "decoder": {
          "type": "Strip",
          "content": " ",
          "start": 2,
          "stop": 1
        }
      },
      "crate_panics": [
        [
          2
        ],
        [
          2,
          2
        ]
      ],
      "cases": [
        {
          "ids": [
            0
          ],
          "decoded": "hello"
        },
        {
          "ids": [
            1
          ],
          "decoded": "x"
        },
        {
          "ids": [
            3
          ],
          "decoded": "no"
        },
        {
          "ids": [
            0,
            3
          ],
          "decoded": "hellono"
        }
      ]
    },
    "bytefallback_hex_case": {
      "why": "which byte-token spellings the crate reads as a byte",
      "spec": {
        "version": "1.0",
        "truncation": null,
        "padding": null,
        "added_tokens": [],
        "normalizer": null,
        "pre_tokenizer": null,
        "post_processor": null,
        "model": {
          "type": "BPE",
          "dropout": null,
          "unk_token": null,
          "continuing_subword_prefix": null,
          "end_of_word_suffix": null,
          "fuse_unk": false,
          "byte_fallback": true,
          "ignore_merges": true,
          "vocab": {
            "<0xFF>": 0,
            "<0xff>": 1,
            "<0x41>": 2,
            "<0x61>": 3,
            "a": 4,
            "<0x+5>": 5,
            "<0x 5>": 6,
            "<0XFF>": 7,
            "<0xfF>": 8,
            "<0xF3>": 9,
            "<0xB0>": 10,
            "<0x80>": 11
          },
          "merges": []
        },
        "decoder": {
          "type": "ByteFallback"
        }
      },
      "cases": [
        {
          "ids": [
            2
          ],
          "decoded": "A"
        },
        {
          "ids": [
            3
          ],
          "decoded": "a"
        },
        {
          "ids": [
            1
          ],
          "decoded": "�"
        },
        {
          "ids": [
            0
          ],
          "decoded": "�"
        },
        {
          "ids": [
            2,
            3
          ],
          "decoded": "Aa"
        },
        {
          "ids": [
            0,
            0
          ],
          "decoded": "��"
        },
        {
          "ids": [
            1,
            4
          ],
          "decoded": "�a"
        },
        {
          "ids": [
            5
          ],
          "decoded": "\u0005"
        },
        {
          "ids": [
            6
          ],
          "decoded": "<0x 5>"
        },
        {
          "ids": [
            7
          ],
          "decoded": "<0XFF>"
        },
        {
          "ids": [
            8
          ],
          "decoded": "�"
        },
        {
          "ids": [
            5,
            4
          ],
          "decoded": "\u0005a"
        },
        {
          "ids": [
            6,
            4
          ],
          "decoded": "<0x 5>a"
        },
        {
          "ids": [
            7,
            4
          ],
          "decoded": "<0XFF>a"
        },
        {
          "ids": [
            9,
            10
          ],
          "decoded": "��"
        },
        {
          "ids": [
            9,
            10,
            11
          ],
          "decoded": "���"
        },
        {
          "ids": [
            9,
            10,
            11,
            11
          ],
          "decoded": "󰀀"
        },
        {
          "ids": [
            4,
            9,
            10,
            4
          ],
          "decoded": "a��a"
        }
      ]
    },
    "no_decoder": {
      "why": "no decoder section: the separator the crate falls back to",
      "spec": {
        "version": "1.0",
        "truncation": null,
        "padding": null,
        "added_tokens": [],
        "normalizer": null,
        "pre_tokenizer": null,
        "post_processor": null,
        "model": {
          "type": "BPE",
          "dropout": null,
          "unk_token": null,
          "continuing_subword_prefix": null,
          "end_of_word_suffix": null,
          "fuse_unk": false,
          "byte_fallback": true,
          "ignore_merges": true,
          "vocab": {
            "ab": 0,
            "cd": 1,
            "ef": 2,
            " ": 3
          },
          "merges": []
        }
      },
      "cases": [
        {
          "ids": [
            0
          ],
          "decoded": "ab"
        },
        {
          "ids": [
            0,
            1
          ],
          "decoded": "ab cd"
        },
        {
          "ids": [
            0,
            1,
            2
          ],
          "decoded": "ab cd ef"
        },
        {
          "ids": [
            0,
            3,
            1
          ],
          "decoded": "ab   cd"
        },
        {
          "ids": [
            3,
            3
          ],
          "decoded": "   "
        }
      ]
    },
    "nested_sequence": {
      "why": "a Sequence nested in a Sequence keeps its pieces",
      "spec": {
        "version": "1.0",
        "truncation": null,
        "padding": null,
        "added_tokens": [],
        "normalizer": null,
        "pre_tokenizer": null,
        "post_processor": null,
        "model": {
          "type": "BPE",
          "dropout": null,
          "unk_token": null,
          "continuing_subword_prefix": null,
          "end_of_word_suffix": null,
          "fuse_unk": false,
          "byte_fallback": true,
          "ignore_merges": true,
          "vocab": {
            "▁Hello": 0,
            "▁world": 1,
            "▁": 2,
            "▁a": 3
          },
          "merges": []
        },
        "decoder": {
          "type": "Sequence",
          "decoders": [
            {
              "type": "Sequence",
              "decoders": [
                {
                  "type": "Replace",
                  "pattern": {
                    "String": "▁"
                  },
                  "content": " "
                },
                {
                  "type": "ByteFallback"
                }
              ]
            },
            {
              "type": "Strip",
              "content": " ",
              "start": 1,
              "stop": 0
            }
          ]
        }
      },
      "cases": [
        {
          "ids": [
            0,
            1
          ],
          "decoded": "Helloworld"
        },
        {
          "ids": [
            2,
            2,
            0
          ],
          "decoded": "Hello"
        },
        {
          "ids": [
            2,
            3,
            2
          ],
          "decoded": "a"
        },
        {
          "ids": [
            0
          ],
          "decoded": "Hello"
        },
        {
          "ids": [
            3
          ],
          "decoded": "a"
        }
      ]
    },
    "bytefallback_piece_boundaries": {
      "why": "a failed byte run is one piece per byte, not one piece of N",
      "spec": {
        "version": "1.0",
        "truncation": null,
        "padding": null,
        "added_tokens": [],
        "normalizer": null,
        "pre_tokenizer": null,
        "post_processor": null,
        "model": {
          "type": "BPE",
          "dropout": null,
          "unk_token": null,
          "continuing_subword_prefix": null,
          "end_of_word_suffix": null,
          "fuse_unk": false,
          "byte_fallback": true,
          "ignore_merges": true,
          "vocab": {
            "<0xF3>": 0,
            "<0xB0>": 1,
            "<0x80>": 2,
            "hi": 3
          },
          "merges": []
        },
        "decoder": {
          "type": "Sequence",
          "decoders": [
            {
              "type": "ByteFallback"
            },
            {
              "type": "Replace",
              "pattern": {
                "String": "��"
              },
              "content": "X"
            }
          ]
        }
      },
      "cases": [
        {
          "ids": [
            0,
            1
          ],
          "decoded": "��"
        },
        {
          "ids": [
            0,
            1,
            2
          ],
          "decoded": "���"
        },
        {
          "ids": [
            3,
            0,
            1,
            3
          ],
          "decoded": "hi��hi"
        },
        {
          "ids": [
            0,
            1,
            2,
            2
          ],
          "decoded": "󰀀"
        },
        {
          "ids": [
            3
          ],
          "decoded": "hi"
        }
      ]
    }
  }
}