Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

feat(ts): decode official tokenizers for windowed prediction

wow476562743e38
diff --git a/laya-ts/fixtures/tokenizer-decode.json b/laya-ts/fixtures/tokenizer-decode.json
new file mode 100644
index 00000000..6793f1b2
--- /dev/null
+++ b/laya-ts/fixtures/tokenizer-decode.json
@@ -0,0 +1,1668 @@
+{
+ "reference": {
+ "repository": "convaiinnovations/laya",
+ "revision": "55cf4c4ebb4ebe31b2550e8bdf3bd21b99753851",
+ "tokenizers": "0.23.2",
+ "files": {
+ "tokenizer/tokenizer.json": "6c8aaa9a542084f2457eab775d4eeb51f92a70c0fd9de28d5edb0ddec3c08d30",
+ "multilingual/tokenizer/tokenizer.json": "609d8f4c067cd3950f88594c5a802616cea245823836ef5848ee4fc40aab5b6f"
+ }
+ },
+ "tokenizers": {
+ "english": {
+ "tokenizer": {
+ "version": "1.0",
+ "truncation": null,
+ "padding": null,
+ "added_tokens": [
+ {
+ "id": 50276,
+ "content": " ",
+ "single_word": false,
+ "lstrip": false,
+ "rstrip": false,
+ "normalized": true,
+ "special": false
+ },
+ {
+ "id": 50281,
+ "content": "[CLS]",
+ "single_word": false,
+ "lstrip": false,
+ "rstrip": false,
+ "normalized": false,
+ "special": true
+ },
+ {
+ "id": 50282,
+ "content": "[SEP]",
+ "single_word": false,
+ "lstrip": false,
+ "rstrip": false,
+ "normalized": false,
+ "special": true
+ },
+ {
+ "id": 50284,
+ "content": "[MASK]",
+ "single_word": false,
+ "lstrip": true,
+ "rstrip": false,
+ "normalized": false,
+ "special": true
+ }
+ ],
+ "normalizer": {
+ "type": "NFC"
+ },
+ "pre_tokenizer": {
+ "type": "ByteLevel",
+ "add_prefix_space": false,
+ "trim_offsets": true,
+ "use_regex": true
+ },
+ "post_processor": null,
+ "decoder": {
+ "type": "ByteLevel",
+ "add_prefix_space": false,
+ "trim_offsets": true,
+ "use_regex": true
+ },
+ "model": {
+ "type": "BPE",
+ "dropout": null,
+ "unk_token": null,
+ "continuing_subword_prefix": null,
+ "end_of_word_suffix": null,
+ "fuse_unk": false,
+ "byte_fallback": false,
+ "ignore_merges": false,
+ "vocab": {
+ "H": 41,
+ "a": 66,
+ "b": 67,
+ "c": 68,
+ "d": 69,
+ "e": 70,
+ "f": 71,
+ "h": 73,
+ "l": 77,
+ "o": 80,
+ "r": 83,
+ "w": 88,
+ "¢": 97,
+ "©": 104,
+ "«": 106,
+ "¬": 107,
+ "®": 108,
+ "·": 117,
+ "»": 121,
+ "½": 123,
+ "¾": 124,
+ "¿": 125,
+ "Ã": 127,
+ "å": 161,
+ "æ": 162,
+ "é": 165,
+ "ï": 171,
+ "ð": 172,
+ "Ā": 177,
+ "ĉ": 186,
+ "Ċ": 187,
+ "Ġ": 209,
+ "Ģ": 211,
+ "ģ": 212,
+ "Ĥ": 213,
+ "Ī": 219,
+ "ĺ": 235,
+ "Ł": 242,
+ "ł": 243,
+ "he": 248,
+ "Ġc": 260,
+ "or": 263,
+ "el": 293,
+ "Ġhe": 344,
+ "ld": 392,
+ "ell": 437,
+ "ll": 620,
+ "é": 860,
+ "af": 2320,
+ "orld": 2875,
+ "Ġé": 3974,
+ "ello": 6646,
+ "å®": 6916,
+ "Ġhell": 7085,
+ "æĪ": 8650,
+ "world": 10186,
+ "éĢ": 11016,
+ "Hello": 12092,
+ "ðŁ": 14931,
+ "Ġcaf": 15660,
+ "Ġï": 16141,
+ "¿½": 19675,
+ "�": 21478,
+ "ĠðŁ": 22692,
+ "Ġhello": 23120,
+ "»¿": 24384,
+ "æ¬": 25715,
+ "Ġcafé": 36241,
+ "": 41497,
+ "��": 42481,
+ "æĪ·": 46136,
+ "ĠðŁĺ": 49042,
+ " ": 50276,
+ "[CLS]": 50281,
+ "[SEP]": 50282,
+ "[MASK]": 50284
+ },
+ "merges": [
+ [
+ "h",
+ "e"
+ ],
+ [
+ "Ġ",
+ "c"
+ ],
+ [
+ "o",
+ "r"
+ ],
+ [
+ "e",
+ "l"
+ ],
+ [
+ "Ġ",
+ "he"
+ ],
+ [
+ "l",
+ "d"
+ ],
+ [
+ "el",
+ "l"
+ ],
+ [
+ "l",
+ "l"
+ ],
+ [
+ "Ã",
+ "©"
+ ],
+ [
+ "a",
+ "f"
+ ],
+ [
+ "or",
+ "ld"
+ ],
+ [
+ "Ġ",
+ "é"
+ ],
+ [
+ "ell",
+ "o"
+ ],
+ [
+ "å",
+ "®"
+ ],
+ [
+ "Ġhe",
+ "ll"
+ ],
+ [
+ "æ",
+ "Ī"
+ ],
+ [
+ "w",
+ "orld"
+ ],
+ [
+ "é",
+ "Ģ"
+ ],
+ [
+ "H",
+ "ello"
+ ],
+ [
+ "ð",
+ "Ł"
+ ],
+ [
+ "Ġc",
+ "af"
+ ],
+ [
+ "Ġ",
+ "ï"
+ ],
+ [
+ "¿",
+ "½"
+ ],
+ [
+ "ï",
+ "¿½"
+ ],
+ [
+ "Ġ",
+ "ðŁ"
+ ],
+ [
+ "Ġhell",
+ "o"
+ ],
+ [
+ "»",
+ "¿"
+ ],
+ [
+ "æ",
+ "¬"
+ ],
+ [
+ "Ġcaf",
+ "é"
+ ],
+ [
+ "ï",
+ "»¿"
+ ],
+ [
+ "�",
+ "�"
+ ],
+ [
+ "æĪ",
+ "·"
+ ],
+ [
+ "ĠðŁ",
+ "ĺ"
+ ]
+ ]
+ }
+ },
+ "cases": [
+ {
+ "name": "empty",
+ "ids": [],
+ "decoded": "",
+ "reencoded": []
+ },
+ {
+ "name": "whitespace",
+ "ids": [
+ 50276,
+ 12092,
+ 187,
+ 186,
+ 10186,
+ 50276
+ ],
+ "decoded": " Hello\n\tworld ",
+ "reencoded": [
+ 50276,
+ 12092,
+ 187,
+ 186,
+ 10186,
+ 50276
+ ]
+ },
+ {
+ "name": "unicode",
+ "ids": [
+ 6916,
+ 97,
+ 46136,
+ 11016,
+ 211,
+ 25715,
+ 124,
+ 36241,
+ 3974,
+ 49042,
+ 211,
+ 14931,
+ 106,
+ 243,
+ 209,
+ 172,
+ 243,
+ 211,
+ 211,
+ 172,
+ 243,
+ 211,
+ 212
+ ],
+ "decoded": "客户退款 café é 😀🫠 𠀀𠀁",
+ "reencoded": [
+ 6916,
+ 97,
+ 46136,
+ 11016,
+ 211,
+ 25715,
+ 124,
+ 36241,
+ 3974,
+ 49042,
+ 211,
+ 14931,
+ 106,
+ 243,
+ 209,
+ 172,
+ 243,
+ 211,
+ 211,
+ 172,
+ 243,
+ 211,
+ 212
+ ]
+ },
+ {
+ "name": "bom-nul",
+ "ids": [
+ 41497,
+ 66,
+ 177,
+ 67,
+ 41497
+ ],
+ "decoded": "a\u0000b",
+ "reencoded": [
+ 41497,
+ 66,
+ 177,
+ 67,
+ 41497
+ ]
+ },
+ {
+ "name": "special",
+ "ids": [
+ 50281,
+ 23120,
+ 50284,
+ 209,
+ 50282
+ ],
+ "decoded": "[CLS] hello[MASK] [SEP]",
+ "reencoded": [
+ 50281,
+ 23120,
+ 50284,
+ 209,
+ 50282
+ ]
+ },
+ {
+ "name": "boundary-0",
+ "ids": [
+ 172,
+ 243,
+ 211
+ ],
+ "decoded": "�",
+ "reencoded": [
+ 21478
+ ]
+ },
+ {
+ "name": "boundary-1",
+ "ids": [
+ 243,
+ 211,
+ 211
+ ],
+ "decoded": "���",
+ "reencoded": [
+ 42481,
+ 21478
+ ]
+ },
+ {
+ "name": "boundary-2",
+ "ids": [
+ 211,
+ 211,
+ 172
+ ],
+ "decoded": "���",
+ "reencoded": [
+ 42481,
+ 21478
+ ]
+ },
+ {
+ "name": "boundary-3",
+ "ids": [
+ 211,
+ 172,
+ 243
+ ],
+ "decoded": "��",
+ "reencoded": [
+ 42481
+ ]
+ },
+ {
+ "name": "boundary-4",
+ "ids": [
+ 172,
+ 243,
+ 211
+ ],
+ "decoded": "�",
+ "reencoded": [
+ 21478
+ ]
+ },
+ {
+ "name": "boundary-5",
+ "ids": [
+ 243,
+ 211,
+ 212
+ ],
+ "decoded": "���",
+ "reencoded": [
+ 42481,
+ 21478
+ ]
+ },
+ {
+ "name": "boundary-6",
+ "ids": [
+ 211,
+ 212,
+ 172
+ ],
+ "decoded": "���",
+ "reencoded": [
+ 42481,
+ 21478
+ ]
+ },
+ {
+ "name": "boundary-7",
+ "ids": [
+ 212,
+ 172,
+ 243
+ ],
+ "decoded": "��",
+ "reencoded": [
+ 42481
+ ]
+ },
+ {
+ "name": "boundary-8",
+ "ids": [
+ 172,
+ 243,
+ 211
+ ],
+ "decoded": "�",
+ "reencoded": [
+ 21478
+ ]
+ },
+ {
+ "name": "boundary-9",
+ "ids": [
+ 243,
+ 211,
+ 213
+ ],
+ "decoded": "���",
+ "reencoded": [
+ 42481,
+ 21478
+ ]
+ },
+ {
+ "name": "boundary-10",
+ "ids": [
+ 211,
+ 213,
+ 49042
+ ],
+ "decoded": "�� �",
+ "reencoded": [
+ 42481,
+ 16141,
+ 19675
+ ]
+ },
+ {
+ "name": "boundary-11",
+ "ids": [
+ 213,
+ 49042,
+ 211
+ ],
+ "decoded": "� 😀",
+ "reencoded": [
+ 21478,
+ 49042,
+ 211
+ ]
+ },
+ {
+ "name": "unknown-id",
+ "ids": [
+ 2147483647
+ ],
+ "decoded": "",
+ "reencoded": []
+ }
+ ]
+ },
+ "multilingual": {
+ "tokenizer": {
+ "version": "1.0",
+ "truncation": null,
+ "padding": null,
+ "added_tokens": [
+ {
+ "id": 1,
+ "content": "<eos>",
+ "single_word": false,
+ "lstrip": false,
+ "rstrip": false,
+ "normalized": false,
+ "special": true
+ },
+ {
+ "id": 2,
+ "content": "<bos>",
+ "single_word": false,
+ "lstrip": false,
+ "rstrip": false,
+ "normalized": false,
+ "special": true
+ },
+ {
+ "id": 4,
+ "content": "<mask>",
+ "single_word": false,
+ "lstrip": true,
+ "rstrip": false,
+ "normalized": false,
+ "special": true
+ },
+ {
+ "id": 108,
+ "content": "\n",
+ "single_word": false,
+ "lstrip": false,
+ "rstrip": false,
+ "normalized": false,
+ "special": false
+ }
+ ],
+ "normalizer": {
+ "type": "Replace",
+ "pattern": {
+ "String": " "
+ },
+ "content": "▁"
+ },
+ "pre_tokenizer": {
+ "type": "Metaspace",
+ "replacement": "▁",
+ "prepend_scheme": "always",
+ "split": true
+ },
+ "post_processor": null,
+ "decoder": {
+ "type": "Sequence",
+ "decoders": [
+ {
+ "type": "Replace",
+ "pattern": {
+ "String": "▁"
+ },
+ "content": " "
+ },
+ {
+ "type": "ByteFallback"
+ },
+ {
+ "type": "Fuse"
+ }
+ ]
+ },
+ "model": {
+ "type": "BPE",
+ "dropout": null,
+ "unk_token": "<unk>",
+ "continuing_subword_prefix": null,
+ "end_of_word_suffix": null,
+ "fuse_unk": true,
+ "byte_fallback": true,
+ "ignore_merges": false,
+ "vocab": {
+ "<eos>": 1,
+ "<bos>": 2,
+ "<unk>": 3,
+ "<mask>": 4,
+ "\n": 108,
+ "<0x00>": 217,
+ "\t": 226,
+ "<0x61>": 314,
+ "<0x80>": 345,
+ "<0x81>": 346,
+ "<0x82>": 347,
+ "<0x98>": 369,
+ "<0x9F>": 376,
+ "<0xA0>": 377,
+ "<0xBB>": 404,
+ "<0xBF>": 408,
+ "<0xEF>": 456,
+ "<0xF0>": 457,
+ "he": 480,
+ "or": 483,
+ "▁c": 498,
+ "el": 521,
+ "▁e": 526,
+ "ll": 529,
+ "▁h": 531,
+ "lo": 545,
+ "ca": 557,
+ "▁H": 640,
+ "▁he": 693,
+ "ld": 706,
+ "ell": 806,
+ "wo": 1100,
+ "▁He": 1315,
+ "▁hel": 1487,
+ "He": 1949,
+ "▁ca": 1962,
+ "Hel": 2405,
+ "af": 2518,
+ "orld": 2780,
+ "hel": 3549,
+ "llo": 3979,
+ "Hello": 4521,
+ "▁Hel": 5447,
+ "fé": 5909,
+ "我们": 7078,
+ "world": 9097,
+ "了一": 10151,
+ "已经": 11546,
+ "▁hell": 11944,
+ "ello": 13444,
+ "用户": 16464,
+ "hello": 17534,
+ "必要": 17986,
+ "wor": 17987,
+ "▁café": 18688,
+ "hell": 20256,
+ "原因": 22064,
+ "��": 23484,
+ "▁Hell": 24598,
+ "▁hello": 25612,
+ "▁Hello": 25957,
+ "客户": 30379,
+ "记录": 31274,
+ "Hell": 34074,
+ "说明": 34310,
+ "检查": 37251,
+ "▁caf": 41680,
+ "申请": 43501,
+ "▁": 45835,
+ "支付": 46753,
+ "▁�": 53152,
+ "安排": 54402,
+ "▁客": 62958,
+ "确认": 64753,
+ "caf": 67504,
+ "费用": 71465,
+ "重复": 79825,
+ "▁😀": 79937,
+ "café": 81453,
+ "账户": 93391,
+ "并在": 130004,
+ "���": 135502,
+ "退款": 186612,
+ "▁��": 228059,
+ "▁客户": 231537,
+ "▁": 235248,
+ "e": 235249,
+ "a": 235250,
+ "o": 235253,
+ "r": 235255,
+ "l": 235257,
+ "d": 235258,
+ "h": 235259,
+ "c": 235260,
+ "f": 235266,
+ "b": 235268,
+ "w": 235271,
+ "H": 235314,
+ "é": 235335,
+ "。": 235362,
+ ",": 235365,
+ "一": 235411,
+ "了": 235445,
+ "在": 235473,
+ "我": 235509,
+ "用": 235522,
+ "要": 235556,
+ "时": 235635,
+ "": 235705,
+ "们": 235707,
+ "说": 235726,
+ "明": 235781,
+ "重": 235846,
+ "次": 235847,
+ "安": 235859,
+ "原": 235875,
+ "只": 235918,
+ "实": 235930,
+ "经": 235931,
+ "因": 235933,
+ "已": 236023,
+ "付": 236076,
+ "必": 236188,
+ "�": 236193,
+ "并": 236203,
+ "客": 236224,
+ "查": 236293,
+ "户": 236308,
+ "支": 236324,
+ "请": 236343,
+ "记": 236364,
+ "认": 236458,
+ "复": 236465,
+ "排": 236551,
+ "录": 236586,
+ "确": 236601,
+ "款": 236604,
+ "费": 236699,
+ "́": 236703,
+ "续": 236812,
+ "退": 236924,
+ "申": 237026,
+ "检": 237147,
+ "核": 237260,
+ "订": 237743,
+ "账": 238157,
+ "扣": 238498,
+ "😀": 239061,
+ "🫠": 244497
+ },
+ "merges": [
+ [
+ "h",
+ "e"
+ ],
+ [
+ "o",
+ "r"
+ ],
+ [
+ "▁",
+ "c"
+ ],
+ [
+ "e",
+ "l"
+ ],
+ [
+ "▁",
+ "e"
+ ],
+ [
+ "l",
+ "l"
+ ],
+ [
+ "▁",
+ "h"
+ ],
+ [
+ "l",
+ "o"
+ ],
+ [
+ "c",
+ "a"
+ ],
+ [
+ "▁",
+ "H"
+ ],
+ [
+ "▁h",
+ "e"
+ ],
+ [
+ "▁",
+ "he"
+ ],
+ [
+ "l",
+ "d"
+ ],
+ [
+ "el",
+ "l"
+ ],
+ [
+ "e",
+ "ll"
+ ],
+ [
+ "w",
+ "o"
+ ],
+ [
+ "▁H",
+ "e"
+ ],
+ [
+ "▁",
+ "He"
+ ],
+ [
+ "▁he",
+ "l"
+ ],
+ [
+ "▁h",
+ "el"
+ ],
+ [
+ "▁",
+ "hel"
+ ],
+ [
+ "H",
+ "e"
+ ],
+ [
+ "▁c",
+ "a"
+ ],
+ [
+ "▁",
+ "ca"
+ ],
+ [
+ "He",
+ "l"
+ ],
+ [
+ "H",
+ "el"
+ ],
+ [
+ "a",
+ "f"
+ ],
+ [
+ "or",
+ "ld"
+ ],
+ [
+ "he",
+ "l"
+ ],
+ [
+ "h",
+ "el"
+ ],
+ [
+ "ll",
+ "o"
+ ],
+ [
+ "l",
+ "lo"
+ ],
+ [
+ "Hell",
+ "o"
+ ],
+ [
+ "Hel",
+ "lo"
+ ],
+ [
+ "He",
+ "llo"
+ ],
+ [
+ "H",
+ "ello"
+ ],
+ [
+ "▁He",
+ "l"
+ ],
+ [
+ "▁H",
+ "el"
+ ],
+ [
+ "▁",
+ "Hel"
+ ],
+ [
+ "f",
+ "é"
+ ],
+ [
+ "我",
+ "们"
+ ],
+ [
+ "wor",
+ "ld"
+ ],
+ [
+ "w",
+ "orld"
+ ],
+ [
+ "了",
+ "一"
+ ],
+ [
+ "已",
+ "经"
+ ],
+ [
+ "▁hel",
+ "l"
+ ],
+ [
+ "▁he",
+ "ll"
+ ],
+ [
+ "▁h",
+ "ell"
+ ],
+ [
+ "▁",
+ "hell"
+ ],
+ [
+ "ell",
+ "o"
+ ],
+ [
+ "el",
+ "lo"
+ ],
+ [
+ "e",
+ "llo"
+ ],
+ [
+ "用",
+ "户"
+ ],
+ [
+ "hell",
+ "o"
+ ],
+ [
+ "hel",
+ "lo"
+ ],
+ [
+ "he",
+ "llo"
+ ],
+ [
+ "h",
+ "ello"
+ ],
+ [
+ "必",
+ "要"
+ ],
+ [
+ "wo",
+ "r"
+ ],
+ [
+ "w",
+ "or"
+ ],
+ [
+ "▁caf",
+ "é"
+ ],
+ [
+ "▁ca",
+ "fé"
+ ],
+ [
+ "▁",
+ "café"
+ ],
+ [
+ "hel",
+ "l"
+ ],
+ [
+ "he",
+ "ll"
+ ],
+ [
+ "h",
+ "ell"
+ ],
+ [
+ "原",
+ "因"
+ ],
+ [
+ "�",
+ "�"
+ ],
+ [
+ "▁Hel",
+ "l"
+ ],
+ [
+ "▁He",
+ "ll"
+ ],
+ [
+ "▁H",
+ "ell"
+ ],
+ [
+ "▁",
+ "Hell"
+ ],
+ [
+ "▁hell",
+ "o"
+ ],
+ [
+ "▁hel",
+ "lo"
+ ],
+ [
+ "▁he",
+ "llo"
+ ],
+ [
+ "▁h",
+ "ello"
+ ],
+ [
+ "▁",
+ "hello"
+ ],
+ [
+ "▁Hell",
+ "o"
+ ],
+ [
+ "▁Hel",
+ "lo"
+ ],
+ [
+ "▁He",
+ "llo"
+ ],
+ [
+ "▁H",
+ "ello"
+ ],
+ [
+ "▁",
+ "Hello"
+ ],
+ [
+ "客",
+ "户"
+ ],
+ [
+ "记",
+ "录"
+ ],
+ [
+ "Hel",
+ "l"
+ ],
+ [
+ "He",
+ "ll"
+ ],
+ [
+ "H",
+ "ell"
+ ],
+ [
+ "说",
+ "明"
+ ],
+ [
+ "检",
+ "查"
+ ],
+ [
+ "▁ca",
+ "f"
+ ],
+ [
+ "▁c",
+ "af"
+ ],
+ [
+ "▁",
+ "caf"
+ ],
+ [
+ "申",
+ "请"
+ ],
+ [
+ "▁",
+ ""
+ ],
+ [
+ "支",
+ "付"
+ ],
+ [
+ "▁",
+ "�"
+ ],
+ [
+ "安",
+ "排"
+ ],
+ [
+ "▁",
+ "客"
+ ],
+ [
+ "确",
+ "认"
+ ],
+ [
+ "ca",
+ "f"
+ ],
+ [
+ "c",
+ "af"
+ ],
+ [
+ "费",
+ "用"
+ ],
+ [
+ "重",
+ "复"
+ ],
+ [
+ "▁",
+ "😀"
+ ],
+ [
+ "caf",
+ "é"
+ ],
+ [
+ "ca",
+ "fé"
+ ],
+ [
+ "账",
+ "户"
+ ],
+ [
+ "并",
+ "在"
+ ],
+ [
+ "��",
+ "�"
+ ],
+ [
+ "�",
+ "��"
+ ],
+ [
+ "退",
+ "款"
+ ],
+ [
+ "▁�",
+ "�"
+ ],
+ [
+ "▁",
+ "��"
+ ],
+ [
+ "▁客",
+ "户"
+ ],
+ [
+ "▁",
+ "客户"
+ ]
+ ]
+ }
+ },
+ "cases": [
+ {
+ "name": "empty",
+ "ids": [],
+ "decoded": "",
+ "reencoded": []
+ },
+ {
+ "name": "whitespace",
+ "ids": [
+ 235248,
+ 25957,
+ 108,
+ 235248,
+ 226,
+ 9097,
+ 235248,
+ 235248
+ ],
+ "decoded": " Hello\n \tworld ",
+ "reencoded": [
+ 235248,
+ 25957,
+ 108,
+ 235248,
+ 226,
+ 9097,
+ 235248,
+ 235248
+ ]
+ },
+ {
+ "name": "unicode",
+ "ids": [
+ 231537,
+ 186612,
+ 18688,
+ 526,
+ 236703,
+ 79937,
+ 244497,
+ 235248,
+ 457,
+ 377,
+ 345,
+ 345,
+ 457,
+ 377,
+ 345,
+ 346
+ ],
+ "decoded": " 客户退款 café é 😀🫠 𠀀𠀁",
+ "reencoded": [
+ 231537,
+ 186612,
+ 18688,
+ 526,
+ 236703,
+ 79937,
+ 244497,
+ 235248,
+ 457,
+ 377,
+ 345,
+ 345,
+ 457,
+ 377,
+ 345,
+ 346
+ ]
+ },
+ {
+ "name": "bom-nul",
+ "ids": [
+ 45835,
+ 235250,
+ 217,
+ 235268,
+ 235705
+ ],
+ "decoded": " a\u0000b",
+ "reencoded": [
+ 45835,
+ 235250,
+ 217,
+ 235268,
+ 235705
+ ]
+ },
+ {
+ "name": "special",
+ "ids": [
+ 2,
+ 25612,
+ 4,
+ 1
+ ],
+ "decoded": "<bos> hello<mask><eos>",
+ "reencoded": [
+ 2,
+ 25612,
+ 4,
+ 1
+ ]
+ },
+ {
+ "name": "boundary-0",
+ "ids": [
+ 235248,
+ 457,
+ 377
+ ],
+ "decoded": " ��",
+ "reencoded": [
+ 228059
+ ]
+ },
+ {
+ "name": "boundary-1",
+ "ids": [
+ 457,
+ 377,
+ 345
+ ],
+ "decoded": "���",
+ "reencoded": [
+ 235248,
+ 135502
+ ]
+ },
+ {
+ "name": "boundary-2",
+ "ids": [
+ 377,
+ 345,
+ 345
+ ],
+ "decoded": "���",
+ "reencoded": [
+ 235248,
+ 135502
+ ]
+ },
+ {
+ "name": "boundary-3",
+ "ids": [
+ 345,
+ 345,
+ 457
+ ],
+ "decoded": "���",
+ "reencoded": [
+ 235248,
+ 135502
+ ]
+ },
+ {
+ "name": "boundary-4",
+ "ids": [
+ 345,
+ 457,
+ 377
+ ],
+ "decoded": "���",
+ "reencoded": [
+ 235248,
+ 135502
+ ]
+ },
+ {
+ "name": "boundary-5",
+ "ids": [
+ 457,
+ 377,
+ 345
+ ],
+ "decoded": "���",
+ "reencoded": [
+ 235248,
+ 135502
+ ]
+ },
+ {
+ "name": "boundary-6",
+ "ids": [
+ 377,
+ 345,
+ 346
+ ],
+ "decoded": "���",
+ "reencoded": [
+ 235248,
+ 135502
+ ]
+ },
+ {
+ "name": "boundary-7",
+ "ids": [
+ 345,
+ 346,
+ 457
+ ],
+ "decoded": "���",
+ "reencoded": [
+ 235248,
+ 135502
+ ]
+ },
+ {
+ "name": "boundary-8",
+ "ids": [
+ 346,
+ 457,
+ 377
+ ],
+ "decoded": "���",
+ "reencoded": [
+ 235248,
+ 135502
+ ]
+ },
+ {
+ "name": "boundary-9",
+ "ids": [
+ 457,
+ 377,
+ 345
+ ],
+ "decoded": "���",
+ "reencoded": [
+ 235248,
+ 135502
+ ]
+ },
+ {
+ "name": "boundary-10",
+ "ids": [
+ 377,
+ 345,
+ 347
+ ],
+ "decoded": "���",
+ "reencoded": [
+ 235248,
+ 135502
+ ]
+ },
+ {
+ "name": "boundary-11",
+ "ids": [
+ 345,
+ 347,
+ 79937
+ ],
+ "decoded": "�� 😀",
+ "reencoded": [
+ 228059,
+ 79937
+ ]
+ },
+ {
+ "name": "fallback-valid",
+ "ids": [
+ 457,
+ 376,
+ 369,
+ 345
+ ],
+ "decoded": "😀",
+ "reencoded": [
+ 79937
+ ]
+ },
+ {
+ "name": "fallback-invalid-prefix",
+ "ids": [
+ 314,
+ 457
+ ],
+ "decoded": "��",
+ "reencoded": [
+ 228059
+ ]
+ },
+ {
+ "name": "fallback-invalid-suffix",
+ "ids": [
+ 457,
+ 314
+ ],
+ "decoded": "��",
+ "reencoded": [
+ 228059
+ ]
+ },
+ {
+ "name": "fallback-bom",
+ "ids": [
+ 456,
+ 404,
+ 408,
+ 217
+ ],
+ "decoded": "\u0000",
+ "reencoded": [
+ 45835,
+ 217
+ ]
+ },
+ {
+ "name": "reencoding-growth",
+ "ids": [
+ 71465,
+ 235362,
+ 7078,
+ 11546,
+ 37251,
+ 93391,
+ 31274,
+ 235365,
+ 64753,
+ 16464,
+ 235918,
+ 43501,
+ 10151,
+ 235847,
+ 236812,
+ 237743,
+ 235362,
+ 236343,
+ 237260,
+ 235930,
+ 46753,
+ 31274,
+ 235365,
+ 34310,
+ 79825,
+ 238498,
+ 236699,
+ 22064,
+ 235365,
+ 130004,
+ 17986,
+ 235635,
+ 54402
+ ],
+ "decoded": "费用。我们已经检查账户记录,确认用户只申请了一次续订。请核实支付记录,说明重复扣费原因,并在必要时安排",
+ "reencoded": [
+ 235248,
+ 71465,
+ 235362,
+ 7078,
+ 11546,
+ 37251,
+ 93391,
+ 31274,
+ 235365,
+ 64753,
+ 16464,
+ 235918,
+ 43501,
+ 10151,
+ 235847,
+ 236812,
+ 237743,
+ 235362,
+ 236343,
+ 237260,
+ 235930,
+ 46753,
+ 31274,
+ 235365,
+ 34310,
+ 79825,
+ 238498,
+ 236699,
+ 22064,
+ 235365,
+ 130004,
+ 17986,
+ 235635,
+ 54402
+ ]
+ },
+ {
+ "name": "unknown-id",
+ "ids": [
+ 2147483647
+ ],
+ "decoded": "",
+ "reencoded": []
+ }
+ ]
+ }
+ }
+}
diff --git a/laya-ts/scripts/tokenizer_decode_fixtures.py b/laya-ts/scripts/tokenizer_decode_fixtures.py
new file mode 100644
index 00000000..9f2142c9
--- /dev/null
+++ b/laya-ts/scripts/tokenizer_decode_fixtures.py
@@ -0,0 +1,152 @@
+"""Record/check checkpoint decode -> re-encode vectors with Hugging Face tokenizers.
+
+ python laya-ts/scripts/tokenizer_decode_fixtures.py --checkpoint-root SNAPSHOT
+ python laya-ts/scripts/tokenizer_decode_fixtures.py --check
+
+The checked-in tokenizer subsets retain the original token IDs and merge order.
+Recording checks each subset against the complete checkpoint tokenizer; --check
+executes the same HF decoder/encoder without downloading model assets. Supply
+--checkpoint-root with --check to additionally verify the original asset hashes.
+"""
+from __future__ import annotations
+
+import argparse
+from copy import deepcopy
+import hashlib
+import json
+from pathlib import Path
+
+import tokenizers
+from tokenizers import Tokenizer
+
+
+FIXTURE = Path(__file__).resolve().parents[1] / "fixtures/tokenizer-decode.json"
+REVISION = "55cf4c4ebb4ebe31b2550e8bdf3bd21b99753851"
+FILES = {"english": "tokenizer/tokenizer.json", "multilingual": "multilingual/tokenizer/tokenizer.json"}
+SOURCE_HASHES = {
+ "tokenizer/tokenizer.json": "6c8aaa9a542084f2457eab775d4eeb51f92a70c0fd9de28d5edb0ddec3c08d30",
+ "multilingual/tokenizer/tokenizer.json": "609d8f4c067cd3950f88594c5a802616cea245823836ef5848ee4fc40aab5b6f",
+}
+
+
+def vectors(tok, kind):
+ texts = [
+ ("empty", ""), ("whitespace", " Hello\n\tworld "),
+ ("unicode", "客户退款 café e\u0301 😀🫠 𠀀𠀁"),
+ ("bom-nul", "\ufeffa\0b\ufeff"),
+ ("special", "[CLS] hello [MASK] [SEP]" if kind == "english" else "<bos> hello <mask><eos>"),
+ ]
+ cases = []
+
+ def add(name, ids):
+ decoded = tok.decode(ids, skip_special_tokens=False)
+ cases.append({"name": name, "ids": ids, "decoded": decoded,
+ "reencoded": tok.encode(decoded, add_special_tokens=False).ids})
+
+ for name, text in texts:
+ add(name, tok.encode(text, add_special_tokens=False).ids)
+ boundary = tok.encode("𠀀𠀁𠀂 😀🫠 café 客户退款", add_special_tokens=False).ids
+ for start in range(min(12, len(boundary))):
+ add(f"boundary-{start}", boundary[start:start + 3])
+ if kind == "multilingual":
+ for name, values in [
+ ("fallback-valid", [0xF0, 0x9F, 0x98, 0x80]),
+ ("fallback-invalid-prefix", [0x61, 0xF0]),
+ ("fallback-invalid-suffix", [0xF0, 0x61]),
+ ("fallback-bom", [0xEF, 0xBB, 0xBF, 0]),
+ ]:
+ add(name, [tok.token_to_id(f"<0x{v:02X}>") for v in values])
+ paragraph = ("客户表示,最近的账单中同一项订阅被收取了两次费用。"
+ "我们已经检查账户记录,确认用户只申请了一次续订。"
+ "请核实支付记录,说明重复扣费原因,并在必要时安排退款。"
+ "客户可以提供发票号码和付款凭证。\n\n")
+ ids = tok.encode(paragraph * 4, add_special_tokens=False).ids
+ add("reencoding-growth", ids[16:49])
+ add("unknown-id", [2**31 - 1])
+ return cases
+
+
+def subset(raw, cases):
+ """Keep every merge decomposition of the used tokens, in original rank order."""
+ vocab = raw["model"]["vocab"]
+ by_id = {v: k for k, v in vocab.items()}
+ used_ids = {i for case in cases for key in ("ids", "reencoded") for i in case[key]}
+ added = [t for t in raw["added_tokens"] if t["id"] in used_ids
+ or any(t["content"] in case["decoded"] for case in cases)]
+ wanted = {by_id[i] for case in cases for key in ("ids", "reencoded")
+ for i in case[key] if i in by_id}
+ wanted.update(t["content"] for t in added if t["content"] in vocab)
+ if raw["model"].get("unk_token"):
+ wanted.add(raw["model"]["unk_token"])
+ merges = [m.split(" ") if isinstance(m, str) else m for m in raw["model"]["merges"]]
+ decompositions = {}
+ for pair in merges:
+ decompositions.setdefault("".join(pair), []).append(pair)
+ pending = list(wanted)
+ while pending:
+ for pair in decompositions.get(pending.pop(), []):
+ for token in pair:
+ if token not in wanted:
+ wanted.add(token)
+ pending.append(token)
+ result = deepcopy(raw)
+ result["added_tokens"] = added
+ result["model"]["vocab"] = {k: v for k, v in vocab.items() if k in wanted}
+ # HF otherwise allocates newly added token IDs from the smaller vocabulary size.
+ result["model"]["vocab"].update({t["content"]: t["id"] for t in added})
+ result["model"]["merges"] = [pair for pair in merges if "".join(pair) in wanted]
+ # Encoding requests no special tokens. Avoid unused processor references in the subset.
+ result["post_processor"] = None
+ return result
+
+
+def verify(data):
+ count = 0
+ for kind, section in data["tokenizers"].items():
+ tok = Tokenizer.from_str(json.dumps(section["tokenizer"]))
+ for case in section["cases"]:
+ decoded = tok.decode(case["ids"], skip_special_tokens=False)
+ encoded = tok.encode(decoded, add_special_tokens=False).ids
+ if decoded != case["decoded"] or encoded != case["reencoded"]:
+ raise AssertionError(f"{kind}/{case['name']}: HF decode/re-encode changed")
+ count += 1
+ return count
+
+
+def record(root):
+ data = {"reference": {"repository": "convaiinnovations/laya", "revision": REVISION,
+ "tokenizers": tokenizers.__version__, "files": {}}, "tokenizers": {}}
+ for kind, relative in FILES.items():
+ contents = (root / relative).read_bytes()
+ digest = hashlib.sha256(contents).hexdigest()
+ if digest != SOURCE_HASHES[relative]:
+ raise ValueError(f"{relative}: tokenizer does not match the pinned snapshot {REVISION}")
+ data["reference"]["files"][relative] = digest
+ raw = json.loads(contents)
+ cases = vectors(Tokenizer.from_str(contents.decode("utf-8")), kind)
+ data["tokenizers"][kind] = {"tokenizer": subset(raw, cases), "cases": cases}
+ verify(data)
+ return data
+
+
+def main():
+ parser = argparse.ArgumentParser(description=__doc__)
+ parser.add_argument("--checkpoint-root", type=Path)
+ parser.add_argument("--check", action="store_true")
+ args = parser.parse_args()
+ if args.check:
+ stored = json.loads(FIXTURE.read_text(encoding="utf-8"))
+ count = verify(stored)
+ if args.checkpoint_root and record(args.checkpoint_root) != stored:
+ raise AssertionError("checkpoint tokenizer fixture differs; review before recording")
+ print(f"PASS: {count} HF decode/re-encode vectors (tokenizers {tokenizers.__version__})")
+ else:
+ if not args.checkpoint_root:
+ parser.error("recording requires --checkpoint-root at the pinned snapshot")
+ data = record(args.checkpoint_root)
+ FIXTURE.write_text(json.dumps(data, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
+ print(f"Recorded {FIXTURE}")
+
+
+if __name__ == "__main__":
+ main()
diff --git a/laya-ts/src/agent.ts b/laya-ts/src/agent.ts
index 4b439b1f..bfef3b9f 100644
--- a/laya-ts/src/agent.ts
+++ b/laya-ts/src/agent.ts
@@ -420,6 +420,7 @@ function tokenizerFromHF(tokenizerJson: unknown): TokenizerLike | null {
padId: data.ids.pad,
maskToken: data.maskToken,
encode: (text: string) => encodeWithData(data, text),
+ ...(data.decode ? { decode: data.decode } : {}),
};
}
diff --git a/laya-ts/src/tokenizer.ts b/laya-ts/src/tokenizer.ts
index 3e460d0d..be8da866 100644
--- a/laya-ts/src/tokenizer.ts
+++ b/laya-ts/src/tokenizer.ts
@@ -3,6 +3,8 @@ export interface TokenizerLike {
readonly maskId: number; readonly padId: number;
readonly maskToken: string;
encode(text: string): number[];
+ /** Decode without removing special tokens; needed only when predictLong splits the state. */
+ decode?(ids: readonly number[]): string;
}
/** Special ids of the laya ModernBERT checkpoint (HF added_tokens). */
@@ -31,6 +33,8 @@ export interface TokenizerData {
byteFallback?: boolean;
/** Tokens cut out of the text before it is tokenized (HF added_tokens). */
added?: AddedToken[];
+ /** Absent when tokenizer.json uses a decoder this runtime does not implement. */
+ decode?: (ids: readonly number[]) => string;
}
export interface AddedToken { content: string; id: number; normalized: boolean; lstrip: boolean; rstrip: boolean }
@@ -411,6 +415,64 @@ function collectReplaces(node: unknown, out: Array<[string, string]>): void {
for (const c of childNodes(node)) collectReplaces(c, out);
}
+/** Compile the two checkpoint decoders, retaining HF's skip_special_tokens=False behavior. */
+function checkpointDecoder(
+ raw: unknown, vocab: Map<string, number>, added: AddedToken[],
+): ((ids: readonly number[]) => string) | undefined {
+ if (!raw || typeof raw !== "object") return undefined;
+ const decoder = raw as Record<string, unknown>;
+ let decodeTokens: (tokens: string[]) => string;
+ // ignoreBOM preserves U+FEFF in a decoded window, just as the HF decoder does.
+ const utf8 = new TextDecoder("utf-8", { ignoreBOM: true });
+ if (decoder.type === "ByteLevel") {
+ decodeTokens = (tokens) => {
+ const bytes: number[] = [];
+ const { u2b } = maps();
+ for (const token of tokens) {
+ const mapped = Array.from(token, (ch) => u2b.get(ch));
+ // HF falls back to the whole token's UTF-8 bytes if any character is unmapped.
+ const part = mapped.every((b) => b !== undefined)
+ ? mapped as number[] : (sharedEncoder ??= new TextEncoder()).encode(token);
+ for (const b of part) bytes.push(b);
+ }
+ return utf8.decode(new Uint8Array(bytes));
+ };
+ } else if (decoder.type === "Sequence" && Array.isArray(decoder.decoders)) {
+ const steps = decoder.decoders as Array<Record<string, unknown> | null>;
+ const pattern = steps[0]?.pattern as Record<string, unknown> | undefined;
+ if (steps.length !== 3 || steps[0]?.type !== "Replace"
+ || typeof pattern?.String !== "string" || pattern.String.length === 0
+ || typeof steps[0].content !== "string"
+ || steps[1]?.type !== "ByteFallback" || steps[2]?.type !== "Fuse") return undefined;
+ const from = pattern.String, to = steps[0].content;
+ const strict = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true });
+ decodeTokens = (tokens) => {
+ const out: string[] = [], bytes: number[] = [];
+ const flush = (): void => {
+ if (bytes.length === 0) return;
+ // ByteFallback rejects an entire invalid run, including any otherwise valid prefix.
+ try { out.push(strict.decode(new Uint8Array(bytes))); }
+ catch { out.push("\uFFFD".repeat(bytes.length)); }
+ bytes.length = 0;
+ };
+ for (const token of tokens) {
+ const replaced = token.split(from).join(to);
+ const match = /^<0x([\dA-Fa-f]{2})>$/.exec(replaced);
+ if (match) bytes.push(parseInt(match[1], 16));
+ else { flush(); out.push(replaced); }
+ }
+ flush();
+ return out.join("");
+ };
+ } else return undefined;
+ const byId = new Map(Array.from(vocab, ([token, id]) => [id, token]));
+ for (const token of added) byId.set(token.id, token.content);
+ return (ids) => decodeTokens(ids.flatMap((id) => {
+ const token = byId.get(id);
+ return token === undefined ? [] : [token];
+ }));
+}
+
/** Parse an HF tokenizer.json ({model vocab/merges, normalizer, pre_tokenizer, added_tokens}). */
export function parseTokenizerJson(raw: unknown): TokenizerData | null {
try {
@@ -418,6 +480,7 @@ export function parseTokenizerJson(raw: unknown): TokenizerData | null {
model?: { vocab?: Record<string, number>; merges?: Array<string | [string, string]>; byte_fallback?: boolean };
normalizer?: unknown;
pre_tokenizer?: unknown;
+ decoder?: unknown;
added_tokens?: Array<{ id?: number; content?: string; normalized?: boolean; lstrip?: boolean; rstrip?: boolean }>;
};
const vocabObj = r?.model?.vocab;
@@ -463,6 +526,7 @@ export function parseTokenizerJson(raw: unknown): TokenizerData | null {
replaces,
byteFallback: r.model?.byte_fallback === true,
added: addedTokens,
+ decode: checkpointDecoder(r.decoder, vocab, addedTokens),
};
} catch {
return null;
diff --git a/laya-ts/tests/tokenizer-decode.test.ts b/laya-ts/tests/tokenizer-decode.test.ts
new file mode 100644
index 00000000..fa1afb1f
--- /dev/null
+++ b/laya-ts/tests/tokenizer-decode.test.ts
@@ -0,0 +1,79 @@
+import { readFileSync } from "node:fs";
+import { describe, expect, it, vi } from "vitest";
+import { encodeWithData, parseTokenizerJson } from "../src/tokenizer.js";
+
+const fixture = JSON.parse(readFileSync(new URL("../fixtures/tokenizer-decode.json", import.meta.url), "utf8"));
+
+describe("checkpoint decode/re-encode parity", () => {
+ for (const [kind, raw] of Object.entries(fixture.tokenizers)) {
+ const section = raw as {
+ tokenizer: unknown;
+ cases: Array<{ name: string; ids: number[]; decoded: string; reencoded: number[] }>;
+ };
+ const data = parseTokenizerJson(section.tokenizer)!;
+ for (const vector of section.cases) {
+ it(`${kind}: ${vector.name}`, () => {
+ const decoded = data.decode!(vector.ids);
+ expect(decoded).toBe(vector.decoded);
+ expect(encodeWithData(data, decoded)).toEqual(vector.reencoded);
+ });
+ }
+ }
+
+ it("records actual multilingual re-encoding growth instead of promising lossless windows", () => {
+ const vector = fixture.tokenizers.multilingual.cases.find((c: { name: string }) => c.name === "reencoding-growth");
+ expect(vector.ids).toHaveLength(33);
+ expect(vector.reencoded).toHaveLength(34);
+ });
+});
+
+describe("decoder boundaries", () => {
+ const byteLevel = (vocab: Record<string, number>, added_tokens: unknown[] = []) => parseTokenizerJson({
+ model: { vocab, merges: [] }, decoder: { type: "ByteLevel" }, added_tokens,
+ })!;
+
+ it("decodes UTF-8 across token boundaries and preserves BOM/NUL", () => {
+ const data = byteLevel({ "ð": 0, "Ł": 1, "ĺ": 2, "Ģ": 3 });
+ expect(data.decode!([0, 1, 2, 3])).toBe("😀");
+ const bom = byteLevel({ "ï": 0, "»": 1, "¿": 2, "Ā": 3 });
+ expect(bom.decode!([0, 1, 2, 3])).toBe("\ufeff\0");
+ });
+
+ it("falls back to an unmappable added token as a whole, retaining special tokens", () => {
+ const data = byteLevel({ "Ġ": 0 }, [
+ { id: 1, content: "Ġ🫠", special: true },
+ { id: 2, content: "[MASK]", special: true },
+ ]);
+ expect(data.decode!([0, 1, 2, 999])).toBe(" Ġ🫠[MASK]");
+ });
+
+ it("keeps encoding available for absent or unsupported decoders", () => {
+ for (const decoder of [undefined, { type: "Metaspace" }, { type: "Sequence", decoders: [
+ { type: "Replace", pattern: { Regex: "▁" }, content: " " },
+ { type: "ByteFallback" }, { type: "Fuse" },
+ ] }, { type: "Sequence", decoders: [
+ { type: "Replace", pattern: { String: "▁" }, content: " " },
+ { type: "ByteFallback" }, { type: "Fuse" }, { type: "Strip" },
+ ] }]) {
+ const data = parseTokenizerJson({ model: { vocab: { a: 1 }, merges: [] }, decoder })!;
+ expect(data.decode).toBeUndefined();
+ expect(encodeWithData(data, "a")).toEqual([1]);
+ }
+ });
+
+ it("loads a decoder into Agent without needing a Node tokenizer dependency", async () => {
+ const providers = await import("../src/providers.js");
+ const { Agent } = await import("../src/agent.js");
+ const bundle = vi.spyOn(providers, "loadWebBundle").mockResolvedValue({
+ cfg: {}, tokenizerJson: fixture.tokenizers.english.tokenizer,
+ dir: "https://example.test/model", revision: null,
+ } as any);
+ const provider = vi.spyOn(providers, "createWebProvider").mockResolvedValue({} as any);
+ vi.stubGlobal("window", {});
+ try {
+ const agent = await Agent.load("https://example.test/model");
+ const vector = fixture.tokenizers.english.cases.find((c: { name: string }) => c.name === "special");
+ expect(agent.tok.decode!(vector.ids)).toBe(vector.decoded);
+ } finally { bundle.mockRestore(); provider.mockRestore(); vi.unstubAllGlobals(); }
+ });
+});