Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
feat(ts): decode official tokenizers for windowed prediction
wow476562743e38
diff --git a/laya-ts/fixtures/tokenizer-decode.json b/laya-ts/fixtures/tokenizer-decode.jsonnew file mode 100644index 00000000..6793f1b2--- /dev/null+++ b/laya-ts/fixtures/tokenizer-decode.json@@ -0,0 +1,1668 @@+{+ "reference": {+ "repository": "convaiinnovations/laya",+ "revision": "55cf4c4ebb4ebe31b2550e8bdf3bd21b99753851",+ "tokenizers": "0.23.2",+ "files": {+ "tokenizer/tokenizer.json": "6c8aaa9a542084f2457eab775d4eeb51f92a70c0fd9de28d5edb0ddec3c08d30",+ "multilingual/tokenizer/tokenizer.json": "609d8f4c067cd3950f88594c5a802616cea245823836ef5848ee4fc40aab5b6f"+ }+ },+ "tokenizers": {+ "english": {+ "tokenizer": {+ "version": "1.0",+ "truncation": null,+ "padding": null,+ "added_tokens": [+ {+ "id": 50276,+ "content": " ",+ "single_word": false,+ "lstrip": false,+ "rstrip": false,+ "normalized": true,+ "special": false+ },+ {+ "id": 50281,+ "content": "[CLS]",+ "single_word": false,+ "lstrip": false,+ "rstrip": false,+ "normalized": false,+ "special": true+ },+ {+ "id": 50282,+ "content": "[SEP]",+ "single_word": false,+ "lstrip": false,+ "rstrip": false,+ "normalized": false,+ "special": true+ },+ {+ "id": 50284,+ "content": "[MASK]",+ "single_word": false,+ "lstrip": true,+ "rstrip": false,+ "normalized": false,+ "special": true+ }+ ],+ "normalizer": {+ "type": "NFC"+ },+ "pre_tokenizer": {+ "type": "ByteLevel",+ "add_prefix_space": false,+ "trim_offsets": true,+ "use_regex": true+ },+ "post_processor": null,+ "decoder": {+ "type": "ByteLevel",+ "add_prefix_space": false,+ "trim_offsets": true,+ "use_regex": true+ },+ "model": {+ "type": "BPE",+ "dropout": null,+ "unk_token": null,+ "continuing_subword_prefix": null,+ "end_of_word_suffix": null,+ "fuse_unk": false,+ "byte_fallback": false,+ "ignore_merges": false,+ "vocab": {+ "H": 41,+ "a": 66,+ "b": 67,+ "c": 68,+ "d": 69,+ "e": 70,+ "f": 71,+ "h": 73,+ "l": 77,+ "o": 80,+ "r": 83,+ "w": 88,+ "¢": 97,+ "©": 104,+ "«": 106,+ "¬": 107,+ "®": 108,+ "·": 117,+ "»": 121,+ "½": 123,+ "¾": 124,+ "¿": 125,+ "Ã": 127,+ "å": 161,+ "æ": 162,+ "é": 165,+ "ï": 171,+ "ð": 172,+ "Ā": 177,+ "ĉ": 186,+ "Ċ": 187,+ "Ġ": 209,+ "Ģ": 211,+ "ģ": 212,+ "Ĥ": 213,+ "Ī": 219,+ "ĺ": 235,+ "Ł": 242,+ "ł": 243,+ "he": 248,+ "Ġc": 260,+ "or": 263,+ "el": 293,+ "Ġhe": 344,+ "ld": 392,+ "ell": 437,+ "ll": 620,+ "é": 860,+ "af": 2320,+ "orld": 2875,+ "Ġé": 3974,+ "ello": 6646,+ "å®": 6916,+ "Ġhell": 7085,+ "æĪ": 8650,+ "world": 10186,+ "éĢ": 11016,+ "Hello": 12092,+ "ðŁ": 14931,+ "Ġcaf": 15660,+ "Ġï": 16141,+ "¿½": 19675,+ "�": 21478,+ "ĠðŁ": 22692,+ "Ġhello": 23120,+ "»¿": 24384,+ "æ¬": 25715,+ "Ġcafé": 36241,+ "": 41497,+ "��": 42481,+ "æĪ·": 46136,+ "ĠðŁĺ": 49042,+ " ": 50276,+ "[CLS]": 50281,+ "[SEP]": 50282,+ "[MASK]": 50284+ },+ "merges": [+ [+ "h",+ "e"+ ],+ [+ "Ġ",+ "c"+ ],+ [+ "o",+ "r"+ ],+ [+ "e",+ "l"+ ],+ [+ "Ġ",+ "he"+ ],+ [+ "l",+ "d"+ ],+ [+ "el",+ "l"+ ],+ [+ "l",+ "l"+ ],+ [+ "Ã",+ "©"+ ],+ [+ "a",+ "f"+ ],+ [+ "or",+ "ld"+ ],+ [+ "Ġ",+ "é"+ ],+ [+ "ell",+ "o"+ ],+ [+ "å",+ "®"+ ],+ [+ "Ġhe",+ "ll"+ ],+ [+ "æ",+ "Ī"+ ],+ [+ "w",+ "orld"+ ],+ [+ "é",+ "Ģ"+ ],+ [+ "H",+ "ello"+ ],+ [+ "ð",+ "Ł"+ ],+ [+ "Ġc",+ "af"+ ],+ [+ "Ġ",+ "ï"+ ],+ [+ "¿",+ "½"+ ],+ [+ "ï",+ "¿½"+ ],+ [+ "Ġ",+ "ðŁ"+ ],+ [+ "Ġhell",+ "o"+ ],+ [+ "»",+ "¿"+ ],+ [+ "æ",+ "¬"+ ],+ [+ "Ġcaf",+ "é"+ ],+ [+ "ï",+ "»¿"+ ],+ [+ "�",+ "�"+ ],+ [+ "æĪ",+ "·"+ ],+ [+ "ĠðŁ",+ "ĺ"+ ]+ ]+ }+ },+ "cases": [+ {+ "name": "empty",+ "ids": [],+ "decoded": "",+ "reencoded": []+ },+ {+ "name": "whitespace",+ "ids": [+ 50276,+ 12092,+ 187,+ 186,+ 10186,+ 50276+ ],+ "decoded": " Hello\n\tworld ",+ "reencoded": [+ 50276,+ 12092,+ 187,+ 186,+ 10186,+ 50276+ ]+ },+ {+ "name": "unicode",+ "ids": [+ 6916,+ 97,+ 46136,+ 11016,+ 211,+ 25715,+ 124,+ 36241,+ 3974,+ 49042,+ 211,+ 14931,+ 106,+ 243,+ 209,+ 172,+ 243,+ 211,+ 211,+ 172,+ 243,+ 211,+ 212+ ],+ "decoded": "客户退款 café é 😀🫠 𠀀𠀁",+ "reencoded": [+ 6916,+ 97,+ 46136,+ 11016,+ 211,+ 25715,+ 124,+ 36241,+ 3974,+ 49042,+ 211,+ 14931,+ 106,+ 243,+ 209,+ 172,+ 243,+ 211,+ 211,+ 172,+ 243,+ 211,+ 212+ ]+ },+ {+ "name": "bom-nul",+ "ids": [+ 41497,+ 66,+ 177,+ 67,+ 41497+ ],+ "decoded": "a\u0000b",+ "reencoded": [+ 41497,+ 66,+ 177,+ 67,+ 41497+ ]+ },+ {+ "name": "special",+ "ids": [+ 50281,+ 23120,+ 50284,+ 209,+ 50282+ ],+ "decoded": "[CLS] hello[MASK] [SEP]",+ "reencoded": [+ 50281,+ 23120,+ 50284,+ 209,+ 50282+ ]+ },+ {+ "name": "boundary-0",+ "ids": [+ 172,+ 243,+ 211+ ],+ "decoded": "�",+ "reencoded": [+ 21478+ ]+ },+ {+ "name": "boundary-1",+ "ids": [+ 243,+ 211,+ 211+ ],+ "decoded": "���",+ "reencoded": [+ 42481,+ 21478+ ]+ },+ {+ "name": "boundary-2",+ "ids": [+ 211,+ 211,+ 172+ ],+ "decoded": "���",+ "reencoded": [+ 42481,+ 21478+ ]+ },+ {+ "name": "boundary-3",+ "ids": [+ 211,+ 172,+ 243+ ],+ "decoded": "��",+ "reencoded": [+ 42481+ ]+ },+ {+ "name": "boundary-4",+ "ids": [+ 172,+ 243,+ 211+ ],+ "decoded": "�",+ "reencoded": [+ 21478+ ]+ },+ {+ "name": "boundary-5",+ "ids": [+ 243,+ 211,+ 212+ ],+ "decoded": "���",+ "reencoded": [+ 42481,+ 21478+ ]+ },+ {+ "name": "boundary-6",+ "ids": [+ 211,+ 212,+ 172+ ],+ "decoded": "���",+ "reencoded": [+ 42481,+ 21478+ ]+ },+ {+ "name": "boundary-7",+ "ids": [+ 212,+ 172,+ 243+ ],+ "decoded": "��",+ "reencoded": [+ 42481+ ]+ },+ {+ "name": "boundary-8",+ "ids": [+ 172,+ 243,+ 211+ ],+ "decoded": "�",+ "reencoded": [+ 21478+ ]+ },+ {+ "name": "boundary-9",+ "ids": [+ 243,+ 211,+ 213+ ],+ "decoded": "���",+ "reencoded": [+ 42481,+ 21478+ ]+ },+ {+ "name": "boundary-10",+ "ids": [+ 211,+ 213,+ 49042+ ],+ "decoded": "�� �",+ "reencoded": [+ 42481,+ 16141,+ 19675+ ]+ },+ {+ "name": "boundary-11",+ "ids": [+ 213,+ 49042,+ 211+ ],+ "decoded": "� 😀",+ "reencoded": [+ 21478,+ 49042,+ 211+ ]+ },+ {+ "name": "unknown-id",+ "ids": [+ 2147483647+ ],+ "decoded": "",+ "reencoded": []+ }+ ]+ },+ "multilingual": {+ "tokenizer": {+ "version": "1.0",+ "truncation": null,+ "padding": null,+ "added_tokens": [+ {+ "id": 1,+ "content": "<eos>",+ "single_word": false,+ "lstrip": false,+ "rstrip": false,+ "normalized": false,+ "special": true+ },+ {+ "id": 2,+ "content": "<bos>",+ "single_word": false,+ "lstrip": false,+ "rstrip": false,+ "normalized": false,+ "special": true+ },+ {+ "id": 4,+ "content": "<mask>",+ "single_word": false,+ "lstrip": true,+ "rstrip": false,+ "normalized": false,+ "special": true+ },+ {+ "id": 108,+ "content": "\n",+ "single_word": false,+ "lstrip": false,+ "rstrip": false,+ "normalized": false,+ "special": false+ }+ ],+ "normalizer": {+ "type": "Replace",+ "pattern": {+ "String": " "+ },+ "content": "▁"+ },+ "pre_tokenizer": {+ "type": "Metaspace",+ "replacement": "▁",+ "prepend_scheme": "always",+ "split": true+ },+ "post_processor": null,+ "decoder": {+ "type": "Sequence",+ "decoders": [+ {+ "type": "Replace",+ "pattern": {+ "String": "▁"+ },+ "content": " "+ },+ {+ "type": "ByteFallback"+ },+ {+ "type": "Fuse"+ }+ ]+ },+ "model": {+ "type": "BPE",+ "dropout": null,+ "unk_token": "<unk>",+ "continuing_subword_prefix": null,+ "end_of_word_suffix": null,+ "fuse_unk": true,+ "byte_fallback": true,+ "ignore_merges": false,+ "vocab": {+ "<eos>": 1,+ "<bos>": 2,+ "<unk>": 3,+ "<mask>": 4,+ "\n": 108,+ "<0x00>": 217,+ "\t": 226,+ "<0x61>": 314,+ "<0x80>": 345,+ "<0x81>": 346,+ "<0x82>": 347,+ "<0x98>": 369,+ "<0x9F>": 376,+ "<0xA0>": 377,+ "<0xBB>": 404,+ "<0xBF>": 408,+ "<0xEF>": 456,+ "<0xF0>": 457,+ "he": 480,+ "or": 483,+ "▁c": 498,+ "el": 521,+ "▁e": 526,+ "ll": 529,+ "▁h": 531,+ "lo": 545,+ "ca": 557,+ "▁H": 640,+ "▁he": 693,+ "ld": 706,+ "ell": 806,+ "wo": 1100,+ "▁He": 1315,+ "▁hel": 1487,+ "He": 1949,+ "▁ca": 1962,+ "Hel": 2405,+ "af": 2518,+ "orld": 2780,+ "hel": 3549,+ "llo": 3979,+ "Hello": 4521,+ "▁Hel": 5447,+ "fé": 5909,+ "我们": 7078,+ "world": 9097,+ "了一": 10151,+ "已经": 11546,+ "▁hell": 11944,+ "ello": 13444,+ "用户": 16464,+ "hello": 17534,+ "必要": 17986,+ "wor": 17987,+ "▁café": 18688,+ "hell": 20256,+ "原因": 22064,+ "��": 23484,+ "▁Hell": 24598,+ "▁hello": 25612,+ "▁Hello": 25957,+ "客户": 30379,+ "记录": 31274,+ "Hell": 34074,+ "说明": 34310,+ "检查": 37251,+ "▁caf": 41680,+ "申请": 43501,+ "▁": 45835,+ "支付": 46753,+ "▁�": 53152,+ "安排": 54402,+ "▁客": 62958,+ "确认": 64753,+ "caf": 67504,+ "费用": 71465,+ "重复": 79825,+ "▁😀": 79937,+ "café": 81453,+ "账户": 93391,+ "并在": 130004,+ "���": 135502,+ "退款": 186612,+ "▁��": 228059,+ "▁客户": 231537,+ "▁": 235248,+ "e": 235249,+ "a": 235250,+ "o": 235253,+ "r": 235255,+ "l": 235257,+ "d": 235258,+ "h": 235259,+ "c": 235260,+ "f": 235266,+ "b": 235268,+ "w": 235271,+ "H": 235314,+ "é": 235335,+ "。": 235362,+ ",": 235365,+ "一": 235411,+ "了": 235445,+ "在": 235473,+ "我": 235509,+ "用": 235522,+ "要": 235556,+ "时": 235635,+ "": 235705,+ "们": 235707,+ "说": 235726,+ "明": 235781,+ "重": 235846,+ "次": 235847,+ "安": 235859,+ "原": 235875,+ "只": 235918,+ "实": 235930,+ "经": 235931,+ "因": 235933,+ "已": 236023,+ "付": 236076,+ "必": 236188,+ "�": 236193,+ "并": 236203,+ "客": 236224,+ "查": 236293,+ "户": 236308,+ "支": 236324,+ "请": 236343,+ "记": 236364,+ "认": 236458,+ "复": 236465,+ "排": 236551,+ "录": 236586,+ "确": 236601,+ "款": 236604,+ "费": 236699,+ "́": 236703,+ "续": 236812,+ "退": 236924,+ "申": 237026,+ "检": 237147,+ "核": 237260,+ "订": 237743,+ "账": 238157,+ "扣": 238498,+ "😀": 239061,+ "🫠": 244497+ },+ "merges": [+ [+ "h",+ "e"+ ],+ [+ "o",+ "r"+ ],+ [+ "▁",+ "c"+ ],+ [+ "e",+ "l"+ ],+ [+ "▁",+ "e"+ ],+ [+ "l",+ "l"+ ],+ [+ "▁",+ "h"+ ],+ [+ "l",+ "o"+ ],+ [+ "c",+ "a"+ ],+ [+ "▁",+ "H"+ ],+ [+ "▁h",+ "e"+ ],+ [+ "▁",+ "he"+ ],+ [+ "l",+ "d"+ ],+ [+ "el",+ "l"+ ],+ [+ "e",+ "ll"+ ],+ [+ "w",+ "o"+ ],+ [+ "▁H",+ "e"+ ],+ [+ "▁",+ "He"+ ],+ [+ "▁he",+ "l"+ ],+ [+ "▁h",+ "el"+ ],+ [+ "▁",+ "hel"+ ],+ [+ "H",+ "e"+ ],+ [+ "▁c",+ "a"+ ],+ [+ "▁",+ "ca"+ ],+ [+ "He",+ "l"+ ],+ [+ "H",+ "el"+ ],+ [+ "a",+ "f"+ ],+ [+ "or",+ "ld"+ ],+ [+ "he",+ "l"+ ],+ [+ "h",+ "el"+ ],+ [+ "ll",+ "o"+ ],+ [+ "l",+ "lo"+ ],+ [+ "Hell",+ "o"+ ],+ [+ "Hel",+ "lo"+ ],+ [+ "He",+ "llo"+ ],+ [+ "H",+ "ello"+ ],+ [+ "▁He",+ "l"+ ],+ [+ "▁H",+ "el"+ ],+ [+ "▁",+ "Hel"+ ],+ [+ "f",+ "é"+ ],+ [+ "我",+ "们"+ ],+ [+ "wor",+ "ld"+ ],+ [+ "w",+ "orld"+ ],+ [+ "了",+ "一"+ ],+ [+ "已",+ "经"+ ],+ [+ "▁hel",+ "l"+ ],+ [+ "▁he",+ "ll"+ ],+ [+ "▁h",+ "ell"+ ],+ [+ "▁",+ "hell"+ ],+ [+ "ell",+ "o"+ ],+ [+ "el",+ "lo"+ ],+ [+ "e",+ "llo"+ ],+ [+ "用",+ "户"+ ],+ [+ "hell",+ "o"+ ],+ [+ "hel",+ "lo"+ ],+ [+ "he",+ "llo"+ ],+ [+ "h",+ "ello"+ ],+ [+ "必",+ "要"+ ],+ [+ "wo",+ "r"+ ],+ [+ "w",+ "or"+ ],+ [+ "▁caf",+ "é"+ ],+ [+ "▁ca",+ "fé"+ ],+ [+ "▁",+ "café"+ ],+ [+ "hel",+ "l"+ ],+ [+ "he",+ "ll"+ ],+ [+ "h",+ "ell"+ ],+ [+ "原",+ "因"+ ],+ [+ "�",+ "�"+ ],+ [+ "▁Hel",+ "l"+ ],+ [+ "▁He",+ "ll"+ ],+ [+ "▁H",+ "ell"+ ],+ [+ "▁",+ "Hell"+ ],+ [+ "▁hell",+ "o"+ ],+ [+ "▁hel",+ "lo"+ ],+ [+ "▁he",+ "llo"+ ],+ [+ "▁h",+ "ello"+ ],+ [+ "▁",+ "hello"+ ],+ [+ "▁Hell",+ "o"+ ],+ [+ "▁Hel",+ "lo"+ ],+ [+ "▁He",+ "llo"+ ],+ [+ "▁H",+ "ello"+ ],+ [+ "▁",+ "Hello"+ ],+ [+ "客",+ "户"+ ],+ [+ "记",+ "录"+ ],+ [+ "Hel",+ "l"+ ],+ [+ "He",+ "ll"+ ],+ [+ "H",+ "ell"+ ],+ [+ "说",+ "明"+ ],+ [+ "检",+ "查"+ ],+ [+ "▁ca",+ "f"+ ],+ [+ "▁c",+ "af"+ ],+ [+ "▁",+ "caf"+ ],+ [+ "申",+ "请"+ ],+ [+ "▁",+ ""+ ],+ [+ "支",+ "付"+ ],+ [+ "▁",+ "�"+ ],+ [+ "安",+ "排"+ ],+ [+ "▁",+ "客"+ ],+ [+ "确",+ "认"+ ],+ [+ "ca",+ "f"+ ],+ [+ "c",+ "af"+ ],+ [+ "费",+ "用"+ ],+ [+ "重",+ "复"+ ],+ [+ "▁",+ "😀"+ ],+ [+ "caf",+ "é"+ ],+ [+ "ca",+ "fé"+ ],+ [+ "账",+ "户"+ ],+ [+ "并",+ "在"+ ],+ [+ "��",+ "�"+ ],+ [+ "�",+ "��"+ ],+ [+ "退",+ "款"+ ],+ [+ "▁�",+ "�"+ ],+ [+ "▁",+ "��"+ ],+ [+ "▁客",+ "户"+ ],+ [+ "▁",+ "客户"+ ]+ ]+ }+ },+ "cases": [+ {+ "name": "empty",+ "ids": [],+ "decoded": "",+ "reencoded": []+ },+ {+ "name": "whitespace",+ "ids": [+ 235248,+ 25957,+ 108,+ 235248,+ 226,+ 9097,+ 235248,+ 235248+ ],+ "decoded": " Hello\n \tworld ",+ "reencoded": [+ 235248,+ 25957,+ 108,+ 235248,+ 226,+ 9097,+ 235248,+ 235248+ ]+ },+ {+ "name": "unicode",+ "ids": [+ 231537,+ 186612,+ 18688,+ 526,+ 236703,+ 79937,+ 244497,+ 235248,+ 457,+ 377,+ 345,+ 345,+ 457,+ 377,+ 345,+ 346+ ],+ "decoded": " 客户退款 café é 😀🫠 𠀀𠀁",+ "reencoded": [+ 231537,+ 186612,+ 18688,+ 526,+ 236703,+ 79937,+ 244497,+ 235248,+ 457,+ 377,+ 345,+ 345,+ 457,+ 377,+ 345,+ 346+ ]+ },+ {+ "name": "bom-nul",+ "ids": [+ 45835,+ 235250,+ 217,+ 235268,+ 235705+ ],+ "decoded": " a\u0000b",+ "reencoded": [+ 45835,+ 235250,+ 217,+ 235268,+ 235705+ ]+ },+ {+ "name": "special",+ "ids": [+ 2,+ 25612,+ 4,+ 1+ ],+ "decoded": "<bos> hello<mask><eos>",+ "reencoded": [+ 2,+ 25612,+ 4,+ 1+ ]+ },+ {+ "name": "boundary-0",+ "ids": [+ 235248,+ 457,+ 377+ ],+ "decoded": " ��",+ "reencoded": [+ 228059+ ]+ },+ {+ "name": "boundary-1",+ "ids": [+ 457,+ 377,+ 345+ ],+ "decoded": "���",+ "reencoded": [+ 235248,+ 135502+ ]+ },+ {+ "name": "boundary-2",+ "ids": [+ 377,+ 345,+ 345+ ],+ "decoded": "���",+ "reencoded": [+ 235248,+ 135502+ ]+ },+ {+ "name": "boundary-3",+ "ids": [+ 345,+ 345,+ 457+ ],+ "decoded": "���",+ "reencoded": [+ 235248,+ 135502+ ]+ },+ {+ "name": "boundary-4",+ "ids": [+ 345,+ 457,+ 377+ ],+ "decoded": "���",+ "reencoded": [+ 235248,+ 135502+ ]+ },+ {+ "name": "boundary-5",+ "ids": [+ 457,+ 377,+ 345+ ],+ "decoded": "���",+ "reencoded": [+ 235248,+ 135502+ ]+ },+ {+ "name": "boundary-6",+ "ids": [+ 377,+ 345,+ 346+ ],+ "decoded": "���",+ "reencoded": [+ 235248,+ 135502+ ]+ },+ {+ "name": "boundary-7",+ "ids": [+ 345,+ 346,+ 457+ ],+ "decoded": "���",+ "reencoded": [+ 235248,+ 135502+ ]+ },+ {+ "name": "boundary-8",+ "ids": [+ 346,+ 457,+ 377+ ],+ "decoded": "���",+ "reencoded": [+ 235248,+ 135502+ ]+ },+ {+ "name": "boundary-9",+ "ids": [+ 457,+ 377,+ 345+ ],+ "decoded": "���",+ "reencoded": [+ 235248,+ 135502+ ]+ },+ {+ "name": "boundary-10",+ "ids": [+ 377,+ 345,+ 347+ ],+ "decoded": "���",+ "reencoded": [+ 235248,+ 135502+ ]+ },+ {+ "name": "boundary-11",+ "ids": [+ 345,+ 347,+ 79937+ ],+ "decoded": "�� 😀",+ "reencoded": [+ 228059,+ 79937+ ]+ },+ {+ "name": "fallback-valid",+ "ids": [+ 457,+ 376,+ 369,+ 345+ ],+ "decoded": "😀",+ "reencoded": [+ 79937+ ]+ },+ {+ "name": "fallback-invalid-prefix",+ "ids": [+ 314,+ 457+ ],+ "decoded": "��",+ "reencoded": [+ 228059+ ]+ },+ {+ "name": "fallback-invalid-suffix",+ "ids": [+ 457,+ 314+ ],+ "decoded": "��",+ "reencoded": [+ 228059+ ]+ },+ {+ "name": "fallback-bom",+ "ids": [+ 456,+ 404,+ 408,+ 217+ ],+ "decoded": "\u0000",+ "reencoded": [+ 45835,+ 217+ ]+ },+ {+ "name": "reencoding-growth",+ "ids": [+ 71465,+ 235362,+ 7078,+ 11546,+ 37251,+ 93391,+ 31274,+ 235365,+ 64753,+ 16464,+ 235918,+ 43501,+ 10151,+ 235847,+ 236812,+ 237743,+ 235362,+ 236343,+ 237260,+ 235930,+ 46753,+ 31274,+ 235365,+ 34310,+ 79825,+ 238498,+ 236699,+ 22064,+ 235365,+ 130004,+ 17986,+ 235635,+ 54402+ ],+ "decoded": "费用。我们已经检查账户记录,确认用户只申请了一次续订。请核实支付记录,说明重复扣费原因,并在必要时安排",+ "reencoded": [+ 235248,+ 71465,+ 235362,+ 7078,+ 11546,+ 37251,+ 93391,+ 31274,+ 235365,+ 64753,+ 16464,+ 235918,+ 43501,+ 10151,+ 235847,+ 236812,+ 237743,+ 235362,+ 236343,+ 237260,+ 235930,+ 46753,+ 31274,+ 235365,+ 34310,+ 79825,+ 238498,+ 236699,+ 22064,+ 235365,+ 130004,+ 17986,+ 235635,+ 54402+ ]+ },+ {+ "name": "unknown-id",+ "ids": [+ 2147483647+ ],+ "decoded": "",+ "reencoded": []+ }+ ]+ }+ }+}diff --git a/laya-ts/scripts/tokenizer_decode_fixtures.py b/laya-ts/scripts/tokenizer_decode_fixtures.pynew file mode 100644index 00000000..9f2142c9--- /dev/null+++ b/laya-ts/scripts/tokenizer_decode_fixtures.py@@ -0,0 +1,152 @@+"""Record/check checkpoint decode -> re-encode vectors with Hugging Face tokenizers.++ python laya-ts/scripts/tokenizer_decode_fixtures.py --checkpoint-root SNAPSHOT+ python laya-ts/scripts/tokenizer_decode_fixtures.py --check++The checked-in tokenizer subsets retain the original token IDs and merge order.+Recording checks each subset against the complete checkpoint tokenizer; --check+executes the same HF decoder/encoder without downloading model assets. Supply+--checkpoint-root with --check to additionally verify the original asset hashes.+"""+from __future__ import annotations++import argparse+from copy import deepcopy+import hashlib+import json+from pathlib import Path++import tokenizers+from tokenizers import Tokenizer+++FIXTURE = Path(__file__).resolve().parents[1] / "fixtures/tokenizer-decode.json"+REVISION = "55cf4c4ebb4ebe31b2550e8bdf3bd21b99753851"+FILES = {"english": "tokenizer/tokenizer.json", "multilingual": "multilingual/tokenizer/tokenizer.json"}+SOURCE_HASHES = {+ "tokenizer/tokenizer.json": "6c8aaa9a542084f2457eab775d4eeb51f92a70c0fd9de28d5edb0ddec3c08d30",+ "multilingual/tokenizer/tokenizer.json": "609d8f4c067cd3950f88594c5a802616cea245823836ef5848ee4fc40aab5b6f",+}+++def vectors(tok, kind):+ texts = [+ ("empty", ""), ("whitespace", " Hello\n\tworld "),+ ("unicode", "客户退款 café e\u0301 😀🫠 𠀀𠀁"),+ ("bom-nul", "\ufeffa\0b\ufeff"),+ ("special", "[CLS] hello [MASK] [SEP]" if kind == "english" else "<bos> hello <mask><eos>"),+ ]+ cases = []++ def add(name, ids):+ decoded = tok.decode(ids, skip_special_tokens=False)+ cases.append({"name": name, "ids": ids, "decoded": decoded,+ "reencoded": tok.encode(decoded, add_special_tokens=False).ids})++ for name, text in texts:+ add(name, tok.encode(text, add_special_tokens=False).ids)+ boundary = tok.encode("𠀀𠀁𠀂 😀🫠 café 客户退款", add_special_tokens=False).ids+ for start in range(min(12, len(boundary))):+ add(f"boundary-{start}", boundary[start:start + 3])+ if kind == "multilingual":+ for name, values in [+ ("fallback-valid", [0xF0, 0x9F, 0x98, 0x80]),+ ("fallback-invalid-prefix", [0x61, 0xF0]),+ ("fallback-invalid-suffix", [0xF0, 0x61]),+ ("fallback-bom", [0xEF, 0xBB, 0xBF, 0]),+ ]:+ add(name, [tok.token_to_id(f"<0x{v:02X}>") for v in values])+ paragraph = ("客户表示,最近的账单中同一项订阅被收取了两次费用。"+ "我们已经检查账户记录,确认用户只申请了一次续订。"+ "请核实支付记录,说明重复扣费原因,并在必要时安排退款。"+ "客户可以提供发票号码和付款凭证。\n\n")+ ids = tok.encode(paragraph * 4, add_special_tokens=False).ids+ add("reencoding-growth", ids[16:49])+ add("unknown-id", [2**31 - 1])+ return cases+++def subset(raw, cases):+ """Keep every merge decomposition of the used tokens, in original rank order."""+ vocab = raw["model"]["vocab"]+ by_id = {v: k for k, v in vocab.items()}+ used_ids = {i for case in cases for key in ("ids", "reencoded") for i in case[key]}+ added = [t for t in raw["added_tokens"] if t["id"] in used_ids+ or any(t["content"] in case["decoded"] for case in cases)]+ wanted = {by_id[i] for case in cases for key in ("ids", "reencoded")+ for i in case[key] if i in by_id}+ wanted.update(t["content"] for t in added if t["content"] in vocab)+ if raw["model"].get("unk_token"):+ wanted.add(raw["model"]["unk_token"])+ merges = [m.split(" ") if isinstance(m, str) else m for m in raw["model"]["merges"]]+ decompositions = {}+ for pair in merges:+ decompositions.setdefault("".join(pair), []).append(pair)+ pending = list(wanted)+ while pending:+ for pair in decompositions.get(pending.pop(), []):+ for token in pair:+ if token not in wanted:+ wanted.add(token)+ pending.append(token)+ result = deepcopy(raw)+ result["added_tokens"] = added+ result["model"]["vocab"] = {k: v for k, v in vocab.items() if k in wanted}+ # HF otherwise allocates newly added token IDs from the smaller vocabulary size.+ result["model"]["vocab"].update({t["content"]: t["id"] for t in added})+ result["model"]["merges"] = [pair for pair in merges if "".join(pair) in wanted]+ # Encoding requests no special tokens. Avoid unused processor references in the subset.+ result["post_processor"] = None+ return result+++def verify(data):+ count = 0+ for kind, section in data["tokenizers"].items():+ tok = Tokenizer.from_str(json.dumps(section["tokenizer"]))+ for case in section["cases"]:+ decoded = tok.decode(case["ids"], skip_special_tokens=False)+ encoded = tok.encode(decoded, add_special_tokens=False).ids+ if decoded != case["decoded"] or encoded != case["reencoded"]:+ raise AssertionError(f"{kind}/{case['name']}: HF decode/re-encode changed")+ count += 1+ return count+++def record(root):+ data = {"reference": {"repository": "convaiinnovations/laya", "revision": REVISION,+ "tokenizers": tokenizers.__version__, "files": {}}, "tokenizers": {}}+ for kind, relative in FILES.items():+ contents = (root / relative).read_bytes()+ digest = hashlib.sha256(contents).hexdigest()+ if digest != SOURCE_HASHES[relative]:+ raise ValueError(f"{relative}: tokenizer does not match the pinned snapshot {REVISION}")+ data["reference"]["files"][relative] = digest+ raw = json.loads(contents)+ cases = vectors(Tokenizer.from_str(contents.decode("utf-8")), kind)+ data["tokenizers"][kind] = {"tokenizer": subset(raw, cases), "cases": cases}+ verify(data)+ return data+++def main():+ parser = argparse.ArgumentParser(description=__doc__)+ parser.add_argument("--checkpoint-root", type=Path)+ parser.add_argument("--check", action="store_true")+ args = parser.parse_args()+ if args.check:+ stored = json.loads(FIXTURE.read_text(encoding="utf-8"))+ count = verify(stored)+ if args.checkpoint_root and record(args.checkpoint_root) != stored:+ raise AssertionError("checkpoint tokenizer fixture differs; review before recording")+ print(f"PASS: {count} HF decode/re-encode vectors (tokenizers {tokenizers.__version__})")+ else:+ if not args.checkpoint_root:+ parser.error("recording requires --checkpoint-root at the pinned snapshot")+ data = record(args.checkpoint_root)+ FIXTURE.write_text(json.dumps(data, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")+ print(f"Recorded {FIXTURE}")+++if __name__ == "__main__":+ main()diff --git a/laya-ts/src/agent.ts b/laya-ts/src/agent.tsindex 4b439b1f..bfef3b9f 100644--- a/laya-ts/src/agent.ts+++ b/laya-ts/src/agent.ts@@ -420,6 +420,7 @@ function tokenizerFromHF(tokenizerJson: unknown): TokenizerLike | null {padId: data.ids.pad,maskToken: data.maskToken,encode: (text: string) => encodeWithData(data, text),+ ...(data.decode ? { decode: data.decode } : {}),};}diff --git a/laya-ts/src/tokenizer.ts b/laya-ts/src/tokenizer.tsindex 3e460d0d..be8da866 100644--- a/laya-ts/src/tokenizer.ts+++ b/laya-ts/src/tokenizer.ts@@ -3,6 +3,8 @@ export interface TokenizerLike {readonly maskId: number; readonly padId: number;readonly maskToken: string;encode(text: string): number[];+ /** Decode without removing special tokens; needed only when predictLong splits the state. */+ decode?(ids: readonly number[]): string;}/** Special ids of the laya ModernBERT checkpoint (HF added_tokens). */@@ -31,6 +33,8 @@ export interface TokenizerData {byteFallback?: boolean;/** Tokens cut out of the text before it is tokenized (HF added_tokens). */added?: AddedToken[];+ /** Absent when tokenizer.json uses a decoder this runtime does not implement. */+ decode?: (ids: readonly number[]) => string;}export interface AddedToken { content: string; id: number; normalized: boolean; lstrip: boolean; rstrip: boolean }@@ -411,6 +415,64 @@ function collectReplaces(node: unknown, out: Array<[string, string]>): void {for (const c of childNodes(node)) collectReplaces(c, out);}+/** Compile the two checkpoint decoders, retaining HF's skip_special_tokens=False behavior. */+function checkpointDecoder(+ raw: unknown, vocab: Map<string, number>, added: AddedToken[],+): ((ids: readonly number[]) => string) | undefined {+ if (!raw || typeof raw !== "object") return undefined;+ const decoder = raw as Record<string, unknown>;+ let decodeTokens: (tokens: string[]) => string;+ // ignoreBOM preserves U+FEFF in a decoded window, just as the HF decoder does.+ const utf8 = new TextDecoder("utf-8", { ignoreBOM: true });+ if (decoder.type === "ByteLevel") {+ decodeTokens = (tokens) => {+ const bytes: number[] = [];+ const { u2b } = maps();+ for (const token of tokens) {+ const mapped = Array.from(token, (ch) => u2b.get(ch));+ // HF falls back to the whole token's UTF-8 bytes if any character is unmapped.+ const part = mapped.every((b) => b !== undefined)+ ? mapped as number[] : (sharedEncoder ??= new TextEncoder()).encode(token);+ for (const b of part) bytes.push(b);+ }+ return utf8.decode(new Uint8Array(bytes));+ };+ } else if (decoder.type === "Sequence" && Array.isArray(decoder.decoders)) {+ const steps = decoder.decoders as Array<Record<string, unknown> | null>;+ const pattern = steps[0]?.pattern as Record<string, unknown> | undefined;+ if (steps.length !== 3 || steps[0]?.type !== "Replace"+ || typeof pattern?.String !== "string" || pattern.String.length === 0+ || typeof steps[0].content !== "string"+ || steps[1]?.type !== "ByteFallback" || steps[2]?.type !== "Fuse") return undefined;+ const from = pattern.String, to = steps[0].content;+ const strict = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true });+ decodeTokens = (tokens) => {+ const out: string[] = [], bytes: number[] = [];+ const flush = (): void => {+ if (bytes.length === 0) return;+ // ByteFallback rejects an entire invalid run, including any otherwise valid prefix.+ try { out.push(strict.decode(new Uint8Array(bytes))); }+ catch { out.push("\uFFFD".repeat(bytes.length)); }+ bytes.length = 0;+ };+ for (const token of tokens) {+ const replaced = token.split(from).join(to);+ const match = /^<0x([\dA-Fa-f]{2})>$/.exec(replaced);+ if (match) bytes.push(parseInt(match[1], 16));+ else { flush(); out.push(replaced); }+ }+ flush();+ return out.join("");+ };+ } else return undefined;+ const byId = new Map(Array.from(vocab, ([token, id]) => [id, token]));+ for (const token of added) byId.set(token.id, token.content);+ return (ids) => decodeTokens(ids.flatMap((id) => {+ const token = byId.get(id);+ return token === undefined ? [] : [token];+ }));+}+/** Parse an HF tokenizer.json ({model vocab/merges, normalizer, pre_tokenizer, added_tokens}). */export function parseTokenizerJson(raw: unknown): TokenizerData | null {try {@@ -418,6 +480,7 @@ export function parseTokenizerJson(raw: unknown): TokenizerData | null {model?: { vocab?: Record<string, number>; merges?: Array<string | [string, string]>; byte_fallback?: boolean };normalizer?: unknown;pre_tokenizer?: unknown;+ decoder?: unknown;added_tokens?: Array<{ id?: number; content?: string; normalized?: boolean; lstrip?: boolean; rstrip?: boolean }>;};const vocabObj = r?.model?.vocab;@@ -463,6 +526,7 @@ export function parseTokenizerJson(raw: unknown): TokenizerData | null {replaces,byteFallback: r.model?.byte_fallback === true,added: addedTokens,+ decode: checkpointDecoder(r.decoder, vocab, addedTokens),};} catch {return null;diff --git a/laya-ts/tests/tokenizer-decode.test.ts b/laya-ts/tests/tokenizer-decode.test.tsnew file mode 100644index 00000000..fa1afb1f--- /dev/null+++ b/laya-ts/tests/tokenizer-decode.test.ts@@ -0,0 +1,79 @@+import { readFileSync } from "node:fs";+import { describe, expect, it, vi } from "vitest";+import { encodeWithData, parseTokenizerJson } from "../src/tokenizer.js";++const fixture = JSON.parse(readFileSync(new URL("../fixtures/tokenizer-decode.json", import.meta.url), "utf8"));++describe("checkpoint decode/re-encode parity", () => {+ for (const [kind, raw] of Object.entries(fixture.tokenizers)) {+ const section = raw as {+ tokenizer: unknown;+ cases: Array<{ name: string; ids: number[]; decoded: string; reencoded: number[] }>;+ };+ const data = parseTokenizerJson(section.tokenizer)!;+ for (const vector of section.cases) {+ it(`${kind}: ${vector.name}`, () => {+ const decoded = data.decode!(vector.ids);+ expect(decoded).toBe(vector.decoded);+ expect(encodeWithData(data, decoded)).toEqual(vector.reencoded);+ });+ }+ }++ it("records actual multilingual re-encoding growth instead of promising lossless windows", () => {+ const vector = fixture.tokenizers.multilingual.cases.find((c: { name: string }) => c.name === "reencoding-growth");+ expect(vector.ids).toHaveLength(33);+ expect(vector.reencoded).toHaveLength(34);+ });+});++describe("decoder boundaries", () => {+ const byteLevel = (vocab: Record<string, number>, added_tokens: unknown[] = []) => parseTokenizerJson({+ model: { vocab, merges: [] }, decoder: { type: "ByteLevel" }, added_tokens,+ })!;++ it("decodes UTF-8 across token boundaries and preserves BOM/NUL", () => {+ const data = byteLevel({ "ð": 0, "Ł": 1, "ĺ": 2, "Ģ": 3 });+ expect(data.decode!([0, 1, 2, 3])).toBe("😀");+ const bom = byteLevel({ "ï": 0, "»": 1, "¿": 2, "Ā": 3 });+ expect(bom.decode!([0, 1, 2, 3])).toBe("\ufeff\0");+ });++ it("falls back to an unmappable added token as a whole, retaining special tokens", () => {+ const data = byteLevel({ "Ġ": 0 }, [+ { id: 1, content: "Ġ🫠", special: true },+ { id: 2, content: "[MASK]", special: true },+ ]);+ expect(data.decode!([0, 1, 2, 999])).toBe(" Ġ🫠[MASK]");+ });++ it("keeps encoding available for absent or unsupported decoders", () => {+ for (const decoder of [undefined, { type: "Metaspace" }, { type: "Sequence", decoders: [+ { type: "Replace", pattern: { Regex: "▁" }, content: " " },+ { type: "ByteFallback" }, { type: "Fuse" },+ ] }, { type: "Sequence", decoders: [+ { type: "Replace", pattern: { String: "▁" }, content: " " },+ { type: "ByteFallback" }, { type: "Fuse" }, { type: "Strip" },+ ] }]) {+ const data = parseTokenizerJson({ model: { vocab: { a: 1 }, merges: [] }, decoder })!;+ expect(data.decode).toBeUndefined();+ expect(encodeWithData(data, "a")).toEqual([1]);+ }+ });++ it("loads a decoder into Agent without needing a Node tokenizer dependency", async () => {+ const providers = await import("../src/providers.js");+ const { Agent } = await import("../src/agent.js");+ const bundle = vi.spyOn(providers, "loadWebBundle").mockResolvedValue({+ cfg: {}, tokenizerJson: fixture.tokenizers.english.tokenizer,+ dir: "https://example.test/model", revision: null,+ } as any);+ const provider = vi.spyOn(providers, "createWebProvider").mockResolvedValue({} as any);+ vi.stubGlobal("window", {});+ try {+ const agent = await Agent.load("https://example.test/model");+ const vector = fixture.tokenizers.english.cases.find((c: { name: string }) => c.name === "special");+ expect(agent.tok.decode!(vector.ids)).toBe(vector.decoded);+ } finally { bundle.mockRestore(); provider.mockRestore(); vi.unstubAllGlobals(); }+ });+});