Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

fix(java): the pre-tokenizer's letter class was probed by an unsound question

Adeshina Lasisi64e4fc8706c9
diff --git a/laya-java/laya-java/src/main/java/com/convaiinnovations/laya/tokenizer/PreTokenizerTables.java b/laya-java/laya-java/src/main/java/com/convaiinnovations/laya/tokenizer/PreTokenizerTables.java
index 97f3ec83..e9d7fe72 100644
--- a/laya-java/laya-java/src/main/java/com/convaiinnovations/laya/tokenizer/PreTokenizerTables.java
+++ b/laya-java/laya-java/src/main/java/com/convaiinnovations/laya/tokenizer/PreTokenizerTables.java
@@ -30,139 +30,123 @@ public final class PreTokenizerTables {
/**
* The pattern's {@code \\p{L}}: what the reference treats as a letter.
*
- * <p>777 ranges covering 142963 code points, probed from
+ * <p>677 ranges covering 141028 code points, probed from
* the reference tokenizer.
*/
private static final int[] LETTER = letterTable();
private static int[] letterTable() {
return new int[] {
- 9, 13, 48, 57, 65, 90, 97, 122, 133, 133, 160, 160, 170, 170, 178, 179, 181, 181,
- 185, 186, 188, 190, 192, 214, 216, 246, 248, 705, 710, 721, 736, 740, 748, 748,
- 750, 750, 880, 884, 886, 887, 890, 893, 895, 895, 902, 902, 904, 906, 908, 908,
- 910, 929, 931, 1013, 1015, 1153, 1162, 1327, 1329, 1366, 1369, 1369, 1376, 1416,
- 1488, 1514, 1519, 1522, 1568, 1610, 1632, 1641, 1646, 1647, 1649, 1747, 1749, 1749,
- 1765, 1766, 1774, 1788, 1791, 1791, 1808, 1808, 1810, 1839, 1869, 1957, 1969, 1969,
- 1984, 2026, 2036, 2037, 2042, 2042, 2048, 2069, 2074, 2074, 2084, 2084, 2088, 2088,
- 2112, 2136, 2144, 2154, 2160, 2183, 2185, 2190, 2208, 2249, 2308, 2361, 2365, 2365,
- 2384, 2384, 2392, 2401, 2406, 2415, 2417, 2432, 2437, 2444, 2447, 2448, 2451, 2472,
- 2474, 2480, 2482, 2482, 2486, 2489, 2493, 2493, 2510, 2510, 2524, 2525, 2527, 2529,
- 2534, 2545, 2548, 2553, 2556, 2556, 2565, 2570, 2575, 2576, 2579, 2600, 2602, 2608,
- 2610, 2611, 2613, 2614, 2616, 2617, 2649, 2652, 2654, 2654, 2662, 2671, 2674, 2676,
- 2693, 2701, 2703, 2705, 2707, 2728, 2730, 2736, 2738, 2739, 2741, 2745, 2749, 2749,
- 2768, 2768, 2784, 2785, 2790, 2799, 2809, 2809, 2821, 2828, 2831, 2832, 2835, 2856,
- 2858, 2864, 2866, 2867, 2869, 2873, 2877, 2877, 2908, 2909, 2911, 2913, 2918, 2927,
- 2929, 2935, 2947, 2947, 2949, 2954, 2958, 2960, 2962, 2965, 2969, 2970, 2972, 2972,
- 2974, 2975, 2979, 2980, 2984, 2986, 2990, 3001, 3024, 3024, 3046, 3058, 3077, 3084,
- 3086, 3088, 3090, 3112, 3114, 3129, 3133, 3133, 3160, 3162, 3165, 3165, 3168, 3169,
- 3174, 3183, 3192, 3198, 3200, 3200, 3205, 3212, 3214, 3216, 3218, 3240, 3242, 3251,
- 3253, 3257, 3261, 3261, 3293, 3294, 3296, 3297, 3302, 3311, 3313, 3314, 3332, 3340,
- 3342, 3344, 3346, 3386, 3389, 3389, 3406, 3406, 3412, 3414, 3416, 3425, 3430, 3448,
- 3450, 3455, 3461, 3478, 3482, 3505, 3507, 3515, 3517, 3517, 3520, 3526, 3558, 3567,
- 3585, 3632, 3634, 3635, 3648, 3654, 3664, 3673, 3713, 3714, 3716, 3716, 3718, 3722,
- 3724, 3747, 3749, 3749, 3751, 3760, 3762, 3763, 3773, 3773, 3776, 3780, 3782, 3782,
- 3792, 3801, 3804, 3807, 3840, 3840, 3872, 3891, 3904, 3911, 3913, 3948, 3976, 3980,
- 4096, 4138, 4159, 4169, 4176, 4181, 4186, 4189, 4193, 4193, 4197, 4198, 4206, 4208,
- 4213, 4225, 4238, 4238, 4240, 4249, 4256, 4293, 4295, 4295, 4301, 4301, 4304, 4346,
- 4348, 4680, 4682, 4685, 4688, 4694, 4696, 4696, 4698, 4701, 4704, 4744, 4746, 4749,
- 4752, 4784, 4786, 4789, 4792, 4798, 4800, 4800, 4802, 4805, 4808, 4822, 4824, 4880,
- 4882, 4885, 4888, 4954, 4969, 4988, 4992, 5007, 5024, 5109, 5112, 5117, 5121, 5740,
- 5743, 5786, 5792, 5866, 5870, 5880, 5888, 5905, 5919, 5937, 5952, 5969, 5984, 5996,
- 5998, 6000, 6016, 6067, 6103, 6103, 6108, 6108, 6112, 6121, 6128, 6137, 6160, 6169,
- 6176, 6264, 6272, 6276, 6279, 6312, 6314, 6314, 6320, 6389, 6400, 6430, 6470, 6509,
- 6512, 6516, 6528, 6571, 6576, 6601, 6608, 6618, 6656, 6678, 6688, 6740, 6784, 6793,
- 6800, 6809, 6823, 6823, 6917, 6963, 6981, 6988, 6992, 7001, 7043, 7072, 7086, 7141,
- 7168, 7203, 7232, 7241, 7245, 7293, 7296, 7306, 7312, 7354, 7357, 7359, 7401, 7404,
- 7406, 7411, 7413, 7414, 7418, 7418, 7424, 7615, 7680, 7957, 7960, 7965, 7968, 8005,
- 8008, 8013, 8016, 8023, 8025, 8025, 8027, 8027, 8029, 8029, 8031, 8061, 8064, 8116,
- 8118, 8124, 8126, 8126, 8130, 8132, 8134, 8140, 8144, 8147, 8150, 8155, 8160, 8172,
- 8178, 8180, 8182, 8188, 8192, 8202, 8232, 8233, 8239, 8239, 8287, 8287, 8304, 8305,
- 8308, 8313, 8319, 8329, 8336, 8348, 8450, 8450, 8455, 8455, 8458, 8467, 8469, 8469,
- 8473, 8477, 8484, 8484, 8486, 8486, 8488, 8488, 8490, 8493, 8495, 8505, 8508, 8511,
- 8517, 8521, 8526, 8526, 8528, 8585, 9312, 9371, 9450, 9471, 10102, 10131, 11264,
- 11492, 11499, 11502, 11506, 11507, 11517, 11517, 11520, 11557, 11559, 11559, 11565,
- 11565, 11568, 11623, 11631, 11631, 11648, 11670, 11680, 11686, 11688, 11694, 11696,
- 11702, 11704, 11710, 11712, 11718, 11720, 11726, 11728, 11734, 11736, 11742, 11823,
- 11823, 12288, 12288, 12293, 12295, 12321, 12329, 12337, 12341, 12344, 12348, 12353,
- 12438, 12445, 12447, 12449, 12538, 12540, 12543, 12549, 12591, 12593, 12686, 12690,
- 12693, 12704, 12735, 12784, 12799, 12832, 12841, 12872, 12879, 12881, 12895, 12928,
- 12937, 12977, 12991, 13312, 19903, 19968, 42124, 42192, 42237, 42240, 42508, 42512,
- 42539, 42560, 42606, 42623, 42653, 42656, 42735, 42775, 42783, 42786, 42888, 42891,
- 42957, 42960, 42961, 42963, 42963, 42965, 42972, 42994, 43009, 43011, 43013, 43015,
- 43018, 43020, 43042, 43056, 43061, 43072, 43123, 43138, 43187, 43216, 43225, 43250,
- 43255, 43259, 43259, 43261, 43262, 43264, 43301, 43312, 43334, 43360, 43388, 43396,
- 43442, 43471, 43481, 43488, 43492, 43494, 43518, 43520, 43560, 43584, 43586, 43588,
- 43595, 43600, 43609, 43616, 43638, 43642, 43642, 43646, 43695, 43697, 43697, 43701,
- 43702, 43705, 43709, 43712, 43712, 43714, 43714, 43739, 43741, 43744, 43754, 43762,
- 43764, 43777, 43782, 43785, 43790, 43793, 43798, 43808, 43814, 43816, 43822, 43824,
- 43866, 43868, 43881, 43888, 44002, 44016, 44025, 44032, 55203, 55216, 55238, 55243,
- 55291, 63744, 64109, 64112, 64217, 64256, 64262, 64275, 64279, 64285, 64285, 64287,
- 64296, 64298, 64310, 64312, 64316, 64318, 64318, 64320, 64321, 64323, 64324, 64326,
- 64433, 64467, 64829, 64848, 64911, 64914, 64967, 65008, 65019, 65136, 65140, 65142,
- 65276, 65296, 65305, 65313, 65338, 65345, 65370, 65382, 65470, 65474, 65479, 65482,
- 65487, 65490, 65495, 65498, 65500, 65536, 65547, 65549, 65574, 65576, 65594, 65596,
- 65597, 65599, 65613, 65616, 65629, 65664, 65786, 65799, 65843, 65856, 65912, 65930,
- 65931, 66176, 66204, 66208, 66256, 66273, 66299, 66304, 66339, 66349, 66378, 66384,
- 66421, 66432, 66461, 66464, 66499, 66504, 66511, 66513, 66517, 66560, 66717, 66720,
- 66729, 66736, 66771, 66776, 66811, 66816, 66855, 66864, 66915, 66928, 66938, 66940,
- 66954, 66956, 66962, 66964, 66965, 66967, 66977, 66979, 66993, 66995, 67001, 67003,
- 67004, 67008, 67059, 67072, 67382, 67392, 67413, 67424, 67431, 67456, 67461, 67463,
- 67504, 67506, 67514, 67584, 67589, 67592, 67592, 67594, 67637, 67639, 67640, 67644,
- 67644, 67647, 67669, 67672, 67702, 67705, 67742, 67751, 67759, 67808, 67826, 67828,
- 67829, 67835, 67867, 67872, 67897, 67968, 68023, 68028, 68047, 68050, 68096, 68112,
- 68115, 68117, 68119, 68121, 68149, 68160, 68168, 68192, 68222, 68224, 68255, 68288,
- 68295, 68297, 68324, 68331, 68335, 68352, 68405, 68416, 68437, 68440, 68466, 68472,
- 68497, 68521, 68527, 68608, 68680, 68736, 68786, 68800, 68850, 68858, 68899, 68912,
- 68921, 68928, 68965, 68975, 68997, 69216, 69246, 69248, 69289, 69296, 69297, 69314,
- 69316, 69376, 69415, 69424, 69445, 69457, 69460, 69488, 69505, 69552, 69579, 69600,
- 69622, 69635, 69687, 69714, 69743, 69745, 69746, 69749, 69749, 69763, 69807, 69840,
- 69864, 69872, 69881, 69891, 69926, 69942, 69951, 69956, 69956, 69959, 69959, 69968,
- 70002, 70006, 70006, 70019, 70066, 70081, 70084, 70096, 70106, 70108, 70108, 70113,
- 70132, 70144, 70161, 70163, 70187, 70207, 70208, 70272, 70278, 70280, 70280, 70282,
- 70285, 70287, 70301, 70303, 70312, 70320, 70366, 70384, 70393, 70405, 70412, 70415,
- 70416, 70419, 70440, 70442, 70448, 70450, 70451, 70453, 70457, 70461, 70461, 70480,
- 70480, 70493, 70497, 70528, 70537, 70539, 70539, 70542, 70542, 70544, 70581, 70583,
- 70583, 70609, 70609, 70611, 70611, 70656, 70708, 70727, 70730, 70736, 70745, 70751,
- 70753, 70784, 70831, 70852, 70853, 70855, 70855, 70864, 70873, 71040, 71086, 71128,
- 71131, 71168, 71215, 71236, 71236, 71248, 71257, 71296, 71338, 71352, 71352, 71360,
- 71369, 71376, 71395, 71424, 71450, 71472, 71483, 71488, 71494, 71680, 71723, 71840,
- 71922, 71935, 71942, 71945, 71945, 71948, 71955, 71957, 71958, 71960, 71983, 71999,
- 71999, 72001, 72001, 72016, 72025, 72096, 72103, 72106, 72144, 72161, 72161, 72163,
- 72163, 72192, 72192, 72203, 72242, 72250, 72250, 72272, 72272, 72284, 72329, 72349,
- 72349, 72368, 72440, 72640, 72672, 72688, 72697, 72704, 72712, 72714, 72750, 72768,
- 72768, 72784, 72812, 72818, 72847, 72960, 72966, 72968, 72969, 72971, 73008, 73030,
- 73030, 73040, 73049, 73056, 73061, 73063, 73064, 73066, 73097, 73112, 73112, 73120,
- 73129, 73440, 73458, 73474, 73474, 73476, 73488, 73490, 73523, 73552, 73561, 73648,
- 73648, 73664, 73684, 73728, 74649, 74752, 74862, 74880, 75075, 77712, 77808, 77824,
- 78895, 78913, 78918, 78944, 82938, 82944, 83526, 90368, 90397, 90416, 90425, 92160,
- 92728, 92736, 92766, 92768, 92777, 92784, 92862, 92864, 92873, 92880, 92909, 92928,
- 92975, 92992, 92995, 93008, 93017, 93019, 93025, 93027, 93047, 93053, 93071, 93504,
- 93548, 93552, 93561, 93760, 93846, 93952, 94026, 94032, 94032, 94099, 94111, 94176,
- 94177, 94179, 94179, 94208, 100343, 100352, 101589, 101631, 101640, 110576, 110579,
- 110581, 110587, 110589, 110590, 110592, 110882, 110898, 110898, 110928, 110930,
- 110933, 110933, 110948, 110951, 110960, 111355, 113664, 113770, 113776, 113788,
- 113792, 113800, 113808, 113817, 118000, 118009, 119488, 119507, 119520, 119539,
- 119648, 119672, 119808, 119892, 119894, 119964, 119966, 119967, 119970, 119970,
- 119973, 119974, 119977, 119980, 119982, 119993, 119995, 119995, 119997, 120003,
- 120005, 120069, 120071, 120074, 120077, 120084, 120086, 120092, 120094, 120121,
- 120123, 120126, 120128, 120132, 120134, 120134, 120138, 120144, 120146, 120485,
- 120488, 120512, 120514, 120538, 120540, 120570, 120572, 120596, 120598, 120628,
- 120630, 120654, 120656, 120686, 120688, 120712, 120714, 120744, 120746, 120770,
- 120772, 120779, 120782, 120831, 122624, 122654, 122661, 122666, 122928, 122989,
- 123136, 123180, 123191, 123197, 123200, 123209, 123214, 123214, 123536, 123565,
- 123584, 123627, 123632, 123641, 124112, 124139, 124144, 124153, 124368, 124397,
- 124400, 124410, 124896, 124902, 124904, 124907, 124909, 124910, 124912, 124926,
- 124928, 125124, 125127, 125135, 125184, 125251, 125259, 125259, 125264, 125273,
- 126065, 126123, 126125, 126127, 126129, 126132, 126209, 126253, 126255, 126269,
- 126464, 126467, 126469, 126495, 126497, 126498, 126500, 126500, 126503, 126503,
- 126505, 126514, 126516, 126519, 126521, 126521, 126523, 126523, 126530, 126530,
- 126535, 126535, 126537, 126537, 126539, 126539, 126541, 126543, 126545, 126546,
- 126548, 126548, 126551, 126551, 126553, 126553, 126555, 126555, 126557, 126557,
- 126559, 126559, 126561, 126562, 126564, 126564, 126567, 126570, 126572, 126578,
- 126580, 126583, 126585, 126588, 126590, 126590, 126592, 126601, 126603, 126619,
- 126625, 126627, 126629, 126633, 126635, 126651, 127232, 127244, 130032, 130041,
- 131072, 173791, 173824, 177977, 177984, 178205, 178208, 183969, 183984, 191456,
- 191472, 192093, 194560, 195101, 196608, 201546, 201552, 205743
+ 65, 90, 97, 122, 170, 170, 181, 181, 186, 186, 192, 214, 216, 246, 248, 705, 710,
+ 721, 736, 740, 748, 748, 750, 750, 880, 884, 886, 887, 890, 893, 895, 895, 902,
+ 902, 904, 906, 908, 908, 910, 929, 931, 1013, 1015, 1153, 1162, 1327, 1329, 1366,
+ 1369, 1369, 1376, 1416, 1488, 1514, 1519, 1522, 1568, 1610, 1646, 1647, 1649, 1747,
+ 1749, 1749, 1765, 1766, 1774, 1775, 1786, 1788, 1791, 1791, 1808, 1808, 1810, 1839,
+ 1869, 1957, 1969, 1969, 1994, 2026, 2036, 2037, 2042, 2042, 2048, 2069, 2074, 2074,
+ 2084, 2084, 2088, 2088, 2112, 2136, 2144, 2154, 2160, 2183, 2185, 2190, 2208, 2249,
+ 2308, 2361, 2365, 2365, 2384, 2384, 2392, 2401, 2417, 2432, 2437, 2444, 2447, 2448,
+ 2451, 2472, 2474, 2480, 2482, 2482, 2486, 2489, 2493, 2493, 2510, 2510, 2524, 2525,
+ 2527, 2529, 2544, 2545, 2556, 2556, 2565, 2570, 2575, 2576, 2579, 2600, 2602, 2608,
+ 2610, 2611, 2613, 2614, 2616, 2617, 2649, 2652, 2654, 2654, 2674, 2676, 2693, 2701,
+ 2703, 2705, 2707, 2728, 2730, 2736, 2738, 2739, 2741, 2745, 2749, 2749, 2768, 2768,
+ 2784, 2785, 2809, 2809, 2821, 2828, 2831, 2832, 2835, 2856, 2858, 2864, 2866, 2867,
+ 2869, 2873, 2877, 2877, 2908, 2909, 2911, 2913, 2929, 2929, 2947, 2947, 2949, 2954,
+ 2958, 2960, 2962, 2965, 2969, 2970, 2972, 2972, 2974, 2975, 2979, 2980, 2984, 2986,
+ 2990, 3001, 3024, 3024, 3077, 3084, 3086, 3088, 3090, 3112, 3114, 3129, 3133, 3133,
+ 3160, 3162, 3165, 3165, 3168, 3169, 3200, 3200, 3205, 3212, 3214, 3216, 3218, 3240,
+ 3242, 3251, 3253, 3257, 3261, 3261, 3293, 3294, 3296, 3297, 3313, 3314, 3332, 3340,
+ 3342, 3344, 3346, 3386, 3389, 3389, 3406, 3406, 3412, 3414, 3423, 3425, 3450, 3455,
+ 3461, 3478, 3482, 3505, 3507, 3515, 3517, 3517, 3520, 3526, 3585, 3632, 3634, 3635,
+ 3648, 3654, 3713, 3714, 3716, 3716, 3718, 3722, 3724, 3747, 3749, 3749, 3751, 3760,
+ 3762, 3763, 3773, 3773, 3776, 3780, 3782, 3782, 3804, 3807, 3840, 3840, 3904, 3911,
+ 3913, 3948, 3976, 3980, 4096, 4138, 4159, 4159, 4176, 4181, 4186, 4189, 4193, 4193,
+ 4197, 4198, 4206, 4208, 4213, 4225, 4238, 4238, 4256, 4293, 4295, 4295, 4301, 4301,
+ 4304, 4346, 4348, 4680, 4682, 4685, 4688, 4694, 4696, 4696, 4698, 4701, 4704, 4744,
+ 4746, 4749, 4752, 4784, 4786, 4789, 4792, 4798, 4800, 4800, 4802, 4805, 4808, 4822,
+ 4824, 4880, 4882, 4885, 4888, 4954, 4992, 5007, 5024, 5109, 5112, 5117, 5121, 5740,
+ 5743, 5759, 5761, 5786, 5792, 5866, 5873, 5880, 5888, 5905, 5919, 5937, 5952, 5969,
+ 5984, 5996, 5998, 6000, 6016, 6067, 6103, 6103, 6108, 6108, 6176, 6264, 6272, 6276,
+ 6279, 6312, 6314, 6314, 6320, 6389, 6400, 6430, 6480, 6509, 6512, 6516, 6528, 6571,
+ 6576, 6601, 6656, 6678, 6688, 6740, 6823, 6823, 6917, 6963, 6981, 6988, 7043, 7072,
+ 7086, 7087, 7098, 7141, 7168, 7203, 7245, 7247, 7258, 7293, 7296, 7306, 7312, 7354,
+ 7357, 7359, 7401, 7404, 7406, 7411, 7413, 7414, 7418, 7418, 7424, 7615, 7680, 7957,
+ 7960, 7965, 7968, 8005, 8008, 8013, 8016, 8023, 8025, 8025, 8027, 8027, 8029, 8029,
+ 8031, 8061, 8064, 8116, 8118, 8124, 8126, 8126, 8130, 8132, 8134, 8140, 8144, 8147,
+ 8150, 8155, 8160, 8172, 8178, 8180, 8182, 8188, 8305, 8305, 8319, 8319, 8336, 8348,
+ 8450, 8450, 8455, 8455, 8458, 8467, 8469, 8469, 8473, 8477, 8484, 8484, 8486, 8486,
+ 8488, 8488, 8490, 8493, 8495, 8505, 8508, 8511, 8517, 8521, 8526, 8526, 8579, 8580,
+ 11264, 11492, 11499, 11502, 11506, 11507, 11520, 11557, 11559, 11559, 11565, 11565,
+ 11568, 11623, 11631, 11631, 11648, 11670, 11680, 11686, 11688, 11694, 11696, 11702,
+ 11704, 11710, 11712, 11718, 11720, 11726, 11728, 11734, 11736, 11742, 11823, 11823,
+ 12293, 12294, 12337, 12341, 12347, 12348, 12353, 12438, 12445, 12447, 12449, 12538,
+ 12540, 12543, 12549, 12591, 12593, 12686, 12704, 12735, 12784, 12799, 13312, 19903,
+ 19968, 42124, 42192, 42237, 42240, 42508, 42512, 42527, 42538, 42539, 42560, 42606,
+ 42623, 42653, 42656, 42725, 42775, 42783, 42786, 42888, 42891, 42957, 42960, 42961,
+ 42963, 42963, 42965, 42972, 42994, 43009, 43011, 43013, 43015, 43018, 43020, 43042,
+ 43072, 43123, 43138, 43187, 43250, 43255, 43259, 43259, 43261, 43262, 43274, 43301,
+ 43312, 43334, 43360, 43388, 43396, 43442, 43471, 43471, 43488, 43492, 43494, 43503,
+ 43514, 43518, 43520, 43560, 43584, 43586, 43588, 43595, 43616, 43638, 43642, 43642,
+ 43646, 43695, 43697, 43697, 43701, 43702, 43705, 43709, 43712, 43712, 43714, 43714,
+ 43739, 43741, 43744, 43754, 43762, 43764, 43777, 43782, 43785, 43790, 43793, 43798,
+ 43808, 43814, 43816, 43822, 43824, 43866, 43868, 43881, 43888, 44002, 44032, 55203,
+ 55216, 55238, 55243, 55291, 63744, 64109, 64112, 64217, 64256, 64262, 64275, 64279,
+ 64285, 64285, 64287, 64296, 64298, 64310, 64312, 64316, 64318, 64318, 64320, 64321,
+ 64323, 64324, 64326, 64433, 64467, 64829, 64848, 64911, 64914, 64967, 65008, 65019,
+ 65136, 65140, 65142, 65276, 65313, 65338, 65345, 65370, 65382, 65470, 65474, 65479,
+ 65482, 65487, 65490, 65495, 65498, 65500, 65536, 65547, 65549, 65574, 65576, 65594,
+ 65596, 65597, 65599, 65613, 65616, 65629, 65664, 65786, 66176, 66204, 66208, 66256,
+ 66304, 66335, 66349, 66368, 66370, 66377, 66384, 66421, 66432, 66461, 66464, 66499,
+ 66504, 66511, 66560, 66717, 66736, 66771, 66776, 66811, 66816, 66855, 66864, 66915,
+ 66928, 66938, 66940, 66954, 66956, 66962, 66964, 66965, 66967, 66977, 66979, 66993,
+ 66995, 67001, 67003, 67004, 67008, 67059, 67072, 67382, 67392, 67413, 67424, 67431,
+ 67456, 67461, 67463, 67504, 67506, 67514, 67584, 67589, 67592, 67592, 67594, 67637,
+ 67639, 67640, 67644, 67644, 67647, 67669, 67680, 67702, 67712, 67742, 67808, 67826,
+ 67828, 67829, 67840, 67861, 67872, 67897, 67968, 68023, 68030, 68031, 68096, 68096,
+ 68112, 68115, 68117, 68119, 68121, 68149, 68192, 68220, 68224, 68252, 68288, 68295,
+ 68297, 68324, 68352, 68405, 68416, 68437, 68448, 68466, 68480, 68497, 68608, 68680,
+ 68736, 68786, 68800, 68850, 68864, 68899, 68938, 68965, 68975, 68997, 69248, 69289,
+ 69296, 69297, 69314, 69316, 69376, 69404, 69415, 69415, 69424, 69445, 69488, 69505,
+ 69552, 69572, 69600, 69622, 69635, 69687, 69745, 69746, 69749, 69749, 69763, 69807,
+ 69840, 69864, 69891, 69926, 69956, 69956, 69959, 69959, 69968, 70002, 70006, 70006,
+ 70019, 70066, 70081, 70084, 70106, 70106, 70108, 70108, 70144, 70161, 70163, 70187,
+ 70207, 70208, 70272, 70278, 70280, 70280, 70282, 70285, 70287, 70301, 70303, 70312,
+ 70320, 70366, 70405, 70412, 70415, 70416, 70419, 70440, 70442, 70448, 70450, 70451,
+ 70453, 70457, 70461, 70461, 70480, 70480, 70493, 70497, 70528, 70537, 70539, 70539,
+ 70542, 70542, 70544, 70581, 70583, 70583, 70609, 70609, 70611, 70611, 70656, 70708,
+ 70727, 70730, 70751, 70753, 70784, 70831, 70852, 70853, 70855, 70855, 71040, 71086,
+ 71128, 71131, 71168, 71215, 71236, 71236, 71296, 71338, 71352, 71352, 71424, 71450,
+ 71488, 71494, 71680, 71723, 71840, 71903, 71935, 71942, 71945, 71945, 71948, 71955,
+ 71957, 71958, 71960, 71983, 71999, 71999, 72001, 72001, 72096, 72103, 72106, 72144,
+ 72161, 72161, 72163, 72163, 72192, 72192, 72203, 72242, 72250, 72250, 72272, 72272,
+ 72284, 72329, 72349, 72349, 72368, 72440, 72640, 72672, 72704, 72712, 72714, 72750,
+ 72768, 72768, 72818, 72847, 72960, 72966, 72968, 72969, 72971, 73008, 73030, 73030,
+ 73056, 73061, 73063, 73064, 73066, 73097, 73112, 73112, 73440, 73458, 73474, 73474,
+ 73476, 73488, 73490, 73523, 73648, 73648, 73728, 74649, 74880, 75075, 77712, 77808,
+ 77824, 78895, 78913, 78918, 78944, 82938, 82944, 83526, 90368, 90397, 92160, 92728,
+ 92736, 92766, 92784, 92862, 92880, 92909, 92928, 92975, 92992, 92995, 93027, 93047,
+ 93053, 93071, 93504, 93548, 93760, 93823, 93952, 94026, 94032, 94032, 94099, 94111,
+ 94176, 94177, 94179, 94179, 94208, 100343, 100352, 101589, 101631, 101640, 110576,
+ 110579, 110581, 110587, 110589, 110590, 110592, 110882, 110898, 110898, 110928,
+ 110930, 110933, 110933, 110948, 110951, 110960, 111355, 113664, 113770, 113776,
+ 113788, 113792, 113800, 113808, 113817, 119808, 119892, 119894, 119964, 119966,
+ 119967, 119970, 119970, 119973, 119974, 119977, 119980, 119982, 119993, 119995,
+ 119995, 119997, 120003, 120005, 120069, 120071, 120074, 120077, 120084, 120086,
+ 120092, 120094, 120121, 120123, 120126, 120128, 120132, 120134, 120134, 120138,
+ 120144, 120146, 120485, 120488, 120512, 120514, 120538, 120540, 120570, 120572,
+ 120596, 120598, 120628, 120630, 120654, 120656, 120686, 120688, 120712, 120714,
+ 120744, 120746, 120770, 120772, 120779, 122624, 122654, 122661, 122666, 122928,
+ 122989, 123136, 123180, 123191, 123197, 123214, 123214, 123536, 123565, 123584,
+ 123627, 124112, 124139, 124368, 124397, 124400, 124400, 124896, 124902, 124904,
+ 124907, 124909, 124910, 124912, 124926, 124928, 125124, 125184, 125251, 125259,
+ 125259, 126464, 126467, 126469, 126495, 126497, 126498, 126500, 126500, 126503,
+ 126503, 126505, 126514, 126516, 126519, 126521, 126521, 126523, 126523, 126530,
+ 126530, 126535, 126535, 126537, 126537, 126539, 126539, 126541, 126543, 126545,
+ 126546, 126548, 126548, 126551, 126551, 126553, 126553, 126555, 126555, 126557,
+ 126557, 126559, 126559, 126561, 126562, 126564, 126564, 126567, 126570, 126572,
+ 126578, 126580, 126583, 126585, 126588, 126590, 126590, 126592, 126601, 126603,
+ 126619, 126625, 126627, 126629, 126633, 126635, 126651, 131072, 173791, 173824,
+ 177977, 177984, 178205, 178208, 183969, 183984, 191456, 191472, 192093, 194560,
+ 195101, 196608, 201546, 201552, 205743
};
}
diff --git a/laya-java/laya-java/src/test/java/com/convaiinnovations/laya/tokenizer/PreTokenizerTablesTest.java b/laya-java/laya-java/src/test/java/com/convaiinnovations/laya/tokenizer/PreTokenizerTablesTest.java
new file mode 100644
index 00000000..68d03d6f
--- /dev/null
+++ b/laya-java/laya-java/src/test/java/com/convaiinnovations/laya/tokenizer/PreTokenizerTablesTest.java
@@ -0,0 +1,135 @@
+package com.convaiinnovations.laya.tokenizer;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import org.junit.jupiter.api.DisplayName;
+import org.junit.jupiter.api.Test;
+
+/**
+ * The letter and number classes are recorded from the reference by a behavioural probe, because
+ * the property is not exposed. A probe can therefore be wrong in a way a round-trip through the
+ * generator cannot catch -- regenerating agrees with itself whatever it asks -- so the classes are
+ * checked here against properties the GPT-2 pattern's three alternatives imply.
+ *
+ * <p>These exist because a probe WAS wrong. It asked whether {@code "<cp>'s"} leaves {@code "'s"}
+ * as its own piece, which is true of every code point that fails to absorb the apostrophe: every
+ * whitespace code point and all 1,911 digits were recorded as letters. A letter run then swallowed
+ * the inner whitespace of a state, and only a corpus fixture needing a checkpoint caught it.
+ */
+final class PreTokenizerTablesTest {
+
+ /** The shipped english checkpoint's own pre_tokenizer node, verbatim. */
+ private static PreTokenizer byteLevel() {
+ Map<String, Object> node = new LinkedHashMap<>();
+ node.put("type", "ByteLevel");
+ node.put("add_prefix_space", Boolean.FALSE);
+ node.put("trim_offsets", Boolean.TRUE);
+ node.put("use_regex", Boolean.TRUE);
+ return PreTokenizer.from(node);
+ }
+
+ @Test
+ @DisplayName("no whitespace code point is a letter or a number")
+ void whitespaceIsOutsideBothClasses() {
+ // The alternatives ` ?\p{L}+` and ` ?\p{N}+` are reached before `\s+`, so a whitespace
+ // code point inside either class is consumed by a word run and never becomes its own
+ // piece -- which is what produced 11 ids where the reference produces 14.
+ int examined = 0;
+ for (int cp = 0; cp < 0x110000; cp++) {
+ final int at = cp;
+ if (Unicode.isWhiteSpace(cp)) {
+ examined++;
+ assertFalse(PreTokenizerTables.isLetter(cp),
+ () -> String.format("U+%04X is whitespace and must not be a letter", at));
+ assertFalse(PreTokenizerTables.isNumber(cp),
+ () -> String.format("U+%04X is whitespace and must not be a number", at));
+ }
+ }
+ assertEquals(25, examined, "the Unicode White_Space set is 25 code points");
+ }
+
+ @Test
+ @DisplayName("the two classes are disjoint, and ASCII digits are only numbers")
+ void classesAreDisjoint() {
+ for (int cp = 0; cp < 0x110000; cp++) {
+ final int at = cp;
+ if (PreTokenizerTables.isLetter(cp)) {
+ assertFalse(PreTokenizerTables.isNumber(cp),
+ () -> String.format("U+%04X is in both classes", at));
+ }
+ }
+ for (int cp = '0'; cp <= '9'; cp++) {
+ assertTrue(PreTokenizerTables.isNumber(cp), "an ASCII digit is a number");
+ assertFalse(PreTokenizerTables.isLetter(cp), "an ASCII digit is not a letter");
+ }
+ for (int cp = 'a'; cp <= 'z'; cp++) {
+ assertTrue(PreTokenizerTables.isLetter(cp), "an ASCII lowercase letter is a letter");
+ assertFalse(PreTokenizerTables.isNumber(cp), "an ASCII letter is not a number");
+ }
+ }
+
+ @Test
+ @DisplayName("the tables carry the Unicode the JDK does not")
+ void carriesTheReferencesUnicode() {
+ // Each is a letter to the reference tokenizer and not to Corretto 17's Unicode 13.0.
+ // Written as ranges rather than single values so the assertion does not pass on a table
+ // that happens to include one code point of a block it is missing.
+ int[][] laterThanTheJdk = {
+ {0x2EBF0, 0x2EE5D}, // CJK Unified Ideographs Extension I (Unicode 15.1)
+ {0x105C0, 0x105F3}, // Todhri (16.0)
+ {0x10D4A, 0x10D65}, // Garay (16.0)
+ {0x113A0, 0x113B2}, // Tulu-Tigalari (16.0)
+ {0x16100, 0x1611D}, // Gurung Khema (16.0)
+ };
+ for (int[] block : laterThanTheJdk) {
+ for (int cp = block[0]; cp <= block[1]; cp++) {
+ final int at = cp;
+ assertTrue(PreTokenizerTables.isLetter(cp),
+ () -> String.format("U+%04X is a letter to the reference", at));
+ }
+ }
+ }
+
+ @Test
+ @DisplayName("an inner whitespace run is cut the way the reference cuts it")
+ void innerWhitespaceRunsAreNotSwallowed() {
+ // Recorded from the reference's own pre_tokenize_str. The two interesting rules are both
+ // here: a space run before a word leaves its LAST space to the word, because ` ?\p{L}+`
+ // is tried before `\s+(?!\S)`; and a TAB or newline run does not, because the optional
+ // leading space of that alternative is a literal U+0020. So " b" is two pieces and
+ // "\t\tc" is three.
+ PreTokenizer byteLevel = byteLevel();
+ assertEquals(
+ List.of("a",
+ "Ġ",
+ "Ġb",
+ "ĉ",
+ "ĉ",
+ "c",
+ "Ċ",
+ "Ċ",
+ "d",
+ "č",
+ "Ċ",
+ "e",
+ "ĠĠ",
+ "Ġf"),
+ byteLevel.preTokenize("a b\t\tc\n\nd\r\ne f"));
+ }
+
+ @Test
+ @DisplayName("a digit run stays its own piece next to a word")
+ void digitRunsAreNotSwallowedByLetterRuns() {
+ // The same bad probe put every digit in the letter class. Because ` ?\p{L}+` is tried
+ // first, "abc123" became one piece instead of the reference's two.
+ PreTokenizer byteLevel = byteLevel();
+ assertEquals(List.of("abc", "123"), byteLevel.preTokenize("abc123"));
+ assertEquals(List.of("123", "abc"), byteLevel.preTokenize("123abc"));
+ assertEquals(List.of("a", "Ġ1", "Ġb"), byteLevel.preTokenize("a 1 b"));
+ }
+}
diff --git a/laya-java/scripts/gen_pretokenizer_tables.py b/laya-java/scripts/gen_pretokenizer_tables.py
index e4579fcd..629f0600 100644
--- a/laya-java/scripts/gen_pretokenizer_tables.py
+++ b/laya-java/scripts/gen_pretokenizer_tables.py
@@ -1,5 +1,5 @@
#!/usr/bin/env python3
-"""Record the GPT-2 pre-tokenizer's letter and number classes FROM THE REFERENCE, as Java source.
+r"""Record the GPT-2 pre-tokenizer's letter and number classes FROM THE REFERENCE, as Java source.
WHY THIS EXISTS. The GPT-2 pre-tokenizer pattern is
@@ -20,8 +20,16 @@ and the same jar gives different ids on a different JDK.
So the classes are recorded here from the reference and compiled in, which is what every other
table in this port does. The probe is behavioural rather than a property lookup, because the
-property is not exposed: a code point is a LETTER if "<cp>'s" leaves "'s" as its own piece, and a
-NUMBER if "1<cp>" stays one piece. Both follow from the pattern above.
+property is not exposed. A code point is a LETTER if "a<cp>" stays ONE piece and a NUMBER if
+"1<cp>" stays one piece: the only way the first match can reach past the "a" is ` ?\p{L}+`
+consuming the code point, and past the "1" is ` ?\p{N}+`, so one piece means membership and two
+means non-membership, with no third reading.
+
+An earlier probe asked instead whether "<cp>'s" leaves "'s" as its own piece. That is NOT a test
+of membership: it is true for every code point that fails to absorb the apostrophe, which includes
+every whitespace code point and every digit. It marked TAB, LF, CR, NBSP, U+2007 and all 1,911
+digits as letters, so a letter run swallowed the inner whitespace of "a b\t\tc" and the scanner
+returned 11 ids where the reference returns 14.
python laya-java/scripts/gen_pretokenizer_tables.py --tokenizer <path/to/tokenizer.json>
python laya-java/scripts/gen_pretokenizer_tables.py --check --tokenizer <path>
@@ -46,7 +54,7 @@ def classes_from_reference(tokenizer_path):
pre = Tokenizer.from_file(tokenizer_path).pre_tokenizer
letters, numbers = [], []
for name, probe, judge in (
- ("letter", lambda ch: ch + "'s", lambda pieces: pieces[-1][0] == "'s"),
+ ("letter", lambda ch: "a" + ch, lambda pieces: len(pieces) == 1),
("number", lambda ch: "1" + ch, lambda pieces: len(pieces) == 1),
):
runs, start = [], None
@@ -54,8 +62,10 @@ def classes_from_reference(tokenizer_path):
hit = False
if cp not in SURROGATES:
ch = chr(cp)
- # A whitespace or control code point can make the probe degenerate; the judge
- # below is written so that only a genuine class membership answers true.
+ # Both probes are anchored by a character of the class being tested, so a
+ # second piece means the code point is outside it. Nothing else can join the
+ # anchor, which is what makes the judge a membership test rather than a
+ # statement about the apostrophe.
try:
hit = judge(pre.pre_tokenize_str(probe(ch)))
except Exception: