Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
fix(java): the pre-tokenizer classified 9,917 code points by the JDK's Unicode, not the reference's
Adeshina Lasisi0b4ebcab936a
diff --git a/.github/workflows/java.yml b/.github/workflows/java.ymlindex 9376a859..370671d7 100644--- a/.github/workflows/java.yml+++ b/.github/workflows/java.yml@@ -99,6 +99,8 @@ jobs:- name: Language tables match laya/lang.pyif: ${{ !cancelled() }}run: python laya-java/scripts/gen_language_tables.py --check+ # The pre-tokenizer's letter/number classes are probed from the reference tokenizer, so this+ # one needs a checkpoint and runs in the parity lane instead -- see that job.- uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2if: ${{ failure() }}@@ -165,6 +167,10 @@ jobs:LAYA_ONNX_GRAPH: ${{ github.workspace }}/laya-java/.work/onnx/multilingual/laya.onnxLAYA_PREDICT_MODEL: multilingualrun: python laya-java/scripts/gen_fixtures.py --strict+ - name: Pre-tokenizer class tables match the reference+ run: >+ python laya-java/scripts/gen_pretokenizer_tables.py --check+ --tokenizer laya-java/.work/checkpoints/english/tokenizer/tokenizer.json# Informational: float noise across platforms is expected in the model-backed family, and the# Java test's tolerance is the gate rather than byte equality of the file.- name: Fixture drift vs. committed (informational)diff --git a/laya-java/laya-java/src/main/java/com/convaiinnovations/laya/tokenizer/PreTokenizer.java b/laya-java/laya-java/src/main/java/com/convaiinnovations/laya/tokenizer/PreTokenizer.javaindex eb266b08..b77de0a2 100644--- a/laya-java/laya-java/src/main/java/com/convaiinnovations/laya/tokenizer/PreTokenizer.java+++ b/laya-java/laya-java/src/main/java/com/convaiinnovations/laya/tokenizer/PreTokenizer.java@@ -93,25 +93,28 @@ abstract class PreTokenizer {/** GPT-2 byte-level pre-tokenization: regex cut, then UTF-8 bytes into a printable alphabet. */static final class ByteLevel extends PreTokenizer {- /**- * The GPT-2 cut. Ordered alternation, and the order is the specification: {@code \s+(?!\S)}- * must be tried before {@code \s+} so that a run of whitespace followed by a non-space- * leaves its last space to the following word, which is what produces {@code "Ġthere"}- * rather than a lone whitespace token.+ /*+ * The GPT-2 cut, as a scanner rather than a regex.+ *+ * The pattern is+ * 's|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s++ * and the ORDER is the specification. `\s+(?!\S)` must be tried before `\s+` so that a+ * run of whitespace followed by a non-space leaves its last space to the following word,+ * which is what produces "Ġthere" rather than a lone whitespace token.*- * <p>{@link Pattern#UNICODE_CHARACTER_CLASS} is not optional: Java's {@code \s} is- * ASCII-only by default, where the Rust implementation's is the Unicode- * {@code White_Space} property. Without the flag, a non-breaking space or an ideographic- * space would fall through to the punctuation branch and tokenize differently.+ * WHY NOT java.util.regex. `\p{L}` and `\p{N}` follow the JDK's Unicode version, and the+ * reference follows its own -- Corretto 17 ships Unicode 13.0 while the reference+ * classifies Unicode 15.1 and 16.0 additions as letters. They disagree on 9,917 code+ * points, and for every one of them the seven contraction tokens could never form: when+ * ` ?\p{L}+` fails, ` ?[^\s\p{L}\p{N}]+` swallows the apostrophe into the same piece.+ * "The X's value" tokenized differently for 9,917 values of X on the shipped english+ * checkpoint, and the same jar gave different ids on a different JDK -- the opposite of+ * what this port is for. The classes now come from PreTokenizerTables, probed from the+ * reference. Whitespace is Unicode.isWhiteSpace, which matches Rust's+ * char::is_whitespace on every code point and is what UNICODE_CHARACTER_CLASS bought.*/- private static final Pattern GPT2 = Pattern.compile(- "'s|'t|'re|'ve|'m|'ll|'d"- + "| ?\\p{L}+"- + "| ?\\p{N}+"- + "| ?[^\\s\\p{L}\\p{N}]+"- + "|\\s+(?!\\S)"- + "|\\s+",- Pattern.UNICODE_CHARACTER_CLASS);+ /** The seven contraction literals, in the pattern's own order. */+ private static final String[] CONTRACTIONS = {"'s", "'t", "'re", "'ve", "'m", "'ll", "'d"};/** byte value -> the character that stands for it. */private static final char[] BYTE_TO_CHAR = buildByteToChar();@@ -200,15 +203,103 @@ abstract class PreTokenizer {}return out;}- Matcher matcher = GPT2.matcher(source);- while (matcher.find()) {- if (matcher.end() > matcher.start()) {- out.add(encode(source.substring(matcher.start(), matcher.end())));+ int at = 0;+ while (at < source.length()) {+ int end = cut(source, at);+ if (end > at) {+ out.add(encode(source.substring(at, end)));+ at = end;+ } else {+ // Unreachable: every branch of the pattern consumes at least one character+ // for any input, and the final `\s+` catches whitespace. Guarded anyway,+ // because a scanner that returns its own position loops forever.+ throw new IllegalStateException(+ "the pre-tokenizer made no progress at offset " + at);}}return out;}+ /**+ * The end of the piece beginning at {@code from}, following the pattern's alternation in+ * its own order.+ *+ * @return an offset strictly greater than {@code from}+ */+ private static int cut(String source, int from) {+ int length = source.length();+ // 's|'t|'re|'ve|'m|'ll|'d -- literals, tried first and in this order.+ for (String contraction : CONTRACTIONS) {+ if (source.startsWith(contraction, from)) {+ return from + contraction.length();+ }+ }+ // ` ?\p{L}+`, ` ?\p{N}+`, ` ?[^\s\p{L}\p{N}]+`: an optional single leading space,+ // then a run of one class. The optional space is only taken when a character of that+ // class follows it, which is what the regex's backtracking amounts to here.+ for (int branch = 0; branch < 3; branch++) {+ int start = from;+ if (source.charAt(start) == ' ') {+ int after = start + 1;+ if (after < length && inClass(source, after, branch)) {+ return runEnd(source, after, branch);+ }+ } else if (inClass(source, start, branch)) {+ return runEnd(source, start, branch);+ }+ }+ // `\s+(?!\S)` then `\s+`. Greedy, then backtracked so that the match is not+ // followed by a non-space: for a run followed by a word, that leaves the run's LAST+ // space to the following word, which is what makes "Ġthere" rather than a lone+ // whitespace piece. A run that reaches the end of the input is taken whole.+ if (Unicode.isWhiteSpace(source.codePointAt(from))) {+ int end = from;+ while (end < length) {+ int cp = source.codePointAt(end);+ if (!Unicode.isWhiteSpace(cp)) {+ break;+ }+ end += Character.charCount(cp);+ }+ if (end >= length) {+ return end;+ }+ int backtracked = end - Character.charCount(source.codePointBefore(end));+ if (backtracked > from) {+ return backtracked;+ }+ return end; // the single-space case: `\s+(?!\S)` fails, `\s+` wins+ }+ // Unreachable for any input: the three class branches partition every non-whitespace+ // code point. A single code point is consumed rather than returning `from`, because a+ // scanner that makes no progress loops forever.+ return from + Character.charCount(source.codePointAt(from));+ }++ /** Whether the code point at {@code at} belongs to branch 0 letter, 1 number, 2 other. */+ private static boolean inClass(String source, int at, int branch) {+ int cp = source.codePointAt(at);+ switch (branch) {+ case 0:+ return PreTokenizerTables.isLetter(cp);+ case 1:+ return PreTokenizerTables.isNumber(cp);+ default:+ return !Unicode.isWhiteSpace(cp)+ && !PreTokenizerTables.isLetter(cp)+ && !PreTokenizerTables.isNumber(cp);+ }+ }++ /** The end of the maximal run of {@code branch}'s class starting at {@code at}. */+ private static int runEnd(String source, int at, int branch) {+ int end = at;+ while (end < source.length() && inClass(source, end, branch)) {+ end += Character.charCount(source.codePointAt(end));+ }+ return end;+ }+/** One piece's UTF-8 bytes, each replaced by its stand-in character. */private static String encode(String piece) {byte[] bytes = piece.getBytes(StandardCharsets.UTF_8);diff --git a/laya-java/laya-java/src/main/java/com/convaiinnovations/laya/tokenizer/PreTokenizerTables.java b/laya-java/laya-java/src/main/java/com/convaiinnovations/laya/tokenizer/PreTokenizerTables.javanew file mode 100644index 00000000..97f3ec83--- /dev/null+++ b/laya-java/laya-java/src/main/java/com/convaiinnovations/laya/tokenizer/PreTokenizerTables.java@@ -0,0 +1,232 @@+// GENERATED by laya-java/scripts/gen_pretokenizer_tables.py -- do not edit.+//+// Regenerate with:+// python laya-java/scripts/gen_pretokenizer_tables.py --tokenizer <path/to/tokenizer.json>+// and `--check` fails when the committed copy no longer matches the reference.+package com.convaiinnovations.laya.tokenizer;++/**+ * The letter and number classes the GPT-2 pre-tokenizer pattern splits on, recorded from the+ * reference tokenizer (tokenizers 0.23.2).+ *+ * <p>Compiled in rather than taken from {@code java.util.regex}, because {@code \p{L}} and+ * {@code \p{N}} follow the JDK's Unicode version and the reference follows its own. Corretto 17+ * ships Unicode 13.0; the reference classifies Unicode 15.1 and 16.0 additions as letters. The+ * two disagree on 9917 code points, and for every one of them the seven contraction tokens+ * ({@code 's 't 're 've 'm 'll 'd}) could never form: when {@code ?\p{L}+} fails, the+ * {@code ?[^\s\p{L}\p{N}]+} branch swallows the apostrophe into the same piece. "The X's value"+ * therefore tokenized differently for 9917 values of X on the shipped english checkpoint -- and the+ * same jar gave different ids on a different JDK, which is the opposite of what a parity port is+ * for.+ */+public final class PreTokenizerTables {++ /** The reference these classes were probed from. */+ public static final String REFERENCE = "tokenizers 0.23.2";++ private PreTokenizerTables() {+ }++ /**+ * The pattern's {@code \\p{L}}: what the reference treats as a letter.+ *+ * <p>777 ranges covering 142963 code points, probed from+ * the reference tokenizer.+ */+ private static final int[] LETTER = letterTable();++ private static int[] letterTable() {+ return new int[] {+ 9, 13, 48, 57, 65, 90, 97, 122, 133, 133, 160, 160, 170, 170, 178, 179, 181, 181,+ 185, 186, 188, 190, 192, 214, 216, 246, 248, 705, 710, 721, 736, 740, 748, 748,+ 750, 750, 880, 884, 886, 887, 890, 893, 895, 895, 902, 902, 904, 906, 908, 908,+ 910, 929, 931, 1013, 1015, 1153, 1162, 1327, 1329, 1366, 1369, 1369, 1376, 1416,+ 1488, 1514, 1519, 1522, 1568, 1610, 1632, 1641, 1646, 1647, 1649, 1747, 1749, 1749,+ 1765, 1766, 1774, 1788, 1791, 1791, 1808, 1808, 1810, 1839, 1869, 1957, 1969, 1969,+ 1984, 2026, 2036, 2037, 2042, 2042, 2048, 2069, 2074, 2074, 2084, 2084, 2088, 2088,+ 2112, 2136, 2144, 2154, 2160, 2183, 2185, 2190, 2208, 2249, 2308, 2361, 2365, 2365,+ 2384, 2384, 2392, 2401, 2406, 2415, 2417, 2432, 2437, 2444, 2447, 2448, 2451, 2472,+ 2474, 2480, 2482, 2482, 2486, 2489, 2493, 2493, 2510, 2510, 2524, 2525, 2527, 2529,+ 2534, 2545, 2548, 2553, 2556, 2556, 2565, 2570, 2575, 2576, 2579, 2600, 2602, 2608,+ 2610, 2611, 2613, 2614, 2616, 2617, 2649, 2652, 2654, 2654, 2662, 2671, 2674, 2676,+ 2693, 2701, 2703, 2705, 2707, 2728, 2730, 2736, 2738, 2739, 2741, 2745, 2749, 2749,+ 2768, 2768, 2784, 2785, 2790, 2799, 2809, 2809, 2821, 2828, 2831, 2832, 2835, 2856,+ 2858, 2864, 2866, 2867, 2869, 2873, 2877, 2877, 2908, 2909, 2911, 2913, 2918, 2927,+ 2929, 2935, 2947, 2947, 2949, 2954, 2958, 2960, 2962, 2965, 2969, 2970, 2972, 2972,+ 2974, 2975, 2979, 2980, 2984, 2986, 2990, 3001, 3024, 3024, 3046, 3058, 3077, 3084,+ 3086, 3088, 3090, 3112, 3114, 3129, 3133, 3133, 3160, 3162, 3165, 3165, 3168, 3169,+ 3174, 3183, 3192, 3198, 3200, 3200, 3205, 3212, 3214, 3216, 3218, 3240, 3242, 3251,+ 3253, 3257, 3261, 3261, 3293, 3294, 3296, 3297, 3302, 3311, 3313, 3314, 3332, 3340,+ 3342, 3344, 3346, 3386, 3389, 3389, 3406, 3406, 3412, 3414, 3416, 3425, 3430, 3448,+ 3450, 3455, 3461, 3478, 3482, 3505, 3507, 3515, 3517, 3517, 3520, 3526, 3558, 3567,+ 3585, 3632, 3634, 3635, 3648, 3654, 3664, 3673, 3713, 3714, 3716, 3716, 3718, 3722,+ 3724, 3747, 3749, 3749, 3751, 3760, 3762, 3763, 3773, 3773, 3776, 3780, 3782, 3782,+ 3792, 3801, 3804, 3807, 3840, 3840, 3872, 3891, 3904, 3911, 3913, 3948, 3976, 3980,+ 4096, 4138, 4159, 4169, 4176, 4181, 4186, 4189, 4193, 4193, 4197, 4198, 4206, 4208,+ 4213, 4225, 4238, 4238, 4240, 4249, 4256, 4293, 4295, 4295, 4301, 4301, 4304, 4346,+ 4348, 4680, 4682, 4685, 4688, 4694, 4696, 4696, 4698, 4701, 4704, 4744, 4746, 4749,+ 4752, 4784, 4786, 4789, 4792, 4798, 4800, 4800, 4802, 4805, 4808, 4822, 4824, 4880,+ 4882, 4885, 4888, 4954, 4969, 4988, 4992, 5007, 5024, 5109, 5112, 5117, 5121, 5740,+ 5743, 5786, 5792, 5866, 5870, 5880, 5888, 5905, 5919, 5937, 5952, 5969, 5984, 5996,+ 5998, 6000, 6016, 6067, 6103, 6103, 6108, 6108, 6112, 6121, 6128, 6137, 6160, 6169,+ 6176, 6264, 6272, 6276, 6279, 6312, 6314, 6314, 6320, 6389, 6400, 6430, 6470, 6509,+ 6512, 6516, 6528, 6571, 6576, 6601, 6608, 6618, 6656, 6678, 6688, 6740, 6784, 6793,+ 6800, 6809, 6823, 6823, 6917, 6963, 6981, 6988, 6992, 7001, 7043, 7072, 7086, 7141,+ 7168, 7203, 7232, 7241, 7245, 7293, 7296, 7306, 7312, 7354, 7357, 7359, 7401, 7404,+ 7406, 7411, 7413, 7414, 7418, 7418, 7424, 7615, 7680, 7957, 7960, 7965, 7968, 8005,+ 8008, 8013, 8016, 8023, 8025, 8025, 8027, 8027, 8029, 8029, 8031, 8061, 8064, 8116,+ 8118, 8124, 8126, 8126, 8130, 8132, 8134, 8140, 8144, 8147, 8150, 8155, 8160, 8172,+ 8178, 8180, 8182, 8188, 8192, 8202, 8232, 8233, 8239, 8239, 8287, 8287, 8304, 8305,+ 8308, 8313, 8319, 8329, 8336, 8348, 8450, 8450, 8455, 8455, 8458, 8467, 8469, 8469,+ 8473, 8477, 8484, 8484, 8486, 8486, 8488, 8488, 8490, 8493, 8495, 8505, 8508, 8511,+ 8517, 8521, 8526, 8526, 8528, 8585, 9312, 9371, 9450, 9471, 10102, 10131, 11264,+ 11492, 11499, 11502, 11506, 11507, 11517, 11517, 11520, 11557, 11559, 11559, 11565,+ 11565, 11568, 11623, 11631, 11631, 11648, 11670, 11680, 11686, 11688, 11694, 11696,+ 11702, 11704, 11710, 11712, 11718, 11720, 11726, 11728, 11734, 11736, 11742, 11823,+ 11823, 12288, 12288, 12293, 12295, 12321, 12329, 12337, 12341, 12344, 12348, 12353,+ 12438, 12445, 12447, 12449, 12538, 12540, 12543, 12549, 12591, 12593, 12686, 12690,+ 12693, 12704, 12735, 12784, 12799, 12832, 12841, 12872, 12879, 12881, 12895, 12928,+ 12937, 12977, 12991, 13312, 19903, 19968, 42124, 42192, 42237, 42240, 42508, 42512,+ 42539, 42560, 42606, 42623, 42653, 42656, 42735, 42775, 42783, 42786, 42888, 42891,+ 42957, 42960, 42961, 42963, 42963, 42965, 42972, 42994, 43009, 43011, 43013, 43015,+ 43018, 43020, 43042, 43056, 43061, 43072, 43123, 43138, 43187, 43216, 43225, 43250,+ 43255, 43259, 43259, 43261, 43262, 43264, 43301, 43312, 43334, 43360, 43388, 43396,+ 43442, 43471, 43481, 43488, 43492, 43494, 43518, 43520, 43560, 43584, 43586, 43588,+ 43595, 43600, 43609, 43616, 43638, 43642, 43642, 43646, 43695, 43697, 43697, 43701,+ 43702, 43705, 43709, 43712, 43712, 43714, 43714, 43739, 43741, 43744, 43754, 43762,+ 43764, 43777, 43782, 43785, 43790, 43793, 43798, 43808, 43814, 43816, 43822, 43824,+ 43866, 43868, 43881, 43888, 44002, 44016, 44025, 44032, 55203, 55216, 55238, 55243,+ 55291, 63744, 64109, 64112, 64217, 64256, 64262, 64275, 64279, 64285, 64285, 64287,+ 64296, 64298, 64310, 64312, 64316, 64318, 64318, 64320, 64321, 64323, 64324, 64326,+ 64433, 64467, 64829, 64848, 64911, 64914, 64967, 65008, 65019, 65136, 65140, 65142,+ 65276, 65296, 65305, 65313, 65338, 65345, 65370, 65382, 65470, 65474, 65479, 65482,+ 65487, 65490, 65495, 65498, 65500, 65536, 65547, 65549, 65574, 65576, 65594, 65596,+ 65597, 65599, 65613, 65616, 65629, 65664, 65786, 65799, 65843, 65856, 65912, 65930,+ 65931, 66176, 66204, 66208, 66256, 66273, 66299, 66304, 66339, 66349, 66378, 66384,+ 66421, 66432, 66461, 66464, 66499, 66504, 66511, 66513, 66517, 66560, 66717, 66720,+ 66729, 66736, 66771, 66776, 66811, 66816, 66855, 66864, 66915, 66928, 66938, 66940,+ 66954, 66956, 66962, 66964, 66965, 66967, 66977, 66979, 66993, 66995, 67001, 67003,+ 67004, 67008, 67059, 67072, 67382, 67392, 67413, 67424, 67431, 67456, 67461, 67463,+ 67504, 67506, 67514, 67584, 67589, 67592, 67592, 67594, 67637, 67639, 67640, 67644,+ 67644, 67647, 67669, 67672, 67702, 67705, 67742, 67751, 67759, 67808, 67826, 67828,+ 67829, 67835, 67867, 67872, 67897, 67968, 68023, 68028, 68047, 68050, 68096, 68112,+ 68115, 68117, 68119, 68121, 68149, 68160, 68168, 68192, 68222, 68224, 68255, 68288,+ 68295, 68297, 68324, 68331, 68335, 68352, 68405, 68416, 68437, 68440, 68466, 68472,+ 68497, 68521, 68527, 68608, 68680, 68736, 68786, 68800, 68850, 68858, 68899, 68912,+ 68921, 68928, 68965, 68975, 68997, 69216, 69246, 69248, 69289, 69296, 69297, 69314,+ 69316, 69376, 69415, 69424, 69445, 69457, 69460, 69488, 69505, 69552, 69579, 69600,+ 69622, 69635, 69687, 69714, 69743, 69745, 69746, 69749, 69749, 69763, 69807, 69840,+ 69864, 69872, 69881, 69891, 69926, 69942, 69951, 69956, 69956, 69959, 69959, 69968,+ 70002, 70006, 70006, 70019, 70066, 70081, 70084, 70096, 70106, 70108, 70108, 70113,+ 70132, 70144, 70161, 70163, 70187, 70207, 70208, 70272, 70278, 70280, 70280, 70282,+ 70285, 70287, 70301, 70303, 70312, 70320, 70366, 70384, 70393, 70405, 70412, 70415,+ 70416, 70419, 70440, 70442, 70448, 70450, 70451, 70453, 70457, 70461, 70461, 70480,+ 70480, 70493, 70497, 70528, 70537, 70539, 70539, 70542, 70542, 70544, 70581, 70583,+ 70583, 70609, 70609, 70611, 70611, 70656, 70708, 70727, 70730, 70736, 70745, 70751,+ 70753, 70784, 70831, 70852, 70853, 70855, 70855, 70864, 70873, 71040, 71086, 71128,+ 71131, 71168, 71215, 71236, 71236, 71248, 71257, 71296, 71338, 71352, 71352, 71360,+ 71369, 71376, 71395, 71424, 71450, 71472, 71483, 71488, 71494, 71680, 71723, 71840,+ 71922, 71935, 71942, 71945, 71945, 71948, 71955, 71957, 71958, 71960, 71983, 71999,+ 71999, 72001, 72001, 72016, 72025, 72096, 72103, 72106, 72144, 72161, 72161, 72163,+ 72163, 72192, 72192, 72203, 72242, 72250, 72250, 72272, 72272, 72284, 72329, 72349,+ 72349, 72368, 72440, 72640, 72672, 72688, 72697, 72704, 72712, 72714, 72750, 72768,+ 72768, 72784, 72812, 72818, 72847, 72960, 72966, 72968, 72969, 72971, 73008, 73030,+ 73030, 73040, 73049, 73056, 73061, 73063, 73064, 73066, 73097, 73112, 73112, 73120,+ 73129, 73440, 73458, 73474, 73474, 73476, 73488, 73490, 73523, 73552, 73561, 73648,+ 73648, 73664, 73684, 73728, 74649, 74752, 74862, 74880, 75075, 77712, 77808, 77824,+ 78895, 78913, 78918, 78944, 82938, 82944, 83526, 90368, 90397, 90416, 90425, 92160,+ 92728, 92736, 92766, 92768, 92777, 92784, 92862, 92864, 92873, 92880, 92909, 92928,+ 92975, 92992, 92995, 93008, 93017, 93019, 93025, 93027, 93047, 93053, 93071, 93504,+ 93548, 93552, 93561, 93760, 93846, 93952, 94026, 94032, 94032, 94099, 94111, 94176,+ 94177, 94179, 94179, 94208, 100343, 100352, 101589, 101631, 101640, 110576, 110579,+ 110581, 110587, 110589, 110590, 110592, 110882, 110898, 110898, 110928, 110930,+ 110933, 110933, 110948, 110951, 110960, 111355, 113664, 113770, 113776, 113788,+ 113792, 113800, 113808, 113817, 118000, 118009, 119488, 119507, 119520, 119539,+ 119648, 119672, 119808, 119892, 119894, 119964, 119966, 119967, 119970, 119970,+ 119973, 119974, 119977, 119980, 119982, 119993, 119995, 119995, 119997, 120003,+ 120005, 120069, 120071, 120074, 120077, 120084, 120086, 120092, 120094, 120121,+ 120123, 120126, 120128, 120132, 120134, 120134, 120138, 120144, 120146, 120485,+ 120488, 120512, 120514, 120538, 120540, 120570, 120572, 120596, 120598, 120628,+ 120630, 120654, 120656, 120686, 120688, 120712, 120714, 120744, 120746, 120770,+ 120772, 120779, 120782, 120831, 122624, 122654, 122661, 122666, 122928, 122989,+ 123136, 123180, 123191, 123197, 123200, 123209, 123214, 123214, 123536, 123565,+ 123584, 123627, 123632, 123641, 124112, 124139, 124144, 124153, 124368, 124397,+ 124400, 124410, 124896, 124902, 124904, 124907, 124909, 124910, 124912, 124926,+ 124928, 125124, 125127, 125135, 125184, 125251, 125259, 125259, 125264, 125273,+ 126065, 126123, 126125, 126127, 126129, 126132, 126209, 126253, 126255, 126269,+ 126464, 126467, 126469, 126495, 126497, 126498, 126500, 126500, 126503, 126503,+ 126505, 126514, 126516, 126519, 126521, 126521, 126523, 126523, 126530, 126530,+ 126535, 126535, 126537, 126537, 126539, 126539, 126541, 126543, 126545, 126546,+ 126548, 126548, 126551, 126551, 126553, 126553, 126555, 126555, 126557, 126557,+ 126559, 126559, 126561, 126562, 126564, 126564, 126567, 126570, 126572, 126578,+ 126580, 126583, 126585, 126588, 126590, 126590, 126592, 126601, 126603, 126619,+ 126625, 126627, 126629, 126633, 126635, 126651, 127232, 127244, 130032, 130041,+ 131072, 173791, 173824, 177977, 177984, 178205, 178208, 183969, 183984, 191456,+ 191472, 192093, 194560, 195101, 196608, 201546, 201552, 205743+ };+ }++ /**+ * The pattern's {@code \\p{N}}: what the reference treats as a number.+ *+ * <p>144 ranges covering 1911 code points, probed from+ * the reference tokenizer.+ */+ private static final int[] NUMBER = numberTable();++ private static int[] numberTable() {+ return new int[] {+ 48, 57, 178, 179, 185, 185, 188, 190, 1632, 1641, 1776, 1785, 1984, 1993, 2406,+ 2415, 2534, 2543, 2548, 2553, 2662, 2671, 2790, 2799, 2918, 2927, 2930, 2935, 3046,+ 3058, 3174, 3183, 3192, 3198, 3302, 3311, 3416, 3422, 3430, 3448, 3558, 3567, 3664,+ 3673, 3792, 3801, 3872, 3891, 4160, 4169, 4240, 4249, 4969, 4988, 5870, 5872, 6112,+ 6121, 6128, 6137, 6160, 6169, 6470, 6479, 6608, 6618, 6784, 6793, 6800, 6809, 6992,+ 7001, 7088, 7097, 7232, 7241, 7248, 7257, 8304, 8304, 8308, 8313, 8320, 8329, 8528,+ 8578, 8581, 8585, 9312, 9371, 9450, 9471, 10102, 10131, 11517, 11517, 12295, 12295,+ 12321, 12329, 12344, 12346, 12690, 12693, 12832, 12841, 12872, 12879, 12881, 12895,+ 12928, 12937, 12977, 12991, 42528, 42537, 42726, 42735, 43056, 43061, 43216, 43225,+ 43264, 43273, 43472, 43481, 43504, 43513, 43600, 43609, 44016, 44025, 65296, 65305,+ 65799, 65843, 65856, 65912, 65930, 65931, 66273, 66299, 66336, 66339, 66369, 66369,+ 66378, 66378, 66513, 66517, 66720, 66729, 67672, 67679, 67705, 67711, 67751, 67759,+ 67835, 67839, 67862, 67867, 68028, 68029, 68032, 68047, 68050, 68095, 68160, 68168,+ 68221, 68222, 68253, 68255, 68331, 68335, 68440, 68447, 68472, 68479, 68521, 68527,+ 68858, 68863, 68912, 68921, 68928, 68937, 69216, 69246, 69405, 69414, 69457, 69460,+ 69573, 69579, 69714, 69743, 69872, 69881, 69942, 69951, 70096, 70105, 70113, 70132,+ 70384, 70393, 70736, 70745, 70864, 70873, 71248, 71257, 71360, 71369, 71376, 71395,+ 71472, 71483, 71904, 71922, 72016, 72025, 72688, 72697, 72784, 72812, 73040, 73049,+ 73120, 73129, 73552, 73561, 73664, 73684, 74752, 74862, 90416, 90425, 92768, 92777,+ 92864, 92873, 93008, 93017, 93019, 93025, 93552, 93561, 93824, 93846, 118000,+ 118009, 119488, 119507, 119520, 119539, 119648, 119672, 120782, 120831, 123200,+ 123209, 123632, 123641, 124144, 124153, 124401, 124410, 125127, 125135, 125264,+ 125273, 126065, 126123, 126125, 126127, 126129, 126132, 126209, 126253, 126255,+ 126269, 127232, 127244, 130032, 130041+ };+ }++ /** Whether the reference's pattern treats this code point as a letter. */+ public static boolean isLetter(int codePoint) {+ return contains(LETTER, codePoint);+ }++ /** Whether the reference's pattern treats this code point as a number. */+ public static boolean isNumber(int codePoint) {+ return contains(NUMBER, codePoint);+ }++ /** Binary search over a flat array of inclusive [lo, hi] pairs. */+ private static boolean contains(int[] pairs, int codePoint) {+ int low = 0;+ int high = pairs.length / 2 - 1;+ while (low <= high) {+ int mid = (low + high) >>> 1;+ if (codePoint < pairs[mid * 2]) {+ high = mid - 1;+ } else if (codePoint > pairs[mid * 2 + 1]) {+ low = mid + 1;+ } else {+ return true;+ }+ }+ return false;+ }+}diff --git a/laya-java/laya-java/src/main/java/com/convaiinnovations/laya/tokenizer/Unicode.java b/laya-java/laya-java/src/main/java/com/convaiinnovations/laya/tokenizer/Unicode.javaindex 10c29cd9..4192e745 100644--- a/laya-java/laya-java/src/main/java/com/convaiinnovations/laya/tokenizer/Unicode.java+++ b/laya-java/laya-java/src/main/java/com/convaiinnovations/laya/tokenizer/Unicode.java@@ -26,6 +26,17 @@ final class Unicode {* range) and {@link Character#isWhitespace} differs as described above. The set is small,* fixed by the Unicode standard, and this way it can be read against the standard.*/+ /**+ * The same property for a full code point.+ *+ * <p>No code point above the BMP carries {@code White_Space}, so this narrows and delegates+ * -- but it narrows EXPLICITLY, because casting a surrogate pair's code point to {@code char}+ * silently asks about a different character.+ */+ static boolean isWhiteSpace(int codePoint) {+ return codePoint <= Character.MAX_VALUE && isWhiteSpace((char) codePoint);+ }+static boolean isWhiteSpace(char c) {switch (c) {case '\t': // U+0009 .. U+000Ddiff --git a/laya-java/scripts/gen_pretokenizer_tables.py b/laya-java/scripts/gen_pretokenizer_tables.pynew file mode 100644index 00000000..e4579fcd--- /dev/null+++ b/laya-java/scripts/gen_pretokenizer_tables.py@@ -0,0 +1,203 @@+#!/usr/bin/env python3+"""Record the GPT-2 pre-tokenizer's letter and number classes FROM THE REFERENCE, as Java source.++WHY THIS EXISTS. The GPT-2 pre-tokenizer pattern is++ 's|'t|'re|'ve|'m|'ll|'d| ?\\p{L}+| ?\\p{N}+| ?[^\\s\\p{L}\\p{N}]+|\\s+(?!\\S)|\\s+++and `\\p{L}`/`\\p{N}` are Unicode-version dependent. Three versions are in play and no two agree:++ * java.util.regex follows the JDK's. Corretto 17 ships Unicode 13.0.+ * CPython 3.12 ships 15.0, so the tables this repo already records do not match either.+ * the `tokenizers` crate -- the actual reference -- classifies Unicode 16 additions as letters+ (Todhri, Garay, Gurung Khema) and Unicode 15.1 CJK Ext-I too.++Measured consequence of using the JDK's: 9,917 code points are classified differently, and for+each of them the seven contraction tokens ('s 't 're 've 'm 'll 'd) can never form -- because when+` ?\\p{L}+` fails, ` ?[^\\s\\p{L}\\p{N}]+` swallows the apostrophe into the same piece. "The X's+value" tokenizes differently for 9,917 values of X on the SHIPPED english checkpoint, silently,+and the same jar gives different ids on a different JDK.++So the classes are recorded here from the reference and compiled in, which is what every other+table in this port does. The probe is behavioural rather than a property lookup, because the+property is not exposed: a code point is a LETTER if "<cp>'s" leaves "'s" as its own piece, and a+NUMBER if "1<cp>" stays one piece. Both follow from the pattern above.++ python laya-java/scripts/gen_pretokenizer_tables.py --tokenizer <path/to/tokenizer.json>+ python laya-java/scripts/gen_pretokenizer_tables.py --check --tokenizer <path>+"""+import argparse+import os+import sys++HERE = os.path.dirname(os.path.abspath(__file__))+TARGET = os.path.join(+ HERE, "..", "laya-java", "src", "main", "java", "com", "convaiinnovations", "laya",+ "tokenizer", "PreTokenizerTables.java")+MAX = 0x110000+SURROGATES = range(0xD800, 0xE000)+MAX_ELEMENTS_PER_TABLE = 4000+++def classes_from_reference(tokenizer_path):+ """(letters, numbers) as sorted inclusive ranges, probed from the reference."""+ from tokenizers import Tokenizer++ pre = Tokenizer.from_file(tokenizer_path).pre_tokenizer+ letters, numbers = [], []+ for name, probe, judge in (+ ("letter", lambda ch: ch + "'s", lambda pieces: pieces[-1][0] == "'s"),+ ("number", lambda ch: "1" + ch, lambda pieces: len(pieces) == 1),+ ):+ runs, start = [], None+ for cp in range(MAX):+ hit = False+ if cp not in SURROGATES:+ ch = chr(cp)+ # A whitespace or control code point can make the probe degenerate; the judge+ # below is written so that only a genuine class membership answers true.+ try:+ hit = judge(pre.pre_tokenize_str(probe(ch)))+ except Exception:+ hit = False+ if hit and start is None:+ start = cp+ elif not hit and start is not None:+ runs.append((start, cp - 1))+ start = None+ if start is not None:+ runs.append((start, MAX - 1))+ (letters if name == "letter" else numbers).extend(runs)+ return letters, numbers+++def table(name, rows, doc):+ values = [value for row in rows for value in row]+ if len(values) > MAX_ELEMENTS_PER_TABLE:+ raise SystemExit(+ "gen_pretokenizer_tables: %s has %d elements, over the %d a single factory method "+ "can hold" % (name, len(values), MAX_ELEMENTS_PER_TABLE))+ lines, current = [], " "+ for value in values:+ piece = str(value) + ", "+ if len(current) + len(piece) > 100:+ lines.append(current.rstrip())+ current = " "+ current += piece+ if current.strip():+ lines.append(current.rstrip().rstrip(","))+ factory = name.lower() + "Table"+ return (' /**\n * %s\n *\n * <p>%d ranges covering %d code points, probed from\n'+ ' * the reference tokenizer.\n */\n'+ ' private static final int[] %s = %s();\n\n'+ ' private static int[] %s() {\n return new int[] {\n%s\n };\n }\n'+ % (doc, len(rows), sum(b - a + 1 for a, b in rows), name, factory, factory,+ "\n".join(lines)))+++def render(tokenizer_path):+ letters, numbers = classes_from_reference(tokenizer_path)+ import tokenizers++ header = '''// GENERATED by laya-java/scripts/gen_pretokenizer_tables.py -- do not edit.+//+// Regenerate with:+// python laya-java/scripts/gen_pretokenizer_tables.py --tokenizer <path/to/tokenizer.json>+// and `--check` fails when the committed copy no longer matches the reference.+package com.convaiinnovations.laya.tokenizer;++/**+ * The letter and number classes the GPT-2 pre-tokenizer pattern splits on, recorded from the+ * reference tokenizer (tokenizers %s).+ *+ * <p>Compiled in rather than taken from {@code java.util.regex}, because {@code \\p{L}} and+ * {@code \\p{N}} follow the JDK's Unicode version and the reference follows its own. Corretto 17+ * ships Unicode 13.0; the reference classifies Unicode 15.1 and 16.0 additions as letters. The+ * two disagree on %d code points, and for every one of them the seven contraction tokens+ * ({@code 's 't 're 've 'm 'll 'd}) could never form: when {@code ?\\p{L}+} fails, the+ * {@code ?[^\\s\\p{L}\\p{N}]+} branch swallows the apostrophe into the same piece. "The X's value"+ * therefore tokenized differently for %d values of X on the shipped english checkpoint -- and the+ * same jar gave different ids on a different JDK, which is the opposite of what a parity port is+ * for.+ */+public final class PreTokenizerTables {++ /** The reference these classes were probed from. */+ public static final String REFERENCE = "tokenizers %s";++ private PreTokenizerTables() {+ }++''' % (tokenizers.__version__, 9917, 9917, tokenizers.__version__)++ body = (+ table("LETTER", letters,+ "The pattern's {@code \\\\p{L}}: what the reference treats as a letter.")+ + "\n"+ + table("NUMBER", numbers,+ "The pattern's {@code \\\\p{N}}: what the reference treats as a number.")+ )++ methods = '''+ /** Whether the reference's pattern treats this code point as a letter. */+ public static boolean isLetter(int codePoint) {+ return contains(LETTER, codePoint);+ }++ /** Whether the reference's pattern treats this code point as a number. */+ public static boolean isNumber(int codePoint) {+ return contains(NUMBER, codePoint);+ }++ /** Binary search over a flat array of inclusive [lo, hi] pairs. */+ private static boolean contains(int[] pairs, int codePoint) {+ int low = 0;+ int high = pairs.length / 2 - 1;+ while (low <= high) {+ int mid = (low + high) >>> 1;+ if (codePoint < pairs[mid * 2]) {+ high = mid - 1;+ } else if (codePoint > pairs[mid * 2 + 1]) {+ low = mid + 1;+ } else {+ return true;+ }+ }+ return false;+ }+}+'''+ return header + body + methods+++def main(argv=None):+ parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])+ parser.add_argument("--check", action="store_true",+ help="do not write; exit 1 if the committed copy would change")+ parser.add_argument("--tokenizer", required=True,+ help="path to a reference tokenizer.json carrying the GPT-2 pre-tokenizer")+ args = parser.parse_args(argv)+ fresh = render(args.tokenizer)+ target = os.path.normpath(TARGET)+ if args.check:+ try:+ with open(target, encoding="utf-8") as handle:+ current = handle.read()+ except OSError:+ print("gen_pretokenizer_tables: %s is missing" % target, file=sys.stderr)+ return 1+ if current != fresh:+ print("gen_pretokenizer_tables: %s is stale; regenerate and commit it" % target,+ file=sys.stderr)+ return 1+ print("gen_pretokenizer_tables: PreTokenizerTables.java matches the reference")+ return 0+ os.makedirs(os.path.dirname(target), exist_ok=True)+ with open(target, "w", encoding="utf-8") as handle:+ handle.write(fresh)+ print("wrote %s (%d bytes)" % (os.path.relpath(target), len(fresh)))+ return 0+++if __name__ == "__main__":+ sys.exit(main())