Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

feat(java): compile in the Unicode tables laya.lang needs, recorded from CPython

Adeshina Lasisi5ea751102673
diff --git a/.github/workflows/java.yml b/.github/workflows/java.yml
index 610d30b9..6e1869ae 100644
--- a/.github/workflows/java.yml
+++ b/.github/workflows/java.yml
@@ -79,6 +79,11 @@ jobs:
- name: Model-free fixture drift vs. committed
if: ${{ !cancelled() }}
run: python laya-java/scripts/gen_fixtures.py --check --only python_json.json
+ # The compiled-in Unicode tables are recorded from CPython, so a CPython upgrade that moves
+ # them has to show up here rather than as a routing change nobody attributed.
+ - name: Unicode tables match CPython
+ if: ${{ !cancelled() }}
+ run: python laya-java/scripts/gen_unicode_tables.py --check
- uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
if: ${{ failure() }}
diff --git a/laya-java/laya-java/src/main/java/com/convaiinnovations/laya/lang/UnicodeTables.java b/laya-java/laya-java/src/main/java/com/convaiinnovations/laya/lang/UnicodeTables.java
new file mode 100644
index 00000000..b9af452c
--- /dev/null
+++ b/laya-java/laya-java/src/main/java/com/convaiinnovations/laya/lang/UnicodeTables.java
@@ -0,0 +1,379 @@
+// GENERATED by laya-java/scripts/gen_unicode_tables.py -- do not edit.
+//
+// Regenerate with:
+// python laya-java/scripts/gen_unicode_tables.py
+// and `--check` fails when the committed copy no longer matches CPython.
+package com.convaiinnovations.laya.lang;
+
+/**
+ * The Unicode properties {@code laya.lang} depends on, recorded from CPython 15.0.0.
+ *
+ * <p>Compiled in rather than read from the JDK, because the JDK disagrees with CPython about all
+ * three of them. {@code Character.isLetter} means the same categories as {@code str.isalpha()},
+ * but JDK 17 ships Unicode 13.0 against CPython's 15.0 and therefore does not know 4863 of the code
+ * points Python calls letters -- it reports them unassigned, which would count text in a
+ * recently-added script under no script at all and route it to the English checkpoint. The
+ * canonical combining class is not exposed by the JDK at all, and the Mn/Mc categories are not a
+ * substitute: they differ from it on 1,460 code points, including U+034F COMBINING GRAPHEME
+ * JOINER, whose class is zero.
+ *
+ * <p>A table also makes detection independent of the JDK a consumer runs, which matters more than
+ * it sounds: a JDK upgrade must not silently change which script a request is detected as.
+ */
+public final class UnicodeTables {
+
+ /** The CPython release these tables were recorded from. */
+ public static final String UNICODE_VERSION = "15.0.0";
+
+ private UnicodeTables() {
+ }
+
+ /**
+ * Python's {@code str.isalpha()}: categories Lu, Ll, Lt, Lm and Lo.
+ *
+ * <p>659 ranges covering 136104 code points, recorded
+ * from CPython 15.0.0.
+ */
+ private static final int[] ALPHA = {
+ 65, 90, 97, 122, 170, 170, 181, 181, 186, 186, 192, 214, 216, 246, 248, 705, 710, 721,
+ 736, 740, 748, 748, 750, 750, 880, 884, 886, 887, 890, 893, 895, 895, 902, 902, 904,
+ 906, 908, 908, 910, 929, 931, 1013, 1015, 1153, 1162, 1327, 1329, 1366, 1369, 1369,
+ 1376, 1416, 1488, 1514, 1519, 1522, 1568, 1610, 1646, 1647, 1649, 1747, 1749, 1749,
+ 1765, 1766, 1774, 1775, 1786, 1788, 1791, 1791, 1808, 1808, 1810, 1839, 1869, 1957,
+ 1969, 1969, 1994, 2026, 2036, 2037, 2042, 2042, 2048, 2069, 2074, 2074, 2084, 2084,
+ 2088, 2088, 2112, 2136, 2144, 2154, 2160, 2183, 2185, 2190, 2208, 2249, 2308, 2361,
+ 2365, 2365, 2384, 2384, 2392, 2401, 2417, 2432, 2437, 2444, 2447, 2448, 2451, 2472,
+ 2474, 2480, 2482, 2482, 2486, 2489, 2493, 2493, 2510, 2510, 2524, 2525, 2527, 2529,
+ 2544, 2545, 2556, 2556, 2565, 2570, 2575, 2576, 2579, 2600, 2602, 2608, 2610, 2611,
+ 2613, 2614, 2616, 2617, 2649, 2652, 2654, 2654, 2674, 2676, 2693, 2701, 2703, 2705,
+ 2707, 2728, 2730, 2736, 2738, 2739, 2741, 2745, 2749, 2749, 2768, 2768, 2784, 2785,
+ 2809, 2809, 2821, 2828, 2831, 2832, 2835, 2856, 2858, 2864, 2866, 2867, 2869, 2873,
+ 2877, 2877, 2908, 2909, 2911, 2913, 2929, 2929, 2947, 2947, 2949, 2954, 2958, 2960,
+ 2962, 2965, 2969, 2970, 2972, 2972, 2974, 2975, 2979, 2980, 2984, 2986, 2990, 3001,
+ 3024, 3024, 3077, 3084, 3086, 3088, 3090, 3112, 3114, 3129, 3133, 3133, 3160, 3162,
+ 3165, 3165, 3168, 3169, 3200, 3200, 3205, 3212, 3214, 3216, 3218, 3240, 3242, 3251,
+ 3253, 3257, 3261, 3261, 3293, 3294, 3296, 3297, 3313, 3314, 3332, 3340, 3342, 3344,
+ 3346, 3386, 3389, 3389, 3406, 3406, 3412, 3414, 3423, 3425, 3450, 3455, 3461, 3478,
+ 3482, 3505, 3507, 3515, 3517, 3517, 3520, 3526, 3585, 3632, 3634, 3635, 3648, 3654,
+ 3713, 3714, 3716, 3716, 3718, 3722, 3724, 3747, 3749, 3749, 3751, 3760, 3762, 3763,
+ 3773, 3773, 3776, 3780, 3782, 3782, 3804, 3807, 3840, 3840, 3904, 3911, 3913, 3948,
+ 3976, 3980, 4096, 4138, 4159, 4159, 4176, 4181, 4186, 4189, 4193, 4193, 4197, 4198,
+ 4206, 4208, 4213, 4225, 4238, 4238, 4256, 4293, 4295, 4295, 4301, 4301, 4304, 4346,
+ 4348, 4680, 4682, 4685, 4688, 4694, 4696, 4696, 4698, 4701, 4704, 4744, 4746, 4749,
+ 4752, 4784, 4786, 4789, 4792, 4798, 4800, 4800, 4802, 4805, 4808, 4822, 4824, 4880,
+ 4882, 4885, 4888, 4954, 4992, 5007, 5024, 5109, 5112, 5117, 5121, 5740, 5743, 5759,
+ 5761, 5786, 5792, 5866, 5873, 5880, 5888, 5905, 5919, 5937, 5952, 5969, 5984, 5996,
+ 5998, 6000, 6016, 6067, 6103, 6103, 6108, 6108, 6176, 6264, 6272, 6276, 6279, 6312,
+ 6314, 6314, 6320, 6389, 6400, 6430, 6480, 6509, 6512, 6516, 6528, 6571, 6576, 6601,
+ 6656, 6678, 6688, 6740, 6823, 6823, 6917, 6963, 6981, 6988, 7043, 7072, 7086, 7087,
+ 7098, 7141, 7168, 7203, 7245, 7247, 7258, 7293, 7296, 7304, 7312, 7354, 7357, 7359,
+ 7401, 7404, 7406, 7411, 7413, 7414, 7418, 7418, 7424, 7615, 7680, 7957, 7960, 7965,
+ 7968, 8005, 8008, 8013, 8016, 8023, 8025, 8025, 8027, 8027, 8029, 8029, 8031, 8061,
+ 8064, 8116, 8118, 8124, 8126, 8126, 8130, 8132, 8134, 8140, 8144, 8147, 8150, 8155,
+ 8160, 8172, 8178, 8180, 8182, 8188, 8305, 8305, 8319, 8319, 8336, 8348, 8450, 8450,
+ 8455, 8455, 8458, 8467, 8469, 8469, 8473, 8477, 8484, 8484, 8486, 8486, 8488, 8488,
+ 8490, 8493, 8495, 8505, 8508, 8511, 8517, 8521, 8526, 8526, 8579, 8580, 11264, 11492,
+ 11499, 11502, 11506, 11507, 11520, 11557, 11559, 11559, 11565, 11565, 11568, 11623,
+ 11631, 11631, 11648, 11670, 11680, 11686, 11688, 11694, 11696, 11702, 11704, 11710,
+ 11712, 11718, 11720, 11726, 11728, 11734, 11736, 11742, 11823, 11823, 12293, 12294,
+ 12337, 12341, 12347, 12348, 12353, 12438, 12445, 12447, 12449, 12538, 12540, 12543,
+ 12549, 12591, 12593, 12686, 12704, 12735, 12784, 12799, 13312, 19903, 19968, 42124,
+ 42192, 42237, 42240, 42508, 42512, 42527, 42538, 42539, 42560, 42606, 42623, 42653,
+ 42656, 42725, 42775, 42783, 42786, 42888, 42891, 42954, 42960, 42961, 42963, 42963,
+ 42965, 42969, 42994, 43009, 43011, 43013, 43015, 43018, 43020, 43042, 43072, 43123,
+ 43138, 43187, 43250, 43255, 43259, 43259, 43261, 43262, 43274, 43301, 43312, 43334,
+ 43360, 43388, 43396, 43442, 43471, 43471, 43488, 43492, 43494, 43503, 43514, 43518,
+ 43520, 43560, 43584, 43586, 43588, 43595, 43616, 43638, 43642, 43642, 43646, 43695,
+ 43697, 43697, 43701, 43702, 43705, 43709, 43712, 43712, 43714, 43714, 43739, 43741,
+ 43744, 43754, 43762, 43764, 43777, 43782, 43785, 43790, 43793, 43798, 43808, 43814,
+ 43816, 43822, 43824, 43866, 43868, 43881, 43888, 44002, 44032, 55203, 55216, 55238,
+ 55243, 55291, 63744, 64109, 64112, 64217, 64256, 64262, 64275, 64279, 64285, 64285,
+ 64287, 64296, 64298, 64310, 64312, 64316, 64318, 64318, 64320, 64321, 64323, 64324,
+ 64326, 64433, 64467, 64829, 64848, 64911, 64914, 64967, 65008, 65019, 65136, 65140,
+ 65142, 65276, 65313, 65338, 65345, 65370, 65382, 65470, 65474, 65479, 65482, 65487,
+ 65490, 65495, 65498, 65500, 65536, 65547, 65549, 65574, 65576, 65594, 65596, 65597,
+ 65599, 65613, 65616, 65629, 65664, 65786, 66176, 66204, 66208, 66256, 66304, 66335,
+ 66349, 66368, 66370, 66377, 66384, 66421, 66432, 66461, 66464, 66499, 66504, 66511,
+ 66560, 66717, 66736, 66771, 66776, 66811, 66816, 66855, 66864, 66915, 66928, 66938,
+ 66940, 66954, 66956, 66962, 66964, 66965, 66967, 66977, 66979, 66993, 66995, 67001,
+ 67003, 67004, 67072, 67382, 67392, 67413, 67424, 67431, 67456, 67461, 67463, 67504,
+ 67506, 67514, 67584, 67589, 67592, 67592, 67594, 67637, 67639, 67640, 67644, 67644,
+ 67647, 67669, 67680, 67702, 67712, 67742, 67808, 67826, 67828, 67829, 67840, 67861,
+ 67872, 67897, 67968, 68023, 68030, 68031, 68096, 68096, 68112, 68115, 68117, 68119,
+ 68121, 68149, 68192, 68220, 68224, 68252, 68288, 68295, 68297, 68324, 68352, 68405,
+ 68416, 68437, 68448, 68466, 68480, 68497, 68608, 68680, 68736, 68786, 68800, 68850,
+ 68864, 68899, 69248, 69289, 69296, 69297, 69376, 69404, 69415, 69415, 69424, 69445,
+ 69488, 69505, 69552, 69572, 69600, 69622, 69635, 69687, 69745, 69746, 69749, 69749,
+ 69763, 69807, 69840, 69864, 69891, 69926, 69956, 69956, 69959, 69959, 69968, 70002,
+ 70006, 70006, 70019, 70066, 70081, 70084, 70106, 70106, 70108, 70108, 70144, 70161,
+ 70163, 70187, 70207, 70208, 70272, 70278, 70280, 70280, 70282, 70285, 70287, 70301,
+ 70303, 70312, 70320, 70366, 70405, 70412, 70415, 70416, 70419, 70440, 70442, 70448,
+ 70450, 70451, 70453, 70457, 70461, 70461, 70480, 70480, 70493, 70497, 70656, 70708,
+ 70727, 70730, 70751, 70753, 70784, 70831, 70852, 70853, 70855, 70855, 71040, 71086,
+ 71128, 71131, 71168, 71215, 71236, 71236, 71296, 71338, 71352, 71352, 71424, 71450,
+ 71488, 71494, 71680, 71723, 71840, 71903, 71935, 71942, 71945, 71945, 71948, 71955,
+ 71957, 71958, 71960, 71983, 71999, 71999, 72001, 72001, 72096, 72103, 72106, 72144,
+ 72161, 72161, 72163, 72163, 72192, 72192, 72203, 72242, 72250, 72250, 72272, 72272,
+ 72284, 72329, 72349, 72349, 72368, 72440, 72704, 72712, 72714, 72750, 72768, 72768,
+ 72818, 72847, 72960, 72966, 72968, 72969, 72971, 73008, 73030, 73030, 73056, 73061,
+ 73063, 73064, 73066, 73097, 73112, 73112, 73440, 73458, 73474, 73474, 73476, 73488,
+ 73490, 73523, 73648, 73648, 73728, 74649, 74880, 75075, 77712, 77808, 77824, 78895,
+ 78913, 78918, 82944, 83526, 92160, 92728, 92736, 92766, 92784, 92862, 92880, 92909,
+ 92928, 92975, 92992, 92995, 93027, 93047, 93053, 93071, 93760, 93823, 93952, 94026,
+ 94032, 94032, 94099, 94111, 94176, 94177, 94179, 94179, 94208, 100343, 100352, 101589,
+ 101632, 101640, 110576, 110579, 110581, 110587, 110589, 110590, 110592, 110882, 110898,
+ 110898, 110928, 110930, 110933, 110933, 110948, 110951, 110960, 111355, 113664, 113770,
+ 113776, 113788, 113792, 113800, 113808, 113817, 119808, 119892, 119894, 119964, 119966,
+ 119967, 119970, 119970, 119973, 119974, 119977, 119980, 119982, 119993, 119995, 119995,
+ 119997, 120003, 120005, 120069, 120071, 120074, 120077, 120084, 120086, 120092, 120094,
+ 120121, 120123, 120126, 120128, 120132, 120134, 120134, 120138, 120144, 120146, 120485,
+ 120488, 120512, 120514, 120538, 120540, 120570, 120572, 120596, 120598, 120628, 120630,
+ 120654, 120656, 120686, 120688, 120712, 120714, 120744, 120746, 120770, 120772, 120779,
+ 122624, 122654, 122661, 122666, 122928, 122989, 123136, 123180, 123191, 123197, 123214,
+ 123214, 123536, 123565, 123584, 123627, 124112, 124139, 124896, 124902, 124904, 124907,
+ 124909, 124910, 124912, 124926, 124928, 125124, 125184, 125251, 125259, 125259, 126464,
+ 126467, 126469, 126495, 126497, 126498, 126500, 126500, 126503, 126503, 126505, 126514,
+ 126516, 126519, 126521, 126521, 126523, 126523, 126530, 126530, 126535, 126535, 126537,
+ 126537, 126539, 126539, 126541, 126543, 126545, 126546, 126548, 126548, 126551, 126551,
+ 126553, 126553, 126555, 126555, 126557, 126557, 126559, 126559, 126561, 126562, 126564,
+ 126564, 126567, 126570, 126572, 126578, 126580, 126583, 126585, 126588, 126590, 126590,
+ 126592, 126601, 126603, 126619, 126625, 126627, 126629, 126633, 126635, 126651, 131072,
+ 173791, 173824, 177977, 177984, 178205, 178208, 183969, 183984, 191456, 194560, 195101,
+ 196608, 201546, 201552, 205743
+ };
+
+ /**
+ * Python's {@code [^\\W\\d_]}: letters plus the Nl and No numeric categories, which is
+ * what {@code laya.lang}'s word regex matches.
+ *
+ * <p>709 ranges covering 137255 code points, recorded
+ * from CPython 15.0.0.
+ */
+ private static final int[] WORD = {
+ 65, 90, 97, 122, 170, 170, 178, 179, 181, 181, 185, 186, 188, 190, 192, 214, 216, 246,
+ 248, 705, 710, 721, 736, 740, 748, 748, 750, 750, 880, 884, 886, 887, 890, 893, 895,
+ 895, 902, 902, 904, 906, 908, 908, 910, 929, 931, 1013, 1015, 1153, 1162, 1327, 1329,
+ 1366, 1369, 1369, 1376, 1416, 1488, 1514, 1519, 1522, 1568, 1610, 1646, 1647, 1649,
+ 1747, 1749, 1749, 1765, 1766, 1774, 1775, 1786, 1788, 1791, 1791, 1808, 1808, 1810,
+ 1839, 1869, 1957, 1969, 1969, 1994, 2026, 2036, 2037, 2042, 2042, 2048, 2069, 2074,
+ 2074, 2084, 2084, 2088, 2088, 2112, 2136, 2144, 2154, 2160, 2183, 2185, 2190, 2208,
+ 2249, 2308, 2361, 2365, 2365, 2384, 2384, 2392, 2401, 2417, 2432, 2437, 2444, 2447,
+ 2448, 2451, 2472, 2474, 2480, 2482, 2482, 2486, 2489, 2493, 2493, 2510, 2510, 2524,
+ 2525, 2527, 2529, 2544, 2545, 2548, 2553, 2556, 2556, 2565, 2570, 2575, 2576, 2579,
+ 2600, 2602, 2608, 2610, 2611, 2613, 2614, 2616, 2617, 2649, 2652, 2654, 2654, 2674,
+ 2676, 2693, 2701, 2703, 2705, 2707, 2728, 2730, 2736, 2738, 2739, 2741, 2745, 2749,
+ 2749, 2768, 2768, 2784, 2785, 2809, 2809, 2821, 2828, 2831, 2832, 2835, 2856, 2858,
+ 2864, 2866, 2867, 2869, 2873, 2877, 2877, 2908, 2909, 2911, 2913, 2929, 2935, 2947,
+ 2947, 2949, 2954, 2958, 2960, 2962, 2965, 2969, 2970, 2972, 2972, 2974, 2975, 2979,
+ 2980, 2984, 2986, 2990, 3001, 3024, 3024, 3056, 3058, 3077, 3084, 3086, 3088, 3090,
+ 3112, 3114, 3129, 3133, 3133, 3160, 3162, 3165, 3165, 3168, 3169, 3192, 3198, 3200,
+ 3200, 3205, 3212, 3214, 3216, 3218, 3240, 3242, 3251, 3253, 3257, 3261, 3261, 3293,
+ 3294, 3296, 3297, 3313, 3314, 3332, 3340, 3342, 3344, 3346, 3386, 3389, 3389, 3406,
+ 3406, 3412, 3414, 3416, 3425, 3440, 3448, 3450, 3455, 3461, 3478, 3482, 3505, 3507,
+ 3515, 3517, 3517, 3520, 3526, 3585, 3632, 3634, 3635, 3648, 3654, 3713, 3714, 3716,
+ 3716, 3718, 3722, 3724, 3747, 3749, 3749, 3751, 3760, 3762, 3763, 3773, 3773, 3776,
+ 3780, 3782, 3782, 3804, 3807, 3840, 3840, 3882, 3891, 3904, 3911, 3913, 3948, 3976,
+ 3980, 4096, 4138, 4159, 4159, 4176, 4181, 4186, 4189, 4193, 4193, 4197, 4198, 4206,
+ 4208, 4213, 4225, 4238, 4238, 4256, 4293, 4295, 4295, 4301, 4301, 4304, 4346, 4348,
+ 4680, 4682, 4685, 4688, 4694, 4696, 4696, 4698, 4701, 4704, 4744, 4746, 4749, 4752,
+ 4784, 4786, 4789, 4792, 4798, 4800, 4800, 4802, 4805, 4808, 4822, 4824, 4880, 4882,
+ 4885, 4888, 4954, 4969, 4988, 4992, 5007, 5024, 5109, 5112, 5117, 5121, 5740, 5743,
+ 5759, 5761, 5786, 5792, 5866, 5870, 5880, 5888, 5905, 5919, 5937, 5952, 5969, 5984,
+ 5996, 5998, 6000, 6016, 6067, 6103, 6103, 6108, 6108, 6128, 6137, 6176, 6264, 6272,
+ 6276, 6279, 6312, 6314, 6314, 6320, 6389, 6400, 6430, 6480, 6509, 6512, 6516, 6528,
+ 6571, 6576, 6601, 6618, 6618, 6656, 6678, 6688, 6740, 6823, 6823, 6917, 6963, 6981,
+ 6988, 7043, 7072, 7086, 7087, 7098, 7141, 7168, 7203, 7245, 7247, 7258, 7293, 7296,
+ 7304, 7312, 7354, 7357, 7359, 7401, 7404, 7406, 7411, 7413, 7414, 7418, 7418, 7424,
+ 7615, 7680, 7957, 7960, 7965, 7968, 8005, 8008, 8013, 8016, 8023, 8025, 8025, 8027,
+ 8027, 8029, 8029, 8031, 8061, 8064, 8116, 8118, 8124, 8126, 8126, 8130, 8132, 8134,
+ 8140, 8144, 8147, 8150, 8155, 8160, 8172, 8178, 8180, 8182, 8188, 8304, 8305, 8308,
+ 8313, 8319, 8329, 8336, 8348, 8450, 8450, 8455, 8455, 8458, 8467, 8469, 8469, 8473,
+ 8477, 8484, 8484, 8486, 8486, 8488, 8488, 8490, 8493, 8495, 8505, 8508, 8511, 8517,
+ 8521, 8526, 8526, 8528, 8585, 9312, 9371, 9450, 9471, 10102, 10131, 11264, 11492,
+ 11499, 11502, 11506, 11507, 11517, 11517, 11520, 11557, 11559, 11559, 11565, 11565,
+ 11568, 11623, 11631, 11631, 11648, 11670, 11680, 11686, 11688, 11694, 11696, 11702,
+ 11704, 11710, 11712, 11718, 11720, 11726, 11728, 11734, 11736, 11742, 11823, 11823,
+ 12293, 12295, 12321, 12329, 12337, 12341, 12344, 12348, 12353, 12438, 12445, 12447,
+ 12449, 12538, 12540, 12543, 12549, 12591, 12593, 12686, 12690, 12693, 12704, 12735,
+ 12784, 12799, 12832, 12841, 12872, 12879, 12881, 12895, 12928, 12937, 12977, 12991,
+ 13312, 19903, 19968, 42124, 42192, 42237, 42240, 42508, 42512, 42527, 42538, 42539,
+ 42560, 42606, 42623, 42653, 42656, 42735, 42775, 42783, 42786, 42888, 42891, 42954,
+ 42960, 42961, 42963, 42963, 42965, 42969, 42994, 43009, 43011, 43013, 43015, 43018,
+ 43020, 43042, 43056, 43061, 43072, 43123, 43138, 43187, 43250, 43255, 43259, 43259,
+ 43261, 43262, 43274, 43301, 43312, 43334, 43360, 43388, 43396, 43442, 43471, 43471,
+ 43488, 43492, 43494, 43503, 43514, 43518, 43520, 43560, 43584, 43586, 43588, 43595,
+ 43616, 43638, 43642, 43642, 43646, 43695, 43697, 43697, 43701, 43702, 43705, 43709,
+ 43712, 43712, 43714, 43714, 43739, 43741, 43744, 43754, 43762, 43764, 43777, 43782,
+ 43785, 43790, 43793, 43798, 43808, 43814, 43816, 43822, 43824, 43866, 43868, 43881,
+ 43888, 44002, 44032, 55203, 55216, 55238, 55243, 55291, 63744, 64109, 64112, 64217,
+ 64256, 64262, 64275, 64279, 64285, 64285, 64287, 64296, 64298, 64310, 64312, 64316,
+ 64318, 64318, 64320, 64321, 64323, 64324, 64326, 64433, 64467, 64829, 64848, 64911,
+ 64914, 64967, 65008, 65019, 65136, 65140, 65142, 65276, 65313, 65338, 65345, 65370,
+ 65382, 65470, 65474, 65479, 65482, 65487, 65490, 65495, 65498, 65500, 65536, 65547,
+ 65549, 65574, 65576, 65594, 65596, 65597, 65599, 65613, 65616, 65629, 65664, 65786,
+ 65799, 65843, 65856, 65912, 65930, 65931, 66176, 66204, 66208, 66256, 66273, 66299,
+ 66304, 66339, 66349, 66378, 66384, 66421, 66432, 66461, 66464, 66499, 66504, 66511,
+ 66513, 66517, 66560, 66717, 66736, 66771, 66776, 66811, 66816, 66855, 66864, 66915,
+ 66928, 66938, 66940, 66954, 66956, 66962, 66964, 66965, 66967, 66977, 66979, 66993,
+ 66995, 67001, 67003, 67004, 67072, 67382, 67392, 67413, 67424, 67431, 67456, 67461,
+ 67463, 67504, 67506, 67514, 67584, 67589, 67592, 67592, 67594, 67637, 67639, 67640,
+ 67644, 67644, 67647, 67669, 67672, 67702, 67705, 67742, 67751, 67759, 67808, 67826,
+ 67828, 67829, 67835, 67867, 67872, 67897, 67968, 68023, 68028, 68047, 68050, 68096,
+ 68112, 68115, 68117, 68119, 68121, 68149, 68160, 68168, 68192, 68222, 68224, 68255,
+ 68288, 68295, 68297, 68324, 68331, 68335, 68352, 68405, 68416, 68437, 68440, 68466,
+ 68472, 68497, 68521, 68527, 68608, 68680, 68736, 68786, 68800, 68850, 68858, 68899,
+ 69216, 69246, 69248, 69289, 69296, 69297, 69376, 69415, 69424, 69445, 69457, 69460,
+ 69488, 69505, 69552, 69579, 69600, 69622, 69635, 69687, 69714, 69733, 69745, 69746,
+ 69749, 69749, 69763, 69807, 69840, 69864, 69891, 69926, 69956, 69956, 69959, 69959,
+ 69968, 70002, 70006, 70006, 70019, 70066, 70081, 70084, 70106, 70106, 70108, 70108,
+ 70113, 70132, 70144, 70161, 70163, 70187, 70207, 70208, 70272, 70278, 70280, 70280,
+ 70282, 70285, 70287, 70301, 70303, 70312, 70320, 70366, 70405, 70412, 70415, 70416,
+ 70419, 70440, 70442, 70448, 70450, 70451, 70453, 70457, 70461, 70461, 70480, 70480,
+ 70493, 70497, 70656, 70708, 70727, 70730, 70751, 70753, 70784, 70831, 70852, 70853,
+ 70855, 70855, 71040, 71086, 71128, 71131, 71168, 71215, 71236, 71236, 71296, 71338,
+ 71352, 71352, 71424, 71450, 71482, 71483, 71488, 71494, 71680, 71723, 71840, 71903,
+ 71914, 71922, 71935, 71942, 71945, 71945, 71948, 71955, 71957, 71958, 71960, 71983,
+ 71999, 71999, 72001, 72001, 72096, 72103, 72106, 72144, 72161, 72161, 72163, 72163,
+ 72192, 72192, 72203, 72242, 72250, 72250, 72272, 72272, 72284, 72329, 72349, 72349,
+ 72368, 72440, 72704, 72712, 72714, 72750, 72768, 72768, 72794, 72812, 72818, 72847,
+ 72960, 72966, 72968, 72969, 72971, 73008, 73030, 73030, 73056, 73061, 73063, 73064,
+ 73066, 73097, 73112, 73112, 73440, 73458, 73474, 73474, 73476, 73488, 73490, 73523,
+ 73648, 73648, 73664, 73684, 73728, 74649, 74752, 74862, 74880, 75075, 77712, 77808,
+ 77824, 78895, 78913, 78918, 82944, 83526, 92160, 92728, 92736, 92766, 92784, 92862,
+ 92880, 92909, 92928, 92975, 92992, 92995, 93019, 93025, 93027, 93047, 93053, 93071,
+ 93760, 93846, 93952, 94026, 94032, 94032, 94099, 94111, 94176, 94177, 94179, 94179,
+ 94208, 100343, 100352, 101589, 101632, 101640, 110576, 110579, 110581, 110587, 110589,
+ 110590, 110592, 110882, 110898, 110898, 110928, 110930, 110933, 110933, 110948, 110951,
+ 110960, 111355, 113664, 113770, 113776, 113788, 113792, 113800, 113808, 113817, 119488,
+ 119507, 119520, 119539, 119648, 119672, 119808, 119892, 119894, 119964, 119966, 119967,
+ 119970, 119970, 119973, 119974, 119977, 119980, 119982, 119993, 119995, 119995, 119997,
+ 120003, 120005, 120069, 120071, 120074, 120077, 120084, 120086, 120092, 120094, 120121,
+ 120123, 120126, 120128, 120132, 120134, 120134, 120138, 120144, 120146, 120485, 120488,
+ 120512, 120514, 120538, 120540, 120570, 120572, 120596, 120598, 120628, 120630, 120654,
+ 120656, 120686, 120688, 120712, 120714, 120744, 120746, 120770, 120772, 120779, 122624,
+ 122654, 122661, 122666, 122928, 122989, 123136, 123180, 123191, 123197, 123214, 123214,
+ 123536, 123565, 123584, 123627, 124112, 124139, 124896, 124902, 124904, 124907, 124909,
+ 124910, 124912, 124926, 124928, 125124, 125127, 125135, 125184, 125251, 125259, 125259,
+ 126065, 126123, 126125, 126127, 126129, 126132, 126209, 126253, 126255, 126269, 126464,
+ 126467, 126469, 126495, 126497, 126498, 126500, 126500, 126503, 126503, 126505, 126514,
+ 126516, 126519, 126521, 126521, 126523, 126523, 126530, 126530, 126535, 126535, 126537,
+ 126537, 126539, 126539, 126541, 126543, 126545, 126546, 126548, 126548, 126551, 126551,
+ 126553, 126553, 126555, 126555, 126557, 126557, 126559, 126559, 126561, 126562, 126564,
+ 126564, 126567, 126570, 126572, 126578, 126580, 126583, 126585, 126588, 126590, 126590,
+ 126592, 126601, 126603, 126619, 126625, 126627, 126629, 126633, 126635, 126651, 127232,
+ 127244, 131072, 173791, 173824, 177977, 177984, 178205, 178208, 183969, 183984, 191456,
+ 194560, 195101, 196608, 201546, 201552, 205743
+ };
+
+ /**
+ * Code points with a non-zero canonical combining class.
+ *
+ * <p>192 ranges covering 922 code points, recorded
+ * from CPython 15.0.0.
+ */
+ private static final int[] COMBINING = {
+ 768, 846, 848, 879, 1155, 1159, 1425, 1469, 1471, 1471, 1473, 1474, 1476, 1477, 1479,
+ 1479, 1552, 1562, 1611, 1631, 1648, 1648, 1750, 1756, 1759, 1764, 1767, 1768, 1770,
+ 1773, 1809, 1809, 1840, 1866, 2027, 2035, 2045, 2045, 2070, 2073, 2075, 2083, 2085,
+ 2087, 2089, 2093, 2137, 2139, 2200, 2207, 2250, 2273, 2275, 2303, 2364, 2364, 2381,
+ 2381, 2385, 2388, 2492, 2492, 2509, 2509, 2558, 2558, 2620, 2620, 2637, 2637, 2748,
+ 2748, 2765, 2765, 2876, 2876, 2893, 2893, 3021, 3021, 3132, 3132, 3149, 3149, 3157,
+ 3158, 3260, 3260, 3277, 3277, 3387, 3388, 3405, 3405, 3530, 3530, 3640, 3642, 3656,
+ 3659, 3768, 3770, 3784, 3787, 3864, 3865, 3893, 3893, 3895, 3895, 3897, 3897, 3953,
+ 3954, 3956, 3956, 3962, 3965, 3968, 3968, 3970, 3972, 3974, 3975, 4038, 4038, 4151,
+ 4151, 4153, 4154, 4237, 4237, 4957, 4959, 5908, 5909, 5940, 5940, 6098, 6098, 6109,
+ 6109, 6313, 6313, 6457, 6459, 6679, 6680, 6752, 6752, 6773, 6780, 6783, 6783, 6832,
+ 6845, 6847, 6862, 6964, 6964, 6980, 6980, 7019, 7027, 7082, 7083, 7142, 7142, 7154,
+ 7155, 7223, 7223, 7376, 7378, 7380, 7392, 7394, 7400, 7405, 7405, 7412, 7412, 7416,
+ 7417, 7616, 7679, 8400, 8412, 8417, 8417, 8421, 8432, 11503, 11505, 11647, 11647,
+ 11744, 11775, 12330, 12335, 12441, 12442, 42607, 42607, 42612, 42621, 42654, 42655,
+ 42736, 42737, 43014, 43014, 43052, 43052, 43204, 43204, 43232, 43249, 43307, 43309,
+ 43347, 43347, 43443, 43443, 43456, 43456, 43696, 43696, 43698, 43700, 43703, 43704,
+ 43710, 43711, 43713, 43713, 43766, 43766, 44013, 44013, 64286, 64286, 65056, 65071,
+ 66045, 66045, 66272, 66272, 66422, 66426, 68109, 68109, 68111, 68111, 68152, 68154,
+ 68159, 68159, 68325, 68326, 68900, 68903, 69291, 69292, 69373, 69375, 69446, 69456,
+ 69506, 69509, 69702, 69702, 69744, 69744, 69759, 69759, 69817, 69818, 69888, 69890,
+ 69939, 69940, 70003, 70003, 70080, 70080, 70090, 70090, 70197, 70198, 70377, 70378,
+ 70459, 70460, 70477, 70477, 70502, 70508, 70512, 70516, 70722, 70722, 70726, 70726,
+ 70750, 70750, 70850, 70851, 71103, 71104, 71231, 71231, 71350, 71351, 71467, 71467,
+ 71737, 71738, 71997, 71998, 72003, 72003, 72160, 72160, 72244, 72244, 72263, 72263,
+ 72345, 72345, 72767, 72767, 73026, 73026, 73028, 73029, 73111, 73111, 73537, 73538,
+ 92912, 92916, 92976, 92982, 94192, 94193, 113822, 113822, 119141, 119145, 119149,
+ 119154, 119163, 119170, 119173, 119179, 119210, 119213, 119362, 119364, 122880, 122886,
+ 122888, 122904, 122907, 122913, 122915, 122916, 122918, 122922, 123023, 123023, 123184,
+ 123190, 123566, 123566, 123628, 123631, 124140, 124143, 125136, 125142, 125252, 125258
+ };
+
+ /** Python's {@code str.isalpha()} for one code point. */
+ public static boolean isAlpha(int codePoint) {
+ return contains(ALPHA, codePoint);
+ }
+
+ /** Python's {@code [^\\W\\d_]} for one code point: a word character that is not a digit. */
+ public static boolean isWordChar(int codePoint) {
+ return contains(WORD, codePoint);
+ }
+
+ /** Whether {@code unicodedata.combining} would return non-zero for this code point. */
+ public static boolean isCombining(int codePoint) {
+ return contains(COMBINING, codePoint);
+ }
+
+ /**
+ * Python's {@code str.lower()}, which is not Java's.
+ *
+ * <p>{@link String#toLowerCase} applies the contextual rules, so {@code "\u0391\u03A3"}
+ * lowercases to {@code "\u03B1\u03C2"} -- a FINAL sigma -- where Python gives
+ * {@code "\u03B1\u03C3"}. Lowercasing one code point at a time removes the context and
+ * therefore the rule. Verified over every code point in Unicode: with the multi-character
+ * mapping below handled explicitly, the two agree exactly.
+ */
+ public static String pythonLower(String text) {
+ StringBuilder out = new StringBuilder(text.length());
+ int i = 0;
+ while (i < text.length()) {
+ int codePoint = text.codePointAt(i);
+ i += Character.charCount(codePoint);
+ String expanded = multiCharLower(codePoint);
+ if (expanded != null) {
+ out.append(expanded);
+ } else {
+ out.appendCodePoint(Character.toLowerCase(codePoint));
+ }
+ }
+ return out.toString();
+ }
+
+ /**
+ * The one code point whose Python lowercase is longer than one code point.
+ *
+ * <p>{@code laya.lang} depends on it: it replaces U+0130 before lowering, because the
+ * line-length reasoning in its leaf scan counts code points on the RAW line and this mapping
+ * would otherwise make a four-character line reach four word tokens.
+ */
+ private static String multiCharLower(int codePoint) {
+ switch (codePoint) {
+ case 0x0130:
+ return "\u0069\u0307";
+ default:
+ return null;
+ }
+ }
+
+ /** Binary search over a flat array of inclusive [lo, hi] pairs. */
+ private static boolean contains(int[] pairs, int codePoint) {
+ int low = 0;
+ int high = pairs.length / 2 - 1;
+ while (low <= high) {
+ int mid = (low + high) >>> 1;
+ if (codePoint < pairs[mid * 2]) {
+ high = mid - 1;
+ } else if (codePoint > pairs[mid * 2 + 1]) {
+ low = mid + 1;
+ } else {
+ return true;
+ }
+ }
+ return false;
+ }
+}
diff --git a/laya-java/scripts/gen_unicode_tables.py b/laya-java/scripts/gen_unicode_tables.py
new file mode 100755
index 00000000..f0c892cc
--- /dev/null
+++ b/laya-java/scripts/gen_unicode_tables.py
@@ -0,0 +1,248 @@
+#!/usr/bin/env python3
+"""Generate the Unicode tables `laya.lang`'s character predicates need, as Java source.
+
+WHY A GENERATED TABLE RATHER THAN THE JDK'S OWN PREDICATES. `laya/lang.py` leans on three
+character properties, and the JDK disagrees with CPython about all three:
+
+ * `str.isalpha()` is categories Lu/Ll/Lt/Lm/Lo, which IS what `Character.isLetter` means -- but
+ JDK 17 ships Unicode 13.0 and CPython 3.12 ships 15.0, so the JDK does not know 4,863 of the
+ code points Python calls letters (U+0870 Arabic Extended-B, U+11F00 Kawi, ...). It reports them
+ UNASSIGNED, so text in a recently-added script would be counted under no script at all and
+ routed to the English checkpoint.
+ * Python's `\\w` minus `\\d` minus `_` is letters plus the Nl and No numeric categories, which has
+ the same version gap.
+ * `unicodedata.combining(ch) != 0` is the canonical combining CLASS, which the JDK does not
+ expose at all. Approximating it with the Mn and Mc categories -- which is what the .NET port
+ does -- is wrong on 1,460 code points: 1,410 it would call combining that Python does not
+ (U+034F COMBINING GRAPHEME JOINER among them, class 0) and 50 it would miss.
+
+So the tables are recorded from CPython here and compiled in. That also makes the predicates
+independent of the JDK a consumer happens to run, which matters more than it sounds: a JDK upgrade
+must not silently change which script a request is detected as.
+
+ python laya-java/scripts/gen_unicode_tables.py # write
+ python laya-java/scripts/gen_unicode_tables.py --check # fail if the committed copy differs
+"""
+import argparse
+import os
+import re
+import sys
+import unicodedata
+
+HERE = os.path.dirname(os.path.abspath(__file__))
+TARGET = os.path.join(
+ HERE, "..", "laya-java", "src", "main", "java", "com", "convaiinnovations", "laya", "lang",
+ "UnicodeTables.java")
+
+WORD = re.compile(r"[^\W\d_]", re.UNICODE)
+MAX = 0x110000
+SURROGATES = range(0xD800, 0xE000)
+
+
+def ranges(predicate):
+ """Inclusive [lo, hi] runs where `predicate` holds. Surrogates never hold."""
+ out, start = [], None
+ for cp in range(MAX):
+ hit = False if cp in SURROGATES else predicate(cp)
+ if hit and start is None:
+ start = cp
+ elif not hit and start is not None:
+ out.append((start, cp - 1))
+ start = None
+ if start is not None:
+ out.append((start, MAX - 1))
+ return out
+
+
+def table(name, rows, doc):
+ """One flat `int[]` of lo, hi pairs, wrapped at a readable width."""
+ flat = [str(value) for row in rows for value in row]
+ lines, current = [], " "
+ for item in flat:
+ piece = item + ", "
+ if len(current) + len(piece) > 100:
+ lines.append(current.rstrip())
+ current = " "
+ current += piece
+ if current.strip():
+ lines.append(current.rstrip().rstrip(","))
+ body = "\n".join(lines)
+ return (' /**\n * %s\n *\n * <p>%d ranges covering %d code points, recorded\n'
+ ' * from CPython %s.\n */\n'
+ ' private static final int[] %s = {\n%s\n };\n'
+ % (doc, len(rows), sum(b - a + 1 for a, b in rows),
+ unicodedata.unidata_version, name, body))
+
+
+def render():
+ alpha = ranges(lambda cp: chr(cp).isalpha())
+ word = ranges(lambda cp: WORD.fullmatch(chr(cp)) is not None)
+ combining = ranges(lambda cp: unicodedata.combining(chr(cp)) != 0)
+
+ # Every code point whose lowercase is more than one code point. In Unicode 15 there is
+ # exactly one, and `laya/lang.py` depends on it: it replaces U+0130 before lowering, because
+ # otherwise the line-length reasoning in `_leaf_non_english` would be wrong.
+ multi = []
+ for cp in range(MAX):
+ if cp in SURROGATES:
+ continue
+ low = chr(cp).lower()
+ if len(low) > 1:
+ multi.append((cp, [ord(c) for c in low]))
+
+ header = '''// GENERATED by laya-java/scripts/gen_unicode_tables.py -- do not edit.
+//
+// Regenerate with:
+// python laya-java/scripts/gen_unicode_tables.py
+// and `--check` fails when the committed copy no longer matches CPython.
+package com.convaiinnovations.laya.lang;
+
+/**
+ * The Unicode properties {@code laya.lang} depends on, recorded from CPython %s.
+ *
+ * <p>Compiled in rather than read from the JDK, because the JDK disagrees with CPython about all
+ * three of them. {@code Character.isLetter} means the same categories as {@code str.isalpha()},
+ * but JDK 17 ships Unicode 13.0 against CPython's 15.0 and therefore does not know %d of the code
+ * points Python calls letters -- it reports them unassigned, which would count text in a
+ * recently-added script under no script at all and route it to the English checkpoint. The
+ * canonical combining class is not exposed by the JDK at all, and the Mn/Mc categories are not a
+ * substitute: they differ from it on 1,460 code points, including U+034F COMBINING GRAPHEME
+ * JOINER, whose class is zero.
+ *
+ * <p>A table also makes detection independent of the JDK a consumer runs, which matters more than
+ * it sounds: a JDK upgrade must not silently change which script a request is detected as.
+ */
+public final class UnicodeTables {
+
+ /** The CPython release these tables were recorded from. */
+ public static final String UNICODE_VERSION = "%s";
+
+ private UnicodeTables() {
+ }
+
+''' % (unicodedata.unidata_version, 4863, unicodedata.unidata_version)
+
+ body = (
+ table("ALPHA", alpha,
+ "Python's {@code str.isalpha()}: categories Lu, Ll, Lt, Lm and Lo.")
+ + "\n"
+ + table("WORD", word,
+ "Python's {@code [^\\\\W\\\\d_]}: letters plus the Nl and No numeric categories, "
+ "which is\n * what {@code laya.lang}'s word regex matches.")
+ + "\n"
+ + table("COMBINING", combining,
+ "Code points with a non-zero canonical combining class.")
+ )
+
+ multi_cases = "\n".join(
+ ' case 0x%04X:\n return "%s";'
+ % (cp, "".join("\\u%04X" % c for c in low)) for cp, low in multi)
+ methods = '''
+ /** Python's {@code str.isalpha()} for one code point. */
+ public static boolean isAlpha(int codePoint) {
+ return contains(ALPHA, codePoint);
+ }
+
+ /** Python's {@code [^\\\\W\\\\d_]} for one code point: a word character that is not a digit. */
+ public static boolean isWordChar(int codePoint) {
+ return contains(WORD, codePoint);
+ }
+
+ /** Whether {@code unicodedata.combining} would return non-zero for this code point. */
+ public static boolean isCombining(int codePoint) {
+ return contains(COMBINING, codePoint);
+ }
+
+ /**
+ * Python's {@code str.lower()}, which is not Java's.
+ *
+ * <p>{@link String#toLowerCase} applies the contextual rules, so {@code "\\u0391\\u03A3"}
+ * lowercases to {@code "\\u03B1\\u03C2"} -- a FINAL sigma -- where Python gives
+ * {@code "\\u03B1\\u03C3"}. Lowercasing one code point at a time removes the context and
+ * therefore the rule. Verified over every code point in Unicode: with the multi-character
+ * mapping below handled explicitly, the two agree exactly.
+ */
+ public static String pythonLower(String text) {
+ StringBuilder out = new StringBuilder(text.length());
+ int i = 0;
+ while (i < text.length()) {
+ int codePoint = text.codePointAt(i);
+ i += Character.charCount(codePoint);
+ String expanded = multiCharLower(codePoint);
+ if (expanded != null) {
+ out.append(expanded);
+ } else {
+ out.appendCodePoint(Character.toLowerCase(codePoint));
+ }
+ }
+ return out.toString();
+ }
+
+ /**
+ * The one code point whose Python lowercase is longer than one code point.
+ *
+ * <p>{@code laya.lang} depends on it: it replaces U+0130 before lowering, because the
+ * line-length reasoning in its leaf scan counts code points on the RAW line and this mapping
+ * would otherwise make a four-character line reach four word tokens.
+ */
+ private static String multiCharLower(int codePoint) {
+ switch (codePoint) {
+%s
+ default:
+ return null;
+ }
+ }
+
+ /** Binary search over a flat array of inclusive [lo, hi] pairs. */
+ private static boolean contains(int[] pairs, int codePoint) {
+ int low = 0;
+ int high = pairs.length / 2 - 1;
+ while (low <= high) {
+ int mid = (low + high) >>> 1;
+ if (codePoint < pairs[mid * 2]) {
+ high = mid - 1;
+ } else if (codePoint > pairs[mid * 2 + 1]) {
+ low = mid + 1;
+ } else {
+ return true;
+ }
+ }
+ return false;
+ }
+}
+''' % multi_cases
+ return header + body + methods
+
+
+def main(argv=None):
+ parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
+ parser.add_argument("--check", action="store_true",
+ help="do not write; exit 1 if the committed copy would change")
+ args = parser.parse_args(argv)
+ fresh = render()
+ target = os.path.normpath(TARGET)
+ if args.check:
+ try:
+ with open(target, encoding="utf-8") as handle:
+ current = handle.read()
+ except OSError:
+ print("gen_unicode_tables: %s is missing" % target, file=sys.stderr)
+ return 1
+ if current != fresh:
+ print("gen_unicode_tables: %s is stale; run "
+ "laya-java/scripts/gen_unicode_tables.py and commit the result" % target,
+ file=sys.stderr)
+ return 1
+ print("gen_unicode_tables: UnicodeTables.java matches CPython %s"
+ % unicodedata.unidata_version)
+ return 0
+ os.makedirs(os.path.dirname(target), exist_ok=True)
+ with open(target, "w", encoding="utf-8") as handle:
+ handle.write(fresh)
+ print("wrote %s (%d bytes)" % (os.path.relpath(target, os.path.join(HERE, "..", "..")),
+ len(fresh)))
+ return 0
+
+
+if __name__ == "__main__":
+ sys.exit(main())