Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
feat(java): the two Unicode tables laya.email needs
Adeshina Lasisife77867b27e2
diff --git a/laya-java/fixtures/lang_detect.json b/laya-java/fixtures/lang_detect.jsonindex 2746dcfd..93f65181 100644--- a/laya-java/fixtures/lang_detect.json+++ b/laya-java/fixtures/lang_detect.json@@ -15,6 +15,8 @@"upper": "494a3792b2eb7d335604ad353d2f5db88d1b11178e46de4f1a03e4de080697e7","lower": "f3603abf8d8e964512a2905c9d81e74c6e6fc282f2ca86f5052f80717af9be30","space": "6c4eda9112d4db3cd23936f071127b489c7a69988e8758a098bb879af25f141e",+ "mark": "a2063cf78c2c2e836f232a36c26930e7385b336d3468018fdd2b485f86586314",+ "initial": "5b136bd2b3f50965fc5a10ee3188a68778c03257d11d699bcf9b3cae315d37ac","python_lower": "a93175724463d190c5c2bf756ecd8363f3fc3ee72bf93dbe5b6c2c27db80fbaf","script_of": "863dbf19c0b480abf327c9d5d85f373da1ab7f7724d2a5fcfb086b63419ead75"},diff --git a/laya-java/laya-java/src/main/java/com/convaiinnovations/laya/lang/UnicodeTables.java b/laya-java/laya-java/src/main/java/com/convaiinnovations/laya/lang/UnicodeTables.javaindex 6f1c2b70..b619fbe9 100644--- a/laya-java/laya-java/src/main/java/com/convaiinnovations/laya/lang/UnicodeTables.java+++ b/laya-java/laya-java/src/main/java/com/convaiinnovations/laya/lang/UnicodeTables.java@@ -711,6 +711,258 @@ public final class UnicodeTables {}+ /**+ * The Mn/Mc/Me categories: what {@code laya.email} drops before matching a+sign-off's tail, and what the other ports spell {@code \p{M}}.+ *+ * <p>310 ranges covering 2450 code points, recorded+ * from CPython 15.0.0.+ */+ private static final int[] MARK = markTable();++ private static int[] markTable() {+ return new int[] {+ 768, 879, 1155, 1161, 1425, 1469, 1471, 1471, 1473, 1474, 1476, 1477, 1479, 1479,+ 1552, 1562, 1611, 1631, 1648, 1648, 1750, 1756, 1759, 1764, 1767, 1768, 1770, 1773,+ 1809, 1809, 1840, 1866, 1958, 1968, 2027, 2035, 2045, 2045, 2070, 2073, 2075, 2083,+ 2085, 2087, 2089, 2093, 2137, 2139, 2200, 2207, 2250, 2273, 2275, 2307, 2362, 2364,+ 2366, 2383, 2385, 2391, 2402, 2403, 2433, 2435, 2492, 2492, 2494, 2500, 2503, 2504,+ 2507, 2509, 2519, 2519, 2530, 2531, 2558, 2558, 2561, 2563, 2620, 2620, 2622, 2626,+ 2631, 2632, 2635, 2637, 2641, 2641, 2672, 2673, 2677, 2677, 2689, 2691, 2748, 2748,+ 2750, 2757, 2759, 2761, 2763, 2765, 2786, 2787, 2810, 2815, 2817, 2819, 2876, 2876,+ 2878, 2884, 2887, 2888, 2891, 2893, 2901, 2903, 2914, 2915, 2946, 2946, 3006, 3010,+ 3014, 3016, 3018, 3021, 3031, 3031, 3072, 3076, 3132, 3132, 3134, 3140, 3142, 3144,+ 3146, 3149, 3157, 3158, 3170, 3171, 3201, 3203, 3260, 3260, 3262, 3268, 3270, 3272,+ 3274, 3277, 3285, 3286, 3298, 3299, 3315, 3315, 3328, 3331, 3387, 3388, 3390, 3396,+ 3398, 3400, 3402, 3405, 3415, 3415, 3426, 3427, 3457, 3459, 3530, 3530, 3535, 3540,+ 3542, 3542, 3544, 3551, 3570, 3571, 3633, 3633, 3636, 3642, 3655, 3662, 3761, 3761,+ 3764, 3772, 3784, 3790, 3864, 3865, 3893, 3893, 3895, 3895, 3897, 3897, 3902, 3903,+ 3953, 3972, 3974, 3975, 3981, 3991, 3993, 4028, 4038, 4038, 4139, 4158, 4182, 4185,+ 4190, 4192, 4194, 4196, 4199, 4205, 4209, 4212, 4226, 4237, 4239, 4239, 4250, 4253,+ 4957, 4959, 5906, 5909, 5938, 5940, 5970, 5971, 6002, 6003, 6068, 6099, 6109, 6109,+ 6155, 6157, 6159, 6159, 6277, 6278, 6313, 6313, 6432, 6443, 6448, 6459, 6679, 6683,+ 6741, 6750, 6752, 6780, 6783, 6783, 6832, 6862, 6912, 6916, 6964, 6980, 7019, 7027,+ 7040, 7042, 7073, 7085, 7142, 7155, 7204, 7223, 7376, 7378, 7380, 7400, 7405, 7405,+ 7412, 7412, 7415, 7417, 7616, 7679, 8400, 8432, 11503, 11505, 11647, 11647, 11744,+ 11775, 12330, 12335, 12441, 12442, 42607, 42610, 42612, 42621, 42654, 42655, 42736,+ 42737, 43010, 43010, 43014, 43014, 43019, 43019, 43043, 43047, 43052, 43052, 43136,+ 43137, 43188, 43205, 43232, 43249, 43263, 43263, 43302, 43309, 43335, 43347, 43392,+ 43395, 43443, 43456, 43493, 43493, 43561, 43574, 43587, 43587, 43596, 43597, 43643,+ 43645, 43696, 43696, 43698, 43700, 43703, 43704, 43710, 43711, 43713, 43713, 43755,+ 43759, 43765, 43766, 44003, 44010, 44012, 44013, 64286, 64286, 65024, 65039, 65056,+ 65071, 66045, 66045, 66272, 66272, 66422, 66426, 68097, 68099, 68101, 68102, 68108,+ 68111, 68152, 68154, 68159, 68159, 68325, 68326, 68900, 68903, 69291, 69292, 69373,+ 69375, 69446, 69456, 69506, 69509, 69632, 69634, 69688, 69702, 69744, 69744, 69747,+ 69748, 69759, 69762, 69808, 69818, 69826, 69826, 69888, 69890, 69927, 69940, 69957,+ 69958, 70003, 70003, 70016, 70018, 70067, 70080, 70089, 70092, 70094, 70095, 70188,+ 70199, 70206, 70206, 70209, 70209, 70367, 70378, 70400, 70403, 70459, 70460, 70462,+ 70468, 70471, 70472, 70475, 70477, 70487, 70487, 70498, 70499, 70502, 70508, 70512,+ 70516, 70709, 70726, 70750, 70750, 70832, 70851, 71087, 71093, 71096, 71104, 71132,+ 71133, 71216, 71232, 71339, 71351, 71453, 71467, 71724, 71738, 71984, 71989, 71991,+ 71992, 71995, 71998, 72000, 72000, 72002, 72003, 72145, 72151, 72154, 72160, 72164,+ 72164, 72193, 72202, 72243, 72249, 72251, 72254, 72263, 72263, 72273, 72283, 72330,+ 72345, 72751, 72758, 72760, 72767, 72850, 72871, 72873, 72886, 73009, 73014, 73018,+ 73018, 73020, 73021, 73023, 73029, 73031, 73031, 73098, 73102, 73104, 73105, 73107,+ 73111, 73459, 73462, 73472, 73473, 73475, 73475, 73524, 73530, 73534, 73538, 78912,+ 78912, 78919, 78933, 92912, 92916, 92976, 92982, 94031, 94031, 94033, 94087, 94095,+ 94098, 94180, 94180, 94192, 94193, 113821, 113822, 118528, 118573, 118576, 118598,+ 119141, 119145, 119149, 119154, 119163, 119170, 119173, 119179, 119210, 119213,+ 119362, 119364, 121344, 121398, 121403, 121452, 121461, 121461, 121476, 121476,+ 121499, 121503, 121505, 121519, 122880, 122886, 122888, 122904, 122907, 122913,+ 122915, 122916, 122918, 122922, 123023, 123023, 123184, 123190, 123566, 123566,+ 123628, 123631, 124140, 124143, 125136, 125142, 125252, 125258, 917760, 917999+ };+ }+ /**+ * The Lu/Lt/Lo categories: the letters a name may begin with, in any script.+ *+ * <p>1156 ranges covering 133474 code points, recorded+ * from CPython 15.0.0.+ */+ private static final int[] INITIAL = initialTable();++ private static int[] initialTable() {+ return new int[] {+ 65, 90, 170, 170, 186, 186, 192, 214, 216, 222, 256, 256, 258, 258, 260, 260, 262,+ 262, 264, 264, 266, 266, 268, 268, 270, 270, 272, 272, 274, 274, 276, 276, 278,+ 278, 280, 280, 282, 282, 284, 284, 286, 286, 288, 288, 290, 290, 292, 292, 294,+ 294, 296, 296, 298, 298, 300, 300, 302, 302, 304, 304, 306, 306, 308, 308, 310,+ 310, 313, 313, 315, 315, 317, 317, 319, 319, 321, 321, 323, 323, 325, 325, 327,+ 327, 330, 330, 332, 332, 334, 334, 336, 336, 338, 338, 340, 340, 342, 342, 344,+ 344, 346, 346, 348, 348, 350, 350, 352, 352, 354, 354, 356, 356, 358, 358, 360,+ 360, 362, 362, 364, 364, 366, 366, 368, 368, 370, 370, 372, 372, 374, 374, 376,+ 377, 379, 379, 381, 381, 385, 386, 388, 388, 390, 391, 393, 395, 398, 401, 403,+ 404, 406, 408, 412, 413, 415, 416, 418, 418, 420, 420, 422, 423, 425, 425, 428,+ 428, 430, 431, 433, 435, 437, 437, 439, 440, 443, 444, 448, 453, 455, 456, 458,+ 459, 461, 461, 463, 463, 465, 465, 467, 467, 469, 469, 471, 471, 473, 473, 475,+ 475, 478, 478, 480, 480, 482, 482, 484, 484, 486, 486, 488, 488, 490, 490, 492,+ 492, 494, 494, 497, 498, 500, 500, 502, 504, 506, 506, 508, 508, 510, 510, 512,+ 512, 514, 514, 516, 516, 518, 518, 520, 520, 522, 522, 524, 524, 526, 526, 528,+ 528, 530, 530, 532, 532, 534, 534, 536, 536, 538, 538, 540, 540, 542, 542, 544,+ 544, 546, 546, 548, 548, 550, 550, 552, 552, 554, 554, 556, 556, 558, 558, 560,+ 560, 562, 562, 570, 571, 573, 574, 577, 577, 579, 582, 584, 584, 586, 586, 588,+ 588, 590, 590, 660, 660, 880, 880, 882, 882, 886, 886, 895, 895, 902, 902, 904,+ 906, 908, 908, 910, 911, 913, 929, 931, 939, 975, 975, 978, 980, 984, 984, 986,+ 986, 988, 988, 990, 990, 992, 992, 994, 994, 996, 996, 998, 998, 1000, 1000, 1002,+ 1002, 1004, 1004, 1006, 1006, 1012, 1012, 1015, 1015, 1017, 1018, 1021, 1071, 1120,+ 1120, 1122, 1122, 1124, 1124, 1126, 1126, 1128, 1128, 1130, 1130, 1132, 1132, 1134,+ 1134, 1136, 1136, 1138, 1138, 1140, 1140, 1142, 1142, 1144, 1144, 1146, 1146, 1148,+ 1148, 1150, 1150, 1152, 1152, 1162, 1162, 1164, 1164, 1166, 1166, 1168, 1168, 1170,+ 1170, 1172, 1172, 1174, 1174, 1176, 1176, 1178, 1178, 1180, 1180, 1182, 1182, 1184,+ 1184, 1186, 1186, 1188, 1188, 1190, 1190, 1192, 1192, 1194, 1194, 1196, 1196, 1198,+ 1198, 1200, 1200, 1202, 1202, 1204, 1204, 1206, 1206, 1208, 1208, 1210, 1210, 1212,+ 1212, 1214, 1214, 1216, 1217, 1219, 1219, 1221, 1221, 1223, 1223, 1225, 1225, 1227,+ 1227, 1229, 1229, 1232, 1232, 1234, 1234, 1236, 1236, 1238, 1238, 1240, 1240, 1242,+ 1242, 1244, 1244, 1246, 1246, 1248, 1248, 1250, 1250, 1252, 1252, 1254, 1254, 1256,+ 1256, 1258, 1258, 1260, 1260, 1262, 1262, 1264, 1264, 1266, 1266, 1268, 1268, 1270,+ 1270, 1272, 1272, 1274, 1274, 1276, 1276, 1278, 1278, 1280, 1280, 1282, 1282, 1284,+ 1284, 1286, 1286, 1288, 1288, 1290, 1290, 1292, 1292, 1294, 1294, 1296, 1296, 1298,+ 1298, 1300, 1300, 1302, 1302, 1304, 1304, 1306, 1306, 1308, 1308, 1310, 1310, 1312,+ 1312, 1314, 1314, 1316, 1316, 1318, 1318, 1320, 1320, 1322, 1322, 1324, 1324, 1326,+ 1326, 1329, 1366, 1488, 1514, 1519, 1522, 1568, 1599, 1601, 1610, 1646, 1647, 1649,+ 1747, 1749, 1749, 1774, 1775, 1786, 1788, 1791, 1791, 1808, 1808, 1810, 1839, 1869,+ 1957, 1969, 1969, 1994, 2026, 2048, 2069, 2112, 2136, 2144, 2154, 2160, 2183, 2185,+ 2190, 2208, 2248, 2308, 2361, 2365, 2365, 2384, 2384, 2392, 2401, 2418, 2432, 2437,+ 2444, 2447, 2448, 2451, 2472, 2474, 2480, 2482, 2482, 2486, 2489, 2493, 2493, 2510,+ 2510, 2524, 2525, 2527, 2529, 2544, 2545, 2556, 2556, 2565, 2570, 2575, 2576, 2579,+ 2600, 2602, 2608, 2610, 2611, 2613, 2614, 2616, 2617, 2649, 2652, 2654, 2654, 2674,+ 2676, 2693, 2701, 2703, 2705, 2707, 2728, 2730, 2736, 2738, 2739, 2741, 2745, 2749,+ 2749, 2768, 2768, 2784, 2785, 2809, 2809, 2821, 2828, 2831, 2832, 2835, 2856, 2858,+ 2864, 2866, 2867, 2869, 2873, 2877, 2877, 2908, 2909, 2911, 2913, 2929, 2929, 2947,+ 2947, 2949, 2954, 2958, 2960, 2962, 2965, 2969, 2970, 2972, 2972, 2974, 2975, 2979,+ 2980, 2984, 2986, 2990, 3001, 3024, 3024, 3077, 3084, 3086, 3088, 3090, 3112, 3114,+ 3129, 3133, 3133, 3160, 3162, 3165, 3165, 3168, 3169, 3200, 3200, 3205, 3212, 3214,+ 3216, 3218, 3240, 3242, 3251, 3253, 3257, 3261, 3261, 3293, 3294, 3296, 3297, 3313,+ 3314, 3332, 3340, 3342, 3344, 3346, 3386, 3389, 3389, 3406, 3406, 3412, 3414, 3423,+ 3425, 3450, 3455, 3461, 3478, 3482, 3505, 3507, 3515, 3517, 3517, 3520, 3526, 3585,+ 3632, 3634, 3635, 3648, 3653, 3713, 3714, 3716, 3716, 3718, 3722, 3724, 3747, 3749,+ 3749, 3751, 3760, 3762, 3763, 3773, 3773, 3776, 3780, 3804, 3807, 3840, 3840, 3904,+ 3911, 3913, 3948, 3976, 3980, 4096, 4138, 4159, 4159, 4176, 4181, 4186, 4189, 4193,+ 4193, 4197, 4198, 4206, 4208, 4213, 4225, 4238, 4238, 4256, 4293, 4295, 4295, 4301,+ 4301, 4352, 4680, 4682, 4685, 4688, 4694, 4696, 4696, 4698, 4701, 4704, 4744, 4746,+ 4749, 4752, 4784, 4786, 4789, 4792, 4798, 4800, 4800, 4802, 4805, 4808, 4822, 4824,+ 4880, 4882, 4885, 4888, 4954, 4992, 5007, 5024, 5109, 5121, 5740, 5743, 5759, 5761,+ 5786, 5792, 5866, 5873, 5880, 5888, 5905, 5919, 5937, 5952, 5969, 5984, 5996, 5998,+ 6000, 6016, 6067, 6108, 6108, 6176, 6210, 6212, 6264, 6272, 6276, 6279, 6312, 6314,+ 6314, 6320, 6389, 6400, 6430, 6480, 6509, 6512, 6516, 6528, 6571, 6576, 6601, 6656,+ 6678, 6688, 6740, 6917, 6963, 6981, 6988, 7043, 7072, 7086, 7087, 7098, 7141, 7168,+ 7203, 7245, 7247, 7258, 7287, 7312, 7354, 7357, 7359, 7401, 7404, 7406, 7411, 7413,+ 7414, 7418, 7418, 7680, 7680, 7682, 7682, 7684, 7684, 7686, 7686, 7688, 7688, 7690,+ 7690, 7692, 7692, 7694, 7694, 7696, 7696, 7698, 7698, 7700, 7700, 7702, 7702, 7704,+ 7704, 7706, 7706, 7708, 7708, 7710, 7710, 7712, 7712, 7714, 7714, 7716, 7716, 7718,+ 7718, 7720, 7720, 7722, 7722, 7724, 7724, 7726, 7726, 7728, 7728, 7730, 7730, 7732,+ 7732, 7734, 7734, 7736, 7736, 7738, 7738, 7740, 7740, 7742, 7742, 7744, 7744, 7746,+ 7746, 7748, 7748, 7750, 7750, 7752, 7752, 7754, 7754, 7756, 7756, 7758, 7758, 7760,+ 7760, 7762, 7762, 7764, 7764, 7766, 7766, 7768, 7768, 7770, 7770, 7772, 7772, 7774,+ 7774, 7776, 7776, 7778, 7778, 7780, 7780, 7782, 7782, 7784, 7784, 7786, 7786, 7788,+ 7788, 7790, 7790, 7792, 7792, 7794, 7794, 7796, 7796, 7798, 7798, 7800, 7800, 7802,+ 7802, 7804, 7804, 7806, 7806, 7808, 7808, 7810, 7810, 7812, 7812, 7814, 7814, 7816,+ 7816, 7818, 7818, 7820, 7820, 7822, 7822, 7824, 7824, 7826, 7826, 7828, 7828, 7838,+ 7838, 7840, 7840, 7842, 7842, 7844, 7844, 7846, 7846, 7848, 7848, 7850, 7850, 7852,+ 7852, 7854, 7854, 7856, 7856, 7858, 7858, 7860, 7860, 7862, 7862, 7864, 7864, 7866,+ 7866, 7868, 7868, 7870, 7870, 7872, 7872, 7874, 7874, 7876, 7876, 7878, 7878, 7880,+ 7880, 7882, 7882, 7884, 7884, 7886, 7886, 7888, 7888, 7890, 7890, 7892, 7892, 7894,+ 7894, 7896, 7896, 7898, 7898, 7900, 7900, 7902, 7902, 7904, 7904, 7906, 7906, 7908,+ 7908, 7910, 7910, 7912, 7912, 7914, 7914, 7916, 7916, 7918, 7918, 7920, 7920, 7922,+ 7922, 7924, 7924, 7926, 7926, 7928, 7928, 7930, 7930, 7932, 7932, 7934, 7934, 7944,+ 7951, 7960, 7965, 7976, 7983, 7992, 7999, 8008, 8013, 8025, 8025, 8027, 8027, 8029,+ 8029, 8031, 8031, 8040, 8047, 8072, 8079, 8088, 8095, 8104, 8111, 8120, 8124, 8136,+ 8140, 8152, 8155, 8168, 8172, 8184, 8188, 8450, 8450, 8455, 8455, 8459, 8461, 8464,+ 8466, 8469, 8469, 8473, 8477, 8484, 8484, 8486, 8486, 8488, 8488, 8490, 8493, 8496,+ 8499, 8501, 8504, 8510, 8511, 8517, 8517, 8579, 8579, 11264, 11311, 11360, 11360,+ 11362, 11364, 11367, 11367, 11369, 11369, 11371, 11371, 11373, 11376, 11378, 11378,+ 11381, 11381, 11390, 11392, 11394, 11394, 11396, 11396, 11398, 11398, 11400, 11400,+ 11402, 11402, 11404, 11404, 11406, 11406, 11408, 11408, 11410, 11410, 11412, 11412,+ 11414, 11414, 11416, 11416, 11418, 11418, 11420, 11420, 11422, 11422, 11424, 11424,+ 11426, 11426, 11428, 11428, 11430, 11430, 11432, 11432, 11434, 11434, 11436, 11436,+ 11438, 11438, 11440, 11440, 11442, 11442, 11444, 11444, 11446, 11446, 11448, 11448,+ 11450, 11450, 11452, 11452, 11454, 11454, 11456, 11456, 11458, 11458, 11460, 11460,+ 11462, 11462, 11464, 11464, 11466, 11466, 11468, 11468, 11470, 11470, 11472, 11472,+ 11474, 11474, 11476, 11476, 11478, 11478, 11480, 11480, 11482, 11482, 11484, 11484,+ 11486, 11486, 11488, 11488, 11490, 11490, 11499, 11499, 11501, 11501, 11506, 11506,+ 11568, 11623, 11648, 11670, 11680, 11686, 11688, 11694, 11696, 11702, 11704, 11710,+ 11712, 11718, 11720, 11726, 11728, 11734, 11736, 11742, 12294, 12294, 12348, 12348,+ 12353, 12438, 12447, 12447, 12449, 12538, 12543, 12543, 12549, 12591, 12593, 12686,+ 12704, 12735, 12784, 12799, 13312, 19903, 19968, 40980, 40982, 42124, 42192, 42231,+ 42240, 42507, 42512, 42527, 42538, 42539, 42560, 42560, 42562, 42562, 42564, 42564,+ 42566, 42566, 42568, 42568, 42570, 42570, 42572, 42572, 42574, 42574, 42576, 42576,+ 42578, 42578, 42580, 42580, 42582, 42582, 42584, 42584, 42586, 42586, 42588, 42588,+ 42590, 42590, 42592, 42592, 42594, 42594, 42596, 42596, 42598, 42598, 42600, 42600,+ 42602, 42602, 42604, 42604, 42606, 42606, 42624, 42624, 42626, 42626, 42628, 42628,+ 42630, 42630, 42632, 42632, 42634, 42634, 42636, 42636, 42638, 42638, 42640, 42640,+ 42642, 42642, 42644, 42644, 42646, 42646, 42648, 42648, 42650, 42650, 42656, 42725,+ 42786, 42786, 42788, 42788, 42790, 42790, 42792, 42792, 42794, 42794, 42796, 42796,+ 42798, 42798, 42802, 42802, 42804, 42804, 42806, 42806, 42808, 42808, 42810, 42810,+ 42812, 42812, 42814, 42814, 42816, 42816, 42818, 42818, 42820, 42820, 42822, 42822,+ 42824, 42824, 42826, 42826, 42828, 42828, 42830, 42830, 42832, 42832, 42834, 42834,+ 42836, 42836, 42838, 42838, 42840, 42840, 42842, 42842, 42844, 42844, 42846, 42846,+ 42848, 42848, 42850, 42850, 42852, 42852, 42854, 42854, 42856, 42856, 42858, 42858,+ 42860, 42860, 42862, 42862, 42873, 42873, 42875, 42875, 42877, 42878, 42880, 42880,+ 42882, 42882, 42884, 42884, 42886, 42886, 42891, 42891, 42893, 42893, 42895, 42896,+ 42898, 42898, 42902, 42902, 42904, 42904, 42906, 42906, 42908, 42908, 42910, 42910,+ 42912, 42912, 42914, 42914, 42916, 42916, 42918, 42918, 42920, 42920, 42922, 42926,+ 42928, 42932, 42934, 42934, 42936, 42936, 42938, 42938, 42940, 42940, 42942, 42942,+ 42944, 42944, 42946, 42946, 42948, 42951, 42953, 42953, 42960, 42960, 42966, 42966,+ 42968, 42968, 42997, 42997, 42999, 42999, 43003, 43009, 43011, 43013, 43015, 43018,+ 43020, 43042, 43072, 43123, 43138, 43187, 43250, 43255, 43259, 43259, 43261, 43262,+ 43274, 43301, 43312, 43334, 43360, 43388, 43396, 43442, 43488, 43492, 43495, 43503,+ 43514, 43518, 43520, 43560, 43584, 43586, 43588, 43595, 43616, 43631, 43633, 43638,+ 43642, 43642, 43646, 43695, 43697, 43697, 43701, 43702, 43705, 43709, 43712, 43712,+ 43714, 43714, 43739, 43740, 43744, 43754, 43762, 43762, 43777, 43782, 43785, 43790,+ 43793, 43798, 43808, 43814, 43816, 43822, 43968, 44002, 44032, 55203, 55216, 55238,+ 55243, 55291, 63744, 64109, 64112, 64217, 64285, 64285, 64287, 64296, 64298, 64310,+ 64312, 64316, 64318, 64318, 64320, 64321, 64323, 64324, 64326, 64433, 64467, 64829,+ 64848, 64911, 64914, 64967, 65008, 65019, 65136, 65140, 65142, 65276, 65313, 65338,+ 65382, 65391, 65393, 65437, 65440, 65470, 65474, 65479, 65482, 65487, 65490, 65495,+ 65498, 65500, 65536, 65547, 65549, 65574, 65576, 65594, 65596, 65597, 65599, 65613,+ 65616, 65629, 65664, 65786, 66176, 66204, 66208, 66256, 66304, 66335, 66349, 66368,+ 66370, 66377, 66384, 66421, 66432, 66461, 66464, 66499, 66504, 66511, 66560, 66599,+ 66640, 66717, 66736, 66771, 66816, 66855, 66864, 66915, 66928, 66938, 66940, 66954,+ 66956, 66962, 66964, 66965, 67072, 67382, 67392, 67413, 67424, 67431, 67584, 67589,+ 67592, 67592, 67594, 67637, 67639, 67640, 67644, 67644, 67647, 67669, 67680, 67702,+ 67712, 67742, 67808, 67826, 67828, 67829, 67840, 67861, 67872, 67897, 67968, 68023,+ 68030, 68031, 68096, 68096, 68112, 68115, 68117, 68119, 68121, 68149, 68192, 68220,+ 68224, 68252, 68288, 68295, 68297, 68324, 68352, 68405, 68416, 68437, 68448, 68466,+ 68480, 68497, 68608, 68680, 68736, 68786, 68864, 68899, 69248, 69289, 69296, 69297,+ 69376, 69404, 69415, 69415, 69424, 69445, 69488, 69505, 69552, 69572, 69600, 69622,+ 69635, 69687, 69745, 69746, 69749, 69749, 69763, 69807, 69840, 69864, 69891, 69926,+ 69956, 69956, 69959, 69959, 69968, 70002, 70006, 70006, 70019, 70066, 70081, 70084,+ 70106, 70106, 70108, 70108, 70144, 70161, 70163, 70187, 70207, 70208, 70272, 70278,+ 70280, 70280, 70282, 70285, 70287, 70301, 70303, 70312, 70320, 70366, 70405, 70412,+ 70415, 70416, 70419, 70440, 70442, 70448, 70450, 70451, 70453, 70457, 70461, 70461,+ 70480, 70480, 70493, 70497, 70656, 70708, 70727, 70730, 70751, 70753, 70784, 70831,+ 70852, 70853, 70855, 70855, 71040, 71086, 71128, 71131, 71168, 71215, 71236, 71236,+ 71296, 71338, 71352, 71352, 71424, 71450, 71488, 71494, 71680, 71723, 71840, 71871,+ 71935, 71942, 71945, 71945, 71948, 71955, 71957, 71958, 71960, 71983, 71999, 71999,+ 72001, 72001, 72096, 72103, 72106, 72144, 72161, 72161, 72163, 72163, 72192, 72192,+ 72203, 72242, 72250, 72250, 72272, 72272, 72284, 72329, 72349, 72349, 72368, 72440,+ 72704, 72712, 72714, 72750, 72768, 72768, 72818, 72847, 72960, 72966, 72968, 72969,+ 72971, 73008, 73030, 73030, 73056, 73061, 73063, 73064, 73066, 73097, 73112, 73112,+ 73440, 73458, 73474, 73474, 73476, 73488, 73490, 73523, 73648, 73648, 73728, 74649,+ 74880, 75075, 77712, 77808, 77824, 78895, 78913, 78918, 82944, 83526, 92160, 92728,+ 92736, 92766, 92784, 92862, 92880, 92909, 92928, 92975, 93027, 93047, 93053, 93071,+ 93760, 93791, 93952, 94026, 94032, 94032, 94208, 100343, 100352, 101589, 101632,+ 101640, 110592, 110882, 110898, 110898, 110928, 110930, 110933, 110933, 110948,+ 110951, 110960, 111355, 113664, 113770, 113776, 113788, 113792, 113800, 113808,+ 113817, 119808, 119833, 119860, 119885, 119912, 119937, 119964, 119964, 119966,+ 119967, 119970, 119970, 119973, 119974, 119977, 119980, 119982, 119989, 120016,+ 120041, 120068, 120069, 120071, 120074, 120077, 120084, 120086, 120092, 120120,+ 120121, 120123, 120126, 120128, 120132, 120134, 120134, 120138, 120144, 120172,+ 120197, 120224, 120249, 120276, 120301, 120328, 120353, 120380, 120405, 120432,+ 120457, 120488, 120512, 120546, 120570, 120604, 120628, 120662, 120686, 120720,+ 120744, 120778, 120778, 122634, 122634, 123136, 123180, 123214, 123214, 123536,+ 123565, 123584, 123627, 124112, 124138, 124896, 124902, 124904, 124907, 124909,+ 124910, 124912, 124926, 124928, 125124, 125184, 125217, 126464, 126467, 126469,+ 126495, 126497, 126498, 126500, 126500, 126503, 126503, 126505, 126514, 126516,+ 126519, 126521, 126521, 126523, 126523, 126530, 126530, 126535, 126535, 126537,+ 126537, 126539, 126539, 126541, 126543, 126545, 126546, 126548, 126548, 126551,+ 126551, 126553, 126553, 126555, 126555, 126557, 126557, 126559, 126559, 126561,+ 126562, 126564, 126564, 126567, 126570, 126572, 126578, 126580, 126583, 126585,+ 126588, 126590, 126590, 126592, 126601, 126603, 126619, 126625, 126627, 126629,+ 126633, 126635, 126651, 131072, 173791, 173824, 177977, 177984, 178205, 178208,+ 183969, 183984, 191456, 194560, 195101, 196608, 201546, 201552, 205743+ };+ }/*** Python's {@code str.isprintable()}, which is what {@code repr} leaves* unescaped. Everything else it spells as a numeric escape.@@ -992,6 +1244,12 @@ public final class UnicodeTables {private static final long PRINTABLE_ASCII_LOW = -4294967296L;private static final long PRINTABLE_ASCII_HIGH = 9223372036854775807L;+ private static final long MARK_ASCII_LOW = 0L;+ private static final long MARK_ASCII_HIGH = 0L;++ private static final long INITIAL_ASCII_LOW = 0L;+ private static final long INITIAL_ASCII_HIGH = 134217726L;+/** Python's {@code str.isalpha()} for one code point. */public static boolean isAlpha(int codePoint) {@@ -1156,6 +1414,26 @@ public final class UnicodeTables {return contains(SPACE, codePoint);}+ /**+ * Whether this code point is a combining mark: the Mn, Mc and Me categories.+ *+ * <p>Not {@link #isCombining}, which is the canonical combining CLASS. They disagree on 1,528+ * code points that are category M with class zero, so the two are not interchangeable here.+ */+ public static boolean isMark(int codePoint) {+ return contains(MARK, codePoint);+ }++ /**+ * Whether a name may begin with this code point: the Lu, Lt and Lo categories.+ *+ * <p>Not {@link #isUpper}, which is Python's {@code str.isupper()} -- the Uppercase property,+ * which holds caseless symbols this does not and misses every caseless script this does.+ */+ public static boolean isInitial(int codePoint) {+ return contains(INITIAL, codePoint);+ }+/** Python's {@code str.strip()} with no argument. */public static String strip(String text) {int start = 0;diff --git a/laya-java/laya-java/src/test/java/com/convaiinnovations/laya/lang/UnicodeTablesTest.java b/laya-java/laya-java/src/test/java/com/convaiinnovations/laya/lang/UnicodeTablesTest.javaindex d07132a8..5e44f03a 100644--- a/laya-java/laya-java/src/test/java/com/convaiinnovations/laya/lang/UnicodeTablesTest.java+++ b/laya-java/laya-java/src/test/java/com/convaiinnovations/laya/lang/UnicodeTablesTest.java@@ -124,6 +124,54 @@ class UnicodeTablesTest {assertPredicate("space", UnicodeTables::isSpace);}+ @Test+ @DisplayName("isMark is the Mn/Mc/Me categories on every code point")+ void mark() {+ assertPredicate("mark", UnicodeTables::isMark);+ }++ @Test+ @DisplayName("isInitial is the Lu/Lt/Lo categories on every code point")+ void initial() {+ assertPredicate("initial", UnicodeTables::isInitial);+ }++ @Test+ @DisplayName("isMark is not the combining class, and isInitial is not str.isupper")+ void theTwoNewTablesAreNotTheOldOnes() {+ // Both were nearly implemented by reusing a table that already existed, and both reuses+ // would have been wrong on a large, specific set. Asserted so that a later simplification+ // has to argue with a number rather than with a comment.+ int markWithZeroCombiningClass = 0;+ int initialThatIsNotUpper = 0;+ int upperThatIsNotInitial = 0;+ for (int cp = 0; cp < 0x110000; cp++) {+ if (UnicodeTables.isMark(cp) && !UnicodeTables.isCombining(cp)) {+ markWithZeroCombiningClass++;+ }+ if (UnicodeTables.isInitial(cp) && !UnicodeTables.isUpper(cp)) {+ initialThatIsNotUpper++;+ }+ if (UnicodeTables.isUpper(cp) && !UnicodeTables.isInitial(cp)) {+ upperThatIsNotInitial++;+ }+ }+ assertEquals(1528, markWithZeroCombiningClass,+ "isCombining is the canonical combining CLASS; these are category M with class 0");+ assertEquals(131643, initialThatIsNotUpper,+ "every caseless script: str.isupper() is false for all of them");+ assertEquals(120, upperThatIsNotInitial,+ "the Uppercase property holds symbols that are not letters, such as U+24B6");+ // The three that make the difference observable in a sign-off's name.+ assertTrue(UnicodeTables.isInitial(0x01C5), "U+01C5 is Lt, a name may begin with it");+ assertFalse(UnicodeTables.isUpper(0x01C5), "and str.isupper() rejects it");+ assertTrue(UnicodeTables.isInitial(0x5C71), "U+5C71 is Lo, a name may begin with it");+ assertTrue(UnicodeTables.isUpper(0x24B6), "U+24B6 is Uppercase...");+ assertFalse(UnicodeTables.isInitial(0x24B6), "...but it is category So, not a letter");+ assertTrue(UnicodeTables.isMark(0x034F), "U+034F is Mn...");+ assertFalse(UnicodeTables.isCombining(0x034F), "...with a canonical combining class of 0");+ }+@Test@DisplayName("pythonLower maps every code point the way Python's str.lower does")void pythonLower() {diff --git a/laya-java/scripts/gen_fixtures.py b/laya-java/scripts/gen_fixtures.pyindex 629088f9..632dab71 100644--- a/laya-java/scripts/gen_fixtures.py+++ b/laya-java/scripts/gen_fixtures.py@@ -100,7 +100,7 @@ def _unicode_digests():A digest rather than a table: the committed Java tables already hold the ranges, and whatneeds proving is that they still agree with CPython over the WHOLE of Unicode and not just- over the corpus below. Nine hashes do that in 500 bytes of fixture, and they fail the moment+ over the corpus below. Eleven hashes do that in 700 bytes of fixture, and they fail the momenta JDK upgrade, a table edit or a CPython bump moves a single code point.Surrogates are included for the predicates -- Python reports every one of them false, and so@@ -118,6 +118,13 @@ def _unicode_digests():"upper": lambda ch: ch.isupper(),"lower": lambda ch: ch.islower(),"space": lambda ch: ch.isspace(),+ # `laya.email`'s two. `mark` is the Mn/Mc/Me categories, which the `combining` digest+ # above does NOT cover: that is the canonical combining class, and the two disagree on+ # 1,528 code points. `initial` is Lu/Lt/Lo, which `upper` does not cover either --+ # `str.isupper()` is the Uppercase property, and the two sets are not nested in either+ # direction.+ "mark": lambda ch: unicodedata.category(ch).startswith("M"),+ "initial": lambda ch: unicodedata.category(ch) in ("Lu", "Lt", "Lo"),}out = {}for name, predicate in predicates.items():diff --git a/laya-java/scripts/gen_unicode_tables.py b/laya-java/scripts/gen_unicode_tables.pyindex c5d896d2..603038e9 100755--- a/laya-java/scripts/gen_unicode_tables.py+++ b/laya-java/scripts/gen_unicode_tables.py@@ -161,6 +161,22 @@ def render():lower = ranges(lambda cp: chr(cp).islower())space = ranges(lambda cp: chr(cp).isspace())printable = ranges(lambda cp: chr(cp).isprintable())+ # `laya.email` needs two properties none of the tables above supply.+ #+ # MARK is the Mn/Mc/Me categories, which `_drop_marks` spells `unicodedata.category(ch)+ # .startswith("M")` and the TS port spells `\p{M}`. COMBINING is NOT a substitute: it is the+ # canonical combining CLASS, and 1,528 code points are category M with class zero --+ # U+034F COMBINING GRAPHEME JOINER, the Cyrillic Me numerals, the Thaana vowels. Using it+ # would leave those marks in the tail that `_is_english_signoff` matches structurally, so a+ # name carrying one would stop reading as a name.+ #+ # INITIAL is the Lu/Lt/Lo categories, the test `_is_english_signoff` applies to each token's+ # first letter. UPPER is not a substitute either: `str.isupper()` is the Uppercase property,+ # true of 1,951 code points against these 133,474, and the two sets are not nested -- it+ # holds circled and squared letters that are category So, and rejects every caseless script,+ # so `Obrigado, 山田` would stop being a sign-off and `Thanks, Ⓐ` would start being one.+ mark = ranges(lambda cp: unicodedata.category(chr(cp)).startswith("M"))+ initial = ranges(lambda cp: unicodedata.category(chr(cp)) in ("Lu", "Lt", "Lo"))# Every code point whose lowercase is more than one code point. In Unicode 15 there is# exactly one, and `laya/lang.py` depends on it: it replaces U+0130 before lowering, because@@ -241,6 +257,11 @@ public final class UnicodeTables {" * sorted. {@link #LOWER_TO} holds what each one maps to.")+ "\n"+ "\n"+ + table("MARK", mark,+ "The Mn/Mc/Me categories: what {@code laya.email} drops before matching a\n"+ "sign-off's tail, and what the other ports spell {@code \\p{M}}.")+ + table("INITIAL", initial,+ "The Lu/Lt/Lo categories: the letters a name may begin with, in any script.")+ table("PRINTABLE", printable,"Python's {@code str.isprintable()}, which is what {@code repr} leaves\n"" * unescaped. Everything else it spells as a numeric escape.")@@ -262,6 +283,8 @@ public final class UnicodeTables {("LOWER", lambda cp: chr(cp).islower()),("SPACE", lambda cp: chr(cp).isspace()),("PRINTABLE", lambda cp: chr(cp).isprintable()),+ ("MARK", lambda cp: unicodedata.category(chr(cp)).startswith("M")),+ ("INITIAL", lambda cp: unicodedata.category(chr(cp)) in ("Lu", "Lt", "Lo")),)))@@ -395,6 +418,26 @@ public final class UnicodeTables {return contains(SPACE, codePoint);}+ /**+ * Whether this code point is a combining mark: the Mn, Mc and Me categories.+ *+ * <p>Not {@link #isCombining}, which is the canonical combining CLASS. They disagree on 1,528+ * code points that are category M with class zero, so the two are not interchangeable here.+ */+ public static boolean isMark(int codePoint) {+ return contains(MARK, codePoint);+ }++ /**+ * Whether a name may begin with this code point: the Lu, Lt and Lo categories.+ *+ * <p>Not {@link #isUpper}, which is Python's {@code str.isupper()} -- the Uppercase property,+ * which holds caseless symbols this does not and misses every caseless script this does.+ */+ public static boolean isInitial(int codePoint) {+ return contains(INITIAL, codePoint);+ }+/** Python's {@code str.strip()} with no argument. */public static String strip(String text) {int start = 0;