Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

feat(go): IsMark and IsInitial in internal/unicodedata

Adeshina Lasisi849a46709ff2
diff --git a/laya-go/internal/unicodedata/tables_gen.go b/laya-go/internal/unicodedata/tables_gen.go
index f85eb26e..c7b02a4d 100644
--- a/laya-go/internal/unicodedata/tables_gen.go
+++ b/laya-go/internal/unicodedata/tables_gen.go
@@ -562,6 +562,217 @@ var printable = [...]rune{
var printableASCII = [2]uint64{0xffffffff00000000, 0x7fffffffffffffff}
+// The Mn, Mc and Me categories, which laya/email.py spells unicodedata.category(ch)
+// startswith("M").
+// 310 ranges covering 2450 code points.
+var mark = [...]rune{
+ 768, 879, 1155, 1161, 1425, 1469, 1471, 1471, 1473, 1474, 1476, 1477, 1479, 1479, 1552, 1562,
+ 1611, 1631, 1648, 1648, 1750, 1756, 1759, 1764, 1767, 1768, 1770, 1773, 1809, 1809, 1840, 1866,
+ 1958, 1968, 2027, 2035, 2045, 2045, 2070, 2073, 2075, 2083, 2085, 2087, 2089, 2093, 2137, 2139,
+ 2200, 2207, 2250, 2273, 2275, 2307, 2362, 2364, 2366, 2383, 2385, 2391, 2402, 2403, 2433, 2435,
+ 2492, 2492, 2494, 2500, 2503, 2504, 2507, 2509, 2519, 2519, 2530, 2531, 2558, 2558, 2561, 2563,
+ 2620, 2620, 2622, 2626, 2631, 2632, 2635, 2637, 2641, 2641, 2672, 2673, 2677, 2677, 2689, 2691,
+ 2748, 2748, 2750, 2757, 2759, 2761, 2763, 2765, 2786, 2787, 2810, 2815, 2817, 2819, 2876, 2876,
+ 2878, 2884, 2887, 2888, 2891, 2893, 2901, 2903, 2914, 2915, 2946, 2946, 3006, 3010, 3014, 3016,
+ 3018, 3021, 3031, 3031, 3072, 3076, 3132, 3132, 3134, 3140, 3142, 3144, 3146, 3149, 3157, 3158,
+ 3170, 3171, 3201, 3203, 3260, 3260, 3262, 3268, 3270, 3272, 3274, 3277, 3285, 3286, 3298, 3299,
+ 3315, 3315, 3328, 3331, 3387, 3388, 3390, 3396, 3398, 3400, 3402, 3405, 3415, 3415, 3426, 3427,
+ 3457, 3459, 3530, 3530, 3535, 3540, 3542, 3542, 3544, 3551, 3570, 3571, 3633, 3633, 3636, 3642,
+ 3655, 3662, 3761, 3761, 3764, 3772, 3784, 3790, 3864, 3865, 3893, 3893, 3895, 3895, 3897, 3897,
+ 3902, 3903, 3953, 3972, 3974, 3975, 3981, 3991, 3993, 4028, 4038, 4038, 4139, 4158, 4182, 4185,
+ 4190, 4192, 4194, 4196, 4199, 4205, 4209, 4212, 4226, 4237, 4239, 4239, 4250, 4253, 4957, 4959,
+ 5906, 5909, 5938, 5940, 5970, 5971, 6002, 6003, 6068, 6099, 6109, 6109, 6155, 6157, 6159, 6159,
+ 6277, 6278, 6313, 6313, 6432, 6443, 6448, 6459, 6679, 6683, 6741, 6750, 6752, 6780, 6783, 6783,
+ 6832, 6862, 6912, 6916, 6964, 6980, 7019, 7027, 7040, 7042, 7073, 7085, 7142, 7155, 7204, 7223,
+ 7376, 7378, 7380, 7400, 7405, 7405, 7412, 7412, 7415, 7417, 7616, 7679, 8400, 8432, 11503, 11505,
+ 11647, 11647, 11744, 11775, 12330, 12335, 12441, 12442, 42607, 42610, 42612, 42621, 42654, 42655,
+ 42736, 42737, 43010, 43010, 43014, 43014, 43019, 43019, 43043, 43047, 43052, 43052, 43136, 43137,
+ 43188, 43205, 43232, 43249, 43263, 43263, 43302, 43309, 43335, 43347, 43392, 43395, 43443, 43456,
+ 43493, 43493, 43561, 43574, 43587, 43587, 43596, 43597, 43643, 43645, 43696, 43696, 43698, 43700,
+ 43703, 43704, 43710, 43711, 43713, 43713, 43755, 43759, 43765, 43766, 44003, 44010, 44012, 44013,
+ 64286, 64286, 65024, 65039, 65056, 65071, 66045, 66045, 66272, 66272, 66422, 66426, 68097, 68099,
+ 68101, 68102, 68108, 68111, 68152, 68154, 68159, 68159, 68325, 68326, 68900, 68903, 69291, 69292,
+ 69373, 69375, 69446, 69456, 69506, 69509, 69632, 69634, 69688, 69702, 69744, 69744, 69747, 69748,
+ 69759, 69762, 69808, 69818, 69826, 69826, 69888, 69890, 69927, 69940, 69957, 69958, 70003, 70003,
+ 70016, 70018, 70067, 70080, 70089, 70092, 70094, 70095, 70188, 70199, 70206, 70206, 70209, 70209,
+ 70367, 70378, 70400, 70403, 70459, 70460, 70462, 70468, 70471, 70472, 70475, 70477, 70487, 70487,
+ 70498, 70499, 70502, 70508, 70512, 70516, 70709, 70726, 70750, 70750, 70832, 70851, 71087, 71093,
+ 71096, 71104, 71132, 71133, 71216, 71232, 71339, 71351, 71453, 71467, 71724, 71738, 71984, 71989,
+ 71991, 71992, 71995, 71998, 72000, 72000, 72002, 72003, 72145, 72151, 72154, 72160, 72164, 72164,
+ 72193, 72202, 72243, 72249, 72251, 72254, 72263, 72263, 72273, 72283, 72330, 72345, 72751, 72758,
+ 72760, 72767, 72850, 72871, 72873, 72886, 73009, 73014, 73018, 73018, 73020, 73021, 73023, 73029,
+ 73031, 73031, 73098, 73102, 73104, 73105, 73107, 73111, 73459, 73462, 73472, 73473, 73475, 73475,
+ 73524, 73530, 73534, 73538, 78912, 78912, 78919, 78933, 92912, 92916, 92976, 92982, 94031, 94031,
+ 94033, 94087, 94095, 94098, 94180, 94180, 94192, 94193, 113821, 113822, 118528, 118573, 118576,
+ 118598, 119141, 119145, 119149, 119154, 119163, 119170, 119173, 119179, 119210, 119213, 119362,
+ 119364, 121344, 121398, 121403, 121452, 121461, 121461, 121476, 121476, 121499, 121503, 121505,
+ 121519, 122880, 122886, 122888, 122904, 122907, 122913, 122915, 122916, 122918, 122922, 123023,
+ 123023, 123184, 123190, 123566, 123566, 123628, 123631, 124140, 124143, 125136, 125142, 125252,
+ 125258, 917760, 917999,
+}
+
+var markASCII = [2]uint64{0x0, 0x0}
+
+// The Lu, Lt and Lo categories: a letter that is not lowercase.
+// 1156 ranges covering 133474 code points.
+var initial = [...]rune{
+ 65, 90, 170, 170, 186, 186, 192, 214, 216, 222, 256, 256, 258, 258, 260, 260, 262, 262, 264, 264,
+ 266, 266, 268, 268, 270, 270, 272, 272, 274, 274, 276, 276, 278, 278, 280, 280, 282, 282, 284,
+ 284, 286, 286, 288, 288, 290, 290, 292, 292, 294, 294, 296, 296, 298, 298, 300, 300, 302, 302,
+ 304, 304, 306, 306, 308, 308, 310, 310, 313, 313, 315, 315, 317, 317, 319, 319, 321, 321, 323,
+ 323, 325, 325, 327, 327, 330, 330, 332, 332, 334, 334, 336, 336, 338, 338, 340, 340, 342, 342,
+ 344, 344, 346, 346, 348, 348, 350, 350, 352, 352, 354, 354, 356, 356, 358, 358, 360, 360, 362,
+ 362, 364, 364, 366, 366, 368, 368, 370, 370, 372, 372, 374, 374, 376, 377, 379, 379, 381, 381,
+ 385, 386, 388, 388, 390, 391, 393, 395, 398, 401, 403, 404, 406, 408, 412, 413, 415, 416, 418,
+ 418, 420, 420, 422, 423, 425, 425, 428, 428, 430, 431, 433, 435, 437, 437, 439, 440, 443, 444,
+ 448, 453, 455, 456, 458, 459, 461, 461, 463, 463, 465, 465, 467, 467, 469, 469, 471, 471, 473,
+ 473, 475, 475, 478, 478, 480, 480, 482, 482, 484, 484, 486, 486, 488, 488, 490, 490, 492, 492,
+ 494, 494, 497, 498, 500, 500, 502, 504, 506, 506, 508, 508, 510, 510, 512, 512, 514, 514, 516,
+ 516, 518, 518, 520, 520, 522, 522, 524, 524, 526, 526, 528, 528, 530, 530, 532, 532, 534, 534,
+ 536, 536, 538, 538, 540, 540, 542, 542, 544, 544, 546, 546, 548, 548, 550, 550, 552, 552, 554,
+ 554, 556, 556, 558, 558, 560, 560, 562, 562, 570, 571, 573, 574, 577, 577, 579, 582, 584, 584,
+ 586, 586, 588, 588, 590, 590, 660, 660, 880, 880, 882, 882, 886, 886, 895, 895, 902, 902, 904,
+ 906, 908, 908, 910, 911, 913, 929, 931, 939, 975, 975, 978, 980, 984, 984, 986, 986, 988, 988,
+ 990, 990, 992, 992, 994, 994, 996, 996, 998, 998, 1000, 1000, 1002, 1002, 1004, 1004, 1006, 1006,
+ 1012, 1012, 1015, 1015, 1017, 1018, 1021, 1071, 1120, 1120, 1122, 1122, 1124, 1124, 1126, 1126,
+ 1128, 1128, 1130, 1130, 1132, 1132, 1134, 1134, 1136, 1136, 1138, 1138, 1140, 1140, 1142, 1142,
+ 1144, 1144, 1146, 1146, 1148, 1148, 1150, 1150, 1152, 1152, 1162, 1162, 1164, 1164, 1166, 1166,
+ 1168, 1168, 1170, 1170, 1172, 1172, 1174, 1174, 1176, 1176, 1178, 1178, 1180, 1180, 1182, 1182,
+ 1184, 1184, 1186, 1186, 1188, 1188, 1190, 1190, 1192, 1192, 1194, 1194, 1196, 1196, 1198, 1198,
+ 1200, 1200, 1202, 1202, 1204, 1204, 1206, 1206, 1208, 1208, 1210, 1210, 1212, 1212, 1214, 1214,
+ 1216, 1217, 1219, 1219, 1221, 1221, 1223, 1223, 1225, 1225, 1227, 1227, 1229, 1229, 1232, 1232,
+ 1234, 1234, 1236, 1236, 1238, 1238, 1240, 1240, 1242, 1242, 1244, 1244, 1246, 1246, 1248, 1248,
+ 1250, 1250, 1252, 1252, 1254, 1254, 1256, 1256, 1258, 1258, 1260, 1260, 1262, 1262, 1264, 1264,
+ 1266, 1266, 1268, 1268, 1270, 1270, 1272, 1272, 1274, 1274, 1276, 1276, 1278, 1278, 1280, 1280,
+ 1282, 1282, 1284, 1284, 1286, 1286, 1288, 1288, 1290, 1290, 1292, 1292, 1294, 1294, 1296, 1296,
+ 1298, 1298, 1300, 1300, 1302, 1302, 1304, 1304, 1306, 1306, 1308, 1308, 1310, 1310, 1312, 1312,
+ 1314, 1314, 1316, 1316, 1318, 1318, 1320, 1320, 1322, 1322, 1324, 1324, 1326, 1326, 1329, 1366,
+ 1488, 1514, 1519, 1522, 1568, 1599, 1601, 1610, 1646, 1647, 1649, 1747, 1749, 1749, 1774, 1775,
+ 1786, 1788, 1791, 1791, 1808, 1808, 1810, 1839, 1869, 1957, 1969, 1969, 1994, 2026, 2048, 2069,
+ 2112, 2136, 2144, 2154, 2160, 2183, 2185, 2190, 2208, 2248, 2308, 2361, 2365, 2365, 2384, 2384,
+ 2392, 2401, 2418, 2432, 2437, 2444, 2447, 2448, 2451, 2472, 2474, 2480, 2482, 2482, 2486, 2489,
+ 2493, 2493, 2510, 2510, 2524, 2525, 2527, 2529, 2544, 2545, 2556, 2556, 2565, 2570, 2575, 2576,
+ 2579, 2600, 2602, 2608, 2610, 2611, 2613, 2614, 2616, 2617, 2649, 2652, 2654, 2654, 2674, 2676,
+ 2693, 2701, 2703, 2705, 2707, 2728, 2730, 2736, 2738, 2739, 2741, 2745, 2749, 2749, 2768, 2768,
+ 2784, 2785, 2809, 2809, 2821, 2828, 2831, 2832, 2835, 2856, 2858, 2864, 2866, 2867, 2869, 2873,
+ 2877, 2877, 2908, 2909, 2911, 2913, 2929, 2929, 2947, 2947, 2949, 2954, 2958, 2960, 2962, 2965,
+ 2969, 2970, 2972, 2972, 2974, 2975, 2979, 2980, 2984, 2986, 2990, 3001, 3024, 3024, 3077, 3084,
+ 3086, 3088, 3090, 3112, 3114, 3129, 3133, 3133, 3160, 3162, 3165, 3165, 3168, 3169, 3200, 3200,
+ 3205, 3212, 3214, 3216, 3218, 3240, 3242, 3251, 3253, 3257, 3261, 3261, 3293, 3294, 3296, 3297,
+ 3313, 3314, 3332, 3340, 3342, 3344, 3346, 3386, 3389, 3389, 3406, 3406, 3412, 3414, 3423, 3425,
+ 3450, 3455, 3461, 3478, 3482, 3505, 3507, 3515, 3517, 3517, 3520, 3526, 3585, 3632, 3634, 3635,
+ 3648, 3653, 3713, 3714, 3716, 3716, 3718, 3722, 3724, 3747, 3749, 3749, 3751, 3760, 3762, 3763,
+ 3773, 3773, 3776, 3780, 3804, 3807, 3840, 3840, 3904, 3911, 3913, 3948, 3976, 3980, 4096, 4138,
+ 4159, 4159, 4176, 4181, 4186, 4189, 4193, 4193, 4197, 4198, 4206, 4208, 4213, 4225, 4238, 4238,
+ 4256, 4293, 4295, 4295, 4301, 4301, 4352, 4680, 4682, 4685, 4688, 4694, 4696, 4696, 4698, 4701,
+ 4704, 4744, 4746, 4749, 4752, 4784, 4786, 4789, 4792, 4798, 4800, 4800, 4802, 4805, 4808, 4822,
+ 4824, 4880, 4882, 4885, 4888, 4954, 4992, 5007, 5024, 5109, 5121, 5740, 5743, 5759, 5761, 5786,
+ 5792, 5866, 5873, 5880, 5888, 5905, 5919, 5937, 5952, 5969, 5984, 5996, 5998, 6000, 6016, 6067,
+ 6108, 6108, 6176, 6210, 6212, 6264, 6272, 6276, 6279, 6312, 6314, 6314, 6320, 6389, 6400, 6430,
+ 6480, 6509, 6512, 6516, 6528, 6571, 6576, 6601, 6656, 6678, 6688, 6740, 6917, 6963, 6981, 6988,
+ 7043, 7072, 7086, 7087, 7098, 7141, 7168, 7203, 7245, 7247, 7258, 7287, 7312, 7354, 7357, 7359,
+ 7401, 7404, 7406, 7411, 7413, 7414, 7418, 7418, 7680, 7680, 7682, 7682, 7684, 7684, 7686, 7686,
+ 7688, 7688, 7690, 7690, 7692, 7692, 7694, 7694, 7696, 7696, 7698, 7698, 7700, 7700, 7702, 7702,
+ 7704, 7704, 7706, 7706, 7708, 7708, 7710, 7710, 7712, 7712, 7714, 7714, 7716, 7716, 7718, 7718,
+ 7720, 7720, 7722, 7722, 7724, 7724, 7726, 7726, 7728, 7728, 7730, 7730, 7732, 7732, 7734, 7734,
+ 7736, 7736, 7738, 7738, 7740, 7740, 7742, 7742, 7744, 7744, 7746, 7746, 7748, 7748, 7750, 7750,
+ 7752, 7752, 7754, 7754, 7756, 7756, 7758, 7758, 7760, 7760, 7762, 7762, 7764, 7764, 7766, 7766,
+ 7768, 7768, 7770, 7770, 7772, 7772, 7774, 7774, 7776, 7776, 7778, 7778, 7780, 7780, 7782, 7782,
+ 7784, 7784, 7786, 7786, 7788, 7788, 7790, 7790, 7792, 7792, 7794, 7794, 7796, 7796, 7798, 7798,
+ 7800, 7800, 7802, 7802, 7804, 7804, 7806, 7806, 7808, 7808, 7810, 7810, 7812, 7812, 7814, 7814,
+ 7816, 7816, 7818, 7818, 7820, 7820, 7822, 7822, 7824, 7824, 7826, 7826, 7828, 7828, 7838, 7838,
+ 7840, 7840, 7842, 7842, 7844, 7844, 7846, 7846, 7848, 7848, 7850, 7850, 7852, 7852, 7854, 7854,
+ 7856, 7856, 7858, 7858, 7860, 7860, 7862, 7862, 7864, 7864, 7866, 7866, 7868, 7868, 7870, 7870,
+ 7872, 7872, 7874, 7874, 7876, 7876, 7878, 7878, 7880, 7880, 7882, 7882, 7884, 7884, 7886, 7886,
+ 7888, 7888, 7890, 7890, 7892, 7892, 7894, 7894, 7896, 7896, 7898, 7898, 7900, 7900, 7902, 7902,
+ 7904, 7904, 7906, 7906, 7908, 7908, 7910, 7910, 7912, 7912, 7914, 7914, 7916, 7916, 7918, 7918,
+ 7920, 7920, 7922, 7922, 7924, 7924, 7926, 7926, 7928, 7928, 7930, 7930, 7932, 7932, 7934, 7934,
+ 7944, 7951, 7960, 7965, 7976, 7983, 7992, 7999, 8008, 8013, 8025, 8025, 8027, 8027, 8029, 8029,
+ 8031, 8031, 8040, 8047, 8072, 8079, 8088, 8095, 8104, 8111, 8120, 8124, 8136, 8140, 8152, 8155,
+ 8168, 8172, 8184, 8188, 8450, 8450, 8455, 8455, 8459, 8461, 8464, 8466, 8469, 8469, 8473, 8477,
+ 8484, 8484, 8486, 8486, 8488, 8488, 8490, 8493, 8496, 8499, 8501, 8504, 8510, 8511, 8517, 8517,
+ 8579, 8579, 11264, 11311, 11360, 11360, 11362, 11364, 11367, 11367, 11369, 11369, 11371, 11371,
+ 11373, 11376, 11378, 11378, 11381, 11381, 11390, 11392, 11394, 11394, 11396, 11396, 11398, 11398,
+ 11400, 11400, 11402, 11402, 11404, 11404, 11406, 11406, 11408, 11408, 11410, 11410, 11412, 11412,
+ 11414, 11414, 11416, 11416, 11418, 11418, 11420, 11420, 11422, 11422, 11424, 11424, 11426, 11426,
+ 11428, 11428, 11430, 11430, 11432, 11432, 11434, 11434, 11436, 11436, 11438, 11438, 11440, 11440,
+ 11442, 11442, 11444, 11444, 11446, 11446, 11448, 11448, 11450, 11450, 11452, 11452, 11454, 11454,
+ 11456, 11456, 11458, 11458, 11460, 11460, 11462, 11462, 11464, 11464, 11466, 11466, 11468, 11468,
+ 11470, 11470, 11472, 11472, 11474, 11474, 11476, 11476, 11478, 11478, 11480, 11480, 11482, 11482,
+ 11484, 11484, 11486, 11486, 11488, 11488, 11490, 11490, 11499, 11499, 11501, 11501, 11506, 11506,
+ 11568, 11623, 11648, 11670, 11680, 11686, 11688, 11694, 11696, 11702, 11704, 11710, 11712, 11718,
+ 11720, 11726, 11728, 11734, 11736, 11742, 12294, 12294, 12348, 12348, 12353, 12438, 12447, 12447,
+ 12449, 12538, 12543, 12543, 12549, 12591, 12593, 12686, 12704, 12735, 12784, 12799, 13312, 19903,
+ 19968, 40980, 40982, 42124, 42192, 42231, 42240, 42507, 42512, 42527, 42538, 42539, 42560, 42560,
+ 42562, 42562, 42564, 42564, 42566, 42566, 42568, 42568, 42570, 42570, 42572, 42572, 42574, 42574,
+ 42576, 42576, 42578, 42578, 42580, 42580, 42582, 42582, 42584, 42584, 42586, 42586, 42588, 42588,
+ 42590, 42590, 42592, 42592, 42594, 42594, 42596, 42596, 42598, 42598, 42600, 42600, 42602, 42602,
+ 42604, 42604, 42606, 42606, 42624, 42624, 42626, 42626, 42628, 42628, 42630, 42630, 42632, 42632,
+ 42634, 42634, 42636, 42636, 42638, 42638, 42640, 42640, 42642, 42642, 42644, 42644, 42646, 42646,
+ 42648, 42648, 42650, 42650, 42656, 42725, 42786, 42786, 42788, 42788, 42790, 42790, 42792, 42792,
+ 42794, 42794, 42796, 42796, 42798, 42798, 42802, 42802, 42804, 42804, 42806, 42806, 42808, 42808,
+ 42810, 42810, 42812, 42812, 42814, 42814, 42816, 42816, 42818, 42818, 42820, 42820, 42822, 42822,
+ 42824, 42824, 42826, 42826, 42828, 42828, 42830, 42830, 42832, 42832, 42834, 42834, 42836, 42836,
+ 42838, 42838, 42840, 42840, 42842, 42842, 42844, 42844, 42846, 42846, 42848, 42848, 42850, 42850,
+ 42852, 42852, 42854, 42854, 42856, 42856, 42858, 42858, 42860, 42860, 42862, 42862, 42873, 42873,
+ 42875, 42875, 42877, 42878, 42880, 42880, 42882, 42882, 42884, 42884, 42886, 42886, 42891, 42891,
+ 42893, 42893, 42895, 42896, 42898, 42898, 42902, 42902, 42904, 42904, 42906, 42906, 42908, 42908,
+ 42910, 42910, 42912, 42912, 42914, 42914, 42916, 42916, 42918, 42918, 42920, 42920, 42922, 42926,
+ 42928, 42932, 42934, 42934, 42936, 42936, 42938, 42938, 42940, 42940, 42942, 42942, 42944, 42944,
+ 42946, 42946, 42948, 42951, 42953, 42953, 42960, 42960, 42966, 42966, 42968, 42968, 42997, 42997,
+ 42999, 42999, 43003, 43009, 43011, 43013, 43015, 43018, 43020, 43042, 43072, 43123, 43138, 43187,
+ 43250, 43255, 43259, 43259, 43261, 43262, 43274, 43301, 43312, 43334, 43360, 43388, 43396, 43442,
+ 43488, 43492, 43495, 43503, 43514, 43518, 43520, 43560, 43584, 43586, 43588, 43595, 43616, 43631,
+ 43633, 43638, 43642, 43642, 43646, 43695, 43697, 43697, 43701, 43702, 43705, 43709, 43712, 43712,
+ 43714, 43714, 43739, 43740, 43744, 43754, 43762, 43762, 43777, 43782, 43785, 43790, 43793, 43798,
+ 43808, 43814, 43816, 43822, 43968, 44002, 44032, 55203, 55216, 55238, 55243, 55291, 63744, 64109,
+ 64112, 64217, 64285, 64285, 64287, 64296, 64298, 64310, 64312, 64316, 64318, 64318, 64320, 64321,
+ 64323, 64324, 64326, 64433, 64467, 64829, 64848, 64911, 64914, 64967, 65008, 65019, 65136, 65140,
+ 65142, 65276, 65313, 65338, 65382, 65391, 65393, 65437, 65440, 65470, 65474, 65479, 65482, 65487,
+ 65490, 65495, 65498, 65500, 65536, 65547, 65549, 65574, 65576, 65594, 65596, 65597, 65599, 65613,
+ 65616, 65629, 65664, 65786, 66176, 66204, 66208, 66256, 66304, 66335, 66349, 66368, 66370, 66377,
+ 66384, 66421, 66432, 66461, 66464, 66499, 66504, 66511, 66560, 66599, 66640, 66717, 66736, 66771,
+ 66816, 66855, 66864, 66915, 66928, 66938, 66940, 66954, 66956, 66962, 66964, 66965, 67072, 67382,
+ 67392, 67413, 67424, 67431, 67584, 67589, 67592, 67592, 67594, 67637, 67639, 67640, 67644, 67644,
+ 67647, 67669, 67680, 67702, 67712, 67742, 67808, 67826, 67828, 67829, 67840, 67861, 67872, 67897,
+ 67968, 68023, 68030, 68031, 68096, 68096, 68112, 68115, 68117, 68119, 68121, 68149, 68192, 68220,
+ 68224, 68252, 68288, 68295, 68297, 68324, 68352, 68405, 68416, 68437, 68448, 68466, 68480, 68497,
+ 68608, 68680, 68736, 68786, 68864, 68899, 69248, 69289, 69296, 69297, 69376, 69404, 69415, 69415,
+ 69424, 69445, 69488, 69505, 69552, 69572, 69600, 69622, 69635, 69687, 69745, 69746, 69749, 69749,
+ 69763, 69807, 69840, 69864, 69891, 69926, 69956, 69956, 69959, 69959, 69968, 70002, 70006, 70006,
+ 70019, 70066, 70081, 70084, 70106, 70106, 70108, 70108, 70144, 70161, 70163, 70187, 70207, 70208,
+ 70272, 70278, 70280, 70280, 70282, 70285, 70287, 70301, 70303, 70312, 70320, 70366, 70405, 70412,
+ 70415, 70416, 70419, 70440, 70442, 70448, 70450, 70451, 70453, 70457, 70461, 70461, 70480, 70480,
+ 70493, 70497, 70656, 70708, 70727, 70730, 70751, 70753, 70784, 70831, 70852, 70853, 70855, 70855,
+ 71040, 71086, 71128, 71131, 71168, 71215, 71236, 71236, 71296, 71338, 71352, 71352, 71424, 71450,
+ 71488, 71494, 71680, 71723, 71840, 71871, 71935, 71942, 71945, 71945, 71948, 71955, 71957, 71958,
+ 71960, 71983, 71999, 71999, 72001, 72001, 72096, 72103, 72106, 72144, 72161, 72161, 72163, 72163,
+ 72192, 72192, 72203, 72242, 72250, 72250, 72272, 72272, 72284, 72329, 72349, 72349, 72368, 72440,
+ 72704, 72712, 72714, 72750, 72768, 72768, 72818, 72847, 72960, 72966, 72968, 72969, 72971, 73008,
+ 73030, 73030, 73056, 73061, 73063, 73064, 73066, 73097, 73112, 73112, 73440, 73458, 73474, 73474,
+ 73476, 73488, 73490, 73523, 73648, 73648, 73728, 74649, 74880, 75075, 77712, 77808, 77824, 78895,
+ 78913, 78918, 82944, 83526, 92160, 92728, 92736, 92766, 92784, 92862, 92880, 92909, 92928, 92975,
+ 93027, 93047, 93053, 93071, 93760, 93791, 93952, 94026, 94032, 94032, 94208, 100343, 100352,
+ 101589, 101632, 101640, 110592, 110882, 110898, 110898, 110928, 110930, 110933, 110933, 110948,
+ 110951, 110960, 111355, 113664, 113770, 113776, 113788, 113792, 113800, 113808, 113817, 119808,
+ 119833, 119860, 119885, 119912, 119937, 119964, 119964, 119966, 119967, 119970, 119970, 119973,
+ 119974, 119977, 119980, 119982, 119989, 120016, 120041, 120068, 120069, 120071, 120074, 120077,
+ 120084, 120086, 120092, 120120, 120121, 120123, 120126, 120128, 120132, 120134, 120134, 120138,
+ 120144, 120172, 120197, 120224, 120249, 120276, 120301, 120328, 120353, 120380, 120405, 120432,
+ 120457, 120488, 120512, 120546, 120570, 120604, 120628, 120662, 120686, 120720, 120744, 120778,
+ 120778, 122634, 122634, 123136, 123180, 123214, 123214, 123536, 123565, 123584, 123627, 124112,
+ 124138, 124896, 124902, 124904, 124907, 124909, 124910, 124912, 124926, 124928, 125124, 125184,
+ 125217, 126464, 126467, 126469, 126495, 126497, 126498, 126500, 126500, 126503, 126503, 126505,
+ 126514, 126516, 126519, 126521, 126521, 126523, 126523, 126530, 126530, 126535, 126535, 126537,
+ 126537, 126539, 126539, 126541, 126543, 126545, 126546, 126548, 126548, 126551, 126551, 126553,
+ 126553, 126555, 126555, 126557, 126557, 126559, 126559, 126561, 126562, 126564, 126564, 126567,
+ 126570, 126572, 126578, 126580, 126583, 126585, 126588, 126590, 126590, 126592, 126601, 126603,
+ 126619, 126625, 126627, 126629, 126633, 126635, 126651, 131072, 173791, 173824, 177977, 177984,
+ 178205, 178208, 183969, 183984, 191456, 194560, 195101, 196608, 201546, 201552, 205743,
+}
+
+var initialASCII = [2]uint64{0x0, 0x7fffffe}
+
// The Lt category: cased, but neither upper nor lower, so str.isupper() on a
// string holding one is false.
// 10 ranges covering 31 code points.
diff --git a/laya-go/internal/unicodedata/unicodedata.go b/laya-go/internal/unicodedata/unicodedata.go
index 2487743c..f59cbcc1 100644
--- a/laya-go/internal/unicodedata/unicodedata.go
+++ b/laya-go/internal/unicodedata/unicodedata.go
@@ -59,6 +59,12 @@ func IsLower(r rune) bool { return lookup(&lowerASCII, lower[:], r) }
// IsSpace is Python's str.isspace() for one character.
func IsSpace(r rune) bool { return lookup(&spaceASCII, space[:], r) }
+// IsMark is Python's unicodedata.category(ch).startswith("M"): the Mn, Mc and Me categories.
+func IsMark(r rune) bool { return lookup(&markASCII, mark[:], r) }
+
+// IsInitial reports category Lu, Lt or Lo: a letter that is not lowercase.
+func IsInitial(r rune) bool { return lookup(&initialASCII, initial[:], r) }
+
// IsPrintable is Python's str.isprintable() for one character.
func IsPrintable(r rune) bool { return lookup(&printableASCII, printable[:], r) }
diff --git a/laya-go/internal/unicodedata/unicodedata_test.go b/laya-go/internal/unicodedata/unicodedata_test.go
index acfcb19f..0d1c9b61 100644
--- a/laya-go/internal/unicodedata/unicodedata_test.go
+++ b/laya-go/internal/unicodedata/unicodedata_test.go
@@ -50,6 +50,8 @@ func TestPredicatesMatchCPythonEverywhere(t *testing.T) {
"upper": IsUpper,
"lower": IsLower,
"space": IsSpace,
+ "mark": IsMark,
+ "initial": IsInitial,
} {
t.Run(name, func(t *testing.T) {
want := fixtures.Str(t, d.Must(name))
diff --git a/laya-go/scripts/gen_tables.py b/laya-go/scripts/gen_tables.py
index 2e0508a5..9e814f07 100644
--- a/laya-go/scripts/gen_tables.py
+++ b/laya-go/scripts/gen_tables.py
@@ -102,6 +102,10 @@ def render_unicode():
("space", "Python's str.isspace().", lambda cp: chr(cp).isspace()),
("printable", "Python's str.isprintable(), which is what repr leaves unescaped.",
lambda cp: chr(cp).isprintable()),
+ ("mark", "The Mn, Mc and Me categories, which laya/email.py spells unicodedata.category(ch)\n"
+ "// startswith(\"M\").", lambda cp: unicodedata.category(chr(cp)).startswith("M")),
+ ("initial", "The Lu, Lt and Lo categories: a letter that is not lowercase.",
+ lambda cp: unicodedata.category(chr(cp)) in ("Lu", "Lt", "Lo")),
("title", "The Lt category: cased, but neither upper nor lower, so str.isupper() on a\n"
"// string holding one is false.", lambda cp: unicodedata.category(chr(cp)) == "Lt"),
]