Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
feat(java): CPython's repr and %.0f, which the router's reasons are built from
Adeshina Lasisid1ccb913d0cf
diff --git a/laya-java/fixtures/python_json.json b/laya-java/fixtures/python_json.jsonindex 0a56ff8f..3bad4084 100644--- a/laya-java/fixtures/python_json.json+++ b/laya-java/fixtures/python_json.json@@ -9380,5 +9380,266 @@"r4": 0.3457}]- }+ },+ "repr_strings": [+ [+ "",+ "''"+ ],+ [+ "english",+ "'english'"+ ],+ [+ "multilingual",+ "'multilingual'"+ ],+ [+ "pt",+ "'pt'"+ ],+ [+ "typed_decisions",+ "'typed_decisions'"+ ],+ [+ "it's here",+ "\"it's here\""+ ],+ [+ "say \"hi\"",+ "'say \"hi\"'"+ ],+ [+ "both ' and \"",+ "'both \\' and \"'"+ ],+ [+ "'",+ "\"'\""+ ],+ [+ "\"",+ "'\"'"+ ],+ [+ "'\"",+ "'\\'\"'"+ ],+ [+ "line\nbreak",+ "'line\\nbreak'"+ ],+ [+ "tab\there",+ "'tab\\there'"+ ],+ [+ "carriage\rreturn",+ "'carriage\\rreturn'"+ ],+ [+ "back\\slash",+ "'back\\\\slash'"+ ],+ [+ "bell\u0007",+ "'bell\\x07'"+ ],+ [+ "null\u0000",+ "'null\\x00'"+ ],+ [+ "café",+ "'café'"+ ],+ [+ "中文",+ "'中文'"+ ],+ [+ "😀",+ "'😀'"+ ],+ [+ "𑼀",+ "'𑼀'"+ ],+ [+ "a b",+ "'a\\xa0b'"+ ],+ [+ "ab",+ "'a\\xadb'"+ ],+ [+ "ab",+ "'a\\u200bb'"+ ],+ [+ "a b",+ "'a\\u2028b'"+ ],+ [+ "a b",+ "'a\\x85b'"+ ],+ [+ "ab",+ "'a\\ufeffb'"+ ],+ [+ "ab",+ "'a\\u202eb'"+ ],+ [+ "ab",+ "'a\\ue000b'"+ ],+ [+ "ab",+ "'a\\u0378b'"+ ],+ [+ "ab",+ "'a\\x7fb'"+ ],+ [+ "áb",+ "'áb'"+ ],+ [+ "Nao consigo entrar na minha conta e a senha nao funciona",+ "'Nao consigo entrar na minha conta e a senha nao funciona'"+ ],+ [+ "Ich kann mich nicht in mein Konto einloggen und das Passwort",+ "'Ich kann mich nicht in mein Konto einloggen und das Passwort'"+ ],+ [+ "MON LA EST COM DES",+ "'MON LA EST COM DES'"+ ],+ [+ "mixed and and 'quotes' in one line",+ "\"mixed \\xa0 and \\u200b and 'quotes' in one line\""+ ]+ ],+ "repr_digest": "68e2a834d7a1633529456b61b01649ed5440a7cb387297afc3f0aa721cfac0d9",+ "percent0": [+ [+ 0.0,+ "0"+ ],+ [+ 1.0,+ "100"+ ],+ [+ 0.005,+ "0"+ ],+ [+ 0.015,+ "2"+ ],+ [+ 0.025,+ "2"+ ],+ [+ 0.045,+ "4"+ ],+ [+ 0.125,+ "12"+ ],+ [+ 0.135,+ "14"+ ],+ [+ 0.205,+ "20"+ ],+ [+ 0.5,+ "50"+ ],+ [+ 0.625,+ "62"+ ],+ [+ 0.0312,+ "3"+ ],+ [+ 0.0313,+ "3"+ ],+ [+ 0.1,+ "10"+ ],+ [+ 0.2222,+ "22"+ ],+ [+ 0.2941,+ "29"+ ],+ [+ 0.3333,+ "33"+ ],+ [+ 0.4,+ "40"+ ],+ [+ 0.6667,+ "67"+ ],+ [+ 0.7778,+ "78"+ ],+ [+ 0.9999,+ "100"+ ],+ [+ 0.3333333333333333,+ "33"+ ],+ [+ 0.6666666666666666,+ "67"+ ],+ [+ 0.125,+ "12"+ ],+ [+ 0.375,+ "38"+ ],+ [+ 0.625,+ "62"+ ],+ [+ 0.875,+ "88"+ ],+ [+ 0.0625,+ "6"+ ]+ ]}diff --git a/laya-java/laya-java/src/main/java/com/convaiinnovations/laya/json/PythonJson.java b/laya-java/laya-java/src/main/java/com/convaiinnovations/laya/json/PythonJson.javaindex d2dfd9b2..35f3acc4 100644--- a/laya-java/laya-java/src/main/java/com/convaiinnovations/laya/json/PythonJson.java+++ b/laya-java/laya-java/src/main/java/com/convaiinnovations/laya/json/PythonJson.java@@ -1,5 +1,6 @@package com.convaiinnovations.laya.json;+import com.convaiinnovations.laya.lang.UnicodeTables;import java.math.BigInteger;import java.util.List;import java.util.Locale;@@ -188,6 +189,83 @@ public final class PythonJson {* JSON, and is deliberately what CPython does by default, so a state carrying one produces the* same tokens here as there rather than a different error.*/+ /**+ * CPython's {@code repr} for a string, which is what a {@code %r} in a message interpolates.+ *+ * <p>Needed because the router's reason strings are built with {@code %r} and one of them+ * interpolates the mixed segment -- a slice of the caller's own text. So this sees arbitrary+ * input, not a short language code, and every rule below is reachable from a pasted ticket.+ *+ * <p>The rules, in CPython's order:+ *+ * <ul>+ * <li>The quote is a single quote, unless the string holds a single quote and no double+ * quote, in which case the whole thing is double-quoted and nothing needs escaping.+ * A string holding both is single-quoted with its single quotes escaped.+ * <li>A backslash and the chosen quote are backslash-escaped; newline, carriage return and+ * tab get their letter escapes.+ * <li>Anything {@link UnicodeTables#isPrintable} calls unprintable becomes a numeric escape+ * -- backslash-x and two hex digits below U+0100, backslash-u and four below U+10000,+ * backslash-U and eight above it -- in LOWERCASE hex. (Spelled out rather than shown:+ * a backslash-u sequence in a Java comment is translated before the file is lexed, so+ * writing the escape here would stop this file compiling.)+ * <li>Everything else passes through as itself, including every non-ASCII printable+ * character: CPython 3 does not escape those.+ * </ul>+ *+ * <p>That last pair is where an ASCII-only escape check goes wrong, and it is not a corner:+ * U+00A0 NO-BREAK SPACE is unprintable to CPython and arrives in pasted text constantly, so+ * stopping at U+007F would write a raw control character into an API response.+ */+ public static String repr(String value) {+ if (value == null) {+ return "None";+ }+ char quote = value.indexOf('\'') >= 0 && value.indexOf('"') < 0 ? '"' : '\'';+ StringBuilder out = new StringBuilder(value.length() + 2);+ out.append(quote);+ int i = 0;+ while (i < value.length()) {+ int cp = value.codePointAt(i);+ i += Character.charCount(cp);+ if (cp == quote || cp == '\\') {+ out.append('\\').appendCodePoint(cp);+ } else if (cp == '\n') {+ out.append("\\n");+ } else if (cp == '\r') {+ out.append("\\r");+ } else if (cp == '\t') {+ out.append("\\t");+ } else if (UnicodeTables.isPrintable(cp)) {+ out.appendCodePoint(cp);+ } else if (cp < 0x100) {+ out.append(String.format(Locale.ROOT, "\\x%02x", cp));+ } else if (cp < 0x10000) {+ out.append(String.format(Locale.ROOT, "\\u%04x", cp));+ } else {+ out.append(String.format(Locale.ROOT, "\\U%08x", cp));+ }+ }+ return out.append(quote).toString();+ }++ /**+ * CPython's {@code %.0f}: the nearest integer, halves to EVEN.+ *+ * <p>{@code String.format("%.0f", v)} rounds halves UP, so the two disagree on every halfway+ * value -- {@code 12.5} is {@code "12"} in CPython and {@code "13"} in Java. The router+ * reports a percentage of letters with this, and a share of one in eight letters is exactly+ * 12.5, so the disagreement is reachable rather than theoretical.+ */+ public static String percent0(double fraction) {+ double scaled = 100.0 * fraction;+ if (Double.isNaN(scaled) || Double.isInfinite(scaled)) {+ return repr(scaled);+ }+ // Math.rint is IEEE ties-to-even, which is the rule; a long keeps a share above 2^31.+ return Long.toString((long) Math.rint(scaled));+ }+public static String repr(double value) {if (Double.isNaN(value)) {return "NaN";diff --git a/laya-java/laya-java/src/main/java/com/convaiinnovations/laya/lang/UnicodeTables.java b/laya-java/laya-java/src/main/java/com/convaiinnovations/laya/lang/UnicodeTables.javaindex 5733c926..6f1c2b70 100644--- a/laya-java/laya-java/src/main/java/com/convaiinnovations/laya/lang/UnicodeTables.java+++ b/laya-java/laya-java/src/main/java/com/convaiinnovations/laya/lang/UnicodeTables.java@@ -710,6 +710,139 @@ public final class UnicodeTables {};}++ /**+ * Python's {@code str.isprintable()}, which is what {@code repr} leaves+ * unescaped. Everything else it spells as a numeric escape.+ *+ * <p>711 ranges covering 148998 code points, recorded+ * from CPython 15.0.0.+ */+ private static final int[] PRINTABLE = printableTable();++ private static int[] printableTable() {+ return new int[] {+ 32, 126, 161, 172, 174, 887, 890, 895, 900, 906, 908, 908, 910, 929, 931, 1327,+ 1329, 1366, 1369, 1418, 1421, 1423, 1425, 1479, 1488, 1514, 1519, 1524, 1542, 1563,+ 1565, 1756, 1758, 1805, 1808, 1866, 1869, 1969, 1984, 2042, 2045, 2093, 2096, 2110,+ 2112, 2139, 2142, 2142, 2144, 2154, 2160, 2190, 2200, 2273, 2275, 2435, 2437, 2444,+ 2447, 2448, 2451, 2472, 2474, 2480, 2482, 2482, 2486, 2489, 2492, 2500, 2503, 2504,+ 2507, 2510, 2519, 2519, 2524, 2525, 2527, 2531, 2534, 2558, 2561, 2563, 2565, 2570,+ 2575, 2576, 2579, 2600, 2602, 2608, 2610, 2611, 2613, 2614, 2616, 2617, 2620, 2620,+ 2622, 2626, 2631, 2632, 2635, 2637, 2641, 2641, 2649, 2652, 2654, 2654, 2662, 2678,+ 2689, 2691, 2693, 2701, 2703, 2705, 2707, 2728, 2730, 2736, 2738, 2739, 2741, 2745,+ 2748, 2757, 2759, 2761, 2763, 2765, 2768, 2768, 2784, 2787, 2790, 2801, 2809, 2815,+ 2817, 2819, 2821, 2828, 2831, 2832, 2835, 2856, 2858, 2864, 2866, 2867, 2869, 2873,+ 2876, 2884, 2887, 2888, 2891, 2893, 2901, 2903, 2908, 2909, 2911, 2915, 2918, 2935,+ 2946, 2947, 2949, 2954, 2958, 2960, 2962, 2965, 2969, 2970, 2972, 2972, 2974, 2975,+ 2979, 2980, 2984, 2986, 2990, 3001, 3006, 3010, 3014, 3016, 3018, 3021, 3024, 3024,+ 3031, 3031, 3046, 3066, 3072, 3084, 3086, 3088, 3090, 3112, 3114, 3129, 3132, 3140,+ 3142, 3144, 3146, 3149, 3157, 3158, 3160, 3162, 3165, 3165, 3168, 3171, 3174, 3183,+ 3191, 3212, 3214, 3216, 3218, 3240, 3242, 3251, 3253, 3257, 3260, 3268, 3270, 3272,+ 3274, 3277, 3285, 3286, 3293, 3294, 3296, 3299, 3302, 3311, 3313, 3315, 3328, 3340,+ 3342, 3344, 3346, 3396, 3398, 3400, 3402, 3407, 3412, 3427, 3430, 3455, 3457, 3459,+ 3461, 3478, 3482, 3505, 3507, 3515, 3517, 3517, 3520, 3526, 3530, 3530, 3535, 3540,+ 3542, 3542, 3544, 3551, 3558, 3567, 3570, 3572, 3585, 3642, 3647, 3675, 3713, 3714,+ 3716, 3716, 3718, 3722, 3724, 3747, 3749, 3749, 3751, 3773, 3776, 3780, 3782, 3782,+ 3784, 3790, 3792, 3801, 3804, 3807, 3840, 3911, 3913, 3948, 3953, 3991, 3993, 4028,+ 4030, 4044, 4046, 4058, 4096, 4293, 4295, 4295, 4301, 4301, 4304, 4680, 4682, 4685,+ 4688, 4694, 4696, 4696, 4698, 4701, 4704, 4744, 4746, 4749, 4752, 4784, 4786, 4789,+ 4792, 4798, 4800, 4800, 4802, 4805, 4808, 4822, 4824, 4880, 4882, 4885, 4888, 4954,+ 4957, 4988, 4992, 5017, 5024, 5109, 5112, 5117, 5120, 5759, 5761, 5788, 5792, 5880,+ 5888, 5909, 5919, 5942, 5952, 5971, 5984, 5996, 5998, 6000, 6002, 6003, 6016, 6109,+ 6112, 6121, 6128, 6137, 6144, 6157, 6159, 6169, 6176, 6264, 6272, 6314, 6320, 6389,+ 6400, 6430, 6432, 6443, 6448, 6459, 6464, 6464, 6468, 6509, 6512, 6516, 6528, 6571,+ 6576, 6601, 6608, 6618, 6622, 6683, 6686, 6750, 6752, 6780, 6783, 6793, 6800, 6809,+ 6816, 6829, 6832, 6862, 6912, 6988, 6992, 7038, 7040, 7155, 7164, 7223, 7227, 7241,+ 7245, 7304, 7312, 7354, 7357, 7367, 7376, 7418, 7424, 7957, 7960, 7965, 7968, 8005,+ 8008, 8013, 8016, 8023, 8025, 8025, 8027, 8027, 8029, 8029, 8031, 8061, 8064, 8116,+ 8118, 8132, 8134, 8147, 8150, 8155, 8157, 8175, 8178, 8180, 8182, 8190, 8208, 8231,+ 8240, 8286, 8304, 8305, 8308, 8334, 8336, 8348, 8352, 8384, 8400, 8432, 8448, 8587,+ 8592, 9254, 9280, 9290, 9312, 11123, 11126, 11157, 11159, 11507, 11513, 11557,+ 11559, 11559, 11565, 11565, 11568, 11623, 11631, 11632, 11647, 11670, 11680, 11686,+ 11688, 11694, 11696, 11702, 11704, 11710, 11712, 11718, 11720, 11726, 11728, 11734,+ 11736, 11742, 11744, 11869, 11904, 11929, 11931, 12019, 12032, 12245, 12272, 12283,+ 12289, 12351, 12353, 12438, 12441, 12543, 12549, 12591, 12593, 12686, 12688, 12771,+ 12784, 12830, 12832, 42124, 42128, 42182, 42192, 42539, 42560, 42743, 42752, 42954,+ 42960, 42961, 42963, 42963, 42965, 42969, 42994, 43052, 43056, 43065, 43072, 43127,+ 43136, 43205, 43214, 43225, 43232, 43347, 43359, 43388, 43392, 43469, 43471, 43481,+ 43486, 43518, 43520, 43574, 43584, 43597, 43600, 43609, 43612, 43714, 43739, 43766,+ 43777, 43782, 43785, 43790, 43793, 43798, 43808, 43814, 43816, 43822, 43824, 43883,+ 43888, 44013, 44016, 44025, 44032, 55203, 55216, 55238, 55243, 55291, 63744, 64109,+ 64112, 64217, 64256, 64262, 64275, 64279, 64285, 64310, 64312, 64316, 64318, 64318,+ 64320, 64321, 64323, 64324, 64326, 64450, 64467, 64911, 64914, 64967, 64975, 64975,+ 65008, 65049, 65056, 65106, 65108, 65126, 65128, 65131, 65136, 65140, 65142, 65276,+ 65281, 65470, 65474, 65479, 65482, 65487, 65490, 65495, 65498, 65500, 65504, 65510,+ 65512, 65518, 65532, 65533, 65536, 65547, 65549, 65574, 65576, 65594, 65596, 65597,+ 65599, 65613, 65616, 65629, 65664, 65786, 65792, 65794, 65799, 65843, 65847, 65934,+ 65936, 65948, 65952, 65952, 66000, 66045, 66176, 66204, 66208, 66256, 66272, 66299,+ 66304, 66339, 66349, 66378, 66384, 66426, 66432, 66461, 66463, 66499, 66504, 66517,+ 66560, 66717, 66720, 66729, 66736, 66771, 66776, 66811, 66816, 66855, 66864, 66915,+ 66927, 66938, 66940, 66954, 66956, 66962, 66964, 66965, 66967, 66977, 66979, 66993,+ 66995, 67001, 67003, 67004, 67072, 67382, 67392, 67413, 67424, 67431, 67456, 67461,+ 67463, 67504, 67506, 67514, 67584, 67589, 67592, 67592, 67594, 67637, 67639, 67640,+ 67644, 67644, 67647, 67669, 67671, 67742, 67751, 67759, 67808, 67826, 67828, 67829,+ 67835, 67867, 67871, 67897, 67903, 67903, 67968, 68023, 68028, 68047, 68050, 68099,+ 68101, 68102, 68108, 68115, 68117, 68119, 68121, 68149, 68152, 68154, 68159, 68168,+ 68176, 68184, 68192, 68255, 68288, 68326, 68331, 68342, 68352, 68405, 68409, 68437,+ 68440, 68466, 68472, 68497, 68505, 68508, 68521, 68527, 68608, 68680, 68736, 68786,+ 68800, 68850, 68858, 68903, 68912, 68921, 69216, 69246, 69248, 69289, 69291, 69293,+ 69296, 69297, 69373, 69415, 69424, 69465, 69488, 69513, 69552, 69579, 69600, 69622,+ 69632, 69709, 69714, 69749, 69759, 69820, 69822, 69826, 69840, 69864, 69872, 69881,+ 69888, 69940, 69942, 69959, 69968, 70006, 70016, 70111, 70113, 70132, 70144, 70161,+ 70163, 70209, 70272, 70278, 70280, 70280, 70282, 70285, 70287, 70301, 70303, 70313,+ 70320, 70378, 70384, 70393, 70400, 70403, 70405, 70412, 70415, 70416, 70419, 70440,+ 70442, 70448, 70450, 70451, 70453, 70457, 70459, 70468, 70471, 70472, 70475, 70477,+ 70480, 70480, 70487, 70487, 70493, 70499, 70502, 70508, 70512, 70516, 70656, 70747,+ 70749, 70753, 70784, 70855, 70864, 70873, 71040, 71093, 71096, 71133, 71168, 71236,+ 71248, 71257, 71264, 71276, 71296, 71353, 71360, 71369, 71424, 71450, 71453, 71467,+ 71472, 71494, 71680, 71739, 71840, 71922, 71935, 71942, 71945, 71945, 71948, 71955,+ 71957, 71958, 71960, 71989, 71991, 71992, 71995, 72006, 72016, 72025, 72096, 72103,+ 72106, 72151, 72154, 72164, 72192, 72263, 72272, 72354, 72368, 72440, 72448, 72457,+ 72704, 72712, 72714, 72758, 72760, 72773, 72784, 72812, 72816, 72847, 72850, 72871,+ 72873, 72886, 72960, 72966, 72968, 72969, 72971, 73014, 73018, 73018, 73020, 73021,+ 73023, 73031, 73040, 73049, 73056, 73061, 73063, 73064, 73066, 73102, 73104, 73105,+ 73107, 73112, 73120, 73129, 73440, 73464, 73472, 73488, 73490, 73530, 73534, 73561,+ 73648, 73648, 73664, 73713, 73727, 74649, 74752, 74862, 74864, 74868, 74880, 75075,+ 77712, 77810, 77824, 78895, 78912, 78933, 82944, 83526, 92160, 92728, 92736, 92766,+ 92768, 92777, 92782, 92862, 92864, 92873, 92880, 92909, 92912, 92917, 92928, 92997,+ 93008, 93017, 93019, 93025, 93027, 93047, 93053, 93071, 93760, 93850, 93952, 94026,+ 94031, 94087, 94095, 94111, 94176, 94180, 94192, 94193, 94208, 100343, 100352,+ 101589, 101632, 101640, 110576, 110579, 110581, 110587, 110589, 110590, 110592,+ 110882, 110898, 110898, 110928, 110930, 110933, 110933, 110948, 110951, 110960,+ 111355, 113664, 113770, 113776, 113788, 113792, 113800, 113808, 113817, 113820,+ 113823, 118528, 118573, 118576, 118598, 118608, 118723, 118784, 119029, 119040,+ 119078, 119081, 119154, 119163, 119274, 119296, 119365, 119488, 119507, 119520,+ 119539, 119552, 119638, 119648, 119672, 119808, 119892, 119894, 119964, 119966,+ 119967, 119970, 119970, 119973, 119974, 119977, 119980, 119982, 119993, 119995,+ 119995, 119997, 120003, 120005, 120069, 120071, 120074, 120077, 120084, 120086,+ 120092, 120094, 120121, 120123, 120126, 120128, 120132, 120134, 120134, 120138,+ 120144, 120146, 120485, 120488, 120779, 120782, 121483, 121499, 121503, 121505,+ 121519, 122624, 122654, 122661, 122666, 122880, 122886, 122888, 122904, 122907,+ 122913, 122915, 122916, 122918, 122922, 122928, 122989, 123023, 123023, 123136,+ 123180, 123184, 123197, 123200, 123209, 123214, 123215, 123536, 123566, 123584,+ 123641, 123647, 123647, 124112, 124153, 124896, 124902, 124904, 124907, 124909,+ 124910, 124912, 124926, 124928, 125124, 125127, 125142, 125184, 125259, 125264,+ 125273, 125278, 125279, 126065, 126132, 126209, 126269, 126464, 126467, 126469,+ 126495, 126497, 126498, 126500, 126500, 126503, 126503, 126505, 126514, 126516,+ 126519, 126521, 126521, 126523, 126523, 126530, 126530, 126535, 126535, 126537,+ 126537, 126539, 126539, 126541, 126543, 126545, 126546, 126548, 126548, 126551,+ 126551, 126553, 126553, 126555, 126555, 126557, 126557, 126559, 126559, 126561,+ 126562, 126564, 126564, 126567, 126570, 126572, 126578, 126580, 126583, 126585,+ 126588, 126590, 126590, 126592, 126601, 126603, 126619, 126625, 126627, 126629,+ 126633, 126635, 126651, 126704, 126705, 126976, 127019, 127024, 127123, 127136,+ 127150, 127153, 127167, 127169, 127183, 127185, 127221, 127232, 127405, 127462,+ 127490, 127504, 127547, 127552, 127560, 127568, 127569, 127584, 127589, 127744,+ 128727, 128732, 128748, 128752, 128764, 128768, 128886, 128891, 128985, 128992,+ 129003, 129008, 129008, 129024, 129035, 129040, 129095, 129104, 129113, 129120,+ 129159, 129168, 129197, 129200, 129201, 129280, 129619, 129632, 129645, 129648,+ 129660, 129664, 129672, 129680, 129725, 129727, 129733, 129742, 129755, 129760,+ 129768, 129776, 129784, 129792, 129938, 129940, 129994, 130032, 130041, 131072,+ 173791, 173824, 177977, 177984, 178205, 178208, 183969, 183984, 191456, 194560,+ 195101, 196608, 201546, 201552, 205743, 917760, 917999+ };+ }+/*** The lowercase of each entry of {@link #LOWER_FROM}, in the same order.*@@ -856,6 +989,9 @@ public final class UnicodeTables {private static final long SPACE_ASCII_LOW = 8321515008L;private static final long SPACE_ASCII_HIGH = 0L;+ private static final long PRINTABLE_ASCII_LOW = -4294967296L;+ private static final long PRINTABLE_ASCII_HIGH = 9223372036854775807L;+/** Python's {@code str.isalpha()} for one code point. */public static boolean isAlpha(int codePoint) {@@ -982,6 +1118,25 @@ public final class UnicodeTables {return false;}+ /**+ * Python's {@code str.isprintable()} for one code point.+ *+ * <p>What {@code repr} leaves alone. Everything else it spells as a numeric escape, and that+ * includes code points no ASCII-only check catches: U+00A0 NO-BREAK SPACE, U+00AD SOFT+ * HYPHEN, U+200B ZERO WIDTH SPACE, U+FEFF, and the private-use and unassigned planes. A+ * pasted ticket is full of the first one, and it reaches a route's reason string through the+ * mixed segment, so an escape check that stopped at U+007F would put a raw control character+ * into an API response.+ */+ public static boolean isPrintable(int codePoint) {+ if (codePoint < 0x80) {+ long mask = codePoint < 64 ? PRINTABLE_ASCII_LOW >>> codePoint+ : PRINTABLE_ASCII_HIGH >>> (codePoint - 64);+ return (mask & 1L) != 0L;+ }+ return contains(PRINTABLE, codePoint);+ }+/*** Python's {@code str.isspace()} for one code point.*diff --git a/laya-java/laya-java/src/test/java/com/convaiinnovations/laya/json/PythonJsonTest.java b/laya-java/laya-java/src/test/java/com/convaiinnovations/laya/json/PythonJsonTest.javaindex f49f547f..bd251ed3 100644--- a/laya-java/laya-java/src/test/java/com/convaiinnovations/laya/json/PythonJsonTest.java+++ b/laya-java/laya-java/src/test/java/com/convaiinnovations/laya/json/PythonJsonTest.java@@ -1,8 +1,12 @@package com.convaiinnovations.laya.json;import static org.junit.jupiter.api.Assertions.assertEquals;+import static org.junit.jupiter.api.Assertions.assertTrue;import com.convaiinnovations.laya.Fixtures;+import java.nio.charset.StandardCharsets;+import java.security.MessageDigest;+import java.security.NoSuchAlgorithmException;import java.util.List;import java.util.Map;import org.junit.jupiter.api.DisplayName;@@ -122,6 +126,92 @@ final class PythonJsonTest {() -> PythonJson.dumps(map));}+ @Test+ @DisplayName("repr spells every recorded string the way CPython does")+ void reprMatchesCpython() {+ @SuppressWarnings("unchecked")+ List<Object> cases = (List<Object>) Fixtures.load("python_json.json").get("repr_strings");+ int escaped = 0;+ int doubleQuoted = 0;+ for (Object entry : cases) {+ @SuppressWarnings("unchecked")+ List<Object> pair = (List<Object>) entry;+ String input = (String) pair.get(0);+ String expected = (String) pair.get(1);+ assertEquals(expected, PythonJson.repr(input),+ "repr of " + expected + " (" + input.length() + " chars)");+ if (expected.indexOf('\\') >= 0) {+ escaped++;+ }+ if (expected.startsWith("\"")) {+ doubleQuoted++;+ }+ }+ assertTrue(cases.size() >= 30, "only " + cases.size() + " repr cases");+ assertTrue(escaped >= 15, "only " + escaped + " cases actually escape something");+ assertTrue(doubleQuoted >= 1, "no case exercises the double-quoted form, which is the"+ + " branch for a string holding a single quote and no double quote");+ }++ @Test+ @DisplayName("repr escapes every unprintable code point in Unicode, and no printable one")+ void reprDigestMatchesCpython() {+ // The listed cases cover the rules a reader can name; this covers the boundary nobody+ // enumerated. 148,998 code points are printable and the rest are not, and an ASCII-only+ // escape check -- which is what the .NET port does -- passes every named case above while+ // leaving U+00A0 and U+200B raw in an API response.+ MessageDigest digest;+ try {+ digest = MessageDigest.getInstance("SHA-256");+ } catch (NoSuchAlgorithmException impossible) {+ throw new IllegalStateException("SHA-256 is required of every JVM", impossible);+ }+ for (int cp = 0; cp < 0x110000; cp++) {+ if (cp >= 0xD800 && cp <= 0xDFFF) {+ continue;+ }+ String value = "a" + new String(Character.toChars(cp)) + "b";+ digest.update(PythonJson.repr(value).getBytes(StandardCharsets.UTF_8));+ }+ StringBuilder hex = new StringBuilder(64);+ for (byte value : digest.digest()) {+ hex.append(Character.forDigit((value >> 4) & 0xF, 16));+ hex.append(Character.forDigit(value & 0xF, 16));+ }+ assertEquals(Fixtures.load("python_json.json").get("repr_digest"), hex.toString(),+ "repr disagrees with CPython somewhere in Unicode");+ }++ @Test+ @DisplayName("percent0 rounds halves to even, as CPython's %.0f does")+ void percent0MatchesCpython() {+ @SuppressWarnings("unchecked")+ List<Object> cases = (List<Object>) Fixtures.load("python_json.json").get("percent0");+ int distinguishing = 0;+ for (Object entry : cases) {+ @SuppressWarnings("unchecked")+ List<Object> pair = (List<Object>) entry;+ double fraction = ((Number) pair.get(0)).doubleValue();+ String expected = (String) pair.get(1);+ assertEquals(expected, PythonJson.percent0(fraction), "percent0 of " + fraction);+ // Java's own formatter rounds halves up, so count how many of these would be wrong+ // with it -- a count of zero would mean this test proves nothing.+ if (!expected.equals(String.format(java.util.Locale.ROOT, "%.0f", 100.0 * fraction))) {+ distinguishing++;+ }+ }+ assertTrue(cases.size() >= 25, "only " + cases.size() + " percent cases");+ assertTrue(distinguishing >= 3,+ "only " + distinguishing + " of these differ from String.format(\"%.0f\"), so"+ + " this test would not notice half-up rounding");+ }++ @Test+ @DisplayName("repr of null is None, as a %r of None would print")+ void reprOfNull() {+ assertEquals("None", PythonJson.repr((String) null));+ }+@Test@DisplayName("the same container appearing twice is NOT a cycle")void repeatedContainerIsNotACycle() {diff --git a/laya-java/scripts/gen_unicode_tables.py b/laya-java/scripts/gen_unicode_tables.pyindex 84dc6a04..c5d896d2 100755--- a/laya-java/scripts/gen_unicode_tables.py+++ b/laya-java/scripts/gen_unicode_tables.py@@ -160,6 +160,7 @@ def render():upper = ranges(lambda cp: chr(cp).isupper())lower = ranges(lambda cp: chr(cp).islower())space = ranges(lambda cp: chr(cp).isspace())+ printable = ranges(lambda cp: chr(cp).isprintable())# Every code point whose lowercase is more than one code point. In Unicode 15 there is# exactly one, and `laya/lang.py` depends on it: it replaces U+0130 before lowering, because@@ -239,6 +240,11 @@ public final class UnicodeTables {"Every code point whose Python lowercase is one DIFFERENT code point,\n"" * sorted. {@link #LOWER_TO} holds what each one maps to.")+ "\n"+ + "\n"+ + table("PRINTABLE", printable,+ "Python's {@code str.isprintable()}, which is what {@code repr} leaves\n"+ " * unescaped. Everything else it spells as a numeric escape.")+ + "\n"+ flat_table("LOWER_TO", [low for _, low in simple],"The lowercase of each entry of {@link #LOWER_FROM}, in the same order.")+ "\n"@@ -255,6 +261,7 @@ public final class UnicodeTables {("UPPER", lambda cp: chr(cp).isupper()),("LOWER", lambda cp: chr(cp).islower()),("SPACE", lambda cp: chr(cp).isspace()),+ ("PRINTABLE", lambda cp: chr(cp).isprintable()),)))@@ -360,6 +367,20 @@ public final class UnicodeTables {return false;}+ /**+ * Python's {@code str.isprintable()} for one code point.+ *+ * <p>What {@code repr} leaves alone. Everything else it spells as a numeric escape, and that+ * includes code points no ASCII-only check catches: U+00A0 NO-BREAK SPACE, U+00AD SOFT+ * HYPHEN, U+200B ZERO WIDTH SPACE, U+FEFF, and the private-use and unassigned planes. A+ * pasted ticket is full of the first one, and it reaches a route's reason string through the+ * mixed segment, so an escape check that stopped at U+007F would put a raw control character+ * into an API response.+ */+ public static boolean isPrintable(int codePoint) {+ return contains(PRINTABLE, codePoint);+ }+/*** Python's {@code str.isspace()} for one code point.*@@ -527,7 +548,8 @@ public final class UnicodeTables {# Splice the ASCII fast path into each accessor. Done here rather than in the template so the# template stays readable, and asserted so a renamed accessor cannot silently lose its fast# path and quietly become ten times slower on ordinary Latin text.- for name in ("ALPHA", "WORD", "COMBINING", "DIGIT", "UPPER", "LOWER", "SPACE"):+ for name in ("ALPHA", "WORD", "COMBINING", "DIGIT", "UPPER", "LOWER", "SPACE",+ "PRINTABLE"):marker = " return contains(%s, codePoint);\n" % nameif methods.count(marker) != 1:raise SystemExit(