Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

feat(java): CPython's repr and %.0f, which the router's reasons are built from

Adeshina Lasisid1ccb913d0cf
diff --git a/laya-java/fixtures/python_json.json b/laya-java/fixtures/python_json.json
index 0a56ff8f..3bad4084 100644
--- a/laya-java/fixtures/python_json.json
+++ b/laya-java/fixtures/python_json.json
@@ -9380,5 +9380,266 @@
"r4": 0.3457
}
]
- }
+ },
+ "repr_strings": [
+ [
+ "",
+ "''"
+ ],
+ [
+ "english",
+ "'english'"
+ ],
+ [
+ "multilingual",
+ "'multilingual'"
+ ],
+ [
+ "pt",
+ "'pt'"
+ ],
+ [
+ "typed_decisions",
+ "'typed_decisions'"
+ ],
+ [
+ "it's here",
+ "\"it's here\""
+ ],
+ [
+ "say \"hi\"",
+ "'say \"hi\"'"
+ ],
+ [
+ "both ' and \"",
+ "'both \\' and \"'"
+ ],
+ [
+ "'",
+ "\"'\""
+ ],
+ [
+ "\"",
+ "'\"'"
+ ],
+ [
+ "'\"",
+ "'\\'\"'"
+ ],
+ [
+ "line\nbreak",
+ "'line\\nbreak'"
+ ],
+ [
+ "tab\there",
+ "'tab\\there'"
+ ],
+ [
+ "carriage\rreturn",
+ "'carriage\\rreturn'"
+ ],
+ [
+ "back\\slash",
+ "'back\\\\slash'"
+ ],
+ [
+ "bell\u0007",
+ "'bell\\x07'"
+ ],
+ [
+ "null\u0000",
+ "'null\\x00'"
+ ],
+ [
+ "café",
+ "'café'"
+ ],
+ [
+ "中文",
+ "'中文'"
+ ],
+ [
+ "😀",
+ "'😀'"
+ ],
+ [
+ "𑼀",
+ "'𑼀'"
+ ],
+ [
+ "a b",
+ "'a\\xa0b'"
+ ],
+ [
+ "a­b",
+ "'a\\xadb'"
+ ],
+ [
+ "a​b",
+ "'a\\u200bb'"
+ ],
+ [
+ "a
b",
+ "'a\\u2028b'"
+ ],
+ [
+ "a…b",
+ "'a\\x85b'"
+ ],
+ [
+ "ab",
+ "'a\\ufeffb'"
+ ],
+ [
+ "a‮b",
+ "'a\\u202eb'"
+ ],
+ [
+ "ab",
+ "'a\\ue000b'"
+ ],
+ [
+ "a͸b",
+ "'a\\u0378b'"
+ ],
+ [
+ "ab",
+ "'a\\x7fb'"
+ ],
+ [
+ "áb",
+ "'áb'"
+ ],
+ [
+ "Nao consigo entrar na minha conta e a senha nao funciona",
+ "'Nao consigo entrar na minha conta e a senha nao funciona'"
+ ],
+ [
+ "Ich kann mich nicht in mein Konto einloggen und das Passwort",
+ "'Ich kann mich nicht in mein Konto einloggen und das Passwort'"
+ ],
+ [
+ "MON LA EST COM DES",
+ "'MON LA EST COM DES'"
+ ],
+ [
+ "mixed   and ​ and 'quotes' in one line",
+ "\"mixed \\xa0 and \\u200b and 'quotes' in one line\""
+ ]
+ ],
+ "repr_digest": "68e2a834d7a1633529456b61b01649ed5440a7cb387297afc3f0aa721cfac0d9",
+ "percent0": [
+ [
+ 0.0,
+ "0"
+ ],
+ [
+ 1.0,
+ "100"
+ ],
+ [
+ 0.005,
+ "0"
+ ],
+ [
+ 0.015,
+ "2"
+ ],
+ [
+ 0.025,
+ "2"
+ ],
+ [
+ 0.045,
+ "4"
+ ],
+ [
+ 0.125,
+ "12"
+ ],
+ [
+ 0.135,
+ "14"
+ ],
+ [
+ 0.205,
+ "20"
+ ],
+ [
+ 0.5,
+ "50"
+ ],
+ [
+ 0.625,
+ "62"
+ ],
+ [
+ 0.0312,
+ "3"
+ ],
+ [
+ 0.0313,
+ "3"
+ ],
+ [
+ 0.1,
+ "10"
+ ],
+ [
+ 0.2222,
+ "22"
+ ],
+ [
+ 0.2941,
+ "29"
+ ],
+ [
+ 0.3333,
+ "33"
+ ],
+ [
+ 0.4,
+ "40"
+ ],
+ [
+ 0.6667,
+ "67"
+ ],
+ [
+ 0.7778,
+ "78"
+ ],
+ [
+ 0.9999,
+ "100"
+ ],
+ [
+ 0.3333333333333333,
+ "33"
+ ],
+ [
+ 0.6666666666666666,
+ "67"
+ ],
+ [
+ 0.125,
+ "12"
+ ],
+ [
+ 0.375,
+ "38"
+ ],
+ [
+ 0.625,
+ "62"
+ ],
+ [
+ 0.875,
+ "88"
+ ],
+ [
+ 0.0625,
+ "6"
+ ]
+ ]
}
diff --git a/laya-java/laya-java/src/main/java/com/convaiinnovations/laya/json/PythonJson.java b/laya-java/laya-java/src/main/java/com/convaiinnovations/laya/json/PythonJson.java
index d2dfd9b2..35f3acc4 100644
--- a/laya-java/laya-java/src/main/java/com/convaiinnovations/laya/json/PythonJson.java
+++ b/laya-java/laya-java/src/main/java/com/convaiinnovations/laya/json/PythonJson.java
@@ -1,5 +1,6 @@
package com.convaiinnovations.laya.json;
+import com.convaiinnovations.laya.lang.UnicodeTables;
import java.math.BigInteger;
import java.util.List;
import java.util.Locale;
@@ -188,6 +189,83 @@ public final class PythonJson {
* JSON, and is deliberately what CPython does by default, so a state carrying one produces the
* same tokens here as there rather than a different error.
*/
+ /**
+ * CPython's {@code repr} for a string, which is what a {@code %r} in a message interpolates.
+ *
+ * <p>Needed because the router's reason strings are built with {@code %r} and one of them
+ * interpolates the mixed segment -- a slice of the caller's own text. So this sees arbitrary
+ * input, not a short language code, and every rule below is reachable from a pasted ticket.
+ *
+ * <p>The rules, in CPython's order:
+ *
+ * <ul>
+ * <li>The quote is a single quote, unless the string holds a single quote and no double
+ * quote, in which case the whole thing is double-quoted and nothing needs escaping.
+ * A string holding both is single-quoted with its single quotes escaped.
+ * <li>A backslash and the chosen quote are backslash-escaped; newline, carriage return and
+ * tab get their letter escapes.
+ * <li>Anything {@link UnicodeTables#isPrintable} calls unprintable becomes a numeric escape
+ * -- backslash-x and two hex digits below U+0100, backslash-u and four below U+10000,
+ * backslash-U and eight above it -- in LOWERCASE hex. (Spelled out rather than shown:
+ * a backslash-u sequence in a Java comment is translated before the file is lexed, so
+ * writing the escape here would stop this file compiling.)
+ * <li>Everything else passes through as itself, including every non-ASCII printable
+ * character: CPython 3 does not escape those.
+ * </ul>
+ *
+ * <p>That last pair is where an ASCII-only escape check goes wrong, and it is not a corner:
+ * U+00A0 NO-BREAK SPACE is unprintable to CPython and arrives in pasted text constantly, so
+ * stopping at U+007F would write a raw control character into an API response.
+ */
+ public static String repr(String value) {
+ if (value == null) {
+ return "None";
+ }
+ char quote = value.indexOf('\'') >= 0 && value.indexOf('"') < 0 ? '"' : '\'';
+ StringBuilder out = new StringBuilder(value.length() + 2);
+ out.append(quote);
+ int i = 0;
+ while (i < value.length()) {
+ int cp = value.codePointAt(i);
+ i += Character.charCount(cp);
+ if (cp == quote || cp == '\\') {
+ out.append('\\').appendCodePoint(cp);
+ } else if (cp == '\n') {
+ out.append("\\n");
+ } else if (cp == '\r') {
+ out.append("\\r");
+ } else if (cp == '\t') {
+ out.append("\\t");
+ } else if (UnicodeTables.isPrintable(cp)) {
+ out.appendCodePoint(cp);
+ } else if (cp < 0x100) {
+ out.append(String.format(Locale.ROOT, "\\x%02x", cp));
+ } else if (cp < 0x10000) {
+ out.append(String.format(Locale.ROOT, "\\u%04x", cp));
+ } else {
+ out.append(String.format(Locale.ROOT, "\\U%08x", cp));
+ }
+ }
+ return out.append(quote).toString();
+ }
+
+ /**
+ * CPython's {@code %.0f}: the nearest integer, halves to EVEN.
+ *
+ * <p>{@code String.format("%.0f", v)} rounds halves UP, so the two disagree on every halfway
+ * value -- {@code 12.5} is {@code "12"} in CPython and {@code "13"} in Java. The router
+ * reports a percentage of letters with this, and a share of one in eight letters is exactly
+ * 12.5, so the disagreement is reachable rather than theoretical.
+ */
+ public static String percent0(double fraction) {
+ double scaled = 100.0 * fraction;
+ if (Double.isNaN(scaled) || Double.isInfinite(scaled)) {
+ return repr(scaled);
+ }
+ // Math.rint is IEEE ties-to-even, which is the rule; a long keeps a share above 2^31.
+ return Long.toString((long) Math.rint(scaled));
+ }
+
public static String repr(double value) {
if (Double.isNaN(value)) {
return "NaN";
diff --git a/laya-java/laya-java/src/main/java/com/convaiinnovations/laya/lang/UnicodeTables.java b/laya-java/laya-java/src/main/java/com/convaiinnovations/laya/lang/UnicodeTables.java
index 5733c926..6f1c2b70 100644
--- a/laya-java/laya-java/src/main/java/com/convaiinnovations/laya/lang/UnicodeTables.java
+++ b/laya-java/laya-java/src/main/java/com/convaiinnovations/laya/lang/UnicodeTables.java
@@ -710,6 +710,139 @@ public final class UnicodeTables {
};
}
+
+ /**
+ * Python's {@code str.isprintable()}, which is what {@code repr} leaves
+ * unescaped. Everything else it spells as a numeric escape.
+ *
+ * <p>711 ranges covering 148998 code points, recorded
+ * from CPython 15.0.0.
+ */
+ private static final int[] PRINTABLE = printableTable();
+
+ private static int[] printableTable() {
+ return new int[] {
+ 32, 126, 161, 172, 174, 887, 890, 895, 900, 906, 908, 908, 910, 929, 931, 1327,
+ 1329, 1366, 1369, 1418, 1421, 1423, 1425, 1479, 1488, 1514, 1519, 1524, 1542, 1563,
+ 1565, 1756, 1758, 1805, 1808, 1866, 1869, 1969, 1984, 2042, 2045, 2093, 2096, 2110,
+ 2112, 2139, 2142, 2142, 2144, 2154, 2160, 2190, 2200, 2273, 2275, 2435, 2437, 2444,
+ 2447, 2448, 2451, 2472, 2474, 2480, 2482, 2482, 2486, 2489, 2492, 2500, 2503, 2504,
+ 2507, 2510, 2519, 2519, 2524, 2525, 2527, 2531, 2534, 2558, 2561, 2563, 2565, 2570,
+ 2575, 2576, 2579, 2600, 2602, 2608, 2610, 2611, 2613, 2614, 2616, 2617, 2620, 2620,
+ 2622, 2626, 2631, 2632, 2635, 2637, 2641, 2641, 2649, 2652, 2654, 2654, 2662, 2678,
+ 2689, 2691, 2693, 2701, 2703, 2705, 2707, 2728, 2730, 2736, 2738, 2739, 2741, 2745,
+ 2748, 2757, 2759, 2761, 2763, 2765, 2768, 2768, 2784, 2787, 2790, 2801, 2809, 2815,
+ 2817, 2819, 2821, 2828, 2831, 2832, 2835, 2856, 2858, 2864, 2866, 2867, 2869, 2873,
+ 2876, 2884, 2887, 2888, 2891, 2893, 2901, 2903, 2908, 2909, 2911, 2915, 2918, 2935,
+ 2946, 2947, 2949, 2954, 2958, 2960, 2962, 2965, 2969, 2970, 2972, 2972, 2974, 2975,
+ 2979, 2980, 2984, 2986, 2990, 3001, 3006, 3010, 3014, 3016, 3018, 3021, 3024, 3024,
+ 3031, 3031, 3046, 3066, 3072, 3084, 3086, 3088, 3090, 3112, 3114, 3129, 3132, 3140,
+ 3142, 3144, 3146, 3149, 3157, 3158, 3160, 3162, 3165, 3165, 3168, 3171, 3174, 3183,
+ 3191, 3212, 3214, 3216, 3218, 3240, 3242, 3251, 3253, 3257, 3260, 3268, 3270, 3272,
+ 3274, 3277, 3285, 3286, 3293, 3294, 3296, 3299, 3302, 3311, 3313, 3315, 3328, 3340,
+ 3342, 3344, 3346, 3396, 3398, 3400, 3402, 3407, 3412, 3427, 3430, 3455, 3457, 3459,
+ 3461, 3478, 3482, 3505, 3507, 3515, 3517, 3517, 3520, 3526, 3530, 3530, 3535, 3540,
+ 3542, 3542, 3544, 3551, 3558, 3567, 3570, 3572, 3585, 3642, 3647, 3675, 3713, 3714,
+ 3716, 3716, 3718, 3722, 3724, 3747, 3749, 3749, 3751, 3773, 3776, 3780, 3782, 3782,
+ 3784, 3790, 3792, 3801, 3804, 3807, 3840, 3911, 3913, 3948, 3953, 3991, 3993, 4028,
+ 4030, 4044, 4046, 4058, 4096, 4293, 4295, 4295, 4301, 4301, 4304, 4680, 4682, 4685,
+ 4688, 4694, 4696, 4696, 4698, 4701, 4704, 4744, 4746, 4749, 4752, 4784, 4786, 4789,
+ 4792, 4798, 4800, 4800, 4802, 4805, 4808, 4822, 4824, 4880, 4882, 4885, 4888, 4954,
+ 4957, 4988, 4992, 5017, 5024, 5109, 5112, 5117, 5120, 5759, 5761, 5788, 5792, 5880,
+ 5888, 5909, 5919, 5942, 5952, 5971, 5984, 5996, 5998, 6000, 6002, 6003, 6016, 6109,
+ 6112, 6121, 6128, 6137, 6144, 6157, 6159, 6169, 6176, 6264, 6272, 6314, 6320, 6389,
+ 6400, 6430, 6432, 6443, 6448, 6459, 6464, 6464, 6468, 6509, 6512, 6516, 6528, 6571,
+ 6576, 6601, 6608, 6618, 6622, 6683, 6686, 6750, 6752, 6780, 6783, 6793, 6800, 6809,
+ 6816, 6829, 6832, 6862, 6912, 6988, 6992, 7038, 7040, 7155, 7164, 7223, 7227, 7241,
+ 7245, 7304, 7312, 7354, 7357, 7367, 7376, 7418, 7424, 7957, 7960, 7965, 7968, 8005,
+ 8008, 8013, 8016, 8023, 8025, 8025, 8027, 8027, 8029, 8029, 8031, 8061, 8064, 8116,
+ 8118, 8132, 8134, 8147, 8150, 8155, 8157, 8175, 8178, 8180, 8182, 8190, 8208, 8231,
+ 8240, 8286, 8304, 8305, 8308, 8334, 8336, 8348, 8352, 8384, 8400, 8432, 8448, 8587,
+ 8592, 9254, 9280, 9290, 9312, 11123, 11126, 11157, 11159, 11507, 11513, 11557,
+ 11559, 11559, 11565, 11565, 11568, 11623, 11631, 11632, 11647, 11670, 11680, 11686,
+ 11688, 11694, 11696, 11702, 11704, 11710, 11712, 11718, 11720, 11726, 11728, 11734,
+ 11736, 11742, 11744, 11869, 11904, 11929, 11931, 12019, 12032, 12245, 12272, 12283,
+ 12289, 12351, 12353, 12438, 12441, 12543, 12549, 12591, 12593, 12686, 12688, 12771,
+ 12784, 12830, 12832, 42124, 42128, 42182, 42192, 42539, 42560, 42743, 42752, 42954,
+ 42960, 42961, 42963, 42963, 42965, 42969, 42994, 43052, 43056, 43065, 43072, 43127,
+ 43136, 43205, 43214, 43225, 43232, 43347, 43359, 43388, 43392, 43469, 43471, 43481,
+ 43486, 43518, 43520, 43574, 43584, 43597, 43600, 43609, 43612, 43714, 43739, 43766,
+ 43777, 43782, 43785, 43790, 43793, 43798, 43808, 43814, 43816, 43822, 43824, 43883,
+ 43888, 44013, 44016, 44025, 44032, 55203, 55216, 55238, 55243, 55291, 63744, 64109,
+ 64112, 64217, 64256, 64262, 64275, 64279, 64285, 64310, 64312, 64316, 64318, 64318,
+ 64320, 64321, 64323, 64324, 64326, 64450, 64467, 64911, 64914, 64967, 64975, 64975,
+ 65008, 65049, 65056, 65106, 65108, 65126, 65128, 65131, 65136, 65140, 65142, 65276,
+ 65281, 65470, 65474, 65479, 65482, 65487, 65490, 65495, 65498, 65500, 65504, 65510,
+ 65512, 65518, 65532, 65533, 65536, 65547, 65549, 65574, 65576, 65594, 65596, 65597,
+ 65599, 65613, 65616, 65629, 65664, 65786, 65792, 65794, 65799, 65843, 65847, 65934,
+ 65936, 65948, 65952, 65952, 66000, 66045, 66176, 66204, 66208, 66256, 66272, 66299,
+ 66304, 66339, 66349, 66378, 66384, 66426, 66432, 66461, 66463, 66499, 66504, 66517,
+ 66560, 66717, 66720, 66729, 66736, 66771, 66776, 66811, 66816, 66855, 66864, 66915,
+ 66927, 66938, 66940, 66954, 66956, 66962, 66964, 66965, 66967, 66977, 66979, 66993,
+ 66995, 67001, 67003, 67004, 67072, 67382, 67392, 67413, 67424, 67431, 67456, 67461,
+ 67463, 67504, 67506, 67514, 67584, 67589, 67592, 67592, 67594, 67637, 67639, 67640,
+ 67644, 67644, 67647, 67669, 67671, 67742, 67751, 67759, 67808, 67826, 67828, 67829,
+ 67835, 67867, 67871, 67897, 67903, 67903, 67968, 68023, 68028, 68047, 68050, 68099,
+ 68101, 68102, 68108, 68115, 68117, 68119, 68121, 68149, 68152, 68154, 68159, 68168,
+ 68176, 68184, 68192, 68255, 68288, 68326, 68331, 68342, 68352, 68405, 68409, 68437,
+ 68440, 68466, 68472, 68497, 68505, 68508, 68521, 68527, 68608, 68680, 68736, 68786,
+ 68800, 68850, 68858, 68903, 68912, 68921, 69216, 69246, 69248, 69289, 69291, 69293,
+ 69296, 69297, 69373, 69415, 69424, 69465, 69488, 69513, 69552, 69579, 69600, 69622,
+ 69632, 69709, 69714, 69749, 69759, 69820, 69822, 69826, 69840, 69864, 69872, 69881,
+ 69888, 69940, 69942, 69959, 69968, 70006, 70016, 70111, 70113, 70132, 70144, 70161,
+ 70163, 70209, 70272, 70278, 70280, 70280, 70282, 70285, 70287, 70301, 70303, 70313,
+ 70320, 70378, 70384, 70393, 70400, 70403, 70405, 70412, 70415, 70416, 70419, 70440,
+ 70442, 70448, 70450, 70451, 70453, 70457, 70459, 70468, 70471, 70472, 70475, 70477,
+ 70480, 70480, 70487, 70487, 70493, 70499, 70502, 70508, 70512, 70516, 70656, 70747,
+ 70749, 70753, 70784, 70855, 70864, 70873, 71040, 71093, 71096, 71133, 71168, 71236,
+ 71248, 71257, 71264, 71276, 71296, 71353, 71360, 71369, 71424, 71450, 71453, 71467,
+ 71472, 71494, 71680, 71739, 71840, 71922, 71935, 71942, 71945, 71945, 71948, 71955,
+ 71957, 71958, 71960, 71989, 71991, 71992, 71995, 72006, 72016, 72025, 72096, 72103,
+ 72106, 72151, 72154, 72164, 72192, 72263, 72272, 72354, 72368, 72440, 72448, 72457,
+ 72704, 72712, 72714, 72758, 72760, 72773, 72784, 72812, 72816, 72847, 72850, 72871,
+ 72873, 72886, 72960, 72966, 72968, 72969, 72971, 73014, 73018, 73018, 73020, 73021,
+ 73023, 73031, 73040, 73049, 73056, 73061, 73063, 73064, 73066, 73102, 73104, 73105,
+ 73107, 73112, 73120, 73129, 73440, 73464, 73472, 73488, 73490, 73530, 73534, 73561,
+ 73648, 73648, 73664, 73713, 73727, 74649, 74752, 74862, 74864, 74868, 74880, 75075,
+ 77712, 77810, 77824, 78895, 78912, 78933, 82944, 83526, 92160, 92728, 92736, 92766,
+ 92768, 92777, 92782, 92862, 92864, 92873, 92880, 92909, 92912, 92917, 92928, 92997,
+ 93008, 93017, 93019, 93025, 93027, 93047, 93053, 93071, 93760, 93850, 93952, 94026,
+ 94031, 94087, 94095, 94111, 94176, 94180, 94192, 94193, 94208, 100343, 100352,
+ 101589, 101632, 101640, 110576, 110579, 110581, 110587, 110589, 110590, 110592,
+ 110882, 110898, 110898, 110928, 110930, 110933, 110933, 110948, 110951, 110960,
+ 111355, 113664, 113770, 113776, 113788, 113792, 113800, 113808, 113817, 113820,
+ 113823, 118528, 118573, 118576, 118598, 118608, 118723, 118784, 119029, 119040,
+ 119078, 119081, 119154, 119163, 119274, 119296, 119365, 119488, 119507, 119520,
+ 119539, 119552, 119638, 119648, 119672, 119808, 119892, 119894, 119964, 119966,
+ 119967, 119970, 119970, 119973, 119974, 119977, 119980, 119982, 119993, 119995,
+ 119995, 119997, 120003, 120005, 120069, 120071, 120074, 120077, 120084, 120086,
+ 120092, 120094, 120121, 120123, 120126, 120128, 120132, 120134, 120134, 120138,
+ 120144, 120146, 120485, 120488, 120779, 120782, 121483, 121499, 121503, 121505,
+ 121519, 122624, 122654, 122661, 122666, 122880, 122886, 122888, 122904, 122907,
+ 122913, 122915, 122916, 122918, 122922, 122928, 122989, 123023, 123023, 123136,
+ 123180, 123184, 123197, 123200, 123209, 123214, 123215, 123536, 123566, 123584,
+ 123641, 123647, 123647, 124112, 124153, 124896, 124902, 124904, 124907, 124909,
+ 124910, 124912, 124926, 124928, 125124, 125127, 125142, 125184, 125259, 125264,
+ 125273, 125278, 125279, 126065, 126132, 126209, 126269, 126464, 126467, 126469,
+ 126495, 126497, 126498, 126500, 126500, 126503, 126503, 126505, 126514, 126516,
+ 126519, 126521, 126521, 126523, 126523, 126530, 126530, 126535, 126535, 126537,
+ 126537, 126539, 126539, 126541, 126543, 126545, 126546, 126548, 126548, 126551,
+ 126551, 126553, 126553, 126555, 126555, 126557, 126557, 126559, 126559, 126561,
+ 126562, 126564, 126564, 126567, 126570, 126572, 126578, 126580, 126583, 126585,
+ 126588, 126590, 126590, 126592, 126601, 126603, 126619, 126625, 126627, 126629,
+ 126633, 126635, 126651, 126704, 126705, 126976, 127019, 127024, 127123, 127136,
+ 127150, 127153, 127167, 127169, 127183, 127185, 127221, 127232, 127405, 127462,
+ 127490, 127504, 127547, 127552, 127560, 127568, 127569, 127584, 127589, 127744,
+ 128727, 128732, 128748, 128752, 128764, 128768, 128886, 128891, 128985, 128992,
+ 129003, 129008, 129008, 129024, 129035, 129040, 129095, 129104, 129113, 129120,
+ 129159, 129168, 129197, 129200, 129201, 129280, 129619, 129632, 129645, 129648,
+ 129660, 129664, 129672, 129680, 129725, 129727, 129733, 129742, 129755, 129760,
+ 129768, 129776, 129784, 129792, 129938, 129940, 129994, 130032, 130041, 131072,
+ 173791, 173824, 177977, 177984, 178205, 178208, 183969, 183984, 191456, 194560,
+ 195101, 196608, 201546, 201552, 205743, 917760, 917999
+ };
+ }
+
/**
* The lowercase of each entry of {@link #LOWER_FROM}, in the same order.
*
@@ -856,6 +989,9 @@ public final class UnicodeTables {
private static final long SPACE_ASCII_LOW = 8321515008L;
private static final long SPACE_ASCII_HIGH = 0L;
+ private static final long PRINTABLE_ASCII_LOW = -4294967296L;
+ private static final long PRINTABLE_ASCII_HIGH = 9223372036854775807L;
+
/** Python's {@code str.isalpha()} for one code point. */
public static boolean isAlpha(int codePoint) {
@@ -982,6 +1118,25 @@ public final class UnicodeTables {
return false;
}
+ /**
+ * Python's {@code str.isprintable()} for one code point.
+ *
+ * <p>What {@code repr} leaves alone. Everything else it spells as a numeric escape, and that
+ * includes code points no ASCII-only check catches: U+00A0 NO-BREAK SPACE, U+00AD SOFT
+ * HYPHEN, U+200B ZERO WIDTH SPACE, U+FEFF, and the private-use and unassigned planes. A
+ * pasted ticket is full of the first one, and it reaches a route's reason string through the
+ * mixed segment, so an escape check that stopped at U+007F would put a raw control character
+ * into an API response.
+ */
+ public static boolean isPrintable(int codePoint) {
+ if (codePoint < 0x80) {
+ long mask = codePoint < 64 ? PRINTABLE_ASCII_LOW >>> codePoint
+ : PRINTABLE_ASCII_HIGH >>> (codePoint - 64);
+ return (mask & 1L) != 0L;
+ }
+ return contains(PRINTABLE, codePoint);
+ }
+
/**
* Python's {@code str.isspace()} for one code point.
*
diff --git a/laya-java/laya-java/src/test/java/com/convaiinnovations/laya/json/PythonJsonTest.java b/laya-java/laya-java/src/test/java/com/convaiinnovations/laya/json/PythonJsonTest.java
index f49f547f..bd251ed3 100644
--- a/laya-java/laya-java/src/test/java/com/convaiinnovations/laya/json/PythonJsonTest.java
+++ b/laya-java/laya-java/src/test/java/com/convaiinnovations/laya/json/PythonJsonTest.java
@@ -1,8 +1,12 @@
package com.convaiinnovations.laya.json;
import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
import com.convaiinnovations.laya.Fixtures;
+import java.nio.charset.StandardCharsets;
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
import java.util.List;
import java.util.Map;
import org.junit.jupiter.api.DisplayName;
@@ -122,6 +126,92 @@ final class PythonJsonTest {
() -> PythonJson.dumps(map));
}
+ @Test
+ @DisplayName("repr spells every recorded string the way CPython does")
+ void reprMatchesCpython() {
+ @SuppressWarnings("unchecked")
+ List<Object> cases = (List<Object>) Fixtures.load("python_json.json").get("repr_strings");
+ int escaped = 0;
+ int doubleQuoted = 0;
+ for (Object entry : cases) {
+ @SuppressWarnings("unchecked")
+ List<Object> pair = (List<Object>) entry;
+ String input = (String) pair.get(0);
+ String expected = (String) pair.get(1);
+ assertEquals(expected, PythonJson.repr(input),
+ "repr of " + expected + " (" + input.length() + " chars)");
+ if (expected.indexOf('\\') >= 0) {
+ escaped++;
+ }
+ if (expected.startsWith("\"")) {
+ doubleQuoted++;
+ }
+ }
+ assertTrue(cases.size() >= 30, "only " + cases.size() + " repr cases");
+ assertTrue(escaped >= 15, "only " + escaped + " cases actually escape something");
+ assertTrue(doubleQuoted >= 1, "no case exercises the double-quoted form, which is the"
+ + " branch for a string holding a single quote and no double quote");
+ }
+
+ @Test
+ @DisplayName("repr escapes every unprintable code point in Unicode, and no printable one")
+ void reprDigestMatchesCpython() {
+ // The listed cases cover the rules a reader can name; this covers the boundary nobody
+ // enumerated. 148,998 code points are printable and the rest are not, and an ASCII-only
+ // escape check -- which is what the .NET port does -- passes every named case above while
+ // leaving U+00A0 and U+200B raw in an API response.
+ MessageDigest digest;
+ try {
+ digest = MessageDigest.getInstance("SHA-256");
+ } catch (NoSuchAlgorithmException impossible) {
+ throw new IllegalStateException("SHA-256 is required of every JVM", impossible);
+ }
+ for (int cp = 0; cp < 0x110000; cp++) {
+ if (cp >= 0xD800 && cp <= 0xDFFF) {
+ continue;
+ }
+ String value = "a" + new String(Character.toChars(cp)) + "b";
+ digest.update(PythonJson.repr(value).getBytes(StandardCharsets.UTF_8));
+ }
+ StringBuilder hex = new StringBuilder(64);
+ for (byte value : digest.digest()) {
+ hex.append(Character.forDigit((value >> 4) & 0xF, 16));
+ hex.append(Character.forDigit(value & 0xF, 16));
+ }
+ assertEquals(Fixtures.load("python_json.json").get("repr_digest"), hex.toString(),
+ "repr disagrees with CPython somewhere in Unicode");
+ }
+
+ @Test
+ @DisplayName("percent0 rounds halves to even, as CPython's %.0f does")
+ void percent0MatchesCpython() {
+ @SuppressWarnings("unchecked")
+ List<Object> cases = (List<Object>) Fixtures.load("python_json.json").get("percent0");
+ int distinguishing = 0;
+ for (Object entry : cases) {
+ @SuppressWarnings("unchecked")
+ List<Object> pair = (List<Object>) entry;
+ double fraction = ((Number) pair.get(0)).doubleValue();
+ String expected = (String) pair.get(1);
+ assertEquals(expected, PythonJson.percent0(fraction), "percent0 of " + fraction);
+ // Java's own formatter rounds halves up, so count how many of these would be wrong
+ // with it -- a count of zero would mean this test proves nothing.
+ if (!expected.equals(String.format(java.util.Locale.ROOT, "%.0f", 100.0 * fraction))) {
+ distinguishing++;
+ }
+ }
+ assertTrue(cases.size() >= 25, "only " + cases.size() + " percent cases");
+ assertTrue(distinguishing >= 3,
+ "only " + distinguishing + " of these differ from String.format(\"%.0f\"), so"
+ + " this test would not notice half-up rounding");
+ }
+
+ @Test
+ @DisplayName("repr of null is None, as a %r of None would print")
+ void reprOfNull() {
+ assertEquals("None", PythonJson.repr((String) null));
+ }
+
@Test
@DisplayName("the same container appearing twice is NOT a cycle")
void repeatedContainerIsNotACycle() {
diff --git a/laya-java/scripts/gen_unicode_tables.py b/laya-java/scripts/gen_unicode_tables.py
index 84dc6a04..c5d896d2 100755
--- a/laya-java/scripts/gen_unicode_tables.py
+++ b/laya-java/scripts/gen_unicode_tables.py
@@ -160,6 +160,7 @@ def render():
upper = ranges(lambda cp: chr(cp).isupper())
lower = ranges(lambda cp: chr(cp).islower())
space = ranges(lambda cp: chr(cp).isspace())
+ printable = ranges(lambda cp: chr(cp).isprintable())
# Every code point whose lowercase is more than one code point. In Unicode 15 there is
# exactly one, and `laya/lang.py` depends on it: it replaces U+0130 before lowering, because
@@ -239,6 +240,11 @@ public final class UnicodeTables {
"Every code point whose Python lowercase is one DIFFERENT code point,\n"
" * sorted. {@link #LOWER_TO} holds what each one maps to.")
+ "\n"
+ + "\n"
+ + table("PRINTABLE", printable,
+ "Python's {@code str.isprintable()}, which is what {@code repr} leaves\n"
+ " * unescaped. Everything else it spells as a numeric escape.")
+ + "\n"
+ flat_table("LOWER_TO", [low for _, low in simple],
"The lowercase of each entry of {@link #LOWER_FROM}, in the same order.")
+ "\n"
@@ -255,6 +261,7 @@ public final class UnicodeTables {
("UPPER", lambda cp: chr(cp).isupper()),
("LOWER", lambda cp: chr(cp).islower()),
("SPACE", lambda cp: chr(cp).isspace()),
+ ("PRINTABLE", lambda cp: chr(cp).isprintable()),
))
)
@@ -360,6 +367,20 @@ public final class UnicodeTables {
return false;
}
+ /**
+ * Python's {@code str.isprintable()} for one code point.
+ *
+ * <p>What {@code repr} leaves alone. Everything else it spells as a numeric escape, and that
+ * includes code points no ASCII-only check catches: U+00A0 NO-BREAK SPACE, U+00AD SOFT
+ * HYPHEN, U+200B ZERO WIDTH SPACE, U+FEFF, and the private-use and unassigned planes. A
+ * pasted ticket is full of the first one, and it reaches a route's reason string through the
+ * mixed segment, so an escape check that stopped at U+007F would put a raw control character
+ * into an API response.
+ */
+ public static boolean isPrintable(int codePoint) {
+ return contains(PRINTABLE, codePoint);
+ }
+
/**
* Python's {@code str.isspace()} for one code point.
*
@@ -527,7 +548,8 @@ public final class UnicodeTables {
# Splice the ASCII fast path into each accessor. Done here rather than in the template so the
# template stays readable, and asserted so a renamed accessor cannot silently lose its fast
# path and quietly become ten times slower on ordinary Latin text.
- for name in ("ALPHA", "WORD", "COMBINING", "DIGIT", "UPPER", "LOWER", "SPACE"):
+ for name in ("ALPHA", "WORD", "COMBINING", "DIGIT", "UPPER", "LOWER", "SPACE",
+ "PRINTABLE"):
marker = " return contains(%s, codePoint);\n" % name
if methods.count(marker) != 1:
raise SystemExit(