Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
fix(java): four defects the email port had outside the BMP
Adeshina Lasisia57f3c16dffc
diff --git a/laya-java/fixtures/email.json b/laya-java/fixtures/email.jsonindex 34d25b50..178ebf0d 100644--- a/laya-java/fixtures/email.json+++ b/laya-java/fixtures/email.json@@ -400,6 +400,31 @@"body": "I cannot log in and need a password reset.\n\nFrom: Maria Souza\nDate: 3 Sep २०२५\nolder","result": "I cannot log in and need a password reset."},+ {+ "name": "astral-word-char-breaks-no-boundary",+ "body": "Esta mensagem𒐲 e confidencial.",+ "result": "Esta mensagem𒐲 e confidencial."+ },+ {+ "name": "astral-elsewhere-still-matches",+ "body": "Esta mensagem e confidencial e de uso exclusivo do destinatario𒐲.",+ "result": ""+ },+ {+ "name": "astral-signoff-at-the-width-limit",+ "body": "I cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nRegards, 𝐀AAAAAAAAAAAAAAAAAAAAAAAAAAAAAA",+ "result": "I cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset."+ },+ {+ "name": "astral-signoff-just-over-the-limit",+ "body": "I cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nRegards, 𝐀AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA",+ "result": "I cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nRegards, 𝐀AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"+ },+ {+ "name": "astral-mark-in-a-disclaimer",+ "body": "Esta mensagem e confidencial𑀁 e de uso exclusivo do destinatario.",+ "result": ""+ },{"name": "full-stack","body": "I cannot log in and need a password reset.\nPlease check the attached receipt.\n\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nAtenciosamente,\nAna Souza\nEnviado do meu iPhone\n\nEsta mensagem e confidencial e de uso exclusivo do destinatario.\n\nEm ter., 3 de set. de 2025, Bruno <[email protected]> escreveu:\n> mensagem antiga",@@ -436,6 +461,30 @@"body": "I cannot log in and need a password reset.","max_chars": 0,"result": ""+ },+ {+ "name": "budget-splits-a-surrogate-pair",+ "body": "ab𝐀cd𝐀ef",+ "max_chars": 3,+ "result": "ab𝐀"+ },+ {+ "name": "budget-ends-on-a-surrogate-pair",+ "body": "ab𝐀cd",+ "max_chars": 4,+ "result": "ab𝐀c"+ },+ {+ "name": "budget-all-astral",+ "body": "𝐀𝐁𝐂𝐃",+ "max_chars": 2,+ "result": "𝐀𝐁"+ },+ {+ "name": "budget-four-x-bound-astral",+ "body": "𝐀𝐀𝐀 This email is confidential and intended solely for the addressee.",+ "max_chars": 11,+ "result": ""}],"states": [diff --git a/laya-java/laya-java/src/main/java/com/convaiinnovations/laya/LayaEmail.java b/laya-java/laya-java/src/main/java/com/convaiinnovations/laya/LayaEmail.javaindex 686b782a..512d1fce 100644--- a/laya-java/laya-java/src/main/java/com/convaiinnovations/laya/LayaEmail.java+++ b/laya-java/laya-java/src/main/java/com/convaiinnovations/laya/LayaEmail.java@@ -134,6 +134,26 @@ public final class LayaEmail {return Pattern.compile(regex, Pattern.UNIX_LINES);}+ /**+ * {@link #compile} plus {@code UNICODE_CHARACTER_CLASS}, for the one pattern that uses+ * {@code \b}.+ *+ * <p>The flag is what makes {@code \b} read Java's Unicode word class; without it {@code \b}+ * reads {@code [a-zA-Z0-9_]}, and then {@link #boundaryView} -- which reconciles Python's+ * class with the Unicode one -- is reconciling against a class nothing consults. The symptom+ * was a false positive: "Esta mensagem" followed by an astral letter has no word boundary to+ * Python, an ASCII {@code \b} found one anyway, and a paragraph the reference keeps was+ * deleted whole.+ *+ * <p>Only {@code \b} changes meaning here. This pattern has no {@code \s}, {@code \d} or+ * {@code \w} of its own -- those are the classes built from the tables, spelled out -- so the+ * flag cannot reach anything else.+ */+ private static Pattern compileWithBoundaries(String regex) {+ return Pattern.compile(regex, Pattern.CASE_INSENSITIVE | Pattern.UNICODE_CASE+ | Pattern.UNIX_LINES | Pattern.UNICODE_CHARACTER_CLASS);+ }+/** The default body budget, which {@link #emailState} passes through. */public static final int DEFAULT_MAX_CHARS = 3000;@@ -260,7 +280,7 @@ public final class LayaEmail {+ "|max|mini|plus|using [a-z][a-z0-9_.+-]*))*"+ "|(obter o|get) outlook (para|for) (ios|android))[" + inner(S) + ".!]*$");- static final Pattern DISCLAIMER = compile(+ static final Pattern DISCLAIMER = compileWithBoundaries(// English: tied to a disclaimer noun AND a disclaimer tail, the way the// Portuguese branches are. The bare word matched any sentence that merely// mentioned it, so "Is this confidential?" was deleted whole. `[^.]` rather than@@ -620,9 +640,13 @@ public final class LayaEmail {.replace("\r\n", "\n").replace("\r", "\n").replace("\\n", "\n");// Bound regex work before the expensive patterns below: DISCLAIMER uses [^.]{0,60/80/100}// alternations whose cost grows with input length, and only maxChars are ever returned.- if (maxChars > 0 && text.length() > (long) maxChars * 4) {- text = text.substring(0, maxChars * 4);- } else if (maxChars <= 0) {+ //+ // In CODE POINTS, because Python slices a str by code point. Cutting by char index splits+ // a surrogate pair, and the replacement character that comes out is not what the+ // reference returns -- measured on 4,641 cases of an astral corpus.+ if (maxChars > 0) {+ text = cutToCodePoints(text, (long) maxChars * 4);+ } else {// Python slices with max_chars * 4, which is <= 0 here: `text[:0]` is empty, and so is// the result. Kept explicit because a negative length is an exception in Java.text = "";@@ -655,7 +679,11 @@ public final class LayaEmail {int cut = lines.size();int start = Math.max(1, Math.min((int) (lines.size() * 0.6), lines.size() - 8));for (int i = start; i < lines.size(); i++) {- int width = UnicodeTables.strip(lines.get(i)).length();+ // Code points, as Python's len() counts them. An astral character is two chars to+ // Java, so counting chars made a 40-code-point closing 41 wide and left a signature+ // in the model's input.+ String candidate = UnicodeTables.strip(lines.get(i));+ int width = candidate.codePointCount(0, candidate.length());if ((width <= 40 && isSignature(lines.get(i)))|| (width <= 60 && Patterns.DEVICE_FOOTER.matcher(lines.get(i)).lookingAt())) {cut = i;@@ -683,7 +711,29 @@ public final class LayaEmail {out.append(cleaned);}String result = Patterns.SPACES_AND_TABS.matcher(out.toString()).replaceAll(" ");- return result.length() > maxChars ? result.substring(0, Math.max(maxChars, 0)) : result;+ return cutToCodePoints(result, Math.max(maxChars, 0));+ }++ /**+ * The first {@code limit} CODE POINTS of {@code text}, which is what Python's {@code [:n]}+ * takes.+ *+ * <p>Never cuts between a high and a low surrogate: a half pair is not a character the+ * reference can return, and the budget cases of the astral corpus are 4,641 proofs that+ * cutting by char index returns one.+ */+ private static String cutToCodePoints(String text, long limit) {+ if (limit <= 0) {+ return "";+ }+ if (text.length() <= limit) {+ return text; // cannot hold more code points than chars+ }+ int capped = (int) Math.min(limit, text.length());+ if (text.codePointCount(0, text.length()) <= capped) {+ return text;+ }+ return text.substring(0, text.offsetByCodePoints(0, capped));}private static boolean matchesAny(List<Pattern> patterns, String line) {diff --git a/laya-java/scripts/gen_fixtures.py b/laya-java/scripts/gen_fixtures.pyindex f9dd25f3..a5c1b466 100644--- a/laya-java/scripts/gen_fixtures.py+++ b/laya-java/scripts/gen_fixtures.py@@ -1895,6 +1895,31 @@ def email_clean():("header-next-devanagari-year",q + "\n\nFrom: Maria Souza\nDate: 3 Sep \u0968\u0966\u0968\u096B\nolder"),+ # -- outside the BMP. The first sweep of this port had no astral character in it and+ # passed on 40,000 cases; a corpus of them found four defects at once. Each case below+ # is one of them, because the units the reference counts are CODE POINTS and the units+ # Java reaches for are chars.+ #+ # A word boundary. U+12432 is a letter-number, which is a word character to Python, so+ # there is no boundary after "mensagem" and this paragraph is NOT a disclaimer. An ASCII+ # `\\b` finds a boundary anyway and deletes the sentence whole.+ ("astral-word-char-breaks-no-boundary",+ "Esta mensagem\U00012432 e confidencial."),+ # And the same shape where the reference DOES match, so the fix cannot be "never match".+ ("astral-elsewhere-still-matches",+ "Esta mensagem e confidencial e de uso exclusivo do destinatario\U00012432."),+ # A 40-code-point closing is 41 Java chars with one astral letter in it, so a char count+ # leaves the signature in.+ ("astral-signoff-at-the-width-limit",+ "\n".join([q] * 9 + ["Regards, \U0001D400" + "A" * 30])),+ ("astral-signoff-just-over-the-limit",+ "\n".join([q] * 9 + ["Regards, \U0001D400" + "A" * 31])),+ # A combining mark outside the BMP: Java's word class holds it and Python's does not, so+ # the view substitutes it -- one code point for one code point, which is what keeps the+ # `[^.]{0,80}` window counting the same.+ ("astral-mark-in-a-disclaimer",+ "Esta mensagem e confidencial\U00011001 e de uso exclusivo do destinatario."),+# -- everything at once("full-stack", "\n".join([q, "Please check the attached receipt.", ""]@@ -1912,6 +1937,22 @@ def email_clean():("budget-four-x-bound", ("word " * 400) + "This email is confidential and intended solely ""for the addressee.", 100),("budget-zero", q, 0),+ # A budget that lands between a high and a low surrogate. Python slices by code point and+ # returns the whole character; a char-index cut returns half of one, which is not a+ # character the reference can produce.+ ("budget-splits-a-surrogate-pair", "ab\U0001D400cd\U0001D400ef", 3),+ ("budget-ends-on-a-surrogate-pair", "ab\U0001D400cd", 4),+ ("budget-all-astral", "\U0001D400\U0001D401\U0001D402\U0001D403", 2),+ # And the pre-truncation bound, which is max_chars * 4 code points.+ # The pre-truncation bound is max_chars * 4 CODE POINTS, and it decides whether the+ # disclaimer below is inside the window at all. Cut by char index the window is half as+ # wide, the disclaimer arrives truncated, the truncated fragment no longer matches, and+ # it survives into the model's input instead of being stripped. The final cut hides this+ # for most inputs, which is why the case is this specific: 3 astral characters and a+ # budget of 11.+ ("budget-four-x-bound-astral",+ "\U0001D400\U0001D400\U0001D400 This email is confidential and intended solely for "+ "the addressee.", 11),]states = [("state-plain", "Refund request", q, None, True, 3000, {}),