Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

fix(java): four defects the email port had outside the BMP

Adeshina Lasisia57f3c16dffc
diff --git a/laya-java/fixtures/email.json b/laya-java/fixtures/email.json
index 34d25b50..178ebf0d 100644
--- a/laya-java/fixtures/email.json
+++ b/laya-java/fixtures/email.json
@@ -400,6 +400,31 @@
"body": "I cannot log in and need a password reset.\n\nFrom: Maria Souza\nDate: 3 Sep २०२५\nolder",
"result": "I cannot log in and need a password reset."
},
+ {
+ "name": "astral-word-char-breaks-no-boundary",
+ "body": "Esta mensagem𒐲 e confidencial.",
+ "result": "Esta mensagem𒐲 e confidencial."
+ },
+ {
+ "name": "astral-elsewhere-still-matches",
+ "body": "Esta mensagem e confidencial e de uso exclusivo do destinatario𒐲.",
+ "result": ""
+ },
+ {
+ "name": "astral-signoff-at-the-width-limit",
+ "body": "I cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nRegards, 𝐀AAAAAAAAAAAAAAAAAAAAAAAAAAAAAA",
+ "result": "I cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset."
+ },
+ {
+ "name": "astral-signoff-just-over-the-limit",
+ "body": "I cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nRegards, 𝐀AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA",
+ "result": "I cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nRegards, 𝐀AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"
+ },
+ {
+ "name": "astral-mark-in-a-disclaimer",
+ "body": "Esta mensagem e confidencial𑀁 e de uso exclusivo do destinatario.",
+ "result": ""
+ },
{
"name": "full-stack",
"body": "I cannot log in and need a password reset.\nPlease check the attached receipt.\n\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nAtenciosamente,\nAna Souza\nEnviado do meu iPhone\n\nEsta mensagem e confidencial e de uso exclusivo do destinatario.\n\nEm ter., 3 de set. de 2025, Bruno <[email protected]> escreveu:\n> mensagem antiga",
@@ -436,6 +461,30 @@
"body": "I cannot log in and need a password reset.",
"max_chars": 0,
"result": ""
+ },
+ {
+ "name": "budget-splits-a-surrogate-pair",
+ "body": "ab𝐀cd𝐀ef",
+ "max_chars": 3,
+ "result": "ab𝐀"
+ },
+ {
+ "name": "budget-ends-on-a-surrogate-pair",
+ "body": "ab𝐀cd",
+ "max_chars": 4,
+ "result": "ab𝐀c"
+ },
+ {
+ "name": "budget-all-astral",
+ "body": "𝐀𝐁𝐂𝐃",
+ "max_chars": 2,
+ "result": "𝐀𝐁"
+ },
+ {
+ "name": "budget-four-x-bound-astral",
+ "body": "𝐀𝐀𝐀 This email is confidential and intended solely for the addressee.",
+ "max_chars": 11,
+ "result": ""
}
],
"states": [
diff --git a/laya-java/laya-java/src/main/java/com/convaiinnovations/laya/LayaEmail.java b/laya-java/laya-java/src/main/java/com/convaiinnovations/laya/LayaEmail.java
index 686b782a..512d1fce 100644
--- a/laya-java/laya-java/src/main/java/com/convaiinnovations/laya/LayaEmail.java
+++ b/laya-java/laya-java/src/main/java/com/convaiinnovations/laya/LayaEmail.java
@@ -134,6 +134,26 @@ public final class LayaEmail {
return Pattern.compile(regex, Pattern.UNIX_LINES);
}
+ /**
+ * {@link #compile} plus {@code UNICODE_CHARACTER_CLASS}, for the one pattern that uses
+ * {@code \b}.
+ *
+ * <p>The flag is what makes {@code \b} read Java's Unicode word class; without it {@code \b}
+ * reads {@code [a-zA-Z0-9_]}, and then {@link #boundaryView} -- which reconciles Python's
+ * class with the Unicode one -- is reconciling against a class nothing consults. The symptom
+ * was a false positive: "Esta mensagem" followed by an astral letter has no word boundary to
+ * Python, an ASCII {@code \b} found one anyway, and a paragraph the reference keeps was
+ * deleted whole.
+ *
+ * <p>Only {@code \b} changes meaning here. This pattern has no {@code \s}, {@code \d} or
+ * {@code \w} of its own -- those are the classes built from the tables, spelled out -- so the
+ * flag cannot reach anything else.
+ */
+ private static Pattern compileWithBoundaries(String regex) {
+ return Pattern.compile(regex, Pattern.CASE_INSENSITIVE | Pattern.UNICODE_CASE
+ | Pattern.UNIX_LINES | Pattern.UNICODE_CHARACTER_CLASS);
+ }
+
/** The default body budget, which {@link #emailState} passes through. */
public static final int DEFAULT_MAX_CHARS = 3000;
@@ -260,7 +280,7 @@ public final class LayaEmail {
+ "|max|mini|plus|using [a-z][a-z0-9_.+-]*))*"
+ "|(obter o|get) outlook (para|for) (ios|android))[" + inner(S) + ".!]*$");
- static final Pattern DISCLAIMER = compile(
+ static final Pattern DISCLAIMER = compileWithBoundaries(
// English: tied to a disclaimer noun AND a disclaimer tail, the way the
// Portuguese branches are. The bare word matched any sentence that merely
// mentioned it, so "Is this confidential?" was deleted whole. `[^.]` rather than
@@ -620,9 +640,13 @@ public final class LayaEmail {
.replace("\r\n", "\n").replace("\r", "\n").replace("\\n", "\n");
// Bound regex work before the expensive patterns below: DISCLAIMER uses [^.]{0,60/80/100}
// alternations whose cost grows with input length, and only maxChars are ever returned.
- if (maxChars > 0 && text.length() > (long) maxChars * 4) {
- text = text.substring(0, maxChars * 4);
- } else if (maxChars <= 0) {
+ //
+ // In CODE POINTS, because Python slices a str by code point. Cutting by char index splits
+ // a surrogate pair, and the replacement character that comes out is not what the
+ // reference returns -- measured on 4,641 cases of an astral corpus.
+ if (maxChars > 0) {
+ text = cutToCodePoints(text, (long) maxChars * 4);
+ } else {
// Python slices with max_chars * 4, which is <= 0 here: `text[:0]` is empty, and so is
// the result. Kept explicit because a negative length is an exception in Java.
text = "";
@@ -655,7 +679,11 @@ public final class LayaEmail {
int cut = lines.size();
int start = Math.max(1, Math.min((int) (lines.size() * 0.6), lines.size() - 8));
for (int i = start; i < lines.size(); i++) {
- int width = UnicodeTables.strip(lines.get(i)).length();
+ // Code points, as Python's len() counts them. An astral character is two chars to
+ // Java, so counting chars made a 40-code-point closing 41 wide and left a signature
+ // in the model's input.
+ String candidate = UnicodeTables.strip(lines.get(i));
+ int width = candidate.codePointCount(0, candidate.length());
if ((width <= 40 && isSignature(lines.get(i)))
|| (width <= 60 && Patterns.DEVICE_FOOTER.matcher(lines.get(i)).lookingAt())) {
cut = i;
@@ -683,7 +711,29 @@ public final class LayaEmail {
out.append(cleaned);
}
String result = Patterns.SPACES_AND_TABS.matcher(out.toString()).replaceAll(" ");
- return result.length() > maxChars ? result.substring(0, Math.max(maxChars, 0)) : result;
+ return cutToCodePoints(result, Math.max(maxChars, 0));
+ }
+
+ /**
+ * The first {@code limit} CODE POINTS of {@code text}, which is what Python's {@code [:n]}
+ * takes.
+ *
+ * <p>Never cuts between a high and a low surrogate: a half pair is not a character the
+ * reference can return, and the budget cases of the astral corpus are 4,641 proofs that
+ * cutting by char index returns one.
+ */
+ private static String cutToCodePoints(String text, long limit) {
+ if (limit <= 0) {
+ return "";
+ }
+ if (text.length() <= limit) {
+ return text; // cannot hold more code points than chars
+ }
+ int capped = (int) Math.min(limit, text.length());
+ if (text.codePointCount(0, text.length()) <= capped) {
+ return text;
+ }
+ return text.substring(0, text.offsetByCodePoints(0, capped));
}
private static boolean matchesAny(List<Pattern> patterns, String line) {
diff --git a/laya-java/scripts/gen_fixtures.py b/laya-java/scripts/gen_fixtures.py
index f9dd25f3..a5c1b466 100644
--- a/laya-java/scripts/gen_fixtures.py
+++ b/laya-java/scripts/gen_fixtures.py
@@ -1895,6 +1895,31 @@ def email_clean():
("header-next-devanagari-year",
q + "\n\nFrom: Maria Souza\nDate: 3 Sep \u0968\u0966\u0968\u096B\nolder"),
+ # -- outside the BMP. The first sweep of this port had no astral character in it and
+ # passed on 40,000 cases; a corpus of them found four defects at once. Each case below
+ # is one of them, because the units the reference counts are CODE POINTS and the units
+ # Java reaches for are chars.
+ #
+ # A word boundary. U+12432 is a letter-number, which is a word character to Python, so
+ # there is no boundary after "mensagem" and this paragraph is NOT a disclaimer. An ASCII
+ # `\\b` finds a boundary anyway and deletes the sentence whole.
+ ("astral-word-char-breaks-no-boundary",
+ "Esta mensagem\U00012432 e confidencial."),
+ # And the same shape where the reference DOES match, so the fix cannot be "never match".
+ ("astral-elsewhere-still-matches",
+ "Esta mensagem e confidencial e de uso exclusivo do destinatario\U00012432."),
+ # A 40-code-point closing is 41 Java chars with one astral letter in it, so a char count
+ # leaves the signature in.
+ ("astral-signoff-at-the-width-limit",
+ "\n".join([q] * 9 + ["Regards, \U0001D400" + "A" * 30])),
+ ("astral-signoff-just-over-the-limit",
+ "\n".join([q] * 9 + ["Regards, \U0001D400" + "A" * 31])),
+ # A combining mark outside the BMP: Java's word class holds it and Python's does not, so
+ # the view substitutes it -- one code point for one code point, which is what keeps the
+ # `[^.]{0,80}` window counting the same.
+ ("astral-mark-in-a-disclaimer",
+ "Esta mensagem e confidencial\U00011001 e de uso exclusivo do destinatario."),
+
# -- everything at once
("full-stack", "\n".join(
[q, "Please check the attached receipt.", ""]
@@ -1912,6 +1937,22 @@ def email_clean():
("budget-four-x-bound", ("word " * 400) + "This email is confidential and intended solely "
"for the addressee.", 100),
("budget-zero", q, 0),
+ # A budget that lands between a high and a low surrogate. Python slices by code point and
+ # returns the whole character; a char-index cut returns half of one, which is not a
+ # character the reference can produce.
+ ("budget-splits-a-surrogate-pair", "ab\U0001D400cd\U0001D400ef", 3),
+ ("budget-ends-on-a-surrogate-pair", "ab\U0001D400cd", 4),
+ ("budget-all-astral", "\U0001D400\U0001D401\U0001D402\U0001D403", 2),
+ # And the pre-truncation bound, which is max_chars * 4 code points.
+ # The pre-truncation bound is max_chars * 4 CODE POINTS, and it decides whether the
+ # disclaimer below is inside the window at all. Cut by char index the window is half as
+ # wide, the disclaimer arrives truncated, the truncated fragment no longer matches, and
+ # it survives into the model's input instead of being stripped. The final cut hides this
+ # for most inputs, which is why the case is this specific: 3 astral characters and a
+ # budget of 11.
+ ("budget-four-x-bound-astral",
+ "\U0001D400\U0001D400\U0001D400 This email is confidential and intended solely for "
+ "the addressee.", 11),
]
states = [
("state-plain", "Refund request", q, None, True, 3000, {}),