Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
fix(java): what the adversarial review of the email port found
Adeshina Lasisic6fbc970b44f
diff --git a/laya-java/fixtures/email.json b/laya-java/fixtures/email.jsonindex 178ebf0d..c2cd66de 100644--- a/laya-java/fixtures/email.json+++ b/laya-java/fixtures/email.json@@ -400,6 +400,51 @@"body": "I cannot log in and need a password reset.\n\nFrom: Maria Souza\nDate: 3 Sep २०२५\nolder","result": "I cannot log in and need a password reset."},+ {+ "name": "window-floor-of-one",+ "body": "I cannot log in and need a password reset.\nRegards, Ana",+ "result": "I cannot log in and need a password reset."+ },+ {+ "name": "window-offset-dominates",+ "body": "I cannot log in and need a password reset.\nRegards, Ana\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.",+ "result": "I cannot log in and need a password reset."+ },+ {+ "name": "window-offset-excludes",+ "body": "I cannot log in and need a password reset.\nRegards, Ana\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.",+ "result": "I cannot log in and need a password reset.\nRegards, Ana\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset."+ },+ {+ "name": "window-ratio-excludes",+ "body": "I cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nRegards, Ana\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.",+ "result": "I cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nRegards, Ana\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset."+ },+ {+ "name": "window-ratio-includes",+ "body": "I cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nRegards, Ana\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.",+ "result": "I cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset."+ },+ {+ "name": "footer-at-the-width-limit",+ "body": "I cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nSent from my iphone using aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",+ "result": "I cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset."+ },+ {+ "name": "footer-just-over-the-limit",+ "body": "I cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nSent from my iphone using aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",+ "result": "I cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nSent from my iphone using aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"+ },+ {+ "name": "body-strip-is-pythons",+ "body": " Refund please ",+ "result": "Refund please"+ },+ {+ "name": "body-strip-both-ends",+ "body": " Please refund ",+ "result": "Please refund"+ },{"name": "astral-word-char-breaks-no-boundary","body": "Esta mensagem𒐲 e confidencial.",diff --git a/laya-java/laya-java/src/main/java/com/convaiinnovations/laya/LayaEmail.java b/laya-java/laya-java/src/main/java/com/convaiinnovations/laya/LayaEmail.javaindex 512d1fce..03243417 100644--- a/laya-java/laya-java/src/main/java/com/convaiinnovations/laya/LayaEmail.java+++ b/laya-java/laya-java/src/main/java/com/convaiinnovations/laya/LayaEmail.java@@ -95,6 +95,17 @@ public final class LayaEmail {}cp++;}+ // An empty class is `[]` or `[^]`, which Java rejects -- and it would arrive as+ // ExceptionInInitializerError on the first call and NoClassDefFoundError on every one+ // after, naming neither the table nor this method. A table that lost its contents is a+ // generator regression, so it says so.+ if (space.length() == 1 || digit.length() == 1) {+ throw new IllegalStateException(+ "UnicodeTables gave an empty class: isSpace matched "+ + (space.length() == 1 ? "no" : "some") + " code points and isDigit matched "+ + (digit.length() == 1 ? "no" : "some")+ + "; regenerate with laya-java/scripts/gen_unicode_tables.py");+ }String spaces = space.append(']').toString();return new String[] {spaces, "[^" + spaces.substring(1), digit.append(']').toString()};}@@ -160,21 +171,20 @@ public final class LayaEmail {/*** The patterns, built on first use.*- * <p>A holder rather than fields of {@code LayaEmail}, because deriving the character classes- * from {@link UnicodeTables} walks all 1,114,112 code points and costs about 80 ms. That is a- * fair price for classes that cannot disagree with CPython, but only for a caller who cleans- * an email: paid in {@code LayaEmail}'s own initialiser it would land on every caller of the- * package, including one that only ever opens an {@link Agent}.+ * <p>A holder rather than fields of {@code LayaEmail}, because this costs about 55 ms --+ * one walk of all 1,114,112 code points to build the character classes, plus compiling the+ * patterns. That is a fair price for classes that cannot disagree with CPython, but only for+ * a caller who cleans an email: paid in {@code LayaEmail}'s own initialiser it would land on+ * every caller of the package, including one that only ever opens an {@link Agent}.*/private static final class Patterns {/*** {@code \s}, {@code \S} and {@code \d}, built in ONE walk of Unicode.*- * <p>Three calls to {@code classOf} is three walks of 1,114,112 code points, and this is- * the whole cost of the first call into this class. Measured on the first- * {@code cleanEmailBody}: 160 ms when the unused word class was still built here, about- * 100 ms with three walks, about 35 ms with one.+ * <p>One walk of 1,114,112 code points rather than one per class. The walk itself is+ * 15.8 ms warm; the first {@code cleanEmailBody} is 52 to 57 ms, most of it compiling+ * the patterns. It was about 160 ms when a fourth, unused class was built here too.*/private static final String[] CLASSES = classes();@@ -201,9 +211,6 @@ public final class LayaEmail {static final String BB = "\\b";static final String BA = "\\b";- /** Java's {@code \w}, probed rather than assumed, because it is the JDK's and not ours. */- static final Pattern JAVA_WORD =- Pattern.compile("\\w", Pattern.UNICODE_CHARACTER_CLASS);static final List<Pattern> QUOTE_HEADERS = List.of(compile("^" + S + "*On .{0,300}wrote:" + S + "*$"),@@ -231,7 +238,14 @@ public final class LayaEmail {// next line. That tail cuts too, and takes the `On/Em/El/Le ...` head it belongs to with// it. The French tail keeps its spaced colon.static final Pattern ATTRIBUTION_TAIL = compile(- "^.{0,120}" + NS + "@" + NS + "+" + S + "+(wrote|escreveu|escribi[óo]|a [eé]crit)"+ // `\S++` POSSESSIVE, not greedy. `.{0,120}` offers 121 starting points, and at+ // each one a greedy `\S+` runs to the end of the line and then gives every+ // character back, because the `\s+` after it can never match where a `\S` was+ // just surrendered -- the two classes are complements. The result was 121 full+ // scans of the line: measured 55 ms on a 4,000,000-character line at the default+ // budget and 4.8 s at `maxChars` of 300,000, against 0.018 ms for an ordinary+ // message. Possessive forbids the backtracking that could never have succeeded.+ "^.{0,120}" + NS + "@" + NS + "++" + S + "+(wrote|escreveu|escribi[óo]|a [eé]crit)"+ S + "*:" + S + "*$");static final Pattern ATTRIBUTION_HEAD = compile("^" + S + "*(On|Em|El|Le) (?=.*" + D + ")");@@ -314,27 +328,47 @@ public final class LayaEmail {+ BB + "(usage exclusif|exclusivement|uniquement)" + BA + "[^.]{0,30}destinataire)");/**- * A necessary condition for DISCLAIMER, checked first because it is 40 times cheaper.+ * A necessary condition for DISCLAIMER, checked first.+ *+ * <p>Each of DISCLAIMER's twelve top-level alternations requires at least one of these+ * literals, so a paragraph holding none of them cannot match and does not have to be+ * scanned by the full pattern. Eleven of the twelve carry word boundaries -- 26 of them+ * between them; the "received this ... in error" branch carries none.*- * <p>Every branch of DISCLAIMER requires at least one of these literals, so a paragraph- * holding none of them cannot match and does not have to be scanned by a pattern whose- * twenty alternations each carry two word-boundary lookarounds over a 10,000-character- * class. Measured: DISCLAIMER alone was 4.9 ms per case on the sweep corpus, which was- * the whole cost of cleaning a message.+ * <p>It is worth far less than it was. When `\b` was a lookaround over a+ * 10,622-character class this filter took DISCLAIMER from 4.9 ms per case to 0.018 ms and+ * was the difference between usable and not. Once `\b` became Java's own, DISCLAIMER+ * stopped being expensive, and this is now an ordinary literal alternation of the same+ * shape as the pattern it guards -- measured at parity with it on a paragraph holding no+ * literal, and worth 1.16x on the committed corpus and 1.4x on ordinary paragraphs. It+ * is kept for that, and because a cheap necessary condition in front of a complicated+ * sufficient one is the right shape, not because it is 40 times anything.** <p>This is a filter, never a decision: a hit still runs DISCLAIMER, so a false positive* costs only time. What would be a defect is a false NEGATIVE, so the literals are- * matched with the same flags as DISCLAIMER itself -- `re.I` folds more than lowercasing- * does, U+017F LATIN SMALL LETTER LONG S against `s` among it, and a filter that lowered- * the text instead would reject a paragraph the pattern accepts. EmailTest asserts the- * implication over every branch and over the whole corpus.+ * matched case-insensitively with Unicode folding, exactly as `re.I` on a str pattern+ * folds -- it folds more than lowercasing does, U+017F LATIN SMALL LETTER LONG S against+ * `s` among it, and a filter that lowered the text instead would reject a paragraph the+ * pattern accepts. DISCLAIMER additionally carries UNICODE_CHARACTER_CLASS, which it+ * needs for `\b` and this has no use for, having no class of its own.*/static final Pattern DISCLAIMER_HINT = compile(- "confidential|confidencia|sigilos|privilegiad|confidentiel|privil[eé]gi"+ // Each literal is the SHORTEST form that covers its branch, and every one of+ // them is load-bearing: `EmailTest` gives each a disclaimer whose only hint+ // literal is that one, and asserts that removing it stops the filter matching.+ // An earlier version listed the branches' own words -- `confidential`,+ // `recebeu|recebido|receber`, `uso exclusivo|exclusivamente|usage exclusif` --+ // and five of those were deletable with the suite still green, because every+ // example that reached the filter satisfied it through some OTHER literal.+ //+ // Shortening is always safe in this direction. The filter has to be a NECESSARY+ // condition for DISCLAIMER and nothing more, so a literal that matches more than+ // its branch costs time on a paragraph that then fails the real pattern; a+ // literal that matches less lets a disclaimer through to the model.+ "confidenc|confidenti|sigilos|privil[eé]gi"+ "|antes de imprimir|received this"- + "|recebeu|recebido|receber|ha recibido|recibi[óo]|recibe|avez re[çc]u"- + "|uso exclusivo|exclusivamente|[úu]nicamente|usage exclusif|exclusivement"- + "|uniquement");+ + "|receb|recib|avez re[çc]u"+ + "|exclusiv|[úu]nicamente|uniquement");static final Pattern SENTENCE = compile("(?<=[.!?])" + S + "+");static final Pattern BLANK_LINE = compile("\n" + S + "*\n");@@ -360,9 +394,11 @@ public final class LayaEmail {* <p>Two substitutes, chosen so a replacement cannot change which literal matches:* <ul>* <li>Java says word, Python says not: U+0000, which Java also calls a non-word character.- * <li>Python says word, Java says not: {@code '0'}, which Java calls a word character. No- * literal in any pattern here contains a digit, so a digit can neither complete nor- * break one.+ * <li>Python says word, Java says not: {@code '0'}, which Java calls a word character.+ * DISCLAIMER is the only pattern matched against the view, and no literal in it+ * contains a digit, so a digit can neither complete nor break one. (Other patterns do+ * hold digits -- DEVICE_FOOTER's {@code [a-z0-9_.+-]}, HEADER_NEXT's four-digit year --+ * and none of them ever sees this string.)* </ul>* Both substitutes are matched by the {@code [^.]} windows the patterns measure, so a window* counts the same number of characters as it does in the original.@@ -408,7 +444,7 @@ public final class LayaEmail {* <p>Spelled out from the definition {@code java.util.regex.Pattern} documents for* {@code \w} under {@code UNICODE_CHARACTER_CLASS} -- alphabetic, the three mark categories,* decimal digits, connector punctuation and the join controls -- rather than asked of a- * {@link Patterns#JAVA_WORD} matcher, which allocates a string and a matcher per character+ * a matcher on Java's own {@code \\w}, which allocates a string and a matcher per character* and cost more than the rest of the cleaner put together. Every term reads the JDK's own* Unicode, so this stays whatever that JDK's {@code \w} is, which is the point: the view is* built by comparing the JDK against the recorded table, so this side has to be the JDK's@@ -642,8 +678,8 @@ public final class LayaEmail {// alternations whose cost grows with input length, and only maxChars are ever returned.//// In CODE POINTS, because Python slices a str by code point. Cutting by char index splits- // a surrogate pair, and the replacement character that comes out is not what the- // reference returns -- measured on 4,641 cases of an astral corpus.+ // a surrogate pair, and the half character that comes out is not what the reference+ // returns -- 4,641 of 20,000 astral bodies in a one-off differential sweep.if (maxChars > 0) {text = cutToCodePoints(text, (long) maxChars * 4);} else {@@ -663,7 +699,14 @@ public final class LayaEmail {&& Patterns.HEADER_NEXT.matcher(source[i + 1]).lookingAt()) {break;}- if (Patterns.ATTRIBUTION_TAIL.matcher(line).lookingAt() && !lines.isEmpty()) {+ // `!lines.isEmpty()` and the trailing colon first, because both are necessary and+ // both are O(1) against a pattern whose cost grows with the line. The colon really is+ // necessary: the pattern ends `\s*:\s*$`, a line split on "\n" holds no newline,+ // and UNIX_LINES makes `$` the end of input -- so a tail that matches is a line whose+ // rstrip ends with a colon. Together with the possessive quantifier above this is+ // 55 ms to 11 ms at the default budget, and 4.8 s to 0.1 s at 300,000.+ if (!lines.isEmpty() && stripTrailing(line).endsWith(":")+ && Patterns.ATTRIBUTION_TAIL.matcher(line).lookingAt()) {if (Patterns.ATTRIBUTION_HEAD.matcher(lines.get(lines.size() - 1)).lookingAt()) {lines.remove(lines.size() - 1);}@@ -719,8 +762,11 @@ public final class LayaEmail {* takes.** <p>Never cuts between a high and a low surrogate: a half pair is not a character the- * reference can return, and the budget cases of the astral corpus are 4,641 proofs that- * cutting by char index returns one.+ * reference can return. Cutting by char index returned one, and a one-off differential sweep+ * of 20,000 astral bodies against the reference disagreed on 4,641 of them for this reason+ * alone. That sweep is not in the tree -- generated corpora are not committed here -- so the+ * cases that hold the line are the four named {@code budget-*} entries of+ * {@code fixtures/email.json}, each of which fails if this reverts to a char index.*/private static String cutToCodePoints(String text, long limit) {if (limit <= 0) {@@ -796,8 +842,12 @@ public final class LayaEmail {* behaviour and is why it is not validated. A {@code null} value is dropped, as the reference* drops it.*- * @return a mutable, insertion-ordered map: {@code subject}, {@code body}, then {@code from}- * if a sender was given, then the extras+ * @return a mutable, insertion-ordered map: {@code subject}, {@code body}, then+ * {@code from} if a sender was given, then the extras. An extra named+ * {@code subject}, {@code body} or {@code from} REPLACES that value and keeps the+ * original position, as Python's {@code dict.update} does; a null key is permitted,+ * again as the reference permits it. Both are input mutations rather than errors,+ * which is the reference's choice and is why neither is validated.*/public static Map<String, Object> emailState(String subject, String body, String sender,boolean clean, int maxChars,diff --git a/laya-java/laya-java/src/main/java/com/convaiinnovations/laya/lang/UnicodeTables.java b/laya-java/laya-java/src/main/java/com/convaiinnovations/laya/lang/UnicodeTables.javaindex b619fbe9..34ce49f0 100644--- a/laya-java/laya-java/src/main/java/com/convaiinnovations/laya/lang/UnicodeTables.java+++ b/laya-java/laya-java/src/main/java/com/convaiinnovations/laya/lang/UnicodeTables.java@@ -1421,6 +1421,11 @@ sign-off's tail, and what the other ports spell {@code \p{M}}.* code points that are category M with class zero, so the two are not interchangeable here.*/public static boolean isMark(int codePoint) {+ if (codePoint < 0x80) {+ long mask = codePoint < 64 ? MARK_ASCII_LOW >>> codePoint+ : MARK_ASCII_HIGH >>> (codePoint - 64);+ return (mask & 1L) != 0L;+ }return contains(MARK, codePoint);}@@ -1431,6 +1436,11 @@ sign-off's tail, and what the other ports spell {@code \p{M}}.* which holds caseless symbols this does not and misses every caseless script this does.*/public static boolean isInitial(int codePoint) {+ if (codePoint < 0x80) {+ long mask = codePoint < 64 ? INITIAL_ASCII_LOW >>> codePoint+ : INITIAL_ASCII_HIGH >>> (codePoint - 64);+ return (mask & 1L) != 0L;+ }return contains(INITIAL, codePoint);}diff --git a/laya-java/laya-java/src/test/java/com/convaiinnovations/laya/EmailTest.java b/laya-java/laya-java/src/test/java/com/convaiinnovations/laya/EmailTest.javaindex 36cdb262..e7d48583 100644--- a/laya-java/laya-java/src/test/java/com/convaiinnovations/laya/EmailTest.java+++ b/laya-java/laya-java/src/test/java/com/convaiinnovations/laya/EmailTest.java@@ -2,6 +2,8 @@ package com.convaiinnovations.laya;import static org.junit.jupiter.api.Assertions.assertEquals;import static org.junit.jupiter.api.Assertions.assertFalse;+import static org.junit.jupiter.api.Assertions.assertNotEquals;+import static org.junit.jupiter.api.Assertions.assertNotSame;import static org.junit.jupiter.api.Assertions.assertSame;import static org.junit.jupiter.api.Assertions.assertTrue;import static org.junit.jupiter.api.DynamicTest.dynamicTest;@@ -30,17 +32,80 @@ import org.junit.jupiter.api.TestFactory;*/final class EmailTest {+ /**+ * Cases the corpus must hold by NAME, not by count.+ *+ * <p>A count is not coverage. Trimmed to its old floors -- which were 70 against 86 and 5+ * against 9 -- this corpus still passed while losing every astral case and eleven+ * near-misses, and three mutants that the full corpus kills then survived: the code-point+ * width gate, the surrogate-safe cut and the UNICODE_CHARACTER_CLASS flag on `\b`. Each+ * name below is a rule nothing else in the corpus pins.+ */+ private static final List<String> REQUIRED_CLEANED = List.of(+ // near-misses: the cleaner must NOT fire. More of these than hits, because a cleaner+ // that is too eager deletes what the sender actually wrote.+ "not-signoff-pt-with-name", "not-signoff-sentence",+ "not-signoff-lowercase-other-script", "not-signoff-symbol-name", "not-signoff-zwnj",+ "not-signoff-obrigado-mas", "not-signoff-merci-mais", "not-footer-with-request",+ "not-disclaimer-question", "not-disclaimer-colon", "not-disclaimer-print",+ "not-disclaimer-contract", "not-disclaimer-fr-contract",+ "not-signoff-baseless-mark", "not-signoff-too-wide", "not-signoff-four-tokens",+ // outside the BMP, where the reference counts code points and Java reaches for chars+ "astral-word-char-breaks-no-boundary", "astral-elsewhere-still-matches",+ "astral-signoff-at-the-width-limit", "astral-signoff-just-over-the-limit",+ "astral-mark-in-a-disclaimer",+ // the signature window's five parameters, the footer width in both directions, and+ // the body-side strip -- each the only case that tells its rule from a plausible+ // wrong one+ "window-floor-of-one", "window-offset-dominates", "window-offset-excludes",+ "window-ratio-excludes", "window-ratio-includes",+ "footer-at-the-width-limit", "footer-just-over-the-limit",+ "body-strip-is-pythons", "body-strip-both-ends");++ private static final List<String> REQUIRED_BUDGETS = List.of(+ "budget-four-x-bound", "budget-splits-a-surrogate-pair",+ "budget-ends-on-a-surrogate-pair", "budget-all-astral",+ "budget-four-x-bound-astral");+@SuppressWarnings("unchecked")private static List<Object> rows(String key) {return (List<Object>) Fixtures.load("email.json").get(key);}+ private static List<String> names(String key) {+ List<String> out = new ArrayList<>();+ for (Object entry : rows(key)) {+ out.add((String) asMap(entry).get("name"));+ }+ return out;+ }++ @Test+ @DisplayName("the corpus still holds every case that pins a rule nothing else pins")+ void corpusHoldsItsLoadBearingCases() {+ List<String> cleaned = names("cleaned");+ for (String required : REQUIRED_CLEANED) {+ assertTrue(cleaned.contains(required),+ () -> "the corpus lost " + required + ", which is the only case pinning its "+ + "rule; regenerate from a gen_fixtures.py that still has it");+ }+ List<String> budgets = names("budgets");+ for (String required : REQUIRED_BUDGETS) {+ assertTrue(budgets.contains(required),+ () -> "the corpus lost the budget case " + required);+ }+ assertEquals(95, cleaned.size(), "cases were added or removed; update the count knowingly");+ assertEquals(9, budgets.size(), "budget cases were added or removed");+ assertEquals(7, names("states").size(), "state cases were added or removed");+ }+@TestFactory@DisplayName("every cleaned body is byte-identical to the reference")List<DynamicTest> cleanedMatches() {List<Object> cases = rows("cleaned");- assertTrue(cases.size() >= 70,- () -> "the email corpus lost cases: " + cases.size() + " of at least 70");+ assertEquals(95, cases.size(),+ "the email corpus changed size; corpusHoldsItsLoadBearingCases says which "+ + "cases may not go");List<DynamicTest> tests = new ArrayList<>();for (Object entry : cases) {Map<String, Object> row = asMap(entry);@@ -56,7 +121,7 @@ final class EmailTest {@DisplayName("every budget cuts where the reference cuts")List<DynamicTest> budgetsMatch() {List<Object> cases = rows("budgets");- assertTrue(cases.size() >= 5, "the budget cases vanished");+ assertEquals(9, cases.size(), "the budget cases changed size");List<DynamicTest> tests = new ArrayList<>();for (Object entry : cases) {Map<String, Object> row = asMap(entry);@@ -74,7 +139,7 @@ final class EmailTest {@SuppressWarnings("unchecked")List<DynamicTest> statesMatch() {List<Object> cases = rows("states");- assertTrue(cases.size() >= 7, "the state cases vanished");+ assertEquals(7, cases.size(), "the state cases changed size");List<DynamicTest> tests = new ArrayList<>();for (Object entry : cases) {Map<String, Object> row = asMap(entry);@@ -96,13 +161,44 @@ final class EmailTest {}@Test- @DisplayName("the question set is the same object graph as the preset")+ @DisplayName("the re-export is the preset, and hands out a fresh map each time")void questionsAreThePreset() {- assertEquals(Presets.email().keySet(), LayaEmail.emailQuestions().keySet());+ // The DisplayName used to say "the same object graph", which is the opposite of true and+ // of what is wanted: `Presets.email()` builds a fresh map, so a caller cannot corrupt a+ // later caller's questions. Comparing two key sets could not tell the difference.+ Map<String, Question> first = LayaEmail.emailQuestions();+ Map<String, Question> second = LayaEmail.emailQuestions();+ assertEquals(Presets.email().keySet(), first.keySet());+ assertNotSame(first, second, "each call must hand out its own map");+ first.remove(first.keySet().iterator().next());+ assertEquals(Presets.email().keySet(), LayaEmail.emailQuestions().keySet(),+ "mutating a returned map changed what a later caller gets");assertEquals(Boolean.TRUE, Fixtures.load("email.json").get("questions_match_presets"),"the reference's re-export no longer equals laya.presets.email_questions");}+ @Test+ @DisplayName("the categories overload passes the caller's categories through")+ void questionsWithCategories() {+ // Untested until a review pointed it out: `Presets.email(categories)` could have been+ // `Presets.email(null)` -- silently ignoring the argument -- with nothing failing.+ Map<String, String> categories = new LinkedHashMap<>();+ categories.put("refund", "the sender wants money back");+ categories.put("access", "the sender cannot log in");+ Map<String, Question> questions = LayaEmail.emailQuestions(categories);+ assertEquals(Presets.email(categories).keySet(), questions.keySet());+ // The labels, not toString(): `Question` has no toString, so comparing those compared+ // identity hashes and was true however the method behaved. The first version of this+ // test did exactly that and the mutant that drops the argument survived it.+ assertEquals(List.of("refund", "access"), questions.get("category").labels(),+ "the overload returned the DEFAULT questions, so the categories were dropped");+ assertEquals(Presets.email(categories).get("category").renderOptions(),+ questions.get("category").renderOptions());+ assertNotEquals(Presets.email().get("category").labels(),+ questions.get("category").labels(),+ "the caller's categories are indistinguishable from the defaults");+ }+// ------------------------------------------------------------------ the sign-off rule@Test@@ -110,12 +206,12 @@ final class EmailTest {void signoffInitials() {// Each of these is a rule UPPER would get wrong; see UnicodeTablesTest for the counts.assertTrue(LayaEmail.isEnglishSignoff("Regards, Ana"));- assertTrue(LayaEmail.isEnglishSignoff("Regards, Łukasz"), "Lu outside Latin-1");- assertTrue(LayaEmail.isEnglishSignoff("Regards, 山田"), "Lo: a caseless script");- assertTrue(LayaEmail.isEnglishSignoff("Regards, Džarko"), "Lt: titlecase");- assertFalse(LayaEmail.isEnglishSignoff("Thanks, Ⓐ"),+ assertTrue(LayaEmail.isEnglishSignoff("Regards, \u0141ukasz"), "Lu outside Latin-1");+ assertTrue(LayaEmail.isEnglishSignoff("Regards, \u5C71\u7530"), "Lo: a caseless script");+ assertTrue(LayaEmail.isEnglishSignoff("Regards, \u01C5arko"), "Lt: titlecase");+ assertFalse(LayaEmail.isEnglishSignoff("Thanks, \u24B6"),"U+24B6 is Uppercase but category So, so it is not a letter");- assertFalse(LayaEmail.isEnglishSignoff("Thanks, żaneta"), "a lowercase name is prose");+ assertFalse(LayaEmail.isEnglishSignoff("Thanks, \u017Caneta"), "a lowercase name is prose");assertFalse(LayaEmail.isEnglishSignoff("Thanks for the quick reply."));assertFalse(LayaEmail.isEnglishSignoff("Regards, Ana Maria Souza Lima"),"the tail allows at most three tokens");@@ -124,16 +220,16 @@ final class EmailTest {@Test@DisplayName("a mark rides on its base, and a mark without one still separates tokens")void marksAreDroppedOnlyWithABase() {- assertEquals("Jose", LayaEmail.dropMarks("José"), "a mark with a base goes");- assertEquals("Jose", LayaEmail.dropMarks("Jo͏se"),+ assertEquals("Jose", LayaEmail.dropMarks("Jose\u0301"), "a mark with a base goes");+ assertEquals("Jose", LayaEmail.dropMarks("Jo\u034Fse"),"including U+034F, whose canonical combining class is zero");- assertEquals("́Ana", LayaEmail.dropMarks("́Ana"),+ assertEquals("\u0301Ana", LayaEmail.dropMarks("\u0301Ana"),"a mark opening the string has no base");- assertEquals("a ́b", LayaEmail.dropMarks("a ́b"),+ assertEquals("a \u0301b", LayaEmail.dropMarks("a \u0301b"),"nor does one following a space");// and the consequence, which is the reason the rule exists- assertTrue(LayaEmail.isEnglishSignoff("Regards, José"));- assertFalse(LayaEmail.isEnglishSignoff("Regards, ́Ana"),+ assertTrue(LayaEmail.isEnglishSignoff("Regards, Jose\u0301"));+ assertFalse(LayaEmail.isEnglishSignoff("Regards, \u0301Ana"),"a kept mark is not a token opener, so this is not a sign-off");}@@ -160,7 +256,17 @@ final class EmailTest {assertEquals(pattern, spelled, String.format("U+%04X", cp));}}- assertEquals(0x110000 - 2048, examined, "every code point but the surrogates");+ // Not asserted against a constant the loop itself produces -- that only restates the+ // loop. What matters is that the comparison ran on a lot of code points and on the ones+ // where the two classes are known to disagree.+ final int compared = examined;+ assertTrue(compared > 1_000_000, () -> "only " + compared + " code points examined");+ for (int codePoint : new int[] {0x0301, 0x200C, 0x200D, 0x00BD, 0x2160, 0x11001, 0x10107}) {+ final int cp = codePoint;+ boolean spelled = (Boolean) javaWord.invoke(null, cp);+ assertEquals(word.matcher(new String(Character.toChars(cp))).matches(), spelled,+ () -> String.format("U+%04X, a code point the two word classes argue about", cp));+ }}@Test@@ -168,18 +274,24 @@ final class EmailTest {void boundaryViewSubstitutes() {assertSame("plain ascii", LayaEmail.boundaryView("plain ascii"),"an ASCII-only message is returned without copying");- assertSame("café résumé", LayaEmail.boundaryView("café résumé"),+ assertSame("caf\u00E9 r\u00E9sum\u00E9", LayaEmail.boundaryView("caf\u00E9 r\u00E9sum\u00E9"),"ordinary accented letters are word characters in both, so nothing is rewritten");// Java says word, Python says not -> U+0000, a non-word character to Java too.- assertEquals("a b", LayaEmail.boundaryView("áb"), "a combining mark");- assertEquals("a b", LayaEmail.boundaryView("ab"), "ZWNJ, a join control");- assertEquals("a b", LayaEmail.boundaryView("ab"), "ZWJ");+ assertEquals("a\u0000b", LayaEmail.boundaryView("a\u0301b"), "a combining mark");+ assertEquals("a\u0000b", LayaEmail.boundaryView("a\u200Cb"), "ZWNJ, a join control");+ assertEquals("a\u0000b", LayaEmail.boundaryView("a\u200Db"), "ZWJ");// Python says word, Java says not -> '0', which no literal in any pattern contains.- assertEquals("a0b", LayaEmail.boundaryView("a½b"), "U+00BD is No: Python's word, not Java's");- // Length is preserved, so the [^.]{0,N} windows count the same characters.- for (String text : List.of("áb", "ab", "a½b", "x́½y")) {- assertEquals(text.length(), LayaEmail.boundaryView(text).length(),- () -> "boundaryView changed the length of " + text);+ assertEquals("a0b", LayaEmail.boundaryView("a\u00BDb"), "U+00BD is No: Python's word, not Java's");+ // CODE POINT count is preserved, which is the unit a regex quantifier counts in -- and+ // not the char count, which an astral disagreement does change: U+1D16D is two chars and+ // one code point, and its substitute is one of each. Asserting chars passed only because+ // every case in the list was in the BMP.+ for (String text : List.of("a\u0301b", "a\u200Cb", "a\u00BDb", "x\u0301\u00BD\u200Cy",+ "a\uD834\uDD6Db", "a\uD800\uDD07b")) {+ String view = LayaEmail.boundaryView(text);+ assertEquals(text.codePointCount(0, text.length()),+ view.codePointCount(0, view.length()),+ () -> "boundaryView changed the code point count of " + text);}}@@ -187,32 +299,107 @@ final class EmailTest {@DisplayName("neither substitute can appear in a pattern literal")void substitutesAreNotLiterals() throws Exception {// The whole argument for substituting is that the replacement cannot complete or break a- // literal. That holds only while no pattern here holds a digit or a NUL as a literal, so- // it is checked rather than asserted in a comment.+ // literal. Checked BEHAVIOURALLY: an earlier version scanned the pattern source for a+ // raw NUL, which no pattern can hold -- `appendEscaped` writes every class member as a+ // backslash-u escape -- while the two forms a NUL literal actually takes in a Java+ // regex, backslash-x-zero-zero and backslash-u-four-zeros, both read as ordinary text to+ // `indexOf`. So it could not fail from any plausible edit. (Spelled out rather than+ // written, because the compiler translates a backslash-u escape before it lexes, even+ // inside a comment, and an invalid one there is a compile error.)+ String nul = "\u0000";for (Pattern pattern : allPatterns()) {- String source = pattern.pattern();- assertFalse(source.indexOf(' ') >= 0,- () -> "a pattern holds a literal NUL: " + source.substring(0, 40));+ assertFalse(pattern.matcher(nul).find(),+ () -> "this pattern matches a lone NUL, so U+0000 is no longer a safe "+ + "substitute: " + head(pattern.pattern()));}- // A digit may only appear inside the generated \d class or as a quantifier bound, never- // as a literal to be matched, so the DISCLAIMER branches are checked directly.- Pattern disclaimer = (Pattern) patternField("DISCLAIMER");- assertFalse(disclaimer.pattern().matches(".*[^{,0-9]\\d[^}0-9,].*"),+ // And the digit side, for DISCLAIMER, which is the only pattern the view ever reaches.+ // Quantifier bounds are stripped first: `[^.]{0,60}` is not a digit literal, and a check+ // that did not strip them could not see `covid19` either, because every digit there is+ // next to another one.+ String disclaimer = ((Pattern) patternField("DISCLAIMER")).pattern()+ .replaceAll("\\{\\d+(,\\d+)?\\}", "");+ assertFalse(disclaimer.matches(".*\\d.*"),"a digit became a literal in DISCLAIMER; '0' is no longer a safe substitute");+ // the guard above is only meaningful if it would notice one+ assertTrue(("x" + disclaimer + "7").matches(".*\\d.*"),+ "the digit check cannot see a digit, so it proves nothing");+ }++ /** A pattern's opening, for a failure message, without risking its length. */+ private static String head(String source) {+ return source.substring(0, Math.min(40, source.length()));}@Test- @DisplayName("the disclaimer filter never rejects what the pattern would accept")- void hintIsANecessaryCondition() throws Exception {+ @DisplayName("every literal in the disclaimer filter is load-bearing")+ void everyHintLiteralIsNecessary() throws Exception {+ Pattern hint = (Pattern) patternField("DISCLAIMER_HINT");+ Pattern disclaimer = (Pattern) patternField("DISCLAIMER");++ // One disclaimer per literal, chosen so that literal is the ONLY one the filter can+ // match it by. That is what makes each row a test of necessity rather than of+ // membership: the previous version of this test checked thirteen examples against the+ // whole filter, every example was satisfied by SOME literal, and five literals could be+ // deleted with the suite green -- leaking "Esta mensagem e sigilosa.", "Destinada+ // exclusivamente ao destinatario." and "Reserve uniquement au destinataire." into the+ // model's input.+ String[][] perLiteral = {+ {"confidenc", "Esta mensagem e confidencial."},+ {"confidenti", "This email is confidential and intended solely for the addressee."},+ {"sigilos", "Esta mensagem e sigilosa."},+ {"privil[e\u00e9]gi", "Ce message est privilegie."},+ {"antes de imprimir", "Antes de imprimir pense no meio ambiente."},+ {"received this", "If you have received this email in error please delete it."},+ {"receb", "Se recebeu esta mensagem por engano, apague."},+ {"recib", "Usted ha recibido este mensaje por error."},+ {"avez re[\u00e7c]u", "Vous avez recu ce message par erreur."},+ {"exclusiv", "Uso exclusivo do destinatario desta mensagem."},+ {"[\u00fau]nicamente", "Unicamente ao destinatario desta mensagem."},+ {"uniquement", "Reserve uniquement au destinataire."},+ };+ List<String> alternatives = List.of(hint.pattern().split("\\|"));+ assertEquals(perLiteral.length, alternatives.size(),+ () -> "the filter has " + alternatives.size() + " literals and this test names "+ + perLiteral.length + "; a literal with no example is a literal that can "+ + "be deleted without failing anything");++ for (String[] row : perLiteral) {+ String literal = row[0];+ String example = row[1];+ assertTrue(alternatives.contains(literal),+ () -> "the filter no longer holds the literal " + literal);+ // the example really is a disclaimer, so leaking it would matter+ assertTrue(disclaimer.matcher(LayaEmail.boundaryView(example)).find(),+ () -> "this is no longer a disclaimer at all: " + example);+ assertTrue(hint.matcher(example).find(),+ () -> "the filter rejects a disclaimer the pattern accepts: " + example);+ // and WITHOUT this literal the filter misses it, which is what "necessary" means+ StringBuilder reduced = new StringBuilder();+ for (String alternative : alternatives) {+ if (alternative.equals(literal)) {+ continue;+ }+ if (reduced.length() > 0) {+ reduced.append('|');+ }+ reduced.append(alternative);+ }+ Pattern without = Pattern.compile(reduced.toString(),+ Pattern.CASE_INSENSITIVE | Pattern.UNICODE_CASE | Pattern.UNIX_LINES);+ assertFalse(without.matcher(example).find(),+ () -> "deleting " + literal + " from the filter would still catch "+ + example + ", so this row does not test that literal");+ }+ }++ @Test+ @DisplayName("the filter rejects nothing the pattern accepts, over the whole corpus")+ void hintIsNecessaryOverTheCorpus() throws Exception {Pattern hint = (Pattern) patternField("DISCLAIMER_HINT");Pattern disclaimer = (Pattern) patternField("DISCLAIMER");- // A filter is only sound as a NECESSARY condition: a false positive costs time, a false- // negative keeps a disclaimer in the model's input. Checked over every recorded case,- // which includes one example of every branch of the pattern.int fired = 0;for (Object entry : rows("cleaned")) {- Map<String, Object> row = asMap(entry);- String body = (String) row.get("body");+ String body = (String) asMap(entry).get("body");for (String raw : body.split("\n", -1)) {final String paragraph = raw;if (disclaimer.matcher(LayaEmail.boundaryView(paragraph)).find()) {@@ -223,34 +410,13 @@ final class EmailTest {}}}+ // A secondary check. `everyHintLiteralIsNecessary` is the coverage gate -- it proves each+ // literal is load-bearing -- and this one proves the implication holds on real bodies as+ // well as on the twelve constructed ones.final int hits = fired;- assertTrue(hits >= 9,- () -> "the corpus no longer exercises the disclaimer pattern: " + hits + " hits");-- // The corpus count is a floor, not coverage. Every BRANCH of the pattern is named here,- // so a branch whose required literal is missing from the filter fails this rather than- // waiting for a message in production to be the first to notice.- String[] oneExamplePerBranch = {- "This email and its contents are confidential and intended solely for the addressee.",- "This message is confidential and may also privileged.",- "If you have received this email in error please delete it.",- "Esta mensagem e seus anexos sao confidenciais e de uso exclusivo do destinatario.",- "Este mensaje es confidencial y para uso exclusivo del destinatario.",- "Uso exclusivo do destinatario desta mensagem.",- "Se recebeu esta mensagem por engano, apague-a.",- "Usted ha recibido este mensaje por error.",- "Antes de imprimir pense no meio ambiente.",- "Pense no meio ambiente antes de imprimir.",- "Ce message est confidentiel et destine uniquement au destinataire.",- "Vous avez recu ce message par erreur.",- "Usage exclusif du destinataire.",- };- for (String example : oneExamplePerBranch) {- assertTrue(disclaimer.matcher(LayaEmail.boundaryView(example)).find(),- () -> "this no longer matches DISCLAIMER at all: " + example);- assertTrue(hint.matcher(example).find(),- () -> "the filter would reject a disclaimer the pattern accepts: " + example);- }+ assertEquals(11, hits,+ "the corpus's disclaimer coverage changed; this is a count, so change it "+ + "knowingly rather than lowering it");}// ------------------------------------------------------------------------- the state@@ -295,7 +461,7 @@ final class EmailTest {void subjectStripIsPythons() {// U+00A0 and U+2007 are whitespace to str.strip() and not to String.trim(). The same gap// sat in the router's checkpoint names and routed a caller to the wrong model.- assertEquals("Refund", LayaEmail.emailState(" Refund ", "b").get("subject"));+ assertEquals("Refund", LayaEmail.emailState("\u00A0Refund\u2007", "b").get("subject"));assertTrue(UnicodeTables.isSpace(0x00a0) && UnicodeTables.isSpace(0x2007),"the premise of the assertion above");}diff --git a/laya-java/scripts/gen_fixtures.py b/laya-java/scripts/gen_fixtures.pyindex a5c1b466..54a03c2d 100644--- a/laya-java/scripts/gen_fixtures.py+++ b/laya-java/scripts/gen_fixtures.py@@ -1895,6 +1895,32 @@ def email_clean():("header-next-devanagari-year",q + "\n\nFrom: Maria Souza\nDate: 3 Sep \u0968\u0966\u0968\u096B\nolder"),+ # -- the signature window, `max(1, min(int(len * 0.6), len - 8))`. Every part of that+ # formula was unpinned: a review mutated the ratio to 0.55 and to 0.9, the offset to 9+ # and to 2, and the floor to 2, and all five left the suite green. These five cases are+ # the smallest inputs under 60 lines on which each mutant's window reaches a different+ # line from the real one -- found by searching, not derived, because deriving them by+ # hand gets the `min` the wrong way round.+ ("window-floor-of-one", "\n".join(["Regards, Ana" if i == 1 else q for i in range(2)])),+ ("window-offset-dominates", "\n".join(["Regards, Ana" if i == 1 else q for i in range(4)])),+ ("window-offset-excludes", "\n".join(["Regards, Ana" if i == 1 else q for i in range(10)])),+ ("window-ratio-excludes", "\n".join(["Regards, Ana" if i == 9 else q for i in range(18)])),+ ("window-ratio-includes", "\n".join(["Regards, Ana" if i == 12 else q for i in range(21)])),++ # -- the device-footer width gate, 60 code points against the sign-off's 40. Both+ # directions, because a review found `<= 60` mutable to 41 and to 200 with nothing+ # failing.+ ("footer-at-the-width-limit",+ "\n".join([q] * 9 + ["Sent from my iphone using " + "a" * 34])),+ ("footer-just-over-the-limit",+ "\n".join([q] * 9 + ["Sent from my iphone using " + "a" * 35])),++ # -- the body-side strip is Python's `str.strip()`, which takes U+00A0 and U+2007 where+ # `String.trim()` does not. The subject side was pinned and the body side was not, and+ # swapping it for `trim()` left the suite green while returning a leading space.+ ("body-strip-is-pythons", "\u00a0Refund please\u2007"),+ ("body-strip-both-ends", "\u2007\u00a0Please refund\u00a0"),+# -- outside the BMP. The first sweep of this port had no astral character in it and# passed on 40,000 cases; a corpus of them found four defects at once. Each case below# is one of them, because the units the reference counts are CODE POINTS and the unitsdiff --git a/laya-java/scripts/gen_unicode_tables.py b/laya-java/scripts/gen_unicode_tables.pyindex 603038e9..126db7b4 100755--- a/laya-java/scripts/gen_unicode_tables.py+++ b/laya-java/scripts/gen_unicode_tables.py@@ -425,6 +425,11 @@ public final class UnicodeTables {* code points that are category M with class zero, so the two are not interchangeable here.*/public static boolean isMark(int codePoint) {+ if (codePoint < 0x80) {+ long mask = codePoint < 64 ? MARK_ASCII_LOW >>> codePoint+ : MARK_ASCII_HIGH >>> (codePoint - 64);+ return (mask & 1L) != 0L;+ }return contains(MARK, codePoint);}@@ -435,6 +440,11 @@ public final class UnicodeTables {* which holds caseless symbols this does not and misses every caseless script this does.*/public static boolean isInitial(int codePoint) {+ if (codePoint < 0x80) {+ long mask = codePoint < 64 ? INITIAL_ASCII_LOW >>> codePoint+ : INITIAL_ASCII_HIGH >>> (codePoint - 64);+ return (mask & 1L) != 0L;+ }return contains(INITIAL, codePoint);}