Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

fix(java): what the adversarial review of the email port found

Adeshina Lasisic6fbc970b44f
diff --git a/laya-java/fixtures/email.json b/laya-java/fixtures/email.json
index 178ebf0d..c2cd66de 100644
--- a/laya-java/fixtures/email.json
+++ b/laya-java/fixtures/email.json
@@ -400,6 +400,51 @@
"body": "I cannot log in and need a password reset.\n\nFrom: Maria Souza\nDate: 3 Sep २०२५\nolder",
"result": "I cannot log in and need a password reset."
},
+ {
+ "name": "window-floor-of-one",
+ "body": "I cannot log in and need a password reset.\nRegards, Ana",
+ "result": "I cannot log in and need a password reset."
+ },
+ {
+ "name": "window-offset-dominates",
+ "body": "I cannot log in and need a password reset.\nRegards, Ana\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.",
+ "result": "I cannot log in and need a password reset."
+ },
+ {
+ "name": "window-offset-excludes",
+ "body": "I cannot log in and need a password reset.\nRegards, Ana\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.",
+ "result": "I cannot log in and need a password reset.\nRegards, Ana\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset."
+ },
+ {
+ "name": "window-ratio-excludes",
+ "body": "I cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nRegards, Ana\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.",
+ "result": "I cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nRegards, Ana\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset."
+ },
+ {
+ "name": "window-ratio-includes",
+ "body": "I cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nRegards, Ana\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.",
+ "result": "I cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset."
+ },
+ {
+ "name": "footer-at-the-width-limit",
+ "body": "I cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nSent from my iphone using aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
+ "result": "I cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset."
+ },
+ {
+ "name": "footer-just-over-the-limit",
+ "body": "I cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nSent from my iphone using aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
+ "result": "I cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nI cannot log in and need a password reset.\nSent from my iphone using aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
+ },
+ {
+ "name": "body-strip-is-pythons",
+ "body": " Refund please ",
+ "result": "Refund please"
+ },
+ {
+ "name": "body-strip-both-ends",
+ "body": "  Please refund ",
+ "result": "Please refund"
+ },
{
"name": "astral-word-char-breaks-no-boundary",
"body": "Esta mensagem𒐲 e confidencial.",
diff --git a/laya-java/laya-java/src/main/java/com/convaiinnovations/laya/LayaEmail.java b/laya-java/laya-java/src/main/java/com/convaiinnovations/laya/LayaEmail.java
index 512d1fce..03243417 100644
--- a/laya-java/laya-java/src/main/java/com/convaiinnovations/laya/LayaEmail.java
+++ b/laya-java/laya-java/src/main/java/com/convaiinnovations/laya/LayaEmail.java
@@ -95,6 +95,17 @@ public final class LayaEmail {
}
cp++;
}
+ // An empty class is `[]` or `[^]`, which Java rejects -- and it would arrive as
+ // ExceptionInInitializerError on the first call and NoClassDefFoundError on every one
+ // after, naming neither the table nor this method. A table that lost its contents is a
+ // generator regression, so it says so.
+ if (space.length() == 1 || digit.length() == 1) {
+ throw new IllegalStateException(
+ "UnicodeTables gave an empty class: isSpace matched "
+ + (space.length() == 1 ? "no" : "some") + " code points and isDigit matched "
+ + (digit.length() == 1 ? "no" : "some")
+ + "; regenerate with laya-java/scripts/gen_unicode_tables.py");
+ }
String spaces = space.append(']').toString();
return new String[] {spaces, "[^" + spaces.substring(1), digit.append(']').toString()};
}
@@ -160,21 +171,20 @@ public final class LayaEmail {
/**
* The patterns, built on first use.
*
- * <p>A holder rather than fields of {@code LayaEmail}, because deriving the character classes
- * from {@link UnicodeTables} walks all 1,114,112 code points and costs about 80 ms. That is a
- * fair price for classes that cannot disagree with CPython, but only for a caller who cleans
- * an email: paid in {@code LayaEmail}'s own initialiser it would land on every caller of the
- * package, including one that only ever opens an {@link Agent}.
+ * <p>A holder rather than fields of {@code LayaEmail}, because this costs about 55 ms --
+ * one walk of all 1,114,112 code points to build the character classes, plus compiling the
+ * patterns. That is a fair price for classes that cannot disagree with CPython, but only for
+ * a caller who cleans an email: paid in {@code LayaEmail}'s own initialiser it would land on
+ * every caller of the package, including one that only ever opens an {@link Agent}.
*/
private static final class Patterns {
/**
* {@code \s}, {@code \S} and {@code \d}, built in ONE walk of Unicode.
*
- * <p>Three calls to {@code classOf} is three walks of 1,114,112 code points, and this is
- * the whole cost of the first call into this class. Measured on the first
- * {@code cleanEmailBody}: 160 ms when the unused word class was still built here, about
- * 100 ms with three walks, about 35 ms with one.
+ * <p>One walk of 1,114,112 code points rather than one per class. The walk itself is
+ * 15.8 ms warm; the first {@code cleanEmailBody} is 52 to 57 ms, most of it compiling
+ * the patterns. It was about 160 ms when a fourth, unused class was built here too.
*/
private static final String[] CLASSES = classes();
@@ -201,9 +211,6 @@ public final class LayaEmail {
static final String BB = "\\b";
static final String BA = "\\b";
- /** Java's {@code \w}, probed rather than assumed, because it is the JDK's and not ours. */
- static final Pattern JAVA_WORD =
- Pattern.compile("\\w", Pattern.UNICODE_CHARACTER_CLASS);
static final List<Pattern> QUOTE_HEADERS = List.of(
compile("^" + S + "*On .{0,300}wrote:" + S + "*$"),
@@ -231,7 +238,14 @@ public final class LayaEmail {
// next line. That tail cuts too, and takes the `On/Em/El/Le ...` head it belongs to with
// it. The French tail keeps its spaced colon.
static final Pattern ATTRIBUTION_TAIL = compile(
- "^.{0,120}" + NS + "@" + NS + "+" + S + "+(wrote|escreveu|escribi[óo]|a [eé]crit)"
+ // `\S++` POSSESSIVE, not greedy. `.{0,120}` offers 121 starting points, and at
+ // each one a greedy `\S+` runs to the end of the line and then gives every
+ // character back, because the `\s+` after it can never match where a `\S` was
+ // just surrendered -- the two classes are complements. The result was 121 full
+ // scans of the line: measured 55 ms on a 4,000,000-character line at the default
+ // budget and 4.8 s at `maxChars` of 300,000, against 0.018 ms for an ordinary
+ // message. Possessive forbids the backtracking that could never have succeeded.
+ "^.{0,120}" + NS + "@" + NS + "++" + S + "+(wrote|escreveu|escribi[óo]|a [eé]crit)"
+ S + "*:" + S + "*$");
static final Pattern ATTRIBUTION_HEAD = compile("^" + S + "*(On|Em|El|Le) (?=.*" + D + ")");
@@ -314,27 +328,47 @@ public final class LayaEmail {
+ BB + "(usage exclusif|exclusivement|uniquement)" + BA + "[^.]{0,30}destinataire)");
/**
- * A necessary condition for DISCLAIMER, checked first because it is 40 times cheaper.
+ * A necessary condition for DISCLAIMER, checked first.
+ *
+ * <p>Each of DISCLAIMER's twelve top-level alternations requires at least one of these
+ * literals, so a paragraph holding none of them cannot match and does not have to be
+ * scanned by the full pattern. Eleven of the twelve carry word boundaries -- 26 of them
+ * between them; the "received this ... in error" branch carries none.
*
- * <p>Every branch of DISCLAIMER requires at least one of these literals, so a paragraph
- * holding none of them cannot match and does not have to be scanned by a pattern whose
- * twenty alternations each carry two word-boundary lookarounds over a 10,000-character
- * class. Measured: DISCLAIMER alone was 4.9 ms per case on the sweep corpus, which was
- * the whole cost of cleaning a message.
+ * <p>It is worth far less than it was. When `\b` was a lookaround over a
+ * 10,622-character class this filter took DISCLAIMER from 4.9 ms per case to 0.018 ms and
+ * was the difference between usable and not. Once `\b` became Java's own, DISCLAIMER
+ * stopped being expensive, and this is now an ordinary literal alternation of the same
+ * shape as the pattern it guards -- measured at parity with it on a paragraph holding no
+ * literal, and worth 1.16x on the committed corpus and 1.4x on ordinary paragraphs. It
+ * is kept for that, and because a cheap necessary condition in front of a complicated
+ * sufficient one is the right shape, not because it is 40 times anything.
*
* <p>This is a filter, never a decision: a hit still runs DISCLAIMER, so a false positive
* costs only time. What would be a defect is a false NEGATIVE, so the literals are
- * matched with the same flags as DISCLAIMER itself -- `re.I` folds more than lowercasing
- * does, U+017F LATIN SMALL LETTER LONG S against `s` among it, and a filter that lowered
- * the text instead would reject a paragraph the pattern accepts. EmailTest asserts the
- * implication over every branch and over the whole corpus.
+ * matched case-insensitively with Unicode folding, exactly as `re.I` on a str pattern
+ * folds -- it folds more than lowercasing does, U+017F LATIN SMALL LETTER LONG S against
+ * `s` among it, and a filter that lowered the text instead would reject a paragraph the
+ * pattern accepts. DISCLAIMER additionally carries UNICODE_CHARACTER_CLASS, which it
+ * needs for `\b` and this has no use for, having no class of its own.
*/
static final Pattern DISCLAIMER_HINT = compile(
- "confidential|confidencia|sigilos|privilegiad|confidentiel|privil[eé]gi"
+ // Each literal is the SHORTEST form that covers its branch, and every one of
+ // them is load-bearing: `EmailTest` gives each a disclaimer whose only hint
+ // literal is that one, and asserts that removing it stops the filter matching.
+ // An earlier version listed the branches' own words -- `confidential`,
+ // `recebeu|recebido|receber`, `uso exclusivo|exclusivamente|usage exclusif` --
+ // and five of those were deletable with the suite still green, because every
+ // example that reached the filter satisfied it through some OTHER literal.
+ //
+ // Shortening is always safe in this direction. The filter has to be a NECESSARY
+ // condition for DISCLAIMER and nothing more, so a literal that matches more than
+ // its branch costs time on a paragraph that then fails the real pattern; a
+ // literal that matches less lets a disclaimer through to the model.
+ "confidenc|confidenti|sigilos|privil[eé]gi"
+ "|antes de imprimir|received this"
- + "|recebeu|recebido|receber|ha recibido|recibi[óo]|recibe|avez re[çc]u"
- + "|uso exclusivo|exclusivamente|[úu]nicamente|usage exclusif|exclusivement"
- + "|uniquement");
+ + "|receb|recib|avez re[çc]u"
+ + "|exclusiv|[úu]nicamente|uniquement");
static final Pattern SENTENCE = compile("(?<=[.!?])" + S + "+");
static final Pattern BLANK_LINE = compile("\n" + S + "*\n");
@@ -360,9 +394,11 @@ public final class LayaEmail {
* <p>Two substitutes, chosen so a replacement cannot change which literal matches:
* <ul>
* <li>Java says word, Python says not: U+0000, which Java also calls a non-word character.
- * <li>Python says word, Java says not: {@code '0'}, which Java calls a word character. No
- * literal in any pattern here contains a digit, so a digit can neither complete nor
- * break one.
+ * <li>Python says word, Java says not: {@code '0'}, which Java calls a word character.
+ * DISCLAIMER is the only pattern matched against the view, and no literal in it
+ * contains a digit, so a digit can neither complete nor break one. (Other patterns do
+ * hold digits -- DEVICE_FOOTER's {@code [a-z0-9_.+-]}, HEADER_NEXT's four-digit year --
+ * and none of them ever sees this string.)
* </ul>
* Both substitutes are matched by the {@code [^.]} windows the patterns measure, so a window
* counts the same number of characters as it does in the original.
@@ -408,7 +444,7 @@ public final class LayaEmail {
* <p>Spelled out from the definition {@code java.util.regex.Pattern} documents for
* {@code \w} under {@code UNICODE_CHARACTER_CLASS} -- alphabetic, the three mark categories,
* decimal digits, connector punctuation and the join controls -- rather than asked of a
- * {@link Patterns#JAVA_WORD} matcher, which allocates a string and a matcher per character
+ * a matcher on Java's own {@code \\w}, which allocates a string and a matcher per character
* and cost more than the rest of the cleaner put together. Every term reads the JDK's own
* Unicode, so this stays whatever that JDK's {@code \w} is, which is the point: the view is
* built by comparing the JDK against the recorded table, so this side has to be the JDK's
@@ -642,8 +678,8 @@ public final class LayaEmail {
// alternations whose cost grows with input length, and only maxChars are ever returned.
//
// In CODE POINTS, because Python slices a str by code point. Cutting by char index splits
- // a surrogate pair, and the replacement character that comes out is not what the
- // reference returns -- measured on 4,641 cases of an astral corpus.
+ // a surrogate pair, and the half character that comes out is not what the reference
+ // returns -- 4,641 of 20,000 astral bodies in a one-off differential sweep.
if (maxChars > 0) {
text = cutToCodePoints(text, (long) maxChars * 4);
} else {
@@ -663,7 +699,14 @@ public final class LayaEmail {
&& Patterns.HEADER_NEXT.matcher(source[i + 1]).lookingAt()) {
break;
}
- if (Patterns.ATTRIBUTION_TAIL.matcher(line).lookingAt() && !lines.isEmpty()) {
+ // `!lines.isEmpty()` and the trailing colon first, because both are necessary and
+ // both are O(1) against a pattern whose cost grows with the line. The colon really is
+ // necessary: the pattern ends `\s*:\s*$`, a line split on "\n" holds no newline,
+ // and UNIX_LINES makes `$` the end of input -- so a tail that matches is a line whose
+ // rstrip ends with a colon. Together with the possessive quantifier above this is
+ // 55 ms to 11 ms at the default budget, and 4.8 s to 0.1 s at 300,000.
+ if (!lines.isEmpty() && stripTrailing(line).endsWith(":")
+ && Patterns.ATTRIBUTION_TAIL.matcher(line).lookingAt()) {
if (Patterns.ATTRIBUTION_HEAD.matcher(lines.get(lines.size() - 1)).lookingAt()) {
lines.remove(lines.size() - 1);
}
@@ -719,8 +762,11 @@ public final class LayaEmail {
* takes.
*
* <p>Never cuts between a high and a low surrogate: a half pair is not a character the
- * reference can return, and the budget cases of the astral corpus are 4,641 proofs that
- * cutting by char index returns one.
+ * reference can return. Cutting by char index returned one, and a one-off differential sweep
+ * of 20,000 astral bodies against the reference disagreed on 4,641 of them for this reason
+ * alone. That sweep is not in the tree -- generated corpora are not committed here -- so the
+ * cases that hold the line are the four named {@code budget-*} entries of
+ * {@code fixtures/email.json}, each of which fails if this reverts to a char index.
*/
private static String cutToCodePoints(String text, long limit) {
if (limit <= 0) {
@@ -796,8 +842,12 @@ public final class LayaEmail {
* behaviour and is why it is not validated. A {@code null} value is dropped, as the reference
* drops it.
*
- * @return a mutable, insertion-ordered map: {@code subject}, {@code body}, then {@code from}
- * if a sender was given, then the extras
+ * @return a mutable, insertion-ordered map: {@code subject}, {@code body}, then
+ * {@code from} if a sender was given, then the extras. An extra named
+ * {@code subject}, {@code body} or {@code from} REPLACES that value and keeps the
+ * original position, as Python's {@code dict.update} does; a null key is permitted,
+ * again as the reference permits it. Both are input mutations rather than errors,
+ * which is the reference's choice and is why neither is validated.
*/
public static Map<String, Object> emailState(String subject, String body, String sender,
boolean clean, int maxChars,
diff --git a/laya-java/laya-java/src/main/java/com/convaiinnovations/laya/lang/UnicodeTables.java b/laya-java/laya-java/src/main/java/com/convaiinnovations/laya/lang/UnicodeTables.java
index b619fbe9..34ce49f0 100644
--- a/laya-java/laya-java/src/main/java/com/convaiinnovations/laya/lang/UnicodeTables.java
+++ b/laya-java/laya-java/src/main/java/com/convaiinnovations/laya/lang/UnicodeTables.java
@@ -1421,6 +1421,11 @@ sign-off's tail, and what the other ports spell {@code \p{M}}.
* code points that are category M with class zero, so the two are not interchangeable here.
*/
public static boolean isMark(int codePoint) {
+ if (codePoint < 0x80) {
+ long mask = codePoint < 64 ? MARK_ASCII_LOW >>> codePoint
+ : MARK_ASCII_HIGH >>> (codePoint - 64);
+ return (mask & 1L) != 0L;
+ }
return contains(MARK, codePoint);
}
@@ -1431,6 +1436,11 @@ sign-off's tail, and what the other ports spell {@code \p{M}}.
* which holds caseless symbols this does not and misses every caseless script this does.
*/
public static boolean isInitial(int codePoint) {
+ if (codePoint < 0x80) {
+ long mask = codePoint < 64 ? INITIAL_ASCII_LOW >>> codePoint
+ : INITIAL_ASCII_HIGH >>> (codePoint - 64);
+ return (mask & 1L) != 0L;
+ }
return contains(INITIAL, codePoint);
}
diff --git a/laya-java/laya-java/src/test/java/com/convaiinnovations/laya/EmailTest.java b/laya-java/laya-java/src/test/java/com/convaiinnovations/laya/EmailTest.java
index 36cdb262..e7d48583 100644
--- a/laya-java/laya-java/src/test/java/com/convaiinnovations/laya/EmailTest.java
+++ b/laya-java/laya-java/src/test/java/com/convaiinnovations/laya/EmailTest.java
@@ -2,6 +2,8 @@ package com.convaiinnovations.laya;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNotEquals;
+import static org.junit.jupiter.api.Assertions.assertNotSame;
import static org.junit.jupiter.api.Assertions.assertSame;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.junit.jupiter.api.DynamicTest.dynamicTest;
@@ -30,17 +32,80 @@ import org.junit.jupiter.api.TestFactory;
*/
final class EmailTest {
+ /**
+ * Cases the corpus must hold by NAME, not by count.
+ *
+ * <p>A count is not coverage. Trimmed to its old floors -- which were 70 against 86 and 5
+ * against 9 -- this corpus still passed while losing every astral case and eleven
+ * near-misses, and three mutants that the full corpus kills then survived: the code-point
+ * width gate, the surrogate-safe cut and the UNICODE_CHARACTER_CLASS flag on `\b`. Each
+ * name below is a rule nothing else in the corpus pins.
+ */
+ private static final List<String> REQUIRED_CLEANED = List.of(
+ // near-misses: the cleaner must NOT fire. More of these than hits, because a cleaner
+ // that is too eager deletes what the sender actually wrote.
+ "not-signoff-pt-with-name", "not-signoff-sentence",
+ "not-signoff-lowercase-other-script", "not-signoff-symbol-name", "not-signoff-zwnj",
+ "not-signoff-obrigado-mas", "not-signoff-merci-mais", "not-footer-with-request",
+ "not-disclaimer-question", "not-disclaimer-colon", "not-disclaimer-print",
+ "not-disclaimer-contract", "not-disclaimer-fr-contract",
+ "not-signoff-baseless-mark", "not-signoff-too-wide", "not-signoff-four-tokens",
+ // outside the BMP, where the reference counts code points and Java reaches for chars
+ "astral-word-char-breaks-no-boundary", "astral-elsewhere-still-matches",
+ "astral-signoff-at-the-width-limit", "astral-signoff-just-over-the-limit",
+ "astral-mark-in-a-disclaimer",
+ // the signature window's five parameters, the footer width in both directions, and
+ // the body-side strip -- each the only case that tells its rule from a plausible
+ // wrong one
+ "window-floor-of-one", "window-offset-dominates", "window-offset-excludes",
+ "window-ratio-excludes", "window-ratio-includes",
+ "footer-at-the-width-limit", "footer-just-over-the-limit",
+ "body-strip-is-pythons", "body-strip-both-ends");
+
+ private static final List<String> REQUIRED_BUDGETS = List.of(
+ "budget-four-x-bound", "budget-splits-a-surrogate-pair",
+ "budget-ends-on-a-surrogate-pair", "budget-all-astral",
+ "budget-four-x-bound-astral");
+
@SuppressWarnings("unchecked")
private static List<Object> rows(String key) {
return (List<Object>) Fixtures.load("email.json").get(key);
}
+ private static List<String> names(String key) {
+ List<String> out = new ArrayList<>();
+ for (Object entry : rows(key)) {
+ out.add((String) asMap(entry).get("name"));
+ }
+ return out;
+ }
+
+ @Test
+ @DisplayName("the corpus still holds every case that pins a rule nothing else pins")
+ void corpusHoldsItsLoadBearingCases() {
+ List<String> cleaned = names("cleaned");
+ for (String required : REQUIRED_CLEANED) {
+ assertTrue(cleaned.contains(required),
+ () -> "the corpus lost " + required + ", which is the only case pinning its "
+ + "rule; regenerate from a gen_fixtures.py that still has it");
+ }
+ List<String> budgets = names("budgets");
+ for (String required : REQUIRED_BUDGETS) {
+ assertTrue(budgets.contains(required),
+ () -> "the corpus lost the budget case " + required);
+ }
+ assertEquals(95, cleaned.size(), "cases were added or removed; update the count knowingly");
+ assertEquals(9, budgets.size(), "budget cases were added or removed");
+ assertEquals(7, names("states").size(), "state cases were added or removed");
+ }
+
@TestFactory
@DisplayName("every cleaned body is byte-identical to the reference")
List<DynamicTest> cleanedMatches() {
List<Object> cases = rows("cleaned");
- assertTrue(cases.size() >= 70,
- () -> "the email corpus lost cases: " + cases.size() + " of at least 70");
+ assertEquals(95, cases.size(),
+ "the email corpus changed size; corpusHoldsItsLoadBearingCases says which "
+ + "cases may not go");
List<DynamicTest> tests = new ArrayList<>();
for (Object entry : cases) {
Map<String, Object> row = asMap(entry);
@@ -56,7 +121,7 @@ final class EmailTest {
@DisplayName("every budget cuts where the reference cuts")
List<DynamicTest> budgetsMatch() {
List<Object> cases = rows("budgets");
- assertTrue(cases.size() >= 5, "the budget cases vanished");
+ assertEquals(9, cases.size(), "the budget cases changed size");
List<DynamicTest> tests = new ArrayList<>();
for (Object entry : cases) {
Map<String, Object> row = asMap(entry);
@@ -74,7 +139,7 @@ final class EmailTest {
@SuppressWarnings("unchecked")
List<DynamicTest> statesMatch() {
List<Object> cases = rows("states");
- assertTrue(cases.size() >= 7, "the state cases vanished");
+ assertEquals(7, cases.size(), "the state cases changed size");
List<DynamicTest> tests = new ArrayList<>();
for (Object entry : cases) {
Map<String, Object> row = asMap(entry);
@@ -96,13 +161,44 @@ final class EmailTest {
}
@Test
- @DisplayName("the question set is the same object graph as the preset")
+ @DisplayName("the re-export is the preset, and hands out a fresh map each time")
void questionsAreThePreset() {
- assertEquals(Presets.email().keySet(), LayaEmail.emailQuestions().keySet());
+ // The DisplayName used to say "the same object graph", which is the opposite of true and
+ // of what is wanted: `Presets.email()` builds a fresh map, so a caller cannot corrupt a
+ // later caller's questions. Comparing two key sets could not tell the difference.
+ Map<String, Question> first = LayaEmail.emailQuestions();
+ Map<String, Question> second = LayaEmail.emailQuestions();
+ assertEquals(Presets.email().keySet(), first.keySet());
+ assertNotSame(first, second, "each call must hand out its own map");
+ first.remove(first.keySet().iterator().next());
+ assertEquals(Presets.email().keySet(), LayaEmail.emailQuestions().keySet(),
+ "mutating a returned map changed what a later caller gets");
assertEquals(Boolean.TRUE, Fixtures.load("email.json").get("questions_match_presets"),
"the reference's re-export no longer equals laya.presets.email_questions");
}
+ @Test
+ @DisplayName("the categories overload passes the caller's categories through")
+ void questionsWithCategories() {
+ // Untested until a review pointed it out: `Presets.email(categories)` could have been
+ // `Presets.email(null)` -- silently ignoring the argument -- with nothing failing.
+ Map<String, String> categories = new LinkedHashMap<>();
+ categories.put("refund", "the sender wants money back");
+ categories.put("access", "the sender cannot log in");
+ Map<String, Question> questions = LayaEmail.emailQuestions(categories);
+ assertEquals(Presets.email(categories).keySet(), questions.keySet());
+ // The labels, not toString(): `Question` has no toString, so comparing those compared
+ // identity hashes and was true however the method behaved. The first version of this
+ // test did exactly that and the mutant that drops the argument survived it.
+ assertEquals(List.of("refund", "access"), questions.get("category").labels(),
+ "the overload returned the DEFAULT questions, so the categories were dropped");
+ assertEquals(Presets.email(categories).get("category").renderOptions(),
+ questions.get("category").renderOptions());
+ assertNotEquals(Presets.email().get("category").labels(),
+ questions.get("category").labels(),
+ "the caller's categories are indistinguishable from the defaults");
+ }
+
// ------------------------------------------------------------------ the sign-off rule
@Test
@@ -110,12 +206,12 @@ final class EmailTest {
void signoffInitials() {
// Each of these is a rule UPPER would get wrong; see UnicodeTablesTest for the counts.
assertTrue(LayaEmail.isEnglishSignoff("Regards, Ana"));
- assertTrue(LayaEmail.isEnglishSignoff("Regards, Łukasz"), "Lu outside Latin-1");
- assertTrue(LayaEmail.isEnglishSignoff("Regards, 山田"), "Lo: a caseless script");
- assertTrue(LayaEmail.isEnglishSignoff("Regards, Džarko"), "Lt: titlecase");
- assertFalse(LayaEmail.isEnglishSignoff("Thanks, Ⓐ"),
+ assertTrue(LayaEmail.isEnglishSignoff("Regards, \u0141ukasz"), "Lu outside Latin-1");
+ assertTrue(LayaEmail.isEnglishSignoff("Regards, \u5C71\u7530"), "Lo: a caseless script");
+ assertTrue(LayaEmail.isEnglishSignoff("Regards, \u01C5arko"), "Lt: titlecase");
+ assertFalse(LayaEmail.isEnglishSignoff("Thanks, \u24B6"),
"U+24B6 is Uppercase but category So, so it is not a letter");
- assertFalse(LayaEmail.isEnglishSignoff("Thanks, żaneta"), "a lowercase name is prose");
+ assertFalse(LayaEmail.isEnglishSignoff("Thanks, \u017Caneta"), "a lowercase name is prose");
assertFalse(LayaEmail.isEnglishSignoff("Thanks for the quick reply."));
assertFalse(LayaEmail.isEnglishSignoff("Regards, Ana Maria Souza Lima"),
"the tail allows at most three tokens");
@@ -124,16 +220,16 @@ final class EmailTest {
@Test
@DisplayName("a mark rides on its base, and a mark without one still separates tokens")
void marksAreDroppedOnlyWithABase() {
- assertEquals("Jose", LayaEmail.dropMarks("José"), "a mark with a base goes");
- assertEquals("Jose", LayaEmail.dropMarks("Jo͏se"),
+ assertEquals("Jose", LayaEmail.dropMarks("Jose\u0301"), "a mark with a base goes");
+ assertEquals("Jose", LayaEmail.dropMarks("Jo\u034Fse"),
"including U+034F, whose canonical combining class is zero");
- assertEquals("́Ana", LayaEmail.dropMarks("́Ana"),
+ assertEquals("\u0301Ana", LayaEmail.dropMarks("\u0301Ana"),
"a mark opening the string has no base");
- assertEquals("a ́b", LayaEmail.dropMarks("a ́b"),
+ assertEquals("a \u0301b", LayaEmail.dropMarks("a \u0301b"),
"nor does one following a space");
// and the consequence, which is the reason the rule exists
- assertTrue(LayaEmail.isEnglishSignoff("Regards, José"));
- assertFalse(LayaEmail.isEnglishSignoff("Regards, ́Ana"),
+ assertTrue(LayaEmail.isEnglishSignoff("Regards, Jose\u0301"));
+ assertFalse(LayaEmail.isEnglishSignoff("Regards, \u0301Ana"),
"a kept mark is not a token opener, so this is not a sign-off");
}
@@ -160,7 +256,17 @@ final class EmailTest {
assertEquals(pattern, spelled, String.format("U+%04X", cp));
}
}
- assertEquals(0x110000 - 2048, examined, "every code point but the surrogates");
+ // Not asserted against a constant the loop itself produces -- that only restates the
+ // loop. What matters is that the comparison ran on a lot of code points and on the ones
+ // where the two classes are known to disagree.
+ final int compared = examined;
+ assertTrue(compared > 1_000_000, () -> "only " + compared + " code points examined");
+ for (int codePoint : new int[] {0x0301, 0x200C, 0x200D, 0x00BD, 0x2160, 0x11001, 0x10107}) {
+ final int cp = codePoint;
+ boolean spelled = (Boolean) javaWord.invoke(null, cp);
+ assertEquals(word.matcher(new String(Character.toChars(cp))).matches(), spelled,
+ () -> String.format("U+%04X, a code point the two word classes argue about", cp));
+ }
}
@Test
@@ -168,18 +274,24 @@ final class EmailTest {
void boundaryViewSubstitutes() {
assertSame("plain ascii", LayaEmail.boundaryView("plain ascii"),
"an ASCII-only message is returned without copying");
- assertSame("café résumé", LayaEmail.boundaryView("café résumé"),
+ assertSame("caf\u00E9 r\u00E9sum\u00E9", LayaEmail.boundaryView("caf\u00E9 r\u00E9sum\u00E9"),
"ordinary accented letters are word characters in both, so nothing is rewritten");
// Java says word, Python says not -> U+0000, a non-word character to Java too.
- assertEquals("ab", LayaEmail.boundaryView("áb"), "a combining mark");
- assertEquals("ab", LayaEmail.boundaryView("a‌b"), "ZWNJ, a join control");
- assertEquals("ab", LayaEmail.boundaryView("a‍b"), "ZWJ");
+ assertEquals("a\u0000b", LayaEmail.boundaryView("a\u0301b"), "a combining mark");
+ assertEquals("a\u0000b", LayaEmail.boundaryView("a\u200Cb"), "ZWNJ, a join control");
+ assertEquals("a\u0000b", LayaEmail.boundaryView("a\u200Db"), "ZWJ");
// Python says word, Java says not -> '0', which no literal in any pattern contains.
- assertEquals("a0b", LayaEmail.boundaryView("a½b"), "U+00BD is No: Python's word, not Java's");
- // Length is preserved, so the [^.]{0,N} windows count the same characters.
- for (String text : List.of("áb", "a‌b", "a½b", "x́½‌y")) {
- assertEquals(text.length(), LayaEmail.boundaryView(text).length(),
- () -> "boundaryView changed the length of " + text);
+ assertEquals("a0b", LayaEmail.boundaryView("a\u00BDb"), "U+00BD is No: Python's word, not Java's");
+ // CODE POINT count is preserved, which is the unit a regex quantifier counts in -- and
+ // not the char count, which an astral disagreement does change: U+1D16D is two chars and
+ // one code point, and its substitute is one of each. Asserting chars passed only because
+ // every case in the list was in the BMP.
+ for (String text : List.of("a\u0301b", "a\u200Cb", "a\u00BDb", "x\u0301\u00BD\u200Cy",
+ "a\uD834\uDD6Db", "a\uD800\uDD07b")) {
+ String view = LayaEmail.boundaryView(text);
+ assertEquals(text.codePointCount(0, text.length()),
+ view.codePointCount(0, view.length()),
+ () -> "boundaryView changed the code point count of " + text);
}
}
@@ -187,32 +299,107 @@ final class EmailTest {
@DisplayName("neither substitute can appear in a pattern literal")
void substitutesAreNotLiterals() throws Exception {
// The whole argument for substituting is that the replacement cannot complete or break a
- // literal. That holds only while no pattern here holds a digit or a NUL as a literal, so
- // it is checked rather than asserted in a comment.
+ // literal. Checked BEHAVIOURALLY: an earlier version scanned the pattern source for a
+ // raw NUL, which no pattern can hold -- `appendEscaped` writes every class member as a
+ // backslash-u escape -- while the two forms a NUL literal actually takes in a Java
+ // regex, backslash-x-zero-zero and backslash-u-four-zeros, both read as ordinary text to
+ // `indexOf`. So it could not fail from any plausible edit. (Spelled out rather than
+ // written, because the compiler translates a backslash-u escape before it lexes, even
+ // inside a comment, and an invalid one there is a compile error.)
+ String nul = "\u0000";
for (Pattern pattern : allPatterns()) {
- String source = pattern.pattern();
- assertFalse(source.indexOf('') >= 0,
- () -> "a pattern holds a literal NUL: " + source.substring(0, 40));
+ assertFalse(pattern.matcher(nul).find(),
+ () -> "this pattern matches a lone NUL, so U+0000 is no longer a safe "
+ + "substitute: " + head(pattern.pattern()));
}
- // A digit may only appear inside the generated \d class or as a quantifier bound, never
- // as a literal to be matched, so the DISCLAIMER branches are checked directly.
- Pattern disclaimer = (Pattern) patternField("DISCLAIMER");
- assertFalse(disclaimer.pattern().matches(".*[^{,0-9]\\d[^}0-9,].*"),
+ // And the digit side, for DISCLAIMER, which is the only pattern the view ever reaches.
+ // Quantifier bounds are stripped first: `[^.]{0,60}` is not a digit literal, and a check
+ // that did not strip them could not see `covid19` either, because every digit there is
+ // next to another one.
+ String disclaimer = ((Pattern) patternField("DISCLAIMER")).pattern()
+ .replaceAll("\\{\\d+(,\\d+)?\\}", "");
+ assertFalse(disclaimer.matches(".*\\d.*"),
"a digit became a literal in DISCLAIMER; '0' is no longer a safe substitute");
+ // the guard above is only meaningful if it would notice one
+ assertTrue(("x" + disclaimer + "7").matches(".*\\d.*"),
+ "the digit check cannot see a digit, so it proves nothing");
+ }
+
+ /** A pattern's opening, for a failure message, without risking its length. */
+ private static String head(String source) {
+ return source.substring(0, Math.min(40, source.length()));
}
@Test
- @DisplayName("the disclaimer filter never rejects what the pattern would accept")
- void hintIsANecessaryCondition() throws Exception {
+ @DisplayName("every literal in the disclaimer filter is load-bearing")
+ void everyHintLiteralIsNecessary() throws Exception {
+ Pattern hint = (Pattern) patternField("DISCLAIMER_HINT");
+ Pattern disclaimer = (Pattern) patternField("DISCLAIMER");
+
+ // One disclaimer per literal, chosen so that literal is the ONLY one the filter can
+ // match it by. That is what makes each row a test of necessity rather than of
+ // membership: the previous version of this test checked thirteen examples against the
+ // whole filter, every example was satisfied by SOME literal, and five literals could be
+ // deleted with the suite green -- leaking "Esta mensagem e sigilosa.", "Destinada
+ // exclusivamente ao destinatario." and "Reserve uniquement au destinataire." into the
+ // model's input.
+ String[][] perLiteral = {
+ {"confidenc", "Esta mensagem e confidencial."},
+ {"confidenti", "This email is confidential and intended solely for the addressee."},
+ {"sigilos", "Esta mensagem e sigilosa."},
+ {"privil[e\u00e9]gi", "Ce message est privilegie."},
+ {"antes de imprimir", "Antes de imprimir pense no meio ambiente."},
+ {"received this", "If you have received this email in error please delete it."},
+ {"receb", "Se recebeu esta mensagem por engano, apague."},
+ {"recib", "Usted ha recibido este mensaje por error."},
+ {"avez re[\u00e7c]u", "Vous avez recu ce message par erreur."},
+ {"exclusiv", "Uso exclusivo do destinatario desta mensagem."},
+ {"[\u00fau]nicamente", "Unicamente ao destinatario desta mensagem."},
+ {"uniquement", "Reserve uniquement au destinataire."},
+ };
+ List<String> alternatives = List.of(hint.pattern().split("\\|"));
+ assertEquals(perLiteral.length, alternatives.size(),
+ () -> "the filter has " + alternatives.size() + " literals and this test names "
+ + perLiteral.length + "; a literal with no example is a literal that can "
+ + "be deleted without failing anything");
+
+ for (String[] row : perLiteral) {
+ String literal = row[0];
+ String example = row[1];
+ assertTrue(alternatives.contains(literal),
+ () -> "the filter no longer holds the literal " + literal);
+ // the example really is a disclaimer, so leaking it would matter
+ assertTrue(disclaimer.matcher(LayaEmail.boundaryView(example)).find(),
+ () -> "this is no longer a disclaimer at all: " + example);
+ assertTrue(hint.matcher(example).find(),
+ () -> "the filter rejects a disclaimer the pattern accepts: " + example);
+ // and WITHOUT this literal the filter misses it, which is what "necessary" means
+ StringBuilder reduced = new StringBuilder();
+ for (String alternative : alternatives) {
+ if (alternative.equals(literal)) {
+ continue;
+ }
+ if (reduced.length() > 0) {
+ reduced.append('|');
+ }
+ reduced.append(alternative);
+ }
+ Pattern without = Pattern.compile(reduced.toString(),
+ Pattern.CASE_INSENSITIVE | Pattern.UNICODE_CASE | Pattern.UNIX_LINES);
+ assertFalse(without.matcher(example).find(),
+ () -> "deleting " + literal + " from the filter would still catch "
+ + example + ", so this row does not test that literal");
+ }
+ }
+
+ @Test
+ @DisplayName("the filter rejects nothing the pattern accepts, over the whole corpus")
+ void hintIsNecessaryOverTheCorpus() throws Exception {
Pattern hint = (Pattern) patternField("DISCLAIMER_HINT");
Pattern disclaimer = (Pattern) patternField("DISCLAIMER");
- // A filter is only sound as a NECESSARY condition: a false positive costs time, a false
- // negative keeps a disclaimer in the model's input. Checked over every recorded case,
- // which includes one example of every branch of the pattern.
int fired = 0;
for (Object entry : rows("cleaned")) {
- Map<String, Object> row = asMap(entry);
- String body = (String) row.get("body");
+ String body = (String) asMap(entry).get("body");
for (String raw : body.split("\n", -1)) {
final String paragraph = raw;
if (disclaimer.matcher(LayaEmail.boundaryView(paragraph)).find()) {
@@ -223,34 +410,13 @@ final class EmailTest {
}
}
}
+ // A secondary check. `everyHintLiteralIsNecessary` is the coverage gate -- it proves each
+ // literal is load-bearing -- and this one proves the implication holds on real bodies as
+ // well as on the twelve constructed ones.
final int hits = fired;
- assertTrue(hits >= 9,
- () -> "the corpus no longer exercises the disclaimer pattern: " + hits + " hits");
-
- // The corpus count is a floor, not coverage. Every BRANCH of the pattern is named here,
- // so a branch whose required literal is missing from the filter fails this rather than
- // waiting for a message in production to be the first to notice.
- String[] oneExamplePerBranch = {
- "This email and its contents are confidential and intended solely for the addressee.",
- "This message is confidential and may also privileged.",
- "If you have received this email in error please delete it.",
- "Esta mensagem e seus anexos sao confidenciais e de uso exclusivo do destinatario.",
- "Este mensaje es confidencial y para uso exclusivo del destinatario.",
- "Uso exclusivo do destinatario desta mensagem.",
- "Se recebeu esta mensagem por engano, apague-a.",
- "Usted ha recibido este mensaje por error.",
- "Antes de imprimir pense no meio ambiente.",
- "Pense no meio ambiente antes de imprimir.",
- "Ce message est confidentiel et destine uniquement au destinataire.",
- "Vous avez recu ce message par erreur.",
- "Usage exclusif du destinataire.",
- };
- for (String example : oneExamplePerBranch) {
- assertTrue(disclaimer.matcher(LayaEmail.boundaryView(example)).find(),
- () -> "this no longer matches DISCLAIMER at all: " + example);
- assertTrue(hint.matcher(example).find(),
- () -> "the filter would reject a disclaimer the pattern accepts: " + example);
- }
+ assertEquals(11, hits,
+ "the corpus's disclaimer coverage changed; this is a count, so change it "
+ + "knowingly rather than lowering it");
}
// ------------------------------------------------------------------------- the state
@@ -295,7 +461,7 @@ final class EmailTest {
void subjectStripIsPythons() {
// U+00A0 and U+2007 are whitespace to str.strip() and not to String.trim(). The same gap
// sat in the router's checkpoint names and routed a caller to the wrong model.
- assertEquals("Refund", LayaEmail.emailState(" Refund ", "b").get("subject"));
+ assertEquals("Refund", LayaEmail.emailState("\u00A0Refund\u2007", "b").get("subject"));
assertTrue(UnicodeTables.isSpace(0x00a0) && UnicodeTables.isSpace(0x2007),
"the premise of the assertion above");
}
diff --git a/laya-java/scripts/gen_fixtures.py b/laya-java/scripts/gen_fixtures.py
index a5c1b466..54a03c2d 100644
--- a/laya-java/scripts/gen_fixtures.py
+++ b/laya-java/scripts/gen_fixtures.py
@@ -1895,6 +1895,32 @@ def email_clean():
("header-next-devanagari-year",
q + "\n\nFrom: Maria Souza\nDate: 3 Sep \u0968\u0966\u0968\u096B\nolder"),
+ # -- the signature window, `max(1, min(int(len * 0.6), len - 8))`. Every part of that
+ # formula was unpinned: a review mutated the ratio to 0.55 and to 0.9, the offset to 9
+ # and to 2, and the floor to 2, and all five left the suite green. These five cases are
+ # the smallest inputs under 60 lines on which each mutant's window reaches a different
+ # line from the real one -- found by searching, not derived, because deriving them by
+ # hand gets the `min` the wrong way round.
+ ("window-floor-of-one", "\n".join(["Regards, Ana" if i == 1 else q for i in range(2)])),
+ ("window-offset-dominates", "\n".join(["Regards, Ana" if i == 1 else q for i in range(4)])),
+ ("window-offset-excludes", "\n".join(["Regards, Ana" if i == 1 else q for i in range(10)])),
+ ("window-ratio-excludes", "\n".join(["Regards, Ana" if i == 9 else q for i in range(18)])),
+ ("window-ratio-includes", "\n".join(["Regards, Ana" if i == 12 else q for i in range(21)])),
+
+ # -- the device-footer width gate, 60 code points against the sign-off's 40. Both
+ # directions, because a review found `<= 60` mutable to 41 and to 200 with nothing
+ # failing.
+ ("footer-at-the-width-limit",
+ "\n".join([q] * 9 + ["Sent from my iphone using " + "a" * 34])),
+ ("footer-just-over-the-limit",
+ "\n".join([q] * 9 + ["Sent from my iphone using " + "a" * 35])),
+
+ # -- the body-side strip is Python's `str.strip()`, which takes U+00A0 and U+2007 where
+ # `String.trim()` does not. The subject side was pinned and the body side was not, and
+ # swapping it for `trim()` left the suite green while returning a leading space.
+ ("body-strip-is-pythons", "\u00a0Refund please\u2007"),
+ ("body-strip-both-ends", "\u2007\u00a0Please refund\u00a0"),
+
# -- outside the BMP. The first sweep of this port had no astral character in it and
# passed on 40,000 cases; a corpus of them found four defects at once. Each case below
# is one of them, because the units the reference counts are CODE POINTS and the units
diff --git a/laya-java/scripts/gen_unicode_tables.py b/laya-java/scripts/gen_unicode_tables.py
index 603038e9..126db7b4 100755
--- a/laya-java/scripts/gen_unicode_tables.py
+++ b/laya-java/scripts/gen_unicode_tables.py
@@ -425,6 +425,11 @@ public final class UnicodeTables {
* code points that are category M with class zero, so the two are not interchangeable here.
*/
public static boolean isMark(int codePoint) {
+ if (codePoint < 0x80) {
+ long mask = codePoint < 64 ? MARK_ASCII_LOW >>> codePoint
+ : MARK_ASCII_HIGH >>> (codePoint - 64);
+ return (mask & 1L) != 0L;
+ }
return contains(MARK, codePoint);
}
@@ -435,6 +440,11 @@ public final class UnicodeTables {
* which holds caseless symbols this does not and misses every caseless script this does.
*/
public static boolean isInitial(int codePoint) {
+ if (codePoint < 0x80) {
+ long mask = codePoint < 64 ? INITIAL_ASCII_LOW >>> codePoint
+ : INITIAL_ASCII_HIGH >>> (codePoint - 64);
+ return (mask & 1L) != 0L;
+ }
return contains(INITIAL, codePoint);
}