Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

fix(ts): read email text the way Python does -- code points, Unicode classes

Aashish4ceaae30db1d
diff --git a/laya-ts/src/email.ts b/laya-ts/src/email.ts
index 49c6a51b..12d902c3 100644
--- a/laya-ts/src/email.ts
+++ b/laya-ts/src/email.ts
@@ -1,26 +1,84 @@
+// Python's `re` classes and its `len()`/slicing are Unicode-wide and count code points; JS's `\s`,
+// `\d`, `\w` and `\b` are ASCII-ish and `.length`/`.slice` count UTF-16 units. The markers below are
+// compiled from the three classes and the two helpers at the bottom of this block, because each
+// difference is a case the reference records in `laya-java/fixtures/email.json` and this port got
+// wrong: a `\x1c` before `On ... escreveu:` was not indentation, an Arabic-Indic or Devanagari date
+// was not a date, a cuneiform sign was not a word character (so a sender's own sentence read as a
+// disclaimer and was deleted whole), and a budget cut an emoji in half.
+//
+// The space class is the measured one: it is exactly the set `re.match(r"\s", ch)` accepts on
+// CPython, which is also the set `str.strip()` removes. JS's `\s` omits U+001C-U+001F and U+0085 and
+// adds U+FEFF, so both the class and `trimPy` below stand in for it.
+const SP_IN =
+ "\\u0009-\\u000D\\u001C-\\u001F\\u0020\\u0085\\u00A0\\u1680\\u2000-\\u200A\\u2028\\u2029" +
+ "\\u202F\\u205F\\u3000";
+const SP = `[${SP_IN}]`; // Python's \s
+const NOT_SP = `[^${SP_IN}]`; // Python's \S
+// Python's \w is `str.isalnum()` plus the underscore: the letter and number categories and nothing
+// else -- measured over the whole repertoire, marks (Mn/Mc/Me) and the join controls are NOT word
+// characters, and every code point CPython accepts is in `[\p{L}\p{N}_]`. The class is wider by the
+// 4,657 code points JS's newer Unicode tables have assigned since CPython's, none of which is a
+// letter or digit Python knows.
+const WORD_IN = "\\p{L}\\p{N}_"; // the class body Python's \w accepts
+const WORD = `[${WORD_IN}]`; // Python's \w
+const DIGIT = "\\p{Nd}"; // Python's \d is the Nd category alone
+const B_L = `(?<!${WORD})`; // Python's \b before a word character
+const B_R = `(?!${WORD})`; // Python's \b after a word character
+
+const LEADING_SP = new RegExp(`^${SP}+`, "u");
+const TRAILING_SP = new RegExp(`${SP}+$`, "u");
+
+/** Python's `str.strip()`, over the space set `re`'s `\s` uses. */
+function trimPy(text: string): string {
+ return text.replace(LEADING_SP, "").replace(TRAILING_SP, "");
+}
+
+/** Python's `len()` on a string: code points, not the UTF-16 units `.length` counts. */
+function width(text: string): number {
+ let count = 0;
+ for (const _ch of text) count++;
+ return count;
+}
+
+/** Python's `text[:count]`, cutting at code points so a surrogate pair is never split in half. */
+function cutCodePoints(text: string, count: number): string {
+ const chars = Array.from(text);
+ const end = count < 0 ? Math.max(0, chars.length + count) : Math.min(count, chars.length);
+ return chars.slice(0, end).join("");
+}
+
const QUOTE_HEADERS: RegExp[] = [
- /^\s*On .{0,300}wrote:\s*$/i,
- /^\s*Em (?=.*\d).{0,300}escreveu:\s*$/i,
- /^\s*El (?=.*\d).{0,300}escribi[óo]:\s*$/i,
+ new RegExp(`^${SP}*On .{0,300}wrote:${SP}*$`, "iu"),
+ // "Em resposta ao que você escreveu:" is body text; a client's attribution always carries a date
+ new RegExp(`^${SP}*Em (?=.*${DIGIT}).{0,300}escreveu:${SP}*$`, "iu"),
+ new RegExp(`^${SP}*El (?=.*${DIGIT}).{0,300}escribi[óo]:${SP}*$`, "iu"),
// Gmail's French attribution carries a date like the others ("Le lun. ... a écrit :"), and
// French typography puts a space before the colon, so the verb allows one
- /^\s*Le (?=.*\d).{0,300}a [eé]crit\s*:\s*$/i,
- /^\s*-{2,}\s*(Original|Forwarded) Message\s*-{2,}/i,
- /^\s*-{2,}\s*(Mensagem (original|encaminhada)|Mensaje (original|reenviado)|Message d'origine)\s*-{2,}/i,
- /^\s*_{8,}\s*$/,
+ new RegExp(`^${SP}*Le (?=.*${DIGIT}).{0,300}a [eé]crit${SP}*:${SP}*$`, "iu"),
+ new RegExp(`^${SP}*-{2,}${SP}*(Original|Forwarded) Message${SP}*-{2,}`, "iu"),
+ new RegExp(
+ `^${SP}*-{2,}${SP}*(Mensagem (original|encaminhada)|Mensaje (original|reenviado)|` +
+ "Message d'origine)" +
+ `${SP}*-{2,}`,
+ "iu",
+ ),
+ new RegExp(`^${SP}*_{8,}${SP}*$`, "u"),
// `From:` opens ordinary prose too ("From: my side the integration works, but please
// refund..."), and a reply header always carries the sender, so the header is only recognised
// when an address follows -- the same rule as `De:` below. A bare `From: Name` header is
// caught by HEADER_FROM_NAME/HEADER_NEXT instead, which need the header's own `Sent:`/`Date:`
// line to tell it apart from a sentence.
- /^\s*From:\s.*[@<]/i,
+ new RegExp(`^${SP}*From:${SP}.*[@<]`, "iu"),
// `De:` also opens ordinary Portuguese/Spanish lines ("De: 10/09 a 15/09"), so the Outlook
// header is only recognised when it carries an address. French Outlook writes `De :` with a
// space, so the marker allows one; the address rule is unchanged
- /^\s*De\s*:\s.*[@<]/i,
+ new RegExp(`^${SP}*De${SP}*:${SP}.*[@<]`, "iu"),
];
-const ATTRIBUTION_TAIL = /^.{0,120}\S@\S+\s+(wrote|escreveu|escribi[óo]|a [eé]crit)\s*:\s*$/i;
-const ATTRIBUTION_HEAD = /^\s*(On|Em|El|Le) (?=.*\d)/i;
+const ATTRIBUTION_TAIL = new RegExp(
+ `^.{0,120}${NOT_SP}@${NOT_SP}+${SP}+(wrote|escreveu|escribi[óo]|a [eé]crit)${SP}*:${SP}*$`,
+ "iu",
+);
+const ATTRIBUTION_HEAD = new RegExp(`^${SP}*(On|Em|El|Le) (?=.*${DIGIT})`, "iu");
// Exchange often leaves the address out of Outlook's reply header ("De: Maria Souza"), so a bare
// `De:` only cuts when the header's own `Enviado:` line, or a dated `Data:`/`Fecha:` line, follows
// it. `Para:` is not enough: "De: 10/09 / Para: 15/09" is how a leave request reads.
@@ -30,41 +88,56 @@ const ATTRIBUTION_HEAD = /^\s*(On|Em|El|Le) (?=.*\d)/i;
// that only looks like prose still has to be told apart from a header by its neighbours, so the
// English pair is "From: <name>" followed by "Sent:"/"Date:". French Outlook writes the same
// header as `De : Marie Dupont` with `Envoyé :` underneath, so both allow the spaced colon.
-const HEADER_FROM_NAME = /^\s*(De|From)\s*:\s+\S/i;
-const HEADER_NEXT =
- /^\s*(Enviad[oa]( em| el)?:\s|Envoy[ée]( le)?\s*:\s|Sent:\s|(Data|Fecha|Date):\s.*\d{4})/i;
+const HEADER_FROM_NAME = new RegExp(`^${SP}*(De|From)${SP}*:${SP}+${NOT_SP}`, "iu");
+const HEADER_NEXT = new RegExp(
+ `^${SP}*(Enviad[oa]( em| el)?:${SP}|Envoy[ée]( le)?${SP}*:${SP}|Sent:${SP}|` +
+ `(Data|Fecha|Date):${SP}.*${DIGIT}{4})`,
+ "iu",
+);
// A closing line is the closing word plus punctuation and at most a name. Anything else on
// the line is a sentence, and the case of the next word is what separates the two: a name is
// capitalised, "for" in "Thanks for the quick reply." is not. JS regexes have no scoped
// case-insensitive groups like Python's (?i:...), so the closing words are matched
// case-insensitively here and the name is checked case-sensitively by SIGNOFF_TAIL.
-const SIGNOFF_HEAD =
- /^\s*(?:best|kind|warmest|warm|many thanks|thanks|thank you|regards|cheers|sincerely)(?:\s+(?:and|&)\s+regards|\s+(?:regards|wishes|again|in advance|a lot|so much|very much))?/i;
+const SIGNOFF_HEAD = new RegExp(
+ `^${SP}*(?:best|kind|warmest|warm|many thanks|thanks|thank you|regards|cheers|sincerely)` +
+ `(?:${SP}+(?:and|&)${SP}+regards|${SP}+(?:regards|wishes|again|in advance|a lot|so much|very much))?`,
+ "iu",
+);
// The name's first letter must not be lowercase, so the check is per character, not per script:
// \p{Lu}/\p{Lt}/\p{Lo} accepts a name capitalised in any script (Regards, Łukasz) or written in
// a script without case (山田). Python asks the same question per token (`_is_english_signoff`).
-const SIGNOFF_TAIL = /^[\s,;:!.]*(?:[\p{Lu}\p{Lt}\p{Lo}][\p{L}\p{M}\p{N}_'-]*[\s,.]*){0,3}$/u;
+const SIGNOFF_TAIL = new RegExp(
+ `^[${SP_IN},;:!.]*(?:[\\p{Lu}\\p{Lt}\\p{Lo}][\\p{L}\\p{M}\\p{N}_'-]*[${SP_IN},.]*){0,3}$`,
+ "u",
+);
function isEnglishSignoff(line: string): boolean {
const m = SIGNOFF_HEAD.exec(line);
return m !== null && SIGNOFF_TAIL.test(line.slice(m[0].length));
}
const SIGNATURE_MARKERS: Array<(line: string) => boolean> = [
- (l) => /^\s*--\s*$/.test(l),
+ (l) => new RegExp(`^${SP}*--${SP}*$`, "u").test(l),
isEnglishSignoff,
(l) =>
- /^\s*(atenciosamente|att|abraços?|abs|um abraço|cordialmente|grat[oa]|(muito )?obrigad[oa]s?( desde já| pela atenção)?|(com os melhores )?cumprimentos|saudações|(un )?saludos?( cordiales)?|atentamente|(muchas )?gracias( de antemano)?|(bien )?cordialement|salutations( distinguées)?|bien à vous|merci( d'avance)?|bonne journée)[\s,!.]*$/i.test(
- l,
- ),
+ new RegExp(
+ `^${SP}*(atenciosamente|att|abraços?|abs|um abraço|cordialmente|grat[oa]` +
+ "|(muito )?obrigad[oa]s?( desde já| pela atenção)?|(com os melhores )?cumprimentos|saudações" +
+ "|(un )?saludos?( cordiales)?|atentamente|(muchas )?gracias( de antemano)?" +
+ "|(bien )?cordialement|salutations( distinguées)?|bien à vous|merci( d'avance)?|bonne journée)" +
+ `[${SP_IN},!.]*$`,
+ "iu",
+ ).test(l),
];
const DEVICE =
"iphone|ipad|android|ios|mobile|celular|telemóvel|móvil|galaxy|smartphone|samsung|tablet|" +
"outlook|yahoo|mail|e-?mail|gmail|windows";
const DEVICE_FOOTER = new RegExp(
- "^\\s*((enviad[oa] (do|pelo|pela|via|desde|a partir do)( meu| minha| mi)?|sent from( my)?|" +
+ `^${SP}*((enviad[oa] (do|pelo|pela|via|desde|a partir do)( meu| minha| mi)?|sent from( my)?|` +
"envoy[ée] (depuis|de)( mon| ma| mes)?)" +
- ` (${DEVICE})( (${DEVICE}|para|for|no|na|\\d+|phone|device|pro|max|mini|plus|using [a-z][\\w.+-]*))*` +
- "|(obter o|get) outlook (para|for) (ios|android))[\\s.!]*$",
- "i",
+ ` (${DEVICE})( (${DEVICE}|para|for|no|na|${DIGIT}+|phone|device|pro|max|mini|plus|` +
+ `using [a-z][${WORD_IN}.+-]*))*` +
+ `|(obter o|get) outlook (para|for) (ios|android))[${SP_IN}.!]*$`,
+ "iu",
);
const DISCLAIMER = new RegExp(
// English is tied to a disclaimer noun and a disclaimer tail, the way the Portuguese branches
@@ -72,26 +145,26 @@ const DISCLAIMER = new RegExp(
// confidential?" and "Confidential: I need a refund." were deleted whole and the model was
// scored on an empty state. `[^.]` rather than `[^.\n]`: a footer wraps, so "are\nconfidential"
// must still match.
- "(\\b(e-?mail|message|information|communication|transmission|contents?)\\b[^.]{0,60}" +
- "\\bconfidential\\b[^.]{0,60}\\b(intended|solely|addressee|recipient|privileged|" +
+ `(${B_L}(e-?mail|message|information|communication|transmission|contents?)${B_R}[^.]{0,60}` +
+ `${B_L}confidential${B_R}[^.]{0,60}${B_L}(intended|solely|addressee|recipient|privileged|` +
"disclos|unauthori[sz]ed)|" +
- "\\bconfidential\\b[^.]{0,60}\\b(and (may|is) (also )?privileged)|" +
+ `${B_L}confidential${B_R}[^.]{0,60}${B_L}(and (may|is) (also )?privileged)|` +
"if you (have )?received this (e-?mail|message) in error|" +
- "\\b(esta|este) (mensagem|e-?mail|mensaje|correo)\\b[^.]{0,80}(confidencia|sigilos|privilegiad)|" +
- "\\b(uso exclusivo|exclusivamente|únicamente|unicamente)\\b[^.]{0,30}" +
+ `${B_L}(esta|este) (mensagem|e-?mail|mensaje|correo)${B_R}[^.]{0,80}(confidencia|sigilos|privilegiad)|` +
+ `${B_L}(uso exclusivo|exclusivamente|únicamente|unicamente)${B_R}[^.]{0,30}` +
"(destinatári|destinatari|pessoa|persona|entidade|entidad)|" +
- "\\b(recebeu|recebido|receber) (esta|este) (mensagem|e-?mail)\\b[^.]{0,20} por (engano|erro)|" +
- "\\b(ha recibido|recibió|recibe) (este|esta) (mensaje|correo)\\b[^.]{0,20} por error|" +
- "\\bantes de imprimir\\b[^.]{0,100}(meio ambiente|medio ambiente|natureza|planeta|realmente necess)|" +
- "\\b(meio|medio) ambiente\\b[^.]{0,30}antes de imprimir|" +
+ `${B_L}(recebeu|recebido|receber) (esta|este) (mensagem|e-?mail)${B_R}[^.]{0,20} por (engano|erro)|` +
+ `${B_L}(ha recibido|recibió|recibe) (este|esta) (mensaje|correo)${B_R}[^.]{0,20} por error|` +
+ `${B_L}antes de imprimir${B_R}[^.]{0,100}(meio ambiente|medio ambiente|natureza|planeta|realmente necess)|` +
+ `${B_L}(meio|medio) ambiente${B_R}[^.]{0,30}antes de imprimir|` +
// French is tied to "ce message/cet e-mail" rather than the bare word `confidentiel`, which a
// sender's own request ("le contrat confidentiel") uses just as often
- "\\b(ce|cet|cette) (message|e-?mail|mail|courriel)\\b[^.]{0,80}(confidentiel|privil[eé]gi)|" +
- "\\bavez re[çc]u (ce|cet|cette) (message|e-?mail|mail)\\b[^.]{0,20} par erreur|" +
- "\\b(usage exclusif|exclusivement|uniquement)\\b[^.]{0,30}destinataire)",
- "i",
+ `${B_L}(ce|cet|cette) (message|e-?mail|mail|courriel)${B_R}[^.]{0,80}(confidentiel|privil[eé]gi)|` +
+ `${B_L}avez re[çc]u (ce|cet|cette) (message|e-?mail|mail)${B_R}[^.]{0,20} par erreur|` +
+ `${B_L}(usage exclusif|exclusivamente|uniquement)${B_R}[^.]{0,30}destinataire)`,
+ "iu",
);
-const SENTENCE = /(?<=[.!?])\s+/;
+const SENTENCE = new RegExp(`(?<=[.!?])${SP}+`, "u");
function startsNewSentence(line: string): boolean {
for (const ch of line) {
@@ -104,7 +177,7 @@ function splitFusedLines(sentence: string): string[] {
if (!sentence.includes("\n")) return [sentence];
const pieces: string[] = [];
let buf = "";
- for (const line of sentence.split("\n").map((ln) => ln.trim())) {
+ for (const line of sentence.split("\n").map(trimPy)) {
if (!line) continue;
if (buf && startsNewSentence(line)) {
pieces.push(buf);
@@ -119,7 +192,7 @@ function splitFusedLines(sentence: string): string[] {
function stripDisclaimer(paragraph: string): string {
if (!DISCLAIMER.test(paragraph)) return paragraph;
- const parts = paragraph.split(SENTENCE).map((p) => p.trim()).filter(Boolean);
+ const parts = paragraph.split(SENTENCE).map(trimPy).filter(Boolean);
const pieces: string[] = [];
for (const p of parts) {
if (DISCLAIMER.test(p)) pieces.push(...splitFusedLines(p));
@@ -132,7 +205,7 @@ export function cleanEmailBody(body: string, maxChars = 3000): string {
let text = (body ?? "").replace(/\r\n?/g, "\n").replace(/\\n/g, "\n");
// Bound regex work before the expensive patterns below (Python does the same): only
// maxChars are ever returned, so nothing past 4x can survive cleaning.
- if (text.length > maxChars * 4) text = text.slice(0, maxChars * 4);
+ if (width(text) > maxChars * 4) text = cutCodePoints(text, maxChars * 4);
const lines: string[] = [];
const src = text.split("\n");
for (let i = 0; i < src.length; i++) {
@@ -143,13 +216,13 @@ export function cleanEmailBody(body: string, maxChars = 3000): string {
if (ATTRIBUTION_HEAD.test(lines[lines.length - 1])) lines.pop();
break;
}
- if (line.replace(/^\s+/, "").startsWith(">")) continue;
- lines.push(line.replace(/\s+$/, ""));
+ if (line.replace(LEADING_SP, "").startsWith(">")) continue;
+ lines.push(line.replace(TRAILING_SP, ""));
}
let cut = lines.length;
const start = Math.max(1, Math.min(Math.floor(lines.length * 0.6), lines.length - 8));
for (let i = start; i < lines.length; i++) {
- const n = lines[i].trim().length;
+ const n = width(trimPy(lines[i]));
if (
(n <= 40 && SIGNATURE_MARKERS.some((p) => p(lines[i]))) ||
(n <= 60 && DEVICE_FOOTER.test(lines[i]))
@@ -161,14 +234,14 @@ export function cleanEmailBody(body: string, maxChars = 3000): string {
const kept = lines.slice(0, cut);
const paragraphs = kept
.join("\n")
- .split(/\n\s*\n/)
+ .split(new RegExp(`\\n${SP}*\\n`, "u"))
.map(stripDisclaimer);
const out = paragraphs
- .map((p) => p.trim())
+ .map(trimPy)
.filter(Boolean)
.join("\n\n")
.replace(/[ \t]+/g, " ");
- return out.slice(0, maxChars);
+ return cutCodePoints(out, maxChars);
}
export function emailState(
@@ -182,7 +255,7 @@ export function emailState(
maxChars = 3000,
): Record<string, unknown> {
const state: Record<string, unknown> = {
- subject: (subject ?? "").trim(),
+ subject: trimPy((subject ?? "")),
body: clean ? cleanEmailBody(body ?? "", maxChars) : (body ?? ""),
};
if (sender) state["from"] = sender;