Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

fix(ts): keep 'Thanks for the reply.' sentences in cleanEmailBody (Python parity) (#301)

Aashish Mahato668a5458513e
diff --git a/laya-ts/src/email.ts b/laya-ts/src/email.ts
index d142ea77..7fe82981 100644
--- a/laya-ts/src/email.ts
+++ b/laya-ts/src/email.ts
@@ -12,11 +12,29 @@ const ATTRIBUTION_TAIL = /^.{0,120}\S@\S+\s+(wrote|escreveu|escribi[óo]):\s*$/i
const ATTRIBUTION_HEAD = /^\s*(On|Em|El) (?=.*\d)/i;
const HEADER_FROM_NAME = /^\s*De:\s+\S/i;
const HEADER_NEXT = /^\s*(Enviad[oa]( em| el)?:\s|(Data|Fecha):\s.*\d{4})/i;
-const SIGNATURE_MARKERS: RegExp[] = [
- /^\s*--\s*$/,
- /^\s*(best|kind|warm|many thanks|thanks|thank you|regards|cheers|sincerely)[\w ,!.]*$/i,
- /^\s*sent from my (iphone|android|mobile|ipad)/i,
- /^\s*(atenciosamente|att|abraços?|abs|um abraço|cordialmente|grat[oa]|(muito )?obrigad[oa]s?( desde já| pela atenção)?|(com os melhores )?cumprimentos|saudações|(un )?saludos?( cordiales)?|atentamente|(muchas )?gracias( de antemano)?)[\s,!.]*$/i,
+// A closing line is the closing word plus punctuation and at most a name. Anything else on
+// the line is a sentence, and the case of the next word is what separates the two: a name is
+// capitalised, "for" in "Thanks for the quick reply." is not. JS regexes have no scoped
+// case-insensitive groups like Python's (?i:...), so the closing words are matched
+// case-insensitively here and the name is checked case-sensitively by SIGNOFF_TAIL.
+const SIGNOFF_HEAD =
+ /^\s*(?:best|kind|warmest|warm|many thanks|thanks|thank you|regards|cheers|sincerely)(?:\s+(?:and|&)\s+regards|\s+(?:regards|wishes|again|in advance|a lot|so much|very much))?/i;
+// Python's name class [^\W\d_a-zß-öø-ÿ] is "a word char that is not a digit, underscore or
+// lowercase letter"; \p{Lu}/\p{Lt}/\p{Lo} is the same intent: a name is capitalised in any
+// script (Regards, Łukasz) or written in a script without case (山田).
+const SIGNOFF_TAIL = /^[\s,;:!.]*(?:[\p{Lu}\p{Lt}\p{Lo}][\p{L}\p{M}\p{N}_'-]*[\s,.]*){0,3}$/u;
+function isEnglishSignoff(line: string): boolean {
+ const m = SIGNOFF_HEAD.exec(line);
+ return m !== null && SIGNOFF_TAIL.test(line.slice(m[0].length));
+}
+const SIGNATURE_MARKERS: Array<(line: string) => boolean> = [
+ (l) => /^\s*--\s*$/.test(l),
+ isEnglishSignoff,
+ (l) => /^\s*sent from my (iphone|android|mobile|ipad)/i.test(l),
+ (l) =>
+ /^\s*(atenciosamente|att|abraços?|abs|um abraço|cordialmente|grat[oa]|(muito )?obrigad[oa]s?( desde já| pela atenção)?|(com os melhores )?cumprimentos|saudações|(un )?saludos?( cordiales)?|atentamente|(muchas )?gracias( de antemano)?)[\s,!.]*$/i.test(
+ l,
+ ),
];
const DEVICE =
"iphone|ipad|android|ios|celular|telemóvel|móvil|galaxy|smartphone|samsung|tablet|" +
@@ -76,7 +94,10 @@ function stripDisclaimer(paragraph: string): string {
}
export function cleanEmailBody(body: string, maxChars = 3000): string {
- const text = (body ?? "").replace(/\r\n?/g, "\n").replace(/\\n/g, "\n");
+ let text = (body ?? "").replace(/\r\n?/g, "\n").replace(/\\n/g, "\n");
+ // Bound regex work before the expensive patterns below (Python does the same): only
+ // maxChars are ever returned, so nothing past 4x can survive cleaning.
+ if (text.length > maxChars * 4) text = text.slice(0, maxChars * 4);
const lines: string[] = [];
const src = text.split("\n");
for (let i = 0; i < src.length; i++) {
@@ -95,7 +116,7 @@ export function cleanEmailBody(body: string, maxChars = 3000): string {
for (let i = start; i < lines.length; i++) {
const n = lines[i].trim().length;
if (
- (n <= 40 && SIGNATURE_MARKERS.some((p) => p.test(lines[i]))) ||
+ (n <= 40 && SIGNATURE_MARKERS.some((p) => p(lines[i]))) ||
(n <= 60 && DEVICE_FOOTER.test(lines[i]))
) {
cut = i;
diff --git a/laya-ts/tests/email-presets.test.ts b/laya-ts/tests/email-presets.test.ts
index 793df497..4f503cdb 100644
--- a/laya-ts/tests/email-presets.test.ts
+++ b/laya-ts/tests/email-presets.test.ts
@@ -15,6 +15,31 @@ describe("email+presets", () => {
expect(out).toContain("Please refund my order");
expect(out).not.toContain("iPhone");
});
+ it("keeps a closing sentence that is not a sign-off (Python parity)", () => {
+ const body = "Please review the draft when you can.\nIt is two pages.\nThanks for the quick reply.";
+ expect(cleanEmailBody(body)).toBe(body);
+ });
+ it("does not cut words that merely start like a closing", () => {
+ const body = "Please review the draft when you can.\nIt is two pages.\nThanksgiving is next week.";
+ expect(cleanEmailBody(body)).toBe(body);
+ });
+ it("still cuts a real sign-off with a capitalised name", () => {
+ const out = cleanEmailBody("Please review the draft when you can.\nIt is two pages.\nThanks,\nMaria");
+ expect(out).toBe("Please review the draft when you can.\nIt is two pages.");
+ });
+ it("cuts warmest regards plus a diacritic name", () => {
+ const out = cleanEmailBody("Please review the draft when you can.\nIt is two pages.\nWarmest regards,\nŁukasz");
+ expect(out).toBe("Please review the draft when you can.\nIt is two pages.");
+ });
+ it("cuts thanks in advance plus a two-word name", () => {
+ const out = cleanEmailBody("Please review the draft when you can.\nIt is two pages.\nThanks in advance,\nPriya Nair");
+ expect(out).toBe("Please review the draft when you can.\nIt is two pages.");
+ });
+ it("bounds input to 4x maxChars before regex work (Python parity)", () => {
+ const out = cleanEmailBody("word ".repeat(3000) + "\nOn Mon, Bob wrote:\nold text");
+ expect(out.length).toBe(3000);
+ expect(out).not.toContain("old text");
+ });
it("guard preset has jailbreak and harm_severity", () => {
const g = guardQuestions() as Record<string, any>;
expect(g.jailbreak.type).toBe("noul");