Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
fix(ts): clean French mail like Python
kevin9327d409bb07fcea
diff --git a/laya-ts/src/email.ts b/laya-ts/src/email.tsindex e56f7c1d..49c6a51b 100644--- a/laya-ts/src/email.ts+++ b/laya-ts/src/email.ts@@ -2,8 +2,11 @@ const QUOTE_HEADERS: RegExp[] = [/^\s*On .{0,300}wrote:\s*$/i,/^\s*Em (?=.*\d).{0,300}escreveu:\s*$/i,/^\s*El (?=.*\d).{0,300}escribi[óo]:\s*$/i,+ // Gmail's French attribution carries a date like the others ("Le lun. ... a écrit :"), and+ // French typography puts a space before the colon, so the verb allows one+ /^\s*Le (?=.*\d).{0,300}a [eé]crit\s*:\s*$/i,/^\s*-{2,}\s*(Original|Forwarded) Message\s*-{2,}/i,- /^\s*-{2,}\s*(Mensagem (original|encaminhada)|Mensaje (original|reenviado))\s*-{2,}/i,+ /^\s*-{2,}\s*(Mensagem (original|encaminhada)|Mensaje (original|reenviado)|Message d'origine)\s*-{2,}/i,/^\s*_{8,}\s*$/,// `From:` opens ordinary prose too ("From: my side the integration works, but please// refund..."), and a reply header always carries the sender, so the header is only recognised@@ -12,11 +15,12 @@ const QUOTE_HEADERS: RegExp[] = [// line to tell it apart from a sentence./^\s*From:\s.*[@<]/i,// `De:` also opens ordinary Portuguese/Spanish lines ("De: 10/09 a 15/09"), so the Outlook- // header is only recognised when it carries an address- /^\s*De:\s.*[@<]/i,+ // header is only recognised when it carries an address. French Outlook writes `De :` with a+ // space, so the marker allows one; the address rule is unchanged+ /^\s*De\s*:\s.*[@<]/i,];-const ATTRIBUTION_TAIL = /^.{0,120}\S@\S+\s+(wrote|escreveu|escribi[óo]):\s*$/i;-const ATTRIBUTION_HEAD = /^\s*(On|Em|El) (?=.*\d)/i;+const ATTRIBUTION_TAIL = /^.{0,120}\S@\S+\s+(wrote|escreveu|escribi[óo]|a [eé]crit)\s*:\s*$/i;+const ATTRIBUTION_HEAD = /^\s*(On|Em|El|Le) (?=.*\d)/i;// Exchange often leaves the address out of Outlook's reply header ("De: Maria Souza"), so a bare// `De:` only cuts when the header's own `Enviado:` line, or a dated `Data:`/`Fecha:` line, follows// it. `Para:` is not enough: "De: 10/09 / Para: 15/09" is how a leave request reads.@@ -24,9 +28,11 @@ const ATTRIBUTION_HEAD = /^\s*(On|Em|El) (?=.*\d)/i;// The same is true of a bare English `From: Maria Souza`, which is why the marker above needs// this rule: the English client lines are the translations of the two `De:` neighbours. A line// that only looks like prose still has to be told apart from a header by its neighbours, so the-// English pair is "From: <name>" followed by "Sent:"/"Date:".-const HEADER_FROM_NAME = /^\s*(De|From):\s+\S/i;-const HEADER_NEXT = /^\s*(Enviad[oa]( em| el)?:\s|Sent:\s|(Data|Fecha|Date):\s.*\d{4})/i;+// English pair is "From: <name>" followed by "Sent:"/"Date:". French Outlook writes the same+// header as `De : Marie Dupont` with `Envoyé :` underneath, so both allow the spaced colon.+const HEADER_FROM_NAME = /^\s*(De|From)\s*:\s+\S/i;+const HEADER_NEXT =+ /^\s*(Enviad[oa]( em| el)?:\s|Envoy[ée]( le)?\s*:\s|Sent:\s|(Data|Fecha|Date):\s.*\d{4})/i;// A closing line is the closing word plus punctuation and at most a name. Anything else on// the line is a sentence, and the case of the next word is what separates the two: a name is// capitalised, "for" in "Thanks for the quick reply." is not. JS regexes have no scoped@@ -46,7 +52,7 @@ const SIGNATURE_MARKERS: Array<(line: string) => boolean> = [(l) => /^\s*--\s*$/.test(l),isEnglishSignoff,(l) =>- /^\s*(atenciosamente|att|abraços?|abs|um abraço|cordialmente|grat[oa]|(muito )?obrigad[oa]s?( desde já| pela atenção)?|(com os melhores )?cumprimentos|saudações|(un )?saludos?( cordiales)?|atentamente|(muchas )?gracias( de antemano)?)[\s,!.]*$/i.test(+ /^\s*(atenciosamente|att|abraços?|abs|um abraço|cordialmente|grat[oa]|(muito )?obrigad[oa]s?( desde já| pela atenção)?|(com os melhores )?cumprimentos|saudações|(un )?saludos?( cordiales)?|atentamente|(muchas )?gracias( de antemano)?|(bien )?cordialement|salutations( distinguées)?|bien à vous|merci( d'avance)?|bonne journée)[\s,!.]*$/i.test(l,),];@@ -54,7 +60,8 @@ const DEVICE ="iphone|ipad|android|ios|mobile|celular|telemóvel|móvil|galaxy|smartphone|samsung|tablet|" +"outlook|yahoo|mail|e-?mail|gmail|windows";const DEVICE_FOOTER = new RegExp(- "^\\s*((enviad[oa] (do|pelo|pela|via|desde|a partir do)( meu| minha| mi)?|sent from( my)?)" ++ "^\\s*((enviad[oa] (do|pelo|pela|via|desde|a partir do)( meu| minha| mi)?|sent from( my)?|" ++ "envoy[ée] (depuis|de)( mon| ma| mes)?)" +` (${DEVICE})( (${DEVICE}|para|for|no|na|\\d+|phone|device|pro|max|mini|plus|using [a-z][\\w.+-]*))*` +"|(obter o|get) outlook (para|for) (ios|android))[\\s.!]*$","i",@@ -76,7 +83,12 @@ const DISCLAIMER = new RegExp("\\b(recebeu|recebido|receber) (esta|este) (mensagem|e-?mail)\\b[^.]{0,20} por (engano|erro)|" +"\\b(ha recibido|recibió|recibe) (este|esta) (mensaje|correo)\\b[^.]{0,20} por error|" +"\\bantes de imprimir\\b[^.]{0,100}(meio ambiente|medio ambiente|natureza|planeta|realmente necess)|" +- "\\b(meio|medio) ambiente\\b[^.]{0,30}antes de imprimir)",+ "\\b(meio|medio) ambiente\\b[^.]{0,30}antes de imprimir|" ++ // French is tied to "ce message/cet e-mail" rather than the bare word `confidentiel`, which a+ // sender's own request ("le contrat confidentiel") uses just as often+ "\\b(ce|cet|cette) (message|e-?mail|mail|courriel)\\b[^.]{0,80}(confidentiel|privil[eé]gi)|" ++ "\\bavez re[çc]u (ce|cet|cette) (message|e-?mail|mail)\\b[^.]{0,20} par erreur|" ++ "\\b(usage exclusif|exclusivement|uniquement)\\b[^.]{0,30}destinataire)","i",);const SENTENCE = /(?<=[.!?])\s+/;diff --git a/laya-ts/tests/email-presets.test.ts b/laya-ts/tests/email-presets.test.tsindex 4093958d..14075bba 100644--- a/laya-ts/tests/email-presets.test.ts+++ b/laya-ts/tests/email-presets.test.ts@@ -146,6 +146,86 @@ describe("email+presets", () => {cleanEmailBody("Please unlock it. This email is confidential and intended solely for the named addressee."),).toBe("Please unlock it.");});+ // French mail, the cases tests/test_email.py checks on the Python side (#736, #913)+ it("cleans a French reply the way Python does (Python parity)", () => {+ const body = [+ "Bonjour,",+ "",+ "J'ai été facturé deux fois sur la facture de mars. Merci de rembourser le double paiement aujourd'hui.",+ "",+ "Cordialement,",+ "Jean Dupont",+ "",+ "Envoyé depuis mon iPhone",+ "",+ "Ce message peut contenir des informations confidentielles. Si vous avez reçu ce message par erreur, merci de le supprimer.",+ "",+ "Le lun. 22 sept. 2026 à 10:14, Support <[email protected]> a écrit :",+ "> Bonjour Jean, nous avons reçu votre demande d'annulation du contrat Enterprise.",+ "",+ ].join("\n");+ expect(cleanEmailBody(body)).toBe(+ "Bonjour,\n\nJ'ai été facturé deux fois sur la facture de mars. " ++ "Merci de rembourser le double paiement aujourd'hui.",+ );+ });+ it("cuts French device footers and keeps French device sentences (Python parity)", () => {+ const request = "Merci de rembourser la facture.";+ for (const footer of [+ "Envoyé depuis mon iPhone",+ "Envoyé de mon iPad.",+ "Envoyé depuis iPhone",+ "Envoyé de iPad",+ "ENVOYÉ DEPUIS MON IPHONE",+ ]) {+ expect(cleanEmailBody(`${request}\n\n${footer}`), footer).toBe(request);+ }+ for (const sentence of ["Envoyé depuis mon iPhone par erreur.", "Envoyé de mon iPad hier."]) {+ const body = `Bonjour,\n${sentence}\n${request}`;+ expect(cleanEmailBody(body), sentence).toBe(body);+ }+ });+ it("cuts French quoted history and sign-offs (Python parity)", () => {+ const cases: Array<[string, string]> = [+ [+ "Voici le justificatif de paiement.\n\n-----Message d'origine-----\n" ++ "De : Marie <[email protected]>\nObjet : résilier le contrat\nNous voulons résilier le contrat.",+ "Voici le justificatif de paiement.",+ ],+ [+ "Voici le justificatif.\n\nDe : Marie Dupont\nEnvoyé : lundi 22 septembre 2026\n" ++ "Objet : résilier le contrat\nNous voulons résilier le contrat.",+ "Voici le justificatif.",+ ],+ [+ "L'accès est rétabli, merci.\n\nLe lun. 22 sept. 2026 à 10:14, Support Technique <\n" ++ "[email protected]> a écrit :\n> ancien texte",+ "L'accès est rétabli, merci.",+ ],+ ["Bonjour,\nLa facture de mars n'est pas arrivée.\nMerci,\nJean", "Bonjour,\nLa facture de mars n'est pas arrivée."],+ [+ "Bonjour,\nLa facture de mars n'est pas arrivée.\nBien à vous,\nMarie",+ "Bonjour,\nLa facture de mars n'est pas arrivée.",+ ],+ [+ "J'ai besoin de la facture de mars.\n\nSi vous avez reçu ce message par erreur, supprimez-le.",+ "J'ai besoin de la facture de mars.",+ ],+ ["Voici le devis demandé.\n\nCe document est à l'usage exclusif du destinataire.", "Voici le devis demandé."],+ ];+ for (const [body, expected] of cases) expect(cleanEmailBody(body), body).toBe(expected);+ });+ it("keeps French requests that only look like markers (Python parity)", () => {+ for (const body of [+ "Le contrat confidentiel doit être signé avant vendredi.",+ "Bonjour,\nMerci pour votre aide.\nRappelez-moi.",+ "Bonjour,\nLe rapport que vous avez écrit :\nla commande 4411 n'est pas arrivée.",+ "Le montant est destiné exclusivement au paiement de la facture. Pouvez-vous confirmer ?",+ "J'ai besoin de congés.\nDe : 10/09 à 15/09\nC'est possible ?",+ ]) {+ expect(cleanEmailBody(body), body).toBe(body);+ }+ });it("guard preset has jailbreak and harm_severity", () => {const g = guardQuestions() as Record<string, any>;expect(g.jailbreak.type).toBe("noul");