Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

fix(ts): clean French mail like Python

kevin9327d409bb07fcea
diff --git a/laya-ts/src/email.ts b/laya-ts/src/email.ts
index e56f7c1d..49c6a51b 100644
--- a/laya-ts/src/email.ts
+++ b/laya-ts/src/email.ts
@@ -2,8 +2,11 @@ const QUOTE_HEADERS: RegExp[] = [
/^\s*On .{0,300}wrote:\s*$/i,
/^\s*Em (?=.*\d).{0,300}escreveu:\s*$/i,
/^\s*El (?=.*\d).{0,300}escribi[óo]:\s*$/i,
+ // Gmail's French attribution carries a date like the others ("Le lun. ... a écrit :"), and
+ // French typography puts a space before the colon, so the verb allows one
+ /^\s*Le (?=.*\d).{0,300}a [eé]crit\s*:\s*$/i,
/^\s*-{2,}\s*(Original|Forwarded) Message\s*-{2,}/i,
- /^\s*-{2,}\s*(Mensagem (original|encaminhada)|Mensaje (original|reenviado))\s*-{2,}/i,
+ /^\s*-{2,}\s*(Mensagem (original|encaminhada)|Mensaje (original|reenviado)|Message d'origine)\s*-{2,}/i,
/^\s*_{8,}\s*$/,
// `From:` opens ordinary prose too ("From: my side the integration works, but please
// refund..."), and a reply header always carries the sender, so the header is only recognised
@@ -12,11 +15,12 @@ const QUOTE_HEADERS: RegExp[] = [
// line to tell it apart from a sentence.
/^\s*From:\s.*[@<]/i,
// `De:` also opens ordinary Portuguese/Spanish lines ("De: 10/09 a 15/09"), so the Outlook
- // header is only recognised when it carries an address
- /^\s*De:\s.*[@<]/i,
+ // header is only recognised when it carries an address. French Outlook writes `De :` with a
+ // space, so the marker allows one; the address rule is unchanged
+ /^\s*De\s*:\s.*[@<]/i,
];
-const ATTRIBUTION_TAIL = /^.{0,120}\S@\S+\s+(wrote|escreveu|escribi[óo]):\s*$/i;
-const ATTRIBUTION_HEAD = /^\s*(On|Em|El) (?=.*\d)/i;
+const ATTRIBUTION_TAIL = /^.{0,120}\S@\S+\s+(wrote|escreveu|escribi[óo]|a [eé]crit)\s*:\s*$/i;
+const ATTRIBUTION_HEAD = /^\s*(On|Em|El|Le) (?=.*\d)/i;
// Exchange often leaves the address out of Outlook's reply header ("De: Maria Souza"), so a bare
// `De:` only cuts when the header's own `Enviado:` line, or a dated `Data:`/`Fecha:` line, follows
// it. `Para:` is not enough: "De: 10/09 / Para: 15/09" is how a leave request reads.
@@ -24,9 +28,11 @@ const ATTRIBUTION_HEAD = /^\s*(On|Em|El) (?=.*\d)/i;
// The same is true of a bare English `From: Maria Souza`, which is why the marker above needs
// this rule: the English client lines are the translations of the two `De:` neighbours. A line
// that only looks like prose still has to be told apart from a header by its neighbours, so the
-// English pair is "From: <name>" followed by "Sent:"/"Date:".
-const HEADER_FROM_NAME = /^\s*(De|From):\s+\S/i;
-const HEADER_NEXT = /^\s*(Enviad[oa]( em| el)?:\s|Sent:\s|(Data|Fecha|Date):\s.*\d{4})/i;
+// English pair is "From: <name>" followed by "Sent:"/"Date:". French Outlook writes the same
+// header as `De : Marie Dupont` with `Envoyé :` underneath, so both allow the spaced colon.
+const HEADER_FROM_NAME = /^\s*(De|From)\s*:\s+\S/i;
+const HEADER_NEXT =
+ /^\s*(Enviad[oa]( em| el)?:\s|Envoy[ée]( le)?\s*:\s|Sent:\s|(Data|Fecha|Date):\s.*\d{4})/i;
// A closing line is the closing word plus punctuation and at most a name. Anything else on
// the line is a sentence, and the case of the next word is what separates the two: a name is
// capitalised, "for" in "Thanks for the quick reply." is not. JS regexes have no scoped
@@ -46,7 +52,7 @@ const SIGNATURE_MARKERS: Array<(line: string) => boolean> = [
(l) => /^\s*--\s*$/.test(l),
isEnglishSignoff,
(l) =>
- /^\s*(atenciosamente|att|abraços?|abs|um abraço|cordialmente|grat[oa]|(muito )?obrigad[oa]s?( desde já| pela atenção)?|(com os melhores )?cumprimentos|saudações|(un )?saludos?( cordiales)?|atentamente|(muchas )?gracias( de antemano)?)[\s,!.]*$/i.test(
+ /^\s*(atenciosamente|att|abraços?|abs|um abraço|cordialmente|grat[oa]|(muito )?obrigad[oa]s?( desde já| pela atenção)?|(com os melhores )?cumprimentos|saudações|(un )?saludos?( cordiales)?|atentamente|(muchas )?gracias( de antemano)?|(bien )?cordialement|salutations( distinguées)?|bien à vous|merci( d'avance)?|bonne journée)[\s,!.]*$/i.test(
l,
),
];
@@ -54,7 +60,8 @@ const DEVICE =
"iphone|ipad|android|ios|mobile|celular|telemóvel|móvil|galaxy|smartphone|samsung|tablet|" +
"outlook|yahoo|mail|e-?mail|gmail|windows";
const DEVICE_FOOTER = new RegExp(
- "^\\s*((enviad[oa] (do|pelo|pela|via|desde|a partir do)( meu| minha| mi)?|sent from( my)?)" +
+ "^\\s*((enviad[oa] (do|pelo|pela|via|desde|a partir do)( meu| minha| mi)?|sent from( my)?|" +
+ "envoy[ée] (depuis|de)( mon| ma| mes)?)" +
` (${DEVICE})( (${DEVICE}|para|for|no|na|\\d+|phone|device|pro|max|mini|plus|using [a-z][\\w.+-]*))*` +
"|(obter o|get) outlook (para|for) (ios|android))[\\s.!]*$",
"i",
@@ -76,7 +83,12 @@ const DISCLAIMER = new RegExp(
"\\b(recebeu|recebido|receber) (esta|este) (mensagem|e-?mail)\\b[^.]{0,20} por (engano|erro)|" +
"\\b(ha recibido|recibió|recibe) (este|esta) (mensaje|correo)\\b[^.]{0,20} por error|" +
"\\bantes de imprimir\\b[^.]{0,100}(meio ambiente|medio ambiente|natureza|planeta|realmente necess)|" +
- "\\b(meio|medio) ambiente\\b[^.]{0,30}antes de imprimir)",
+ "\\b(meio|medio) ambiente\\b[^.]{0,30}antes de imprimir|" +
+ // French is tied to "ce message/cet e-mail" rather than the bare word `confidentiel`, which a
+ // sender's own request ("le contrat confidentiel") uses just as often
+ "\\b(ce|cet|cette) (message|e-?mail|mail|courriel)\\b[^.]{0,80}(confidentiel|privil[eé]gi)|" +
+ "\\bavez re[çc]u (ce|cet|cette) (message|e-?mail|mail)\\b[^.]{0,20} par erreur|" +
+ "\\b(usage exclusif|exclusivement|uniquement)\\b[^.]{0,30}destinataire)",
"i",
);
const SENTENCE = /(?<=[.!?])\s+/;
diff --git a/laya-ts/tests/email-presets.test.ts b/laya-ts/tests/email-presets.test.ts
index 4093958d..14075bba 100644
--- a/laya-ts/tests/email-presets.test.ts
+++ b/laya-ts/tests/email-presets.test.ts
@@ -146,6 +146,86 @@ describe("email+presets", () => {
cleanEmailBody("Please unlock it. This email is confidential and intended solely for the named addressee."),
).toBe("Please unlock it.");
});
+ // French mail, the cases tests/test_email.py checks on the Python side (#736, #913)
+ it("cleans a French reply the way Python does (Python parity)", () => {
+ const body = [
+ "Bonjour,",
+ "",
+ "J'ai été facturé deux fois sur la facture de mars. Merci de rembourser le double paiement aujourd'hui.",
+ "",
+ "Cordialement,",
+ "Jean Dupont",
+ "",
+ "Envoyé depuis mon iPhone",
+ "",
+ "Ce message peut contenir des informations confidentielles. Si vous avez reçu ce message par erreur, merci de le supprimer.",
+ "",
+ "Le lun. 22 sept. 2026 à 10:14, Support <[email protected]> a écrit :",
+ "> Bonjour Jean, nous avons reçu votre demande d'annulation du contrat Enterprise.",
+ "",
+ ].join("\n");
+ expect(cleanEmailBody(body)).toBe(
+ "Bonjour,\n\nJ'ai été facturé deux fois sur la facture de mars. " +
+ "Merci de rembourser le double paiement aujourd'hui.",
+ );
+ });
+ it("cuts French device footers and keeps French device sentences (Python parity)", () => {
+ const request = "Merci de rembourser la facture.";
+ for (const footer of [
+ "Envoyé depuis mon iPhone",
+ "Envoyé de mon iPad.",
+ "Envoyé depuis iPhone",
+ "Envoyé de iPad",
+ "ENVOYÉ DEPUIS MON IPHONE",
+ ]) {
+ expect(cleanEmailBody(`${request}\n\n${footer}`), footer).toBe(request);
+ }
+ for (const sentence of ["Envoyé depuis mon iPhone par erreur.", "Envoyé de mon iPad hier."]) {
+ const body = `Bonjour,\n${sentence}\n${request}`;
+ expect(cleanEmailBody(body), sentence).toBe(body);
+ }
+ });
+ it("cuts French quoted history and sign-offs (Python parity)", () => {
+ const cases: Array<[string, string]> = [
+ [
+ "Voici le justificatif de paiement.\n\n-----Message d'origine-----\n" +
+ "De : Marie <[email protected]>\nObjet : résilier le contrat\nNous voulons résilier le contrat.",
+ "Voici le justificatif de paiement.",
+ ],
+ [
+ "Voici le justificatif.\n\nDe : Marie Dupont\nEnvoyé : lundi 22 septembre 2026\n" +
+ "Objet : résilier le contrat\nNous voulons résilier le contrat.",
+ "Voici le justificatif.",
+ ],
+ [
+ "L'accès est rétabli, merci.\n\nLe lun. 22 sept. 2026 à 10:14, Support Technique <\n" +
+ "[email protected]> a écrit :\n> ancien texte",
+ "L'accès est rétabli, merci.",
+ ],
+ ["Bonjour,\nLa facture de mars n'est pas arrivée.\nMerci,\nJean", "Bonjour,\nLa facture de mars n'est pas arrivée."],
+ [
+ "Bonjour,\nLa facture de mars n'est pas arrivée.\nBien à vous,\nMarie",
+ "Bonjour,\nLa facture de mars n'est pas arrivée.",
+ ],
+ [
+ "J'ai besoin de la facture de mars.\n\nSi vous avez reçu ce message par erreur, supprimez-le.",
+ "J'ai besoin de la facture de mars.",
+ ],
+ ["Voici le devis demandé.\n\nCe document est à l'usage exclusif du destinataire.", "Voici le devis demandé."],
+ ];
+ for (const [body, expected] of cases) expect(cleanEmailBody(body), body).toBe(expected);
+ });
+ it("keeps French requests that only look like markers (Python parity)", () => {
+ for (const body of [
+ "Le contrat confidentiel doit être signé avant vendredi.",
+ "Bonjour,\nMerci pour votre aide.\nRappelez-moi.",
+ "Bonjour,\nLe rapport que vous avez écrit :\nla commande 4411 n'est pas arrivée.",
+ "Le montant est destiné exclusivement au paiement de la facture. Pouvez-vous confirmer ?",
+ "J'ai besoin de congés.\nDe : 10/09 à 15/09\nC'est possible ?",
+ ]) {
+ expect(cleanEmailBody(body), body).toBe(body);
+ }
+ });
it("guard preset has jailbreak and harm_severity", () => {
const g = guardQuestions() as Record<string, any>;
expect(g.jailbreak.type).toBe("noul");