Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

fix(dotnet): preserve email requests and match current client markers

kevin-gatimube14e6bba018
diff --git a/laya-dotnet/src/Laya/LayaEmail.cs b/laya-dotnet/src/Laya/LayaEmail.cs
index ffaf6a98..a1b7f3d3 100644
--- a/laya-dotnet/src/Laya/LayaEmail.cs
+++ b/laya-dotnet/src/Laya/LayaEmail.cs
@@ -1,3 +1,4 @@
+using System.Text;
using System.Text.RegularExpressions;
namespace Laya;
@@ -11,29 +12,70 @@ public static partial class LayaEmail
[GeneratedRegex(@"^\s*On .{0,300}wrote:\s*$", RegexOptions.IgnoreCase)]
private static partial Regex QuoteHeaderOnWrote();
+ [GeneratedRegex(@"^\s*(Em (?=.*\d).{0,300}escreveu:|El (?=.*\d).{0,300}escribi[óo]:|Le (?=.*\d).{0,300}a [eé]crit\s*:)\s*$", RegexOptions.IgnoreCase)]
+ private static partial Regex QuoteHeaderTranslated();
+
[GeneratedRegex(@"^\s*-{2,}\s*(Original|Forwarded) Message\s*-{2,}", RegexOptions.IgnoreCase)]
private static partial Regex QuoteHeaderOriginalOrForwarded();
+ [GeneratedRegex(@"^\s*-{2,}\s*(Mensagem (original|encaminhada)|Mensaje (original|reenviado)|Message d'origine)\s*-{2,}", RegexOptions.IgnoreCase)]
+ private static partial Regex QuoteHeaderTranslatedMessage();
+
[GeneratedRegex(@"^\s*_{8,}\s*$")]
private static partial Regex QuoteHeaderUnderscores();
- [GeneratedRegex(@"^\s*From:\s.+$", RegexOptions.IgnoreCase)]
+ [GeneratedRegex(@"^\s*(From:|De\s*:)\s.*[@<]", RegexOptions.IgnoreCase)]
private static partial Regex QuoteHeaderFromLine();
+ [GeneratedRegex(@"^\s*(De|From)\s*:\s+\S", RegexOptions.IgnoreCase)]
+ private static partial Regex HeaderFromName();
+
+ [GeneratedRegex(@"^\s*(Enviad[oa]( em| el)?:\s|Envoy[ée]( le)?\s*:\s|Sent:\s|(Data|Fecha|Date):\s.*\d{4})", RegexOptions.IgnoreCase)]
+ private static partial Regex HeaderNext();
+
+ [GeneratedRegex(@"^.{0,120}\S@\S+\s+(wrote|escreveu|escribi[óo]|a [eé]crit)\s*:\s*$", RegexOptions.IgnoreCase)]
+ private static partial Regex AttributionTail();
+
+ [GeneratedRegex(@"^\s*(On|Em|El|Le) (?=.*\d)", RegexOptions.IgnoreCase)]
+ private static partial Regex AttributionHead();
+
[GeneratedRegex(@"^\s*--\s*$")]
private static partial Regex SignatureDashDash();
- [GeneratedRegex(@"^\s*(best|kind|warm|many thanks|thanks|thank you|regards|cheers|sincerely)[\w ,!.]*$",
+ [GeneratedRegex(@"^\s*(?i:best|kind|warmest|warm|many thanks|thanks|thank you|regards|cheers|sincerely)"
+ + @"(?i:\s+(?:and|&)\s+regards|\s+(?:regards|wishes|again|in advance|a lot|so much|very much))?")]
+ private static partial Regex SignatureClosingHead();
+
+ [GeneratedRegex(@"^\s*(atenciosamente|att|abraços?|abs|um abraço|cordialmente|grat[oa]|(muito )?obrigad[oa]s?"
+ + @"( desde já| pela atenção)?|(com os melhores )?cumprimentos|saudações|"
+ + @"(un )?saludos?( cordiales)?|atentamente|(muchas )?gracias( de antemano)?|"
+ + @"(bien )?cordialement|salutations( distinguées)?|bien à vous|merci( d'avance)?|bonne journée)[\s,!.]*$",
RegexOptions.IgnoreCase)]
- private static partial Regex SignatureClosing();
+ private static partial Regex SignatureTranslated();
+
+ private const string Device = @"iphone|ipad|android|ios|mobile|celular|telemóvel|móvil|galaxy|smartphone|samsung|tablet|outlook|yahoo|mail|e-?mail|gmail|windows";
- [GeneratedRegex(@"^\s*sent from my (iphone|android|mobile|ipad)", RegexOptions.IgnoreCase)]
+ [GeneratedRegex(@"^\s*((enviad[oa] (do|pelo|pela|via|desde|a partir do)( meu| minha| mi)?|sent from( my)?|"
+ + @"envoy[ée] (depuis|de) (mon |ma |mes )?)"
+ + @" (" + Device + @")( (" + Device + @"|para|for|no|na|\d+|phone|device|pro|max|mini|plus|using [a-z][a-z0-9_.+-]*))*"
+ + @"|(obter o|get) outlook (para|for) (ios|android))[\s.!]*$", RegexOptions.IgnoreCase)]
private static partial Regex SignatureSentFromMobile();
// .search() semantics: no anchors, matches anywhere in the text.
[GeneratedRegex(
- @"(confidential|intended (solely )?for the (use of the )?(named )?(addressee|recipient)|"
- + @"if you (have )?received this (e-?mail|message) in error)",
+ @"(\b(e-?mail|message|information|communication|transmission|contents?)\b[^.]{0,60}"
+ + @"\bconfidential\b[^.]{0,60}\b(intended|solely|addressee|recipient|privileged|disclos|unauthori[sz]ed)|"
+ + @"\bconfidential\b[^.]{0,60}\b(and (may|is) (also )?privileged)|"
+ + @"if you (have )?received this (e-?mail|message) in error|"
+ + @"\b(esta|este) (mensagem|e-?mail|mensaje|correo)\b[^.]{0,80}(confidencia|sigilos|privilegiad)|"
+ + @"\b(uso exclusivo|exclusivamente|únicamente|unicamente)\b[^.]{0,30}(destinatári|destinatari|pessoa|persona|entidade|entidad)|"
+ + @"\b(recebeu|recebido|receber) (esta|este) (mensagem|e-?mail)\b[^.]{0,20} por (engano|erro)|"
+ + @"\b(ha recibido|recibió|recibe) (este|esta) (mensaje|correo)\b[^.]{0,20} por error|"
+ + @"\bantes de imprimir\b[^.]{0,100}(meio ambiente|medio ambiente|natureza|planeta|realmente necess)|"
+ + @"\b(meio|medio) ambiente\b[^.]{0,30}antes de imprimir|"
+ + @"\b(ce|cet|cette) (message|e-?mail|mail|courriel)\b[^.]{0,80}(confidentiel|privil[eé]gi)|"
+ + @"\bavez re[çc]u (ce|cet|cette) (message|e-?mail|mail)\b[^.]{0,20} par erreur|"
+ + @"\b(usage exclusif|exclusivement|uniquement)\b[^.]{0,30}destinataire)",
RegexOptions.IgnoreCase)]
private static partial Regex Disclaimer();
@@ -50,23 +92,35 @@ public static partial class LayaEmail
private static readonly Regex[] QuoteHeaders =
[
- QuoteHeaderOnWrote(), QuoteHeaderOriginalOrForwarded(), QuoteHeaderUnderscores(), QuoteHeaderFromLine(),
+ QuoteHeaderOnWrote(), QuoteHeaderTranslated(), QuoteHeaderOriginalOrForwarded(),
+ QuoteHeaderTranslatedMessage(), QuoteHeaderUnderscores(), QuoteHeaderFromLine(),
];
private static readonly Regex[] SignatureMarkers =
[
- SignatureDashDash(), SignatureClosing(), SignatureSentFromMobile(),
+ SignatureDashDash(), SignatureTranslated(),
];
/// <summary>Remove quoted email history, signatures and disclaimers to keep input focused.</summary>
public static string CleanBody(string? body, int maxChars = 3000)
{
var text = (body ?? string.Empty).Replace("\r\n", "\n").Replace("\r", "\n").Replace("\\n", "\n");
+ if (CodePointLength(text) > maxChars * 4)
+ text = LanguageDetection.SliceCodePoints(text, maxChars * 4);
var lines = new List<string>();
- foreach (var line in text.Split('\n'))
+ var source = text.Split('\n');
+ for (var i = 0; i < source.Length; i++)
{
+ var line = source[i];
if (lines.Count > 0 && Array.Exists(QuoteHeaders, p => p.IsMatch(line))) break;
+ if (lines.Count > 0 && HeaderFromName().IsMatch(line) && i + 1 < source.Length
+ && HeaderNext().IsMatch(source[i + 1])) break;
+ if (lines.Count > 0 && AttributionTail().IsMatch(line))
+ {
+ if (AttributionHead().IsMatch(lines[^1])) lines.RemoveAt(lines.Count - 1);
+ break;
+ }
if (line.TrimStart().StartsWith('>')) continue;
lines.Add(line.TrimEnd());
}
@@ -76,7 +130,9 @@ public static partial class LayaEmail
for (var i = start; i < lines.Count; i++)
{
var stripped = lines[i].Trim();
- if (CodePointLength(stripped) <= 40 && Array.Exists(SignatureMarkers, p => p.IsMatch(lines[i])))
+ var length = CodePointLength(stripped);
+ if ((length <= 40 && (IsEnglishSignoff(lines[i]) || Array.Exists(SignatureMarkers, p => p.IsMatch(lines[i]))))
+ || (length <= 60 && SignatureSentFromMobile().IsMatch(lines[i])))
{
cut = i;
break;
@@ -90,6 +146,38 @@ public static partial class LayaEmail
return LanguageDetection.SliceCodePoints(text, maxChars);
}
+ private static bool IsEnglishSignoff(string line)
+ {
+ var head = SignatureClosingHead().Match(line);
+ if (!head.Success) return false;
+ var tail = new List<Rune>();
+ foreach (var rune in line[head.Length..].EnumerateRunes())
+ {
+ if (Rune.GetUnicodeCategory(rune) is System.Globalization.UnicodeCategory.NonSpacingMark
+ or System.Globalization.UnicodeCategory.SpacingCombiningMark
+ or System.Globalization.UnicodeCategory.EnclosingMark
+ && tail.Count > 0 && !Rune.IsWhiteSpace(tail[^1])) continue;
+ tail.Add(rune);
+ }
+ var i = 0;
+ while (i < tail.Count && (Rune.IsWhiteSpace(tail[i]) || tail[i].Value is ',' or ';' or ':' or '!' or '.')) i++;
+ var tokens = 0;
+ while (i < tail.Count)
+ {
+ if (++tokens > 3 || Rune.GetUnicodeCategory(tail[i]) is not
+ (System.Globalization.UnicodeCategory.UppercaseLetter
+ or System.Globalization.UnicodeCategory.TitlecaseLetter
+ or System.Globalization.UnicodeCategory.OtherLetter)) return false;
+ i++;
+ while (i < tail.Count && (Rune.IsLetterOrDigit(tail[i])
+ || Rune.GetUnicodeCategory(tail[i]) is System.Globalization.UnicodeCategory.LetterNumber
+ or System.Globalization.UnicodeCategory.OtherNumber
+ || tail[i].Value is '_' or '\'' or '-')) i++;
+ while (i < tail.Count && (Rune.IsWhiteSpace(tail[i]) || tail[i].Value is ',' or '.')) i++;
+ }
+ return true;
+ }
+
/// <summary>
/// Drop boilerplate disclaimer text from one paragraph.
/// </summary>
@@ -102,8 +190,24 @@ public static partial class LayaEmail
private static string StripDisclaimer(string paragraph)
{
if (!Disclaimer().IsMatch(paragraph)) return paragraph; // nothing to do: keep the original line structure
- var parts = SentenceBoundary().Split(paragraph).Select(p => p.Trim()).Where(p => p.Length > 0);
- return string.Join(' ', parts.Where(p => !Disclaimer().IsMatch(p)));
+ var pieces = new List<string>();
+ foreach (var part in SentenceBoundary().Split(paragraph).Select(p => p.Trim()).Where(p => p.Length > 0))
+ {
+ if (!Disclaimer().IsMatch(part)) { pieces.Add(part); continue; }
+ var buffer = "";
+ foreach (var line in part.Split('\n').Select(l => l.Trim()).Where(l => l.Length > 0))
+ {
+ var firstLetter = line.EnumerateRunes().FirstOrDefault(Rune.IsLetter);
+ if (buffer.Length > 0 && Rune.IsUpper(firstLetter))
+ {
+ pieces.Add(buffer);
+ buffer = line;
+ }
+ else buffer = buffer.Length > 0 ? buffer + " " + line : line;
+ }
+ if (buffer.Length > 0) pieces.Add(buffer);
+ }
+ return string.Join(' ', pieces.Where(p => !Disclaimer().IsMatch(p)));
}
/// <summary>Construct a clean state dictionary for email classification.</summary>
diff --git a/laya-dotnet/tests/Laya.Tests/LayaEmailTests.cs b/laya-dotnet/tests/Laya.Tests/LayaEmailTests.cs
index cf8ec843..ed891e38 100644
--- a/laya-dotnet/tests/Laya.Tests/LayaEmailTests.cs
+++ b/laya-dotnet/tests/Laya.Tests/LayaEmailTests.cs
@@ -20,7 +20,7 @@ public sealed class LayaEmailTests
[Fact]
public void InlineFooterNoTerminalPunctuationStillRecoversTheRequest() =>
- Assert.Equal("Please unlock it.",
+ Assert.Equal("My account is locked Please unlock it.",
LayaEmail.CleanBody($"My account is locked\n{Disclaimer}\nPlease unlock it."));
[Fact]