Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

feat(dotnet): add email cleaning

kevin-gatimucbad9f9c5b60
diff --git a/laya-dotnet/src/Laya/LayaEmail.cs b/laya-dotnet/src/Laya/LayaEmail.cs
new file mode 100644
index 00000000..ffaf6a98
--- /dev/null
+++ b/laya-dotnet/src/Laya/LayaEmail.cs
@@ -0,0 +1,144 @@
+using System.Text.RegularExpressions;
+
+namespace Laya;
+
+/// <summary>Email utilities for cleaning and structuring email inputs. Ports <c>laya/email.py</c>.</summary>
+public static partial class LayaEmail
+{
+ // re.match anchors at the start only; every pattern below carries its own leading `^`, so
+ // .NET's Regex.IsMatch (which is not implicitly start-anchored) reproduces that exactly.
+ // `re.I` -> RegexOptions.IgnoreCase.
+ [GeneratedRegex(@"^\s*On .{0,300}wrote:\s*$", RegexOptions.IgnoreCase)]
+ private static partial Regex QuoteHeaderOnWrote();
+
+ [GeneratedRegex(@"^\s*-{2,}\s*(Original|Forwarded) Message\s*-{2,}", RegexOptions.IgnoreCase)]
+ private static partial Regex QuoteHeaderOriginalOrForwarded();
+
+ [GeneratedRegex(@"^\s*_{8,}\s*$")]
+ private static partial Regex QuoteHeaderUnderscores();
+
+ [GeneratedRegex(@"^\s*From:\s.+$", RegexOptions.IgnoreCase)]
+ private static partial Regex QuoteHeaderFromLine();
+
+ [GeneratedRegex(@"^\s*--\s*$")]
+ private static partial Regex SignatureDashDash();
+
+ [GeneratedRegex(@"^\s*(best|kind|warm|many thanks|thanks|thank you|regards|cheers|sincerely)[\w ,!.]*$",
+ RegexOptions.IgnoreCase)]
+ private static partial Regex SignatureClosing();
+
+ [GeneratedRegex(@"^\s*sent from my (iphone|android|mobile|ipad)", RegexOptions.IgnoreCase)]
+ private static partial Regex SignatureSentFromMobile();
+
+ // .search() semantics: no anchors, matches anywhere in the text.
+ [GeneratedRegex(
+ @"(confidential|intended (solely )?for the (use of the )?(named )?(addressee|recipient)|"
+ + @"if you (have )?received this (e-?mail|message) in error)",
+ RegexOptions.IgnoreCase)]
+ private static partial Regex Disclaimer();
+
+ // Sentence boundary: split just after a `.`/`!`/`?` and the whitespace that follows it. .NET
+ // Regex supports the lookbehind natively, so this needs no hand-written splitter.
+ [GeneratedRegex(@"(?<=[.!?])\s+")]
+ private static partial Regex SentenceBoundary();
+
+ [GeneratedRegex(@"\n\s*\n")]
+ private static partial Regex ParagraphBreak();
+
+ [GeneratedRegex(@"[ \t]+")]
+ private static partial Regex SpaceTabRun();
+
+ private static readonly Regex[] QuoteHeaders =
+ [
+ QuoteHeaderOnWrote(), QuoteHeaderOriginalOrForwarded(), QuoteHeaderUnderscores(), QuoteHeaderFromLine(),
+ ];
+
+ private static readonly Regex[] SignatureMarkers =
+ [
+ SignatureDashDash(), SignatureClosing(), SignatureSentFromMobile(),
+ ];
+
+ /// <summary>Remove quoted email history, signatures and disclaimers to keep input focused.</summary>
+ public static string CleanBody(string? body, int maxChars = 3000)
+ {
+ var text = (body ?? string.Empty).Replace("\r\n", "\n").Replace("\r", "\n").Replace("\\n", "\n");
+
+ var lines = new List<string>();
+ foreach (var line in text.Split('\n'))
+ {
+ if (lines.Count > 0 && Array.Exists(QuoteHeaders, p => p.IsMatch(line))) break;
+ if (line.TrimStart().StartsWith('>')) continue;
+ lines.Add(line.TrimEnd());
+ }
+
+ var cut = lines.Count;
+ var start = Math.Max(1, Math.Min((int)(lines.Count * 0.6), lines.Count - 8));
+ for (var i = start; i < lines.Count; i++)
+ {
+ var stripped = lines[i].Trim();
+ if (CodePointLength(stripped) <= 40 && Array.Exists(SignatureMarkers, p => p.IsMatch(lines[i])))
+ {
+ cut = i;
+ break;
+ }
+ }
+ lines = lines.GetRange(0, cut);
+
+ var paragraphs = ParagraphBreak().Split(string.Join('\n', lines)).Select(StripDisclaimer);
+ var collapsed = string.Join("\n\n", paragraphs.Select(p => p.Trim()).Where(p => p.Length > 0));
+ text = SpaceTabRun().Replace(collapsed, " ");
+ return LanguageDetection.SliceCodePoints(text, maxChars);
+ }
+
+ /// <summary>
+ /// Drop boilerplate disclaimer text from one paragraph.
+ /// </summary>
+ /// <remarks>
+ /// A paragraph is dropped whole only when *every* sentence in it is boilerplate; otherwise
+ /// only the boilerplate sentences go. A footer that runs on without a blank line would
+ /// otherwise take the sender's actual request with it, which is worse than leaving one
+ /// boilerplate line behind.
+ /// </remarks>
+ private static string StripDisclaimer(string paragraph)
+ {
+ if (!Disclaimer().IsMatch(paragraph)) return paragraph; // nothing to do: keep the original line structure
+ var parts = SentenceBoundary().Split(paragraph).Select(p => p.Trim()).Where(p => p.Length > 0);
+ return string.Join(' ', parts.Where(p => !Disclaimer().IsMatch(p)));
+ }
+
+ /// <summary>Construct a clean state dictionary for email classification.</summary>
+ /// <param name="subject">The email subject. Whitespace-trimmed regardless of <paramref name="clean"/>.</param>
+ /// <param name="body">The email body.</param>
+ /// <param name="sender">Optional sender, stored under the key <c>"from"</c> when non-empty.</param>
+ /// <param name="clean">
+ /// Whether <paramref name="body"/> is run through <see cref="CleanBody"/>. When
+ /// <see langword="false"/>, the raw body is kept as-is.
+ /// </param>
+ /// <param name="extra">
+ /// Additional fields to merge in, in the given order; a <see langword="null"/> value is
+ /// dropped entirely rather than stored, and a key that collides with <c>subject</c>/<c>body</c>/
+ /// <c>from</c> overwrites it in place, matching Python's <c>**extra</c> plus <c>dict.update</c>.
+ /// </param>
+ public static IReadOnlyDictionary<string, object?> State(
+ string? subject, string? body, string? sender = null, bool clean = true,
+ IEnumerable<KeyValuePair<string, object?>>? extra = null)
+ {
+ var state = new OrderedDictionary<string, object?>(StringComparer.Ordinal)
+ {
+ ["subject"] = (subject ?? string.Empty).Trim(),
+ ["body"] = clean ? CleanBody(body) : body ?? string.Empty,
+ };
+ if (!string.IsNullOrEmpty(sender)) state["from"] = sender;
+ if (extra is not null)
+ foreach (var (key, value) in extra)
+ if (value is not null) state[key] = value;
+ return state;
+ }
+
+ private static int CodePointLength(string s)
+ {
+ var count = 0;
+ foreach (var _ in s.EnumerateRunes()) count++;
+ return count;
+ }
+}
diff --git a/laya-dotnet/tests/Laya.Tests/LayaEmailTests.cs b/laya-dotnet/tests/Laya.Tests/LayaEmailTests.cs
new file mode 100644
index 00000000..cf8ec843
--- /dev/null
+++ b/laya-dotnet/tests/Laya.Tests/LayaEmailTests.cs
@@ -0,0 +1,159 @@
+using System.Text.Json;
+
+namespace Laya.Tests;
+
+/// <summary>
+/// Tier 1: email cleaning and state construction, which needs no artifacts. Named cases mirror
+/// <c>tests/test_email.py</c>; <see cref="EveryCleanProbeEntry"/> and
+/// <see cref="EveryStateProbeEntry"/> consume every entry of <c>golden/routing/email_probe.json</c>.
+/// </summary>
+public sealed class LayaEmailTests
+{
+ private const string Disclaimer = "This email is confidential and intended solely for the named addressee.";
+
+ // ── the request survives the footer (regression: disclaimer used to delete whole paragraphs) ──
+
+ [Fact]
+ public void InlineFooterNoBlankLineKeepsTheRequest() =>
+ Assert.Equal("My account is locked. Please unlock it.",
+ LayaEmail.CleanBody($"My account is locked.\n{Disclaimer}\nPlease unlock it."));
+
+ [Fact]
+ public void InlineFooterNoTerminalPunctuationStillRecoversTheRequest() =>
+ Assert.Equal("Please unlock it.",
+ LayaEmail.CleanBody($"My account is locked\n{Disclaimer}\nPlease unlock it."));
+
+ [Fact]
+ public void InlineFooterBodyIsNeverEmptied() =>
+ Assert.Equal("My account is locked.", LayaEmail.CleanBody($"My account is locked. {Disclaimer}"));
+
+ [Fact]
+ public void InlineFooterIsNotEmpty() =>
+ Assert.NotEqual("", LayaEmail.CleanBody($"My account is locked. {Disclaimer}").Trim());
+
+ [Fact]
+ public void EmailStateBodyKeepsTheRequest() =>
+ Assert.Equal("My account is locked.",
+ LayaEmail.State("Locked out", $"My account is locked. {Disclaimer}")["body"]);
+
+ // ── a pure footer is still removed ───────────────────────────────────────
+
+ [Fact]
+ public void StandaloneFooterParagraphIsStillDropped() =>
+ Assert.Equal("My account is locked.", LayaEmail.CleanBody($"My account is locked.\n\n{Disclaimer}"));
+
+ [Fact]
+ public void WrappedStandaloneFooterIsStillDropped() =>
+ Assert.Equal("My account is locked.", LayaEmail.CleanBody(
+ "My account is locked.\n\nThis email and any files transmitted with it are\n"
+ + "confidential and intended solely for the named addressee."));
+
+ [Fact]
+ public void ReceivedInErrorFooterIsStillDropped() =>
+ Assert.Equal("Please reopen ticket 4411.", LayaEmail.CleanBody(
+ "Please reopen ticket 4411.\n\nIf you have received this message in error, delete it."));
+
+ // ── unrelated cleaning is unchanged ──────────────────────────────────────
+
+ [Fact]
+ public void QuotedHistoryIsStillRemoved() =>
+ Assert.Equal("Thanks for the update.",
+ LayaEmail.CleanBody("Thanks for the update.\nOn Mon, Sep 20, Bob wrote:\n> original text"));
+
+ [Fact]
+ public void SignatureBlockIsStillRemoved() =>
+ Assert.Equal("Hi team,\nCan you confirm the refund?",
+ LayaEmail.CleanBody("Hi team,\nCan you confirm the refund?\nRegards,\nAlice"));
+
+ [Fact]
+ public void EmptyBodyStaysEmpty() => Assert.Equal("", LayaEmail.CleanBody(""));
+
+ [Fact]
+ public void NullBodyIsTreatedAsEmpty() => Assert.Equal("", LayaEmail.CleanBody(null));
+
+ // ── email_state ───────────────────────────────────────────────────────────
+
+ [Fact]
+ public void StateWithSenderAddsFromField()
+ {
+ var state = LayaEmail.State("Refund request", "Please refund invoice 4411.", sender: "[email protected]");
+ Assert.Equal("[email protected]", state["from"]);
+ // Insertion order: subject, body, from — matching Python's dict construction order.
+ Assert.Equal(["subject", "body", "from"], state.Keys);
+ }
+
+ [Fact]
+ public void StateSubjectIsWhitespaceStripped() =>
+ Assert.Equal("Refund request", LayaEmail.State(" Refund request ", "Please help.")["subject"]);
+
+ [Fact]
+ public void StateCleanFalseKeepsRawBody() =>
+ Assert.Equal("My account is locked.\nThis is not cleaned.",
+ LayaEmail.State("s", "My account is locked.\nThis is not cleaned.", clean: false)["body"]);
+
+ [Fact]
+ public void StateExtraNoneValuesAreDropped()
+ {
+ var state = LayaEmail.State("Ticket", "Body text.", extra:
+ [
+ new("nullable", null),
+ new("kept", "value"),
+ ]);
+ Assert.False(state.ContainsKey("nullable"));
+ Assert.Equal("value", state["kept"]);
+ }
+
+ // ── every recorded probe entry ────────────────────────────────────────────
+
+ public static IEnumerable<object[]> CleanProbeCases() => ProbeLabels("clean");
+ public static IEnumerable<object[]> StateProbeCases() => ProbeLabels("state");
+
+ private static IEnumerable<object[]> ProbeLabels(string section)
+ {
+ using var doc = RoutingGoldenData.Load("email_probe.json");
+ return
+ [
+ .. doc.RootElement.GetProperty(section).EnumerateArray()
+ .Select(e => new object[] { e.GetProperty("label").GetString()! }),
+ ];
+ }
+
+ private static JsonElement FindEntry(string section, string label)
+ {
+ using var doc = RoutingGoldenData.Load("email_probe.json");
+ foreach (var e in doc.RootElement.GetProperty(section).EnumerateArray())
+ if (e.GetProperty("label").GetString() == label)
+ return JsonDocument.Parse(e.GetRawText()).RootElement;
+ throw new InvalidOperationException($"no email_probe.json[{section}] entry labelled '{label}'");
+ }
+
+ [Theory]
+ [MemberData(nameof(CleanProbeCases))]
+ public void EveryCleanProbeEntry(string label)
+ {
+ var e = FindEntry("clean", label);
+ var body = e.GetProperty("body").GetString();
+ var maxChars = e.GetProperty("max_chars").GetInt32();
+ Assert.Equal(e.GetProperty("result").GetString(), LayaEmail.CleanBody(body, maxChars));
+ }
+
+ [Theory]
+ [MemberData(nameof(StateProbeCases))]
+ public void EveryStateProbeEntry(string label)
+ {
+ var e = FindEntry("state", label);
+ var subject = e.GetProperty("subject").GetString();
+ var body = e.GetProperty("body").GetString();
+ var sender = e.GetProperty("sender").ValueKind == JsonValueKind.Null ? null : e.GetProperty("sender").GetString();
+ var clean = e.GetProperty("clean").GetBoolean();
+ var extra = e.GetProperty("extra").EnumerateObject()
+ .Select(p => new KeyValuePair<string, object?>(p.Name, RoutingGoldenData.ToClr(p.Value)));
+
+ var got = LayaEmail.State(subject, body, sender, clean, extra);
+ var want = e.GetProperty("result");
+
+ Assert.Equal(want.EnumerateObject().Select(p => p.Name), got.Keys);
+ foreach (var p in want.EnumerateObject())
+ Assert.Equal(RoutingGoldenData.ToClr(p.Value), got[p.Name]);
+ }
+}