Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
feat(dotnet): add email cleaning
kevin-gatimucbad9f9c5b60
diff --git a/laya-dotnet/src/Laya/LayaEmail.cs b/laya-dotnet/src/Laya/LayaEmail.csnew file mode 100644index 00000000..ffaf6a98--- /dev/null+++ b/laya-dotnet/src/Laya/LayaEmail.cs@@ -0,0 +1,144 @@+using System.Text.RegularExpressions;++namespace Laya;++/// <summary>Email utilities for cleaning and structuring email inputs. Ports <c>laya/email.py</c>.</summary>+public static partial class LayaEmail+{+ // re.match anchors at the start only; every pattern below carries its own leading `^`, so+ // .NET's Regex.IsMatch (which is not implicitly start-anchored) reproduces that exactly.+ // `re.I` -> RegexOptions.IgnoreCase.+ [GeneratedRegex(@"^\s*On .{0,300}wrote:\s*$", RegexOptions.IgnoreCase)]+ private static partial Regex QuoteHeaderOnWrote();++ [GeneratedRegex(@"^\s*-{2,}\s*(Original|Forwarded) Message\s*-{2,}", RegexOptions.IgnoreCase)]+ private static partial Regex QuoteHeaderOriginalOrForwarded();++ [GeneratedRegex(@"^\s*_{8,}\s*$")]+ private static partial Regex QuoteHeaderUnderscores();++ [GeneratedRegex(@"^\s*From:\s.+$", RegexOptions.IgnoreCase)]+ private static partial Regex QuoteHeaderFromLine();++ [GeneratedRegex(@"^\s*--\s*$")]+ private static partial Regex SignatureDashDash();++ [GeneratedRegex(@"^\s*(best|kind|warm|many thanks|thanks|thank you|regards|cheers|sincerely)[\w ,!.]*$",+ RegexOptions.IgnoreCase)]+ private static partial Regex SignatureClosing();++ [GeneratedRegex(@"^\s*sent from my (iphone|android|mobile|ipad)", RegexOptions.IgnoreCase)]+ private static partial Regex SignatureSentFromMobile();++ // .search() semantics: no anchors, matches anywhere in the text.+ [GeneratedRegex(+ @"(confidential|intended (solely )?for the (use of the )?(named )?(addressee|recipient)|"+ + @"if you (have )?received this (e-?mail|message) in error)",+ RegexOptions.IgnoreCase)]+ private static partial Regex Disclaimer();++ // Sentence boundary: split just after a `.`/`!`/`?` and the whitespace that follows it. .NET+ // Regex supports the lookbehind natively, so this needs no hand-written splitter.+ [GeneratedRegex(@"(?<=[.!?])\s+")]+ private static partial Regex SentenceBoundary();++ [GeneratedRegex(@"\n\s*\n")]+ private static partial Regex ParagraphBreak();++ [GeneratedRegex(@"[ \t]+")]+ private static partial Regex SpaceTabRun();++ private static readonly Regex[] QuoteHeaders =+ [+ QuoteHeaderOnWrote(), QuoteHeaderOriginalOrForwarded(), QuoteHeaderUnderscores(), QuoteHeaderFromLine(),+ ];++ private static readonly Regex[] SignatureMarkers =+ [+ SignatureDashDash(), SignatureClosing(), SignatureSentFromMobile(),+ ];++ /// <summary>Remove quoted email history, signatures and disclaimers to keep input focused.</summary>+ public static string CleanBody(string? body, int maxChars = 3000)+ {+ var text = (body ?? string.Empty).Replace("\r\n", "\n").Replace("\r", "\n").Replace("\\n", "\n");++ var lines = new List<string>();+ foreach (var line in text.Split('\n'))+ {+ if (lines.Count > 0 && Array.Exists(QuoteHeaders, p => p.IsMatch(line))) break;+ if (line.TrimStart().StartsWith('>')) continue;+ lines.Add(line.TrimEnd());+ }++ var cut = lines.Count;+ var start = Math.Max(1, Math.Min((int)(lines.Count * 0.6), lines.Count - 8));+ for (var i = start; i < lines.Count; i++)+ {+ var stripped = lines[i].Trim();+ if (CodePointLength(stripped) <= 40 && Array.Exists(SignatureMarkers, p => p.IsMatch(lines[i])))+ {+ cut = i;+ break;+ }+ }+ lines = lines.GetRange(0, cut);++ var paragraphs = ParagraphBreak().Split(string.Join('\n', lines)).Select(StripDisclaimer);+ var collapsed = string.Join("\n\n", paragraphs.Select(p => p.Trim()).Where(p => p.Length > 0));+ text = SpaceTabRun().Replace(collapsed, " ");+ return LanguageDetection.SliceCodePoints(text, maxChars);+ }++ /// <summary>+ /// Drop boilerplate disclaimer text from one paragraph.+ /// </summary>+ /// <remarks>+ /// A paragraph is dropped whole only when *every* sentence in it is boilerplate; otherwise+ /// only the boilerplate sentences go. A footer that runs on without a blank line would+ /// otherwise take the sender's actual request with it, which is worse than leaving one+ /// boilerplate line behind.+ /// </remarks>+ private static string StripDisclaimer(string paragraph)+ {+ if (!Disclaimer().IsMatch(paragraph)) return paragraph; // nothing to do: keep the original line structure+ var parts = SentenceBoundary().Split(paragraph).Select(p => p.Trim()).Where(p => p.Length > 0);+ return string.Join(' ', parts.Where(p => !Disclaimer().IsMatch(p)));+ }++ /// <summary>Construct a clean state dictionary for email classification.</summary>+ /// <param name="subject">The email subject. Whitespace-trimmed regardless of <paramref name="clean"/>.</param>+ /// <param name="body">The email body.</param>+ /// <param name="sender">Optional sender, stored under the key <c>"from"</c> when non-empty.</param>+ /// <param name="clean">+ /// Whether <paramref name="body"/> is run through <see cref="CleanBody"/>. When+ /// <see langword="false"/>, the raw body is kept as-is.+ /// </param>+ /// <param name="extra">+ /// Additional fields to merge in, in the given order; a <see langword="null"/> value is+ /// dropped entirely rather than stored, and a key that collides with <c>subject</c>/<c>body</c>/+ /// <c>from</c> overwrites it in place, matching Python's <c>**extra</c> plus <c>dict.update</c>.+ /// </param>+ public static IReadOnlyDictionary<string, object?> State(+ string? subject, string? body, string? sender = null, bool clean = true,+ IEnumerable<KeyValuePair<string, object?>>? extra = null)+ {+ var state = new OrderedDictionary<string, object?>(StringComparer.Ordinal)+ {+ ["subject"] = (subject ?? string.Empty).Trim(),+ ["body"] = clean ? CleanBody(body) : body ?? string.Empty,+ };+ if (!string.IsNullOrEmpty(sender)) state["from"] = sender;+ if (extra is not null)+ foreach (var (key, value) in extra)+ if (value is not null) state[key] = value;+ return state;+ }++ private static int CodePointLength(string s)+ {+ var count = 0;+ foreach (var _ in s.EnumerateRunes()) count++;+ return count;+ }+}diff --git a/laya-dotnet/tests/Laya.Tests/LayaEmailTests.cs b/laya-dotnet/tests/Laya.Tests/LayaEmailTests.csnew file mode 100644index 00000000..cf8ec843--- /dev/null+++ b/laya-dotnet/tests/Laya.Tests/LayaEmailTests.cs@@ -0,0 +1,159 @@+using System.Text.Json;++namespace Laya.Tests;++/// <summary>+/// Tier 1: email cleaning and state construction, which needs no artifacts. Named cases mirror+/// <c>tests/test_email.py</c>; <see cref="EveryCleanProbeEntry"/> and+/// <see cref="EveryStateProbeEntry"/> consume every entry of <c>golden/routing/email_probe.json</c>.+/// </summary>+public sealed class LayaEmailTests+{+ private const string Disclaimer = "This email is confidential and intended solely for the named addressee.";++ // ── the request survives the footer (regression: disclaimer used to delete whole paragraphs) ──++ [Fact]+ public void InlineFooterNoBlankLineKeepsTheRequest() =>+ Assert.Equal("My account is locked. Please unlock it.",+ LayaEmail.CleanBody($"My account is locked.\n{Disclaimer}\nPlease unlock it."));++ [Fact]+ public void InlineFooterNoTerminalPunctuationStillRecoversTheRequest() =>+ Assert.Equal("Please unlock it.",+ LayaEmail.CleanBody($"My account is locked\n{Disclaimer}\nPlease unlock it."));++ [Fact]+ public void InlineFooterBodyIsNeverEmptied() =>+ Assert.Equal("My account is locked.", LayaEmail.CleanBody($"My account is locked. {Disclaimer}"));++ [Fact]+ public void InlineFooterIsNotEmpty() =>+ Assert.NotEqual("", LayaEmail.CleanBody($"My account is locked. {Disclaimer}").Trim());++ [Fact]+ public void EmailStateBodyKeepsTheRequest() =>+ Assert.Equal("My account is locked.",+ LayaEmail.State("Locked out", $"My account is locked. {Disclaimer}")["body"]);++ // ── a pure footer is still removed ───────────────────────────────────────++ [Fact]+ public void StandaloneFooterParagraphIsStillDropped() =>+ Assert.Equal("My account is locked.", LayaEmail.CleanBody($"My account is locked.\n\n{Disclaimer}"));++ [Fact]+ public void WrappedStandaloneFooterIsStillDropped() =>+ Assert.Equal("My account is locked.", LayaEmail.CleanBody(+ "My account is locked.\n\nThis email and any files transmitted with it are\n"+ + "confidential and intended solely for the named addressee."));++ [Fact]+ public void ReceivedInErrorFooterIsStillDropped() =>+ Assert.Equal("Please reopen ticket 4411.", LayaEmail.CleanBody(+ "Please reopen ticket 4411.\n\nIf you have received this message in error, delete it."));++ // ── unrelated cleaning is unchanged ──────────────────────────────────────++ [Fact]+ public void QuotedHistoryIsStillRemoved() =>+ Assert.Equal("Thanks for the update.",+ LayaEmail.CleanBody("Thanks for the update.\nOn Mon, Sep 20, Bob wrote:\n> original text"));++ [Fact]+ public void SignatureBlockIsStillRemoved() =>+ Assert.Equal("Hi team,\nCan you confirm the refund?",+ LayaEmail.CleanBody("Hi team,\nCan you confirm the refund?\nRegards,\nAlice"));++ [Fact]+ public void EmptyBodyStaysEmpty() => Assert.Equal("", LayaEmail.CleanBody(""));++ [Fact]+ public void NullBodyIsTreatedAsEmpty() => Assert.Equal("", LayaEmail.CleanBody(null));++ // ── email_state ───────────────────────────────────────────────────────────++ [Fact]+ public void StateWithSenderAddsFromField()+ {+ var state = LayaEmail.State("Refund request", "Please refund invoice 4411.", sender: "[email protected]");+ Assert.Equal("[email protected]", state["from"]);+ // Insertion order: subject, body, from — matching Python's dict construction order.+ Assert.Equal(["subject", "body", "from"], state.Keys);+ }++ [Fact]+ public void StateSubjectIsWhitespaceStripped() =>+ Assert.Equal("Refund request", LayaEmail.State(" Refund request ", "Please help.")["subject"]);++ [Fact]+ public void StateCleanFalseKeepsRawBody() =>+ Assert.Equal("My account is locked.\nThis is not cleaned.",+ LayaEmail.State("s", "My account is locked.\nThis is not cleaned.", clean: false)["body"]);++ [Fact]+ public void StateExtraNoneValuesAreDropped()+ {+ var state = LayaEmail.State("Ticket", "Body text.", extra:+ [+ new("nullable", null),+ new("kept", "value"),+ ]);+ Assert.False(state.ContainsKey("nullable"));+ Assert.Equal("value", state["kept"]);+ }++ // ── every recorded probe entry ────────────────────────────────────────────++ public static IEnumerable<object[]> CleanProbeCases() => ProbeLabels("clean");+ public static IEnumerable<object[]> StateProbeCases() => ProbeLabels("state");++ private static IEnumerable<object[]> ProbeLabels(string section)+ {+ using var doc = RoutingGoldenData.Load("email_probe.json");+ return+ [+ .. doc.RootElement.GetProperty(section).EnumerateArray()+ .Select(e => new object[] { e.GetProperty("label").GetString()! }),+ ];+ }++ private static JsonElement FindEntry(string section, string label)+ {+ using var doc = RoutingGoldenData.Load("email_probe.json");+ foreach (var e in doc.RootElement.GetProperty(section).EnumerateArray())+ if (e.GetProperty("label").GetString() == label)+ return JsonDocument.Parse(e.GetRawText()).RootElement;+ throw new InvalidOperationException($"no email_probe.json[{section}] entry labelled '{label}'");+ }++ [Theory]+ [MemberData(nameof(CleanProbeCases))]+ public void EveryCleanProbeEntry(string label)+ {+ var e = FindEntry("clean", label);+ var body = e.GetProperty("body").GetString();+ var maxChars = e.GetProperty("max_chars").GetInt32();+ Assert.Equal(e.GetProperty("result").GetString(), LayaEmail.CleanBody(body, maxChars));+ }++ [Theory]+ [MemberData(nameof(StateProbeCases))]+ public void EveryStateProbeEntry(string label)+ {+ var e = FindEntry("state", label);+ var subject = e.GetProperty("subject").GetString();+ var body = e.GetProperty("body").GetString();+ var sender = e.GetProperty("sender").ValueKind == JsonValueKind.Null ? null : e.GetProperty("sender").GetString();+ var clean = e.GetProperty("clean").GetBoolean();+ var extra = e.GetProperty("extra").EnumerateObject()+ .Select(p => new KeyValuePair<string, object?>(p.Name, RoutingGoldenData.ToClr(p.Value)));++ var got = LayaEmail.State(subject, body, sender, clean, extra);+ var want = e.GetProperty("result");++ Assert.Equal(want.EnumerateObject().Select(p => p.Name), got.Keys);+ foreach (var p in want.EnumerateObject())+ Assert.Equal(RoutingGoldenData.ToClr(p.Value), got[p.Name]);+ }+}