Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
fix(email): recover the request when a disclaimer shares its paragraph (#115)
Bruce-Yiibff8104e2fb2
diff --git a/laya/email.py b/laya/email.pyindex e1fd563c..cd8fa4dc 100644--- a/laya/email.py+++ b/laya/email.py@@ -7,7 +7,7 @@ cleaning was a no-op: Gmail's `Em ... escreveu:`, Outlook's `-----Mensagem origihistory (often a *different* request) weighed on the answer as much as the new message did."""import re-from typing import Dict, Optional+from typing import Dict, List, Optional_QUOTE_HEADERS = [re.compile(r"^\s*On .{0,300}wrote:\s*$", re.I),@@ -74,6 +74,42 @@ _DISCLAIMER = re.compile(_SENTENCE = re.compile(r"(?<=[.!?])\s+")+def _starts_new_sentence(line: str) -> bool:+ """First letter is uppercase: a fresh sentence, not a wrapped line.++ Lines in uncased scripts (CJK, Devanagari, ...) never start a new piece,+ so wrapped boilerplate in those scripts still drops whole.+ """+ for ch in line:+ if ch.isalpha():+ return ch.isupper()+ return False+++def _split_fused_lines(sentence: str) -> List[str]:+ """Split a fused boilerplate-positive sentence at sentence-starting newlines.++ An unpunctuated request line glued to a disclaimer line ("locked\\nThis ...")+ splits at the newline because the next line starts uppercase; a lowercase+ continuation ("are\\nconfidential") belongs to the same sentence, so a wrapped+ boilerplate footer still drops whole.+ """+ if "\n" not in sentence:+ return [sentence]+ pieces, buf = [], ""+ for line in (ln.strip() for ln in sentence.split("\n")):+ if not line:+ continue+ if buf and _starts_new_sentence(line):+ pieces.append(buf)+ buf = line+ else:+ buf = (buf + " " + line) if buf else line+ if buf:+ pieces.append(buf)+ return pieces++def _strip_disclaimer(paragraph: str) -> str:"""Drop boilerplate disclaimer text from one paragraph.@@ -84,7 +120,10 @@ def _strip_disclaimer(paragraph: str) -> str:if not _DISCLAIMER.search(paragraph):return paragraph # nothing to do: keep the original line structureparts = [p.strip() for p in _SENTENCE.split(paragraph) if p.strip()]- return " ".join(p for p in parts if not _DISCLAIMER.search(p))+ pieces = []+ for p in parts:+ pieces.extend(_split_fused_lines(p) if _DISCLAIMER.search(p) else [p])+ return " ".join(p for p in pieces if not _DISCLAIMER.search(p))def clean_email_body(body: str, max_chars: int = 3000) -> str:diff --git a/tests/test_email.py b/tests/test_email.pyindex 1e6b282a..fdd6a261 100644--- a/tests/test_email.py+++ b/tests/test_email.py@@ -44,9 +44,40 @@ check("My account is locked. Please unlock it.",)check(- "inline footer/no terminal punctuation still recovers the request",+ "inline footer/unpunctuated request line is fully recovered",clean_email_body("My account is locked\n%s\nPlease unlock it." % DISCLAIMER),- "Please unlock it.",+ "My account is locked Please unlock it.",+)+check(+ "inline footer/fused request without a trailing sentence",+ clean_email_body("Please unlock my account\n%s" % DISCLAIMER),+ "Please unlock my account",+)+check(+ "inline footer/fused order reference is fully recovered",+ clean_email_body("RMA 5521 is still pending\n%s\nPlease advise." % DISCLAIMER),+ "RMA 5521 is still pending Please advise.",+)+check(+ # Keep-ward trade-off, documented on purpose: a capitalised continuation of a+ # boilerplate sentence can be a real fragment ("...in error,\nPlease delete it."),+ # so it stays. Leaving one boilerplate line behind is harmless; dropping the+ # request is not.+ "inline footer/capitalised continuation stays",+ clean_email_body("If you have received this message in error,\nPlease delete it."),+ "Please delete it.",+)+check(+ # Documented boundary: an all-lowercase fused request ("locked\nthis email ...")+ # is indistinguishable from a wrapped boilerplate footer, so the first line is+ # still lost. Only a newline followed by an uppercase letter splits.+ "inline footer/lowercase fusion still recovers the tail",+ clean_email_body(+ "my account is locked\n"+ "this email is confidential and intended solely for the named addressee.\n"+ "please unlock it."+ ),+ "please unlock it.",)check("inline footer/body is never emptied",