Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

fix(email): recover the request when a disclaimer shares its paragraph (#115)

Bruce-Yiibff8104e2fb2
diff --git a/laya/email.py b/laya/email.py
index e1fd563c..cd8fa4dc 100644
--- a/laya/email.py
+++ b/laya/email.py
@@ -7,7 +7,7 @@ cleaning was a no-op: Gmail's `Em ... escreveu:`, Outlook's `-----Mensagem origi
history (often a *different* request) weighed on the answer as much as the new message did.
"""
import re
-from typing import Dict, Optional
+from typing import Dict, List, Optional
_QUOTE_HEADERS = [
re.compile(r"^\s*On .{0,300}wrote:\s*$", re.I),
@@ -74,6 +74,42 @@ _DISCLAIMER = re.compile(
_SENTENCE = re.compile(r"(?<=[.!?])\s+")
+def _starts_new_sentence(line: str) -> bool:
+ """First letter is uppercase: a fresh sentence, not a wrapped line.
+
+ Lines in uncased scripts (CJK, Devanagari, ...) never start a new piece,
+ so wrapped boilerplate in those scripts still drops whole.
+ """
+ for ch in line:
+ if ch.isalpha():
+ return ch.isupper()
+ return False
+
+
+def _split_fused_lines(sentence: str) -> List[str]:
+ """Split a fused boilerplate-positive sentence at sentence-starting newlines.
+
+ An unpunctuated request line glued to a disclaimer line ("locked\\nThis ...")
+ splits at the newline because the next line starts uppercase; a lowercase
+ continuation ("are\\nconfidential") belongs to the same sentence, so a wrapped
+ boilerplate footer still drops whole.
+ """
+ if "\n" not in sentence:
+ return [sentence]
+ pieces, buf = [], ""
+ for line in (ln.strip() for ln in sentence.split("\n")):
+ if not line:
+ continue
+ if buf and _starts_new_sentence(line):
+ pieces.append(buf)
+ buf = line
+ else:
+ buf = (buf + " " + line) if buf else line
+ if buf:
+ pieces.append(buf)
+ return pieces
+
+
def _strip_disclaimer(paragraph: str) -> str:
"""Drop boilerplate disclaimer text from one paragraph.
@@ -84,7 +120,10 @@ def _strip_disclaimer(paragraph: str) -> str:
if not _DISCLAIMER.search(paragraph):
return paragraph # nothing to do: keep the original line structure
parts = [p.strip() for p in _SENTENCE.split(paragraph) if p.strip()]
- return " ".join(p for p in parts if not _DISCLAIMER.search(p))
+ pieces = []
+ for p in parts:
+ pieces.extend(_split_fused_lines(p) if _DISCLAIMER.search(p) else [p])
+ return " ".join(p for p in pieces if not _DISCLAIMER.search(p))
def clean_email_body(body: str, max_chars: int = 3000) -> str:
diff --git a/tests/test_email.py b/tests/test_email.py
index 1e6b282a..fdd6a261 100644
--- a/tests/test_email.py
+++ b/tests/test_email.py
@@ -44,9 +44,40 @@ check(
"My account is locked. Please unlock it.",
)
check(
- "inline footer/no terminal punctuation still recovers the request",
+ "inline footer/unpunctuated request line is fully recovered",
clean_email_body("My account is locked\n%s\nPlease unlock it." % DISCLAIMER),
- "Please unlock it.",
+ "My account is locked Please unlock it.",
+)
+check(
+ "inline footer/fused request without a trailing sentence",
+ clean_email_body("Please unlock my account\n%s" % DISCLAIMER),
+ "Please unlock my account",
+)
+check(
+ "inline footer/fused order reference is fully recovered",
+ clean_email_body("RMA 5521 is still pending\n%s\nPlease advise." % DISCLAIMER),
+ "RMA 5521 is still pending Please advise.",
+)
+check(
+ # Keep-ward trade-off, documented on purpose: a capitalised continuation of a
+ # boilerplate sentence can be a real fragment ("...in error,\nPlease delete it."),
+ # so it stays. Leaving one boilerplate line behind is harmless; dropping the
+ # request is not.
+ "inline footer/capitalised continuation stays",
+ clean_email_body("If you have received this message in error,\nPlease delete it."),
+ "Please delete it.",
+)
+check(
+ # Documented boundary: an all-lowercase fused request ("locked\nthis email ...")
+ # is indistinguishable from a wrapped boilerplate footer, so the first line is
+ # still lost. Only a newline followed by an uppercase letter splits.
+ "inline footer/lowercase fusion still recovers the tail",
+ clean_email_body(
+ "my account is locked\n"
+ "this email is confidential and intended solely for the named addressee.\n"
+ "please unlock it."
+ ),
+ "please unlock it.",
)
check(
"inline footer/body is never emptied",