Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

fix(email): keep the request when a disclaimer footer shares its paragraph (#94)

PerryLink4c2c6f9271ec
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 4a7e7150..9fecc50c 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -47,6 +47,9 @@ jobs:
python tests/test_criteria.py
python tests/test_download.py
+ - name: Email cleaning tests
+ run: python tests/test_email.py
+
lint:
name: lint
runs-on: ubuntu-latest
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index c24a2446..f98e49d9 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -39,6 +39,7 @@ jobs:
run: |
python tests/test_router.py
python tests/test_criteria.py
+ python tests/test_email.py
- name: Build
run: python -m build
diff --git a/laya/email.py b/laya/email.py
index e2e41faf..55127417 100644
--- a/laya/email.py
+++ b/laya/email.py
@@ -18,6 +18,20 @@ _DISCLAIMER = re.compile(
r"if you (have )?received this (e-?mail|message) in error)",
re.I,
)
+_SENTENCE = re.compile(r"(?<=[.!?])\s+")
+
+
+def _strip_disclaimer(paragraph: str) -> str:
+ """Drop boilerplate disclaimer text from one paragraph.
+
+ A paragraph is dropped whole only when *every* sentence in it is boilerplate; otherwise only
+ the boilerplate sentences go. A footer that runs on without a blank line used to take the
+ sender's actual request with it, which is worse than leaving one boilerplate line behind.
+ """
+ if not _DISCLAIMER.search(paragraph):
+ return paragraph # nothing to do: keep the original line structure
+ parts = [p.strip() for p in _SENTENCE.split(paragraph) if p.strip()]
+ return " ".join(p for p in parts if not _DISCLAIMER.search(p))
def clean_email_body(body: str, max_chars: int = 3000) -> str:
@@ -36,7 +50,7 @@ def clean_email_body(body: str, max_chars: int = 3000) -> str:
cut = i
break
lines = lines[:cut]
- paragraphs = [p for p in re.split(r"\n\s*\n", "\n".join(lines)) if not _DISCLAIMER.search(p)]
+ paragraphs = [_strip_disclaimer(p) for p in re.split(r"\n\s*\n", "\n".join(lines))]
text = re.sub(r"[ \t]+", " ", "\n\n".join(p.strip() for p in paragraphs if p.strip()))
return text[:max_chars]
diff --git a/tests/test_email.py b/tests/test_email.py
new file mode 100644
index 00000000..8e3d401a
--- /dev/null
+++ b/tests/test_email.py
@@ -0,0 +1,105 @@
+"""Email cleaning: a disclaimer footer must not delete the sender's actual request.
+
+Regression tests for `laya.email.clean_email_body`. `_DISCLAIMER` used to be applied to whole
+paragraphs, so any paragraph that merely *mentioned* boilerplate was deleted outright. When the
+footer ran on without a blank line, the request went with it:
+
+ clean_email_body("My account is locked.\\nThis email is confidential...\\nPlease unlock it.")
+ # before: '' <- the whole body, request included, was deleted
+ # after: 'My account is locked. Please unlock it.'
+
+Dropping the request is silent and severe; leaving one boilerplate line behind is neither, so the
+cleaning errs towards keeping text.
+"""
+import os
+import sys
+
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+from laya.email import clean_email_body, email_state # noqa: E402
+
+PASS, FAIL = [], []
+
+
+def check(name, got, want):
+ if got == want:
+ PASS.append(name)
+ else:
+ FAIL.append("%s:\n got %r\n want %r" % (name, got, want))
+
+
+def check_true(name, cond, detail=""):
+ if cond:
+ PASS.append(name)
+ else:
+ FAIL.append("%s %s" % (name, detail))
+
+
+DISCLAIMER = "This email is confidential and intended solely for the named addressee."
+
+# --------------------------------------------------------------- the request survives the footer
+check(
+ "inline footer/no blank line keeps the request",
+ clean_email_body("My account is locked.\n%s\nPlease unlock it." % DISCLAIMER),
+ "My account is locked. Please unlock it.",
+)
+check(
+ "inline footer/no terminal punctuation still recovers the request",
+ clean_email_body("My account is locked\n%s\nPlease unlock it." % DISCLAIMER),
+ "Please unlock it.",
+)
+check(
+ "inline footer/body is never emptied",
+ clean_email_body("My account is locked. %s" % DISCLAIMER),
+ "My account is locked.",
+)
+check_true(
+ "inline footer/is not empty",
+ clean_email_body("My account is locked. %s" % DISCLAIMER).strip() != "",
+)
+check(
+ "email_state/body keeps the request",
+ email_state("Locked out", "My account is locked. %s" % DISCLAIMER)["body"],
+ "My account is locked.",
+)
+
+# --------------------------------------------------------------- a pure footer is still removed
+check(
+ "standalone footer paragraph is still dropped",
+ clean_email_body("My account is locked.\n\n%s" % DISCLAIMER),
+ "My account is locked.",
+)
+check(
+ "wrapped standalone footer is still dropped",
+ clean_email_body(
+ "My account is locked.\n\nThis email and any files transmitted with it are\n"
+ "confidential and intended solely for the named addressee."
+ ),
+ "My account is locked.",
+)
+check(
+ "received-in-error footer is still dropped",
+ clean_email_body(
+ "Please reopen ticket 4411.\n\nIf you have received this message in error, delete it."
+ ),
+ "Please reopen ticket 4411.",
+)
+
+# --------------------------------------------------------------- unrelated cleaning is unchanged
+check(
+ "quoted history is still removed",
+ clean_email_body("Thanks for the update.\nOn Mon, Sep 20, Bob wrote:\n> original text"),
+ "Thanks for the update.",
+)
+check(
+ "signature block is still removed",
+ clean_email_body("Hi team,\nCan you confirm the refund?\nRegards,\nAlice"),
+ "Hi team,\nCan you confirm the refund?",
+)
+check("empty body stays empty", clean_email_body(""), "")
+
+
+print("\n%d passed, %d failed" % (len(PASS), len(FAIL)))
+for f in FAIL:
+ print(" FAIL " + f)
+sys.exit(1 if FAIL else 0)