Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
fix(email): keep the request when a disclaimer footer shares its paragraph (#94)
PerryLink4c2c6f9271ec
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.ymlindex 4a7e7150..9fecc50c 100644--- a/.github/workflows/ci.yml+++ b/.github/workflows/ci.yml@@ -47,6 +47,9 @@ jobs:python tests/test_criteria.pypython tests/test_download.py+ - name: Email cleaning tests+ run: python tests/test_email.py+lint:name: lintruns-on: ubuntu-latestdiff --git a/.github/workflows/release.yml b/.github/workflows/release.ymlindex c24a2446..f98e49d9 100644--- a/.github/workflows/release.yml+++ b/.github/workflows/release.yml@@ -39,6 +39,7 @@ jobs:run: |python tests/test_router.pypython tests/test_criteria.py+ python tests/test_email.py- name: Buildrun: python -m builddiff --git a/laya/email.py b/laya/email.pyindex e2e41faf..55127417 100644--- a/laya/email.py+++ b/laya/email.py@@ -18,6 +18,20 @@ _DISCLAIMER = re.compile(r"if you (have )?received this (e-?mail|message) in error)",re.I,)+_SENTENCE = re.compile(r"(?<=[.!?])\s+")+++def _strip_disclaimer(paragraph: str) -> str:+ """Drop boilerplate disclaimer text from one paragraph.++ A paragraph is dropped whole only when *every* sentence in it is boilerplate; otherwise only+ the boilerplate sentences go. A footer that runs on without a blank line used to take the+ sender's actual request with it, which is worse than leaving one boilerplate line behind.+ """+ if not _DISCLAIMER.search(paragraph):+ return paragraph # nothing to do: keep the original line structure+ parts = [p.strip() for p in _SENTENCE.split(paragraph) if p.strip()]+ return " ".join(p for p in parts if not _DISCLAIMER.search(p))def clean_email_body(body: str, max_chars: int = 3000) -> str:@@ -36,7 +50,7 @@ def clean_email_body(body: str, max_chars: int = 3000) -> str:cut = ibreaklines = lines[:cut]- paragraphs = [p for p in re.split(r"\n\s*\n", "\n".join(lines)) if not _DISCLAIMER.search(p)]+ paragraphs = [_strip_disclaimer(p) for p in re.split(r"\n\s*\n", "\n".join(lines))]text = re.sub(r"[ \t]+", " ", "\n\n".join(p.strip() for p in paragraphs if p.strip()))return text[:max_chars]diff --git a/tests/test_email.py b/tests/test_email.pynew file mode 100644index 00000000..8e3d401a--- /dev/null+++ b/tests/test_email.py@@ -0,0 +1,105 @@+"""Email cleaning: a disclaimer footer must not delete the sender's actual request.++Regression tests for `laya.email.clean_email_body`. `_DISCLAIMER` used to be applied to whole+paragraphs, so any paragraph that merely *mentioned* boilerplate was deleted outright. When the+footer ran on without a blank line, the request went with it:++ clean_email_body("My account is locked.\\nThis email is confidential...\\nPlease unlock it.")+ # before: '' <- the whole body, request included, was deleted+ # after: 'My account is locked. Please unlock it.'++Dropping the request is silent and severe; leaving one boilerplate line behind is neither, so the+cleaning errs towards keeping text.+"""+import os+import sys++sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))++from laya.email import clean_email_body, email_state # noqa: E402++PASS, FAIL = [], []+++def check(name, got, want):+ if got == want:+ PASS.append(name)+ else:+ FAIL.append("%s:\n got %r\n want %r" % (name, got, want))+++def check_true(name, cond, detail=""):+ if cond:+ PASS.append(name)+ else:+ FAIL.append("%s %s" % (name, detail))+++DISCLAIMER = "This email is confidential and intended solely for the named addressee."++# --------------------------------------------------------------- the request survives the footer+check(+ "inline footer/no blank line keeps the request",+ clean_email_body("My account is locked.\n%s\nPlease unlock it." % DISCLAIMER),+ "My account is locked. Please unlock it.",+)+check(+ "inline footer/no terminal punctuation still recovers the request",+ clean_email_body("My account is locked\n%s\nPlease unlock it." % DISCLAIMER),+ "Please unlock it.",+)+check(+ "inline footer/body is never emptied",+ clean_email_body("My account is locked. %s" % DISCLAIMER),+ "My account is locked.",+)+check_true(+ "inline footer/is not empty",+ clean_email_body("My account is locked. %s" % DISCLAIMER).strip() != "",+)+check(+ "email_state/body keeps the request",+ email_state("Locked out", "My account is locked. %s" % DISCLAIMER)["body"],+ "My account is locked.",+)++# --------------------------------------------------------------- a pure footer is still removed+check(+ "standalone footer paragraph is still dropped",+ clean_email_body("My account is locked.\n\n%s" % DISCLAIMER),+ "My account is locked.",+)+check(+ "wrapped standalone footer is still dropped",+ clean_email_body(+ "My account is locked.\n\nThis email and any files transmitted with it are\n"+ "confidential and intended solely for the named addressee."+ ),+ "My account is locked.",+)+check(+ "received-in-error footer is still dropped",+ clean_email_body(+ "Please reopen ticket 4411.\n\nIf you have received this message in error, delete it."+ ),+ "Please reopen ticket 4411.",+)++# --------------------------------------------------------------- unrelated cleaning is unchanged+check(+ "quoted history is still removed",+ clean_email_body("Thanks for the update.\nOn Mon, Sep 20, Bob wrote:\n> original text"),+ "Thanks for the update.",+)+check(+ "signature block is still removed",+ clean_email_body("Hi team,\nCan you confirm the refund?\nRegards,\nAlice"),+ "Hi team,\nCan you confirm the refund?",+)+check("empty body stays empty", clean_email_body(""), "")+++print("\n%d passed, %d failed" % (len(PASS), len(FAIL)))+for f in FAIL:+ print(" FAIL " + f)+sys.exit(1 if FAIL else 0)