Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
fix(email): the name class excludes lowercase in every script
PerryLink71fe3a9505b2
diff --git a/laya-ts/src/email.ts b/laya-ts/src/email.tsindex 09c1cc24..2959b7a1 100644--- a/laya-ts/src/email.ts+++ b/laya-ts/src/email.ts@@ -19,9 +19,9 @@ const HEADER_NEXT = /^\s*(Enviad[oa]( em| el)?:\s|(Data|Fecha):\s.*\d{4})/i;// case-insensitively here and the name is checked case-sensitively by SIGNOFF_TAIL.const SIGNOFF_HEAD =/^\s*(?:best|kind|warmest|warm|many thanks|thanks|thank you|regards|cheers|sincerely)(?:\s+(?:and|&)\s+regards|\s+(?:regards|wishes|again|in advance|a lot|so much|very much))?/i;-// Python's name class [^\W\d_a-zß-öø-ÿ] is "a word char that is not a digit, underscore or-// lowercase letter"; \p{Lu}/\p{Lt}/\p{Lo} is the same intent: a name is capitalised in any-// script (Regards, Łukasz) or written in a script without case (山田).+// The name's first letter must not be lowercase, so the check is per character, not per script:+// \p{Lu}/\p{Lt}/\p{Lo} accepts a name capitalised in any script (Regards, Łukasz) or written in+// a script without case (山田). Python asks the same question per token (`_is_english_signoff`).const SIGNOFF_TAIL = /^[\s,;:!.]*(?:[\p{Lu}\p{Lt}\p{Lo}][\p{L}\p{M}\p{N}_'-]*[\s,.]*){0,3}$/u;function isEnglishSignoff(line: string): boolean {const m = SIGNOFF_HEAD.exec(line);diff --git a/laya/email.py b/laya/email.pyindex a4a6dcf2..fd7f222c 100644--- a/laya/email.py+++ b/laya/email.py@@ -7,6 +7,7 @@ cleaning was a no-op: Gmail's `Em ... escreveu:`, Outlook's `-----Mensagem origihistory (often a *different* request) weighed on the answer as much as the new message did."""import re+import unicodedatafrom typing import Dict, List, Optional# One definition, in the module that holds the other presets. Re-exported here because@@ -45,20 +46,49 @@ _ATTRIBUTION_HEAD = re.compile(r"^\s*(On|Em|El) (?=.*\d)", re.I)# English pair is "From: <name>" followed by "Sent:"/"Date:"._HEADER_FROM_NAME = re.compile(r"^\s*(De|From):\s+\S", re.I)_HEADER_NEXT = re.compile(r"^\s*(Enviad[oa]( em| el)?:\s|Sent:\s|(Data|Fecha|Date):\s.*\d{4})", re.I)+# A closing's name starts with a letter that is not lowercase: capitalised in any script+# (`Łukasz`, `Дмитрий`) or caseless (`山田`). `re` cannot say "not a lowercase letter in any+# script" -- a class has to list ranges, and `[^\W\d_a-zß-öø-ÿ]` stops at Latin-1, so+# `Thanks, żaneta` read as a name and the line counted as a sign-off. The tail is matched+# structurally instead, and each token's first letter is judged by category below -- the same+# rule as the TS port's `\p{Lu}\p{Lt}\p{Lo}`. Combining marks ride along with the letter before+# them (`Jose\u0301` is `José`), as `\p{M}` allows in the port.+_SIGNOFF_HEAD = re.compile(+ r"^\s*(?i:best|kind|warmest|warm|many thanks|thanks|thank you|regards|cheers|sincerely)"+ r"(?i:\s+(?:and|&)\s+regards|\s+(?:regards|wishes|again|in advance|a lot|so much|very much))?"+)+_SIGNOFF_TAIL = re.compile(r"^[\s,;:!.]*(?:[^\W\d_][\w\u0300-\u036f'-]*[\s,.]*){0,3}$")+_SIGNOFF_TOKEN = re.compile(r"[^\W\d_][\w\u0300-\u036f'-]*")+++def _is_english_signoff(line: str) -> bool:+ """True when a closing word is followed by nothing but punctuation and a short name."""+ m = _SIGNOFF_HEAD.match(line)+ if m is None:+ return False+ tail = line[m.end():]+ if _SIGNOFF_TAIL.match(tail) is None:+ return False+ return all(unicodedata.category(token[0]) in ("Lu", "Lt", "Lo")+ for token in _SIGNOFF_TOKEN.findall(tail))+++def _marker_matches(marker, line: str) -> bool:+ """One `_SIGNATURE_MARKERS` entry: a compiled pattern, or a callable for a rule a pattern+ cannot express (the English sign-off)."""+ return bool(marker(line)) if callable(marker) else bool(marker.match(line))++_SIGNATURE_MARKERS = [re.compile(r"^\s*--\s*$"),# A closing line is the closing word plus punctuation and at most a name. Anything else on# the line is a sentence, and the case of the next word is what separates the two: a name is# capitalised, "for" in "Thanks for the quick reply." is not. The closing words are matched# case-insensitively, the name is not, so the flag is scoped instead of global.- # `warmest` and `and/& regards` are closings the alternation did not reach, and a name is- # capitalised in any script, so the name class excludes the lowercase letters instead of- # listing the uppercase ones: `Regards, Łukasz` is a sign-off, `Thanks for the reply` is not.- re.compile(- r"^\s*(?i:best|kind|warmest|warm|many thanks|thanks|thank you|regards|cheers|sincerely)"- r"(?i:\s+(?:and|&)\s+regards|\s+(?:regards|wishes|again|in advance|a lot|so much|very much))?"- r"[\s,;:!.]*(?:[^\W\d_a-zß-öø-ÿ][\w'-]*[\s,.]*){0,3}$"- ),+ # `warmest` and `and/& regards` are closings the alternation did not reach; the name that may+ # follow is judged in `_is_english_signoff` above, a callable because `re` cannot express its+ # rule. `Regards, Łukasz` is a sign-off, `Thanks for the reply` is not.+ _is_english_signoff,re.compile(r"^\s*sent from my (iphone|android|mobile|ipad)", re.I),# Portuguese/Spanish sign-offs match only on their own: "Obrigado pelo retorno, mas ..." is a# request, not a signature, so unlike the English marker no trailing words are allowed@@ -185,7 +215,7 @@ def clean_email_body(body: str, max_chars: int = 3000) -> str:cut = len(lines)for i in range(max(1, min(int(len(lines) * 0.6), len(lines) - 8)), len(lines)):n = len(lines[i].strip())- if (n <= 40 and any(p.match(lines[i]) for p in _SIGNATURE_MARKERS)) or (+ if (n <= 40 and any(_marker_matches(p, lines[i]) for p in _SIGNATURE_MARKERS)) or (n <= 60 and _DEVICE_FOOTER.match(lines[i])):cut = ibreakdiff --git a/tests/test_email.py b/tests/test_email.pyindex 0d13424a..00081708 100644--- a/tests/test_email.py+++ b/tests/test_email.py@@ -426,6 +426,28 @@ for label, body in []:check("signoff kept/" + label, clean_email_body(body), body)+# ------------------------------------------- the name rule reaches lowercase beyond Latin-1+# #245 replaced the ASCII `[A-Z]` with "exclude the lowercase letters", but the ranges that+# exclude them stop at Latin-1 (`a-zß-öø-ÿ`). A word starting with ż, д or α *is* lowercase --+# not "capitalised in any script" -- yet the class read it as a name, so the line counted as a+# sign-off and the cut dropped whatever followed it. Case is a property of the character, not of+# the script: the port checks it per character (`\p{Lu}\p{Lt}\p{Lo}`), and Python now checks the+# same first-letter categories.+for label, body in [+ ("latin-extended lowercase name", "Hi,\n\nPlease refund invoice 4411.\nThanks, żaneta"),+ ("cyrillic lowercase name", "Hi,\n\nPlease refund invoice 4411.\nThanks, дмитрий"),+ ("greek lowercase name", "Hi,\n\nPlease refund invoice 4411.\nThanks, αλέξανδρος"),+ ("a roman numeral is not a name", "Hi,\n\nPlease refund invoice 4411.\nThanks, Ⅷ"),+]:+ check("signoff kept/" + label, clean_email_body(body), body)+# ...while a combining mark stays part of the letter before it: `Jose\u0301` spells `José` and+# still cuts as a sign-off (the port's `\p{M}`).+check(+ "signoff cut/decomposed accented name",+ clean_email_body("%s\n\n%s" % (BODY, "Regards, Jose\u0301")),+ BODY,+)+# ------------------------------------------------- the word, without the disclaimer# `confidential` was a bare substring of `_DISCLAIMER`, so any sentence that merely