Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

fix(email): the name class excludes lowercase in every script

PerryLink71fe3a9505b2
diff --git a/laya-ts/src/email.ts b/laya-ts/src/email.ts
index 09c1cc24..2959b7a1 100644
--- a/laya-ts/src/email.ts
+++ b/laya-ts/src/email.ts
@@ -19,9 +19,9 @@ const HEADER_NEXT = /^\s*(Enviad[oa]( em| el)?:\s|(Data|Fecha):\s.*\d{4})/i;
// case-insensitively here and the name is checked case-sensitively by SIGNOFF_TAIL.
const SIGNOFF_HEAD =
/^\s*(?:best|kind|warmest|warm|many thanks|thanks|thank you|regards|cheers|sincerely)(?:\s+(?:and|&)\s+regards|\s+(?:regards|wishes|again|in advance|a lot|so much|very much))?/i;
-// Python's name class [^\W\d_a-zß-öø-ÿ] is "a word char that is not a digit, underscore or
-// lowercase letter"; \p{Lu}/\p{Lt}/\p{Lo} is the same intent: a name is capitalised in any
-// script (Regards, Łukasz) or written in a script without case (山田).
+// The name's first letter must not be lowercase, so the check is per character, not per script:
+// \p{Lu}/\p{Lt}/\p{Lo} accepts a name capitalised in any script (Regards, Łukasz) or written in
+// a script without case (山田). Python asks the same question per token (`_is_english_signoff`).
const SIGNOFF_TAIL = /^[\s,;:!.]*(?:[\p{Lu}\p{Lt}\p{Lo}][\p{L}\p{M}\p{N}_'-]*[\s,.]*){0,3}$/u;
function isEnglishSignoff(line: string): boolean {
const m = SIGNOFF_HEAD.exec(line);
diff --git a/laya/email.py b/laya/email.py
index a4a6dcf2..fd7f222c 100644
--- a/laya/email.py
+++ b/laya/email.py
@@ -7,6 +7,7 @@ cleaning was a no-op: Gmail's `Em ... escreveu:`, Outlook's `-----Mensagem origi
history (often a *different* request) weighed on the answer as much as the new message did.
"""
import re
+import unicodedata
from typing import Dict, List, Optional
# One definition, in the module that holds the other presets. Re-exported here because
@@ -45,20 +46,49 @@ _ATTRIBUTION_HEAD = re.compile(r"^\s*(On|Em|El) (?=.*\d)", re.I)
# English pair is "From: <name>" followed by "Sent:"/"Date:".
_HEADER_FROM_NAME = re.compile(r"^\s*(De|From):\s+\S", re.I)
_HEADER_NEXT = re.compile(r"^\s*(Enviad[oa]( em| el)?:\s|Sent:\s|(Data|Fecha|Date):\s.*\d{4})", re.I)
+# A closing's name starts with a letter that is not lowercase: capitalised in any script
+# (`Łukasz`, `Дмитрий`) or caseless (`山田`). `re` cannot say "not a lowercase letter in any
+# script" -- a class has to list ranges, and `[^\W\d_a-zß-öø-ÿ]` stops at Latin-1, so
+# `Thanks, żaneta` read as a name and the line counted as a sign-off. The tail is matched
+# structurally instead, and each token's first letter is judged by category below -- the same
+# rule as the TS port's `\p{Lu}\p{Lt}\p{Lo}`. Combining marks ride along with the letter before
+# them (`Jose\u0301` is `José`), as `\p{M}` allows in the port.
+_SIGNOFF_HEAD = re.compile(
+ r"^\s*(?i:best|kind|warmest|warm|many thanks|thanks|thank you|regards|cheers|sincerely)"
+ r"(?i:\s+(?:and|&)\s+regards|\s+(?:regards|wishes|again|in advance|a lot|so much|very much))?"
+)
+_SIGNOFF_TAIL = re.compile(r"^[\s,;:!.]*(?:[^\W\d_][\w\u0300-\u036f'-]*[\s,.]*){0,3}$")
+_SIGNOFF_TOKEN = re.compile(r"[^\W\d_][\w\u0300-\u036f'-]*")
+
+
+def _is_english_signoff(line: str) -> bool:
+ """True when a closing word is followed by nothing but punctuation and a short name."""
+ m = _SIGNOFF_HEAD.match(line)
+ if m is None:
+ return False
+ tail = line[m.end():]
+ if _SIGNOFF_TAIL.match(tail) is None:
+ return False
+ return all(unicodedata.category(token[0]) in ("Lu", "Lt", "Lo")
+ for token in _SIGNOFF_TOKEN.findall(tail))
+
+
+def _marker_matches(marker, line: str) -> bool:
+ """One `_SIGNATURE_MARKERS` entry: a compiled pattern, or a callable for a rule a pattern
+ cannot express (the English sign-off)."""
+ return bool(marker(line)) if callable(marker) else bool(marker.match(line))
+
+
_SIGNATURE_MARKERS = [
re.compile(r"^\s*--\s*$"),
# A closing line is the closing word plus punctuation and at most a name. Anything else on
# the line is a sentence, and the case of the next word is what separates the two: a name is
# capitalised, "for" in "Thanks for the quick reply." is not. The closing words are matched
# case-insensitively, the name is not, so the flag is scoped instead of global.
- # `warmest` and `and/& regards` are closings the alternation did not reach, and a name is
- # capitalised in any script, so the name class excludes the lowercase letters instead of
- # listing the uppercase ones: `Regards, Łukasz` is a sign-off, `Thanks for the reply` is not.
- re.compile(
- r"^\s*(?i:best|kind|warmest|warm|many thanks|thanks|thank you|regards|cheers|sincerely)"
- r"(?i:\s+(?:and|&)\s+regards|\s+(?:regards|wishes|again|in advance|a lot|so much|very much))?"
- r"[\s,;:!.]*(?:[^\W\d_a-zß-öø-ÿ][\w'-]*[\s,.]*){0,3}$"
- ),
+ # `warmest` and `and/& regards` are closings the alternation did not reach; the name that may
+ # follow is judged in `_is_english_signoff` above, a callable because `re` cannot express its
+ # rule. `Regards, Łukasz` is a sign-off, `Thanks for the reply` is not.
+ _is_english_signoff,
re.compile(r"^\s*sent from my (iphone|android|mobile|ipad)", re.I),
# Portuguese/Spanish sign-offs match only on their own: "Obrigado pelo retorno, mas ..." is a
# request, not a signature, so unlike the English marker no trailing words are allowed
@@ -185,7 +215,7 @@ def clean_email_body(body: str, max_chars: int = 3000) -> str:
cut = len(lines)
for i in range(max(1, min(int(len(lines) * 0.6), len(lines) - 8)), len(lines)):
n = len(lines[i].strip())
- if (n <= 40 and any(p.match(lines[i]) for p in _SIGNATURE_MARKERS)) or (
+ if (n <= 40 and any(_marker_matches(p, lines[i]) for p in _SIGNATURE_MARKERS)) or (
n <= 60 and _DEVICE_FOOTER.match(lines[i])):
cut = i
break
diff --git a/tests/test_email.py b/tests/test_email.py
index 0d13424a..00081708 100644
--- a/tests/test_email.py
+++ b/tests/test_email.py
@@ -426,6 +426,28 @@ for label, body in [
]:
check("signoff kept/" + label, clean_email_body(body), body)
+# ------------------------------------------- the name rule reaches lowercase beyond Latin-1
+# #245 replaced the ASCII `[A-Z]` with "exclude the lowercase letters", but the ranges that
+# exclude them stop at Latin-1 (`a-zß-öø-ÿ`). A word starting with ż, д or α *is* lowercase --
+# not "capitalised in any script" -- yet the class read it as a name, so the line counted as a
+# sign-off and the cut dropped whatever followed it. Case is a property of the character, not of
+# the script: the port checks it per character (`\p{Lu}\p{Lt}\p{Lo}`), and Python now checks the
+# same first-letter categories.
+for label, body in [
+ ("latin-extended lowercase name", "Hi,\n\nPlease refund invoice 4411.\nThanks, żaneta"),
+ ("cyrillic lowercase name", "Hi,\n\nPlease refund invoice 4411.\nThanks, дмитрий"),
+ ("greek lowercase name", "Hi,\n\nPlease refund invoice 4411.\nThanks, αλέξανδρος"),
+ ("a roman numeral is not a name", "Hi,\n\nPlease refund invoice 4411.\nThanks, Ⅷ"),
+]:
+ check("signoff kept/" + label, clean_email_body(body), body)
+# ...while a combining mark stays part of the letter before it: `Jose\u0301` spells `José` and
+# still cuts as a sign-off (the port's `\p{M}`).
+check(
+ "signoff cut/decomposed accented name",
+ clean_email_body("%s\n\n%s" % (BODY, "Regards, Jose\u0301")),
+ BODY,
+)
+
# ------------------------------------------------- the word, without the disclaimer
# `confidential` was a bare substring of `_DISCLAIMER`, so any sentence that merely