Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

fix(email): cut a sign-off whose name carries a mark from any script

Denimworld127512b0e476d1
diff --git a/laya/email.py b/laya/email.py
index 773ba827..05c9641c 100644
--- a/laya/email.py
+++ b/laya/email.py
@@ -52,13 +52,21 @@ _HEADER_NEXT = re.compile(r"^\s*(Enviad[oa]( em| el)?:\s|Sent:\s|(Data|Fecha|Dat
# `Thanks, żaneta` read as a name and the line counted as a sign-off. The tail is matched
# structurally instead, and each token's first letter is judged by category below -- the same
# rule as the TS port's `\p{Lu}\p{Lt}\p{Lo}`. Combining marks ride along with the letter before
-# them (`Jose\u0301` is `José`), as `\p{M}` allows in the port.
+# them (`Jose\u0301` is `José`), as `\p{M}` allows in the port. `re` has no `\p{M}` either, and a
+# class cannot list those ranges any more than it could list the lowercase ones, so the marks
+# are dropped before the tail is matched: they never separate tokens, and the letter each one
+# rides on is what the rule asks about.
_SIGNOFF_HEAD = re.compile(
r"^\s*(?i:best|kind|warmest|warm|many thanks|thanks|thank you|regards|cheers|sincerely)"
r"(?i:\s+(?:and|&)\s+regards|\s+(?:regards|wishes|again|in advance|a lot|so much|very much))?"
)
-_SIGNOFF_TAIL = re.compile(r"^[\s,;:!.]*(?:[^\W\d_][\w\u0300-\u036f'-]*[\s,.]*){0,3}$")
-_SIGNOFF_TOKEN = re.compile(r"[^\W\d_][\w\u0300-\u036f'-]*")
+_SIGNOFF_TAIL = re.compile(r"^[\s,;:!.]*(?:[^\W\d_][\w'-]*[\s,.]*){0,3}$")
+_SIGNOFF_TOKEN = re.compile(r"[^\W\d_][\w'-]*")
+
+
+def _drop_marks(text: str) -> str:
+ """Remove combining marks, the `Mn`/`Mc`/`Me` categories the port spells `\\p{M}`."""
+ return "".join(ch for ch in text if not unicodedata.category(ch).startswith("M"))
def _is_english_signoff(line: str) -> bool:
@@ -66,7 +74,7 @@ def _is_english_signoff(line: str) -> bool:
m = _SIGNOFF_HEAD.match(line)
if m is None:
return False
- tail = line[m.end():]
+ tail = _drop_marks(line[m.end():])
if _SIGNOFF_TAIL.match(tail) is None:
return False
return all(unicodedata.category(token[0]) in ("Lu", "Lt", "Lo")
diff --git a/tests/test_email.py b/tests/test_email.py
index 60eef831..194a8a41 100644
--- a/tests/test_email.py
+++ b/tests/test_email.py
@@ -453,6 +453,29 @@ check(
clean_email_body("%s\n\n%s" % (BODY, "Regards, Jose\u0301")),
BODY,
)
+# ...in every script, not only the Latin one. `\u0300-\u036f` is the Latin combining block
+# alone, so a name carrying a mark from any other script -- a Devanagari virama, an Arabic
+# shadda, a Hebrew point, a Thai tone mark -- failed the tail and the signature stayed in the
+# body. The port's `\p{M}` covers every mark, which is what these names need.
+for label, tail in [
+ ("devanagari", "Thanks, \u0928\u092e\u0938\u094d\u0924\u0947"),
+ ("devanagari name", "Thanks, \u0930\u0935\u093f"),
+ ("arabic", "Thanks, \u0645\u062d\u0645\u0651\u062f"),
+ ("hebrew", "Regards, \u05e9\u05c1\u05dc\u05d5\u05dd"),
+ ("thai", "Thanks, \u0e2a\u0e38\u0e0a\u0e32\u0e15\u0e34\u0e4c"),
+ ("bengali", "Thanks, \u0985\u09ae\u09bf\u09a4"),
+ ("tamil", "Thanks, \u0bb5\u0bc6\u0bb3\u0bcd\u0bb3\u0bbf"),
+]:
+ check("signoff cut/mark beyond latin-1, " + label,
+ clean_email_body("%s\n\n%s" % (BODY, tail)), BODY)
+
+# ...and dropping the marks must not turn a lowercase name into one: the letter a mark rides on
+# is what the case rule asks about, so a marked lowercase name is still not a sign-off.
+for label, body in [
+ ("greek lowercase, accented", "Hi,\n\nPlease refund invoice 4411.\nThanks, \u03b1\u0301\u03bb\u03c6\u03b1"),
+ ("latin lowercase, decomposed", "Hi,\n\nPlease refund invoice 4411.\nThanks, jose\u0301"),
+]:
+ check("signoff kept/" + label, clean_email_body(body), body)
# ------------------------------------------------- the word, without the disclaimer