Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

fix(email): keep a mark that has no base character to ride on

Denimworld12ac91ac81ca6c
diff --git a/laya/email.py b/laya/email.py
index 05c9641c..077b56c7 100644
--- a/laya/email.py
+++ b/laya/email.py
@@ -53,9 +53,11 @@ _HEADER_NEXT = re.compile(r"^\s*(Enviad[oa]( em| el)?:\s|Sent:\s|(Data|Fecha|Dat
# structurally instead, and each token's first letter is judged by category below -- the same
# rule as the TS port's `\p{Lu}\p{Lt}\p{Lo}`. Combining marks ride along with the letter before
# them (`Jose\u0301` is `José`), as `\p{M}` allows in the port. `re` has no `\p{M}` either, and a
-# class cannot list those ranges any more than it could list the lowercase ones, so the marks
-# are dropped before the tail is matched: they never separate tokens, and the letter each one
-# rides on is what the rule asks about.
+# class cannot list those ranges any more than it could list the lowercase ones, so a mark that
+# rides on a character is dropped before the tail is matched and the letter it rides on answers
+# the case question. A mark with no base -- opening the tail, or following a space -- is left
+# where it is, so it still breaks the token as the port's leading `\p{Lu}\p{Lt}\p{Lo}` does.
+# `\p{M}` is marks only: ZWJ and ZWNJ are `Cf`, so `Thanks, क्‌ष` is kept here and in the port alike.
_SIGNOFF_HEAD = re.compile(
r"^\s*(?i:best|kind|warmest|warm|many thanks|thanks|thank you|regards|cheers|sincerely)"
r"(?i:\s+(?:and|&)\s+regards|\s+(?:regards|wishes|again|in advance|a lot|so much|very much))?"
@@ -65,8 +67,17 @@ _SIGNOFF_TOKEN = re.compile(r"[^\W\d_][\w'-]*")
def _drop_marks(text: str) -> str:
- """Remove combining marks, the `Mn`/`Mc`/`Me` categories the port spells `\\p{M}`."""
- return "".join(ch for ch in text if not unicodedata.category(ch).startswith("M"))
+ """Remove combining marks, the `Mn`/`Mc`/`Me` categories the port spells `\\p{M}`.
+
+ Only a mark that rides on a preceding character goes; one that opens the string or
+ follows a space has no base to ride on and stays, so it still separates tokens.
+ """
+ kept = []
+ for ch in text:
+ if unicodedata.category(ch).startswith("M") and kept and not kept[-1].isspace():
+ continue
+ kept.append(ch)
+ return "".join(kept)
def _is_english_signoff(line: str) -> bool:
diff --git a/tests/test_email.py b/tests/test_email.py
index 194a8a41..882ef8c0 100644
--- a/tests/test_email.py
+++ b/tests/test_email.py
@@ -469,6 +469,18 @@ for label, tail in [
check("signoff cut/mark beyond latin-1, " + label,
clean_email_body("%s\n\n%s" % (BODY, tail)), BODY)
+# ...and a mark with no base is left where it is. Stripping one that opens the tail, or that
+# follows a space, would join the tokens around it and cut a line the port keeps: `laya-ts`
+# requires each token's first character to be `\p{Lu}\p{Lt}\p{Lo}`, which a leading mark fails.
+# ZWJ and ZWNJ are `Cf` rather than `M`, so they are untouched here and in the port alike.
+for label, body in [
+ ("mark after a space", "Hi,\n\nPlease refund invoice 4411.\nThanks, Jose \u0301Smith"),
+ ("mark opening the name", "Hi,\n\nPlease refund invoice 4411.\nThanks, \u0301Jose"),
+ ("mark standing alone", "Hi,\n\nPlease refund invoice 4411.\nThanks, \u0301 Jose"),
+ ("zwnj is not a mark", "Hi,\n\nPlease refund invoice 4411.\nThanks, \u0915\u094d\u200c\u0937"),
+]:
+ check("signoff kept/" + label, clean_email_body(body), body)
+
# ...and dropping the marks must not turn a lowercase name into one: the letter a mark rides on
# is what the case rule asks about, so a marked lowercase name is still not a sign-off.
for label, body in [