Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
fix(email): keep a mark that has no base character to ride on
Denimworld12ac91ac81ca6c
diff --git a/laya/email.py b/laya/email.pyindex 05c9641c..077b56c7 100644--- a/laya/email.py+++ b/laya/email.py@@ -53,9 +53,11 @@ _HEADER_NEXT = re.compile(r"^\s*(Enviad[oa]( em| el)?:\s|Sent:\s|(Data|Fecha|Dat# structurally instead, and each token's first letter is judged by category below -- the same# rule as the TS port's `\p{Lu}\p{Lt}\p{Lo}`. Combining marks ride along with the letter before# them (`Jose\u0301` is `José`), as `\p{M}` allows in the port. `re` has no `\p{M}` either, and a-# class cannot list those ranges any more than it could list the lowercase ones, so the marks-# are dropped before the tail is matched: they never separate tokens, and the letter each one-# rides on is what the rule asks about.+# class cannot list those ranges any more than it could list the lowercase ones, so a mark that+# rides on a character is dropped before the tail is matched and the letter it rides on answers+# the case question. A mark with no base -- opening the tail, or following a space -- is left+# where it is, so it still breaks the token as the port's leading `\p{Lu}\p{Lt}\p{Lo}` does.+# `\p{M}` is marks only: ZWJ and ZWNJ are `Cf`, so `Thanks, क्ष` is kept here and in the port alike._SIGNOFF_HEAD = re.compile(r"^\s*(?i:best|kind|warmest|warm|many thanks|thanks|thank you|regards|cheers|sincerely)"r"(?i:\s+(?:and|&)\s+regards|\s+(?:regards|wishes|again|in advance|a lot|so much|very much))?"@@ -65,8 +67,17 @@ _SIGNOFF_TOKEN = re.compile(r"[^\W\d_][\w'-]*")def _drop_marks(text: str) -> str:- """Remove combining marks, the `Mn`/`Mc`/`Me` categories the port spells `\\p{M}`."""- return "".join(ch for ch in text if not unicodedata.category(ch).startswith("M"))+ """Remove combining marks, the `Mn`/`Mc`/`Me` categories the port spells `\\p{M}`.++ Only a mark that rides on a preceding character goes; one that opens the string or+ follows a space has no base to ride on and stays, so it still separates tokens.+ """+ kept = []+ for ch in text:+ if unicodedata.category(ch).startswith("M") and kept and not kept[-1].isspace():+ continue+ kept.append(ch)+ return "".join(kept)def _is_english_signoff(line: str) -> bool:diff --git a/tests/test_email.py b/tests/test_email.pyindex 194a8a41..882ef8c0 100644--- a/tests/test_email.py+++ b/tests/test_email.py@@ -469,6 +469,18 @@ for label, tail in [check("signoff cut/mark beyond latin-1, " + label,clean_email_body("%s\n\n%s" % (BODY, tail)), BODY)+# ...and a mark with no base is left where it is. Stripping one that opens the tail, or that+# follows a space, would join the tokens around it and cut a line the port keeps: `laya-ts`+# requires each token's first character to be `\p{Lu}\p{Lt}\p{Lo}`, which a leading mark fails.+# ZWJ and ZWNJ are `Cf` rather than `M`, so they are untouched here and in the port alike.+for label, body in [+ ("mark after a space", "Hi,\n\nPlease refund invoice 4411.\nThanks, Jose \u0301Smith"),+ ("mark opening the name", "Hi,\n\nPlease refund invoice 4411.\nThanks, \u0301Jose"),+ ("mark standing alone", "Hi,\n\nPlease refund invoice 4411.\nThanks, \u0301 Jose"),+ ("zwnj is not a mark", "Hi,\n\nPlease refund invoice 4411.\nThanks, \u0915\u094d\u200c\u0937"),+]:+ check("signoff kept/" + label, clean_email_body(body), body)+# ...and dropping the marks must not turn a lowercase name into one: the letter a mark rides on# is what the case rule asks about, so a marked lowercase name is still not a sign-off.for label, body in [