Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
fix(email): cut a sign-off whose name carries a mark from any script
Denimworld127512b0e476d1
diff --git a/laya/email.py b/laya/email.pyindex 773ba827..05c9641c 100644--- a/laya/email.py+++ b/laya/email.py@@ -52,13 +52,21 @@ _HEADER_NEXT = re.compile(r"^\s*(Enviad[oa]( em| el)?:\s|Sent:\s|(Data|Fecha|Dat# `Thanks, żaneta` read as a name and the line counted as a sign-off. The tail is matched# structurally instead, and each token's first letter is judged by category below -- the same# rule as the TS port's `\p{Lu}\p{Lt}\p{Lo}`. Combining marks ride along with the letter before-# them (`Jose\u0301` is `José`), as `\p{M}` allows in the port.+# them (`Jose\u0301` is `José`), as `\p{M}` allows in the port. `re` has no `\p{M}` either, and a+# class cannot list those ranges any more than it could list the lowercase ones, so the marks+# are dropped before the tail is matched: they never separate tokens, and the letter each one+# rides on is what the rule asks about._SIGNOFF_HEAD = re.compile(r"^\s*(?i:best|kind|warmest|warm|many thanks|thanks|thank you|regards|cheers|sincerely)"r"(?i:\s+(?:and|&)\s+regards|\s+(?:regards|wishes|again|in advance|a lot|so much|very much))?")-_SIGNOFF_TAIL = re.compile(r"^[\s,;:!.]*(?:[^\W\d_][\w\u0300-\u036f'-]*[\s,.]*){0,3}$")-_SIGNOFF_TOKEN = re.compile(r"[^\W\d_][\w\u0300-\u036f'-]*")+_SIGNOFF_TAIL = re.compile(r"^[\s,;:!.]*(?:[^\W\d_][\w'-]*[\s,.]*){0,3}$")+_SIGNOFF_TOKEN = re.compile(r"[^\W\d_][\w'-]*")+++def _drop_marks(text: str) -> str:+ """Remove combining marks, the `Mn`/`Mc`/`Me` categories the port spells `\\p{M}`."""+ return "".join(ch for ch in text if not unicodedata.category(ch).startswith("M"))def _is_english_signoff(line: str) -> bool:@@ -66,7 +74,7 @@ def _is_english_signoff(line: str) -> bool:m = _SIGNOFF_HEAD.match(line)if m is None:return False- tail = line[m.end():]+ tail = _drop_marks(line[m.end():])if _SIGNOFF_TAIL.match(tail) is None:return Falsereturn all(unicodedata.category(token[0]) in ("Lu", "Lt", "Lo")diff --git a/tests/test_email.py b/tests/test_email.pyindex 60eef831..194a8a41 100644--- a/tests/test_email.py+++ b/tests/test_email.py@@ -453,6 +453,29 @@ check(clean_email_body("%s\n\n%s" % (BODY, "Regards, Jose\u0301")),BODY,)+# ...in every script, not only the Latin one. `\u0300-\u036f` is the Latin combining block+# alone, so a name carrying a mark from any other script -- a Devanagari virama, an Arabic+# shadda, a Hebrew point, a Thai tone mark -- failed the tail and the signature stayed in the+# body. The port's `\p{M}` covers every mark, which is what these names need.+for label, tail in [+ ("devanagari", "Thanks, \u0928\u092e\u0938\u094d\u0924\u0947"),+ ("devanagari name", "Thanks, \u0930\u0935\u093f"),+ ("arabic", "Thanks, \u0645\u062d\u0645\u0651\u062f"),+ ("hebrew", "Regards, \u05e9\u05c1\u05dc\u05d5\u05dd"),+ ("thai", "Thanks, \u0e2a\u0e38\u0e0a\u0e32\u0e15\u0e34\u0e4c"),+ ("bengali", "Thanks, \u0985\u09ae\u09bf\u09a4"),+ ("tamil", "Thanks, \u0bb5\u0bc6\u0bb3\u0bcd\u0bb3\u0bbf"),+]:+ check("signoff cut/mark beyond latin-1, " + label,+ clean_email_body("%s\n\n%s" % (BODY, tail)), BODY)++# ...and dropping the marks must not turn a lowercase name into one: the letter a mark rides on+# is what the case rule asks about, so a marked lowercase name is still not a sign-off.+for label, body in [+ ("greek lowercase, accented", "Hi,\n\nPlease refund invoice 4411.\nThanks, \u03b1\u0301\u03bb\u03c6\u03b1"),+ ("latin lowercase, decomposed", "Hi,\n\nPlease refund invoice 4411.\nThanks, jose\u0301"),+]:+ check("signoff kept/" + label, clean_email_body(body), body)# ------------------------------------------------- the word, without the disclaimer