Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
fix(lang): count letters no script range claims, instead of routing them to English (#169)
PerryLink129007f22a77
diff --git a/laya/lang.py b/laya/lang.pyindex 7edc3700..911509a8 100644--- a/laya/lang.py+++ b/laya/lang.py@@ -179,13 +179,24 @@ def detect_script(text: str) -> str:if not ch.isalpha():continuecp = ord(ch)- if cp < 0x0250 or 0x1E00 <= cp <= 0x1EFF: # Latin + Latin Extended Additional- latin += 1+ if cp < 0x0250 or 0x1E00 <= cp <= 0x1EFF or 0xFF21 <= cp <= 0xFF3A or 0xFF41 <= cp <= 0xFF5A:+ latin += 1 # Latin, Latin Ext-Additional, fullwidthcontinuefor name, ranges in _SCRIPT_RANGES:if any(lo <= cp <= hi for lo, hi in ranges):counts[name] = counts.get(name, 0) + 1break+ else:+ # An alphabetic character no range claims used to be counted nowhere, so text+ # written only in an unlisted script produced a total of 0 and was reported as+ # "unknown" -- and `analyse` treats "unknown" as English, sending it to the+ # checkpoint that has no tokens for it. 68% of Unicode's alphabetic codepoints+ # are outside _SCRIPT_RANGES (the CJK extensions, kana supplements, bopomofo,+ # halfwidth katakana, and dozens of smaller scripts), so enumerating them all is+ # not maintainable. Counting the remainder under "other" keeps them visible and+ # non-Latin, which is the safe direction: an unreadable script must not be+ # handed to the English checkpoint.+ counts["other"] = counts.get("other", 0) + 1counts["latin"] = latintotal = sum(counts.values())if total == 0:@@ -200,13 +211,15 @@ def script_profile(text: str) -> Dict[str, float]:if not ch.isalpha():continuecp = ord(ch)- if cp < 0x0250 or 0x1E00 <= cp <= 0x1EFF:+ if cp < 0x0250 or 0x1E00 <= cp <= 0x1EFF or 0xFF21 <= cp <= 0xFF3A or 0xFF41 <= cp <= 0xFF5A:counts["latin"] += 1continuefor name, ranges in _SCRIPT_RANGES:if any(lo <= cp <= hi for lo, hi in ranges):counts[name] = counts.get(name, 0) + 1break+ else:+ counts["other"] = counts.get("other", 0) + 1total = sum(counts.values())if not total:return {}diff --git a/tests/test_router.py b/tests/test_router.pyindex 9102b283..579c09cc 100644--- a/tests/test_router.py+++ b/tests/test_router.py@@ -649,6 +649,61 @@ check("threads/hot-path loads keep one entry", order_len, 1)check("threads/hot-path loads keep agents consistent", agents_len, 1)check("threads/hot-path order intact", order, ["english"])+++# --------------------------------------------------------------------- unlisted scripts+# `detect_script` counts an alphabetic character only when one of `_SCRIPT_RANGES` claims+# it. Those ranges cover the scripts the checkpoints were measured on, and most of Unicode+# is outside them -- 68% of alphabetic codepoints, including the CJK extensions, the kana+# supplements, bopomofo, halfwidth katakana and dozens of smaller scripts. An unclaimed+# character used to be counted nowhere, so text written only in such a script produced a+# total of 0, was reported as "unknown", and `analyse` treats "unknown" as English. It was+# therefore routed to the English checkpoint, which has no tokens for it at all, with the+# reason "no letters detected in state" -- for text that plainly has letters.+#+# Every case below reported unknown / is_english=True / model "english" before this change.+for label, text in (+ ("halfwidth katakana", "アリガトウ"),+ ("bopomofo", "ㄆㄇㄈㄉ"),+ ("kana supplement", "\U0001B000\U0001B001"),+ ("CJK Ext-B", "\U00020000\U00020001"),+ ("hangul jamo ext-A", "\ua960\ua961"),+ ("Cherokee", "ᏣᎳᎩ"),+ ("Mongolian", "ᠮᠣᠩᠭᠣᠯ"),+ ("Syriac", "ܫܠܡܐ"),+ ("Thaana", "ދިވެހި"),+ ("Tifinagh", "ⵜⴰⵎⴰⵣⵉⵖⵜ"),+ ("Yi", "ꆈꌠ"),+):+ check("unlisted/" + label + " is not called English", analyse(text)["is_english"], False)+ check("unlisted/" + label + " routes to multilingual",+ Router().route(text)["model"], "multilingual")++# Fullwidth Latin is Latin, not an unlisted script.+check("unlisted/fullwidth latin is latin", detect_script("HELLO"), "latin")++# A state with no letters at all must keep behaving exactly as before.+for label, text in (("empty", ""), ("digits only", "12345 67890"), ("emoji only", "😀😀😀")):+ check("unlisted/letterless " + label + " is still unknown",+ analyse(text)["script"], "unknown")+ check("unlisted/letterless " + label + " keeps the default",+ Router().route(text)["model"], "english")++# The scripts the table does name must be untouched.+for label, text, script in (+ ("english", "please cancel my subscription", "latin"),+ ("german", "Mein Konto wurde zweimal belastet, bitte erstatten Sie den Betrag", "latin"),+ ("hindi", "यह एक हिंदी वाक्य है", "devanagari"),+ ("chinese", "请取消我的订阅", "han"),+ ("japanese", "ありがとう", "kana"),+ ("korean", "감사합니다", "hangul"),+ ("russian", "Мой аккаунт был списан дважды", "cyrillic"),+ ("arabic", "تم خصم حسابي مرتين", "arabic"),+ ("greek", "Η χρέωση έγινε δύο φορές", "greek"),+ ("armenian", "Իմ հաշիվը գանձվել է երկու անգամ", "armenian"),+):+ check("unlisted/regression " + label + " script", detect_script(text), script)+# --------------------------------------------------------------------- reportprint("\n%d passed, %d failed" % (len(PASS), len(FAIL)))for f in FAIL: