Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

fix(lang): count letters no script range claims, instead of routing them to English (#169)

PerryLink129007f22a77
diff --git a/laya/lang.py b/laya/lang.py
index 7edc3700..911509a8 100644
--- a/laya/lang.py
+++ b/laya/lang.py
@@ -179,13 +179,24 @@ def detect_script(text: str) -> str:
if not ch.isalpha():
continue
cp = ord(ch)
- if cp < 0x0250 or 0x1E00 <= cp <= 0x1EFF: # Latin + Latin Extended Additional
- latin += 1
+ if cp < 0x0250 or 0x1E00 <= cp <= 0x1EFF or 0xFF21 <= cp <= 0xFF3A or 0xFF41 <= cp <= 0xFF5A:
+ latin += 1 # Latin, Latin Ext-Additional, fullwidth
continue
for name, ranges in _SCRIPT_RANGES:
if any(lo <= cp <= hi for lo, hi in ranges):
counts[name] = counts.get(name, 0) + 1
break
+ else:
+ # An alphabetic character no range claims used to be counted nowhere, so text
+ # written only in an unlisted script produced a total of 0 and was reported as
+ # "unknown" -- and `analyse` treats "unknown" as English, sending it to the
+ # checkpoint that has no tokens for it. 68% of Unicode's alphabetic codepoints
+ # are outside _SCRIPT_RANGES (the CJK extensions, kana supplements, bopomofo,
+ # halfwidth katakana, and dozens of smaller scripts), so enumerating them all is
+ # not maintainable. Counting the remainder under "other" keeps them visible and
+ # non-Latin, which is the safe direction: an unreadable script must not be
+ # handed to the English checkpoint.
+ counts["other"] = counts.get("other", 0) + 1
counts["latin"] = latin
total = sum(counts.values())
if total == 0:
@@ -200,13 +211,15 @@ def script_profile(text: str) -> Dict[str, float]:
if not ch.isalpha():
continue
cp = ord(ch)
- if cp < 0x0250 or 0x1E00 <= cp <= 0x1EFF:
+ if cp < 0x0250 or 0x1E00 <= cp <= 0x1EFF or 0xFF21 <= cp <= 0xFF3A or 0xFF41 <= cp <= 0xFF5A:
counts["latin"] += 1
continue
for name, ranges in _SCRIPT_RANGES:
if any(lo <= cp <= hi for lo, hi in ranges):
counts[name] = counts.get(name, 0) + 1
break
+ else:
+ counts["other"] = counts.get("other", 0) + 1
total = sum(counts.values())
if not total:
return {}
diff --git a/tests/test_router.py b/tests/test_router.py
index 9102b283..579c09cc 100644
--- a/tests/test_router.py
+++ b/tests/test_router.py
@@ -649,6 +649,61 @@ check("threads/hot-path loads keep one entry", order_len, 1)
check("threads/hot-path loads keep agents consistent", agents_len, 1)
check("threads/hot-path order intact", order, ["english"])
+
+
+# --------------------------------------------------------------------- unlisted scripts
+# `detect_script` counts an alphabetic character only when one of `_SCRIPT_RANGES` claims
+# it. Those ranges cover the scripts the checkpoints were measured on, and most of Unicode
+# is outside them -- 68% of alphabetic codepoints, including the CJK extensions, the kana
+# supplements, bopomofo, halfwidth katakana and dozens of smaller scripts. An unclaimed
+# character used to be counted nowhere, so text written only in such a script produced a
+# total of 0, was reported as "unknown", and `analyse` treats "unknown" as English. It was
+# therefore routed to the English checkpoint, which has no tokens for it at all, with the
+# reason "no letters detected in state" -- for text that plainly has letters.
+#
+# Every case below reported unknown / is_english=True / model "english" before this change.
+for label, text in (
+ ("halfwidth katakana", "アリガトウ"),
+ ("bopomofo", "ㄆㄇㄈㄉ"),
+ ("kana supplement", "\U0001B000\U0001B001"),
+ ("CJK Ext-B", "\U00020000\U00020001"),
+ ("hangul jamo ext-A", "\ua960\ua961"),
+ ("Cherokee", "ᏣᎳᎩ"),
+ ("Mongolian", "ᠮᠣᠩᠭᠣᠯ"),
+ ("Syriac", "ܫܠܡܐ"),
+ ("Thaana", "ދިވެހި"),
+ ("Tifinagh", "ⵜⴰⵎⴰⵣⵉⵖⵜ"),
+ ("Yi", "ꆈꌠ"),
+):
+ check("unlisted/" + label + " is not called English", analyse(text)["is_english"], False)
+ check("unlisted/" + label + " routes to multilingual",
+ Router().route(text)["model"], "multilingual")
+
+# Fullwidth Latin is Latin, not an unlisted script.
+check("unlisted/fullwidth latin is latin", detect_script("HELLO"), "latin")
+
+# A state with no letters at all must keep behaving exactly as before.
+for label, text in (("empty", ""), ("digits only", "12345 67890"), ("emoji only", "😀😀😀")):
+ check("unlisted/letterless " + label + " is still unknown",
+ analyse(text)["script"], "unknown")
+ check("unlisted/letterless " + label + " keeps the default",
+ Router().route(text)["model"], "english")
+
+# The scripts the table does name must be untouched.
+for label, text, script in (
+ ("english", "please cancel my subscription", "latin"),
+ ("german", "Mein Konto wurde zweimal belastet, bitte erstatten Sie den Betrag", "latin"),
+ ("hindi", "यह एक हिंदी वाक्य है", "devanagari"),
+ ("chinese", "请取消我的订阅", "han"),
+ ("japanese", "ありがとう", "kana"),
+ ("korean", "감사합니다", "hangul"),
+ ("russian", "Мой аккаунт был списан дважды", "cyrillic"),
+ ("arabic", "تم خصم حسابي مرتين", "arabic"),
+ ("greek", "Η χρέωση έγινε δύο φορές", "greek"),
+ ("armenian", "Իմ հաշիվը գանձվել է երկու անգամ", "armenian"),
+):
+ check("unlisted/regression " + label + " script", detect_script(text), script)
+
# --------------------------------------------------------------------- report
print("\n%d passed, %d failed" % (len(PASS), len(FAIL)))
for f in FAIL: