Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

fix(lang): route CJK text with Latin brand names to the multilingual checkpoint (#113)

Ahmed Abdullaha2f3d004dc57
diff --git a/laya/lang.py b/laya/lang.py
index c6361f27..2584a919 100644
--- a/laya/lang.py
+++ b/laya/lang.py
@@ -11,6 +11,7 @@ Script detection is exact. The Latin-script language guess is a stopword/diacrit
is explicitly best-effort: pass an explicit model or `lang=` when you already know the language.
"""
import re
+import unicodedata
from typing import Dict, List, Optional, Union
# Unicode blocks that the English (ModernBERT-large, 50k English BPE) checkpoint cannot read.
@@ -251,6 +252,51 @@ def script_profile(text: str) -> Dict[str, float]:
# stopword list matches it.
NON_EN_DIACRITIC_RATE = 0.02
+# Non-Latin text is not for the English checkpoint even when Latin letters are the plurality: a
+# brand name or order code outvotes the CJK request around it letter for letter, though one CJK
+# character carries far more than a letter. A short message needs a large share to count; a long
+# payload (ticket fields, English agent turns) dilutes the share, so there a sentence's worth of
+# letters counts too.
+NON_LATIN_FRACTION = 0.2
+NON_LATIN_MIN_FRACTION = 0.1
+NON_LATIN_MIN_LETTERS = 10
+
+
+def _script_of(ch: str) -> Optional[str]:
+ """The named non-Latin script of one letter, or None for Latin and for unclaimed letters."""
+ cp = ord(ch)
+ if cp < 0x0250 or 0x1E00 <= cp <= 0x1EFF or 0xFF21 <= cp <= 0xFF3A or 0xFF41 <= cp <= 0xFF5A:
+ return None
+ for name, ranges in _SCRIPT_RANGES:
+ if any(lo <= cp <= hi for lo, hi in ranges):
+ return name
+ return None
+
+
+def _non_latin_words(text: str) -> List[str]:
+ """Non-Latin runs that read as words rather than as annotation inside English prose.
+
+ English prose carries three kinds of non-Latin letters that are not a request written in
+ another script, and each is excluded here: a symbol (`Set α to 0.05`, one letter), a proper
+ name (`Дмитрий Петрович Савицкий`, capitalised), and a pronunciation (`[vlɐˈdʲimʲɪr]`, which
+ no script range claims). A combining mark belongs to the letter before it and never splits a
+ word, so `Влади́мир` stays one capitalised name rather than becoming `Влади` + `мир`.
+ """
+ runs, cur, script = [], "", None
+ for ch in text:
+ if unicodedata.combining(ch):
+ continue
+ s = _script_of(ch)
+ if s is not None and s == script:
+ cur += ch
+ continue
+ if cur:
+ runs.append(cur)
+ cur, script = (ch, s) if s is not None else ("", None)
+ if cur:
+ runs.append(cur)
+ return [w for w in runs if len(w) >= 2 and not w[0].isupper()]
+
def latin_profile(text: str) -> Dict[str, object]:
"""Evidence behind the Latin-script language guess.
@@ -318,6 +364,11 @@ def analyse(state: Union[str, dict, list, None]) -> Dict[str, object]:
prof = script_profile(text)
script = detect_script(text)
non_latin = round(1.0 - prof.get("latin", 0.0), 4) if prof else 0.0
+ n_non_latin = round(non_latin * sum(ch.isalpha() for ch in text))
+ if script == "latin" and _non_latin_words(text) and (
+ non_latin >= NON_LATIN_FRACTION or (
+ non_latin >= NON_LATIN_MIN_FRACTION and n_non_latin >= NON_LATIN_MIN_LETTERS)):
+ script = max((s for s in prof if s != "latin"), key=prof.get)
if script == "unknown":
return {"script": "unknown", "script_profile": prof, "language": None,
"is_english": True, "language_undecided": True, "diacritic_rate": 0.0,
diff --git a/tests/test_router.py b/tests/test_router.py
index 1f9575ab..d8a06822 100644
--- a/tests/test_router.py
+++ b/tests/test_router.py
@@ -201,6 +201,37 @@ cases = [
("japanese text", {"body": "二重に請求されました"}, Q_GENERIC, {}, "multilingual"),
("korean text", {"body": "두 번 청구되었습니다"}, Q_GENERIC, {}, "multilingual"),
("arabic text", {"body": "تم خصم المبلغ مرتين"}, Q_GENERIC, {}, "multilingual"),
+ # Latin brand names are the letter plurality here, but the request itself is CJK
+ ("chinese with a brand", {"body": "我的 iPhone 15 Pro Max 订单还没到"}, Q_GENERIC, {}, "multilingual"),
+ ("japanese with brands", {"body": "Amazonで買ったiPhoneが届かない"}, Q_GENERIC, {}, "multilingual"),
+ ("korean with a brand", {"body": "Samsung Galaxy 주문이 아직 안 왔어요"}, Q_GENERIC, {}, "multilingual"),
+ ("english with a han name", {"body": "My name is 王小明 and my order is late"}, Q_GENERIC, {}, "english"),
+ # an English wrapper dilutes the share, but the request is still CJK
+ ("chinese in a ticket", {"ticket_id": "TCK-88213", "channel": "web chat",
+ "agent_notes": "Customer asked about a delayed order. Please check shipping status.",
+ "message": "我的订单已经两个星期了还没有到"}, Q_GENERIC, {}, "multilingual"),
+ ("korean after english turns", [{"role": "agent", "text": "Hello! Thanks for contacting support."},
+ {"role": "agent", "text": "Could you share your order number please?"},
+ {"role": "user", "text": "주문번호는 5521이고 아직 배송이 안 됐어요"}],
+ Q_GENERIC, {}, "multilingual"),
+ ("english with greek symbols", {"request": "Compute the mean μ and variance σ of X, then P(|X-μ| > 2σ)."},
+ Q_GENERIC, {}, "english"),
+ # English prose that names someone in their own script: the name is not the request, and a
+ # capitalised run, a lone symbol and a pronunciation are all annotation rather than content.
+ ("english prose, russian name",
+ {"body": "Anton Pavlovich Chekhov (Russian: Антон Павлович Чехов) was a playwright."},
+ Q_GENERIC, {}, "english"),
+ ("english prose, name with IPA",
+ {"body": "Vladimir Nabokov (Russian: Влади́мир Набо́ков [vlɐˈdʲimʲɪr nɐˈbokəf]) wrote Lolita "
+ "and taught literature at Cornell for more than a decade."},
+ Q_GENERIC, {}, "english"),
+ ("english prose, greek name",
+ {"body": "Eleftherios Venizelos (Greek: Ελευθέριος Βενιζέλος) served as prime minister."},
+ Q_GENERIC, {}, "english"),
+ ("english prose, hebrew name",
+ {"body": "Amos Oz (Hebrew: עמוס עוז), born Amos Klausner, was an Israeli writer and professor "
+ "of literature at Ben-Gurion University of the Negev in Beersheba."},
+ Q_GENERIC, {}, "english"),
("german text", {"body": "Der Kunde wurde zweimal belastet und moechte eine Rueckerstattung "
"fuer die Rechnung die nicht korrekt ist"}, Q_GENERIC, {}, "multilingual"),
("explicit model", {"body": "anything"}, Q_GENERIC, {"model": "multilingual"}, "multilingual"),