Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
fix(lang): route CJK text with Latin brand names to the multilingual checkpoint (#113)
Ahmed Abdullaha2f3d004dc57
diff --git a/laya/lang.py b/laya/lang.pyindex c6361f27..2584a919 100644--- a/laya/lang.py+++ b/laya/lang.py@@ -11,6 +11,7 @@ Script detection is exact. The Latin-script language guess is a stopword/diacritis explicitly best-effort: pass an explicit model or `lang=` when you already know the language."""import re+import unicodedatafrom typing import Dict, List, Optional, Union# Unicode blocks that the English (ModernBERT-large, 50k English BPE) checkpoint cannot read.@@ -251,6 +252,51 @@ def script_profile(text: str) -> Dict[str, float]:# stopword list matches it.NON_EN_DIACRITIC_RATE = 0.02+# Non-Latin text is not for the English checkpoint even when Latin letters are the plurality: a+# brand name or order code outvotes the CJK request around it letter for letter, though one CJK+# character carries far more than a letter. A short message needs a large share to count; a long+# payload (ticket fields, English agent turns) dilutes the share, so there a sentence's worth of+# letters counts too.+NON_LATIN_FRACTION = 0.2+NON_LATIN_MIN_FRACTION = 0.1+NON_LATIN_MIN_LETTERS = 10+++def _script_of(ch: str) -> Optional[str]:+ """The named non-Latin script of one letter, or None for Latin and for unclaimed letters."""+ cp = ord(ch)+ if cp < 0x0250 or 0x1E00 <= cp <= 0x1EFF or 0xFF21 <= cp <= 0xFF3A or 0xFF41 <= cp <= 0xFF5A:+ return None+ for name, ranges in _SCRIPT_RANGES:+ if any(lo <= cp <= hi for lo, hi in ranges):+ return name+ return None+++def _non_latin_words(text: str) -> List[str]:+ """Non-Latin runs that read as words rather than as annotation inside English prose.++ English prose carries three kinds of non-Latin letters that are not a request written in+ another script, and each is excluded here: a symbol (`Set α to 0.05`, one letter), a proper+ name (`Дмитрий Петрович Савицкий`, capitalised), and a pronunciation (`[vlɐˈdʲimʲɪr]`, which+ no script range claims). A combining mark belongs to the letter before it and never splits a+ word, so `Влади́мир` stays one capitalised name rather than becoming `Влади` + `мир`.+ """+ runs, cur, script = [], "", None+ for ch in text:+ if unicodedata.combining(ch):+ continue+ s = _script_of(ch)+ if s is not None and s == script:+ cur += ch+ continue+ if cur:+ runs.append(cur)+ cur, script = (ch, s) if s is not None else ("", None)+ if cur:+ runs.append(cur)+ return [w for w in runs if len(w) >= 2 and not w[0].isupper()]+def latin_profile(text: str) -> Dict[str, object]:"""Evidence behind the Latin-script language guess.@@ -318,6 +364,11 @@ def analyse(state: Union[str, dict, list, None]) -> Dict[str, object]:prof = script_profile(text)script = detect_script(text)non_latin = round(1.0 - prof.get("latin", 0.0), 4) if prof else 0.0+ n_non_latin = round(non_latin * sum(ch.isalpha() for ch in text))+ if script == "latin" and _non_latin_words(text) and (+ non_latin >= NON_LATIN_FRACTION or (+ non_latin >= NON_LATIN_MIN_FRACTION and n_non_latin >= NON_LATIN_MIN_LETTERS)):+ script = max((s for s in prof if s != "latin"), key=prof.get)if script == "unknown":return {"script": "unknown", "script_profile": prof, "language": None,"is_english": True, "language_undecided": True, "diacritic_rate": 0.0,diff --git a/tests/test_router.py b/tests/test_router.pyindex 1f9575ab..d8a06822 100644--- a/tests/test_router.py+++ b/tests/test_router.py@@ -201,6 +201,37 @@ cases = [("japanese text", {"body": "二重に請求されました"}, Q_GENERIC, {}, "multilingual"),("korean text", {"body": "두 번 청구되었습니다"}, Q_GENERIC, {}, "multilingual"),("arabic text", {"body": "تم خصم المبلغ مرتين"}, Q_GENERIC, {}, "multilingual"),+ # Latin brand names are the letter plurality here, but the request itself is CJK+ ("chinese with a brand", {"body": "我的 iPhone 15 Pro Max 订单还没到"}, Q_GENERIC, {}, "multilingual"),+ ("japanese with brands", {"body": "Amazonで買ったiPhoneが届かない"}, Q_GENERIC, {}, "multilingual"),+ ("korean with a brand", {"body": "Samsung Galaxy 주문이 아직 안 왔어요"}, Q_GENERIC, {}, "multilingual"),+ ("english with a han name", {"body": "My name is 王小明 and my order is late"}, Q_GENERIC, {}, "english"),+ # an English wrapper dilutes the share, but the request is still CJK+ ("chinese in a ticket", {"ticket_id": "TCK-88213", "channel": "web chat",+ "agent_notes": "Customer asked about a delayed order. Please check shipping status.",+ "message": "我的订单已经两个星期了还没有到"}, Q_GENERIC, {}, "multilingual"),+ ("korean after english turns", [{"role": "agent", "text": "Hello! Thanks for contacting support."},+ {"role": "agent", "text": "Could you share your order number please?"},+ {"role": "user", "text": "주문번호는 5521이고 아직 배송이 안 됐어요"}],+ Q_GENERIC, {}, "multilingual"),+ ("english with greek symbols", {"request": "Compute the mean μ and variance σ of X, then P(|X-μ| > 2σ)."},+ Q_GENERIC, {}, "english"),+ # English prose that names someone in their own script: the name is not the request, and a+ # capitalised run, a lone symbol and a pronunciation are all annotation rather than content.+ ("english prose, russian name",+ {"body": "Anton Pavlovich Chekhov (Russian: Антон Павлович Чехов) was a playwright."},+ Q_GENERIC, {}, "english"),+ ("english prose, name with IPA",+ {"body": "Vladimir Nabokov (Russian: Влади́мир Набо́ков [vlɐˈdʲimʲɪr nɐˈbokəf]) wrote Lolita "+ "and taught literature at Cornell for more than a decade."},+ Q_GENERIC, {}, "english"),+ ("english prose, greek name",+ {"body": "Eleftherios Venizelos (Greek: Ελευθέριος Βενιζέλος) served as prime minister."},+ Q_GENERIC, {}, "english"),+ ("english prose, hebrew name",+ {"body": "Amos Oz (Hebrew: עמוס עוז), born Amos Klausner, was an Israeli writer and professor "+ "of literature at Ben-Gurion University of the Negev in Beersheba."},+ Q_GENERIC, {}, "english"),("german text", {"body": "Der Kunde wurde zweimal belastet und moechte eine Rueckerstattung ""fuer die Rechnung die nicht korrekt ist"}, Q_GENERIC, {}, "multilingual"),("explicit model", {"body": "anything"}, Q_GENERIC, {"model": "multilingual"}, "multilingual"),