Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
fix(lang): a repeated English word that collides with a foreign function word routed English to multilingual
AlKor1312be45494ca8
diff --git a/laya/lang.py b/laya/lang.pyindex d457940b..8522bf93 100644--- a/laya/lang.py+++ b/laya/lang.py@@ -12,6 +12,7 @@ is explicitly best-effort: pass an explicit model or `lang=` when you already kn"""import reimport unicodedata+from collections import Counterfrom collections.abc import Mappingfrom typing import Dict, List, Optional, Union@@ -176,6 +177,18 @@ _SHARED_WORDS = {w for w in {word for words in _STOP.values() for word in words}# English function words no other list holds (`in`, `is`, `as`, `was` are shared with German,# Dutch and Portuguese). They alone carry the English rescue of `latin_profile`._EN_ONLY_WORDS = _STOP["en"] - _SHARED_WORDS+# A few of these function words are also ordinary English words: `come` (Italian), `son` (Spanish),+# `do` (Portuguese), `care` (Romanian), `todo` (a to-do list), `per`, `plus`, and `im` -- German+# `im`, and also how `I'm` is written without the apostrophe. For those a repeat is evidence of the+# foreign language exactly once: "do more, do less" repeats an English word, and counting both hits+# let one such word clear the `best >= 2` bar below and send plain English to the multilingual+# checkpoint. Every other word keeps counting occurrences: `der`, `los` and `sa` are nobody's+# English, so a request whose only evidence is `der` twice ("reduzieren der helligkeit der lichter")+# is still German, and deduping every word instead sent 791 of 148,700 non-English MASSIVE test rows+# to the English checkpoint. Dutch `van` is the word this list deliberately leaves out: "liedje van+# ... van" is ordinary Dutch, and deduping it moved 14 of those rows while rescuing no English one.+_EN_COLLISION_WORDS = {"come", "son", "do", "care", "todo", "im", "per", "plus"} & {+ w for lg, sw in _STOP.items() if lg != "en" for w in sw}_WORD = re.compile(r"[^\W\d_]+", re.UNICODE)# A token whose dot or @ joins word characters is an identifier, not prose: `github.com`,@@ -400,7 +413,10 @@ def latin_profile(text: str) -> Dict[str, object]:return {"language": None, "english_hits": 0, "diacritic_rate": diac_rate,"looks_non_english": non_english}- scores = {lg: sum(1 for w in words if w in sw) for lg, sw in _STOP.items()}+ # A collision word counts once however often it repeats; every other word counts its hits.+ counts = Counter(words)+ scores = {lg: sum(1 if w in _EN_COLLISION_WORDS else n for w, n in counts.items() if w in sw)+ for lg, sw in _STOP.items()}en = scores.get("en", 0)# Only a language that matched at least one word no other list claims may be named. Without# that condition the top score can be pure overlap -- `la` and `e` in Romanian text madediff --git a/tests/test_lang_guess.py b/tests/test_lang_guess.pyindex 17c20d69..d68a3d83 100644--- a/tests/test_lang_guess.py+++ b/tests/test_lang_guess.py@@ -18,6 +18,7 @@ import syssys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))from laya.router import Router, _english_from_code # noqa: E402+from laya.lang import analyse # noqa: E402PASS, FAIL = [], []@@ -176,6 +177,51 @@ for label, s, want in BEFORE:check("unchanged/" + label, r0.route(s, GENERIC)["model"], want)+# A foreign function word that is also an ordinary English word (`come` it, `son` es, `do` pt,+# `care` ro, `todo`/`im`/`per`/`plus`) used to name that language when it merely appeared+# twice, because the score counted occurrences and `best >= 2` was reached by one repeated word.+# Ordinary English then routed to the multilingual checkpoint. Such a word now counts once however+# often it repeats, so one collision no longer clears the bar.+ENGLISH_ON_A_REPEAT = [+ "Come one, come all", # come (it) x2+ "My son, your son", # son (es) x2+ "Do more, do less", # do (pt) x2+ "Care more, care less", # care (ro) x2+ "Add a todo, then another todo item", # todo (es) x2+ "im not able to log in, im stuck", # im (de) x2+ "add 45 to 87 plus 54 plus 43 plus 22", # plus (de, fr) x3 -- a CLINC150 test row+]+for s in ENGLISH_ON_A_REPEAT:+ check("repeat/plain english stays english: %r" % s, analyse(s)["is_english"], True)++# The dedupe is limited to those words. A function word that is nobody's English -- `der`, `des`,+# `di`, `sa` -- still counts every occurrence, so a real request whose only evidence is one such+# word repeated stays foreign. Rows from the MASSIVE test splits, which is where deduping every+# word instead of the collisions cost 791 of 148,700 non-English texts their checkpoint.+REPEAT_STAYS_FOREIGN = [+ ("de", "reduzieren der helligkeit der lichter"),+ ("es", "enumerar todos los horarios de los tren a nueva york"),+ ("fr", "jouer des chansons des beatles"),+ ("it", "numero di telefono di giacomo"),+ ("pt", "pede um pacote de massa chinesa faz um pedido takeaway"),+ ("ro", "ar trebui sa port o pelerina de ploaie inainte sa ies afara"),+ ("nl", "herinner me eraan dat ik dat liedje leuk vind"),+ # `van` is left out of the collision list for exactly this row: it is ordinary Dutch+ ("nl", "hey olly ik hou van muziek van frans bauer"),+]+for want, s in REPEAT_STAYS_FOREIGN:+ check("repeat/%s on a repeated word is still foreign" % want, analyse(s)["is_english"], False)++# and real foreign requests -- two different function words each -- are still detected+STILL_FOREIGN = [+ ("es", "Hola, necesito cancelar mi pedido por favor ahora mismo gracias"),+ ("pt", "Eu quero cancelar o meu pedido por favor agora mesmo obrigado"),+ ("it", "Vorrei annullare il mio ordine per favore adesso grazie mille"),+]+for want, s in STILL_FOREIGN:+ check("repeat/genuine %s still foreign" % want, analyse(s)["is_english"], False)++print("\n%d passed, %d failed" % (len(PASS), len(FAIL)))for f in FAIL:print(" FAIL " + f)