Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
fix(lang): tighten the loanword rescue to evidence only English holds
Linz12481de8c41d0b9d
diff --git a/laya/lang.py b/laya/lang.pyindex 7377b6bc..7d4e52ec 100644--- a/laya/lang.py+++ b/laya/lang.py@@ -172,6 +172,9 @@ _NON_EN_DIACRITICS = set(# way), though it still counts toward the total of a language that also matched a word of its own._SHARED_WORDS = {w for w in {word for words in _STOP.values() for word in words}if sum(w in words for words in _STOP.values()) > 1}+# English function words no other list holds (`in`, `is`, `as`, `was` are shared with German,+# Dutch and Portuguese). They alone carry the English rescue of `latin_profile`.+_EN_ONLY_WORDS = _STOP["en"] - _SHARED_WORDS_WORD = re.compile(r"[^\W\d_]+", re.UNICODE)# A token whose dot or @ joins word characters is an identifier, not prose: `github.com`,@@ -280,8 +283,8 @@ NON_EN_DIACRITIC_RATE = 0.02# One accented loanword or proper noun (`café`, `résumé`, `José`) must not alone pull otherwise# plain English off the English checkpoint: the rate is measured over every character, so a# single `é` in a short sentence clears the floor above. English function words keep their say-# unless the rate is well above the floor, so genuinely non-English text still loses its-# English reading.+# only through the rescue below -- two distinct words no other list holds, at most one+# non-English letter word, and a rate still well above the floor vetoes regardless.ENGLISH_RESCUE_DIACRITIC_RATE = 0.06# Non-Latin text is not for the English checkpoint even when Latin letters are the plurality: a@@ -330,6 +333,23 @@ def _non_latin_words(text: str) -> List[str]:return [w for w in runs if len(w) >= 2 and not w[0].isupper()]+def _english_rescued_by_words(words: List[str], diac_rate: float) -> bool:+ """Whether plain-English function words outvote a marginal diacritic rate (#337).++ The rate is measured over every character, so one accented loanword or proper noun in a+ short sentence clears `NON_EN_DIACRITIC_RATE` outright. English still wins when it shows at+ least two distinct function words no other list holds and at most one word carrying a+ non-English letter: one loanword is not a non-English vocabulary, while the odd word a+ Danish or Swedish sentence picks up (`i`, `at`, `for`, `have`) is not an English sentence+ either. A rate well above `ENGLISH_RESCUE_DIACRITIC_RATE` vetoes regardless.+ """+ if diac_rate >= ENGLISH_RESCUE_DIACRITIC_RATE:+ return False+ if len(set(words) & _EN_ONLY_WORDS) < 2:+ return False+ return sum(1 for w in set(words) if any(ch in _NON_EN_DIACRITICS for ch in w)) <= 1++def latin_profile(text: str) -> Dict[str, object]:"""Evidence behind the Latin-script language guess.@@ -370,9 +390,7 @@ def latin_profile(text: str) -> Dict[str, object]:# Needs two hits here too. One shared function word ("para" in Turkish text) named Spanish# on the strength of the diacritics alone, which is a guess dressed as a detection.lang = best_lg- elif en and (not non_english or diac_rate < ENGLISH_RESCUE_DIACRITIC_RATE):- # A marginal diacritic rate (one loanword) does not outvote English function words;- # only a rate well above the floor keeps its veto.+ elif en and (not non_english or _english_rescued_by_words(words, diac_rate)):lang = "en"return {"language": lang, "english_hits": en, "diacritic_rate": diac_rate,"looks_non_english": non_english}diff --git a/tests/test_router.py b/tests/test_router.pyindex e7c49e38..ce73b95b 100644--- a/tests/test_router.py+++ b/tests/test_router.py@@ -486,9 +486,9 @@ check("latin_lang/french du stays evidence", guess_latin_language("baisse le vol# The diacritic rate is measured over every character, so one `é` in a short English sentence# clears the 0.02 floor and used to veto the English resolution outright: plain English with a# loanword or foreign name (`café`, `résumé`, `José`, `Zürich`) went to the checkpoint the-# README says collapses on English-heavy Latin text. English function words now outvote a-# marginal rate; a rate well above the floor (several diacritics and no English evidence) still-# vetoes.+# README says collapses on English-heavy Latin text. English wins the veto back only through the+# word rescue: at least two distinct function words no other list holds, and at most one word+# carrying a non-English letter.for text in ["Please send me the café menu today please","Could you email me your résumé before the meeting","Send the invoice to José before Friday",@@ -501,6 +501,20 @@ check("latin_lang/accented german stays non-english",is_english("Grüße aus Köln, wir melden uns wegen der Rechnung"), False)check("route/accented german stays multilingual",_r_lat.route("Grüße aus Köln, wir melden uns wegen der Rechnung").model, "multilingual")+# Danish and Swedish hold no list here, and their accented function-word sentences pick up just+# one or two English-shaped words (`i`, `at`, `for`, `have`), which is not the two-distinct-word+# English the rescue requires -- a rescue that counted them sent plain Danish to the English+# checkpoint on the MASSIVE splits.+for text in ["sluk lyset i soveværelset", # da+ "kan jeg få en refundering for det dobbelte beløb", # da+ "stäng av ljuset i sovrummet", # sv+ "jag vill ha en återbetalning för den dubbla avgiften"]: # sv+ check("latin_lang/nordic accented stays non-english " + text, is_english(text), False)+ check("route/nordic accented stays multilingual " + text, _r_lat.route(text).model, "multilingual")+# Two non-English-letter words is a running non-English vocabulary, not one loanword: the rescue+# does not fire even with English function words present.+check("latin_lang/two diacritic words are not one loanword",+ is_english("The naïve façade needs a fresh coat of paint"), False)# --------------------------------------------------------------------- temperature clamp (#35)