Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
fix(lang): one accented loanword no longer pulls plain English off the English checkpoint
Linz124827afcb291060
diff --git a/laya/lang.py b/laya/lang.pyindex 0122c787..7377b6bc 100644--- a/laya/lang.py+++ b/laya/lang.py@@ -277,6 +277,13 @@ def script_profile(text: str) -> Dict[str, float]:# stopword list matches it.NON_EN_DIACRITIC_RATE = 0.02+# One accented loanword or proper noun (`café`, `résumé`, `José`) must not alone pull otherwise+# plain English off the English checkpoint: the rate is measured over every character, so a+# single `é` in a short sentence clears the floor above. English function words keep their say+# unless the rate is well above the floor, so genuinely non-English text still loses its+# English reading.+ENGLISH_RESCUE_DIACRITIC_RATE = 0.06+# Non-Latin text is not for the English checkpoint even when Latin letters are the plurality: a# brand name or order code outvotes the CJK request around it letter for letter, though one CJK# character carries far more than a letter. A short message needs a large share to count; a long@@ -363,7 +370,9 @@ def latin_profile(text: str) -> Dict[str, object]:# Needs two hits here too. One shared function word ("para" in Turkish text) named Spanish# on the strength of the diacritics alone, which is a guess dressed as a detection.lang = best_lg- elif en and not non_english:+ elif en and (not non_english or diac_rate < ENGLISH_RESCUE_DIACRITIC_RATE):+ # A marginal diacritic rate (one loanword) does not outvote English function words;+ # only a rate well above the floor keeps its veto.lang = "en"return {"language": lang, "english_hits": en, "diacritic_rate": diac_rate,"looks_non_english": non_english}diff --git a/tests/test_router.py b/tests/test_router.pyindex 24d65d90..e7c49e38 100644--- a/tests/test_router.py+++ b/tests/test_router.py@@ -482,6 +482,26 @@ for text in ["turn off smart lamp in den", "im so sorry, am an hour late, stuckcheck("latin_lang/spanish es stays evidence", guess_latin_language("que hora es en australia"), "es")check("latin_lang/french du stays evidence", guess_latin_language("baisse le volume du haut-parleur"), "fr")+# ------------------------------------------------------------------ accented loanwords in English (#337)+# The diacritic rate is measured over every character, so one `é` in a short English sentence+# clears the 0.02 floor and used to veto the English resolution outright: plain English with a+# loanword or foreign name (`café`, `résumé`, `José`, `Zürich`) went to the checkpoint the+# README says collapses on English-heavy Latin text. English function words now outvote a+# marginal rate; a rate well above the floor (several diacritics and no English evidence) still+# vetoes.+for text in ["Please send me the café menu today please",+ "Could you email me your résumé before the meeting",+ "Send the invoice to José before Friday",+ "We visited Zürich last summer and loved it"]:+ check("latin_lang/loanword english stays english " + text, guess_latin_language(text), "en")+ check("route/loanword english stays english " + text, _r_lat.route(text).model, "english")+# Genuinely non-English accented text keeps its multilingual routing: a German sentence with+# umlauts and no English function word is not rescued.+check("latin_lang/accented german stays non-english",+ is_english("Grüße aus Köln, wir melden uns wegen der Rechnung"), False)+check("route/accented german stays multilingual",+ _r_lat.route("Grüße aus Köln, wir melden uns wegen der Rechnung").model, "multilingual")+# --------------------------------------------------------------------- temperature clamp (#35)# A fitted temperature below 1 sharpens logits. The shipped `choice:11+` bucket is 0.1006, which