Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

fix(lang): one accented loanword no longer pulls plain English off the English checkpoint

Linz124827afcb291060
diff --git a/laya/lang.py b/laya/lang.py
index 0122c787..7377b6bc 100644
--- a/laya/lang.py
+++ b/laya/lang.py
@@ -277,6 +277,13 @@ def script_profile(text: str) -> Dict[str, float]:
# stopword list matches it.
NON_EN_DIACRITIC_RATE = 0.02
+# One accented loanword or proper noun (`café`, `résumé`, `José`) must not alone pull otherwise
+# plain English off the English checkpoint: the rate is measured over every character, so a
+# single `é` in a short sentence clears the floor above. English function words keep their say
+# unless the rate is well above the floor, so genuinely non-English text still loses its
+# English reading.
+ENGLISH_RESCUE_DIACRITIC_RATE = 0.06
+
# Non-Latin text is not for the English checkpoint even when Latin letters are the plurality: a
# brand name or order code outvotes the CJK request around it letter for letter, though one CJK
# character carries far more than a letter. A short message needs a large share to count; a long
@@ -363,7 +370,9 @@ def latin_profile(text: str) -> Dict[str, object]:
# Needs two hits here too. One shared function word ("para" in Turkish text) named Spanish
# on the strength of the diacritics alone, which is a guess dressed as a detection.
lang = best_lg
- elif en and not non_english:
+ elif en and (not non_english or diac_rate < ENGLISH_RESCUE_DIACRITIC_RATE):
+ # A marginal diacritic rate (one loanword) does not outvote English function words;
+ # only a rate well above the floor keeps its veto.
lang = "en"
return {"language": lang, "english_hits": en, "diacritic_rate": diac_rate,
"looks_non_english": non_english}
diff --git a/tests/test_router.py b/tests/test_router.py
index 24d65d90..e7c49e38 100644
--- a/tests/test_router.py
+++ b/tests/test_router.py
@@ -482,6 +482,26 @@ for text in ["turn off smart lamp in den", "im so sorry, am an hour late, stuck
check("latin_lang/spanish es stays evidence", guess_latin_language("que hora es en australia"), "es")
check("latin_lang/french du stays evidence", guess_latin_language("baisse le volume du haut-parleur"), "fr")
+# ------------------------------------------------------------------ accented loanwords in English (#337)
+# The diacritic rate is measured over every character, so one `é` in a short English sentence
+# clears the 0.02 floor and used to veto the English resolution outright: plain English with a
+# loanword or foreign name (`café`, `résumé`, `José`, `Zürich`) went to the checkpoint the
+# README says collapses on English-heavy Latin text. English function words now outvote a
+# marginal rate; a rate well above the floor (several diacritics and no English evidence) still
+# vetoes.
+for text in ["Please send me the café menu today please",
+ "Could you email me your résumé before the meeting",
+ "Send the invoice to José before Friday",
+ "We visited Zürich last summer and loved it"]:
+ check("latin_lang/loanword english stays english " + text, guess_latin_language(text), "en")
+ check("route/loanword english stays english " + text, _r_lat.route(text).model, "english")
+# Genuinely non-English accented text keeps its multilingual routing: a German sentence with
+# umlauts and no English function word is not rescued.
+check("latin_lang/accented german stays non-english",
+ is_english("Grüße aus Köln, wir melden uns wegen der Rechnung"), False)
+check("route/accented german stays multilingual",
+ _r_lat.route("Grüße aus Köln, wir melden uns wegen der Rechnung").model, "multilingual")
+
# --------------------------------------------------------------------- temperature clamp (#35)
# A fitted temperature below 1 sharpens logits. The shipped `choice:11+` bucket is 0.1006, which