Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

fix(lang): identify plain-ASCII German instead of labelling it English (#130)

Rishit1bee2d0732bc
diff --git a/laya/lang.py b/laya/lang.py
index 559bb077..0122c787 100644
--- a/laya/lang.py
+++ b/laya/lang.py
@@ -64,7 +64,13 @@ _STOP = {
"merci", "bonjour", "jour", "jours", "mois", "fois", "quand", "comment", "pourquoi",
"alors", "donc"},
"de": {"der", "die", "das", "und", "ist", "ein", "eine", "den", "dem", "nicht", "mit", "für",
- "auf", "von", "zu", "sich", "auch", "werden", "wurde", "haben", "sind", "oder", "aber"},
+ "auf", "von", "zu", "sich", "auch", "werden", "wurde", "haben", "sind", "oder", "aber",
+ "ich", "wir", "mir", "mich", "dir", "dich", "uns", "mein", "meine", "meinen",
+ "meinem", "meiner", "diese", "dieser", "diesen", "dieses", "einen", "einem", "einer",
+ "wie", "wo", "wann", "welche", "im", "zum", "zur", "aus", "bei", "nach", "noch", "bitte",
+ "heute", "jetzt", "kann", "kannst", "habe", "gibt", "wird",
+ # shared with English on purpose: counted for English alone, they outvoted short German
+ "in", "was"},
"es": {"el", "los", "las", "que", "por", "con", "para", "una", "es", "se", "del", "como",
"pero", "son", "está", "este", "esta", "todo", "más", "muy", "hay", "sus",
# `de`/`en` are Spanish too, but they are common English tokens as well (`de facto`,
diff --git a/tests/test_lang_guess.py b/tests/test_lang_guess.py
index 75438c16..17c20d69 100644
--- a/tests/test_lang_guess.py
+++ b/tests/test_lang_guess.py
@@ -168,7 +168,7 @@ check("standalone/hint still uses the standalone repo",
# `Router().route(...)` cannot fail, because both sides are the same pure call on an
# equally configured router -- a regression would move both together.
BEFORE = [("plain english", "I was charged twice and want a refund", "english"),
- ("German with umlauts", "Mein Konto wurde zweimal belastet, bitte erstatten Sie", "english"),
+ ("German with umlauts", "Mein Konto wurde zweimal belastet, bitte erstatten Sie", "multilingual"),
("Hindi", "यह एक हिंदी वाक्य है", "multilingual"),
("empty", "", "english"),
("digits", "12345", "english")]
diff --git a/tests/test_router.py b/tests/test_router.py
index 5c79f5f4..24d65d90 100644
--- a/tests/test_router.py
+++ b/tests/test_router.py
@@ -462,6 +462,27 @@ check("latin_lang/romanian with ei stays romanian",
guess_latin_language("Ei nu sunt de acord cu factura, vreau o corecție"), "ro")
+# --------------------------------------------------------------------- plain-ASCII German (#54)
+# No umlaut for the diacritic rate to catch, and `in`/`was` counted for English alone, so these were
+# labelled English and handed to the checkpoint that cannot read them.
+for text in ["trage diesen termin in meinen kalender ein",
+ "wie lautet die temperatur in fulda in hessen",
+ "schalte das licht im wohnzimmer aus",
+ "was ist die aktuelle zeit"]:
+ check("latin_lang/ascii german " + text, guess_latin_language(text), "de")
+ check("route/ascii german " + text, _r_lat.route(text).model, "multilingual")
+check("route/english control for #54",
+ _r_lat.route("I would like to book a flight to Berlin tomorrow").model, "english")
+# English that shares words with the German list stays English. `in` and `den` leave the first, a real
+# en-US MASSIVE utterance, one German hit short of flipping; the chat line carries `im` and flips if
+# any one of the English words `am`, `an` or `so` joins the German list.
+for text in ["turn off smart lamp in den", "im so sorry, am an hour late, stuck in traffic"]:
+ check("route/english sharing german words " + text, _r_lat.route(text).model, "english")
+# German words that Spanish (`es`) or French (`du`) also claim would stop naming those languages
+check("latin_lang/spanish es stays evidence", guess_latin_language("que hora es en australia"), "es")
+check("latin_lang/french du stays evidence", guess_latin_language("baisse le volume du haut-parleur"), "fr")
+
+
# --------------------------------------------------------------------- temperature clamp (#35)
# A fitted temperature below 1 sharpens logits. The shipped `choice:11+` bucket is 0.1006, which
# turned a 0.24 top probability into 0.99 confidence on 13-option skill routing.