Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
fix(lang): identify plain-ASCII German instead of labelling it English (#130)
Rishit1bee2d0732bc
diff --git a/laya/lang.py b/laya/lang.pyindex 559bb077..0122c787 100644--- a/laya/lang.py+++ b/laya/lang.py@@ -64,7 +64,13 @@ _STOP = {"merci", "bonjour", "jour", "jours", "mois", "fois", "quand", "comment", "pourquoi","alors", "donc"},"de": {"der", "die", "das", "und", "ist", "ein", "eine", "den", "dem", "nicht", "mit", "für",- "auf", "von", "zu", "sich", "auch", "werden", "wurde", "haben", "sind", "oder", "aber"},+ "auf", "von", "zu", "sich", "auch", "werden", "wurde", "haben", "sind", "oder", "aber",+ "ich", "wir", "mir", "mich", "dir", "dich", "uns", "mein", "meine", "meinen",+ "meinem", "meiner", "diese", "dieser", "diesen", "dieses", "einen", "einem", "einer",+ "wie", "wo", "wann", "welche", "im", "zum", "zur", "aus", "bei", "nach", "noch", "bitte",+ "heute", "jetzt", "kann", "kannst", "habe", "gibt", "wird",+ # shared with English on purpose: counted for English alone, they outvoted short German+ "in", "was"},"es": {"el", "los", "las", "que", "por", "con", "para", "una", "es", "se", "del", "como","pero", "son", "está", "este", "esta", "todo", "más", "muy", "hay", "sus",# `de`/`en` are Spanish too, but they are common English tokens as well (`de facto`,diff --git a/tests/test_lang_guess.py b/tests/test_lang_guess.pyindex 75438c16..17c20d69 100644--- a/tests/test_lang_guess.py+++ b/tests/test_lang_guess.py@@ -168,7 +168,7 @@ check("standalone/hint still uses the standalone repo",# `Router().route(...)` cannot fail, because both sides are the same pure call on an# equally configured router -- a regression would move both together.BEFORE = [("plain english", "I was charged twice and want a refund", "english"),- ("German with umlauts", "Mein Konto wurde zweimal belastet, bitte erstatten Sie", "english"),+ ("German with umlauts", "Mein Konto wurde zweimal belastet, bitte erstatten Sie", "multilingual"),("Hindi", "यह एक हिंदी वाक्य है", "multilingual"),("empty", "", "english"),("digits", "12345", "english")]diff --git a/tests/test_router.py b/tests/test_router.pyindex 5c79f5f4..24d65d90 100644--- a/tests/test_router.py+++ b/tests/test_router.py@@ -462,6 +462,27 @@ check("latin_lang/romanian with ei stays romanian",guess_latin_language("Ei nu sunt de acord cu factura, vreau o corecție"), "ro")+# --------------------------------------------------------------------- plain-ASCII German (#54)+# No umlaut for the diacritic rate to catch, and `in`/`was` counted for English alone, so these were+# labelled English and handed to the checkpoint that cannot read them.+for text in ["trage diesen termin in meinen kalender ein",+ "wie lautet die temperatur in fulda in hessen",+ "schalte das licht im wohnzimmer aus",+ "was ist die aktuelle zeit"]:+ check("latin_lang/ascii german " + text, guess_latin_language(text), "de")+ check("route/ascii german " + text, _r_lat.route(text).model, "multilingual")+check("route/english control for #54",+ _r_lat.route("I would like to book a flight to Berlin tomorrow").model, "english")+# English that shares words with the German list stays English. `in` and `den` leave the first, a real+# en-US MASSIVE utterance, one German hit short of flipping; the chat line carries `im` and flips if+# any one of the English words `am`, `an` or `so` joins the German list.+for text in ["turn off smart lamp in den", "im so sorry, am an hour late, stuck in traffic"]:+ check("route/english sharing german words " + text, _r_lat.route(text).model, "english")+# German words that Spanish (`es`) or French (`du`) also claim would stop naming those languages+check("latin_lang/spanish es stays evidence", guess_latin_language("que hora es en australia"), "es")+check("latin_lang/french du stays evidence", guess_latin_language("baisse le volume du haut-parleur"), "fr")++# --------------------------------------------------------------------- temperature clamp (#35)# A fitted temperature below 1 sharpens logits. The shipped `choice:11+` bucket is 0.1006, which# turned a 0.24 top probability into 0.99 confidence on 13-option skill routing.