Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
fix(lang): count question words and auxiliaries as English evidence
Rishit Sharma047f18697739
diff --git a/laya/lang.py b/laya/lang.pyindex 28a76aa3..66c0b48d 100644--- a/laya/lang.py+++ b/laya/lang.py@@ -57,7 +57,26 @@ _SCRIPT_RANGES = [_STOP = {"en": {"the", "and", "is", "are", "was", "were", "to", "of", "in", "for", "with", "that","this", "it", "you", "have", "has", "not", "but", "on", "at", "be", "as", "from",- "will", "can", "would", "there", "their", "what", "which", "please", "we", "i"},+ "will", "can", "would", "there", "their", "what", "which", "please", "we", "i",+ # Question words, auxiliaries, pronouns and connectives. Without them a plain request+ # such as "Why hasn't my card been delivered?" matched no word above, stayed undecided+ # and took the router default, the multilingual checkpoint. Each one, added alone,+ # moved none of the 101,650 non-English MASSIVE validation rows to the English+ # checkpoint. Words that did stay out: `over` (Dutch, Danish, Norwegian), `just` and+ # `under` (Swedish), `still` (Norwegian), `he` (Spanish, Afrikaans, Finnish), `had`+ # (Dutch), `or` and `cant` (Welsh), `us` (`us dollar`), and `about`, `up`, `out`+ # (English titles and loanwords inside other languages). `also` is German, so it+ # stays out too.+ # `doesn`, `isn` and the rest are what `_WORD` leaves of a contraction; `doesnt` is how+ # it is typed without the apostrophe.+ "how", "why", "when", "where", "who", "whom", "does", "done", "doing", "been", "being",+ "could", "should", "might", "must", "shall", "doesn", "didn", "isn", "aren", "wasn",+ "weren", "haven", "hasn", "couldn", "wouldn", "won", "dont", "wont", "didnt", "doesnt",+ "isnt", "wasnt", "havent", "ive", "youre", "thats", "whats", "any", "some", "many",+ "much", "every", "each", "these", "those", "both", "few", "they", "them", "she", "him",+ "his", "our", "your", "yours", "into", "onto", "between", "after", "before", "through",+ "during", "without", "within", "down", "if", "nor", "than", "then", "because", "while",+ "until", "here", "only", "very", "too", "ever", "never", "always", "already", "yet"},"fr": {"le", "la", "les", "des", "une", "est", "pour", "dans", "que", "qui", "avec", "sur","pas", "plus", "nous", "vous", "être", "cette", "mais", "sont", "ont", "aux", "ce","et", "du", "au", "ou", "je", "tu", "il", "elle", "ils", "elles", "mon", "ton",diff --git a/tests/test_router.py b/tests/test_router.pyindex c78ea233..a81709e1 100644--- a/tests/test_router.py+++ b/tests/test_router.py@@ -103,6 +103,13 @@ for label, text in [("english", "Please refund the duplicate charge"), ("hindi",check("analyse/keys " + label, set(analyse(text)), _KEYS)# A 0-0 tie between non-English stopword lists is no evidence for any of themcheck("latin_lang/zero tie invents nothing", guess_latin_language("Cât e ora acum la Tokyo"), None)+# Plain English made of question words, auxiliaries and contractions used to match no English+# function word at all, so it stayed undecided and took the router default (multilingual).+for text in ["Why hasn't my card been delivered?",+ "How long does a card delivery take?",+ "Where should my refund show up?"]:+ check("latin_lang/english question words " + text, guess_latin_language(text), "en")+ check("route/english question words " + text, Router().route(text).model, "english")# Known limitation, kept visible on purpose: Romanian short enough to carry no diacritics and an# English function word ("in") still reads as English. A real LID model is the fix, not more@@ -716,6 +723,9 @@ for text in [from laya.lang import _STOP # noqa: E402check("latin_lang/bn list shares no word with another list",sorted(w for w in _STOP.get("bn", ()) for lg, words in _STOP.items() if lg != "bn" and w in words), [])+check("latin_lang/en list shares only in, is, as, was with another list",+ sorted({w for w in _STOP["en"] for lg, words in _STOP.items() if lg != "en" and w in words}),+ ["as", "in", "is", "was"])check("latin_lang/romanian with ei stays romanian",guess_latin_language("Ei nu sunt de acord cu factura, vreau o corecție"), "ro")