Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

fix(lang): count question words and auxiliaries as English evidence

Rishit Sharma047f18697739
diff --git a/laya/lang.py b/laya/lang.py
index 28a76aa3..66c0b48d 100644
--- a/laya/lang.py
+++ b/laya/lang.py
@@ -57,7 +57,26 @@ _SCRIPT_RANGES = [
_STOP = {
"en": {"the", "and", "is", "are", "was", "were", "to", "of", "in", "for", "with", "that",
"this", "it", "you", "have", "has", "not", "but", "on", "at", "be", "as", "from",
- "will", "can", "would", "there", "their", "what", "which", "please", "we", "i"},
+ "will", "can", "would", "there", "their", "what", "which", "please", "we", "i",
+ # Question words, auxiliaries, pronouns and connectives. Without them a plain request
+ # such as "Why hasn't my card been delivered?" matched no word above, stayed undecided
+ # and took the router default, the multilingual checkpoint. Each one, added alone,
+ # moved none of the 101,650 non-English MASSIVE validation rows to the English
+ # checkpoint. Words that did stay out: `over` (Dutch, Danish, Norwegian), `just` and
+ # `under` (Swedish), `still` (Norwegian), `he` (Spanish, Afrikaans, Finnish), `had`
+ # (Dutch), `or` and `cant` (Welsh), `us` (`us dollar`), and `about`, `up`, `out`
+ # (English titles and loanwords inside other languages). `also` is German, so it
+ # stays out too.
+ # `doesn`, `isn` and the rest are what `_WORD` leaves of a contraction; `doesnt` is how
+ # it is typed without the apostrophe.
+ "how", "why", "when", "where", "who", "whom", "does", "done", "doing", "been", "being",
+ "could", "should", "might", "must", "shall", "doesn", "didn", "isn", "aren", "wasn",
+ "weren", "haven", "hasn", "couldn", "wouldn", "won", "dont", "wont", "didnt", "doesnt",
+ "isnt", "wasnt", "havent", "ive", "youre", "thats", "whats", "any", "some", "many",
+ "much", "every", "each", "these", "those", "both", "few", "they", "them", "she", "him",
+ "his", "our", "your", "yours", "into", "onto", "between", "after", "before", "through",
+ "during", "without", "within", "down", "if", "nor", "than", "then", "because", "while",
+ "until", "here", "only", "very", "too", "ever", "never", "always", "already", "yet"},
"fr": {"le", "la", "les", "des", "une", "est", "pour", "dans", "que", "qui", "avec", "sur",
"pas", "plus", "nous", "vous", "être", "cette", "mais", "sont", "ont", "aux", "ce",
"et", "du", "au", "ou", "je", "tu", "il", "elle", "ils", "elles", "mon", "ton",
diff --git a/tests/test_router.py b/tests/test_router.py
index c78ea233..a81709e1 100644
--- a/tests/test_router.py
+++ b/tests/test_router.py
@@ -103,6 +103,13 @@ for label, text in [("english", "Please refund the duplicate charge"), ("hindi",
check("analyse/keys " + label, set(analyse(text)), _KEYS)
# A 0-0 tie between non-English stopword lists is no evidence for any of them
check("latin_lang/zero tie invents nothing", guess_latin_language("Cât e ora acum la Tokyo"), None)
+# Plain English made of question words, auxiliaries and contractions used to match no English
+# function word at all, so it stayed undecided and took the router default (multilingual).
+for text in ["Why hasn't my card been delivered?",
+ "How long does a card delivery take?",
+ "Where should my refund show up?"]:
+ check("latin_lang/english question words " + text, guess_latin_language(text), "en")
+ check("route/english question words " + text, Router().route(text).model, "english")
# Known limitation, kept visible on purpose: Romanian short enough to carry no diacritics and an
# English function word ("in") still reads as English. A real LID model is the fix, not more
@@ -716,6 +723,9 @@ for text in [
from laya.lang import _STOP # noqa: E402
check("latin_lang/bn list shares no word with another list",
sorted(w for w in _STOP.get("bn", ()) for lg, words in _STOP.items() if lg != "bn" and w in words), [])
+check("latin_lang/en list shares only in, is, as, was with another list",
+ sorted({w for w in _STOP["en"] for lg, words in _STOP.items() if lg != "en" and w in words}),
+ ["as", "in", "is", "was"])
check("latin_lang/romanian with ei stays romanian",
guess_latin_language("Ei nu sunt de acord cu factura, vreau o corecție"), "ro")