Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
fix(ts): count the same English function words as Python
Rishit Sharma3a865089f884
diff --git a/laya-ts/src/lang.ts b/laya-ts/src/lang.tsindex a2c3e421..13e89913 100644--- a/laya-ts/src/lang.ts+++ b/laya-ts/src/lang.ts@@ -31,7 +31,18 @@ const SCRIPT_RANGES: ScriptRanges = [const STOP: Record<string, Set<string>> = {en: new Set(["the", "and", "is", "are", "was", "were", "to", "of", "in", "for", "with", "that","this", "it", "you", "have", "has", "not", "but", "on", "at", "be", "as", "from",- "will", "can", "would", "there", "their", "what", "which", "please", "we", "i"]),+ "will", "can", "would", "there", "their", "what", "which", "please", "we", "i",+ // Question words, auxiliaries, pronouns and connectives, so a plain request such as "Why hasn't+ // my card been delivered?" is not left undecided. See _STOP["en"] in laya/lang.py for how they+ // were chosen and which words stay out.+ "how", "why", "when", "where", "who", "whom", "does", "done", "doing", "been", "being",+ "could", "should", "might", "must", "shall", "doesn", "didn", "isn", "aren", "wasn",+ "weren", "haven", "hasn", "couldn", "wouldn", "won", "dont", "wont", "didnt", "doesnt",+ "isnt", "wasnt", "havent", "ive", "youre", "thats", "whats", "any", "some", "many",+ "much", "every", "each", "these", "those", "both", "few", "they", "them", "she", "him",+ "his", "our", "your", "yours", "into", "onto", "between", "after", "before", "through",+ "during", "without", "within", "down", "if", "nor", "than", "then", "because", "while",+ "until", "here", "only", "very", "too", "ever", "never", "always", "already", "yet"]),fr: new Set(["le", "la", "les", "des", "une", "est", "pour", "dans", "que", "qui", "avec", "sur","pas", "plus", "nous", "vous", "être", "cette", "mais", "sont", "ont", "aux", "ce","et", "du", "au", "ou", "je", "tu", "il", "elle", "ils", "elles", "mon", "ton",diff --git a/laya-ts/tests/lang.test.ts b/laya-ts/tests/lang.test.tsindex abfd66ce..fd567e0f 100644--- a/laya-ts/tests/lang.test.ts+++ b/laya-ts/tests/lang.test.ts@@ -79,6 +79,13 @@ describe("lang", () => {expect(analyse("What was the reason for the delay").language).toBe("en");expect(analyse("Where can I find my invoice in the app").isEnglish).toBe(true);});+ it("names English made of question words and contractions", () => {+ for (const text of ["Why hasn't my card been delivered?", "How long does a card delivery take?",+ "Where should my refund show up?"]) {+ expect(guessLatinLanguage(text)).toBe("en");+ expect(analyse(text).languageUndecided).toBe(false);+ }+ });// `es` and `du` are German function words, but Spanish and French claim them, and a word two// lists share names neither language. They stay out of the German list.it("does not pull es or du into German", () => {