Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

fix(ts): count the same English function words as Python

Rishit Sharma3a865089f884
diff --git a/laya-ts/src/lang.ts b/laya-ts/src/lang.ts
index a2c3e421..13e89913 100644
--- a/laya-ts/src/lang.ts
+++ b/laya-ts/src/lang.ts
@@ -31,7 +31,18 @@ const SCRIPT_RANGES: ScriptRanges = [
const STOP: Record<string, Set<string>> = {
en: new Set(["the", "and", "is", "are", "was", "were", "to", "of", "in", "for", "with", "that",
"this", "it", "you", "have", "has", "not", "but", "on", "at", "be", "as", "from",
- "will", "can", "would", "there", "their", "what", "which", "please", "we", "i"]),
+ "will", "can", "would", "there", "their", "what", "which", "please", "we", "i",
+ // Question words, auxiliaries, pronouns and connectives, so a plain request such as "Why hasn't
+ // my card been delivered?" is not left undecided. See _STOP["en"] in laya/lang.py for how they
+ // were chosen and which words stay out.
+ "how", "why", "when", "where", "who", "whom", "does", "done", "doing", "been", "being",
+ "could", "should", "might", "must", "shall", "doesn", "didn", "isn", "aren", "wasn",
+ "weren", "haven", "hasn", "couldn", "wouldn", "won", "dont", "wont", "didnt", "doesnt",
+ "isnt", "wasnt", "havent", "ive", "youre", "thats", "whats", "any", "some", "many",
+ "much", "every", "each", "these", "those", "both", "few", "they", "them", "she", "him",
+ "his", "our", "your", "yours", "into", "onto", "between", "after", "before", "through",
+ "during", "without", "within", "down", "if", "nor", "than", "then", "because", "while",
+ "until", "here", "only", "very", "too", "ever", "never", "always", "already", "yet"]),
fr: new Set(["le", "la", "les", "des", "une", "est", "pour", "dans", "que", "qui", "avec", "sur",
"pas", "plus", "nous", "vous", "être", "cette", "mais", "sont", "ont", "aux", "ce",
"et", "du", "au", "ou", "je", "tu", "il", "elle", "ils", "elles", "mon", "ton",
diff --git a/laya-ts/tests/lang.test.ts b/laya-ts/tests/lang.test.ts
index abfd66ce..fd567e0f 100644
--- a/laya-ts/tests/lang.test.ts
+++ b/laya-ts/tests/lang.test.ts
@@ -79,6 +79,13 @@ describe("lang", () => {
expect(analyse("What was the reason for the delay").language).toBe("en");
expect(analyse("Where can I find my invoice in the app").isEnglish).toBe(true);
});
+ it("names English made of question words and contractions", () => {
+ for (const text of ["Why hasn't my card been delivered?", "How long does a card delivery take?",
+ "Where should my refund show up?"]) {
+ expect(guessLatinLanguage(text)).toBe("en");
+ expect(analyse(text).languageUndecided).toBe(false);
+ }
+ });
// `es` and `du` are German function words, but Spanish and French claim them, and a word two
// lists share names neither language. They stay out of the German list.
it("does not pull es or du into German", () => {