Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
fix(ts): count an English/foreign collision word once, like Python
sameedkhan175c9fa1ab3ee2
diff --git a/laya-ts/src/lang.ts b/laya-ts/src/lang.tsindex 7da5e962..a2c3e421 100644--- a/laya-ts/src/lang.ts+++ b/laya-ts/src/lang.ts@@ -151,6 +151,14 @@ const SHARED_WORDS: Set<string> = (() => {const NORDIC_OVERLAP_WORDS = new Set(["hej", "ja", "nej", "jo", "tack", "mig", "min", "om", "kommer", "får", "skulle", "vi"]);for (const word of NORDIC_OVERLAP_WORDS) SHARED_WORDS.add(word);const EN_ONLY_WORDS = new Set([...STOP["en"]].filter((w) => !SHARED_WORDS.has(w)));+// Foreign function words that are also ordinary English words (`im` is also `I'm` without the+// apostrophe). Each counts once however often it repeats, so "do more, do less" cannot clear the+// `best >= 2` bar on one word. Every other word keeps counting occurrences: `der` twice is still+// German, and Dutch `van` is deliberately left out. See _EN_COLLISION_WORDS in laya/lang.py.+const EN_COLLISION_WORDS = new Set(+ ["come", "son", "do", "care", "todo", "im", "per", "plus"].filter((w) =>+ Object.entries(STOP).some(([lg, sw]) => lg !== "en" && sw.has(w))),+);// Short support fragments need a narrower vocabulary than the general four-word language guess.// Generic words such as "fel" and "hjälp" are deliberately omitted from this subset.@@ -327,10 +335,13 @@ export function latinProfile(text: string): LatinProfile {if (words.length < 4) {return { language: null, englishHits: 0, diacriticRate: diacRate, looksNonEnglish: nonEnglish || nordicOverlap };}+ // A collision word counts once however often it repeats; every other word counts its hits.+ const counts = new Map<string, number>();+ for (const w of words) counts.set(w, (counts.get(w) ?? 0) + 1);const scores: Record<string, number> = {};for (const [lg, sw] of Object.entries(STOP)) {let s = 0;- for (const w of words) if (sw.has(w)) s += 1;+ for (const [w, n] of counts) if (sw.has(w)) s += EN_COLLISION_WORDS.has(w) ? 1 : n;scores[lg] = s;}const en = scores["en"] ?? 0;diff --git a/laya-ts/tests/lang.test.ts b/laya-ts/tests/lang.test.tsindex 5b765bb5..abfd66ce 100644--- a/laya-ts/tests/lang.test.ts+++ b/laya-ts/tests/lang.test.ts@@ -214,4 +214,39 @@ describe("lang", () => {expect(guessLatinLanguage(text)).toBe(language);expect(isEnglish(text)).toBe(false);});++ // A foreign function word that is also an ordinary English word (`come` it, `son` es, `do` pt,+ // `care` ro, `todo`/`im`/`per`/`plus`) named that language when it merely appeared twice, so+ // plain English routed to the multilingual checkpoint. Such a word now counts once however often+ // it repeats. Ported from tests/test_lang_guess.py; expectations from Python's laya.lang.analyse.+ it.each([+ "Come one, come all",+ "My son, your son",+ "Do more, do less",+ "Care more, care less",+ "Add a todo, then another todo item",+ "im not able to log in, im stuck",+ "add 45 to 87 plus 54 plus 43 plus 22",+ ])("plain english with a repeated collision word stays english: %s", (text) => {+ expect(analyse(text).isEnglish).toBe(true);+ });++ // The dedupe is limited to those words: `der`, `des`, `di`, `sa` are nobody's English and still+ // count every occurrence, Dutch `van` is deliberately left off the list, and genuine requests+ // with several different function words are still foreign.+ it.each([+ "reduzieren der helligkeit der lichter",+ "enumerar todos los horarios de los tren a nueva york",+ "jouer des chansons des beatles",+ "numero di telefono di giacomo",+ "pede um pacote de massa chinesa faz um pedido takeaway",+ "ar trebui sa port o pelerina de ploaie inainte sa ies afara",+ "herinner me eraan dat ik dat liedje leuk vind",+ "hey olly ik hou van muziek van frans bauer",+ "Hola, necesito cancelar mi pedido por favor ahora mismo gracias",+ "Eu quero cancelar o meu pedido por favor agora mesmo obrigado",+ "Vorrei annullare il mio ordine per favore adesso grazie mille",+ ])("a repeated non-collision function word is still foreign: %s", (text) => {+ expect(analyse(text).isEnglish).toBe(false);+ });});