Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

fix(ts): count an English/foreign collision word once, like Python

sameedkhan175c9fa1ab3ee2
diff --git a/laya-ts/src/lang.ts b/laya-ts/src/lang.ts
index 7da5e962..a2c3e421 100644
--- a/laya-ts/src/lang.ts
+++ b/laya-ts/src/lang.ts
@@ -151,6 +151,14 @@ const SHARED_WORDS: Set<string> = (() => {
const NORDIC_OVERLAP_WORDS = new Set(["hej", "ja", "nej", "jo", "tack", "mig", "min", "om", "kommer", "får", "skulle", "vi"]);
for (const word of NORDIC_OVERLAP_WORDS) SHARED_WORDS.add(word);
const EN_ONLY_WORDS = new Set([...STOP["en"]].filter((w) => !SHARED_WORDS.has(w)));
+// Foreign function words that are also ordinary English words (`im` is also `I'm` without the
+// apostrophe). Each counts once however often it repeats, so "do more, do less" cannot clear the
+// `best >= 2` bar on one word. Every other word keeps counting occurrences: `der` twice is still
+// German, and Dutch `van` is deliberately left out. See _EN_COLLISION_WORDS in laya/lang.py.
+const EN_COLLISION_WORDS = new Set(
+ ["come", "son", "do", "care", "todo", "im", "per", "plus"].filter((w) =>
+ Object.entries(STOP).some(([lg, sw]) => lg !== "en" && sw.has(w))),
+);
// Short support fragments need a narrower vocabulary than the general four-word language guess.
// Generic words such as "fel" and "hjälp" are deliberately omitted from this subset.
@@ -327,10 +335,13 @@ export function latinProfile(text: string): LatinProfile {
if (words.length < 4) {
return { language: null, englishHits: 0, diacriticRate: diacRate, looksNonEnglish: nonEnglish || nordicOverlap };
}
+ // A collision word counts once however often it repeats; every other word counts its hits.
+ const counts = new Map<string, number>();
+ for (const w of words) counts.set(w, (counts.get(w) ?? 0) + 1);
const scores: Record<string, number> = {};
for (const [lg, sw] of Object.entries(STOP)) {
let s = 0;
- for (const w of words) if (sw.has(w)) s += 1;
+ for (const [w, n] of counts) if (sw.has(w)) s += EN_COLLISION_WORDS.has(w) ? 1 : n;
scores[lg] = s;
}
const en = scores["en"] ?? 0;
diff --git a/laya-ts/tests/lang.test.ts b/laya-ts/tests/lang.test.ts
index 5b765bb5..abfd66ce 100644
--- a/laya-ts/tests/lang.test.ts
+++ b/laya-ts/tests/lang.test.ts
@@ -214,4 +214,39 @@ describe("lang", () => {
expect(guessLatinLanguage(text)).toBe(language);
expect(isEnglish(text)).toBe(false);
});
+
+ // A foreign function word that is also an ordinary English word (`come` it, `son` es, `do` pt,
+ // `care` ro, `todo`/`im`/`per`/`plus`) named that language when it merely appeared twice, so
+ // plain English routed to the multilingual checkpoint. Such a word now counts once however often
+ // it repeats. Ported from tests/test_lang_guess.py; expectations from Python's laya.lang.analyse.
+ it.each([
+ "Come one, come all",
+ "My son, your son",
+ "Do more, do less",
+ "Care more, care less",
+ "Add a todo, then another todo item",
+ "im not able to log in, im stuck",
+ "add 45 to 87 plus 54 plus 43 plus 22",
+ ])("plain english with a repeated collision word stays english: %s", (text) => {
+ expect(analyse(text).isEnglish).toBe(true);
+ });
+
+ // The dedupe is limited to those words: `der`, `des`, `di`, `sa` are nobody's English and still
+ // count every occurrence, Dutch `van` is deliberately left off the list, and genuine requests
+ // with several different function words are still foreign.
+ it.each([
+ "reduzieren der helligkeit der lichter",
+ "enumerar todos los horarios de los tren a nueva york",
+ "jouer des chansons des beatles",
+ "numero di telefono di giacomo",
+ "pede um pacote de massa chinesa faz um pedido takeaway",
+ "ar trebui sa port o pelerina de ploaie inainte sa ies afara",
+ "herinner me eraan dat ik dat liedje leuk vind",
+ "hey olly ik hou van muziek van frans bauer",
+ "Hola, necesito cancelar mi pedido por favor ahora mismo gracias",
+ "Eu quero cancelar o meu pedido por favor agora mesmo obrigado",
+ "Vorrei annullare il mio ordine per favore adesso grazie mille",
+ ])("a repeated non-collision function word is still foreign: %s", (text) => {
+ expect(analyse(text).isEnglish).toBe(false);
+ });
});