Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
fix(ts): keep English with one accented loanword on the English checkpoint (#350 parity)
divit aggarwal9b93d508e92d
diff --git a/laya-ts/src/lang.ts b/laya-ts/src/lang.tsindex 94fec7b7..a3d1fca3 100644--- a/laya-ts/src/lang.ts+++ b/laya-ts/src/lang.ts@@ -113,6 +113,20 @@ const NON_EN_DIACRITICS = new Set();export const NON_EN_DIACRITIC_RATE = 0.02;+const ENGLISH_RESCUE_DIACRITIC_RATE = 0.06;++// One accented loanword (`café`, `José`) clears the rate above on its own; two English-only function words+// and at most one accented word keep the text English, as in laya/lang.py.+function englishRescuedByWords(wordSet: Set<string>, diacRate: number): boolean {+ if (diacRate >= ENGLISH_RESCUE_DIACRITIC_RATE) return false;+ let englishOnly = 0;+ let accented = 0;+ for (const w of wordSet) {+ if (EN_ONLY_WORDS.has(w)) englishOnly += 1;+ if ([...w].some((ch) => NON_EN_DIACRITICS.has(ch))) accented += 1;+ }+ return englishOnly >= 2 && accented <= 1;+}const SHARED_WORDS: Set<string> = (() => {const counts = new Map<string, number>();@@ -123,6 +137,7 @@ const SHARED_WORDS: Set<string> = (() => {for (const [w, n] of counts) if (n > 1) out.add(w);return out;})();+const EN_ONLY_WORDS = new Set([...STOP["en"]].filter((w) => !SHARED_WORDS.has(w)));// JS `\w` is ASCII-only, so Python's `[^\W\d_]` needs the Unicode classes spelled out (Nl/No are in Python's `\w`).const WORD_RE = /[\p{L}\p{Nl}\p{No}]+/gu;@@ -307,7 +322,7 @@ export function latinProfile(text: string): LatinProfile {lang = bestLg;} else if (bestLg && nonEnglish && best >= Math.max(2, en)) {lang = bestLg;- } else if (en && !nonEnglish) {+ } else if (en && (!nonEnglish || englishRescuedByWords(wordSet, diacRate))) {lang = "en";}return { language: lang, englishHits: en, diacriticRate: diacRate, looksNonEnglish: nonEnglish };diff --git a/laya-ts/tests/lang.test.ts b/laya-ts/tests/lang.test.tsindex 342eabab..bbf648c6 100644--- a/laya-ts/tests/lang.test.ts+++ b/laya-ts/tests/lang.test.ts@@ -110,4 +110,26 @@ describe("lang", () => {expect(buried.isEnglish).toBe(false);expect(buried.language).toBe("de");});++ it.each([+ "Please send me the café menu today please",+ "Could you email me your résumé before the meeting",+ "Send the invoice to José before Friday",+ "We visited Zürich last summer and loved it",+ ])("keeps English with one accented loanword English: %s", (text) => {+ expect(guessLatinLanguage(text)).toBe("en");+ expect(isEnglish(text)).toBe(true);+ });++ it.each<[string, string | null]>([+ ["Grüße aus Köln, wir melden uns wegen der Rechnung", "de"],+ ["sluk lyset i soveværelset", null],+ ["kan jeg få en refundering for det dobbelte beløb", null],+ ["stäng av ljuset i sovrummet", null],+ ["jag vill ha en återbetalning för den dubbla avgiften", null],+ ["The naïve façade needs a fresh coat of paint", null],+ ])("does not rescue accented non-English text: %s", (text, language) => {+ expect(guessLatinLanguage(text)).toBe(language);+ expect(isEnglish(text)).toBe(false);+ });});diff --git a/laya-ts/tests/router.test.ts b/laya-ts/tests/router.test.tsindex 3ea49d6f..54e9593d 100644--- a/laya-ts/tests/router.test.ts+++ b/laya-ts/tests/router.test.ts@@ -8,6 +8,14 @@ describe("router", () => {expect(r.route({ body: "मुझसे दो बार शुल्क लिया गया" }, {}).model).toBe("multilingual");expect(r.loaded).toEqual([]);});+ it.each([+ ["Please send me the café menu today please", "english"],+ ["Send the invoice to José before Friday", "english"],+ ["Grüße aus Köln, wir melden uns wegen der Rechnung", "multilingual"],+ ["sluk lyset i soveværelset", "multilingual"],+ ])("routes %s to %s", (text, model) => {+ expect(new Router().route(text, {}).model).toBe(model);+ });it("explicit model wins", () => {expect(new Router().route("hi", {}, { model: "typed-decisions" }).model).toBe("typed-decisions");});