Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

fix(ts): keep English with one accented loanword on the English checkpoint (#350 parity)

divit aggarwal9b93d508e92d
diff --git a/laya-ts/src/lang.ts b/laya-ts/src/lang.ts
index 94fec7b7..a3d1fca3 100644
--- a/laya-ts/src/lang.ts
+++ b/laya-ts/src/lang.ts
@@ -113,6 +113,20 @@ const NON_EN_DIACRITICS = new Set(
);
export const NON_EN_DIACRITIC_RATE = 0.02;
+const ENGLISH_RESCUE_DIACRITIC_RATE = 0.06;
+
+// One accented loanword (`café`, `José`) clears the rate above on its own; two English-only function words
+// and at most one accented word keep the text English, as in laya/lang.py.
+function englishRescuedByWords(wordSet: Set<string>, diacRate: number): boolean {
+ if (diacRate >= ENGLISH_RESCUE_DIACRITIC_RATE) return false;
+ let englishOnly = 0;
+ let accented = 0;
+ for (const w of wordSet) {
+ if (EN_ONLY_WORDS.has(w)) englishOnly += 1;
+ if ([...w].some((ch) => NON_EN_DIACRITICS.has(ch))) accented += 1;
+ }
+ return englishOnly >= 2 && accented <= 1;
+}
const SHARED_WORDS: Set<string> = (() => {
const counts = new Map<string, number>();
@@ -123,6 +137,7 @@ const SHARED_WORDS: Set<string> = (() => {
for (const [w, n] of counts) if (n > 1) out.add(w);
return out;
})();
+const EN_ONLY_WORDS = new Set([...STOP["en"]].filter((w) => !SHARED_WORDS.has(w)));
// JS `\w` is ASCII-only, so Python's `[^\W\d_]` needs the Unicode classes spelled out (Nl/No are in Python's `\w`).
const WORD_RE = /[\p{L}\p{Nl}\p{No}]+/gu;
@@ -307,7 +322,7 @@ export function latinProfile(text: string): LatinProfile {
lang = bestLg;
} else if (bestLg && nonEnglish && best >= Math.max(2, en)) {
lang = bestLg;
- } else if (en && !nonEnglish) {
+ } else if (en && (!nonEnglish || englishRescuedByWords(wordSet, diacRate))) {
lang = "en";
}
return { language: lang, englishHits: en, diacriticRate: diacRate, looksNonEnglish: nonEnglish };
diff --git a/laya-ts/tests/lang.test.ts b/laya-ts/tests/lang.test.ts
index 342eabab..bbf648c6 100644
--- a/laya-ts/tests/lang.test.ts
+++ b/laya-ts/tests/lang.test.ts
@@ -110,4 +110,26 @@ describe("lang", () => {
expect(buried.isEnglish).toBe(false);
expect(buried.language).toBe("de");
});
+
+ it.each([
+ "Please send me the café menu today please",
+ "Could you email me your résumé before the meeting",
+ "Send the invoice to José before Friday",
+ "We visited Zürich last summer and loved it",
+ ])("keeps English with one accented loanword English: %s", (text) => {
+ expect(guessLatinLanguage(text)).toBe("en");
+ expect(isEnglish(text)).toBe(true);
+ });
+
+ it.each<[string, string | null]>([
+ ["Grüße aus Köln, wir melden uns wegen der Rechnung", "de"],
+ ["sluk lyset i soveværelset", null],
+ ["kan jeg få en refundering for det dobbelte beløb", null],
+ ["stäng av ljuset i sovrummet", null],
+ ["jag vill ha en återbetalning för den dubbla avgiften", null],
+ ["The naïve façade needs a fresh coat of paint", null],
+ ])("does not rescue accented non-English text: %s", (text, language) => {
+ expect(guessLatinLanguage(text)).toBe(language);
+ expect(isEnglish(text)).toBe(false);
+ });
});
diff --git a/laya-ts/tests/router.test.ts b/laya-ts/tests/router.test.ts
index 3ea49d6f..54e9593d 100644
--- a/laya-ts/tests/router.test.ts
+++ b/laya-ts/tests/router.test.ts
@@ -8,6 +8,14 @@ describe("router", () => {
expect(r.route({ body: "मुझसे दो बार शुल्क लिया गया" }, {}).model).toBe("multilingual");
expect(r.loaded).toEqual([]);
});
+ it.each([
+ ["Please send me the café menu today please", "english"],
+ ["Send the invoice to José before Friday", "english"],
+ ["Grüße aus Köln, wir melden uns wegen der Rechnung", "multilingual"],
+ ["sluk lyset i soveværelset", "multilingual"],
+ ])("routes %s to %s", (text, model) => {
+ expect(new Router().route(text, {}).model).toBe(model);
+ });
it("explicit model wins", () => {
expect(new Router().route("hi", {}, { model: "typed-decisions" }).model).toBe("typed-decisions");
});