Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
fix(ts): count the Azerbaijani schwa as a non-English letter
PerryLinkd19f9de7c456
diff --git a/laya-ts/src/lang.ts b/laya-ts/src/lang.tsindex 90f54b65..1c8debfc 100644--- a/laya-ts/src/lang.ts+++ b/laya-ts/src/lang.ts@@ -101,14 +101,15 @@ const STOP: Record<string, Set<string>> = {};const NON_EN_DIACRITICS = new Set(- ("àâäãáåçéèêëíìîïñóòôöõøúùûüýÿßæœ" +- "ăâîșțşţ" +- "ąćęłńśźż" +- "čďěňřšťůž" +- "őű" +- "ğı" +- "āēģīķļņūž" +- "đ").split(""),+ ("àâäãáåçéèêëíìîïñóòôöõøúùûüýÿßæœ" + // Western European+ "ăâîșțşţ" + // Romanian+ "ąćęłńśźż" + // Polish+ "čďěňřšťůž" + // Czech / Slovak+ "őű" + // Hungarian+ "ğı" + // Turkish (text is lowercased before matching)+ "āēģīķļņūž" + // Baltic+ "đ" + // Serbo-Croatian / Vietnamese+ "ə").split(""), // Azerbaijani);export const NON_EN_DIACRITIC_RATE = 0.02;diff --git a/laya-ts/tests/lang.test.ts b/laya-ts/tests/lang.test.tsindex 120018fe..03cc24c9 100644--- a/laya-ts/tests/lang.test.ts+++ b/laya-ts/tests/lang.test.ts@@ -26,6 +26,22 @@ describe("lang", () => {expect(a.language).toBe("az");expect(a.isEnglish).toBe(false);});+ // The Azerbaijani stopword list above has always been in the port, but `ə` was missing from+ // NON_EN_DIACRITICS, so the one letter that identifies the language carried no weight. English+ // text with a single schwa read as English here and as undecided in Python. Expectations from+ // Python's laya.lang.analyse on the same inputs.+ it("counts the Azerbaijani schwa as a non-English letter", () => {+ const a = analyse("və the quick brown");+ expect(a.diacriticRate).toBeCloseTo(0.0556, 3);+ expect(a.isEnglish).toBe(false);+ expect(a.language).toBe(null);+ });+ it("still routes plain english without a schwa to english", () => {+ const a = analyse("the quick brown fox");+ expect(a.diacriticRate).toBe(0);+ expect(a.isEnglish).toBe(true);+ expect(a.language).toBe("en");+ });it("a CJK sentence inside an English ticket is not english", () => {const a = analyse("please check the attached logs 請重啟服務器然後再試一次 and tell me what failed");expect(a.script).toBe("han");