Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
fix(ts): route Banglish/Azerbaijani and mixed-script tickets like Python (#307)
Aashish Mahatoefbac1f1fbc0
diff --git a/laya-ts/src/lang.ts b/laya-ts/src/lang.tsindex 64744f35..03d38ea9 100644--- a/laya-ts/src/lang.ts+++ b/laya-ts/src/lang.ts@@ -74,6 +74,24 @@ const STOP: Record<string, Set<string>> = {ro: new Set(["și", "să", "este", "sunt", "care", "pentru", "din", "dar", "după", "până", "fără","ale", "lui", "în", "fost", "acum", "vreau", "trebuie", "foarte", "acest", "această","acesta", "aceasta", "mi", "ți", "vă", "nu"]),+ // Romanized Bangla ("Banglish"): how Bangla is typed in chats, tickets and email when no Bengali+ // keyboard is at hand. It has no diacritics, so without a list it read as undecided-but-English+ // and went to the English checkpoint, which scores 0.08 on Bangla at 0.94 confidence.+ bn: new Set(["ami", "amar", "amake", "amra", "amader", "apni", "apnar", "apnake", "apnara",+ "tumi", "tomar", "tomake", "tomra", "tader", "ota", "eita", "oita",+ "ekta", "ei", "oi", "ki", "keno", "kivabe", "kibhabe", "kothay", "kokhon", "kobe",+ "koto", "kintu", "jodi", "tahole", "ar", "theke", "jonno", "sathe", "shathe", "diye",+ "niye", "moddhe", "kore", "korte", "korchi", "korsi", "korbo", "korechi", "koreche",+ "korun", "koren", "korlam", "hobe", "hoyeche", "hoise", "hocche", "hoyni",+ "chai", "chaina", "lagbe", "parchi", "parbo", "parchina", "peyechi", "paini",+ "dite", "dilam", "diyechi", "nai", "khub", "onek", "ekhon", "akhon", "ekhono",+ "abar", "ekbar", "duibar", "ajke", "kalke", "taka", "bhalo", "valo", "kharap",+ "shomossa", "somossa", "dhonnobad", "bhai", "shob", "keu", "kichu", "bolte", "bolun",+ "parben", "asbe", "jabe", "pabo", "ferot", "dorkar", "hoye", "geche", "gese"]),+ az: new Set(["və", "ve", "bir", "bu", "üçün", "ucun", "ilə", "ile", "olan", "olub", "olmasa",+ "var", "yox", "yoxdur", "mən", "sən", "biz", "siz", "onlar", "daha", "çox", "cox",+ "hər", "nə", "kimi", "görə", "sonra", "əgər", "eger", "deyil", "lakin", "amma",+ "ancaq", "artıq", "artiq", "də", "isə", "həm", "yalnız", "yalniz"]),};const NON_EN_DIACRITICS = new Set(@@ -103,8 +121,22 @@ const WORD_RE = /[^\W\d_]+/gu;const IDENTIFIER_RE = /[\p{L}\p{N}_-]*(?:[.@][\p{L}\p{N}_-]+)+/gu;const IS_ALPHA_RE = /\p{L}/u;+// Python uses two different boundaries on purpose: detect_script/script_profile count the IPA+// Extensions block (0x0250-0x02AF) as Latin, while _script_of (used for non-Latin word runs)+// stops at 0x0250 so a pronunciation like [vlɐˈdʲimʲɪr] is neither Latin nor a script run.function isLatinCp(cp: number): boolean {- return cp < 0x0250 || (0x1e00 <= cp && cp <= 0x1eff) || (0xff21 <= cp && cp <= 0xff3a) || (0xff41 <= cp && cp <= 0xff5a);+ return cp < 0x02b0 || (0x1e00 <= cp && cp <= 0x1eff) || (0xff21 <= cp && cp <= 0xff3a) || (0xff41 <= cp && cp <= 0xff5a);+}++function scriptOf(ch: string): string | null {+ const cp = ch.codePointAt(0)!;+ if (cp < 0x0250 || (0x1e00 <= cp && cp <= 0x1eff) || (0xff21 <= cp && cp <= 0xff3a) || (0xff41 <= cp && cp <= 0xff5a)) {+ return null;+ }+ for (const [name, ranges] of SCRIPT_RANGES) {+ if (ranges.some(([lo, hi]) => lo <= cp && cp <= hi)) return name;+ }+ return null;}export function stateText(state: unknown, maxChars = 4000): string {@@ -167,6 +199,44 @@ export function scriptProfile(text: string): Record<string, number> {return out;}+// Non-Latin text is not for the English checkpoint even when Latin letters are the plurality: a+// brand name or order code outvotes the CJK request around it letter for letter, though one CJK+// character carries far more than a letter. A short message needs a large share to count; a long+// payload (ticket fields, English agent turns) dilutes the share, so there a sentence's worth of+// letters counts too.+const NON_LATIN_FRACTION = 0.2;+const NON_LATIN_MIN_FRACTION = 0.1;+const NON_LATIN_MIN_LETTERS = 10;++/** Non-Latin runs that read as words rather than as annotation inside English prose. */+function nonLatinWords(text: string): string[] {+ // English prose carries three kinds of non-Latin letters that are not a request written in+ // another script, and each is excluded here: a symbol ("Set α to 0.05", one letter), a proper+ // name (capitalised), and a pronunciation ([vlɐˈdʲimʲɪr], which no script range claims). A+ // combining mark belongs to the letter before it and never splits a word.+ const runs: string[] = [];+ let cur = "";+ let script: string | null = null;+ for (const ch of text) {+ if (/^\p{M}$/u.test(ch)) continue;+ const s = scriptOf(ch);+ if (s !== null && s === script) {+ cur += ch;+ continue;+ }+ if (cur) runs.push(cur);+ if (s !== null) {+ cur = ch;+ script = s;+ } else {+ cur = "";+ script = null;+ }+ }+ if (cur) runs.push(cur);+ return runs.filter((w) => [...w].length >= 2 && !/^[\p{Lu}\p{Lt}]/u.test(w));+}+export interface LatinProfile {language: string | null;englishHits: number;@@ -236,8 +306,26 @@ function round4(x: number): number {export function analyse(state: unknown): AnalyseResult {const text = stateText(state);const prof = scriptProfile(text);- const script = detectScript(text);+ let script = detectScript(text);const nonLatin = prof && Object.keys(prof).length ? round4(1.0 - (prof["latin"] ?? 0.0)) : 0.0;+ let nAlpha = 0;+ for (const ch of text) if (IS_ALPHA_RE.test(ch)) nAlpha += 1;+ const nNonLatin = Math.round(nonLatin * nAlpha);+ if (script === "latin" && nonLatinWords(text).length > 0 &&+ (nonLatin >= NON_LATIN_FRACTION ||+ (nonLatin >= NON_LATIN_MIN_FRACTION && nNonLatin >= NON_LATIN_MIN_LETTERS))) {+ // The plurality said Latin, but the non-Latin share is a real message, not a stray name or+ // symbol: re-classify to the dominant non-Latin script, as Python's analyse does.+ let bestScript = script;+ let bestShare = -1;+ for (const [s, share] of Object.entries(prof)) {+ if (s !== "latin" && share > bestShare) {+ bestShare = share;+ bestScript = s;+ }+ }+ script = bestScript;+ }if (script === "unknown") {return {script: "unknown", scriptProfile: prof, language: null,diff --git a/laya-ts/tests/lang.test.ts b/laya-ts/tests/lang.test.tsindex c15f0a7f..9cfce581 100644--- a/laya-ts/tests/lang.test.ts+++ b/laya-ts/tests/lang.test.ts@@ -14,4 +14,29 @@ describe("lang", () => {it("unknown (no letters) is english + undecided", () => {expect(analyse("123 !!!").script).toBe("unknown");});+ // Every expectation below was generated with Python's laya.lang.analyse on the same input.+ it("names romanized Bangla instead of falling back to english", () => {+ const a = analyse("ami ekta ticket khulsi, kalke theke payment hocche na, ekhon ki korte parbo");+ expect(a.script).toBe("latin");+ expect(a.language).toBe("bn");+ expect(a.isEnglish).toBe(false);+ });+ it("names Azerbaijani Latin text", () => {+ const a = analyse("ödənişim iki dəfə tutulub, amma heç bir təsdiq almadım, nə etməliyəm");+ expect(a.language).toBe("az");+ expect(a.isEnglish).toBe(false);+ });+ it("a CJK sentence inside an English ticket is not english", () => {+ const a = analyse("please check the attached logs 請重啟服務器然後再試一次 and tell me what failed");+ expect(a.script).toBe("han");+ expect(a.isEnglish).toBe(false);+ });+ it("a tiny non-latin note does not flip an english ticket", () => {+ const a = analyse("please refund my order, see note 重啟 attached to the ticket");+ expect(a.script).toBe("latin");+ expect(a.isEnglish).toBe(true);+ });+ it("counts IPA extensions as latin, like Python", () => {+ expect(detectScript("ɑ ɒ ɛ ɔ ɪ ʊ æ ʃ θ ð")).toBe("latin");+ });});