Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

fix(ts): route Banglish/Azerbaijani and mixed-script tickets like Python (#307)

Aashish Mahatoefbac1f1fbc0
diff --git a/laya-ts/src/lang.ts b/laya-ts/src/lang.ts
index 64744f35..03d38ea9 100644
--- a/laya-ts/src/lang.ts
+++ b/laya-ts/src/lang.ts
@@ -74,6 +74,24 @@ const STOP: Record<string, Set<string>> = {
ro: new Set(["și", "să", "este", "sunt", "care", "pentru", "din", "dar", "după", "până", "fără",
"ale", "lui", "în", "fost", "acum", "vreau", "trebuie", "foarte", "acest", "această",
"acesta", "aceasta", "mi", "ți", "vă", "nu"]),
+ // Romanized Bangla ("Banglish"): how Bangla is typed in chats, tickets and email when no Bengali
+ // keyboard is at hand. It has no diacritics, so without a list it read as undecided-but-English
+ // and went to the English checkpoint, which scores 0.08 on Bangla at 0.94 confidence.
+ bn: new Set(["ami", "amar", "amake", "amra", "amader", "apni", "apnar", "apnake", "apnara",
+ "tumi", "tomar", "tomake", "tomra", "tader", "ota", "eita", "oita",
+ "ekta", "ei", "oi", "ki", "keno", "kivabe", "kibhabe", "kothay", "kokhon", "kobe",
+ "koto", "kintu", "jodi", "tahole", "ar", "theke", "jonno", "sathe", "shathe", "diye",
+ "niye", "moddhe", "kore", "korte", "korchi", "korsi", "korbo", "korechi", "koreche",
+ "korun", "koren", "korlam", "hobe", "hoyeche", "hoise", "hocche", "hoyni",
+ "chai", "chaina", "lagbe", "parchi", "parbo", "parchina", "peyechi", "paini",
+ "dite", "dilam", "diyechi", "nai", "khub", "onek", "ekhon", "akhon", "ekhono",
+ "abar", "ekbar", "duibar", "ajke", "kalke", "taka", "bhalo", "valo", "kharap",
+ "shomossa", "somossa", "dhonnobad", "bhai", "shob", "keu", "kichu", "bolte", "bolun",
+ "parben", "asbe", "jabe", "pabo", "ferot", "dorkar", "hoye", "geche", "gese"]),
+ az: new Set(["və", "ve", "bir", "bu", "üçün", "ucun", "ilə", "ile", "olan", "olub", "olmasa",
+ "var", "yox", "yoxdur", "mən", "sən", "biz", "siz", "onlar", "daha", "çox", "cox",
+ "hər", "nə", "kimi", "görə", "sonra", "əgər", "eger", "deyil", "lakin", "amma",
+ "ancaq", "artıq", "artiq", "də", "isə", "həm", "yalnız", "yalniz"]),
};
const NON_EN_DIACRITICS = new Set(
@@ -103,8 +121,22 @@ const WORD_RE = /[^\W\d_]+/gu;
const IDENTIFIER_RE = /[\p{L}\p{N}_-]*(?:[.@][\p{L}\p{N}_-]+)+/gu;
const IS_ALPHA_RE = /\p{L}/u;
+// Python uses two different boundaries on purpose: detect_script/script_profile count the IPA
+// Extensions block (0x0250-0x02AF) as Latin, while _script_of (used for non-Latin word runs)
+// stops at 0x0250 so a pronunciation like [vlɐˈdʲimʲɪr] is neither Latin nor a script run.
function isLatinCp(cp: number): boolean {
- return cp < 0x0250 || (0x1e00 <= cp && cp <= 0x1eff) || (0xff21 <= cp && cp <= 0xff3a) || (0xff41 <= cp && cp <= 0xff5a);
+ return cp < 0x02b0 || (0x1e00 <= cp && cp <= 0x1eff) || (0xff21 <= cp && cp <= 0xff3a) || (0xff41 <= cp && cp <= 0xff5a);
+}
+
+function scriptOf(ch: string): string | null {
+ const cp = ch.codePointAt(0)!;
+ if (cp < 0x0250 || (0x1e00 <= cp && cp <= 0x1eff) || (0xff21 <= cp && cp <= 0xff3a) || (0xff41 <= cp && cp <= 0xff5a)) {
+ return null;
+ }
+ for (const [name, ranges] of SCRIPT_RANGES) {
+ if (ranges.some(([lo, hi]) => lo <= cp && cp <= hi)) return name;
+ }
+ return null;
}
export function stateText(state: unknown, maxChars = 4000): string {
@@ -167,6 +199,44 @@ export function scriptProfile(text: string): Record<string, number> {
return out;
}
+// Non-Latin text is not for the English checkpoint even when Latin letters are the plurality: a
+// brand name or order code outvotes the CJK request around it letter for letter, though one CJK
+// character carries far more than a letter. A short message needs a large share to count; a long
+// payload (ticket fields, English agent turns) dilutes the share, so there a sentence's worth of
+// letters counts too.
+const NON_LATIN_FRACTION = 0.2;
+const NON_LATIN_MIN_FRACTION = 0.1;
+const NON_LATIN_MIN_LETTERS = 10;
+
+/** Non-Latin runs that read as words rather than as annotation inside English prose. */
+function nonLatinWords(text: string): string[] {
+ // English prose carries three kinds of non-Latin letters that are not a request written in
+ // another script, and each is excluded here: a symbol ("Set α to 0.05", one letter), a proper
+ // name (capitalised), and a pronunciation ([vlɐˈdʲimʲɪr], which no script range claims). A
+ // combining mark belongs to the letter before it and never splits a word.
+ const runs: string[] = [];
+ let cur = "";
+ let script: string | null = null;
+ for (const ch of text) {
+ if (/^\p{M}$/u.test(ch)) continue;
+ const s = scriptOf(ch);
+ if (s !== null && s === script) {
+ cur += ch;
+ continue;
+ }
+ if (cur) runs.push(cur);
+ if (s !== null) {
+ cur = ch;
+ script = s;
+ } else {
+ cur = "";
+ script = null;
+ }
+ }
+ if (cur) runs.push(cur);
+ return runs.filter((w) => [...w].length >= 2 && !/^[\p{Lu}\p{Lt}]/u.test(w));
+}
+
export interface LatinProfile {
language: string | null;
englishHits: number;
@@ -236,8 +306,26 @@ function round4(x: number): number {
export function analyse(state: unknown): AnalyseResult {
const text = stateText(state);
const prof = scriptProfile(text);
- const script = detectScript(text);
+ let script = detectScript(text);
const nonLatin = prof && Object.keys(prof).length ? round4(1.0 - (prof["latin"] ?? 0.0)) : 0.0;
+ let nAlpha = 0;
+ for (const ch of text) if (IS_ALPHA_RE.test(ch)) nAlpha += 1;
+ const nNonLatin = Math.round(nonLatin * nAlpha);
+ if (script === "latin" && nonLatinWords(text).length > 0 &&
+ (nonLatin >= NON_LATIN_FRACTION ||
+ (nonLatin >= NON_LATIN_MIN_FRACTION && nNonLatin >= NON_LATIN_MIN_LETTERS))) {
+ // The plurality said Latin, but the non-Latin share is a real message, not a stray name or
+ // symbol: re-classify to the dominant non-Latin script, as Python's analyse does.
+ let bestScript = script;
+ let bestShare = -1;
+ for (const [s, share] of Object.entries(prof)) {
+ if (s !== "latin" && share > bestShare) {
+ bestShare = share;
+ bestScript = s;
+ }
+ }
+ script = bestScript;
+ }
if (script === "unknown") {
return {
script: "unknown", scriptProfile: prof, language: null,
diff --git a/laya-ts/tests/lang.test.ts b/laya-ts/tests/lang.test.ts
index c15f0a7f..9cfce581 100644
--- a/laya-ts/tests/lang.test.ts
+++ b/laya-ts/tests/lang.test.ts
@@ -14,4 +14,29 @@ describe("lang", () => {
it("unknown (no letters) is english + undecided", () => {
expect(analyse("123 !!!").script).toBe("unknown");
});
+ // Every expectation below was generated with Python's laya.lang.analyse on the same input.
+ it("names romanized Bangla instead of falling back to english", () => {
+ const a = analyse("ami ekta ticket khulsi, kalke theke payment hocche na, ekhon ki korte parbo");
+ expect(a.script).toBe("latin");
+ expect(a.language).toBe("bn");
+ expect(a.isEnglish).toBe(false);
+ });
+ it("names Azerbaijani Latin text", () => {
+ const a = analyse("ödənişim iki dəfə tutulub, amma heç bir təsdiq almadım, nə etməliyəm");
+ expect(a.language).toBe("az");
+ expect(a.isEnglish).toBe(false);
+ });
+ it("a CJK sentence inside an English ticket is not english", () => {
+ const a = analyse("please check the attached logs 請重啟服務器然後再試一次 and tell me what failed");
+ expect(a.script).toBe("han");
+ expect(a.isEnglish).toBe(false);
+ });
+ it("a tiny non-latin note does not flip an english ticket", () => {
+ const a = analyse("please refund my order, see note 重啟 attached to the ticket");
+ expect(a.script).toBe("latin");
+ expect(a.isEnglish).toBe(true);
+ });
+ it("counts IPA extensions as latin, like Python", () => {
+ expect(detectScript("ɑ ɒ ɛ ɔ ɪ ʊ æ ʃ θ ð")).toBe("latin");
+ });
});