Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
fix(lang): keep Vietnamese off the English checkpoint
Rahul Ranjan3d007e00b635
diff --git a/laya-ts/src/lang.ts b/laya-ts/src/lang.tsindex 8f7ec810..bca4e77c 100644--- a/laya-ts/src/lang.ts+++ b/laya-ts/src/lang.ts@@ -130,7 +130,14 @@ const NON_EN_DIACRITICS = new Set("ğı" + // Turkish (text is lowercased before matching)"āēģīķļņūž" + // Baltic"đ" + // Serbo-Croatian / Vietnamese- "ə").split(""), // Azerbaijani+ "ə" + // Azerbaijani+ // Vietnamese: no stopword list of its own, so its letters are the only signal that keeps+ // it off the English checkpoint, and the tone-marked letters are precomposed codepoints+ // of their own (see laya/lang.py for the same fix).+ "ạảấầẩẫậắằẳẵặ" ++ "ẹẻẽếềểễệỉị" ++ "ọỏốồổỗộớờởỡợụủứừửữự" ++ "ỳỵỷỹơư").split(""),);export const NON_EN_DIACRITIC_RATE = 0.02;diff --git a/laya-ts/tests/lang.test.ts b/laya-ts/tests/lang.test.tsindex 843253b4..5e459b81 100644--- a/laya-ts/tests/lang.test.ts+++ b/laya-ts/tests/lang.test.ts@@ -37,6 +37,23 @@ describe("lang", () => {expect(a.isEnglish).toBe(false);expect(a.language).toBe(null);});+ // Vietnamese: no stopword list of its own, so NON_EN_DIACRITICS is the only signal that keeps+ // it off the English checkpoint -- and the tone-marked letters are precomposed codepoints the+ // set did not hold, so everyday sentences carried a diacritic rate of 0.0 and read as English.+ // Expectations generated with Python's laya.lang.analyse on the same inputs.+ it("counts Vietnamese tone-marked letters as non-English", () => {+ const a = analyse("Cảm ơn bạn nhiều nhẻ");+ expect(a.diacriticRate).toBeCloseTo(0.25, 5);+ expect(a.isEnglish).toBe(false);+ expect(a.language).toBe(null);+ const b = analyse("Mình muốn hỏi về đơn hàng đã đặt");+ expect(b.diacriticRate).toBeCloseTo(0.3438, 4);+ expect(b.isEnglish).toBe(false);+ });+ it("keeps English with Vietnamese loanwords and names English", () => {+ expect(isEnglish("The pho was great and the banh mi was the best in town")).toBe(true);+ expect(isEnglish("Please help Nguyễn with the invoice for the last order")).toBe(true);+ });it("still routes plain english without a schwa to english", () => {const a = analyse("the quick brown fox");expect(a.diacriticRate).toBe(0);diff --git a/laya/lang.py b/laya/lang.pyindex 2079c6ad..feb9ca41 100644--- a/laya/lang.py+++ b/laya/lang.py@@ -315,6 +315,18 @@ _NON_EN_DIACRITICS = set("āēģīķļņūž" # Baltic"đ" # Serbo-Croatian / Vietnamese"ə" # Azerbaijani+ # Vietnamese: it holds no stopword list of its own, so its letters are the only+ # signal that keeps it off the English checkpoint. The set used to stop at `đ`+ # (the comment above already claimed Vietnamese coverage), but the tone-marked+ # letters -- `ả`, `ạ`, `ề`, `ơ`, `ư` and friends -- are precomposed codepoints+ # of their own, so everyday sentences such as "Cảm ơn bạn nhiều nhé" carried a+ # diacritic rate of 0.0 and were routed to the English checkpoint.+ # Text is lowercased before matching, so lowercase forms are enough.+ "ạảấầẩẫậắằẳẵặ" # a-family tone marks+ "ẹẻẽếềểễệỉị" # e-/i-family tone marks+ "ọỏốồổỗộớờởỡợụủứừửữự" # o-/u-family tone marks+ "ỳỵỷỹơư" # y-family tone marks, and the two letters+ # no other listed language uses)# Words that more than one list claims. `la`, `un`, `e`, `que`, `una` and friends are function words# of several of these languages at once, so matching one says "not English" without saying *which*diff --git a/tests/test_router.py b/tests/test_router.pyindex a81709e1..2365792c 100644--- a/tests/test_router.py+++ b/tests/test_router.py@@ -9,6 +9,7 @@ sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))from laya.common import QTYPES, TEMP_MAX, TEMP_MIN, clamp_temperature, temp_bucket # noqa: E402from laya.lang import analyse, detect_script, guess_latin_language, is_english, state_text # noqa: E402+from laya.lang import NON_EN_DIACRITIC_RATE # noqa: E402from laya.lang import _non_english_segment # noqa: E402from laya.router import ( # noqa: E402BUNDLE_REPO,@@ -2286,6 +2287,36 @@ except Exception as _exc: # noqa: BLE001(type(_exc).__name__, _exc))+# --------------------------------------------------------------------- Vietnamese+# Vietnamese is a Latin-script language with no stopword list of its own, so the non-English+# diacritic set is the only signal that keeps it off the English checkpoint -- but every+# tone-marked Vietnamese letter (`ả`, `ạ`, `ề`, `ơ`, `ư`, ...) is a precomposed codepoint the+# set did not hold, and it already claimed Vietnamese coverage through `đ` alone. Everyday+# sentences carried a diacritic rate of 0.0 and routed to the English checkpoint.+for text in [+ "Cảm ơn bạn nhiều nhẻ",+ "Cảm ơn bạn rất nhiều",+ "Mình muốn hỏi về đơn hàng đã đặt",+ "Cho tôi xin hoàn tiền cho đơn hàng này",+ "CẢM ƠN BẠN RẤT NHIỀU",+]:+ check("latin/vietnamese diacritic rate " + text[:24],+ analyse(text)["diacritic_rate"] > NON_EN_DIACRITIC_RATE, True)+ check("is_english/vietnamese " + text[:24], is_english(text), False)+ check("route/vietnamese " + text[:24], _r_lat.route(text).model, "multilingual")+# English must not move: Vietnamese dishes and names that made it into English (`pho`,+# `banh mi`, `ao dai`) are plain ASCII, and an accented Vietnamese surname in an otherwise+# English sentence is one loanword, not a Vietnamese state (#337 rescue).+for text in [+ "The pho was great and the banh mi was the best in town",+ "The ao dai show at the cafe was wonderful",+ "Thanks Nguyen for the meal we had last night",+ "Please help Nguyễn with the invoice for the last order",+]:+ check("is_english/vietnamese control " + text[:24], is_english(text), True)+ check("route/vietnamese control " + text[:24], _r_lat.route(text).model, "english")++# --------------------------------------------------------------------- reportprint("\n%d passed, %d failed" % (len(PASS), len(FAIL)))for f in FAIL: