Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

fix(lang): keep Vietnamese off the English checkpoint

Rahul Ranjan3d007e00b635
diff --git a/laya-ts/src/lang.ts b/laya-ts/src/lang.ts
index 8f7ec810..bca4e77c 100644
--- a/laya-ts/src/lang.ts
+++ b/laya-ts/src/lang.ts
@@ -130,7 +130,14 @@ const NON_EN_DIACRITICS = new Set(
"ğı" + // Turkish (text is lowercased before matching)
"āēģīķļņūž" + // Baltic
"đ" + // Serbo-Croatian / Vietnamese
- "ə").split(""), // Azerbaijani
+ "ə" + // Azerbaijani
+ // Vietnamese: no stopword list of its own, so its letters are the only signal that keeps
+ // it off the English checkpoint, and the tone-marked letters are precomposed codepoints
+ // of their own (see laya/lang.py for the same fix).
+ "ạảấầẩẫậắằẳẵặ" +
+ "ẹẻẽếềểễệỉị" +
+ "ọỏốồổỗộớờởỡợụủứừửữự" +
+ "ỳỵỷỹơư").split(""),
);
export const NON_EN_DIACRITIC_RATE = 0.02;
diff --git a/laya-ts/tests/lang.test.ts b/laya-ts/tests/lang.test.ts
index 843253b4..5e459b81 100644
--- a/laya-ts/tests/lang.test.ts
+++ b/laya-ts/tests/lang.test.ts
@@ -37,6 +37,23 @@ describe("lang", () => {
expect(a.isEnglish).toBe(false);
expect(a.language).toBe(null);
});
+ // Vietnamese: no stopword list of its own, so NON_EN_DIACRITICS is the only signal that keeps
+ // it off the English checkpoint -- and the tone-marked letters are precomposed codepoints the
+ // set did not hold, so everyday sentences carried a diacritic rate of 0.0 and read as English.
+ // Expectations generated with Python's laya.lang.analyse on the same inputs.
+ it("counts Vietnamese tone-marked letters as non-English", () => {
+ const a = analyse("Cảm ơn bạn nhiều nhẻ");
+ expect(a.diacriticRate).toBeCloseTo(0.25, 5);
+ expect(a.isEnglish).toBe(false);
+ expect(a.language).toBe(null);
+ const b = analyse("Mình muốn hỏi về đơn hàng đã đặt");
+ expect(b.diacriticRate).toBeCloseTo(0.3438, 4);
+ expect(b.isEnglish).toBe(false);
+ });
+ it("keeps English with Vietnamese loanwords and names English", () => {
+ expect(isEnglish("The pho was great and the banh mi was the best in town")).toBe(true);
+ expect(isEnglish("Please help Nguyễn with the invoice for the last order")).toBe(true);
+ });
it("still routes plain english without a schwa to english", () => {
const a = analyse("the quick brown fox");
expect(a.diacriticRate).toBe(0);
diff --git a/laya/lang.py b/laya/lang.py
index 2079c6ad..feb9ca41 100644
--- a/laya/lang.py
+++ b/laya/lang.py
@@ -315,6 +315,18 @@ _NON_EN_DIACRITICS = set(
"āēģīķļņūž" # Baltic
"đ" # Serbo-Croatian / Vietnamese
"ə" # Azerbaijani
+ # Vietnamese: it holds no stopword list of its own, so its letters are the only
+ # signal that keeps it off the English checkpoint. The set used to stop at `đ`
+ # (the comment above already claimed Vietnamese coverage), but the tone-marked
+ # letters -- `ả`, `ạ`, `ề`, `ơ`, `ư` and friends -- are precomposed codepoints
+ # of their own, so everyday sentences such as "Cảm ơn bạn nhiều nhé" carried a
+ # diacritic rate of 0.0 and were routed to the English checkpoint.
+ # Text is lowercased before matching, so lowercase forms are enough.
+ "ạảấầẩẫậắằẳẵặ" # a-family tone marks
+ "ẹẻẽếềểễệỉị" # e-/i-family tone marks
+ "ọỏốồổỗộớờởỡợụủứừửữự" # o-/u-family tone marks
+ "ỳỵỷỹơư" # y-family tone marks, and the two letters
+ # no other listed language uses
)
# Words that more than one list claims. `la`, `un`, `e`, `que`, `una` and friends are function words
# of several of these languages at once, so matching one says "not English" without saying *which*
diff --git a/tests/test_router.py b/tests/test_router.py
index a81709e1..2365792c 100644
--- a/tests/test_router.py
+++ b/tests/test_router.py
@@ -9,6 +9,7 @@ sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
from laya.common import QTYPES, TEMP_MAX, TEMP_MIN, clamp_temperature, temp_bucket # noqa: E402
from laya.lang import analyse, detect_script, guess_latin_language, is_english, state_text # noqa: E402
+from laya.lang import NON_EN_DIACRITIC_RATE # noqa: E402
from laya.lang import _non_english_segment # noqa: E402
from laya.router import ( # noqa: E402
BUNDLE_REPO,
@@ -2286,6 +2287,36 @@ except Exception as _exc: # noqa: BLE001
(type(_exc).__name__, _exc))
+# --------------------------------------------------------------------- Vietnamese
+# Vietnamese is a Latin-script language with no stopword list of its own, so the non-English
+# diacritic set is the only signal that keeps it off the English checkpoint -- but every
+# tone-marked Vietnamese letter (`ả`, `ạ`, `ề`, `ơ`, `ư`, ...) is a precomposed codepoint the
+# set did not hold, and it already claimed Vietnamese coverage through `đ` alone. Everyday
+# sentences carried a diacritic rate of 0.0 and routed to the English checkpoint.
+for text in [
+ "Cảm ơn bạn nhiều nhẻ",
+ "Cảm ơn bạn rất nhiều",
+ "Mình muốn hỏi về đơn hàng đã đặt",
+ "Cho tôi xin hoàn tiền cho đơn hàng này",
+ "CẢM ƠN BẠN RẤT NHIỀU",
+]:
+ check("latin/vietnamese diacritic rate " + text[:24],
+ analyse(text)["diacritic_rate"] > NON_EN_DIACRITIC_RATE, True)
+ check("is_english/vietnamese " + text[:24], is_english(text), False)
+ check("route/vietnamese " + text[:24], _r_lat.route(text).model, "multilingual")
+# English must not move: Vietnamese dishes and names that made it into English (`pho`,
+# `banh mi`, `ao dai`) are plain ASCII, and an accented Vietnamese surname in an otherwise
+# English sentence is one loanword, not a Vietnamese state (#337 rescue).
+for text in [
+ "The pho was great and the banh mi was the best in town",
+ "The ao dai show at the cafe was wonderful",
+ "Thanks Nguyen for the meal we had last night",
+ "Please help Nguyễn with the invoice for the last order",
+]:
+ check("is_english/vietnamese control " + text[:24], is_english(text), True)
+ check("route/vietnamese control " + text[:24], _r_lat.route(text).model, "english")
+
+
# --------------------------------------------------------------------- report
print("\n%d passed, %d failed" % (len(PASS), len(FAIL)))
for f in FAIL: