Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

fix(lang): route romanized Bangla to the multilingual checkpoint (#187)

Md Kishor Morolec931e5ee80e
diff --git a/laya/lang.py b/laya/lang.py
index 911509a8..c6361f27 100644
--- a/laya/lang.py
+++ b/laya/lang.py
@@ -115,6 +115,27 @@ _STOP = {
"ro": {"și", "să", "este", "sunt", "care", "pentru", "din", "dar", "după", "până", "fără",
"ale", "lui", "în", "fost", "acum", "vreau", "trebuie", "foarte", "acest", "această",
"acesta", "aceasta", "mi", "ți", "vă", "nu"},
+ # Romanized Bangla ("Banglish"): how Bangla is typed in chats, tickets and email when no Bengali
+ # keyboard is at hand. It has no diacritics, so without a list it read as undecided-but-English
+ # and went to the English checkpoint, which scores 0.08 on Bangla at 0.94 confidence. Spelling
+ # is not standardised, so the common variants are listed (`bhalo`/`valo`, `korchi`/`korsi`).
+ # Left out on purpose: frequent Bangla words that are also English words -- `ache` (is),
+ # `are` (is there), `to` (so), `take` (to him), `age` (before), `pore` (later), `mane`
+ # (meaning), `din` (give), `sob` (all), `tar` (his), `dao` (give), `eta` (this; ETA) --
+ # ordinary words of a neighbouring language (`ora`, `nei`, `vai`), and words another list
+ # already claims (`na`, `o`, `e`, `je`, `ta`, `por`, `hoy`), so adding `bn` cannot move a
+ # state of any other language.
+ "bn": {"ami", "amar", "amake", "amra", "amader", "apni", "apnar", "apnake", "apnara",
+ "tumi", "tomar", "tomake", "tomra", "tader", "ota", "eita", "oita",
+ "ekta", "ei", "oi", "ki", "keno", "kivabe", "kibhabe", "kothay", "kokhon", "kobe",
+ "koto", "kintu", "jodi", "tahole", "ar", "theke", "jonno", "sathe", "shathe", "diye",
+ "niye", "moddhe", "kore", "korte", "korchi", "korsi", "korbo", "korechi", "koreche",
+ "korun", "koren", "korlam", "hobe", "hoyeche", "hoise", "hocche", "hoyni",
+ "chai", "chaina", "lagbe", "parchi", "parbo", "parchina", "peyechi", "paini",
+ "dite", "dilam", "diyechi", "nai", "khub", "onek", "ekhon", "akhon", "ekhono",
+ "abar", "ekbar", "duibar", "ajke", "kalke", "taka", "bhalo", "valo", "kharap",
+ "shomossa", "somossa", "dhonnobad", "bhai", "shob", "keu", "kichu", "bolte", "bolun",
+ "parben", "asbe", "jabe", "pabo", "ferot", "dorkar", "hoye", "geche", "gese"},
}
# Letters that ordinary English does not use. This is the signal that catches a Latin-script
# language we hold no stopwords for at all (Romanian, Polish, Czech, Turkish, Baltic, ...),
diff --git a/tests/test_router.py b/tests/test_router.py
index ae490394..1f9575ab 100644
--- a/tests/test_router.py
+++ b/tests/test_router.py
@@ -377,6 +377,46 @@ for text in [
check("route/pt-br control " + text[:32], _r_lat.route(text).model, "english")
+# --------------------------------------------------------------------- romanized Bangla
+# Bangla is often typed in Latin letters ("Banglish") when no Bengali keyboard is at hand. It has no
+# diacritics and matched no stopword list, so it was reported `is_english=True` and handed to the
+# English checkpoint, which scores 0.08 on Bangla MASSIVE at 0.94 confidence. The Bengali-script
+# spelling of the same text already routed on script alone.
+for text in [
+ "amar kach theke duibar taka kata hoyeche, doya kore ferot din",
+ "ami invoice er jonno duibar charge peyechi, refund chai",
+ "Ami ei product ta niye khub hotash, ekhon e cancel korte chai",
+ "apnara keno amar call dhorchen na? ajke kichu ekta korun",
+ "bhai amar account e login korte parchi na",
+ "taka ekhono ferot paini, kobe pabo?",
+ "order ta kobe asbe bolte parben?",
+]:
+ check("latin_lang/banglish " + text[:32], guess_latin_language(text), "bn")
+ check("is_english/banglish " + text[:32], is_english(text), False)
+ check("route/banglish " + text[:32], _r_lat.route(text).model, "multilingual")
+check("route/bengali script", _r_lat.route("আমার কাছ থেকে দুইবার টাকা কাটা হয়েছে").model, "multilingual")
+
+# English must not move. `chai`, `ar`, `ami`, `koto`, `kore`, `oi` are Bangla words that also turn
+# up in English text as a drink, an acronym or a name; one of them next to English function words
+# stays English.
+for text in [
+ "Chai latte order was charged twice, please refund the extra amount",
+ "The AR team says the ETA for the fix is Friday",
+ "Ami Patel from the Koto office sent the invoice to Kore Ltd",
+ "Our AR and VR demo in Oi Bahia went well, the client wants a quote",
+ "Take the age of the account into account before you refund",
+]:
+ check("is_english/banglish control " + text[:32], is_english(text), True)
+ check("route/banglish control " + text[:32], _r_lat.route(text).model, "english")
+# ...and no other language may move either: the `bn` list claims no word another list holds, and
+# leaves out Romance words such as `ora`, `nei`, `vai`.
+from laya.lang import _STOP # noqa: E402
+check("latin_lang/bn list shares no word with another list",
+ sorted(w for w in _STOP.get("bn", ()) for lg, words in _STOP.items() if lg != "bn" and w in words), [])
+check("latin_lang/romanian with ei stays romanian",
+ guess_latin_language("Ei nu sunt de acord cu factura, vreau o corecție"), "ro")
+
+
# --------------------------------------------------------------------- temperature clamp (#35)
# A fitted temperature below 1 sharpens logits. The shipped `choice:11+` bucket is 0.1006, which
# turned a 0.24 top probability into 0.99 confidence on 13-option skill routing.