Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
fix(lang): recognize Azerbaijani text when routing requests (#36)
Rashad Abdulkhaligovfba30d36e5e2
diff --git a/laya/lang.py b/laya/lang.pyindex 2584a919..93388752 100644--- a/laya/lang.py+++ b/laya/lang.py@@ -137,6 +137,12 @@ _STOP = {"abar", "ekbar", "duibar", "ajke", "kalke", "taka", "bhalo", "valo", "kharap","shomossa", "somossa", "dhonnobad", "bhai", "shob", "keu", "kichu", "bolte", "bolun","parben", "asbe", "jabe", "pabo", "ferot", "dorkar", "hoye", "geche", "gese"},++ # "her", "ne", "men", "de" collide with English, French and Romanian, so they are left out.+ "az": {"və", "ve", "bir", "bu", "ki", "üçün", "ucun", "ilə", "ile", "olan", "olub", "olmasa",+ "var", "yox", "yoxdur", "mən", "sən", "biz", "siz", "onlar", "daha", "çox", "cox",+ "hər", "nə", "kimi", "görə", "sonra", "əgər", "eger", "deyil", "lakin", "amma",+ "ancaq", "artıq", "artiq", "də", "isə", "həm", "yalnız", "yalniz"},}# Letters that ordinary English does not use. This is the signal that catches a Latin-script# language we hold no stopwords for at all (Romanian, Polish, Czech, Turkish, Baltic, ...),@@ -151,6 +157,7 @@ _NON_EN_DIACRITICS = set("ğı" # Turkish (text is lowercased before matching)"āēģīķļņūž" # Baltic"đ" # Serbo-Croatian / Vietnamese+ "ə" # Azerbaijani)# Words that more than one list claims. `la`, `un`, `e`, `que`, `una` and friends are function words# of several of these languages at once, so matching one says "not English" without saying *which*@@ -201,8 +208,8 @@ def detect_script(text: str) -> str:if not ch.isalpha():continuecp = ord(ch)- if cp < 0x0250 or 0x1E00 <= cp <= 0x1EFF or 0xFF21 <= cp <= 0xFF3A or 0xFF41 <= cp <= 0xFF5A:- latin += 1 # Latin, Latin Ext-Additional, fullwidth+ if cp < 0x02B0 or 0x1E00 <= cp <= 0x1EFF or 0xFF21 <= cp <= 0xFF3A or 0xFF41 <= cp <= 0xFF5A:+ latin += 1 # Latin, IPA Extensions, Ext-Additional, fullwidthcontinuefor name, ranges in _SCRIPT_RANGES:if any(lo <= cp <= hi for lo, hi in ranges):@@ -233,7 +240,7 @@ def script_profile(text: str) -> Dict[str, float]:if not ch.isalpha():continuecp = ord(ch)- if cp < 0x0250 or 0x1E00 <= cp <= 0x1EFF or 0xFF21 <= cp <= 0xFF3A or 0xFF41 <= cp <= 0xFF5A:+ if cp < 0x02B0 or 0x1E00 <= cp <= 0x1EFF or 0xFF21 <= cp <= 0xFF3A or 0xFF41 <= cp <= 0xFF5A:counts["latin"] += 1continuefor name, ranges in _SCRIPT_RANGES:@@ -309,7 +316,8 @@ def latin_profile(text: str) -> Dict[str, object]:A non-English language is only named when it matched at least one word that no other listclaims: shared function words alone (`la`, `e`, `o`) identify no particular language."""- words = [w.lower() for w in _WORD.findall(_IDENTIFIER.sub(" ", text))]+ # 'İ'.lower() is 'i' + a combining dot, which matches no word list+ words = _WORD.findall(_IDENTIFIER.sub(" ", text).replace("İ", "i").lower())lowered = text.lower()diac = sum(1 for ch in lowered if ch in _NON_EN_DIACRITICS)diac_rate = diac / max(1, len(lowered))diff --git a/tests/test_router.py b/tests/test_router.pyindex d8a06822..5c79f5f4 100644--- a/tests/test_router.py+++ b/tests/test_router.py@@ -35,6 +35,8 @@ SCRIPTS = [("armenian", "Հայերեն", "armenian"),("armenian uppercase", "ՀԱՅԵՐԵՆ", "armenian"),("armenian punctuation only", "։֊", "unknown"),+ ("azerbaijani lone schwa", "ə", "latin"),+ ("azerbaijani uppercase", "MÜŞTƏRİ İLƏ ƏLAQƏ SAXLAYIN", "latin"),("french", "Le client a été facturé deux fois et demande un remboursement.", "latin"),("hindi", "ग्राहक से दो बार शुल्क लिया गया और वह धनवापसी चाहता है।", "devanagari"),("japanese", "お客様は二重に請求されたため返金を希望しています。", "kana"),@@ -57,6 +59,8 @@ for label, text, want in SCRIPTS:for label, text, want in [("plain english", "Please refund the duplicate charge on invoice 4411 today.", True),("armenian", "Հայերեն", False),+ ("azerbaijani no diacritics", "Sifarisim gelmedi ve pulum geri qaytarilmadi, zehmet olmasa yoxlayin", False),+ ("azerbaijani few diacritics", "Mən sizin xidmətinizdən razı deyiləm və pulumu geri istəyirəm", False),("english short", "refund me", True),("hindi", "ग्राहक से दो बार शुल्क लिया गया", False),("japanese", "お客様は二重に請求されました", False),@@ -108,6 +112,11 @@ for label, text, want in [("french", "Le client a ete facture deux fois et il demande un remboursement pour la facture", "fr"),("german", "Der Kunde wurde zweimal belastet und moechte eine Rueckerstattung fuer die Rechnung", "de"),("spanish", "El cliente fue cobrado dos veces y quiere que le devuelvan el dinero por la factura", "es"),+ ("azerbaijani", "Zəhmət olmasa, sifarişim üçün pulu geri qaytarın, çünki məhsul gəlmədi", "az"),+ # one shared function word is not enough to name a language+ ("azerbaijani single hit stays undecided",+ "Müştəridən iki dəfə pul alınıb və o, geri qaytarılmasını istəyir", None),+ ("azerbaijani uppercase dotted I", "MÜŞTƏRİ İLƏ ƏLAQƏ SAXLAYIN VƏ PULU GERİ QAYTARIN", "az"),("too short", "refund", None),]:check("latin_lang/" + label, guess_latin_language(text), want)@@ -160,6 +169,7 @@ check("state_text/keys ignored",# --------------------------------------------------------------------- workflow signaturescheck("profile/armenian", analyse("Հայերեն")["script_profile"], {"armenian": 1.0})check("profile/armenian mixed with Latin", analyse("Հայերեն abc")["non_latin_fraction"], 0.7)+check("profile/azerbaijani schwa is latin", analyse("ələ")["script_profile"], {"latin": 1.0})TD = {"agent_trace_observability": ["action", "needs_review", "outcome", "risk", "urgency"],@@ -197,6 +207,10 @@ cases = [("english text", {"body": "I was charged twice, please refund."}, Q_GENERIC, {}, "english"),("armenian text", {"body": "Հայերեն"}, Q_GENERIC, {}, "multilingual"),("armenian explicit override", {"body": "Հայերեն"}, Q_GENERIC, {"model": "english"}, "english"),+ ("azerbaijani no diacritics", {"body": "Sifarisim gelmedi ve pulum geri qaytarilmadi, zehmet olmasa yoxlayin"},+ Q_GENERIC, {}, "multilingual"),+ ("azerbaijani explicit override", {"body": "Müştəridən iki dəfə pul alınıb"}, Q_GENERIC,+ {"model": "english"}, "english"),("hindi text", {"body": "मुझसे दो बार शुल्क लिया गया"}, Q_GENERIC, {}, "multilingual"),("japanese text", {"body": "二重に請求されました"}, Q_GENERIC, {}, "multilingual"),("korean text", {"body": "두 번 청구되었습니다"}, Q_GENERIC, {}, "multilingual"),