Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
Merge pull request #460 from AlKor13/fix/latin-distinct-evidence
Nandakishor6b8be28aa06a
diff --cc tests/test_lang_guess.pyindex a1add495,d68a3d83..6bbbbeba--- a/tests/test_lang_guess.py+++ b/tests/test_lang_guess.py@@@ -176,186 -177,49 +177,229 @@@ for label, s, want in BEFOREcheck("unchanged/" + label, r0.route(s, GENERIC)["model"], want)+# ------------------------------------------------------------------ Hindi (Devanagari script)+# Hindi uses Devanagari (U+0900-U+097F), which is in _SCRIPT_RANGES. The English+# checkpoint (ModernBERT-large, 50k English BPE) has no Devanagari tokens, so any+# state written in Hindi must reach the multilingual checkpoint. These cases cover+# the four main customer-support scenarios: billing, technical, account and cancellation.+#+# NOTE on future disambiguation: both Hindi and Marathi use Devanagari (U+0900-U+097F).+# Script detection routes by Unicode block, so it cannot distinguish between languages+# sharing the same script. Both currently route to `multilingual`. If a future language-specific+# checkpoint is added (e.g. a dedicated Hindi or Marathi model), these tests would need to+# assert the specific checkpoint name rather than the generic `multilingual` bucket.++HINDI_CASES = [+ # billing & payment+ ("hindi/billing duplicate charge",+ "मुझसे मार्च महीने में दो बार शुल्क लिया गया है, कृपया डुप्लिकेट राशि वापस करें।",+ "multilingual"),+ ("hindi/payment failed refund",+ "मेरा भुगतान विफल हो गया लेकिन पैसे कट गए, मुझे तुरंत वापसी चाहिए।",+ "multilingual"),+ ("hindi/invoice not received",+ "मुझे अप्रैल माह का इनवॉइस अभी तक नहीं मिला है, कृपया भेजें।",+ "multilingual"),+ # technical support+ ("hindi/app crash on settings",+ "एप्लिकेशन हर बार सेटिंग खोलने पर बंद हो जाती है, कृपया जल्दी ठीक करें।",+ "multilingual"),+ ("hindi/login failure",+ "मैं अपने खाते में लॉग इन नहीं कर पा रहा हूँ, पासवर्ड सही है फिर भी एरर आ रहा है।",+ "multilingual"),+ ("hindi/OTP not received",+ "OTP मेरे मोबाइल पर नहीं आ रहा, मैं सत्यापन पूरा नहीं कर पा रहा।",+ "multilingual"),+ # account & cancellation+ ("hindi/cancel threat",+ "अगर यह समस्या जल्द हल नहीं हुई तो मैं अपनी सदस्यता रद्द कर दूँगा।",+ "multilingual"),+ ("hindi/plan upgrade enquiry",+ "मैं अपना प्लान अपग्रेड करना चाहता हूँ, प्रीमियम के क्या फायदे हैं?",+ "multilingual"),+ # short utterances (still Devanagari -- script detection is exact, not length-dependent)+ ("hindi/short help request",+ "नमस्ते, मुझे मदद चाहिए।",+ "multilingual"),+ ("hindi/single word",+ "धन्यवाद",+ "multilingual"),+]++for label, text, want in HINDI_CASES:+ check(label, r0.route(text, GENERIC)["model"], want)++# lang_guess codes for Hindi: assert model and that detection is None (showing the hint+# was decisive and bypassed built-in script detection).+# We test with English text where lang_guess="hi" forces multilingual routing;+# if lang_guess were ignored, plain English would route to english.+d_hi_dev = r0.route("मुझे मदद चाहिए।", GENERIC, lang_guess="hi")+check("hindi/lang_guess hi routes multilingual on devanagari", d_hi_dev["model"], "multilingual")+check_true("hindi/lang_guess hi bypasses detection (detection is None)", d_hi_dev["detection"] is None)+check_true("hindi/lang_guess hi recorded in reason", "lang_guess" in d_hi_dev["reason"])++d_hi_en = r0.route("I need help with my account billing.", GENERIC, lang_guess="hi")+check("hindi/lang_guess hi overrides English text to multilingual", d_hi_en["model"], "multilingual")+check_true("hindi/lang_guess hi on English sets detection to None", d_hi_en["detection"] is None)++d_hi_in = r0.route("I need help with my account billing.", GENERIC, lang_guess="hi-IN")+check("hindi/lang_guess hi-IN routes multilingual", d_hi_in["model"], "multilingual")+check_true("hindi/lang_guess hi-IN sets detection to None", d_hi_in["detection"] is None)++# script is reported correctly for native Devanagari text (no lang_guess needed)+check("hindi/script detected as devanagari",+ r0.route("यह एक हिंदी वाक्य है।", GENERIC)["detection"]["script"], "devanagari")++# one Devanagari field in a mixed dict is enough to reach multilingual+check("hindi/mixed Hindi-English dict reaches multilingual",+ r0.route({"subject": "Payment issue", "body": "मेरा भुगतान विफल हो गया।"}, GENERIC)["model"],+ "multilingual")++# script detection is not length-weighted: a long English subject must not outvote+# a short Devanagari body field.+check("hindi/long English field does not outvote short Hindi field",+ r0.route(+ {"subject": ("We have been experiencing persistent difficulties with our account billing "+ "over the past several months and need urgent support from customer care."),+ "body": "कृपया मेरा भुगतान वापस करें।"},+ GENERIC)["model"],+ "multilingual")+++# ------------------------------------------------------------------ Marathi (Devanagari script)+# Marathi shares Devanagari with Hindi (same Unicode block, U+0900-U+097F) but is a+# distinct language spoken by ~90 million people. Routing must reach multilingual for+# both -- any regression sending Devanagari to the English checkpoint collapses+# accuracy to near-random (measured at 0.100 on Hindi on MASSIVE at 20 options).+#+# NOTE on future disambiguation: both Hindi and Marathi use Devanagari (U+0900-U+097F).+# Script detection routes by Unicode block, so it cannot distinguish between languages+# sharing the same script. Both currently route to `multilingual`. If a future language-specific+# checkpoint is added (e.g. a dedicated Hindi or Marathi model), these tests would need to+# assert the specific checkpoint name rather than the generic `multilingual` bucket.++MARATHI_CASES = [+ # billing & payment+ ("marathi/billing duplicate charge",+ "मला मार्च महिन्यात दोनदा शुल्क आकारले गेले आहे, कृपया अतिरिक्त रक्कम परत करा.",+ "multilingual"),+ ("marathi/payment failed refund",+ "माझे पेमेंट अयशस्वी झाले पण पैसे कापले गेले, कृपया परतावा द्या.",+ "multilingual"),+ ("marathi/invoice not received",+ "मला एप्रिल महिन्याचे बिल अजून मिळाले नाही, कृपया पाठवा.",+ "multilingual"),+ # technical support (scenarios mirror Hindi exactly)+ ("marathi/app crash on settings",+ "अॅप्लिकेशन सेटिंग उघडताना प्रत्येक वेळी बंद होते, कृपया लवकर सोडवा.",+ "multilingual"),+ ("marathi/login failure",+ "मी माझ्या खात्यात लॉग इन करू शकत नाही, पासवर्ड बरोबर असूनही चूक येते.",+ "multilingual"),+ ("marathi/OTP not received",+ "OTP माझ्या मोबाईलवर येत नाही, मी पडताळणी पूर्ण करू शकत नाही.",+ "multilingual"),+ # account & cancellation+ ("marathi/cancel threat",+ "जर ही समस्या लवकर सुटली नाही तर मी माझी सदस्यता रद्द करेन.",+ "multilingual"),+ ("marathi/plan upgrade enquiry",+ "मला माझा प्लान अपग्रेड करायचा आहे, प्रीमियमचे काय फायदे आहेत?",+ "multilingual"),+ # short utterances+ ("marathi/short help request",+ "नमस्कार, मला मदत हवी आहे.",+ "multilingual"),+ ("marathi/single word",+ "धन्यवाद",+ "multilingual"),+]++for label, text, want in MARATHI_CASES:+ check(label, r0.route(text, GENERIC)["model"], want)++# lang_guess codes for Marathi: assert model and detection is None+d_mr_dev = r0.route("मला मदत हवी आहे.", GENERIC, lang_guess="mr")+check("marathi/lang_guess mr routes multilingual on devanagari", d_mr_dev["model"], "multilingual")+check_true("marathi/lang_guess mr bypasses detection (detection is None)", d_mr_dev["detection"] is None)+check_true("marathi/lang_guess mr recorded in reason", "lang_guess" in d_mr_dev["reason"])++d_mr_en = r0.route("I need help with my account billing.", GENERIC, lang_guess="mr")+check("marathi/lang_guess mr overrides English text to multilingual", d_mr_en["model"], "multilingual")+check_true("marathi/lang_guess mr on English sets detection to None", d_mr_en["detection"] is None)++d_mr_in = r0.route("I need help with my account billing.", GENERIC, lang_guess="mr-IN")+check("marathi/lang_guess mr-IN routes multilingual", d_mr_in["model"], "multilingual")+check_true("marathi/lang_guess mr-IN sets detection to None", d_mr_in["detection"] is None)++# script is reported correctly for Marathi too+check("marathi/script detected as devanagari",+ r0.route("माझे पेमेंट अयशस्वी झाले.", GENERIC)["detection"]["script"], "devanagari")++# one Marathi field in a mixed dict reaches multilingual+check("marathi/mixed Marathi-English dict reaches multilingual",+ r0.route({"subject": "Billing problem", "body": "मला दोनदा शुल्क आकारले गेले."}, GENERIC)["model"],+ "multilingual")++# long English field must not outvote short Marathi field (mirrors the Hindi case above)+check("marathi/long English field does not outvote short Marathi field",+ r0.route(+ {"subject": ("We have been experiencing persistent difficulties with our account billing "+ "over the past several months and need urgent support from customer care."),+ "body": "कृपया माझे पैसे परत करा."},+ GENERIC)["model"],+ "multilingual")+++# ------------------------------------------------------------------ Hindi vs Marathi consistency+# Both languages use Devanagari. The router decides on script, not language identity,+# so routing must be identical for both.+check("hindi_marathi/same checkpoint regardless of language",+ r0.route("मुझे मदद चाहिए।", GENERIC)["model"],+ r0.route("मला मदत हवी आहे.", GENERIC)["model"])+ # A foreign function word that is also an ordinary English word (`come` it, `son` es, `do` pt,+ # `care` ro, `todo`/`im`/`per`/`plus`) used to name that language when it merely appeared+ # twice, because the score counted occurrences and `best >= 2` was reached by one repeated word.+ # Ordinary English then routed to the multilingual checkpoint. Such a word now counts once however+ # often it repeats, so one collision no longer clears the bar.+ ENGLISH_ON_A_REPEAT = [+ "Come one, come all", # come (it) x2+ "My son, your son", # son (es) x2+ "Do more, do less", # do (pt) x2+ "Care more, care less", # care (ro) x2+ "Add a todo, then another todo item", # todo (es) x2+ "im not able to log in, im stuck", # im (de) x2+ "add 45 to 87 plus 54 plus 43 plus 22", # plus (de, fr) x3 -- a CLINC150 test row+ ]+ for s in ENGLISH_ON_A_REPEAT:+ check("repeat/plain english stays english: %r" % s, analyse(s)["is_english"], True)++ # The dedupe is limited to those words. A function word that is nobody's English -- `der`, `des`,+ # `di`, `sa` -- still counts every occurrence, so a real request whose only evidence is one such+ # word repeated stays foreign. Rows from the MASSIVE test splits, which is where deduping every+ # word instead of the collisions cost 791 of 148,700 non-English texts their checkpoint.+ REPEAT_STAYS_FOREIGN = [+ ("de", "reduzieren der helligkeit der lichter"),+ ("es", "enumerar todos los horarios de los tren a nueva york"),+ ("fr", "jouer des chansons des beatles"),+ ("it", "numero di telefono di giacomo"),+ ("pt", "pede um pacote de massa chinesa faz um pedido takeaway"),+ ("ro", "ar trebui sa port o pelerina de ploaie inainte sa ies afara"),+ ("nl", "herinner me eraan dat ik dat liedje leuk vind"),+ # `van` is left out of the collision list for exactly this row: it is ordinary Dutch+ ("nl", "hey olly ik hou van muziek van frans bauer"),+ ]+ for want, s in REPEAT_STAYS_FOREIGN:+ check("repeat/%s on a repeated word is still foreign" % want, analyse(s)["is_english"], False)++ # and real foreign requests -- two different function words each -- are still detected+ STILL_FOREIGN = [+ ("es", "Hola, necesito cancelar mi pedido por favor ahora mismo gracias"),+ ("pt", "Eu quero cancelar o meu pedido por favor agora mesmo obrigado"),+ ("it", "Vorrei annullare il mio ordine per favore adesso grazie mille"),+ ]+ for want, s in STILL_FOREIGN:+ check("repeat/genuine %s still foreign" % want, analyse(s)["is_english"], False)print("\n%d passed, %d failed" % (len(PASS), len(FAIL)))