Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
fix(lang): route a Portuguese ticket with an English log to multilingual
Paulo Carinhena952e125de5e7
diff --git a/laya/lang.py b/laya/lang.pyindex 0122c787..f7995e30 100644--- a/laya/lang.py+++ b/laya/lang.py@@ -380,11 +380,45 @@ def guess_latin_language(text: str) -> Optional[str]:return latin_profile(text)["language"]+# Code is not prose in any language, but split into words it reads as one: `os.path` is Portuguese+# (`os`), `round(el, 2)` Spanish (`el`), `non_english` Italian (`non`). A line pasted from a program+# into an English request must not count as a foreign segment, so a line carrying code syntax --+# `=`, `;`, braces, brackets or a call `name(` -- is skipped, and dotted or underscored identifiers+# are dropped from the rest. Prose keeps "Deu erro (500)": the parenthesis follows a space.+_CODE_LINE = re.compile(r"[=;{}\[\]]|\w\(")+_IDENTIFIER = re.compile(r"\w+(?:[._]\w+)+")+++def _non_english_segment(state: Union[str, dict, list, None], max_chars: int = 4000):+ """First line or field that, read on its own, is named a non-English language, else None.++ Returns (language, segment). A segment needs the same evidence a whole state does -- at least+ four words, and a language named by `latin_profile` -- so this adds no new way to call English+ text foreign; it only stops a longer English part from outvoting it.+ """+ seen = 0+ for leaf in _iter_text(state):+ for seg in leaf.split("\n"):+ if seen >= max_chars:+ return None+ seen += len(seg)+ if _CODE_LINE.search(seg):+ continue+ prose = _IDENTIFIER.sub(" ", seg)+ if len(_WORD.findall(prose)) < 4:+ continue+ lang = latin_profile(prose)["language"]+ if lang not in (None, "en"):+ return lang, seg.strip()+ return None++def analyse(state: Union[str, dict, list, None]) -> Dict[str, object]:"""Full detection result for a state.Returns `script`, `script_profile`, `language` (best effort, may be None),- `is_english` and `non_latin_fraction`.+ `is_english`, `non_latin_fraction` and `mixed_segment` (the line or field that made a mostly+ English state non-English, else None)."""text = state_text(state)prof = script_profile(text)@@ -398,11 +432,11 @@ def analyse(state: Union[str, dict, list, None]) -> Dict[str, object]:if script == "unknown":return {"script": "unknown", "script_profile": prof, "language": None,"is_english": True, "language_undecided": True, "diacritic_rate": 0.0,- "non_latin_fraction": 0.0}+ "non_latin_fraction": 0.0, "mixed_segment": None}if script != "latin":return {"script": script, "script_profile": prof, "language": None,"is_english": False, "language_undecided": True, "diacritic_rate": 0.0,- "non_latin_fraction": non_latin}+ "non_latin_fraction": non_latin, "mixed_segment": None}prof_lat = latin_profile(text)lang = prof_lat["language"]# Undecided is not English. Treating it as English sent every Latin-script language we hold no@@ -411,10 +445,23 @@ def analyse(state: Union[str, dict, list, None]) -> Dict[str, object]:# such letters (including short English) still goes to the English one.undecided = lang is Noneenglish = lang == "en" or (undecided and not prof_lat["looks_non_english"])+ # A Portuguese ticket with an English stack trace, error payload or form template reads as+ # English as a whole, because the English part is longer -- yet the part a question is about is+ # the customer's, and the English checkpoint cannot read it (0.97 confidence at 0.47 accuracy on+ # `pt`). The cost is lopsided: English sent to multilingual loses a few points, the reverse loses+ # calibration. So a state that would go to English is checked line by line and field by field.+ mixed = None+ leaves = _iter_text(state)+ # a single line has no other part to be outvoted by, and was just read whole+ if english and (len(leaves) > 1 or any("\n" in leaf for leaf in leaves)):+ found = _non_english_segment(state)+ if found:+ lang, mixed = found+ english, undecided = False, Falsereturn {"script": "latin", "script_profile": prof, "language": lang,"is_english": english, "language_undecided": undecided,"diacritic_rate": round(float(prof_lat["diacritic_rate"]), 4),- "non_latin_fraction": non_latin}+ "non_latin_fraction": non_latin, "mixed_segment": mixed}def is_english(state: Union[str, dict, list, None]) -> bool:diff --git a/laya/router.py b/laya/router.pyindex 6becc646..e5f75d87 100644--- a/laya/router.py+++ b/laya/router.py@@ -478,7 +478,10 @@ class Router(HookRegistry):det["script"], 100 * float(det["non_latin_fraction"]))elif not det["is_english"]:key = "multilingual"- if det["language"]:+ if det.get("mixed_segment"):+ reason = ("Latin script, mostly English, but a line or field reads as %r (%r); "+ "the English checkpoint cannot read it" % (det["language"], det["mixed_segment"][:60]))+ elif det["language"]:reason = "Latin script but language looks like %r, not English" % det["language"]else:# Unidentified Latin-script language: routed on the non-English letters alone,diff --git a/tests/test_router.py b/tests/test_router.pyindex 24d65d90..f6b27d53 100644--- a/tests/test_router.py+++ b/tests/test_router.py@@ -93,7 +93,7 @@ check("latin/diacritic rate reported", analyse("Gătește-mi o rețetă de sarmacheck("latin/english has no diacritics", analyse("Please refund the duplicate charge today")["diacritic_rate"], 0.0)# every branch of analyse() reports the same keys, so a caller can read one without guarding_KEYS = {"script", "script_profile", "language", "is_english", "language_undecided",- "diacritic_rate", "non_latin_fraction"}+ "diacritic_rate", "non_latin_fraction", "mixed_segment"}for label, text in [("english", "Please refund the duplicate charge"), ("hindi", "ग्राहक से दो बार"),("romanian", "Gătește-mi o rețetă de sarmale"), ("no letters", "12345 ???")]:check("analyse/keys " + label, set(analyse(text)), _KEYS)@@ -328,6 +328,54 @@ check("route/english reason unchanged",_r_lat.route("Please refund the duplicate charge on invoice 4411 today.").reason, "English Latin text")+# --------------------------------------------------------------------- mixed states+# A Portuguese ticket carrying an English stack trace, error payload or form template read as English+# as a whole -- the English part is longer -- and went to the checkpoint that cannot read the+# customer's own words. Any line or field that on its own is named a non-English language now wins.+_TRACE = ("O sistema caiu de novo hoje de manhã, segue o log:\n"+ "Traceback (most recent call last):\n"+ " File \"/app/main.py\", line 42, in handler\n"+ " return self.process(request)\n"+ "ConnectionError: the connection to the database was refused because the pool is "+ "exhausted and there is no available slot for this request")+for label, state, segment in [+ ("portuguese ticket + english traceback", _TRACE, "O sistema caiu de novo hoje de manhã, segue o log:"),+ ("portuguese field + english error payload",+ {"descricao": "O pagamento não foi processado",+ "error": {"code": "card_declined", "message": "Your card was declined. Please try again with a "+ "different card or contact your bank for more information."}},+ "O pagamento não foi processado"),+ ("english form template + portuguese body",+ {"subject": "New ticket from the web form", "body": "Quero cancelar meu plano"},+ "Quero cancelar meu plano"),+ ("parenthesis in prose is not code",+ {"subject": "Urgent: production is down for all customers since the last deploy",+ "body": "Deu erro (500) no login, alguém pode ver isso agora?"},+ "Deu erro (500) no login, alguém pode ver isso agora?"),+]:+ check("mixed/is not english: " + label, is_english(state), False)+ check("mixed/segment reported: " + label, analyse(state)["mixed_segment"], segment)+ check("mixed/routes multilingual: " + label, _r_lat.route(state).model, "multilingual")+check("mixed/reason names the segment",+ "a line or field reads as 'pt'" in _r_lat.route(_TRACE).reason, True)+# English stays English: several English lines, a short foreign sign-off, and code pasted into a request+# (`os.path` reads as Portuguese, `round(el, 2)` as Spanish, `np.mean(na)` as Portuguese)+for label, state in [+ ("multi-line english", "Hi team,\nThe export failed again last night.\nCan you check the logs?\nThanks"),+ ("short portuguese sign-off", "Please resend the invoice for March, the amount is wrong.\nAtenciosamente, Joao"),+ ("os.path", "The build broke after the refactor.\nREPO = os.path.dirname(os.path.dirname(__file__))\n"+ "Please take a look at the import paths when you can."),+ ("round(el)", "The latency script crashes on large runs.\nmix[key] = {\"total_s\": round(el, 2)}\n"+ "Can you check why the stream is empty?"),+ ("np.mean(na)", "The summary is wrong for empty suites.\nif na: non[m] = round(float(np.mean(na)), 4)\n"+ "Please guard the empty case."),+ ("english json", {"status": "open", "priority": "high",+ "message": "The customer was charged twice and wants a refund"}),+]:+ check("mixed/english stays english: " + label, is_english(state), True)+ check("mixed/no segment: " + label, analyse(state)["mixed_segment"], None)++# --------------------------------------------------------------------- plain-ASCII Romance (#172)# A state that lost its accents carries no diacritic rate for the non-English signal to read, and# mail clients and ticket systems strip them routinely, so the function-word lists are the only