Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
fix(lang): a segment needs two different words, and acronyms and slash compounds are not words
Paulo Carinhenab1bb161ecb0f
diff --git a/laya/lang.py b/laya/lang.pyindex f7995e30..4e483ec8 100644--- a/laya/lang.py+++ b/laya/lang.py@@ -386,15 +386,25 @@ def guess_latin_language(text: str) -> Optional[str]:# `=`, `;`, braces, brackets or a call `name(` -- is skipped, and dotted or underscored identifiers# are dropped from the rest. Prose keeps "Deu erro (500)": the parenthesis follows a space._CODE_LINE = re.compile(r"[=;{}\[\]]|\w\(")-_IDENTIFIER = re.compile(r"\w+(?:[._]\w+)+")+_CODE_IDENTIFIER = re.compile(r"\w+(?:[._]\w+)+")+# Slash and backslash compounds are names, not sentences: `Nav/Com` and `OS/2` read as Portuguese+# (`com`, `os`), `C:\DOS\mode` as Portuguese (`dos`), `ESA/UN` as Spanish (`un`).+_SLASH_COMPOUND = re.compile(r"\w+(?:[/\\]\w+)+")+# An all-caps token inside mixed-case text is an acronym or a code: `MON`, `LA`, `EST`, `COM`, `DES`+# are hockey teams, states, time zones and radio bands, not French or Portuguese. A segment written+# entirely in capitals keeps its words -- a customer shouting in Portuguese is still Portuguese.+_LETTER_RUN = re.compile(r"[^\W\d_]{2,}")def _non_english_segment(state: Union[str, dict, list, None], max_chars: int = 4000):"""First line or field that, read on its own, is named a non-English language, else None.- Returns (language, segment). A segment needs the same evidence a whole state does -- at least- four words, and a language named by `latin_profile` -- so this adds no new way to call English- text foreign; it only stops a longer English part from outvoting it.+ Returns (language, segment). A segment needs the evidence a whole state needs -- at least four+ words, and a language named by `latin_profile` -- and, because one line carries far less text+ than a state, two things more: the words that name the language must be two *different* ones+ (`COM ... COM` in an English radio listing is one word seen twice), and acronyms and slash+ compounds are not words. This adds no new way to call English text foreign; it only stops a+ longer English part from outvoting a foreign one."""seen = 0for leaf in _iter_text(state):@@ -404,11 +414,14 @@ def _non_english_segment(state: Union[str, dict, list, None], max_chars: int = 4seen += len(seg)if _CODE_LINE.search(seg):continue- prose = _IDENTIFIER.sub(" ", seg)- if len(_WORD.findall(prose)) < 4:+ prose = _SLASH_COMPOUND.sub(" ", _CODE_IDENTIFIER.sub(" ", seg))+ if any(ch.islower() for ch in prose):+ prose = _LETTER_RUN.sub(lambda m: " " if m.group().isupper() else m.group(), prose)+ tokens = _WORD.findall(prose)+ if len(tokens) < 4:continuelang = latin_profile(prose)["language"]- if lang not in (None, "en"):+ if lang not in (None, "en") and len({w.lower() for w in tokens} & _STOP.get(lang, set())) >= 2:return lang, seg.strip()return Nonediff --git a/tests/test_router.py b/tests/test_router.pyindex 69b44da1..379339d0 100644--- a/tests/test_router.py+++ b/tests/test_router.py@@ -353,6 +353,31 @@ for label, state, segment in ["status page is red for the whole region","body": "Deu erro (500) no login, alguém pode ver isso agora?"},"Deu erro (500) no login, alguém pode ver isso agora?"),+ # the rule runs both ways: an English ticket that pastes a foreign log goes to multilingual too+ ("english ticket + portuguese error log",+ "Our Brazilian branch cannot issue invoices since this morning. The system shows this message:\n"+ "ERRO: Não foi possível emitir a nota fiscal, o certificado digital está vencido\n"+ "Can you help us before the end of the day?",+ "ERRO: Não foi possível emitir a nota fiscal, o certificado digital está vencido"),+ ("english ticket + german error log",+ "The nightly sync to the Munich server keeps failing and we lose the whole batch.\n"+ "Fehler: Die Verbindung zum Server wurde unterbrochen, bitte versuchen Sie es spaeter noch einmal\n"+ "Please check the firewall rules on your side.",+ "Fehler: Die Verbindung zum Server wurde unterbrochen, bitte versuchen Sie es spaeter noch einmal"),+ ("english ticket + spanish error payload",+ {"subject": "Payment failed for a customer in Madrid",+ "description": "The customer tried three times with the same card and each attempt was declined by "+ "the gateway, so we would like to know whether the problem is on our side or with the bank.",+ "error": {"code": "card_declined",+ "message": "La tarjeta fue rechazada por el banco emisor, contacte con su banco"}},+ "La tarjeta fue rechazada por el banco emisor, contacte con su banco"),+ # acronyms are dropped only from mixed-case text: a line written all in capitals keeps its words+ ("all-caps portuguese line",+ "This is the fourth email I have sent about the same order and nobody has answered any of them.\n"+ "The customer wrote this in the chat and then closed the window:\n"+ "QUERO MEU DINHEIRO DE VOLTA AGORA\n"+ "Could someone from the billing team look at order 5512 today?",+ "QUERO MEU DINHEIRO DE VOLTA AGORA"),]:check("mixed/is not english: " + label, is_english(state), False)check("mixed/segment reported: " + label, analyse(state)["mixed_segment"], segment)@@ -372,6 +397,18 @@ for label, state in ["Please guard the empty case."),("english json", {"status": "open", "priority": "high","message": "The customer was charged twice and wants a refund"}),+ # a line carries far less text than a state, so its evidence must be two different words and no+ # acronyms or slash compounds. A ham-radio listing on 20 Newsgroups (misc.forsale/76512) went to+ # multilingual on `COM ... COM` alone; hockey picks on the team codes, OS/2 on `os` and `dos`.+ ("same word twice (Nav/Com, COM)",+ "I'm looking for good deals on the following (used or new):\nAviation Headsets (with mic).\n"+ "Handheld Nav/Com tranciever (may consider COM only).\nPortable GPS or Loran Navigator."),+ ("team codes", "Round two predictions for the pool, as promised.\nQUE vs MON: MON in 7.\n"+ "PIT vs NYI: PIT in 5."),+ ("slash compound", "I need a converter for these image formats.\n"+ "DOS, OS/2 or platform independent programs if possible.\nThanks in advance."),+ ("backslash path", "My modem stopped answering after the upgrade.\nC:\\DOS\\mode COM1:9600,n,8,1,p\n"+ "Is that the right line for a 9600 baud connection?"),]:check("mixed/english stays english: " + label, is_english(state), True)check("mixed/no segment: " + label, analyse(state)["mixed_segment"], None)