Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

fix(lang): a segment needs two different words, and acronyms and slash compounds are not words

Paulo Carinhenab1bb161ecb0f
diff --git a/laya/lang.py b/laya/lang.py
index f7995e30..4e483ec8 100644
--- a/laya/lang.py
+++ b/laya/lang.py
@@ -386,15 +386,25 @@ def guess_latin_language(text: str) -> Optional[str]:
# `=`, `;`, braces, brackets or a call `name(` -- is skipped, and dotted or underscored identifiers
# are dropped from the rest. Prose keeps "Deu erro (500)": the parenthesis follows a space.
_CODE_LINE = re.compile(r"[=;{}\[\]]|\w\(")
-_IDENTIFIER = re.compile(r"\w+(?:[._]\w+)+")
+_CODE_IDENTIFIER = re.compile(r"\w+(?:[._]\w+)+")
+# Slash and backslash compounds are names, not sentences: `Nav/Com` and `OS/2` read as Portuguese
+# (`com`, `os`), `C:\DOS\mode` as Portuguese (`dos`), `ESA/UN` as Spanish (`un`).
+_SLASH_COMPOUND = re.compile(r"\w+(?:[/\\]\w+)+")
+# An all-caps token inside mixed-case text is an acronym or a code: `MON`, `LA`, `EST`, `COM`, `DES`
+# are hockey teams, states, time zones and radio bands, not French or Portuguese. A segment written
+# entirely in capitals keeps its words -- a customer shouting in Portuguese is still Portuguese.
+_LETTER_RUN = re.compile(r"[^\W\d_]{2,}")
def _non_english_segment(state: Union[str, dict, list, None], max_chars: int = 4000):
"""First line or field that, read on its own, is named a non-English language, else None.
- Returns (language, segment). A segment needs the same evidence a whole state does -- at least
- four words, and a language named by `latin_profile` -- so this adds no new way to call English
- text foreign; it only stops a longer English part from outvoting it.
+ Returns (language, segment). A segment needs the evidence a whole state needs -- at least four
+ words, and a language named by `latin_profile` -- and, because one line carries far less text
+ than a state, two things more: the words that name the language must be two *different* ones
+ (`COM ... COM` in an English radio listing is one word seen twice), and acronyms and slash
+ compounds are not words. This adds no new way to call English text foreign; it only stops a
+ longer English part from outvoting a foreign one.
"""
seen = 0
for leaf in _iter_text(state):
@@ -404,11 +414,14 @@ def _non_english_segment(state: Union[str, dict, list, None], max_chars: int = 4
seen += len(seg)
if _CODE_LINE.search(seg):
continue
- prose = _IDENTIFIER.sub(" ", seg)
- if len(_WORD.findall(prose)) < 4:
+ prose = _SLASH_COMPOUND.sub(" ", _CODE_IDENTIFIER.sub(" ", seg))
+ if any(ch.islower() for ch in prose):
+ prose = _LETTER_RUN.sub(lambda m: " " if m.group().isupper() else m.group(), prose)
+ tokens = _WORD.findall(prose)
+ if len(tokens) < 4:
continue
lang = latin_profile(prose)["language"]
- if lang not in (None, "en"):
+ if lang not in (None, "en") and len({w.lower() for w in tokens} & _STOP.get(lang, set())) >= 2:
return lang, seg.strip()
return None
diff --git a/tests/test_router.py b/tests/test_router.py
index 69b44da1..379339d0 100644
--- a/tests/test_router.py
+++ b/tests/test_router.py
@@ -353,6 +353,31 @@ for label, state, segment in [
"status page is red for the whole region",
"body": "Deu erro (500) no login, alguém pode ver isso agora?"},
"Deu erro (500) no login, alguém pode ver isso agora?"),
+ # the rule runs both ways: an English ticket that pastes a foreign log goes to multilingual too
+ ("english ticket + portuguese error log",
+ "Our Brazilian branch cannot issue invoices since this morning. The system shows this message:\n"
+ "ERRO: Não foi possível emitir a nota fiscal, o certificado digital está vencido\n"
+ "Can you help us before the end of the day?",
+ "ERRO: Não foi possível emitir a nota fiscal, o certificado digital está vencido"),
+ ("english ticket + german error log",
+ "The nightly sync to the Munich server keeps failing and we lose the whole batch.\n"
+ "Fehler: Die Verbindung zum Server wurde unterbrochen, bitte versuchen Sie es spaeter noch einmal\n"
+ "Please check the firewall rules on your side.",
+ "Fehler: Die Verbindung zum Server wurde unterbrochen, bitte versuchen Sie es spaeter noch einmal"),
+ ("english ticket + spanish error payload",
+ {"subject": "Payment failed for a customer in Madrid",
+ "description": "The customer tried three times with the same card and each attempt was declined by "
+ "the gateway, so we would like to know whether the problem is on our side or with the bank.",
+ "error": {"code": "card_declined",
+ "message": "La tarjeta fue rechazada por el banco emisor, contacte con su banco"}},
+ "La tarjeta fue rechazada por el banco emisor, contacte con su banco"),
+ # acronyms are dropped only from mixed-case text: a line written all in capitals keeps its words
+ ("all-caps portuguese line",
+ "This is the fourth email I have sent about the same order and nobody has answered any of them.\n"
+ "The customer wrote this in the chat and then closed the window:\n"
+ "QUERO MEU DINHEIRO DE VOLTA AGORA\n"
+ "Could someone from the billing team look at order 5512 today?",
+ "QUERO MEU DINHEIRO DE VOLTA AGORA"),
]:
check("mixed/is not english: " + label, is_english(state), False)
check("mixed/segment reported: " + label, analyse(state)["mixed_segment"], segment)
@@ -372,6 +397,18 @@ for label, state in [
"Please guard the empty case."),
("english json", {"status": "open", "priority": "high",
"message": "The customer was charged twice and wants a refund"}),
+ # a line carries far less text than a state, so its evidence must be two different words and no
+ # acronyms or slash compounds. A ham-radio listing on 20 Newsgroups (misc.forsale/76512) went to
+ # multilingual on `COM ... COM` alone; hockey picks on the team codes, OS/2 on `os` and `dos`.
+ ("same word twice (Nav/Com, COM)",
+ "I'm looking for good deals on the following (used or new):\nAviation Headsets (with mic).\n"
+ "Handheld Nav/Com tranciever (may consider COM only).\nPortable GPS or Loran Navigator."),
+ ("team codes", "Round two predictions for the pool, as promised.\nQUE vs MON: MON in 7.\n"
+ "PIT vs NYI: PIT in 5."),
+ ("slash compound", "I need a converter for these image formats.\n"
+ "DOS, OS/2 or platform independent programs if possible.\nThanks in advance."),
+ ("backslash path", "My modem stopped answering after the upgrade.\nC:\\DOS\\mode COM1:9600,n,8,1,p\n"
+ "Is that the right line for a 9600 baud connection?"),
]:
check("mixed/english stays english: " + label, is_english(state), True)
check("mixed/no segment: " + label, analyse(state)["mixed_segment"], None)