Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

fix(lang): route a Portuguese ticket with an English log to multilingual

Paulo Carinhena952e125de5e7
diff --git a/laya/lang.py b/laya/lang.py
index 0122c787..f7995e30 100644
--- a/laya/lang.py
+++ b/laya/lang.py
@@ -380,11 +380,45 @@ def guess_latin_language(text: str) -> Optional[str]:
return latin_profile(text)["language"]
+# Code is not prose in any language, but split into words it reads as one: `os.path` is Portuguese
+# (`os`), `round(el, 2)` Spanish (`el`), `non_english` Italian (`non`). A line pasted from a program
+# into an English request must not count as a foreign segment, so a line carrying code syntax --
+# `=`, `;`, braces, brackets or a call `name(` -- is skipped, and dotted or underscored identifiers
+# are dropped from the rest. Prose keeps "Deu erro (500)": the parenthesis follows a space.
+_CODE_LINE = re.compile(r"[=;{}\[\]]|\w\(")
+_IDENTIFIER = re.compile(r"\w+(?:[._]\w+)+")
+
+
+def _non_english_segment(state: Union[str, dict, list, None], max_chars: int = 4000):
+ """First line or field that, read on its own, is named a non-English language, else None.
+
+ Returns (language, segment). A segment needs the same evidence a whole state does -- at least
+ four words, and a language named by `latin_profile` -- so this adds no new way to call English
+ text foreign; it only stops a longer English part from outvoting it.
+ """
+ seen = 0
+ for leaf in _iter_text(state):
+ for seg in leaf.split("\n"):
+ if seen >= max_chars:
+ return None
+ seen += len(seg)
+ if _CODE_LINE.search(seg):
+ continue
+ prose = _IDENTIFIER.sub(" ", seg)
+ if len(_WORD.findall(prose)) < 4:
+ continue
+ lang = latin_profile(prose)["language"]
+ if lang not in (None, "en"):
+ return lang, seg.strip()
+ return None
+
+
def analyse(state: Union[str, dict, list, None]) -> Dict[str, object]:
"""Full detection result for a state.
Returns `script`, `script_profile`, `language` (best effort, may be None),
- `is_english` and `non_latin_fraction`.
+ `is_english`, `non_latin_fraction` and `mixed_segment` (the line or field that made a mostly
+ English state non-English, else None).
"""
text = state_text(state)
prof = script_profile(text)
@@ -398,11 +432,11 @@ def analyse(state: Union[str, dict, list, None]) -> Dict[str, object]:
if script == "unknown":
return {"script": "unknown", "script_profile": prof, "language": None,
"is_english": True, "language_undecided": True, "diacritic_rate": 0.0,
- "non_latin_fraction": 0.0}
+ "non_latin_fraction": 0.0, "mixed_segment": None}
if script != "latin":
return {"script": script, "script_profile": prof, "language": None,
"is_english": False, "language_undecided": True, "diacritic_rate": 0.0,
- "non_latin_fraction": non_latin}
+ "non_latin_fraction": non_latin, "mixed_segment": None}
prof_lat = latin_profile(text)
lang = prof_lat["language"]
# Undecided is not English. Treating it as English sent every Latin-script language we hold no
@@ -411,10 +445,23 @@ def analyse(state: Union[str, dict, list, None]) -> Dict[str, object]:
# such letters (including short English) still goes to the English one.
undecided = lang is None
english = lang == "en" or (undecided and not prof_lat["looks_non_english"])
+ # A Portuguese ticket with an English stack trace, error payload or form template reads as
+ # English as a whole, because the English part is longer -- yet the part a question is about is
+ # the customer's, and the English checkpoint cannot read it (0.97 confidence at 0.47 accuracy on
+ # `pt`). The cost is lopsided: English sent to multilingual loses a few points, the reverse loses
+ # calibration. So a state that would go to English is checked line by line and field by field.
+ mixed = None
+ leaves = _iter_text(state)
+ # a single line has no other part to be outvoted by, and was just read whole
+ if english and (len(leaves) > 1 or any("\n" in leaf for leaf in leaves)):
+ found = _non_english_segment(state)
+ if found:
+ lang, mixed = found
+ english, undecided = False, False
return {"script": "latin", "script_profile": prof, "language": lang,
"is_english": english, "language_undecided": undecided,
"diacritic_rate": round(float(prof_lat["diacritic_rate"]), 4),
- "non_latin_fraction": non_latin}
+ "non_latin_fraction": non_latin, "mixed_segment": mixed}
def is_english(state: Union[str, dict, list, None]) -> bool:
diff --git a/laya/router.py b/laya/router.py
index 6becc646..e5f75d87 100644
--- a/laya/router.py
+++ b/laya/router.py
@@ -478,7 +478,10 @@ class Router(HookRegistry):
det["script"], 100 * float(det["non_latin_fraction"]))
elif not det["is_english"]:
key = "multilingual"
- if det["language"]:
+ if det.get("mixed_segment"):
+ reason = ("Latin script, mostly English, but a line or field reads as %r (%r); "
+ "the English checkpoint cannot read it" % (det["language"], det["mixed_segment"][:60]))
+ elif det["language"]:
reason = "Latin script but language looks like %r, not English" % det["language"]
else:
# Unidentified Latin-script language: routed on the non-English letters alone,
diff --git a/tests/test_router.py b/tests/test_router.py
index 24d65d90..f6b27d53 100644
--- a/tests/test_router.py
+++ b/tests/test_router.py
@@ -93,7 +93,7 @@ check("latin/diacritic rate reported", analyse("Gătește-mi o rețetă de sarma
check("latin/english has no diacritics", analyse("Please refund the duplicate charge today")["diacritic_rate"], 0.0)
# every branch of analyse() reports the same keys, so a caller can read one without guarding
_KEYS = {"script", "script_profile", "language", "is_english", "language_undecided",
- "diacritic_rate", "non_latin_fraction"}
+ "diacritic_rate", "non_latin_fraction", "mixed_segment"}
for label, text in [("english", "Please refund the duplicate charge"), ("hindi", "ग्राहक से दो बार"),
("romanian", "Gătește-mi o rețetă de sarmale"), ("no letters", "12345 ???")]:
check("analyse/keys " + label, set(analyse(text)), _KEYS)
@@ -328,6 +328,54 @@ check("route/english reason unchanged",
_r_lat.route("Please refund the duplicate charge on invoice 4411 today.").reason, "English Latin text")
+# --------------------------------------------------------------------- mixed states
+# A Portuguese ticket carrying an English stack trace, error payload or form template read as English
+# as a whole -- the English part is longer -- and went to the checkpoint that cannot read the
+# customer's own words. Any line or field that on its own is named a non-English language now wins.
+_TRACE = ("O sistema caiu de novo hoje de manhã, segue o log:\n"
+ "Traceback (most recent call last):\n"
+ " File \"/app/main.py\", line 42, in handler\n"
+ " return self.process(request)\n"
+ "ConnectionError: the connection to the database was refused because the pool is "
+ "exhausted and there is no available slot for this request")
+for label, state, segment in [
+ ("portuguese ticket + english traceback", _TRACE, "O sistema caiu de novo hoje de manhã, segue o log:"),
+ ("portuguese field + english error payload",
+ {"descricao": "O pagamento não foi processado",
+ "error": {"code": "card_declined", "message": "Your card was declined. Please try again with a "
+ "different card or contact your bank for more information."}},
+ "O pagamento não foi processado"),
+ ("english form template + portuguese body",
+ {"subject": "New ticket from the web form", "body": "Quero cancelar meu plano"},
+ "Quero cancelar meu plano"),
+ ("parenthesis in prose is not code",
+ {"subject": "Urgent: production is down for all customers since the last deploy",
+ "body": "Deu erro (500) no login, alguém pode ver isso agora?"},
+ "Deu erro (500) no login, alguém pode ver isso agora?"),
+]:
+ check("mixed/is not english: " + label, is_english(state), False)
+ check("mixed/segment reported: " + label, analyse(state)["mixed_segment"], segment)
+ check("mixed/routes multilingual: " + label, _r_lat.route(state).model, "multilingual")
+check("mixed/reason names the segment",
+ "a line or field reads as 'pt'" in _r_lat.route(_TRACE).reason, True)
+# English stays English: several English lines, a short foreign sign-off, and code pasted into a request
+# (`os.path` reads as Portuguese, `round(el, 2)` as Spanish, `np.mean(na)` as Portuguese)
+for label, state in [
+ ("multi-line english", "Hi team,\nThe export failed again last night.\nCan you check the logs?\nThanks"),
+ ("short portuguese sign-off", "Please resend the invoice for March, the amount is wrong.\nAtenciosamente, Joao"),
+ ("os.path", "The build broke after the refactor.\nREPO = os.path.dirname(os.path.dirname(__file__))\n"
+ "Please take a look at the import paths when you can."),
+ ("round(el)", "The latency script crashes on large runs.\nmix[key] = {\"total_s\": round(el, 2)}\n"
+ "Can you check why the stream is empty?"),
+ ("np.mean(na)", "The summary is wrong for empty suites.\nif na: non[m] = round(float(np.mean(na)), 4)\n"
+ "Please guard the empty case."),
+ ("english json", {"status": "open", "priority": "high",
+ "message": "The customer was charged twice and wants a refund"}),
+]:
+ check("mixed/english stays english: " + label, is_english(state), True)
+ check("mixed/no segment: " + label, analyse(state)["mixed_segment"], None)
+
+
# --------------------------------------------------------------------- plain-ASCII Romance (#172)
# A state that lost its accents carries no diacritic rate for the non-English signal to read, and
# mail clients and ticket systems strip them routinely, so the function-word lists are the only