Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

fix(lang): drop compounds without an open-ended regex, and bound each segment

Paulo Carinhenab9857255a850
diff --git a/laya/lang.py b/laya/lang.py
index 4e483ec8..a7b181a0 100644
--- a/laya/lang.py
+++ b/laya/lang.py
@@ -386,10 +386,13 @@ def guess_latin_language(text: str) -> Optional[str]:
# `=`, `;`, braces, brackets or a call `name(` -- is skipped, and dotted or underscored identifiers
# are dropped from the rest. Prose keeps "Deu erro (500)": the parenthesis follows a space.
_CODE_LINE = re.compile(r"[=;{}\[\]]|\w\(")
-_CODE_IDENTIFIER = re.compile(r"\w+(?:[._]\w+)+")
# Slash and backslash compounds are names, not sentences: `Nav/Com` and `OS/2` read as Portuguese
-# (`com`, `os`), `C:\DOS\mode` as Portuguese (`dos`), `ESA/UN` as Spanish (`un`).
-_SLASH_COMPOUND = re.compile(r"\w+(?:[/\\]\w+)+")
+# (`com`, `os`), `C:\DOS\mode` as Portuguese (`dos`), `ESA/UN` as Spanish (`un`). A whitespace token
+# holding a letter or digit, a joiner (`.`, `_`, `/`, `\`) and another letter or digit is an
+# identifier or a compound and is dropped whole. The pattern has a fixed length on purpose: an
+# open-ended `\w+(?:[._]\w+)+` backtracks quadratically on a long run of letters with no joiner,
+# and a state is user input.
+_JOINED = re.compile(r"[^\W_][._/\\][^\W_]")
# An all-caps token inside mixed-case text is an acronym or a code: `MON`, `LA`, `EST`, `COM`, `DES`
# are hockey teams, states, time zones and radio bands, not French or Portuguese. A segment written
# entirely in capitals keeps its words -- a customer shouting in Portuguese is still Portuguese.
@@ -404,17 +407,18 @@ def _non_english_segment(state: Union[str, dict, list, None], max_chars: int = 4
than a state, two things more: the words that name the language must be two *different* ones
(`COM ... COM` in an English radio listing is one word seen twice), and acronyms and slash
compounds are not words. This adds no new way to call English text foreign; it only stops a
- longer English part from outvoting a foreign one.
+ longer English part from outvoting a foreign one. Reads at most `max_chars` characters in all.
"""
seen = 0
for leaf in _iter_text(state):
for seg in leaf.split("\n"):
if seen >= max_chars:
return None
+ seg = seg[:max_chars - seen]
seen += len(seg)
if _CODE_LINE.search(seg):
continue
- prose = _SLASH_COMPOUND.sub(" ", _CODE_IDENTIFIER.sub(" ", seg))
+ prose = " ".join(tok for tok in seg.split() if not _JOINED.search(tok))
if any(ch.islower() for ch in prose):
prose = _LETTER_RUN.sub(lambda m: " " if m.group().isupper() else m.group(), prose)
tokens = _WORD.findall(prose)
diff --git a/tests/test_router.py b/tests/test_router.py
index 379339d0..9ae91219 100644
--- a/tests/test_router.py
+++ b/tests/test_router.py
@@ -412,6 +412,15 @@ for label, state in [
]:
check("mixed/english stays english: " + label, is_english(state), True)
check("mixed/no segment: " + label, analyse(state)["mixed_segment"], None)
+# a state is user input: one long line with no joiner took 43 s at 40,000 characters when compounds
+# were stripped with an open-ended regex; the segment check now reads at most the 4,000-character cap
+import time as _time
+_t0 = _time.perf_counter()
+analyse({"subject": "The export failed again last night for the whole region", "body": "a" * 200_000})
+check("mixed/long single-line field stays fast", _time.perf_counter() - _t0 < 5.0, True)
+check("mixed/segment check reads at most the cap",
+ analyse({"log": "The export failed again last night for the whole region. " * 80,
+ "body": "Quero cancelar meu plano agora mesmo"})["mixed_segment"], None)
# --------------------------------------------------------------------- plain-ASCII Romance (#172)