Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

email,lang: bound regex and join work before truncation (#253)

Kunj SHAH16074cee6945
diff --git a/laya/email.py b/laya/email.py
index 24b9ea90..d8bf4f4c 100644
--- a/laya/email.py
+++ b/laya/email.py
@@ -151,6 +151,12 @@ def _strip_disclaimer(paragraph: str) -> str:
def clean_email_body(body: str, max_chars: int = 3000) -> str:
"""Remove quoted email history, signatures and disclaimers to keep input focused."""
text = (body or "").replace("\r\n", "\n").replace("\r", "\n").replace("\\n", "\n")
+ # Bound regex work before the expensive patterns below: _DISCLAIMER uses
+ # [^.]{0,60/80/100} alternations whose cost grows with input length, and only
+ # max_chars are ever returned. Truncate lines too so one MB-long line cannot
+ # dominate matching.
+ if len(text) > max_chars * 4:
+ text = text[:max_chars * 4]
lines = []
src = text.split("\n")
for i, line in enumerate(src):
diff --git a/laya/lang.py b/laya/lang.py
index a0e84637..559bb077 100644
--- a/laya/lang.py
+++ b/laya/lang.py
@@ -198,7 +198,18 @@ def _iter_text(state: Union[str, dict, list, None], _depth: int = 0) -> List[str
def state_text(state: Union[str, dict, list, None], max_chars: int = 4000) -> str:
"""Flatten a state into the text used for detection (keys are ignored: they are usually English)."""
- return " ".join(_iter_text(state))[:max_chars]
+ parts: List[str] = []
+ budget = max_chars
+ for leaf in _iter_text(state):
+ if budget <= 0:
+ break
+ if len(leaf) > budget:
+ parts.append(leaf[:budget])
+ break
+ parts.append(leaf)
+ # Account for the joining space without materializing the full text first.
+ budget -= len(leaf) + 1
+ return " ".join(parts)[:max_chars]
def detect_script(text: str) -> str: