Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

perf(lang): lazy state text traversal and string fast-path

Hamidreza9580d43fafe1
diff --git a/laya/lang.py b/laya/lang.py
index feb9ca41..3a0af646 100644
--- a/laya/lang.py
+++ b/laya/lang.py
@@ -17,7 +17,7 @@ import re
import unicodedata
from collections import Counter
from collections.abc import Mapping
-from typing import Dict, List, Optional, Union
+from typing import Dict, Iterator, List, Optional, Union
# Unicode blocks that the English (ModernBERT-large, 50k English BPE) checkpoint cannot read.
_SCRIPT_RANGES = [
@@ -375,7 +375,7 @@ _IDENTIFIER = re.compile(r"(?<![\w-])[\w-]*(?:[.@][\w-]+)+", re.UNICODE)
def _iter_text(state: Union[str, bytes, Mapping, list, None], _depth: int = 0,
- _paths=None, _path=()) -> List[str]:
+ _paths=None, _path=()) -> Iterator[str]:
"""Collect the string leaves of a state, so detection sees real content.
Keys are ignored: they are usually English field names. Any mapping counts, not only
@@ -383,43 +383,55 @@ def _iter_text(state: Union[str, bytes, Mapping, list, None], _depth: int = 0,
nothing, which made `analyse` report no letters and the router fall through to English.
"""
if _depth > 6 or state is None:
- return []
+ return
if isinstance(state, str):
if _paths is not None:
_paths.append(_path)
- return [state]
+ yield state
+ return
if isinstance(state, (bytes, bytearray)):
try:
text = bytes(state).decode("utf-8")
except UnicodeDecodeError:
- return []
+ return
if _paths is not None:
_paths.append(_path)
- return [text]
+ yield text
+ return
if isinstance(state, Mapping):
- out = []
if _paths is None:
for v in state.values():
- out.extend(_iter_text(v, _depth + 1))
+ yield from _iter_text(v, _depth + 1)
else:
for key, v in state.items():
# Do not stringify arbitrary keys or copy an unbounded field name into a report.
bounded = ((type(key) is str and len(key) <= 128)
or (type(key) is int and key.bit_length() <= 63))
path = _path + (key,) if _path is not None and bounded else None
- out.extend(_iter_text(v, _depth + 1, _paths, path))
- return out
+ yield from _iter_text(v, _depth + 1, _paths, path)
+ return
if isinstance(state, (list, tuple)):
- out = []
for i, v in enumerate(state):
path = _path + (i,) if _paths is not None and _path is not None else _path
- out.extend(_iter_text(v, _depth + 1, _paths, path))
- return out
- return []
+ yield from _iter_text(v, _depth + 1, _paths, path)
+ return
def state_text(state: Union[str, bytes, Mapping, list, None], max_chars: int = 4000) -> str:
"""Flatten a state into the text used for detection (keys are ignored: they are usually English)."""
+ if max_chars <= 0 or state is None:
+ return ""
+ if isinstance(state, str):
+ return state if len(state) <= max_chars else state[:max_chars]
+ if isinstance(state, (bytes, bytearray)):
+ try:
+ text = bytes(state).decode("utf-8")
+ except UnicodeDecodeError:
+ return ""
+ return text if len(text) <= max_chars else text[:max_chars]
+ if not isinstance(state, (Mapping, list, tuple)):
+ return ""
+
parts: List[str] = []
budget = max_chars
for leaf in _iter_text(state):
@@ -1021,9 +1033,10 @@ def analyse(state: Union[str, bytes, Mapping, list, None], *, explain: bool = Fa
# accuracy on `pt`). The cost is lopsided: English sent to multilingual loses a few points,
# the reverse loses calibration. So a state that would go to English is checked line by line
# and field by field.
- leaves = _iter_text(state)
# a single line has no other part to be outvoted by, and was just read whole
- if len(leaves) > 1 or any("\n" in leaf for leaf in leaves):
+ it = iter(_iter_text(state))
+ first = next(it, None)
+ if first is not None and ("\n" in first or next(it, None) is not None):
found = _non_english_segment(state, diagnostic=diagnostic)
if found:
lang, mixed = found
diff --git a/tests/test_router.py b/tests/test_router.py
index 16cf4137..68a5f44f 100644
--- a/tests/test_router.py
+++ b/tests/test_router.py
@@ -175,6 +175,10 @@ check("state_text/dict", "charged twice" in state_text({"body": "charged twice",
check("state_text/nested", "deep" in state_text({"a": {"b": ["deep"]}}), True)
check("state_text/list", "x" in state_text(["x", {"y": "z"}]), True)
check("state_text/none", state_text(None), "")
+check("state_text/plain string", state_text("hello world"), "hello world")
+check("state_text/plain string budget", state_text("hello world", max_chars=5), "hello")
+check("state_text/bytes budget", state_text(b"hello world", max_chars=5), "hello")
+check("state_text/budget truncation halts", state_text(["part1", "part2", "part3"], max_chars=5), "part1")
# keys must not drive detection: English keys around Hindi content stay non-English
check("state_text/keys ignored",
analyse({"subject": "नमस्ते", "body": "ग्राहक से दो बार शुल्क लिया गया"})["is_english"], False)