Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
perf(lang): lazy state text traversal and string fast-path
Hamidreza9580d43fafe1
diff --git a/laya/lang.py b/laya/lang.pyindex feb9ca41..3a0af646 100644--- a/laya/lang.py+++ b/laya/lang.py@@ -17,7 +17,7 @@ import reimport unicodedatafrom collections import Counterfrom collections.abc import Mapping-from typing import Dict, List, Optional, Union+from typing import Dict, Iterator, List, Optional, Union# Unicode blocks that the English (ModernBERT-large, 50k English BPE) checkpoint cannot read._SCRIPT_RANGES = [@@ -375,7 +375,7 @@ _IDENTIFIER = re.compile(r"(?<![\w-])[\w-]*(?:[.@][\w-]+)+", re.UNICODE)def _iter_text(state: Union[str, bytes, Mapping, list, None], _depth: int = 0,- _paths=None, _path=()) -> List[str]:+ _paths=None, _path=()) -> Iterator[str]:"""Collect the string leaves of a state, so detection sees real content.Keys are ignored: they are usually English field names. Any mapping counts, not only@@ -383,43 +383,55 @@ def _iter_text(state: Union[str, bytes, Mapping, list, None], _depth: int = 0,nothing, which made `analyse` report no letters and the router fall through to English."""if _depth > 6 or state is None:- return []+ returnif isinstance(state, str):if _paths is not None:_paths.append(_path)- return [state]+ yield state+ returnif isinstance(state, (bytes, bytearray)):try:text = bytes(state).decode("utf-8")except UnicodeDecodeError:- return []+ returnif _paths is not None:_paths.append(_path)- return [text]+ yield text+ returnif isinstance(state, Mapping):- out = []if _paths is None:for v in state.values():- out.extend(_iter_text(v, _depth + 1))+ yield from _iter_text(v, _depth + 1)else:for key, v in state.items():# Do not stringify arbitrary keys or copy an unbounded field name into a report.bounded = ((type(key) is str and len(key) <= 128)or (type(key) is int and key.bit_length() <= 63))path = _path + (key,) if _path is not None and bounded else None- out.extend(_iter_text(v, _depth + 1, _paths, path))- return out+ yield from _iter_text(v, _depth + 1, _paths, path)+ returnif isinstance(state, (list, tuple)):- out = []for i, v in enumerate(state):path = _path + (i,) if _paths is not None and _path is not None else _path- out.extend(_iter_text(v, _depth + 1, _paths, path))- return out- return []+ yield from _iter_text(v, _depth + 1, _paths, path)+ returndef state_text(state: Union[str, bytes, Mapping, list, None], max_chars: int = 4000) -> str:"""Flatten a state into the text used for detection (keys are ignored: they are usually English)."""+ if max_chars <= 0 or state is None:+ return ""+ if isinstance(state, str):+ return state if len(state) <= max_chars else state[:max_chars]+ if isinstance(state, (bytes, bytearray)):+ try:+ text = bytes(state).decode("utf-8")+ except UnicodeDecodeError:+ return ""+ return text if len(text) <= max_chars else text[:max_chars]+ if not isinstance(state, (Mapping, list, tuple)):+ return ""+parts: List[str] = []budget = max_charsfor leaf in _iter_text(state):@@ -1021,9 +1033,10 @@ def analyse(state: Union[str, bytes, Mapping, list, None], *, explain: bool = Fa# accuracy on `pt`). The cost is lopsided: English sent to multilingual loses a few points,# the reverse loses calibration. So a state that would go to English is checked line by line# and field by field.- leaves = _iter_text(state)# a single line has no other part to be outvoted by, and was just read whole- if len(leaves) > 1 or any("\n" in leaf for leaf in leaves):+ it = iter(_iter_text(state))+ first = next(it, None)+ if first is not None and ("\n" in first or next(it, None) is not None):found = _non_english_segment(state, diagnostic=diagnostic)if found:lang, mixed = founddiff --git a/tests/test_router.py b/tests/test_router.pyindex 16cf4137..68a5f44f 100644--- a/tests/test_router.py+++ b/tests/test_router.py@@ -175,6 +175,10 @@ check("state_text/dict", "charged twice" in state_text({"body": "charged twice",check("state_text/nested", "deep" in state_text({"a": {"b": ["deep"]}}), True)check("state_text/list", "x" in state_text(["x", {"y": "z"}]), True)check("state_text/none", state_text(None), "")+check("state_text/plain string", state_text("hello world"), "hello world")+check("state_text/plain string budget", state_text("hello world", max_chars=5), "hello")+check("state_text/bytes budget", state_text(b"hello world", max_chars=5), "hello")+check("state_text/budget truncation halts", state_text(["part1", "part2", "part3"], max_chars=5), "part1")# keys must not drive detection: English keys around Hindi content stay non-Englishcheck("state_text/keys ignored",analyse({"subject": "नमस्ते", "body": "ग्राहक से दो बार शुल्क लिया गया"})["is_english"], False)