Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
perf(lang): optimize unicode script detection with binary search
Hamidreza92c939d1b857
diff --git a/laya/lang.py b/laya/lang.pyindex 66c0b48d..bce1f8bf 100644--- a/laya/lang.py+++ b/laya/lang.py@@ -10,6 +10,7 @@ secondary signal is whether Latin text is English.Script detection is exact. The Latin-script language guess is a stopword/diacritic heuristic andis explicitly best-effort: pass an explicit model or `lang=` when you already know the language."""+import bisectimport reimport unicodedatafrom collections import Counter@@ -45,6 +46,15 @@ _SCRIPT_RANGES = [("han", ((0x3400, 0x4DBF), (0x4E00, 0x9FFF), (0xF900, 0xFAFF))),]+_FLATTENED_RANGES = sorted(+ ((lo, hi, script) for script, ranges in _SCRIPT_RANGES for lo, hi in ranges),+ key=lambda item: item[0],+)+_STARTS = [lo for lo, hi, script in _FLATTENED_RANGES]+_ENDS = [hi for lo, hi, script in _FLATTENED_RANGES]+_SCRIPTS = [script for lo, hi, script in _FLATTENED_RANGES]+_bisect_right = bisect.bisect_right+# Function words. Latin-script languages overlap heavily (de/la/le/un/e/que), so each hit is# weighted and a margin is required before calling something non-English.#@@ -335,10 +345,10 @@ def _script_counts(text: str) -> Dict[str, int]:if cp < 0x02B0 or 0x1E00 <= cp <= 0x1EFF or 0xFF21 <= cp <= 0xFF3A or 0xFF41 <= cp <= 0xFF5A:latin += 1 # Latin, IPA Extensions, Ext-Additional, fullwidthcontinue- for name, ranges in _SCRIPT_RANGES:- if any(lo <= cp <= hi for lo, hi in ranges):- counts[name] = counts.get(name, 0) + 1- break+ idx = _bisect_right(_STARTS, cp) - 1+ if idx >= 0 and cp <= _ENDS[idx]:+ name = _SCRIPTS[idx]+ counts[name] = counts.get(name, 0) + 1else:# An alphabetic character no range claims used to be counted nowhere, so text# written only in an unlisted script produced a total of 0 and was reported as@@ -416,9 +426,9 @@ def _script_of(ch: str) -> Optional[str]:cp = ord(ch)if cp < 0x0250 or 0x1E00 <= cp <= 0x1EFF or 0xFF21 <= cp <= 0xFF3A or 0xFF41 <= cp <= 0xFF5A:return None- for name, ranges in _SCRIPT_RANGES:- if any(lo <= cp <= hi for lo, hi in ranges):- return name+ idx = _bisect_right(_STARTS, cp) - 1+ if idx >= 0 and cp <= _ENDS[idx]:+ return _SCRIPTS[idx]return None