Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
perf: single-pass script stats; wire test_lang_stats
Emiliano Gandini Outeda232e750d7801
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.ymlindex ce48ff4f..c32d9dff 100644--- a/.github/workflows/ci.yml+++ b/.github/workflows/ci.yml@@ -96,6 +96,7 @@ jobs:python tests/test_lazy_import.pypython tests/test_runtime_fixes.pypython tests/test_lang_guess.py+ python tests/test_lang_stats.pypython tests/test_calibration_persistence.pypython tests/test_context_manager.pypython tests/test_criteria_normalization.pydiff --git a/laya/agent.py b/laya/agent.pyindex b0fb78a7..bf8d2785 100644--- a/laya/agent.py+++ b/laya/agent.py@@ -903,6 +903,9 @@ class Agent(HookRegistry):return _decide(self, state, schema, questions=questions,return_details=return_details, **predict_kwargs)+ def __repr__(self) -> str:+ return "Agent(model_id=%r, device=%s)" % (self.model_id, getattr(self, "device", None))+predict = system_onediff --git a/laya/lang.py b/laya/lang.pyindex 0122c787..421fa194 100644--- a/laya/lang.py+++ b/laya/lang.py@@ -218,8 +218,14 @@ def state_text(state: Union[str, dict, list, None], max_chars: int = 4000) -> streturn " ".join(parts)[:max_chars]-def detect_script(text: str) -> str:- """Dominant script of `text`: 'latin', 'han', 'devanagari', ... or 'unknown' if there are no letters."""+def _script_counts(text: str) -> Dict[str, int]:+ """Count the alphabetic characters of `text` by script, in one pass.++ Latin is inserted last so `_script_from_counts` keeps `detect_script`'s tie-break: a named+ script wins a tie against Latin, because `max` returns the first of equal values and Latin+ is the last key. `analyse` needs both the dominant script and the per-script fractions, and+ used to walk the text twice (once per function) to get them; one pass serves both.+ """counts: Dict[str, int] = {}latin = 0for ch in text:@@ -245,32 +251,37 @@ def detect_script(text: str) -> str:# handed to the English checkpoint.counts["other"] = counts.get("other", 0) + 1counts["latin"] = latin- total = sum(counts.values())- if total == 0:+ return counts+++def _script_from_counts(counts: Dict[str, int]) -> str:+ if not any(counts.values()):return "unknown"return max(counts.items(), key=lambda kv: kv[1])[0]-def script_profile(text: str) -> Dict[str, float]:- """Fraction of alphabetic characters belonging to each detected script."""- counts: Dict[str, int] = {"latin": 0}- for ch in text:- if not ch.isalpha():- continue- cp = ord(ch)- if cp < 0x02B0 or 0x1E00 <= cp <= 0x1EFF or 0xFF21 <= cp <= 0xFF3A or 0xFF41 <= cp <= 0xFF5A:- counts["latin"] += 1- continue- for name, ranges in _SCRIPT_RANGES:- if any(lo <= cp <= hi for lo, hi in ranges):- counts[name] = counts.get(name, 0) + 1- break- else:- counts["other"] = counts.get("other", 0) + 1+def _profile_from_counts(counts: Dict[str, int]) -> Dict[str, float]:total = sum(counts.values())if not total:return {}- return {k: v / total for k, v in counts.items() if v}+ # Keep Latin first in the public mapping, the order callers saw before this was refactored.+ ordered: Dict[str, int] = {}+ if counts.get("latin"):+ ordered["latin"] = counts["latin"]+ for name, value in counts.items():+ if name != "latin":+ ordered[name] = value+ return {k: v / total for k, v in ordered.items() if v}+++def detect_script(text: str) -> str:+ """Dominant script of `text`: 'latin', 'han', 'devanagari', ... or 'unknown' if there are no letters."""+ return _script_from_counts(_script_counts(text))+++def script_profile(text: str) -> Dict[str, float]:+ """Fraction of alphabetic characters belonging to each detected script."""+ return _profile_from_counts(_script_counts(text))# A diacritic rate above this is taken as evidence the text is not English, even when no@@ -387,8 +398,9 @@ def analyse(state: Union[str, dict, list, None]) -> Dict[str, object]:`is_english` and `non_latin_fraction`."""text = state_text(state)- prof = script_profile(text)- script = detect_script(text)+ counts = _script_counts(text)+ prof = _profile_from_counts(counts)+ script = _script_from_counts(counts)non_latin = round(1.0 - prof.get("latin", 0.0), 4) if prof else 0.0n_non_latin = round(non_latin * sum(ch.isalpha() for ch in text))if script == "latin" and _non_latin_words(text) and (diff --git a/tests/test_lang_stats.py b/tests/test_lang_stats.pynew file mode 100644index 00000000..496f656b--- /dev/null+++ b/tests/test_lang_stats.py@@ -0,0 +1,117 @@+"""`detect_script` / `script_profile` must be unchanged by the single-pass refactor in #lang.++The reference implementations below are the pre-refactor versions; the new ones share one pass+over the text, so these checks pin that the counts, the fractions and the tie-break are the same.++Run: python tests/test_lang_stats.py+"""+import os+import sys++sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))++from laya.lang import _SCRIPT_RANGES, analyse, detect_script, script_profile # noqa: E402++PASS, FAIL = [], []+++def check(name, got, want):+ if got == want:+ PASS.append(name)+ else:+ FAIL.append("%s: got %r, want %r" % (name, got, want))+++def _is_latin_cp(cp):+ return cp < 0x02B0 or 0x1E00 <= cp <= 0x1EFF or 0xFF21 <= cp <= 0xFF3A or 0xFF41 <= cp <= 0xFF5A+++def ref_counts(text):+ counts = {}+ latin = 0+ for ch in text:+ if not ch.isalpha():+ continue+ cp = ord(ch)+ if _is_latin_cp(cp):+ latin += 1+ continue+ for name, ranges in _SCRIPT_RANGES:+ if any(lo <= cp <= hi for lo, hi in ranges):+ counts[name] = counts.get(name, 0) + 1+ break+ else:+ counts["other"] = counts.get("other", 0) + 1+ counts["latin"] = latin+ return counts+++def ref_detect(text):+ counts = ref_counts(text)+ if sum(counts.values()) == 0:+ return "unknown"+ return max(counts.items(), key=lambda kv: kv[1])[0]+++def ref_profile(text):+ counts = {"latin": 0}+ for ch in text:+ if not ch.isalpha():+ continue+ cp = ord(ch)+ if _is_latin_cp(cp):+ counts["latin"] += 1+ continue+ for name, ranges in _SCRIPT_RANGES:+ if any(lo <= cp <= hi for lo, hi in ranges):+ counts[name] = counts.get(name, 0) + 1+ break+ else:+ counts["other"] = counts.get("other", 0) + 1+ total = sum(counts.values())+ if not total:+ return {}+ return {k: v / total for k, v in counts.items() if v}+++TEXTS = [+ "",+ " ",+ "hello world",+ "hello мир",+ "мир",+ "世界",+ "日本語です",+ "ΑΒΓΔ",+ "abc ᚠᚢᚦ", # an unlisted script lands under "other"+ "ab αβ", # latin/greek tie+ "αβ аб", # greek/cyrillic tie+ "ᚠᚢᚦabc", # other/latin tie+ "Hallo, meine Bestellung ist zweimal abgebucht worden.",+ "مرحبا كيف حالك",+ "İstanbul",+]++for text in TEXTS:+ check("detect_script(%r)" % text, detect_script(text), ref_detect(text))+ check("script_profile(%r)" % text, script_profile(text), ref_profile(text))++# analyse uses the same counts, so its profile must match too, and it must not regress on a+# non-Latin override (a lowercase non-Latin run flips the dominant script).+for text in ("hello мир", "мир", "مرحبا كيف حالك", "hello world"):+ check("analyse.profile(%r)" % text, analyse(text)["script_profile"], ref_profile(text))+check("analyse/arabic", analyse("مرحبا كيف حالك")["script"], "arabic")+check("analyse/empty is unknown", analyse("")["script"], "unknown")++# a per-script fraction vector still sums to 1+for text in ("hello мир", "abc ᚠᚢᚦ"):+ total = sum(script_profile(text).values())+ check("profile sums to 1 (%r)" % text, round(total, 9), 1.0)+++print("\n%d passed, %d failed" % (len(PASS), len(FAIL)))+for f in FAIL:+ print(" FAIL", f)+if not FAIL:+ print("all lang stats tests passed")+sys.exit(1 if FAIL else 0)