Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

perf: single-pass script stats; wire test_lang_stats

Emiliano Gandini Outeda232e750d7801
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index ce48ff4f..c32d9dff 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -96,6 +96,7 @@ jobs:
python tests/test_lazy_import.py
python tests/test_runtime_fixes.py
python tests/test_lang_guess.py
+ python tests/test_lang_stats.py
python tests/test_calibration_persistence.py
python tests/test_context_manager.py
python tests/test_criteria_normalization.py
diff --git a/laya/agent.py b/laya/agent.py
index b0fb78a7..bf8d2785 100644
--- a/laya/agent.py
+++ b/laya/agent.py
@@ -903,6 +903,9 @@ class Agent(HookRegistry):
return _decide(self, state, schema, questions=questions,
return_details=return_details, **predict_kwargs)
+ def __repr__(self) -> str:
+ return "Agent(model_id=%r, device=%s)" % (self.model_id, getattr(self, "device", None))
+
predict = system_one
diff --git a/laya/lang.py b/laya/lang.py
index 0122c787..421fa194 100644
--- a/laya/lang.py
+++ b/laya/lang.py
@@ -218,8 +218,14 @@ def state_text(state: Union[str, dict, list, None], max_chars: int = 4000) -> st
return " ".join(parts)[:max_chars]
-def detect_script(text: str) -> str:
- """Dominant script of `text`: 'latin', 'han', 'devanagari', ... or 'unknown' if there are no letters."""
+def _script_counts(text: str) -> Dict[str, int]:
+ """Count the alphabetic characters of `text` by script, in one pass.
+
+ Latin is inserted last so `_script_from_counts` keeps `detect_script`'s tie-break: a named
+ script wins a tie against Latin, because `max` returns the first of equal values and Latin
+ is the last key. `analyse` needs both the dominant script and the per-script fractions, and
+ used to walk the text twice (once per function) to get them; one pass serves both.
+ """
counts: Dict[str, int] = {}
latin = 0
for ch in text:
@@ -245,32 +251,37 @@ def detect_script(text: str) -> str:
# handed to the English checkpoint.
counts["other"] = counts.get("other", 0) + 1
counts["latin"] = latin
- total = sum(counts.values())
- if total == 0:
+ return counts
+
+
+def _script_from_counts(counts: Dict[str, int]) -> str:
+ if not any(counts.values()):
return "unknown"
return max(counts.items(), key=lambda kv: kv[1])[0]
-def script_profile(text: str) -> Dict[str, float]:
- """Fraction of alphabetic characters belonging to each detected script."""
- counts: Dict[str, int] = {"latin": 0}
- for ch in text:
- if not ch.isalpha():
- continue
- cp = ord(ch)
- if cp < 0x02B0 or 0x1E00 <= cp <= 0x1EFF or 0xFF21 <= cp <= 0xFF3A or 0xFF41 <= cp <= 0xFF5A:
- counts["latin"] += 1
- continue
- for name, ranges in _SCRIPT_RANGES:
- if any(lo <= cp <= hi for lo, hi in ranges):
- counts[name] = counts.get(name, 0) + 1
- break
- else:
- counts["other"] = counts.get("other", 0) + 1
+def _profile_from_counts(counts: Dict[str, int]) -> Dict[str, float]:
total = sum(counts.values())
if not total:
return {}
- return {k: v / total for k, v in counts.items() if v}
+ # Keep Latin first in the public mapping, the order callers saw before this was refactored.
+ ordered: Dict[str, int] = {}
+ if counts.get("latin"):
+ ordered["latin"] = counts["latin"]
+ for name, value in counts.items():
+ if name != "latin":
+ ordered[name] = value
+ return {k: v / total for k, v in ordered.items() if v}
+
+
+def detect_script(text: str) -> str:
+ """Dominant script of `text`: 'latin', 'han', 'devanagari', ... or 'unknown' if there are no letters."""
+ return _script_from_counts(_script_counts(text))
+
+
+def script_profile(text: str) -> Dict[str, float]:
+ """Fraction of alphabetic characters belonging to each detected script."""
+ return _profile_from_counts(_script_counts(text))
# A diacritic rate above this is taken as evidence the text is not English, even when no
@@ -387,8 +398,9 @@ def analyse(state: Union[str, dict, list, None]) -> Dict[str, object]:
`is_english` and `non_latin_fraction`.
"""
text = state_text(state)
- prof = script_profile(text)
- script = detect_script(text)
+ counts = _script_counts(text)
+ prof = _profile_from_counts(counts)
+ script = _script_from_counts(counts)
non_latin = round(1.0 - prof.get("latin", 0.0), 4) if prof else 0.0
n_non_latin = round(non_latin * sum(ch.isalpha() for ch in text))
if script == "latin" and _non_latin_words(text) and (
diff --git a/tests/test_lang_stats.py b/tests/test_lang_stats.py
new file mode 100644
index 00000000..496f656b
--- /dev/null
+++ b/tests/test_lang_stats.py
@@ -0,0 +1,117 @@
+"""`detect_script` / `script_profile` must be unchanged by the single-pass refactor in #lang.
+
+The reference implementations below are the pre-refactor versions; the new ones share one pass
+over the text, so these checks pin that the counts, the fractions and the tie-break are the same.
+
+Run: python tests/test_lang_stats.py
+"""
+import os
+import sys
+
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+from laya.lang import _SCRIPT_RANGES, analyse, detect_script, script_profile # noqa: E402
+
+PASS, FAIL = [], []
+
+
+def check(name, got, want):
+ if got == want:
+ PASS.append(name)
+ else:
+ FAIL.append("%s: got %r, want %r" % (name, got, want))
+
+
+def _is_latin_cp(cp):
+ return cp < 0x02B0 or 0x1E00 <= cp <= 0x1EFF or 0xFF21 <= cp <= 0xFF3A or 0xFF41 <= cp <= 0xFF5A
+
+
+def ref_counts(text):
+ counts = {}
+ latin = 0
+ for ch in text:
+ if not ch.isalpha():
+ continue
+ cp = ord(ch)
+ if _is_latin_cp(cp):
+ latin += 1
+ continue
+ for name, ranges in _SCRIPT_RANGES:
+ if any(lo <= cp <= hi for lo, hi in ranges):
+ counts[name] = counts.get(name, 0) + 1
+ break
+ else:
+ counts["other"] = counts.get("other", 0) + 1
+ counts["latin"] = latin
+ return counts
+
+
+def ref_detect(text):
+ counts = ref_counts(text)
+ if sum(counts.values()) == 0:
+ return "unknown"
+ return max(counts.items(), key=lambda kv: kv[1])[0]
+
+
+def ref_profile(text):
+ counts = {"latin": 0}
+ for ch in text:
+ if not ch.isalpha():
+ continue
+ cp = ord(ch)
+ if _is_latin_cp(cp):
+ counts["latin"] += 1
+ continue
+ for name, ranges in _SCRIPT_RANGES:
+ if any(lo <= cp <= hi for lo, hi in ranges):
+ counts[name] = counts.get(name, 0) + 1
+ break
+ else:
+ counts["other"] = counts.get("other", 0) + 1
+ total = sum(counts.values())
+ if not total:
+ return {}
+ return {k: v / total for k, v in counts.items() if v}
+
+
+TEXTS = [
+ "",
+ " ",
+ "hello world",
+ "hello мир",
+ "мир",
+ "世界",
+ "日本語です",
+ "ΑΒΓΔ",
+ "abc ᚠᚢᚦ", # an unlisted script lands under "other"
+ "ab αβ", # latin/greek tie
+ "αβ аб", # greek/cyrillic tie
+ "ᚠᚢᚦabc", # other/latin tie
+ "Hallo, meine Bestellung ist zweimal abgebucht worden.",
+ "مرحبا كيف حالك",
+ "İstanbul",
+]
+
+for text in TEXTS:
+ check("detect_script(%r)" % text, detect_script(text), ref_detect(text))
+ check("script_profile(%r)" % text, script_profile(text), ref_profile(text))
+
+# analyse uses the same counts, so its profile must match too, and it must not regress on a
+# non-Latin override (a lowercase non-Latin run flips the dominant script).
+for text in ("hello мир", "мир", "مرحبا كيف حالك", "hello world"):
+ check("analyse.profile(%r)" % text, analyse(text)["script_profile"], ref_profile(text))
+check("analyse/arabic", analyse("مرحبا كيف حالك")["script"], "arabic")
+check("analyse/empty is unknown", analyse("")["script"], "unknown")
+
+# a per-script fraction vector still sums to 1
+for text in ("hello мир", "abc ᚠᚢᚦ"):
+ total = sum(script_profile(text).values())
+ check("profile sums to 1 (%r)" % text, round(total, 9), 1.0)
+
+
+print("\n%d passed, %d failed" % (len(PASS), len(FAIL)))
+for f in FAIL:
+ print(" FAIL", f)
+if not FAIL:
+ print("all lang stats tests passed")
+sys.exit(1 if FAIL else 0)