Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
fix(lang): detect language from every string value in a dict state
Cursor Agente96ac74c0ce1
diff --git a/laya-ts/src/lang.ts b/laya-ts/src/lang.tsindex 90f54b65..34d5b4a0 100644--- a/laya-ts/src/lang.ts+++ b/laya-ts/src/lang.ts@@ -149,16 +149,32 @@ function scriptOf(ch: string): string | null {return null;}+/** String leaves of a state. Keys are ignored: they are usually English field names. */+function iterText(state: unknown, depth = 0): string[] {+ if (depth > 6 || state == null) return [];+ if (typeof state === "string") return [state];+ if (state instanceof Uint8Array) {+ try {+ return [new TextDecoder("utf-8", { fatal: true }).decode(state)];+ } catch {+ return [];+ }+ }+ if (Array.isArray(state)) {+ const out: string[] = [];+ for (const v of state) out.push(...iterText(v, depth + 1));+ return out;+ }+ if (typeof state === "object") {+ const out: string[] = [];+ for (const v of Object.values(state as object)) out.push(...iterText(v, depth + 1));+ return out;+ }+ return [];+}+export function stateText(state: unknown, maxChars = 4000): string {- const parts: string[] = [];- const walk = (v: unknown, d: number): void => {- if (d > 6 || v == null) return;- if (typeof v === "string") { parts.push(v); return; }- if (Array.isArray(v)) { for (const x of v) walk(x, d + 1); return; }- if (typeof v === "object") { for (const x of Object.values(v as object)) walk(x, d + 1); }- };- walk(state, 0);- return parts.join(" ").slice(0, maxChars);+ return iterText(state).join(" ").slice(0, maxChars);}export function detectScript(text: string): string {@@ -313,8 +329,7 @@ function round4(x: number): number {return Math.round(x * 10000) / 10000;}-export function analyse(state: unknown): AnalyseResult {- const text = stateText(state);+function analyseText(text: string): AnalyseResult {const prof = scriptProfile(text);let script = detectScript(text);const nonLatin = prof && Object.keys(prof).length ? round4(1.0 - (prof["latin"] ?? 0.0)) : 0.0;@@ -362,6 +377,53 @@ export function analyse(state: unknown): AnalyseResult {};}+function alphaCount(text: string): number {+ let n = 0;+ for (const ch of text) if (IS_ALPHA_RE.test(ch)) n += 1;+ return n;+}++/** A string value that is itself not safe for the English checkpoint, else null. */+function leafNonEnglish(leaf: string): AnalyseResult | null {+ const sample = leaf.slice(0, 4000);+ if (!sample.trim()) return null;+ const det = analyseText(sample);+ if (det.isEnglish) return null;+ if (det.language !== null && det.language !== "en") return det;+ const nAlpha = alphaCount(sample);+ if (det.script !== "latin" && det.script !== "unknown") {+ if (nonLatinWords(sample).length > 0 && nAlpha >= NON_LATIN_MIN_LETTERS) return det;+ return null;+ }+ const words = sample.match(WORD_RE) ?? [];+ if (det.languageUndecided && det.diacriticRate >= NON_EN_DIACRITIC_RATE && words.length >= 4) {+ return det;+ }+ return null;+}++export function analyse(state: unknown): AnalyseResult {+ // One non-English string value is enough. Joining every value let a long English+ // note fill the window, or outvote a short German message, and that message was+ // then sent to the English checkpoint.+ const result = analyseText(stateText(state));+ if (typeof state === "string" || state == null || state instanceof Uint8Array || !result.isEnglish) {+ return result;+ }+ let best: AnalyseResult | null = null;+ let bestN = -1;+ for (const leaf of iterText(state)) {+ const det = leafNonEnglish(leaf);+ if (!det) continue;+ const n = alphaCount(leaf.slice(0, 4000));+ if (n > bestN) {+ bestN = n;+ best = det;+ }+ }+ return best ?? result;+}+export function isEnglish(state: unknown): boolean {return analyse(state).isEnglish;}diff --git a/laya-ts/tests/lang.test.ts b/laya-ts/tests/lang.test.tsindex 120018fe..92da325d 100644--- a/laya-ts/tests/lang.test.ts+++ b/laya-ts/tests/lang.test.ts@@ -69,4 +69,17 @@ describe("lang", () => {expect(analyse("que hora es en australia").language).toBe("es");expect(analyse("baisse le volume du haut-parleur").language).toBe("fr");});+ it("does not let English sibling fields hide a German value", () => {+ const sentence = "Mein Konto wurde zweimal belastet";+ const asString = analyse(sentence);+ expect(asString.language).toBe("de");+ expect(asString.isEnglish).toBe(false);+ expect(analyse({ message: sentence }).language).toBe(asString.language);+ const buried = analyse({+ agent_notes: "Please check the shipping status and refund the customer if the charge was duplicated. The order was late and we have not heard back.",+ message: sentence,+ });+ expect(buried.isEnglish).toBe(false);+ expect(buried.language).toBe("de");+ });});diff --git a/laya/lang.py b/laya/lang.pyindex b55ac082..01210aeb 100644--- a/laya/lang.py+++ b/laya/lang.py@@ -12,6 +12,7 @@ is explicitly best-effort: pass an explicit model or `lang=` when you already kn"""import reimport unicodedata+from collections.abc import Mappingfrom typing import Dict, List, Optional, Union# Unicode blocks that the English (ModernBERT-large, 50k English BPE) checkpoint cannot read.@@ -190,13 +191,23 @@ _WORD = re.compile(r"[^\W\d_]+", re.UNICODE)_IDENTIFIER = re.compile(r"(?<![\w-])[\w-]*(?:[.@][\w-]+)+", re.UNICODE)-def _iter_text(state: Union[str, dict, list, None], _depth: int = 0) -> List[str]:- """Collect the string leaves of a state (str / dict / list), so detection sees real content."""+def _iter_text(state: Union[str, bytes, Mapping, list, None], _depth: int = 0) -> List[str]:+ """Collect the string leaves of a state, so detection sees real content.++ Keys are ignored: they are usually English field names. Any mapping counts, not only+ `dict` -- a UserDict or mappingproxy has the same string values and used to contribute+ nothing, which made `analyse` report no letters and the router fall through to English.+ """if _depth > 6 or state is None:return []if isinstance(state, str):return [state]- if isinstance(state, dict):+ if isinstance(state, (bytes, bytearray)):+ try:+ return [bytes(state).decode("utf-8")]+ except UnicodeDecodeError:+ return []+ if isinstance(state, Mapping):out = []for v in state.values():out.extend(_iter_text(v, _depth + 1))@@ -209,7 +220,7 @@ def _iter_text(state: Union[str, dict, list, None], _depth: int = 0) -> List[strreturn []-def state_text(state: Union[str, dict, list, None], max_chars: int = 4000) -> str:+def state_text(state: Union[str, bytes, Mapping, list, None], max_chars: int = 4000) -> str:"""Flatten a state into the text used for detection (keys are ignored: they are usually English)."""parts: List[str] = []budget = max_chars@@ -417,7 +428,29 @@ _JOINED = re.compile(r"[^\W_][._/\\][^\W_]")_LETTER_RUN = re.compile(r"[^\W\d_]{2,}")-def _non_english_segment(state: Union[str, dict, list, None], max_chars: int = 4000):+def _named_prose_language(segment: str):+ """Language code for one non-code line, or None when it does not name a foreign language.++ Same evidence bar as `_non_english_segment`: four words, a language `latin_profile` will name,+ and two *different* words of that language. Acronyms and slash compounds are not words.+ """+ if not segment.strip() or _CODE_LINE.search(segment):+ return None+ prose = " ".join(tok for tok in segment.split() if not _JOINED.search(tok))+ if any(ch.islower() for ch in prose):+ prose = _LETTER_RUN.sub(lambda m: " " if m.group().isupper() else m.group(), prose)+ tokens = _WORD.findall(prose)+ if len(tokens) < 4:+ return None+ lang = latin_profile(prose)["language"]+ if lang in (None, "en"):+ return None+ if len({w.lower() for w in tokens} & _STOP.get(lang, set())) < 2:+ return None+ return lang+++def _non_english_segment(state: Union[str, bytes, Mapping, list, None], max_chars: int = 4000):"""First line or field that, read on its own, is named a non-English language, else None.Returns (language, segment). A segment needs the evidence a whole state needs -- at least four@@ -434,28 +467,18 @@ def _non_english_segment(state: Union[str, dict, list, None], max_chars: int = 4return Noneseg = seg[:max_chars - seen]seen += len(seg)- if _CODE_LINE.search(seg):- continue- prose = " ".join(tok for tok in seg.split() if not _JOINED.search(tok))- if any(ch.islower() for ch in prose):- prose = _LETTER_RUN.sub(lambda m: " " if m.group().isupper() else m.group(), prose)- tokens = _WORD.findall(prose)- if len(tokens) < 4:- continue- lang = latin_profile(prose)["language"]- if lang not in (None, "en") and len({w.lower() for w in tokens} & _STOP.get(lang, set())) >= 2:+ lang = _named_prose_language(seg)+ if lang:return lang, seg.strip()return None-def analyse(state: Union[str, dict, list, None]) -> Dict[str, object]:- """Full detection result for a state.+def _analyse_text(text: str) -> Dict[str, object]:+ """Detection result for one already-flattened string.- Returns `script`, `script_profile`, `language` (best effort, may be None),- `is_english`, `non_latin_fraction` and `mixed_segment` (the line or field that made a mostly- English state non-English, else None).+ Does not look for a foreign line inside mostly-English text. `analyse` does that, because it+ needs the original state and not only the joined window."""- text = state_text(state)counts = _script_counts(text)prof = _profile_from_counts(counts)script = _script_from_counts(counts)@@ -481,25 +504,102 @@ def analyse(state: Union[str, dict, list, None]) -> Dict[str, object]:# such letters (including short English) still goes to the English one.undecided = lang is Noneenglish = lang == "en" or (undecided and not prof_lat["looks_non_english"])- # A Portuguese ticket with an English stack trace, error payload or form template reads as- # English as a whole, because the English part is longer -- yet the part a question is about is- # the customer's, and the English checkpoint cannot read it (0.97 confidence at 0.47 accuracy on- # `pt`). The cost is lopsided: English sent to multilingual loses a few points, the reverse loses- # calibration. So a state that would go to English is checked line by line and field by field.- mixed = None- leaves = _iter_text(state)- # a single line has no other part to be outvoted by, and was just read whole- if english and (len(leaves) > 1 or any("\n" in leaf for leaf in leaves)):- found = _non_english_segment(state)- if found:- lang, mixed = found- english, undecided = False, Falsereturn {"script": "latin", "script_profile": prof, "language": lang,"is_english": english, "language_undecided": undecided,"diacritic_rate": round(float(prof_lat["diacritic_rate"]), 4),- "non_latin_fraction": non_latin, "mixed_segment": mixed}+ "non_latin_fraction": non_latin, "mixed_segment": None}++def _leaf_non_english(leaf: str) -> Optional[Dict[str, object]]:+ """A string value that is itself not safe for the English checkpoint, else None.-def is_english(state: Union[str, dict, list, None]) -> bool:+ A one-word name ("José") and a capitalised non-Latin name stay out: the same rules+ `latin_profile` and `_non_latin_words` already use, so a name field cannot pull an+ English ticket onto the multilingual checkpoint. Code lines, acronyms and slash compounds+ stay out too, matching `_non_english_segment`, so a pasted traceback is not a message.+ Each line is capped at 4000 characters; unlike the segment scan, a long earlier field+ does not consume the budget of the next one (#384).+ """+ best_n = -1+ best: Optional[Dict[str, object]] = None+ for line in leaf.split("\n"):+ sample = line[:4000]+ if not sample.strip() or _CODE_LINE.search(sample):+ continue+ det = _analyse_text(sample)+ if det["is_english"]:+ continue+ if det["language"] not in (None, "en"):+ if _named_prose_language(sample) is None:+ continue+ elif det["script"] not in ("latin", "unknown"):+ if not (_non_latin_words(sample) and sum(ch.isalpha() for ch in sample) >= NON_LATIN_MIN_LETTERS):+ continue+ elif not (det["language_undecided"] and float(det["diacritic_rate"]) >= NON_EN_DIACRITIC_RATE+ and len(_WORD.findall(sample)) >= 4):+ continue+ n_alpha = sum(ch.isalpha() for ch in sample)+ if n_alpha > best_n:+ best_n = n_alpha+ best = det+ return best+++def analyse(state: Union[str, bytes, Mapping, list, None]) -> Dict[str, object]:+ """Full detection result for a state.++ Returns `script`, `script_profile`, `language` (best effort, may be None),+ `is_english`, `non_latin_fraction` and `mixed_segment` (the line or field that made a mostly+ English state non-English, else None).++ String values are what get read. When a state has several of them, one non-English value is+ enough: joining every value into one window let a long English note fill the 4000 characters,+ or outvote a short German message, and that message was then sent to the English checkpoint+ (#384). The segment scan still stops at 4000 characters, which is what keeps a huge field+ cheap; a value it did not reach is read on its own afterwards.+ """+ result = _analyse_text(state_text(state))+ if result["script"] == "latin" and result["is_english"]:+ # A Portuguese ticket with an English stack trace, error payload or form template reads as+ # English as a whole, because the English part is longer -- yet the part a question is about+ # is the customer's, and the English checkpoint cannot read it (0.97 confidence at 0.47+ # accuracy on `pt`). The cost is lopsided: English sent to multilingual loses a few points,+ # the reverse loses calibration. So a state that would go to English is checked line by line+ # and field by field.+ leaves = _iter_text(state)+ # a single line has no other part to be outvoted by, and was just read whole+ if len(leaves) > 1 or any("\n" in leaf for leaf in leaves):+ found = _non_english_segment(state)+ if found:+ lang, mixed = found+ result = dict(result)+ result["language"] = lang+ result["is_english"] = False+ result["language_undecided"] = False+ result["mixed_segment"] = mixed+ # A plain string was just read whole. A structured state can still hide a message past the+ # segment cap, or in a script `latin_profile` does not name.+ if isinstance(state, (str, bytes, bytearray)) or state is None or not result["is_english"]:+ return result+ best_n = -1+ best: Optional[Dict[str, object]] = None+ for leaf in _iter_text(state):+ det = _leaf_non_english(leaf)+ if det is None:+ continue+ n_alpha = sum(ch.isalpha() for ch in leaf[:4000])+ if n_alpha > best_n:+ best_n = n_alpha+ best = det+ if best is None:+ return result+ updated = dict(result)+ updated["language"] = best["language"]+ updated["is_english"] = False+ updated["language_undecided"] = best["language_undecided"]+ return updated+++def is_english(state: Union[str, bytes, Mapping, list, None]) -> bool:"""True when the English checkpoint can be expected to read this state."""return bool(analyse(state)["is_english"])diff --git a/tests/test_router.py b/tests/test_router.pyindex b9c66c62..00207a77 100644--- a/tests/test_router.py+++ b/tests/test_router.py@@ -166,6 +166,59 @@ check("state_text/none", state_text(None), "")check("state_text/keys ignored",analyse({"subject": "नमस्ते", "body": "ग्राहक से दो बार शुल्क लिया गया"})["is_english"], False)+# --------------------------------------------------------------------- dict-state language (#384)+# The same German sentence must route the same way as a string and as a dict value.+# A mapping that is not a builtin dict, a bytes value, and English sibling fields used to+# hide that value: detection then reported language_undecided (or English) and the router+# fell through to the English checkpoint.+_DE = "Mein Konto wurde zweimal belastet"+_r_de = Router()+_de_str = _r_de.route(_DE, {})+_de_dict = _r_de.route({"message": _DE}, {})+check("route/dict german matches string", _de_dict.model, _de_str.model)+check("route/dict german is multilingual", _de_dict.model, "multilingual")+check("route/dict german names de", _de_dict["detection"]["language"], "de")+check("route/dict german is not undecided", _de_dict["detection"]["language_undecided"], False)+check("analyse/dict german matches string", analyse({"message": _DE})["language"], analyse(_DE)["language"])+# English notes must not outvote the message, and a long note must not push it out of the window.+_de_ticket = {+ "ticket_id": "TCK-88213",+ "channel": "web chat",+ "agent_notes": "Please check the shipping status and refund the customer if the charge was duplicated.",+ "message": _DE,+}+check("route/dict german beside english notes", _r_de.route(_de_ticket, {}).model, "multilingual")+check("route/dict german beside english notes is not undecided",+ _r_de.route(_de_ticket, {})["detection"]["language_undecided"], False)+_de_long = {+ "agent_notes": "Please check the shipping status and tell the customer about the refund. " * 80,+ "message": _DE,+}+check("route/dict german after long english note", _r_de.route(_de_long, {}).model, "multilingual")+check("route/dict german after long english note names de",+ _r_de.route(_de_long, {})["detection"]["language"], "de")+from collections import UserDict # noqa: E402+from types import MappingProxyType # noqa: E402+check("route/userdict german", _r_de.route(UserDict({"message": _DE}), {}).model, "multilingual")+check("route/userdict german is not undecided",+ _r_de.route(UserDict({"message": _DE}), {})["detection"]["language_undecided"], False)+check("route/mappingproxy german",+ _r_de.route(MappingProxyType({"message": _DE}), {}).model, "multilingual")+check("route/bytes german value",+ _r_de.route({"message": _DE.encode("utf-8")}, {}).model, "multilingual")+# A name beside an English request is not a second message.+check("route/dict cyrillic name stays english",+ _r_de.route({"name": "Антон Павлович Чехов",+ "body": "Please refund the duplicate charge on invoice 4411 today."}, {}).model,+ "english")+check("route/dict jose stays english",+ _r_de.route({"name": "José",+ "body": "Please refund the duplicate charge on invoice 4411 today."}, {}).model,+ "english")+check("route/dict english body stays english",+ _r_de.route({"body": "Please refund the duplicate charge on invoice 4411 today."}, {}).model,+ "english")+# --------------------------------------------------------------------- workflow signaturescheck("profile/armenian", analyse("Հայերեն")["script_profile"], {"armenian": 1.0})