Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

fix(lang): detect language from every string value in a dict state

Cursor Agente96ac74c0ce1
diff --git a/laya-ts/src/lang.ts b/laya-ts/src/lang.ts
index 90f54b65..34d5b4a0 100644
--- a/laya-ts/src/lang.ts
+++ b/laya-ts/src/lang.ts
@@ -149,16 +149,32 @@ function scriptOf(ch: string): string | null {
return null;
}
+/** String leaves of a state. Keys are ignored: they are usually English field names. */
+function iterText(state: unknown, depth = 0): string[] {
+ if (depth > 6 || state == null) return [];
+ if (typeof state === "string") return [state];
+ if (state instanceof Uint8Array) {
+ try {
+ return [new TextDecoder("utf-8", { fatal: true }).decode(state)];
+ } catch {
+ return [];
+ }
+ }
+ if (Array.isArray(state)) {
+ const out: string[] = [];
+ for (const v of state) out.push(...iterText(v, depth + 1));
+ return out;
+ }
+ if (typeof state === "object") {
+ const out: string[] = [];
+ for (const v of Object.values(state as object)) out.push(...iterText(v, depth + 1));
+ return out;
+ }
+ return [];
+}
+
export function stateText(state: unknown, maxChars = 4000): string {
- const parts: string[] = [];
- const walk = (v: unknown, d: number): void => {
- if (d > 6 || v == null) return;
- if (typeof v === "string") { parts.push(v); return; }
- if (Array.isArray(v)) { for (const x of v) walk(x, d + 1); return; }
- if (typeof v === "object") { for (const x of Object.values(v as object)) walk(x, d + 1); }
- };
- walk(state, 0);
- return parts.join(" ").slice(0, maxChars);
+ return iterText(state).join(" ").slice(0, maxChars);
}
export function detectScript(text: string): string {
@@ -313,8 +329,7 @@ function round4(x: number): number {
return Math.round(x * 10000) / 10000;
}
-export function analyse(state: unknown): AnalyseResult {
- const text = stateText(state);
+function analyseText(text: string): AnalyseResult {
const prof = scriptProfile(text);
let script = detectScript(text);
const nonLatin = prof && Object.keys(prof).length ? round4(1.0 - (prof["latin"] ?? 0.0)) : 0.0;
@@ -362,6 +377,53 @@ export function analyse(state: unknown): AnalyseResult {
};
}
+function alphaCount(text: string): number {
+ let n = 0;
+ for (const ch of text) if (IS_ALPHA_RE.test(ch)) n += 1;
+ return n;
+}
+
+/** A string value that is itself not safe for the English checkpoint, else null. */
+function leafNonEnglish(leaf: string): AnalyseResult | null {
+ const sample = leaf.slice(0, 4000);
+ if (!sample.trim()) return null;
+ const det = analyseText(sample);
+ if (det.isEnglish) return null;
+ if (det.language !== null && det.language !== "en") return det;
+ const nAlpha = alphaCount(sample);
+ if (det.script !== "latin" && det.script !== "unknown") {
+ if (nonLatinWords(sample).length > 0 && nAlpha >= NON_LATIN_MIN_LETTERS) return det;
+ return null;
+ }
+ const words = sample.match(WORD_RE) ?? [];
+ if (det.languageUndecided && det.diacriticRate >= NON_EN_DIACRITIC_RATE && words.length >= 4) {
+ return det;
+ }
+ return null;
+}
+
+export function analyse(state: unknown): AnalyseResult {
+ // One non-English string value is enough. Joining every value let a long English
+ // note fill the window, or outvote a short German message, and that message was
+ // then sent to the English checkpoint.
+ const result = analyseText(stateText(state));
+ if (typeof state === "string" || state == null || state instanceof Uint8Array || !result.isEnglish) {
+ return result;
+ }
+ let best: AnalyseResult | null = null;
+ let bestN = -1;
+ for (const leaf of iterText(state)) {
+ const det = leafNonEnglish(leaf);
+ if (!det) continue;
+ const n = alphaCount(leaf.slice(0, 4000));
+ if (n > bestN) {
+ bestN = n;
+ best = det;
+ }
+ }
+ return best ?? result;
+}
+
export function isEnglish(state: unknown): boolean {
return analyse(state).isEnglish;
}
diff --git a/laya-ts/tests/lang.test.ts b/laya-ts/tests/lang.test.ts
index 120018fe..92da325d 100644
--- a/laya-ts/tests/lang.test.ts
+++ b/laya-ts/tests/lang.test.ts
@@ -69,4 +69,17 @@ describe("lang", () => {
expect(analyse("que hora es en australia").language).toBe("es");
expect(analyse("baisse le volume du haut-parleur").language).toBe("fr");
});
+ it("does not let English sibling fields hide a German value", () => {
+ const sentence = "Mein Konto wurde zweimal belastet";
+ const asString = analyse(sentence);
+ expect(asString.language).toBe("de");
+ expect(asString.isEnglish).toBe(false);
+ expect(analyse({ message: sentence }).language).toBe(asString.language);
+ const buried = analyse({
+ agent_notes: "Please check the shipping status and refund the customer if the charge was duplicated. The order was late and we have not heard back.",
+ message: sentence,
+ });
+ expect(buried.isEnglish).toBe(false);
+ expect(buried.language).toBe("de");
+ });
});
diff --git a/laya/lang.py b/laya/lang.py
index b55ac082..01210aeb 100644
--- a/laya/lang.py
+++ b/laya/lang.py
@@ -12,6 +12,7 @@ is explicitly best-effort: pass an explicit model or `lang=` when you already kn
"""
import re
import unicodedata
+from collections.abc import Mapping
from typing import Dict, List, Optional, Union
# Unicode blocks that the English (ModernBERT-large, 50k English BPE) checkpoint cannot read.
@@ -190,13 +191,23 @@ _WORD = re.compile(r"[^\W\d_]+", re.UNICODE)
_IDENTIFIER = re.compile(r"(?<![\w-])[\w-]*(?:[.@][\w-]+)+", re.UNICODE)
-def _iter_text(state: Union[str, dict, list, None], _depth: int = 0) -> List[str]:
- """Collect the string leaves of a state (str / dict / list), so detection sees real content."""
+def _iter_text(state: Union[str, bytes, Mapping, list, None], _depth: int = 0) -> List[str]:
+ """Collect the string leaves of a state, so detection sees real content.
+
+ Keys are ignored: they are usually English field names. Any mapping counts, not only
+ `dict` -- a UserDict or mappingproxy has the same string values and used to contribute
+ nothing, which made `analyse` report no letters and the router fall through to English.
+ """
if _depth > 6 or state is None:
return []
if isinstance(state, str):
return [state]
- if isinstance(state, dict):
+ if isinstance(state, (bytes, bytearray)):
+ try:
+ return [bytes(state).decode("utf-8")]
+ except UnicodeDecodeError:
+ return []
+ if isinstance(state, Mapping):
out = []
for v in state.values():
out.extend(_iter_text(v, _depth + 1))
@@ -209,7 +220,7 @@ def _iter_text(state: Union[str, dict, list, None], _depth: int = 0) -> List[str
return []
-def state_text(state: Union[str, dict, list, None], max_chars: int = 4000) -> str:
+def state_text(state: Union[str, bytes, Mapping, list, None], max_chars: int = 4000) -> str:
"""Flatten a state into the text used for detection (keys are ignored: they are usually English)."""
parts: List[str] = []
budget = max_chars
@@ -417,7 +428,29 @@ _JOINED = re.compile(r"[^\W_][._/\\][^\W_]")
_LETTER_RUN = re.compile(r"[^\W\d_]{2,}")
-def _non_english_segment(state: Union[str, dict, list, None], max_chars: int = 4000):
+def _named_prose_language(segment: str):
+ """Language code for one non-code line, or None when it does not name a foreign language.
+
+ Same evidence bar as `_non_english_segment`: four words, a language `latin_profile` will name,
+ and two *different* words of that language. Acronyms and slash compounds are not words.
+ """
+ if not segment.strip() or _CODE_LINE.search(segment):
+ return None
+ prose = " ".join(tok for tok in segment.split() if not _JOINED.search(tok))
+ if any(ch.islower() for ch in prose):
+ prose = _LETTER_RUN.sub(lambda m: " " if m.group().isupper() else m.group(), prose)
+ tokens = _WORD.findall(prose)
+ if len(tokens) < 4:
+ return None
+ lang = latin_profile(prose)["language"]
+ if lang in (None, "en"):
+ return None
+ if len({w.lower() for w in tokens} & _STOP.get(lang, set())) < 2:
+ return None
+ return lang
+
+
+def _non_english_segment(state: Union[str, bytes, Mapping, list, None], max_chars: int = 4000):
"""First line or field that, read on its own, is named a non-English language, else None.
Returns (language, segment). A segment needs the evidence a whole state needs -- at least four
@@ -434,28 +467,18 @@ def _non_english_segment(state: Union[str, dict, list, None], max_chars: int = 4
return None
seg = seg[:max_chars - seen]
seen += len(seg)
- if _CODE_LINE.search(seg):
- continue
- prose = " ".join(tok for tok in seg.split() if not _JOINED.search(tok))
- if any(ch.islower() for ch in prose):
- prose = _LETTER_RUN.sub(lambda m: " " if m.group().isupper() else m.group(), prose)
- tokens = _WORD.findall(prose)
- if len(tokens) < 4:
- continue
- lang = latin_profile(prose)["language"]
- if lang not in (None, "en") and len({w.lower() for w in tokens} & _STOP.get(lang, set())) >= 2:
+ lang = _named_prose_language(seg)
+ if lang:
return lang, seg.strip()
return None
-def analyse(state: Union[str, dict, list, None]) -> Dict[str, object]:
- """Full detection result for a state.
+def _analyse_text(text: str) -> Dict[str, object]:
+ """Detection result for one already-flattened string.
- Returns `script`, `script_profile`, `language` (best effort, may be None),
- `is_english`, `non_latin_fraction` and `mixed_segment` (the line or field that made a mostly
- English state non-English, else None).
+ Does not look for a foreign line inside mostly-English text. `analyse` does that, because it
+ needs the original state and not only the joined window.
"""
- text = state_text(state)
counts = _script_counts(text)
prof = _profile_from_counts(counts)
script = _script_from_counts(counts)
@@ -481,25 +504,102 @@ def analyse(state: Union[str, dict, list, None]) -> Dict[str, object]:
# such letters (including short English) still goes to the English one.
undecided = lang is None
english = lang == "en" or (undecided and not prof_lat["looks_non_english"])
- # A Portuguese ticket with an English stack trace, error payload or form template reads as
- # English as a whole, because the English part is longer -- yet the part a question is about is
- # the customer's, and the English checkpoint cannot read it (0.97 confidence at 0.47 accuracy on
- # `pt`). The cost is lopsided: English sent to multilingual loses a few points, the reverse loses
- # calibration. So a state that would go to English is checked line by line and field by field.
- mixed = None
- leaves = _iter_text(state)
- # a single line has no other part to be outvoted by, and was just read whole
- if english and (len(leaves) > 1 or any("\n" in leaf for leaf in leaves)):
- found = _non_english_segment(state)
- if found:
- lang, mixed = found
- english, undecided = False, False
return {"script": "latin", "script_profile": prof, "language": lang,
"is_english": english, "language_undecided": undecided,
"diacritic_rate": round(float(prof_lat["diacritic_rate"]), 4),
- "non_latin_fraction": non_latin, "mixed_segment": mixed}
+ "non_latin_fraction": non_latin, "mixed_segment": None}
+
+def _leaf_non_english(leaf: str) -> Optional[Dict[str, object]]:
+ """A string value that is itself not safe for the English checkpoint, else None.
-def is_english(state: Union[str, dict, list, None]) -> bool:
+ A one-word name ("José") and a capitalised non-Latin name stay out: the same rules
+ `latin_profile` and `_non_latin_words` already use, so a name field cannot pull an
+ English ticket onto the multilingual checkpoint. Code lines, acronyms and slash compounds
+ stay out too, matching `_non_english_segment`, so a pasted traceback is not a message.
+ Each line is capped at 4000 characters; unlike the segment scan, a long earlier field
+ does not consume the budget of the next one (#384).
+ """
+ best_n = -1
+ best: Optional[Dict[str, object]] = None
+ for line in leaf.split("\n"):
+ sample = line[:4000]
+ if not sample.strip() or _CODE_LINE.search(sample):
+ continue
+ det = _analyse_text(sample)
+ if det["is_english"]:
+ continue
+ if det["language"] not in (None, "en"):
+ if _named_prose_language(sample) is None:
+ continue
+ elif det["script"] not in ("latin", "unknown"):
+ if not (_non_latin_words(sample) and sum(ch.isalpha() for ch in sample) >= NON_LATIN_MIN_LETTERS):
+ continue
+ elif not (det["language_undecided"] and float(det["diacritic_rate"]) >= NON_EN_DIACRITIC_RATE
+ and len(_WORD.findall(sample)) >= 4):
+ continue
+ n_alpha = sum(ch.isalpha() for ch in sample)
+ if n_alpha > best_n:
+ best_n = n_alpha
+ best = det
+ return best
+
+
+def analyse(state: Union[str, bytes, Mapping, list, None]) -> Dict[str, object]:
+ """Full detection result for a state.
+
+ Returns `script`, `script_profile`, `language` (best effort, may be None),
+ `is_english`, `non_latin_fraction` and `mixed_segment` (the line or field that made a mostly
+ English state non-English, else None).
+
+ String values are what get read. When a state has several of them, one non-English value is
+ enough: joining every value into one window let a long English note fill the 4000 characters,
+ or outvote a short German message, and that message was then sent to the English checkpoint
+ (#384). The segment scan still stops at 4000 characters, which is what keeps a huge field
+ cheap; a value it did not reach is read on its own afterwards.
+ """
+ result = _analyse_text(state_text(state))
+ if result["script"] == "latin" and result["is_english"]:
+ # A Portuguese ticket with an English stack trace, error payload or form template reads as
+ # English as a whole, because the English part is longer -- yet the part a question is about
+ # is the customer's, and the English checkpoint cannot read it (0.97 confidence at 0.47
+ # accuracy on `pt`). The cost is lopsided: English sent to multilingual loses a few points,
+ # the reverse loses calibration. So a state that would go to English is checked line by line
+ # and field by field.
+ leaves = _iter_text(state)
+ # a single line has no other part to be outvoted by, and was just read whole
+ if len(leaves) > 1 or any("\n" in leaf for leaf in leaves):
+ found = _non_english_segment(state)
+ if found:
+ lang, mixed = found
+ result = dict(result)
+ result["language"] = lang
+ result["is_english"] = False
+ result["language_undecided"] = False
+ result["mixed_segment"] = mixed
+ # A plain string was just read whole. A structured state can still hide a message past the
+ # segment cap, or in a script `latin_profile` does not name.
+ if isinstance(state, (str, bytes, bytearray)) or state is None or not result["is_english"]:
+ return result
+ best_n = -1
+ best: Optional[Dict[str, object]] = None
+ for leaf in _iter_text(state):
+ det = _leaf_non_english(leaf)
+ if det is None:
+ continue
+ n_alpha = sum(ch.isalpha() for ch in leaf[:4000])
+ if n_alpha > best_n:
+ best_n = n_alpha
+ best = det
+ if best is None:
+ return result
+ updated = dict(result)
+ updated["language"] = best["language"]
+ updated["is_english"] = False
+ updated["language_undecided"] = best["language_undecided"]
+ return updated
+
+
+def is_english(state: Union[str, bytes, Mapping, list, None]) -> bool:
"""True when the English checkpoint can be expected to read this state."""
return bool(analyse(state)["is_english"])
diff --git a/tests/test_router.py b/tests/test_router.py
index b9c66c62..00207a77 100644
--- a/tests/test_router.py
+++ b/tests/test_router.py
@@ -166,6 +166,59 @@ check("state_text/none", state_text(None), "")
check("state_text/keys ignored",
analyse({"subject": "नमस्ते", "body": "ग्राहक से दो बार शुल्क लिया गया"})["is_english"], False)
+# --------------------------------------------------------------------- dict-state language (#384)
+# The same German sentence must route the same way as a string and as a dict value.
+# A mapping that is not a builtin dict, a bytes value, and English sibling fields used to
+# hide that value: detection then reported language_undecided (or English) and the router
+# fell through to the English checkpoint.
+_DE = "Mein Konto wurde zweimal belastet"
+_r_de = Router()
+_de_str = _r_de.route(_DE, {})
+_de_dict = _r_de.route({"message": _DE}, {})
+check("route/dict german matches string", _de_dict.model, _de_str.model)
+check("route/dict german is multilingual", _de_dict.model, "multilingual")
+check("route/dict german names de", _de_dict["detection"]["language"], "de")
+check("route/dict german is not undecided", _de_dict["detection"]["language_undecided"], False)
+check("analyse/dict german matches string", analyse({"message": _DE})["language"], analyse(_DE)["language"])
+# English notes must not outvote the message, and a long note must not push it out of the window.
+_de_ticket = {
+ "ticket_id": "TCK-88213",
+ "channel": "web chat",
+ "agent_notes": "Please check the shipping status and refund the customer if the charge was duplicated.",
+ "message": _DE,
+}
+check("route/dict german beside english notes", _r_de.route(_de_ticket, {}).model, "multilingual")
+check("route/dict german beside english notes is not undecided",
+ _r_de.route(_de_ticket, {})["detection"]["language_undecided"], False)
+_de_long = {
+ "agent_notes": "Please check the shipping status and tell the customer about the refund. " * 80,
+ "message": _DE,
+}
+check("route/dict german after long english note", _r_de.route(_de_long, {}).model, "multilingual")
+check("route/dict german after long english note names de",
+ _r_de.route(_de_long, {})["detection"]["language"], "de")
+from collections import UserDict # noqa: E402
+from types import MappingProxyType # noqa: E402
+check("route/userdict german", _r_de.route(UserDict({"message": _DE}), {}).model, "multilingual")
+check("route/userdict german is not undecided",
+ _r_de.route(UserDict({"message": _DE}), {})["detection"]["language_undecided"], False)
+check("route/mappingproxy german",
+ _r_de.route(MappingProxyType({"message": _DE}), {}).model, "multilingual")
+check("route/bytes german value",
+ _r_de.route({"message": _DE.encode("utf-8")}, {}).model, "multilingual")
+# A name beside an English request is not a second message.
+check("route/dict cyrillic name stays english",
+ _r_de.route({"name": "Антон Павлович Чехов",
+ "body": "Please refund the duplicate charge on invoice 4411 today."}, {}).model,
+ "english")
+check("route/dict jose stays english",
+ _r_de.route({"name": "José",
+ "body": "Please refund the duplicate charge on invoice 4411 today."}, {}).model,
+ "english")
+check("route/dict english body stays english",
+ _r_de.route({"body": "Please refund the duplicate charge on invoice 4411 today."}, {}).model,
+ "english")
+
# --------------------------------------------------------------------- workflow signatures
check("profile/armenian", analyse("Հայերեն")["script_profile"], {"armenian": 1.0})