Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
fix: never assume an unidentified Latin language is English; clamp temperatures (#35) (#42)
Avinash8e58a6cd8ddb
diff --git a/laya/agent.py b/laya/agent.pyindex c63325be..15e2eb56 100644--- a/laya/agent.py+++ b/laya/agent.py@@ -1,6 +1,7 @@"""High-level inference runtime for laya System 1 decision models."""import jsonimport os+import warningsfrom typing import Any, Dict, Optional, Unionimport numpy as np@@ -8,9 +9,12 @@ import torchfrom .common import (QTYPES,+ TEMP_MAX,+ TEMP_MIN,amp_dtype,build_model,build_sequence,+ clamp_temperature,collate_items,confidence_from_probs,render_options,@@ -197,8 +201,23 @@ class Agent:except Exception:pass- self.temperature = self.cfg.get("temperature", [1.0, 1.0, 1.0])- self.temperature_by_options = self.cfg.get("temperature_by_options", {})+ # Keep what the checkpoint shipped for inspection, but only ever apply clamped values:+ # some buckets are fitted to sharpen rather than soften (see clamp_temperature).+ self.temperature_raw = self.cfg.get("temperature", [1.0, 1.0, 1.0])+ self.temperature_by_options_raw = self.cfg.get("temperature_by_options", {})+ self.temperature = [clamp_temperature(t) for t in self.temperature_raw]+ self.temperature_by_options = {k: clamp_temperature(v)+ for k, v in self.temperature_by_options_raw.items()}+ rejected = ["%s=%.4g" % (k, float(v)) for k, v in self.temperature_by_options_raw.items()+ if clamp_temperature(v) != float(v)]+ rejected += ["temperature[%d]=%.4g" % (i, float(t)) for i, t in enumerate(self.temperature_raw)+ if clamp_temperature(t) != float(t)]+ if rejected:+ warnings.warn(+ "laya: this checkpoint ships temperatures outside [%g, %g] which would distort "+ "confidence; clamping %s. Treat confidence from the affected buckets as uncalibrated."+ % (TEMP_MIN, TEMP_MAX, ", ".join(rejected)),+ RuntimeWarning, stacklevel=2)self.dtype = amp_dtype(self.cfg.get("amp_dtype", "fp16"))if self.device.type == "cuda" and torch.cuda.get_device_capability(self.device)[0] < 8:@@ -308,7 +327,7 @@ class Agent:k = len(items[r]["markers"])qt = QTYPES[q["t"]]t_scale = self.temperature_by_options.get(temp_bucket(qt, k), self.temperature[qt])- z = logits[r, :k] / max(1e-3, float(t_scale))+ z = logits[r, :k] / t_scalep = np.exp(z - z.max())p = p / p.sum()diff --git a/laya/common.py b/laya/common.pyindex fa9c6968..fb900053 100644--- a/laya/common.py+++ b/laya/common.py@@ -221,6 +221,25 @@ def temp_bucket(qtype: int, k: int) -> str:return "%s:%s" % (QTYPE_NAMES[int(qtype)], size)+# A fitted temperature below 1 sharpens the logits instead of softening them. The shipped+# `choice:11+` bucket is 0.1006, which multiplies them ~10x: a 0.24 top probability is published as+# 0.99, so a caller gating on confidence is told a coin flip is a certainty. No honest calibration+# needs to sharpen this hard, so refuse to apply one that does.+TEMP_MIN = 0.5+TEMP_MAX = 5.0+++def clamp_temperature(t, lo: float = TEMP_MIN, hi: float = TEMP_MAX) -> float:+ """A usable temperature: `t` confined to [lo, hi], falling back to 1.0 if it is not a number."""+ try:+ t = float(t)+ except (TypeError, ValueError):+ return 1.0+ if t != t or t in (float("inf"), float("-inf")): # NaN / inf+ return 1.0+ return min(hi, max(lo, t))++def amp_dtype(name: Optional[str]) -> torch.dtype:return torch.bfloat16 if name == "bf16" else torch.float16diff --git a/laya/lang.py b/laya/lang.pyindex e5891ba2..0d1ed7bc 100644--- a/laya/lang.py+++ b/laya/lang.py@@ -60,8 +60,27 @@ _STOP = {"questo", "questa", "anche", "come", "più", "sono", "nella", "alla"},"nl": {"het", "een", "van", "is", "op", "te", "dat", "niet", "met", "voor", "zijn", "aan","door", "maar", "ook", "worden", "deze", "naar", "wordt"},+ # Romanian words that its Romance neighbours do not share, so adding `ro` cannot steal a+ # French/Spanish/Italian/Portuguese state: `la`, `o`, `un`, `de`, `pe`, `ca` are deliberately+ # left out for that reason, and the diacritic signal below carries the rest.+ "ro": {"și", "să", "este", "sunt", "care", "pentru", "din", "dar", "după", "până", "fără",+ "ale", "lui", "în", "fost", "acum", "vreau", "trebuie", "foarte", "acest", "această",+ "acesta", "aceasta", "mi", "ți", "vă", "nu"},}-_NON_EN_DIACRITICS = set("àâäãáåçéèêëíìîïñóòôöõøúùûüýÿßæœđłşţğıåäö")+# Letters that ordinary English does not use. This is the signal that catches a Latin-script+# language we hold no stopwords for at all (Romanian, Polish, Czech, Turkish, Baltic, ...),+# which is the difference between routing it to the multilingual checkpoint and silently+# handing it to the English one.+_NON_EN_DIACRITICS = set(+ "àâäãáåçéèêëíìîïñóòôöõøúùûüýÿßæœ" # Western European+ "ăâîșțşţ" # Romanian+ "ąćęłńśźż" # Polish+ "čďěňřšťůž" # Czech / Slovak+ "őű" # Hungarian+ "ğı" # Turkish (text is lowercased before matching)+ "āēģīķļņūž" # Baltic+ "đ" # Serbo-Croatian / Vietnamese+)_WORD = re.compile(r"[^\W\d_]+", re.UNICODE)@@ -131,30 +150,59 @@ def script_profile(text: str) -> Dict[str, float]:return {k: v / total for k, v in counts.items() if v}-def guess_latin_language(text: str) -> Optional[str]:- """Best-effort language code for Latin-script text, or None when undecided.+# A diacritic rate above this is taken as evidence the text is not English, even when no+# stopword list matches it.+NON_EN_DIACRITIC_RATE = 0.02- Scores function-word hits per language and requires the winner to beat English by a margin,- so ordinary English is never misrouted. Short inputs usually return None on purpose.++def latin_profile(text: str) -> Dict[str, object]:+ """Evidence behind the Latin-script language guess.++ Returns `language` (may be None when undecided), `english_hits`, `diacritic_rate` and+ `looks_non_english`. `analyse` needs the evidence and not just the verdict, because+ "undecided" and "English" are different answers and only one of them is safe to send to the+ English checkpoint."""words = [w.lower() for w in _WORD.findall(text)]- if len(words) < 4:- return None- scores = {lg: sum(1 for w in words if w in sw) for lg, sw in _STOP.items()}lowered = text.lower()diac = sum(1 for ch in lowered if ch in _NON_EN_DIACRITICS)diac_rate = diac / max(1, len(lowered))+ non_english = diac_rate >= NON_EN_DIACRITIC_RATE+ if len(words) < 4:+ return {"language": None, "english_hits": 0, "diacritic_rate": diac_rate,+ "looks_non_english": non_english}++ scores = {lg: sum(1 for w in words if w in sw) for lg, sw in _STOP.items()}en = scores.get("en", 0)best_lg, best = max(((lg, s) for lg, s in scores.items() if lg != "en"),key=lambda kv: kv[1], default=(None, 0))- if best == 0 and diac_rate < 0.02:- return "en" if en else None- # a non-English language needs a clear margin over English function words+ # No stopword hit for any non-English language is no evidence for a *particular* one. Naming+ # the winner of a 0-0 tie invented a language (Romanian text was reported as French), so stay+ # undecided and let the diacritic rate speak.+ if best == 0:+ best_lg = None++ lang = Noneif best_lg and best >= max(2, en + 2):- return best_lg- if diac_rate >= 0.04 and best_lg and best >= en:- return best_lg- return "en" if en else None+ # a non-English language needs a clear margin over English function words+ lang = best_lg+ elif best_lg and non_english and best >= max(2, en):+ # Needs two hits here too. One shared function word ("para" in Turkish text) named Spanish+ # on the strength of the diacritics alone, which is a guess dressed as a detection.+ lang = best_lg+ elif en and not non_english:+ lang = "en"+ return {"language": lang, "english_hits": en, "diacritic_rate": diac_rate,+ "looks_non_english": non_english}+++def guess_latin_language(text: str) -> Optional[str]:+ """Best-effort language code for Latin-script text, or None when undecided.++ Scores function-word hits per language and requires the winner to beat English by a margin,+ so ordinary English is never misrouted. Short inputs usually return None on purpose.+ """+ return latin_profile(text)["language"]def analyse(state: Union[str, dict, list, None]) -> Dict[str, object]:@@ -169,13 +217,24 @@ def analyse(state: Union[str, dict, list, None]) -> Dict[str, object]:non_latin = round(1.0 - prof.get("latin", 0.0), 4) if prof else 0.0if script == "unknown":return {"script": "unknown", "script_profile": prof, "language": None,- "is_english": True, "non_latin_fraction": 0.0}+ "is_english": True, "language_undecided": True, "diacritic_rate": 0.0,+ "non_latin_fraction": 0.0}if script != "latin":return {"script": script, "script_profile": prof, "language": None,- "is_english": False, "non_latin_fraction": non_latin}- lang = guess_latin_language(text)+ "is_english": False, "language_undecided": True, "diacritic_rate": 0.0,+ "non_latin_fraction": non_latin}+ prof_lat = latin_profile(text)+ lang = prof_lat["language"]+ # Undecided is not English. Treating it as English sent every Latin-script language we hold no+ # stopwords for to the checkpoint that cannot read it, silently. When nothing identifies the+ # language, non-English letters are enough to prefer the multilingual checkpoint; text with no+ # such letters (including short English) still goes to the English one.+ undecided = lang is None+ english = lang == "en" or (undecided and not prof_lat["looks_non_english"])return {"script": "latin", "script_profile": prof, "language": lang,- "is_english": lang in (None, "en"), "non_latin_fraction": non_latin}+ "is_english": english, "language_undecided": undecided,+ "diacritic_rate": round(float(prof_lat["diacritic_rate"]), 4),+ "non_latin_fraction": non_latin}def is_english(state: Union[str, dict, list, None]) -> bool:diff --git a/laya/router.py b/laya/router.pyindex 1f123520..6577bec9 100644--- a/laya/router.py+++ b/laya/router.py@@ -298,7 +298,13 @@ class Router:det["script"], 100 * float(det["non_latin_fraction"]))elif not det["is_english"]:key = "multilingual"- reason = "Latin script but language looks like %r, not English" % det["language"]+ if det["language"]:+ reason = "Latin script but language looks like %r, not English" % det["language"]+ else:+ # Unidentified Latin-script language: routed on the non-English letters alone,+ # because no stopword list here covers it.+ reason = ("Latin script, language not identified but %.0f%% non-English letters; "+ "not safe for the English checkpoint" % (100 * float(det["diacritic_rate"])))else:key = "english"reason = "English Latin text"diff --git a/tests/test_router.py b/tests/test_router.pyindex c84b3149..0568f9ef 100644--- a/tests/test_router.py+++ b/tests/test_router.py@@ -6,6 +6,7 @@ import time as _timesys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))+from laya.common import QTYPES, TEMP_MAX, TEMP_MIN, clamp_temperature, temp_bucket # noqa: E402from laya.lang import analyse, detect_script, guess_latin_language, is_english, state_text # noqa: E402from laya.router import ( # noqa: E402BUNDLE_REPO,@@ -63,9 +64,42 @@ for label, text, want in ["facture qui a été payée le mois dernier avec la carte de crédit", False),("german long", "Der Kunde wurde zweimal belastet und möchte eine Rückerstattung für die ""Rechnung die nicht korrekt ist und auch nicht bezahlt wurde", False),+ # Latin-script languages with no stopword list of their own: reported in #35, where Romanian+ # states were handed to the English checkpoint (0.330 accuracy, 0.658 ECE on `ro`) instead of+ # the multilingual one. An unidentified language must never be assumed English.+ ("romanian", "Gătește-mi o rețetă de sarmale de post pentru mâine.", False),+ ("romanian invoice", "Am fost taxat de două ori pentru factura din luna martie și vreau banii", False),+ ("polish", "Klient został obciążony dwukrotnie i chce zwrot pieniędzy za fakturę", False),+ ("czech", "Zákazníkovi byla částka účtována dvakrát a žádá o vrácení peněz", False),+ ("turkish", "Müşteriden iki kez ücret alındı ve para iadesi istiyor lütfen yardım", False),+ ("vietnamese", "Khách hàng đã bị thu phí hai lần và muốn được hoàn tiền ngay", False),+ # English with the odd loanword must not tip over into the multilingual checkpoint+ ("english with loanwords", "We visited a cafe in Zurich and the naive assumption about the "+ "invoice was wrong, so please refund the duplicate charge", True),]:check("is_english/" + label, is_english(text), want)+# Undecided is reported as undecided rather than dressed up as a detection: a single shared+# function word used to name a language ("para" in Turkish text was called Spanish).+check("latin/undecided is flagged", analyse("Müşteriden iki kez ücret alındı ve para iadesi istiyor")["language_undecided"], True)+check("latin/undecided names no language", analyse("Müşteriden iki kez ücret alındı ve para iadesi istiyor")["language"], None)+check("latin/english is not undecided", analyse("Please refund the duplicate charge on the invoice")["language_undecided"], False)+check("latin/diacritic rate reported", analyse("Gătește-mi o rețetă de sarmale")["diacritic_rate"] > 0.02, True)+check("latin/english has no diacritics", analyse("Please refund the duplicate charge today")["diacritic_rate"], 0.0)+# every branch of analyse() reports the same keys, so a caller can read one without guarding+_KEYS = {"script", "script_profile", "language", "is_english", "language_undecided",+ "diacritic_rate", "non_latin_fraction"}+for label, text in [("english", "Please refund the duplicate charge"), ("hindi", "ग्राहक से दो बार"),+ ("romanian", "Gătește-mi o rețetă de sarmale"), ("no letters", "12345 ???")]:+ check("analyse/keys " + label, set(analyse(text)), _KEYS)+# A 0-0 tie between non-English stopword lists is no evidence for any of them+check("latin_lang/zero tie invents nothing", guess_latin_language("Cât e ora acum la Tokyo"), None)++# Known limitation, kept visible on purpose: Romanian short enough to carry no diacritics and an+# English function word ("in") still reads as English. A real LID model is the fix, not more+# stopwords -- see the discussion in #35.+check("latin/KNOWN GAP romanian without diacritics", is_english("Care este ora in Tokyo?"), True)+# --------------------------------------------------------------------- Latin language guessfor label, text, want in [@@ -174,6 +208,42 @@ check("route/custom default", Router(default="multilingual").route("12345", Q_GE"multilingual")+# --------------------------------------------------------------------- unknown-Latin routing (#35)+_r_lat = Router()+for label, text in [+ ("romanian", "Gătește-mi o rețetă de sarmale de post pentru mâine."),+ ("romanian agent request", "Exportă APK-ul pentru Android și pune-l pe Drive ca să-l instalez."),+ ("polish", "Klient został obciążony dwukrotnie i chce zwrot pieniędzy za fakturę"),+ ("turkish", "Müşteriden iki kez ücret alındı ve para iadesi istiyor lütfen yardım"),+]:+ check("route/unknown latin " + label, _r_lat.route(text).model, "multilingual")+# and the reason must say what it actually routed on, not report a language it did not identify+check("route/undecided reason mentions letters",+ "not identified" in _r_lat.route("Müşteriden iki kez ücret alındı ve para iadesi istiyor").reason, True)+check("route/english still english",+ _r_lat.route("Please refund the duplicate charge on invoice 4411 today.").model, "english")+check("route/short english still english", _r_lat.route("refund me").model, "english")+++# --------------------------------------------------------------------- temperature clamp (#35)+# A fitted temperature below 1 sharpens logits. The shipped `choice:11+` bucket is 0.1006, which+# turned a 0.24 top probability into 0.99 confidence on 13-option skill routing.+check("clamp/pathological sharpening", clamp_temperature(0.1006), 0.5)+check("clamp/shipped choice:11+ is rejected", clamp_temperature(0.10058280825614929), TEMP_MIN)+check("clamp/legitimate value untouched", clamp_temperature(1.7601518630981445), 1.7601518630981445)+check("clamp/neutral untouched", clamp_temperature(1.0), 1.0)+check("clamp/upper bound", clamp_temperature(9.0), TEMP_MAX)+check("clamp/zero", clamp_temperature(0.0), TEMP_MIN)+check("clamp/negative", clamp_temperature(-3.0), TEMP_MIN)+check("clamp/none falls back to neutral", clamp_temperature(None), 1.0)+check("clamp/garbage falls back to neutral", clamp_temperature("x"), 1.0)+check("clamp/nan falls back to neutral", clamp_temperature(float("nan")), 1.0)+check("clamp/inf falls back to neutral", clamp_temperature(float("inf")), 1.0)+check("clamp/bounds are sane", TEMP_MIN <= 1.0 <= TEMP_MAX, True)+# 13 options is the bucket the reported skill-router landed in+check("clamp/13 options is the 11+ bucket", temp_bucket(QTYPES["choice"], 13), "choice:11+")++# --------------------------------------------------------------------- LRU bookkeepingclass _Stub:def __init__(self, name):