Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

fix: never assume an unidentified Latin language is English; clamp temperatures (#35) (#42)

Avinash8e58a6cd8ddb
diff --git a/laya/agent.py b/laya/agent.py
index c63325be..15e2eb56 100644
--- a/laya/agent.py
+++ b/laya/agent.py
@@ -1,6 +1,7 @@
"""High-level inference runtime for laya System 1 decision models."""
import json
import os
+import warnings
from typing import Any, Dict, Optional, Union
import numpy as np
@@ -8,9 +9,12 @@ import torch
from .common import (
QTYPES,
+ TEMP_MAX,
+ TEMP_MIN,
amp_dtype,
build_model,
build_sequence,
+ clamp_temperature,
collate_items,
confidence_from_probs,
render_options,
@@ -197,8 +201,23 @@ class Agent:
except Exception:
pass
- self.temperature = self.cfg.get("temperature", [1.0, 1.0, 1.0])
- self.temperature_by_options = self.cfg.get("temperature_by_options", {})
+ # Keep what the checkpoint shipped for inspection, but only ever apply clamped values:
+ # some buckets are fitted to sharpen rather than soften (see clamp_temperature).
+ self.temperature_raw = self.cfg.get("temperature", [1.0, 1.0, 1.0])
+ self.temperature_by_options_raw = self.cfg.get("temperature_by_options", {})
+ self.temperature = [clamp_temperature(t) for t in self.temperature_raw]
+ self.temperature_by_options = {k: clamp_temperature(v)
+ for k, v in self.temperature_by_options_raw.items()}
+ rejected = ["%s=%.4g" % (k, float(v)) for k, v in self.temperature_by_options_raw.items()
+ if clamp_temperature(v) != float(v)]
+ rejected += ["temperature[%d]=%.4g" % (i, float(t)) for i, t in enumerate(self.temperature_raw)
+ if clamp_temperature(t) != float(t)]
+ if rejected:
+ warnings.warn(
+ "laya: this checkpoint ships temperatures outside [%g, %g] which would distort "
+ "confidence; clamping %s. Treat confidence from the affected buckets as uncalibrated."
+ % (TEMP_MIN, TEMP_MAX, ", ".join(rejected)),
+ RuntimeWarning, stacklevel=2)
self.dtype = amp_dtype(self.cfg.get("amp_dtype", "fp16"))
if self.device.type == "cuda" and torch.cuda.get_device_capability(self.device)[0] < 8:
@@ -308,7 +327,7 @@ class Agent:
k = len(items[r]["markers"])
qt = QTYPES[q["t"]]
t_scale = self.temperature_by_options.get(temp_bucket(qt, k), self.temperature[qt])
- z = logits[r, :k] / max(1e-3, float(t_scale))
+ z = logits[r, :k] / t_scale
p = np.exp(z - z.max())
p = p / p.sum()
diff --git a/laya/common.py b/laya/common.py
index fa9c6968..fb900053 100644
--- a/laya/common.py
+++ b/laya/common.py
@@ -221,6 +221,25 @@ def temp_bucket(qtype: int, k: int) -> str:
return "%s:%s" % (QTYPE_NAMES[int(qtype)], size)
+# A fitted temperature below 1 sharpens the logits instead of softening them. The shipped
+# `choice:11+` bucket is 0.1006, which multiplies them ~10x: a 0.24 top probability is published as
+# 0.99, so a caller gating on confidence is told a coin flip is a certainty. No honest calibration
+# needs to sharpen this hard, so refuse to apply one that does.
+TEMP_MIN = 0.5
+TEMP_MAX = 5.0
+
+
+def clamp_temperature(t, lo: float = TEMP_MIN, hi: float = TEMP_MAX) -> float:
+ """A usable temperature: `t` confined to [lo, hi], falling back to 1.0 if it is not a number."""
+ try:
+ t = float(t)
+ except (TypeError, ValueError):
+ return 1.0
+ if t != t or t in (float("inf"), float("-inf")): # NaN / inf
+ return 1.0
+ return min(hi, max(lo, t))
+
+
def amp_dtype(name: Optional[str]) -> torch.dtype:
return torch.bfloat16 if name == "bf16" else torch.float16
diff --git a/laya/lang.py b/laya/lang.py
index e5891ba2..0d1ed7bc 100644
--- a/laya/lang.py
+++ b/laya/lang.py
@@ -60,8 +60,27 @@ _STOP = {
"questo", "questa", "anche", "come", "più", "sono", "nella", "alla"},
"nl": {"het", "een", "van", "is", "op", "te", "dat", "niet", "met", "voor", "zijn", "aan",
"door", "maar", "ook", "worden", "deze", "naar", "wordt"},
+ # Romanian words that its Romance neighbours do not share, so adding `ro` cannot steal a
+ # French/Spanish/Italian/Portuguese state: `la`, `o`, `un`, `de`, `pe`, `ca` are deliberately
+ # left out for that reason, and the diacritic signal below carries the rest.
+ "ro": {"și", "să", "este", "sunt", "care", "pentru", "din", "dar", "după", "până", "fără",
+ "ale", "lui", "în", "fost", "acum", "vreau", "trebuie", "foarte", "acest", "această",
+ "acesta", "aceasta", "mi", "ți", "vă", "nu"},
}
-_NON_EN_DIACRITICS = set("àâäãáåçéèêëíìîïñóòôöõøúùûüýÿßæœđłşţğıåäö")
+# Letters that ordinary English does not use. This is the signal that catches a Latin-script
+# language we hold no stopwords for at all (Romanian, Polish, Czech, Turkish, Baltic, ...),
+# which is the difference between routing it to the multilingual checkpoint and silently
+# handing it to the English one.
+_NON_EN_DIACRITICS = set(
+ "àâäãáåçéèêëíìîïñóòôöõøúùûüýÿßæœ" # Western European
+ "ăâîșțşţ" # Romanian
+ "ąćęłńśźż" # Polish
+ "čďěňřšťůž" # Czech / Slovak
+ "őű" # Hungarian
+ "ğı" # Turkish (text is lowercased before matching)
+ "āēģīķļņūž" # Baltic
+ "đ" # Serbo-Croatian / Vietnamese
+)
_WORD = re.compile(r"[^\W\d_]+", re.UNICODE)
@@ -131,30 +150,59 @@ def script_profile(text: str) -> Dict[str, float]:
return {k: v / total for k, v in counts.items() if v}
-def guess_latin_language(text: str) -> Optional[str]:
- """Best-effort language code for Latin-script text, or None when undecided.
+# A diacritic rate above this is taken as evidence the text is not English, even when no
+# stopword list matches it.
+NON_EN_DIACRITIC_RATE = 0.02
- Scores function-word hits per language and requires the winner to beat English by a margin,
- so ordinary English is never misrouted. Short inputs usually return None on purpose.
+
+def latin_profile(text: str) -> Dict[str, object]:
+ """Evidence behind the Latin-script language guess.
+
+ Returns `language` (may be None when undecided), `english_hits`, `diacritic_rate` and
+ `looks_non_english`. `analyse` needs the evidence and not just the verdict, because
+ "undecided" and "English" are different answers and only one of them is safe to send to the
+ English checkpoint.
"""
words = [w.lower() for w in _WORD.findall(text)]
- if len(words) < 4:
- return None
- scores = {lg: sum(1 for w in words if w in sw) for lg, sw in _STOP.items()}
lowered = text.lower()
diac = sum(1 for ch in lowered if ch in _NON_EN_DIACRITICS)
diac_rate = diac / max(1, len(lowered))
+ non_english = diac_rate >= NON_EN_DIACRITIC_RATE
+ if len(words) < 4:
+ return {"language": None, "english_hits": 0, "diacritic_rate": diac_rate,
+ "looks_non_english": non_english}
+
+ scores = {lg: sum(1 for w in words if w in sw) for lg, sw in _STOP.items()}
en = scores.get("en", 0)
best_lg, best = max(((lg, s) for lg, s in scores.items() if lg != "en"),
key=lambda kv: kv[1], default=(None, 0))
- if best == 0 and diac_rate < 0.02:
- return "en" if en else None
- # a non-English language needs a clear margin over English function words
+ # No stopword hit for any non-English language is no evidence for a *particular* one. Naming
+ # the winner of a 0-0 tie invented a language (Romanian text was reported as French), so stay
+ # undecided and let the diacritic rate speak.
+ if best == 0:
+ best_lg = None
+
+ lang = None
if best_lg and best >= max(2, en + 2):
- return best_lg
- if diac_rate >= 0.04 and best_lg and best >= en:
- return best_lg
- return "en" if en else None
+ # a non-English language needs a clear margin over English function words
+ lang = best_lg
+ elif best_lg and non_english and best >= max(2, en):
+ # Needs two hits here too. One shared function word ("para" in Turkish text) named Spanish
+ # on the strength of the diacritics alone, which is a guess dressed as a detection.
+ lang = best_lg
+ elif en and not non_english:
+ lang = "en"
+ return {"language": lang, "english_hits": en, "diacritic_rate": diac_rate,
+ "looks_non_english": non_english}
+
+
+def guess_latin_language(text: str) -> Optional[str]:
+ """Best-effort language code for Latin-script text, or None when undecided.
+
+ Scores function-word hits per language and requires the winner to beat English by a margin,
+ so ordinary English is never misrouted. Short inputs usually return None on purpose.
+ """
+ return latin_profile(text)["language"]
def analyse(state: Union[str, dict, list, None]) -> Dict[str, object]:
@@ -169,13 +217,24 @@ def analyse(state: Union[str, dict, list, None]) -> Dict[str, object]:
non_latin = round(1.0 - prof.get("latin", 0.0), 4) if prof else 0.0
if script == "unknown":
return {"script": "unknown", "script_profile": prof, "language": None,
- "is_english": True, "non_latin_fraction": 0.0}
+ "is_english": True, "language_undecided": True, "diacritic_rate": 0.0,
+ "non_latin_fraction": 0.0}
if script != "latin":
return {"script": script, "script_profile": prof, "language": None,
- "is_english": False, "non_latin_fraction": non_latin}
- lang = guess_latin_language(text)
+ "is_english": False, "language_undecided": True, "diacritic_rate": 0.0,
+ "non_latin_fraction": non_latin}
+ prof_lat = latin_profile(text)
+ lang = prof_lat["language"]
+ # Undecided is not English. Treating it as English sent every Latin-script language we hold no
+ # stopwords for to the checkpoint that cannot read it, silently. When nothing identifies the
+ # language, non-English letters are enough to prefer the multilingual checkpoint; text with no
+ # such letters (including short English) still goes to the English one.
+ undecided = lang is None
+ english = lang == "en" or (undecided and not prof_lat["looks_non_english"])
return {"script": "latin", "script_profile": prof, "language": lang,
- "is_english": lang in (None, "en"), "non_latin_fraction": non_latin}
+ "is_english": english, "language_undecided": undecided,
+ "diacritic_rate": round(float(prof_lat["diacritic_rate"]), 4),
+ "non_latin_fraction": non_latin}
def is_english(state: Union[str, dict, list, None]) -> bool:
diff --git a/laya/router.py b/laya/router.py
index 1f123520..6577bec9 100644
--- a/laya/router.py
+++ b/laya/router.py
@@ -298,7 +298,13 @@ class Router:
det["script"], 100 * float(det["non_latin_fraction"]))
elif not det["is_english"]:
key = "multilingual"
- reason = "Latin script but language looks like %r, not English" % det["language"]
+ if det["language"]:
+ reason = "Latin script but language looks like %r, not English" % det["language"]
+ else:
+ # Unidentified Latin-script language: routed on the non-English letters alone,
+ # because no stopword list here covers it.
+ reason = ("Latin script, language not identified but %.0f%% non-English letters; "
+ "not safe for the English checkpoint" % (100 * float(det["diacritic_rate"])))
else:
key = "english"
reason = "English Latin text"
diff --git a/tests/test_router.py b/tests/test_router.py
index c84b3149..0568f9ef 100644
--- a/tests/test_router.py
+++ b/tests/test_router.py
@@ -6,6 +6,7 @@ import time as _time
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+from laya.common import QTYPES, TEMP_MAX, TEMP_MIN, clamp_temperature, temp_bucket # noqa: E402
from laya.lang import analyse, detect_script, guess_latin_language, is_english, state_text # noqa: E402
from laya.router import ( # noqa: E402
BUNDLE_REPO,
@@ -63,9 +64,42 @@ for label, text, want in [
"facture qui a été payée le mois dernier avec la carte de crédit", False),
("german long", "Der Kunde wurde zweimal belastet und möchte eine Rückerstattung für die "
"Rechnung die nicht korrekt ist und auch nicht bezahlt wurde", False),
+ # Latin-script languages with no stopword list of their own: reported in #35, where Romanian
+ # states were handed to the English checkpoint (0.330 accuracy, 0.658 ECE on `ro`) instead of
+ # the multilingual one. An unidentified language must never be assumed English.
+ ("romanian", "Gătește-mi o rețetă de sarmale de post pentru mâine.", False),
+ ("romanian invoice", "Am fost taxat de două ori pentru factura din luna martie și vreau banii", False),
+ ("polish", "Klient został obciążony dwukrotnie i chce zwrot pieniędzy za fakturę", False),
+ ("czech", "Zákazníkovi byla částka účtována dvakrát a žádá o vrácení peněz", False),
+ ("turkish", "Müşteriden iki kez ücret alındı ve para iadesi istiyor lütfen yardım", False),
+ ("vietnamese", "Khách hàng đã bị thu phí hai lần và muốn được hoàn tiền ngay", False),
+ # English with the odd loanword must not tip over into the multilingual checkpoint
+ ("english with loanwords", "We visited a cafe in Zurich and the naive assumption about the "
+ "invoice was wrong, so please refund the duplicate charge", True),
]:
check("is_english/" + label, is_english(text), want)
+# Undecided is reported as undecided rather than dressed up as a detection: a single shared
+# function word used to name a language ("para" in Turkish text was called Spanish).
+check("latin/undecided is flagged", analyse("Müşteriden iki kez ücret alındı ve para iadesi istiyor")["language_undecided"], True)
+check("latin/undecided names no language", analyse("Müşteriden iki kez ücret alındı ve para iadesi istiyor")["language"], None)
+check("latin/english is not undecided", analyse("Please refund the duplicate charge on the invoice")["language_undecided"], False)
+check("latin/diacritic rate reported", analyse("Gătește-mi o rețetă de sarmale")["diacritic_rate"] > 0.02, True)
+check("latin/english has no diacritics", analyse("Please refund the duplicate charge today")["diacritic_rate"], 0.0)
+# every branch of analyse() reports the same keys, so a caller can read one without guarding
+_KEYS = {"script", "script_profile", "language", "is_english", "language_undecided",
+ "diacritic_rate", "non_latin_fraction"}
+for label, text in [("english", "Please refund the duplicate charge"), ("hindi", "ग्राहक से दो बार"),
+ ("romanian", "Gătește-mi o rețetă de sarmale"), ("no letters", "12345 ???")]:
+ check("analyse/keys " + label, set(analyse(text)), _KEYS)
+# A 0-0 tie between non-English stopword lists is no evidence for any of them
+check("latin_lang/zero tie invents nothing", guess_latin_language("Cât e ora acum la Tokyo"), None)
+
+# Known limitation, kept visible on purpose: Romanian short enough to carry no diacritics and an
+# English function word ("in") still reads as English. A real LID model is the fix, not more
+# stopwords -- see the discussion in #35.
+check("latin/KNOWN GAP romanian without diacritics", is_english("Care este ora in Tokyo?"), True)
+
# --------------------------------------------------------------------- Latin language guess
for label, text, want in [
@@ -174,6 +208,42 @@ check("route/custom default", Router(default="multilingual").route("12345", Q_GE
"multilingual")
+# --------------------------------------------------------------------- unknown-Latin routing (#35)
+_r_lat = Router()
+for label, text in [
+ ("romanian", "Gătește-mi o rețetă de sarmale de post pentru mâine."),
+ ("romanian agent request", "Exportă APK-ul pentru Android și pune-l pe Drive ca să-l instalez."),
+ ("polish", "Klient został obciążony dwukrotnie i chce zwrot pieniędzy za fakturę"),
+ ("turkish", "Müşteriden iki kez ücret alındı ve para iadesi istiyor lütfen yardım"),
+]:
+ check("route/unknown latin " + label, _r_lat.route(text).model, "multilingual")
+# and the reason must say what it actually routed on, not report a language it did not identify
+check("route/undecided reason mentions letters",
+ "not identified" in _r_lat.route("Müşteriden iki kez ücret alındı ve para iadesi istiyor").reason, True)
+check("route/english still english",
+ _r_lat.route("Please refund the duplicate charge on invoice 4411 today.").model, "english")
+check("route/short english still english", _r_lat.route("refund me").model, "english")
+
+
+# --------------------------------------------------------------------- temperature clamp (#35)
+# A fitted temperature below 1 sharpens logits. The shipped `choice:11+` bucket is 0.1006, which
+# turned a 0.24 top probability into 0.99 confidence on 13-option skill routing.
+check("clamp/pathological sharpening", clamp_temperature(0.1006), 0.5)
+check("clamp/shipped choice:11+ is rejected", clamp_temperature(0.10058280825614929), TEMP_MIN)
+check("clamp/legitimate value untouched", clamp_temperature(1.7601518630981445), 1.7601518630981445)
+check("clamp/neutral untouched", clamp_temperature(1.0), 1.0)
+check("clamp/upper bound", clamp_temperature(9.0), TEMP_MAX)
+check("clamp/zero", clamp_temperature(0.0), TEMP_MIN)
+check("clamp/negative", clamp_temperature(-3.0), TEMP_MIN)
+check("clamp/none falls back to neutral", clamp_temperature(None), 1.0)
+check("clamp/garbage falls back to neutral", clamp_temperature("x"), 1.0)
+check("clamp/nan falls back to neutral", clamp_temperature(float("nan")), 1.0)
+check("clamp/inf falls back to neutral", clamp_temperature(float("inf")), 1.0)
+check("clamp/bounds are sane", TEMP_MIN <= 1.0 <= TEMP_MAX, True)
+# 13 options is the bucket the reported skill-router landed in
+check("clamp/13 options is the 11+ bucket", temp_bucket(QTYPES["choice"], 13), "choice:11+")
+
+
# --------------------------------------------------------------------- LRU bookkeeping
class _Stub:
def __init__(self, name):