Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
feat(lang): detect Swedish with measured MASSIVE results
Daniel Nylanderee499bb09558
diff --git a/laya-ts/src/lang.ts b/laya-ts/src/lang.tsindex 2b16adcc..7da5e962 100644--- a/laya-ts/src/lang.ts+++ b/laya-ts/src/lang.ts@@ -77,6 +77,16 @@ const STOP: Record<string, Set<string>> = {"delle", "dello", "degli", "agli", "alle", "col"]),nl: new Set(["het", "een", "van", "is", "op", "te", "dat", "niet", "met", "voor", "zijn", "aan","door", "maar", "ook", "worden", "deze", "naar", "wordt"]),+ // Keep Swedish routing in parity with Python. Distinctive words help ASCII-normalised ticket text;+ // shared Nordic words are tracked separately so Danish and Norwegian are not named Swedish.+ sv: new Set(["jag", "är", "och", "inte", "att", "från", "till", "behöver", "får", "skulle", "ska",+ "vill", "måste", "också", "dessa", "detta", "säger", "upp", "utan", "mitt", "min", "om",+ "kommer", "här", "två", "vi", "nästa", "gör", "göra", "hjälp", "hjälpa", "mig",+ "återbetalning", "återbetala", "faktura", "gång", "gånger", "hittar", "inställningen",+ "inställningarna", "lösenord", "när", "öppnar", "spårningen", "aterbetalning", "aterbetala",+ "behover", "fel", "ganger", "hjalp", "hjalpa", "installningen", "installningarna", "kraschar",+ "kvittot", "losenord", "nar", "oppnar", "paket", "skicka", "sparningen", "tva", "uppdaterats",+ "blivit", "debiterade", "appen"]),ro: new Set(["și", "să", "este", "sunt", "care", "pentru", "din", "dar", "după", "până", "fără","ale", "lui", "în", "fost", "acum", "vreau", "trebuie", "foarte", "acest", "această","acesta", "aceasta", "mi", "ți", "vă", "nu"]),@@ -137,8 +147,19 @@ const SHARED_WORDS: Set<string> = (() => {for (const [w, n] of counts) if (n > 1) out.add(w);return out;})();+// Danish and Norwegian share many common words with Swedish; they can add score but cannot name sv alone.+const NORDIC_OVERLAP_WORDS = new Set(["hej", "ja", "nej", "jo", "tack", "mig", "min", "om", "kommer", "får", "skulle", "vi"]);+for (const word of NORDIC_OVERLAP_WORDS) SHARED_WORDS.add(word);const EN_ONLY_WORDS = new Set([...STOP["en"]].filter((w) => !SHARED_WORDS.has(w)));+// Short support fragments need a narrower vocabulary than the general four-word language guess.+// Generic words such as "fel" and "hjälp" are deliberately omitted from this subset.+const SHORT_SWEDISH_WORDS = new Set([+ "åtkomst", "atkomst", "lösenord", "losenord", "fakturan", "betalningen", "inloggningen",+ "glömt", "glomt", "behöver", "behover", "återbetalning", "aterbetalning", "kvitto", "kvittot",+ "spårningen", "sparningen", "inställningen", "installningen", "felmeddelande", "abonnemanget",+]);+// JS `\w` is ASCII-only, so Python's `[^\W\d_]` needs the Unicode classes spelled out (Nl/No are in Python's `\w`).const WORD_RE = /[\p{L}\p{Nl}\p{No}]+/gu;// Lookbehind for the same reason as the Python side (see laya/lang.py): without it the@@ -298,8 +319,13 @@ export function latinProfile(text: string): LatinProfile {}const diacRate = diac / Math.max(1, lowered.length);const nonEnglish = diacRate >= NON_EN_DIACRITIC_RATE;+ const nordicOverlap = words.some((w) => NORDIC_OVERLAP_WORDS.has(w)) &&+ !words.some((w) => EN_ONLY_WORDS.has(w));+ if (words.length > 1 && words.length < 4 && words.some((w) => SHORT_SWEDISH_WORDS.has(w))) {+ return { language: "sv", englishHits: 0, diacriticRate: diacRate, looksNonEnglish: nonEnglish };+ }if (words.length < 4) {- return { language: null, englishHits: 0, diacriticRate: diacRate, looksNonEnglish: nonEnglish };+ return { language: null, englishHits: 0, diacriticRate: diacRate, looksNonEnglish: nonEnglish || nordicOverlap };}const scores: Record<string, number> = {};for (const [lg, sw] of Object.entries(STOP)) {@@ -320,12 +346,16 @@ export function latinProfile(text: string): LatinProfile {let lang: string | null = null;if (bestLg && best >= Math.max(2, en + 2)) {lang = bestLg;+ } else if (bestLg === "sv" && words.includes("inte") && words.includes("kan") &&+ ["kan", "jag", "vi"].includes(words[0]) && en <= 1) {+ // Short login requests like "kan inte logga in" contain one English-shaped token (`in`).+ lang = bestLg;} else if (bestLg && nonEnglish && best >= Math.max(2, en)) {lang = bestLg;} else if (en && (!nonEnglish || englishRescuedByWords(wordSet, diacRate))) {lang = "en";}- return { language: lang, englishHits: en, diacriticRate: diacRate, looksNonEnglish: nonEnglish };+ return { language: lang, englishHits: en, diacriticRate: diacRate, looksNonEnglish: nonEnglish || (lang === null && nordicOverlap) };}export function guessLatinLanguage(text: string): string | null {diff --git a/laya-ts/tests/lang.test.ts b/laya-ts/tests/lang.test.tsindex a04b2b9e..5b765bb5 100644--- a/laya-ts/tests/lang.test.ts+++ b/laya-ts/tests/lang.test.ts@@ -208,7 +208,7 @@ describe("lang", () => {["sluk lyset i soveværelset", null],["kan jeg få en refundering for det dobbelte beløb", null],["stäng av ljuset i sovrummet", null],- ["jag vill ha en återbetalning för den dubbla avgiften", null],+ ["jag vill ha en återbetalning för den dubbla avgiften", "sv"],["The naïve façade needs a fresh coat of paint", null],])("does not rescue accented non-English text: %s", (text, language) => {expect(guessLatinLanguage(text)).toBe(language);diff --git a/laya/lang.py b/laya/lang.pyindex f8f0f0d5..2e8a740f 100644--- a/laya/lang.py+++ b/laya/lang.py@@ -118,6 +118,19 @@ _STOP = {"delle", "dello", "degli", "agli", "alle", "col"},"nl": {"het", "een", "van", "is", "op", "te", "dat", "niet", "met", "voor", "zijn", "aan","door", "maar", "ook", "worden", "deze", "naar", "wordt"},+ # Swedish function words and common auxiliaries. Several overlap with English or German+ # (`i`, `kan`, `har`), so the distinctive words below are what lets Swedish text survive an+ # ASCII-normalising ticket pipeline without treating one stray Nordic letter as the only hint.+ "sv": {"jag", "är", "och", "inte", "att", "från", "till", "behöver", "får", "skulle", "ska",+ "vill", "måste", "också", "dessa", "detta", "säger", "upp", "utan", "mitt", "min", "om",+ "kommer", "här", "två", "vi", "nästa", "gör", "göra",+ "hjälp", "hjälpa", "mig", "återbetalning", "återbetala", "faktura", "gång", "gånger",+ "hittar", "inställningen", "inställningarna", "lösenord", "när", "öppnar", "spårningen",+ # Common spellings from ticket systems that strip Swedish diacritics.+ "aterbetalning", "aterbetala", "behover", "fel", "ganger", "hjalp", "hjalpa",+ "installningen", "installningarna", "kraschar", "kvittot", "losenord", "nar",+ "oppnar", "paket", "skicka", "sparningen", "tva", "uppdaterats", "blivit", "debiterade",+ "appen"},# Romanian words that its Romance neighbours do not share, so adding `ro` cannot steal a# French/Spanish/Italian/Portuguese state: `la`, `o`, `un`, `de`, `pe`, `ca` are deliberately# left out for that reason, and the diacritic signal below carries the rest.@@ -153,6 +166,18 @@ _STOP = {"hər", "nə", "kimi", "görə", "sonra", "əgər", "eger", "deyil", "lakin", "amma","ancaq", "artıq", "artiq", "də", "isə", "həm", "yalnız", "yalniz"},}++# Short support fragments often consist of only two or three words, so they do not reach the+# four-token minimum used by the general language guess. These spellings are specific enough to+# identify Swedish in that narrow case; generic words such as `fel`, `hjälp`, `paket` and `appen`+# are deliberately not sufficient on their own. Keep ASCII-normalised variants beside the forms+# users commonly type without Swedish characters.+_SHORT_SWEDISH_WORDS = {+ "åtkomst", "atkomst", "lösenord", "losenord", "fakturan", "betalningen", "inloggningen",+ "glömt", "glomt", "behöver", "behover", "återbetalning", "aterbetalning", "kvitto", "kvittot",+ "spårningen", "sparningen", "inställningen", "installningen", "felmeddelande",+ "abonnemanget",+}# Letters that ordinary English does not use. This is the signal that catches a Latin-script# language we hold no stopwords for at all (Romanian, Polish, Czech, Turkish, Baltic, ...),# which is the difference between routing it to the multilingual checkpoint and silently@@ -174,6 +199,13 @@ _NON_EN_DIACRITICS = set(# way), though it still counts toward the total of a language that also matched a word of its own._SHARED_WORDS = {w for w in {word for words in _STOP.values() for word in words}if sum(w in words for words in _STOP.values()) > 1}+# Danish is not in `_STOP`, but several of its common words also occur in the Swedish list.+# Do not let these words alone name a Danish sentence as Swedish; they remain useful score hits+# when another, more distinctive Swedish word is present.+_NORDIC_OVERLAP_WORDS = {+ "hej", "ja", "nej", "jo", "tack", "mig", "min", "om", "kommer", "får", "skulle", "vi",+}+_SHARED_WORDS.update(_NORDIC_OVERLAP_WORDS)# English function words no other list holds (`in`, `is`, `as`, `was` are shared with German,# Dutch and Portuguese). They alone carry the English rescue of `latin_profile`._EN_ONLY_WORDS = _STOP["en"] - _SHARED_WORDS@@ -396,9 +428,10 @@ def latin_profile(text: str) -> Dict[str, object]:"""Evidence behind the Latin-script language guess.Returns `language` (may be None when undecided), `english_hits`, `diacritic_rate` and- `looks_non_english`. `analyse` needs the evidence and not just the verdict, because- "undecided" and "English" are different answers and only one of them is safe to send to the- English checkpoint.+ `looks_non_english`. The latter can be true for non-English diacritics or an overlapping+ Swedish-Danish marker even when this heuristic cannot name the language. `analyse` needs the+ evidence and not just the verdict, because "undecided" and "English" are different answers+ and only one of them is safe to send to the English checkpoint.A non-English language is only named when it matched at least one word that no other listclaims: shared function words alone (`la`, `e`, `o`) identify no particular language.@@ -409,9 +442,13 @@ def latin_profile(text: str) -> Dict[str, object]:diac = sum(1 for ch in lowered if ch in _NON_EN_DIACRITICS)diac_rate = diac / max(1, len(lowered))non_english = diac_rate >= NON_EN_DIACRITIC_RATE+ nordic_overlap = bool(set(words) & _NORDIC_OVERLAP_WORDS) and not bool(set(words) & _EN_ONLY_WORDS)+ if 1 < len(words) < 4 and set(words) & _SHORT_SWEDISH_WORDS:+ return {"language": "sv", "english_hits": 0, "diacritic_rate": diac_rate,+ "looks_non_english": non_english}if len(words) < 4:return {"language": None, "english_hits": 0, "diacritic_rate": diac_rate,- "looks_non_english": non_english}+ "looks_non_english": non_english or nordic_overlap}# A collision word counts once however often it repeats; every other word counts its hits.counts = Counter(words)@@ -431,6 +468,12 @@ def latin_profile(text: str) -> Dict[str, object]:if best_lg and best >= max(2, en + 2):# a non-English language needs a clear margin over English function wordslang = best_lg+ elif (best_lg == "sv" and "inte" in words and "kan" in words+ and words[0] in {"kan", "jag", "vi"} and en <= 1):+ # Short login requests such as "kan inte logga in" carry a distinctive Swedish phrase+ # but also one English-shaped token (`in`). Do not treat `kan` alone as Swedish: it is+ # common in Danish and Norwegian too.+ lang = best_lgelif best_lg and non_english and best >= max(2, en):# Needs two hits here too. One shared function word ("para" in Turkish text) named Spanish# on the strength of the diacritics alone, which is a guess dressed as a detection.@@ -438,7 +481,7 @@ def latin_profile(text: str) -> Dict[str, object]:elif en and (not non_english or _english_rescued_by_words(words, diac_rate)):lang = "en"return {"language": lang, "english_hits": en, "diacritic_rate": diac_rate,- "looks_non_english": non_english}+ "looks_non_english": non_english or (lang is None and nordic_overlap)}def guess_latin_language(text: str) -> Optional[str]:@@ -543,8 +586,8 @@ def _analyse_text(text: str) -> Dict[str, object]:lang = prof_lat["language"]# Undecided is not English. Treating it as English sent every Latin-script language we hold no# stopwords for to the checkpoint that cannot read it, silently. When nothing identifies the- # language, non-English letters are enough to prefer the multilingual checkpoint; text with no- # such letters (including short English) still goes to the English one.+ # language, non-English letters or a shared Swedish-Danish marker can still prefer the+ # multilingual checkpoint; text with neither signal still goes to the English one.undecided = lang is Noneenglish = lang == "en" or (undecided and not prof_lat["looks_non_english"])return {"script": "latin", "script_profile": prof, "language": lang,diff --git a/research/evals/massive_route_comparison.md b/research/evals/massive_route_comparison.mdnew file mode 100644index 00000000..f6fdd1db--- /dev/null+++ b/research/evals/massive_route_comparison.md@@ -0,0 +1,62 @@+# Swedish routing: MASSIVE full-test comparison++This report compares the `laya.lang` detector on all 51 locale files in the pinned+`mteb/amazon_massive_intent` test split (dataset revision+`940fd47a81eaa7f2cc7b129674d945d618ac38c2`). Each locale contributes 2,974 examples,+for 151,674 examples total. The comparison is model-free: it evaluates the same text+against the baseline `main` detector and the proposed detector without loading either+checkpoint.++## Method++The expected route is `english` for the `en` locale and `multilingual` for every other+locale. The script calls the repository's real `Router._route` for both detectors, so+all other routing rules stay the same, including the configured `english` default when+Latin text is undecided. Route accuracy measures only that checkpoint choice. Exact+named-language accuracy is reported separately: a detector can safely choose+`multilingual` while leaving the specific language undecided. Results for every locale,+including guess counts, are in+[`../results/massive_route_comparison_swedish.json`](../results/massive_route_comparison_swedish.json).++Reproduce from the repository root, passing a baseline and candidate checkout:++```sh+python research/evals/massive_route_comparison.py \+ --before-module /path/to/baseline/laya/lang.py \+ --after-module laya/lang.py \+ --out research/results/massive_route_comparison_swedish.json+```++The report records the source Git revisions and pinned dataset revision. The script+downloads the split from Hugging Face on first run and needs `huggingface_hub` installed.++## Results++| Measure | Baseline | Candidate | Change |+|---|---:|---:|---:|+| Overall checkpoint route accuracy | 77.428% | 77.429% | +0.001 pp |+| Exact named-language accuracy | 8.837% | 8.841% | +0.004 pp |+| Swedish checkpoint route accuracy | 86.954% | 86.954% | 0.000 pp |+| Swedish named-language accuracy | 0.000% | 30.296% | +30.296 pp |+| Danish checkpoint route accuracy | 53.430% | 53.430% | 0.000 pp |+| Norwegian Bokmål route accuracy | 49.630% | 49.630% | 0.000 pp |++Checkpoint route accuracy changes only for `it` (+0.034 pp) and `pt` (+0.034 pp); no+locale decreased. The overall route gain is two examples out of 151,674. Named-language+accuracy also changes slightly for `de`, `it`, `pt`, and `ro`, reflecting shared evidence+rules rather than locale-specific language work. For the 51 locale rows and exact+before/after counts, use the JSON artifact rather than rounded percentages here.++## Interpretation and limits++MASSIVE measures intent classification utterances, not customer-support transcripts.+It supplies a broad per-locale routing check, not evidence that Swedish support+customers are served well end to end. On the pinned split, the Swedish change improves+named-language detection but barely changes which checkpoint is selected; the+configured default already sends most ambiguous examples to English.++Swedish, Danish, and Norwegian share short words. In the candidate results, four+Norwegian examples are named `sv`; all still route to the same multilingual checkpoint+as before. The detector does not claim that checkpoint-routing gains prove perfect+language identification. Broader Swedish support data and a focused false-positive+review remain useful follow-up work.diff --git a/research/evals/massive_route_comparison.py b/research/evals/massive_route_comparison.pynew file mode 100644index 00000000..c848a7ca--- /dev/null+++ b/research/evals/massive_route_comparison.py@@ -0,0 +1,169 @@+"""Compare language routing by locale on the pinned MASSIVE test split.++This model-free diagnostic reports both checkpoint routing and named-language detection before+and after a proposed change to ``laya.lang``. It evaluates the full test split for every locale.++Run from the repository root:++ python research/evals/massive_route_comparison.py \+ --before-module /path/to/baseline/laya/lang.py \+ --after-module laya/lang.py --out research/results/massive_route_comparison.json++No model weights are loaded. A non-English MASSIVE locale should route to ``multilingual``;+English should route to ``english``. Named-language accuracy is reported separately because the+router only needs to choose a readable checkpoint, not identify every language by name.+"""+from __future__ import annotations++import argparse+import gzip+import importlib.util+import json+import subprocess+import sys+from collections import Counter+from pathlib import Path+from typing import Any, Dict++from huggingface_hub import hf_hub_download, list_repo_files+++DATASET = "mteb/amazon_massive_intent"+REVISION = "940fd47a81eaa7f2cc7b129674d945d618ac38c2"+REPO_ROOT = Path(__file__).resolve().parents[2]+++def load_detector(path: Path, name: str):+ """Load a standalone Laya language module from an arbitrary checkout."""+ spec = importlib.util.spec_from_file_location(name, path)+ if spec is None or spec.loader is None:+ raise RuntimeError("cannot load language detector: %s" % path)+ module = importlib.util.module_from_spec(spec)+ sys.modules[name] = module+ spec.loader.exec_module(module)+ return module+++def route(detector: Any, router_module: Any, text: str) -> tuple[str, str]:+ """Use the repository's real Router._route decision path without loading a checkpoint."""+ router = router_module.Router(default="english")+ decision = router._route(text)+ return decision.model, (decision.get("detection") or {}).get("language") or "undecided"+++def source_revision(path: Path) -> str:+ """Return the Git revision owning a detector source file."""+ source = path.resolve()+ root = next((parent for parent in source.parents if (parent / ".git").exists()), source.parent)+ try:+ return subprocess.check_output(+ ["git", "-C", str(root), "rev-parse", "HEAD"], text=True,+ stderr=subprocess.DEVNULL,+ ).strip()+ except (OSError, subprocess.CalledProcessError):+ return "unknown"+++def score_locale(detector: Any, router_module: Any, locale: str, path: Path) -> Dict[str, Any]:+ routes: Counter[str] = Counter()+ named_languages: Counter[str] = Counter()+ named_correct = 0+ examples = 0+ with gzip.open(path, "rt", encoding="utf-8") as source:+ for line in source:+ row = json.loads(line)+ model, named_language = route(detector, router_module, row["text"])+ routes[model] += 1+ named_languages[named_language] += 1+ named_correct += named_language == locale+ examples += 1+ return {+ "examples": examples,+ "route_correct": 0,+ "route_accuracy": None,+ "routed_to": dict(sorted(routes.items())),+ "named_language_correct": named_correct,+ "named_language_accuracy": round(named_correct / examples, 6) if examples else None,+ "named_language_guesses": dict(sorted(named_languages.items())),+ }+++def main(argv=None) -> int:+ parser = argparse.ArgumentParser(description=__doc__)+ parser.add_argument("--before-module", required=True, type=Path,+ help="path to baseline laya/lang.py")+ parser.add_argument("--after-module", required=True, type=Path,+ help="path to candidate laya/lang.py")+ parser.add_argument("--out", type=Path, help="write the JSON report to this path")+ args = parser.parse_args(argv)++ files = list_repo_files(DATASET, repo_type="dataset", revision=REVISION)+ locales = sorted(name[len("test/"):-len(".json.gz")] for name in files+ if name.startswith("test/") and name.endswith(".json.gz"))+ if not locales:+ raise RuntimeError("no per-locale MASSIVE test files at pinned revision")+ before_path = args.before_module.resolve()+ after_path = args.after_module.resolve()+ before = load_detector(before_path, "laya_lang_before")+ after = load_detector(after_path, "laya_lang_after")+ # Use this checkout's Router implementation and replace only its pure detector function for+ # each pass; all checkpoint-selection rules remain the repository code.+ from laya import router as router_module++ report: Dict[str, Any] = {+ "dataset": DATASET,+ "revision": REVISION,+ "split": "test",+ "model_free": True,+ "routing_policy": "english locale -> english checkpoint; every other locale -> multilingual",+ "routing_metric_note": "Router._route decisions are recorded; route accuracy uses expected checkpoint by MASSIVE locale, including Router's configured default for undecided text.",+ "before_module": str(before_path),+ "before_revision": source_revision(before_path),+ "after_module": str(after_path),+ "after_revision": source_revision(after_path),+ "locales": {},+ "overall": {},+ }+ before_total = {"examples": 0, "route_correct": 0, "named_language_correct": 0}+ after_total = dict(before_total)+ for locale in locales:+ local_path = Path(hf_hub_download(+ repo_id=DATASET,+ filename="test/%s.json.gz" % locale,+ repo_type="dataset",+ revision=REVISION,+ ))+ router_module.analyse = before.analyse+ old = score_locale(before, router_module, locale, local_path)+ router_module.analyse = after.analyse+ new = score_locale(after, router_module, locale, local_path)+ expected_model = "english" if locale == "en" else "multilingual"+ for item in (old, new):+ item["expected_checkpoint"] = expected_model+ item["route_correct"] = item["routed_to"].get(expected_model, 0)+ item["route_accuracy"] = round(item["route_correct"] / item["examples"], 6)+ report["locales"][locale] = {"before": old, "after": new,+ "route_accuracy_delta": round(new["route_accuracy"] - old["route_accuracy"], 6),+ "named_language_accuracy_delta": round(new["named_language_accuracy"] - old["named_language_accuracy"], 6)}+ for total, item in ((before_total, old), (after_total, new)):+ total["examples"] += item["examples"]+ total["route_correct"] += item["route_correct"]+ total["named_language_correct"] += item["named_language_correct"]++ for name, total in (("before", before_total), ("after", after_total)):+ count = total["examples"]+ report["overall"][name] = {+ **total,+ "route_accuracy": round(total["route_correct"] / count, 6) if count else None,+ "named_language_accuracy": round(total["named_language_correct"] / count, 6) if count else None,+ }+ if args.out:+ args.out.parent.mkdir(parents=True, exist_ok=True)+ args.out.write_text(json.dumps(report, ensure_ascii=False, indent=2, sort_keys=True) + "\n",+ encoding="utf-8")+ print(json.dumps(report, ensure_ascii=False, indent=2, sort_keys=True))+ return 0+++if __name__ == "__main__":+ raise SystemExit(main())diff --git a/research/results/massive_route_comparison_swedish.json b/research/results/massive_route_comparison_swedish.jsonnew file mode 100644index 00000000..20c7862b--- /dev/null+++ b/research/results/massive_route_comparison_swedish.json@@ -0,0 +1,2018 @@+{+ "after_module": "/Users/danielnylander/Documents/Codex/2026-09-28/kan/work/laya/laya/lang.py",+ "after_revision": "75f2a04a80dbf7dee874addc29c77a7539860ccd",+ "before_module": "/Users/danielnylander/Documents/Codex/2026-09-28/kan/work/laya-split/laya/lang.py",+ "before_revision": "6d942c92081fbc139e736bbd9ac0023223c29b7f",+ "dataset": "mteb/amazon_massive_intent",+ "locales": {+ "af": {+ "after": {+ "examples": 2974,+ "expected_checkpoint": "multilingual",+ "named_language_accuracy": 0.0,+ "named_language_correct": 0,+ "named_language_guesses": {+ "de": 122,+ "en": 734,+ "fr": 1,+ "it": 4,+ "nl": 177,+ "pt": 25,+ "undecided": 1911+ },+ "route_accuracy": 0.167787,+ "route_correct": 499,+ "routed_to": {+ "english": 2475,+ "multilingual": 499+ }+ },+ "before": {+ "examples": 2974,+ "expected_checkpoint": "multilingual",+ "named_language_accuracy": 0.0,+ "named_language_correct": 0,+ "named_language_guesses": {+ "de": 122,+ "en": 734,+ "fr": 1,+ "it": 4,+ "nl": 177,+ "pt": 25,+ "undecided": 1911+ },+ "route_accuracy": 0.167787,+ "route_correct": 499,+ "routed_to": {+ "english": 2475,+ "multilingual": 499+ }+ },+ "named_language_accuracy_delta": 0.0,+ "route_accuracy_delta": 0.0+ },+ "am": {+ "after": {+ "examples": 2974,+ "expected_checkpoint": "multilingual",+ "named_language_accuracy": 0.0,+ "named_language_correct": 0,+ "named_language_guesses": {+ "undecided": 2974+ },+ "route_accuracy": 1.0,+ "route_correct": 2974,+ "routed_to": {+ "multilingual": 2974+ }+ },+ "before": {+ "examples": 2974,+ "expected_checkpoint": "multilingual",+ "named_language_accuracy": 0.0,+ "named_language_correct": 0,+ "named_language_guesses": {+ "undecided": 2974+ },+ "route_accuracy": 1.0,+ "route_correct": 2974,+ "routed_to": {+ "multilingual": 2974+ }+ },+ "named_language_accuracy_delta": 0.0,+ "route_accuracy_delta": 0.0+ },+ "ar": {+ "after": {+ "examples": 2974,+ "expected_checkpoint": "multilingual",+ "named_language_accuracy": 0.0,+ "named_language_correct": 0,+ "named_language_guesses": {+ "en": 24,+ "undecided": 2950+ },+ "route_accuracy": 0.96503,+ "route_correct": 2870,+ "routed_to": {+ "english": 104,+ "multilingual": 2870+ }+ },+ "before": {+ "examples": 2974,+ "expected_checkpoint": "multilingual",+ "named_language_accuracy": 0.0,+ "named_language_correct": 0,+ "named_language_guesses": {+ "en": 24,+ "undecided": 2950+ },+ "route_accuracy": 0.96503,+ "route_correct": 2870,+ "routed_to": {+ "english": 104,+ "multilingual": 2870+ }+ },+ "named_language_accuracy_delta": 0.0,+ "route_accuracy_delta": 0.0+ },+ "az": {+ "after": {+ "examples": 2974,+ "expected_checkpoint": "multilingual",+ "named_language_accuracy": 0.138198,+ "named_language_correct": 411,+ "named_language_guesses": {+ "az": 411,+ "fr": 1,+ "pt": 1,+ "undecided": 2561+ },+ "route_accuracy": 0.973436,+ "route_correct": 2895,+ "routed_to": {+ "english": 79,+ "multilingual": 2895+ }+ },+ "before": {+ "examples": 2974,+ "expected_checkpoint": "multilingual",+ "named_language_accuracy": 0.138198,+ "named_language_correct": 411,+ "named_language_guesses": {+ "az": 411,+ "fr": 1,+ "pt": 1,+ "undecided": 2561+ },+ "route_accuracy": 0.973436,+ "route_correct": 2895,+ "routed_to": {+ "english": 79,+ "multilingual": 2895+ }+ },+ "named_language_accuracy_delta": 0.0,+ "route_accuracy_delta": 0.0+ },+ "bn": {+ "after": {+ "examples": 2974,+ "expected_checkpoint": "multilingual",+ "named_language_accuracy": 0.0,+ "named_language_correct": 0,+ "named_language_guesses": {+ "en": 1,+ "undecided": 2973+ },+ "route_accuracy": 0.998319,+ "route_correct": 2969,+ "routed_to": {+ "english": 5,+ "multilingual": 2969+ }+ },+ "before": {+ "examples": 2974,+ "expected_checkpoint": "multilingual",+ "named_language_accuracy": 0.0,+ "named_language_correct": 0,+ "named_language_guesses": {+ "en": 1,+ "undecided": 2973+ },+ "route_accuracy": 0.998319,+ "route_correct": 2969,+ "routed_to": {+ "english": 5,+ "multilingual": 2969+ }+ },+ "named_language_accuracy_delta": 0.0,+ "route_accuracy_delta": 0.0+ },+ "cy": {+ "after": {+ "examples": 2974,+ "expected_checkpoint": "multilingual",+ "named_language_accuracy": 0.0,+ "named_language_correct": 0,+ "named_language_guesses": {+ "bn": 36,+ "en": 768,+ "es": 98,+ "it": 11,+ "pt": 16,+ "undecided": 2045+ },+ "route_accuracy": 0.132818,+ "route_correct": 395,+ "routed_to": {+ "english": 2579,+ "multilingual": 395+ }+ },+ "before": {+ "examples": 2974,+ "expected_checkpoint": "multilingual",+ "named_language_accuracy": 0.0,+ "named_language_correct": 0,+ "named_language_guesses": {+ "bn": 36,+ "en": 768,+ "es": 98,+ "it": 11,+ "pt": 16,+ "undecided": 2045+ },+ "route_accuracy": 0.132818,+ "route_correct": 395,+ "routed_to": {+ "english": 2579,+ "multilingual": 395+ }+ },+ "named_language_accuracy_delta": 0.0,+ "route_accuracy_delta": 0.0+ },+ "da": {+ "after": {+ "examples": 2974,+ "expected_checkpoint": "multilingual",+ "named_language_accuracy": 0.0,+ "named_language_correct": 0,+ "named_language_guesses": {+ "de": 24,+ "en": 547,+ "fr": 5,+ "it": 10,+ "ro": 1,+ "undecided": 2387+ },+ "route_accuracy": 0.534297,+ "route_correct": 1589,+ "routed_to": {+ "english": 1385,+ "multilingual": 1589+ }+ },+ "before": {+ "examples": 2974,+ "expected_checkpoint": "multilingual",+ "named_language_accuracy": 0.0,+ "named_language_correct": 0,+ "named_language_guesses": {+ "de": 24,+ "en": 547,+ "fr": 5,+ "it": 10,+ "ro": 1,+ "undecided": 2387+ },+ "route_accuracy": 0.534297,+ "route_correct": 1589,+ "routed_to": {+ "english": 1385,+ "multilingual": 1589+ }+ },+ "named_language_accuracy_delta": 0.0,+ "route_accuracy_delta": 0.0+ },+ "de": {+ "after": {+ "examples": 2974,+ "expected_checkpoint": "multilingual",+ "named_language_accuracy": 0.655346,+ "named_language_correct": 1949,+ "named_language_guesses": {+ "de": 1949,+ "en": 82,+ "fr": 3,+ "pt": 1,+ "undecided": 939+ },+ "route_accuracy": 0.752186,+ "route_correct": 2237,+ "routed_to": {+ "english": 737,+ "multilingual": 2237+ }+ },+ "before": {+ "examples": 2974,+ "expected_checkpoint": "multilingual",+ "named_language_accuracy": 0.65501,+ "named_language_correct": 1948,+ "named_language_guesses": {+ "de": 1948,+ "en": 82,+ "fr": 3,+ "pt": 1,+ "undecided": 940+ },+ "route_accuracy": 0.752186,+ "route_correct": 2237,+ "routed_to": {+ "english": 737,+ "multilingual": 2237+ }+ },+ "named_language_accuracy_delta": 0.000336,+ "route_accuracy_delta": 0.0+ },+ "el": {+ "after": {+ "examples": 2974,+ "expected_checkpoint": "multilingual",+ "named_language_accuracy": 0.0,+ "named_language_correct": 0,+ "named_language_guesses": {+ "undecided": 2974+ },+ "route_accuracy": 0.99731,+ "route_correct": 2966,+ "routed_to": {+ "english": 8,+ "multilingual": 2966+ }+ },+ "before": {+ "examples": 2974,+ "expected_checkpoint": "multilingual",+ "named_language_accuracy": 0.0,+ "named_language_correct": 0,+ "named_language_guesses": {+ "undecided": 2974+ },+ "route_accuracy": 0.99731,+ "route_correct": 2966,+ "routed_to": {+ "english": 8,+ "multilingual": 2966+ }+ },+ "named_language_accuracy_delta": 0.0,+ "route_accuracy_delta": 0.0+ },+ "en": {+ "after": {+ "examples": 2974,+ "expected_checkpoint": "english",+ "named_language_accuracy": 0.80195,+ "named_language_correct": 2385,+ "named_language_guesses": {+ "en": 2385,+ "undecided": 589+ },+ "route_accuracy": 1.0,+ "route_correct": 2974,+ "routed_to": {+ "english": 2974+ }+ },+ "before": {+ "examples": 2974,+ "expected_checkpoint": "english",+ "named_language_accuracy": 0.80195,+ "named_language_correct": 2385,+ "named_language_guesses": {+ "en": 2385,+ "undecided": 589+ },+ "route_accuracy": 1.0,+ "route_correct": 2974,+ "routed_to": {+ "english": 2974+ }+ },+ "named_language_accuracy_delta": 0.0,+ "route_accuracy_delta": 0.0+ },+ "es": {+ "after": {+ "examples": 2974,+ "expected_checkpoint": "multilingual",+ "named_language_accuracy": 0.467384,+ "named_language_correct": 1390,+ "named_language_guesses": {+ "en": 6,+ "es": 1390,+ "fr": 3,+ "it": 16,+ "pt": 18,+ "undecided": 1541+ },+ "route_accuracy": 0.711163,+ "route_correct": 2115,+ "routed_to": {+ "english": 859,+ "multilingual": 2115+ }+ },+ "before": {+ "examples": 2974,+ "expected_checkpoint": "multilingual",+ "named_language_accuracy": 0.467384,+ "named_language_correct": 1390,+ "named_language_guesses": {+ "en": 6,+ "es": 1390,+ "fr": 3,+ "it": 16,+ "pt": 18,+ "undecided": 1541+ },+ "route_accuracy": 0.711163,+ "route_correct": 2115,+ "routed_to": {+ "english": 859,+ "multilingual": 2115+ }+ },+ "named_language_accuracy_delta": 0.0,+ "route_accuracy_delta": 0.0+ },+ "fa": {+ "after": {+ "examples": 2974,+ "expected_checkpoint": "multilingual",+ "named_language_accuracy": 0.0,+ "named_language_correct": 0,+ "named_language_guesses": {+ "undecided": 2974+ },+ "route_accuracy": 0.999664,+ "route_correct": 2973,+ "routed_to": {+ "english": 1,+ "multilingual": 2973+ }+ },+ "before": {+ "examples": 2974,+ "expected_checkpoint": "multilingual",+ "named_language_accuracy": 0.0,+ "named_language_correct": 0,+ "named_language_guesses": {+ "undecided": 2974+ },+ "route_accuracy": 0.999664,+ "route_correct": 2973,+ "routed_to": {+ "english": 1,+ "multilingual": 2973+ }+ },+ "named_language_accuracy_delta": 0.0,+ "route_accuracy_delta": 0.0+ },+ "fi": {+ "after": {+ "examples": 2974,+ "expected_checkpoint": "multilingual",+ "named_language_accuracy": 0.0,+ "named_language_correct": 0,+ "named_language_guesses": {+ "en": 97,+ "undecided": 2877+ },+ "route_accuracy": 0.667787,+ "route_correct": 1986,+ "routed_to": {+ "english": 988,+ "multilingual": 1986+ }+ },+ "before": {+ "examples": 2974,+ "expected_checkpoint": "multilingual",+ "named_language_accuracy": 0.0,+ "named_language_correct": 0,+ "named_language_guesses": {+ "en": 97,+ "undecided": 2877+ },+ "route_accuracy": 0.667787,+ "route_correct": 1986,+ "routed_to": {+ "english": 988,+ "multilingual": 1986+ }+ },+ "named_language_accuracy_delta": 0.0,+ "route_accuracy_delta": 0.0+ },+ "fr": {+ "after": {+ "examples": 2974,+ "expected_checkpoint": "multilingual",+ "named_language_accuracy": 0.610962,+ "named_language_correct": 1817,+ "named_language_guesses": {+ "en": 7,+ "es": 14,+ "fr": 1817,+ "it": 7,+ "pt": 6,+ "undecided": 1123+ },+ "route_accuracy": 0.822125,+ "route_correct": 2445,+ "routed_to": {+ "english": 529,+ "multilingual": 2445+ }+ },+ "before": {+ "examples": 2974,+ "expected_checkpoint": "multilingual",+ "named_language_accuracy": 0.610962,+ "named_language_correct": 1817,+ "named_language_guesses": {+ "en": 7,+ "es": 14,+ "fr": 1817,+ "it": 7,+ "pt": 6,+ "undecided": 1123+ },+ "route_accuracy": 0.822125,+ "route_correct": 2445,+ "routed_to": {+ "english": 529,+ "multilingual": 2445+ }+ },+ "named_language_accuracy_delta": 0.0,+ "route_accuracy_delta": 0.0+ },+ "he": {+ "after": {+ "examples": 2974,+ "expected_checkpoint": "multilingual",+ "named_language_accuracy": 0.0,+ "named_language_correct": 0,+ "named_language_guesses": {+ "undecided": 2974+ },+ "route_accuracy": 0.999664,+ "route_correct": 2973,+ "routed_to": {+ "english": 1,+ "multilingual": 2973+ }+ },+ "before": {+ "examples": 2974,+ "expected_checkpoint": "multilingual",+ "named_language_accuracy": 0.0,+ "named_language_correct": 0,+ "named_language_guesses": {+ "undecided": 2974+ },+ "route_accuracy": 0.999664,+ "route_correct": 2973,+ "routed_to": {+ "english": 1,+ "multilingual": 2973+ }+ },+ "named_language_accuracy_delta": 0.0,+ "route_accuracy_delta": 0.0+ },+ "hi": {+ "after": {+ "examples": 2974,+ "expected_checkpoint": "multilingual",+ "named_language_accuracy": 0.0,+ "named_language_correct": 0,+ "named_language_guesses": {+ "en": 5,+ "it": 1,+ "undecided": 2968+ },+ "route_accuracy": 0.993948,+ "route_correct": 2956,+ "routed_to": {+ "english": 18,+ "multilingual": 2956+ }+ },+ "before": {+ "examples": 2974,+ "expected_checkpoint": "multilingual",+ "named_language_accuracy": 0.0,+ "named_language_correct": 0,+ "named_language_guesses": {+ "en": 5,+ "it": 1,+ "undecided": 2968+ },+ "route_accuracy": 0.993948,+ "route_correct": 2956,+ "routed_to": {+ "english": 18,+ "multilingual": 2956+ }+ },+ "named_language_accuracy_delta": 0.0,+ "route_accuracy_delta": 0.0+ },+ "hu": {+ "after": {+ "examples": 2974,+ "expected_checkpoint": "multilingual",+ "named_language_accuracy": 0.0,+ "named_language_correct": 0,+ "named_language_guesses": {+ "en": 7,+ "es": 6,+ "fr": 1,+ "it": 15,+ "nl": 3,+ "undecided": 2942+ },+ "route_accuracy": 0.935777,+ "route_correct": 2783,+ "routed_to": {+ "english": 191,+ "multilingual": 2783+ }+ },+ "before": {+ "examples": 2974,+ "expected_checkpoint": "multilingual",+ "named_language_accuracy": 0.0,+ "named_language_correct": 0,+ "named_language_guesses": {+ "en": 7,+ "es": 6,+ "fr": 1,+ "it": 15,+ "nl": 3,+ "undecided": 2942+ },+ "route_accuracy": 0.935777,+ "route_correct": 2783,+ "routed_to": {+ "english": 191,+ "multilingual": 2783+ }+ },+ "named_language_accuracy_delta": 0.0,+ "route_accuracy_delta": 0.0+ },+ "hy": {+ "after": {+ "examples": 2974,+ "expected_checkpoint": "multilingual",+ "named_language_accuracy": 0.0,+ "named_language_correct": 0,+ "named_language_guesses": {+ "undecided": 2974+ },+ "route_accuracy": 0.999328,+ "route_correct": 2972,+ "routed_to": {+ "english": 2,+ "multilingual": 2972+ }+ },+ "before": {+ "examples": 2974,+ "expected_checkpoint": "multilingual",+ "named_language_accuracy": 0.0,+ "named_language_correct": 0,+ "named_language_guesses": {+ "undecided": 2974+ },+ "route_accuracy": 0.999328,+ "route_correct": 2972,+ "routed_to": {+ "english": 2,+ "multilingual": 2972+ }+ },+ "named_language_accuracy_delta": 0.0,+ "route_accuracy_delta": 0.0+ },+ "id": {+ "after": {+ "examples": 2974,+ "expected_checkpoint": "multilingual",+ "named_language_accuracy": 0.0,+ "named_language_correct": 0,+ "named_language_guesses": {+ "en": 43,+ "es": 1,+ "it": 21,+ "undecided": 2909+ },+ "route_accuracy": 0.007397,+ "route_correct": 22,+ "routed_to": {+ "english": 2952,+ "multilingual": 22+ }+ },+ "before": {+ "examples": 2974,+ "expected_checkpoint": "multilingual",+ "named_language_accuracy": 0.0,+ "named_language_correct": 0,+ "named_language_guesses": {+ "en": 43,+ "es": 1,+ "it": 21,+ "undecided": 2909+ },+ "route_accuracy": 0.007397,+ "route_correct": 22,+ "routed_to": {+ "english": 2952,+ "multilingual": 22+ }+ },+ "named_language_accuracy_delta": 0.0,+ "route_accuracy_delta": 0.0+ },+ "is": {+ "after": {+ "examples": 2974,+ "expected_checkpoint": "multilingual",+ "named_language_accuracy": 0.0,+ "named_language_correct": 0,+ "named_language_guesses": {+ "az": 2,+ "en": 3,+ "pt": 7,+ "sv": 5,+ "undecided": 2957+ },+ "route_accuracy": 0.909213,+ "route_correct": 2704,+ "routed_to": {+ "english": 270,+ "multilingual": 2704+ }+ },+ "before": {+ "examples": 2974,+ "expected_checkpoint": "multilingual",+ "named_language_accuracy": 0.0,+ "named_language_correct": 0,+ "named_language_guesses": {+ "az": 2,+ "en": 3,+ "pt": 7,+ "sv": 5,+ "undecided": 2957+ },+ "route_accuracy": 0.909213,+ "route_correct": 2704,+ "routed_to": {+ "english": 270,+ "multilingual": 2704+ }+ },+ "named_language_accuracy_delta": 0.0,+ "route_accuracy_delta": 0.0+ },+ "it": {+ "after": {+ "examples": 2974,+ "expected_checkpoint": "multilingual",+ "named_language_accuracy": 0.590787,+ "named_language_correct": 1757,+ "named_language_guesses": {+ "en": 159,+ "es": 19,+ "fr": 4,+ "it": 1757,+ "undecided": 1035+ },+ "route_accuracy": 0.635508,+ "route_correct": 1890,+ "routed_to": {+ "english": 1084,+ "multilingual": 1890+ }+ },+ "before": {+ "examples": 2974,+ "expected_checkpoint": "multilingual",+ "named_language_accuracy": 0.589778,+ "named_language_correct": 1754,+ "named_language_guesses": {+ "en": 159,+ "es": 20,+ "fr": 5,+ "it": 1754,+ "undecided": 1036+ },+ "route_accuracy": 0.635171,+ "route_correct": 1889,+ "routed_to": {+ "english": 1085,+ "multilingual": 1889+ }+ },+ "named_language_accuracy_delta": 0.001009,+ "route_accuracy_delta": 0.000337+ },+ "ja": {+ "after": {+ "examples": 2974,+ "expected_checkpoint": "multilingual",+ "named_language_accuracy": 0.0,+ "named_language_correct": 0,+ "named_language_guesses": {+ "undecided": 2974+ },+ "route_accuracy": 0.997983,+ "route_correct": 2968,+ "routed_to": {+ "english": 6,+ "multilingual": 2968+ }+ },+ "before": {+ "examples": 2974,+ "expected_checkpoint": "multilingual",+ "named_language_accuracy": 0.0,+ "named_language_correct": 0,+ "named_language_guesses": {+ "undecided": 2974+ },+ "route_accuracy": 0.997983,+ "route_correct": 2968,+ "routed_to": {+ "english": 6,+ "multilingual": 2968+ }+ },+ "named_language_accuracy_delta": 0.0,+ "route_accuracy_delta": 0.0+ },+ "jv": {+ "after": {+ "examples": 2974,+ "expected_checkpoint": "multilingual",+ "named_language_accuracy": 0.0,+ "named_language_correct": 0,+ "named_language_guesses": {+ "en": 32,+ "fr": 1,+ "it": 18,+ "undecided": 2923+ },+ "route_accuracy": 0.009751,+ "route_correct": 29,+ "routed_to": {+ "english": 2945,+ "multilingual": 29+ }+ },+ "before": {+ "examples": 2974,+ "expected_checkpoint": "multilingual",+ "named_language_accuracy": 0.0,+ "named_language_correct": 0,+ "named_language_guesses": {+ "en": 32,+ "fr": 1,+ "it": 18,+ "undecided": 2923+ },+ "route_accuracy": 0.009751,+ "route_correct": 29,+ "routed_to": {+ "english": 2945,+ "multilingual": 29+ }+ },+ "named_language_accuracy_delta": 0.0,+ "route_accuracy_delta": 0.0+ },+ "ka": {+ "after": {+ "examples": 2974,+ "expected_checkpoint": "multilingual",+ "named_language_accuracy": 0.0,+ "named_language_correct": 0,+ "named_language_guesses": {+ "undecided": 2974+ },+ "route_accuracy": 0.999664,+ "route_correct": 2973,+ "routed_to": {+ "english": 1,+ "multilingual": 2973+ }+ },+ "before": {+ "examples": 2974,+ "expected_checkpoint": "multilingual",+ "named_language_accuracy": 0.0,+ "named_language_correct": 0,+ "named_language_guesses": {+ "undecided": 2974+ },+ "route_accuracy": 0.999664,+ "route_correct": 2973,+ "routed_to": {+ "english": 1,+ "multilingual": 2973+ }+ },+ "named_language_accuracy_delta": 0.0,+ "route_accuracy_delta": 0.0+ },+ "km": {+ "after": {+ "examples": 2974,+ "expected_checkpoint": "multilingual",+ "named_language_accuracy": 0.0,+ "named_language_correct": 0,+ "named_language_guesses": {+ "en": 1,+ "undecided": 2973+ },+ "route_accuracy": 0.992939,+ "route_correct": 2953,+ "routed_to": {+ "english": 21,+ "multilingual": 2953+ }+ },+ "before": {+ "examples": 2974,+ "expected_checkpoint": "multilingual",+ "named_language_accuracy": 0.0,+ "named_language_correct": 0,+ "named_language_guesses": {+ "en": 1,+ "undecided": 2973+ },+ "route_accuracy": 0.992939,+ "route_correct": 2953,+ "routed_to": {+ "english": 21,+ "multilingual": 2953+ }+ },+ "named_language_accuracy_delta": 0.0,+ "route_accuracy_delta": 0.0+ },+ "kn": {+ "after": {+ "examples": 2974,+ "expected_checkpoint": "multilingual",+ "named_language_accuracy": 0.0,+ "named_language_correct": 0,+ "named_language_guesses": {+ "undecided": 2974+ },+ "route_accuracy": 0.999328,+ "route_correct": 2972,+ "routed_to": {+ "english": 2,+ "multilingual": 2972+ }+ },+ "before": {+ "examples": 2974,+ "expected_checkpoint": "multilingual",+ "named_language_accuracy": 0.0,+ "named_language_correct": 0,+ "named_language_guesses": {+ "undecided": 2974+ },+ "route_accuracy": 0.999328,+ "route_correct": 2972,+ "routed_to": {+ "english": 2,+ "multilingual": 2972+ }+ },+ "named_language_accuracy_delta": 0.0,+ "route_accuracy_delta": 0.0+ },+ "ko": {+ "after": {+ "examples": 2974,+ "expected_checkpoint": "multilingual",+ "named_language_accuracy": 0.0,+ "named_language_correct": 0,+ "named_language_guesses": {+ "en": 1,+ "undecided": 2973+ },+ "route_accuracy": 0.997646,+ "route_correct": 2967,+ "routed_to": {+ "english": 7,+ "multilingual": 2967+ }+ },+ "before": {+ "examples": 2974,+ "expected_checkpoint": "multilingual",+ "named_language_accuracy": 0.0,+ "named_language_correct": 0,+ "named_language_guesses": {+ "en": 1,+ "undecided": 2973+ },+ "route_accuracy": 0.997646,+ "route_correct": 2967,+ "routed_to": {+ "english": 7,+ "multilingual": 2967+ }+ },+ "named_language_accuracy_delta": 0.0,+ "route_accuracy_delta": 0.0+ },+ "lv": {+ "after": {+ "examples": 2974,+ "expected_checkpoint": "multilingual",+ "named_language_accuracy": 0.0,+ "named_language_correct": 0,+ "named_language_guesses": {+ "bn": 2,+ "en": 3,+ "es": 23,+ "pt": 1,+ "undecided": 2945+ },+ "route_accuracy": 0.933759,+ "route_correct": 2777,+ "routed_to": {+ "english": 197,+ "multilingual": 2777+ }+ },+ "before": {+ "examples": 2974,+ "expected_checkpoint": "multilingual",+ "named_language_accuracy": 0.0,+ "named_language_correct": 0,+ "named_language_guesses": {+ "bn": 2,+ "en": 3,+ "es": 23,+ "pt": 1,+ "undecided": 2945+ },+ "route_accuracy": 0.933759,+ "route_correct": 2777,+ "routed_to": {+ "english": 197,+ "multilingual": 2777+ }+ },+ "named_language_accuracy_delta": 0.0,+ "route_accuracy_delta": 0.0+ },+ "ml": {+ "after": {+ "examples": 2974,+ "expected_checkpoint": "multilingual",+ "named_language_accuracy": 0.0,+ "named_language_correct": 0,+ "named_language_guesses": {+ "en": 1,+ "undecided": 2973+ },+ "route_accuracy": 0.999664,+ "route_correct": 2973,+ "routed_to": {+ "english": 1,+ "multilingual": 2973+ }+ },+ "before": {+ "examples": 2974,+ "expected_checkpoint": "multilingual",+ "named_language_accuracy": 0.0,+ "named_language_correct": 0,+ "named_language_guesses": {+ "en": 1,+ "undecided": 2973+ },+ "route_accuracy": 0.999664,+ "route_correct": 2973,+ "routed_to": {+ "english": 1,+ "multilingual": 2973+ }+ },+ "named_language_accuracy_delta": 0.0,+ "route_accuracy_delta": 0.0+ },+ "mn": {+ "after": {+ "examples": 2974,+ "expected_checkpoint": "multilingual",+ "named_language_accuracy": 0.0,+ "named_language_correct": 0,+ "named_language_guesses": {+ "undecided": 2974+ },+ "route_accuracy": 0.999328,+ "route_correct": 2972,+ "routed_to": {+ "english": 2,+ "multilingual": 2972+ }+ },+ "before": {+ "examples": 2974,+ "expected_checkpoint": "multilingual",+ "named_language_accuracy": 0.0,+ "named_language_correct": 0,+ "named_language_guesses": {+ "undecided": 2974+ },+ "route_accuracy": 0.999328,+ "route_correct": 2972,+ "routed_to": {+ "english": 2,+ "multilingual": 2972+ }+ },+ "named_language_accuracy_delta": 0.0,+ "route_accuracy_delta": 0.0+ },+ "ms": {+ "after": {+ "examples": 2974,+ "expected_checkpoint": "multilingual",+ "named_language_accuracy": 0.0,+ "named_language_correct": 0,+ "named_language_guesses": {+ "en": 30,+ "it": 12,+ "undecided": 2932+ },+ "route_accuracy": 0.004035,+ "route_correct": 12,+ "routed_to": {+ "english": 2962,+ "multilingual": 12+ }+ },+ "before": {+ "examples": 2974,+ "expected_checkpoint": "multilingual",+ "named_language_accuracy": 0.0,+ "named_language_correct": 0,+ "named_language_guesses": {+ "en": 30,+ "it": 12,+ "undecided": 2932+ },+ "route_accuracy": 0.004035,+ "route_correct": 12,+ "routed_to": {+ "english": 2962,+ "multilingual": 12+ }+ },+ "named_language_accuracy_delta": 0.0,+ "route_accuracy_delta": 0.0+ },+ "my": {+ "after": {+ "examples": 2974,+ "expected_checkpoint": "multilingual",+ "named_language_accuracy": 0.0,+ "named_language_correct": 0,+ "named_language_guesses": {+ "undecided": 2974+ },+ "route_accuracy": 0.998991,+ "route_correct": 2971,+ "routed_to": {+ "english": 3,+ "multilingual": 2971+ }+ },+ "before": {+ "examples": 2974,+ "expected_checkpoint": "multilingual",+ "named_language_accuracy": 0.0,+ "named_language_correct": 0,+ "named_language_guesses": {+ "undecided": 2974+ },+ "route_accuracy": 0.998991,+ "route_correct": 2971,+ "routed_to": {+ "english": 3,+ "multilingual": 2971+ }+ },+ "named_language_accuracy_delta": 0.0,+ "route_accuracy_delta": 0.0+ },+ "nb": {+ "after": {+ "examples": 2974,+ "expected_checkpoint": "multilingual",+ "named_language_accuracy": 0.0,+ "named_language_correct": 0,+ "named_language_guesses": {+ "de": 7,+ "en": 469,+ "fr": 10,+ "it": 12,+ "pt": 1,+ "sv": 4,+ "undecided": 2471+ },+ "route_accuracy": 0.496301,+ "route_correct": 1476,+ "routed_to": {+ "english": 1498,+ "multilingual": 1476+ }+ },+ "before": {+ "examples": 2974,+ "expected_checkpoint": "multilingual",+ "named_language_accuracy": 0.0,+ "named_language_correct": 0,+ "named_language_guesses": {+ "de": 7,+ "en": 469,+ "fr": 10,+ "it": 12,+ "pt": 1,+ "sv": 4,+ "undecided": 2471+ },+ "route_accuracy": 0.496301,+ "route_correct": 1476,+ "routed_to": {+ "english": 1498,+ "multilingual": 1476+ }+ },+ "named_language_accuracy_delta": 0.0,+ "route_accuracy_delta": 0.0+ },+ "nl": {+ "after": {+ "examples": 2974,+ "expected_checkpoint": "multilingual",+ "named_language_accuracy": 0.257566,+ "named_language_correct": 766,+ "named_language_guesses": {+ "de": 3,+ "en": 674,+ "fr": 4,+ "it": 1,+ "nl": 766,+ "pt": 6,+ "undecided": 1520+ },+ "route_accuracy": 0.283457,+ "route_correct": 843,+ "routed_to": {+ "english": 2131,+ "multilingual": 843+ }+ },+ "before": {+ "examples": 2974,+ "expected_checkpoint": "multilingual",+ "named_language_accuracy": 0.257566,+ "named_language_correct": 766,+ "named_language_guesses": {+ "de": 3,+ "en": 674,+ "fr": 4,+ "it": 1,+ "nl": 766,+ "pt": 6,+ "undecided": 1520+ },+ "route_accuracy": 0.283457,+ "route_correct": 843,+ "routed_to": {+ "english": 2131,+ "multilingual": 843+ }+ },+ "named_language_accuracy_delta": 0.0,+ "route_accuracy_delta": 0.0+ },+ "pl": {+ "after": {+ "examples": 2974,+ "expected_checkpoint": "multilingual",+ "named_language_accuracy": 0.0,+ "named_language_correct": 0,+ "named_language_guesses": {+ "de": 1,+ "en": 53,+ "es": 1,+ "pt": 142,+ "undecided": 2777+ },+ "route_accuracy": 0.813046,+ "route_correct": 2418,+ "routed_to": {+ "english": 556,+ "multilingual": 2418+ }+ },+ "before": {+ "examples": 2974,+ "expected_checkpoint": "multilingual",+ "named_language_accuracy": 0.0,+ "named_language_correct": 0,+ "named_language_guesses": {+ "de": 1,+ "en": 53,+ "es": 1,+ "pt": 135,+ "undecided": 2784+ },+ "route_accuracy": 0.813046,+ "route_correct": 2418,+ "routed_to": {+ "english": 556,+ "multilingual": 2418+ }+ },+ "named_language_accuracy_delta": 0.0,+ "route_accuracy_delta": 0.0+ },+ "pt": {+ "after": {+ "examples": 2974,+ "expected_checkpoint": "multilingual",+ "named_language_accuracy": 0.53228,+ "named_language_correct": 1583,+ "named_language_guesses": {+ "en": 62,+ "es": 14,+ "fr": 1,+ "it": 8,+ "pt": 1583,+ "undecided": 1306+ },+ "route_accuracy": 0.765636,+ "route_correct": 2277,+ "routed_to": {+ "english": 697,+ "multilingual": 2277+ }+ },+ "before": {+ "examples": 2974,+ "expected_checkpoint": "multilingual",+ "named_language_accuracy": 0.531944,+ "named_language_correct": 1582,+ "named_language_guesses": {+ "en": 62,+ "es": 14,+ "fr": 1,+ "it": 8,+ "pt": 1582,+ "undecided": 1307+ },+ "route_accuracy": 0.765299,+ "route_correct": 2276,+ "routed_to": {+ "english": 698,+ "multilingual": 2276+ }+ },+ "named_language_accuracy_delta": 0.000336,+ "route_accuracy_delta": 0.000337+ },+ "ro": {+ "after": {+ "examples": 2974,+ "expected_checkpoint": "multilingual",+ "named_language_accuracy": 0.151311,+ "named_language_correct": 450,+ "named_language_guesses": {+ "bn": 2,+ "en": 221,+ "es": 10,+ "fr": 120,+ "it": 98,+ "pt": 24,+ "ro": 450,+ "undecided": 2049+ },+ "route_accuracy": 0.533289,+ "route_correct": 1586,+ "routed_to": {+ "english": 1388,+ "multilingual": 1586+ }+ },+ "before": {+ "examples": 2974,+ "expected_checkpoint": "multilingual",+ "named_language_accuracy": 0.150975,+ "named_language_correct": 449,+ "named_language_guesses": {+ "bn": 2,+ "en": 221,+ "es": 10,+ "fr": 120,+ "it": 99,+ "pt": 24,+ "ro": 449,+ "undecided": 2049+ },+ "route_accuracy": 0.533289,+ "route_correct": 1586,+ "routed_to": {+ "english": 1388,+ "multilingual": 1586+ }+ },+ "named_language_accuracy_delta": 0.000336,+ "route_accuracy_delta": 0.0+ },+ "ru": {+ "after": {+ "examples": 2974,+ "expected_checkpoint": "multilingual",+ "named_language_accuracy": 0.0,+ "named_language_correct": 0,+ "named_language_guesses": {+ "undecided": 2974+ },+ "route_accuracy": 0.999328,+ "route_correct": 2972,+ "routed_to": {+ "english": 2,+ "multilingual": 2972+ }+ },+ "before": {+ "examples": 2974,+ "expected_checkpoint": "multilingual",+ "named_language_accuracy": 0.0,+ "named_language_correct": 0,+ "named_language_guesses": {+ "undecided": 2974+ },+ "route_accuracy": 0.999328,+ "route_correct": 2972,+ "routed_to": {+ "english": 2,+ "multilingual": 2972+ }+ },+ "named_language_accuracy_delta": 0.0,+ "route_accuracy_delta": 0.0+ },+ "sl": {+ "after": {+ "examples": 2974,+ "expected_checkpoint": "multilingual",+ "named_language_accuracy": 0.0,+ "named_language_correct": 0,+ "named_language_guesses": {+ "en": 85,+ "es": 2,+ "fr": 34,+ "it": 14,+ "pt": 108,+ "undecided": 2731+ },+ "route_accuracy": 0.573638,+ "route_correct": 1706,+ "routed_to": {+ "english": 1268,+ "multilingual": 1706+ }+ },+ "before": {+ "examples": 2974,+ "expected_checkpoint": "multilingual",+ "named_language_accuracy": 0.0,+ "named_language_correct": 0,+ "named_language_guesses": {+ "en": 85,+ "es": 2,+ "fr": 34,+ "it": 14,+ "pt": 108,+ "undecided": 2731+ },+ "route_accuracy": 0.573638,+ "route_correct": 1706,+ "routed_to": {+ "english": 1268,+ "multilingual": 1706+ }+ },+ "named_language_accuracy_delta": 0.0,+ "route_accuracy_delta": 0.0+ },+ "sq": {+ "after": {+ "examples": 2974,+ "expected_checkpoint": "multilingual",+ "named_language_accuracy": 0.0,+ "named_language_correct": 0,+ "named_language_guesses": {+ "en": 15,+ "fr": 10,+ "it": 94,+ "nl": 1,+ "pt": 42,+ "undecided": 2812+ },+ "route_accuracy": 0.914929,+ "route_correct": 2721,+ "routed_to": {+ "english": 253,+ "multilingual": 2721+ }+ },+ "before": {+ "examples": 2974,+ "expected_checkpoint": "multilingual",+ "named_language_accuracy": 0.0,+ "named_language_correct": 0,+ "named_language_guesses": {+ "en": 15,+ "fr": 10,+ "it": 94,+ "nl": 1,+ "pt": 39,+ "undecided": 2815+ },+ "route_accuracy": 0.914929,+ "route_correct": 2721,+ "routed_to": {+ "english": 253,+ "multilingual": 2721+ }+ },+ "named_language_accuracy_delta": 0.0,+ "route_accuracy_delta": 0.0+ },+ "sv": {+ "after": {+ "examples": 2974,+ "expected_checkpoint": "multilingual",+ "named_language_accuracy": 0.302959,+ "named_language_correct": 901,+ "named_language_guesses": {+ "de": 7,+ "en": 43,+ "fr": 4,+ "ro": 1,+ "sv": 901,+ "undecided": 2018+ },+ "route_accuracy": 0.869536,+ "route_correct": 2586,+ "routed_to": {+ "english": 388,+ "multilingual": 2586+ }+ },+ "before": {+ "examples": 2974,+ "expected_checkpoint": "multilingual",+ "named_language_accuracy": 0.302959,+ "named_language_correct": 901,+ "named_language_guesses": {+ "de": 7,+ "en": 43,+ "fr": 4,+ "ro": 1,+ "sv": 901,+ "undecided": 2018+ },+ "route_accuracy": 0.869536,+ "route_correct": 2586,+ "routed_to": {+ "english": 388,+ "multilingual": 2586+ }+ },+ "named_language_accuracy_delta": 0.0,+ "route_accuracy_delta": 0.0+ },+ "sw": {+ "after": {+ "examples": 2974,+ "expected_checkpoint": "multilingual",+ "named_language_accuracy": 0.0,+ "named_language_correct": 0,+ "named_language_guesses": {+ "en": 18,+ "es": 360,+ "fr": 6,+ "pt": 40,+ "undecided": 2550+ },+ "route_accuracy": 0.136516,+ "route_correct": 406,+ "routed_to": {+ "english": 2568,+ "multilingual": 406+ }+ },+ "before": {+ "examples": 2974,+ "expected_checkpoint": "multilingual",+ "named_language_accuracy": 0.0,+ "named_language_correct": 0,+ "named_language_guesses": {+ "en": 18,+ "es": 360,+ "fr": 6,+ "pt": 40,+ "undecided": 2550+ },+ "route_accuracy": 0.136516,+ "route_correct": 406,+ "routed_to": {+ "english": 2568,+ "multilingual": 406+ }+ },+ "named_language_accuracy_delta": 0.0,+ "route_accuracy_delta": 0.0+ },+ "ta": {+ "after": {+ "examples": 2974,+ "expected_checkpoint": "multilingual",+ "named_language_accuracy": 0.0,+ "named_language_correct": 0,+ "named_language_guesses": {+ "undecided": 2974+ },+ "route_accuracy": 1.0,+ "route_correct": 2974,+ "routed_to": {+ "multilingual": 2974+ }+ },+ "before": {+ "examples": 2974,+ "expected_checkpoint": "multilingual",+ "named_language_accuracy": 0.0,+ "named_language_correct": 0,+ "named_language_guesses": {+ "undecided": 2974+ },+ "route_accuracy": 1.0,+ "route_correct": 2974,+ "routed_to": {+ "multilingual": 2974+ }+ },+ "named_language_accuracy_delta": 0.0,+ "route_accuracy_delta": 0.0+ },+ "te": {+ "after": {+ "examples": 2974,+ "expected_checkpoint": "multilingual",+ "named_language_accuracy": 0.0,+ "named_language_correct": 0,+ "named_language_guesses": {+ "en": 3,+ "undecided": 2971+ },+ "route_accuracy": 0.998655,+ "route_correct": 2970,+ "routed_to": {+ "english": 4,+ "multilingual": 2970+ }+ },+ "before": {+ "examples": 2974,+ "expected_checkpoint": "multilingual",+ "named_language_accuracy": 0.0,+ "named_language_correct": 0,+ "named_language_guesses": {+ "en": 3,+ "undecided": 2971+ },+ "route_accuracy": 0.998655,+ "route_correct": 2970,+ "routed_to": {+ "english": 4,+ "multilingual": 2970+ }+ },+ "named_language_accuracy_delta": 0.0,+ "route_accuracy_delta": 0.0+ },+ "th": {+ "after": {+ "examples": 2974,+ "expected_checkpoint": "multilingual",+ "named_language_accuracy": 0.0,+ "named_language_correct": 0,+ "named_language_guesses": {+ "undecided": 2974+ },+ "route_accuracy": 0.998319,+ "route_correct": 2969,+ "routed_to": {+ "english": 5,+ "multilingual": 2969+ }+ },+ "before": {+ "examples": 2974,+ "expected_checkpoint": "multilingual",+ "named_language_accuracy": 0.0,+ "named_language_correct": 0,+ "named_language_guesses": {+ "undecided": 2974+ },+ "route_accuracy": 0.998319,+ "route_correct": 2969,+ "routed_to": {+ "english": 5,+ "multilingual": 2969+ }+ },+ "named_language_accuracy_delta": 0.0,+ "route_accuracy_delta": 0.0+ },+ "tl": {+ "after": {+ "examples": 2974,+ "expected_checkpoint": "multilingual",+ "named_language_accuracy": 0.0,+ "named_language_correct": 0,+ "named_language_guesses": {+ "en": 313,+ "es": 7,+ "fr": 257,+ "pt": 77,+ "undecided": 2320+ },+ "route_accuracy": 0.114997,+ "route_correct": 342,+ "routed_to": {+ "english": 2632,+ "multilingual": 342+ }+ },+ "before": {+ "examples": 2974,+ "expected_checkpoint": "multilingual",+ "named_language_accuracy": 0.0,+ "named_language_correct": 0,+ "named_language_guesses": {+ "en": 313,+ "es": 7,+ "fr": 257,+ "pt": 77,+ "undecided": 2320+ },+ "route_accuracy": 0.114997,+ "route_correct": 342,+ "routed_to": {+ "english": 2632,+ "multilingual": 342+ }+ },+ "named_language_accuracy_delta": 0.0,+ "route_accuracy_delta": 0.0+ },+ "tr": {+ "after": {+ "examples": 2974,+ "expected_checkpoint": "multilingual",+ "named_language_accuracy": 0.0,+ "named_language_correct": 0,+ "named_language_guesses": {+ "az": 191,+ "en": 17,+ "es": 4,+ "it": 13,+ "undecided": 2749+ },+ "route_accuracy": 0.905178,+ "route_correct": 2692,+ "routed_to": {+ "english": 282,+ "multilingual": 2692+ }+ },+ "before": {+ "examples": 2974,+ "expected_checkpoint": "multilingual",+ "named_language_accuracy": 0.0,+ "named_language_correct": 0,+ "named_language_guesses": {+ "az": 191,+ "en": 17,+ "es": 4,+ "it": 13,+ "undecided": 2749+ },+ "route_accuracy": 0.905178,+ "route_correct": 2692,+ "routed_to": {+ "english": 282,+ "multilingual": 2692+ }+ },+ "named_language_accuracy_delta": 0.0,+ "route_accuracy_delta": 0.0+ },+ "ur": {+ "after": {+ "examples": 2974,+ "expected_checkpoint": "multilingual",+ "named_language_accuracy": 0.0,+ "named_language_correct": 0,+ "named_language_guesses": {+ "en": 1,+ "undecided": 2973+ },+ "route_accuracy": 0.998319,+ "route_correct": 2969,+ "routed_to": {+ "english": 5,+ "multilingual": 2969+ }+ },+ "before": {+ "examples": 2974,+ "expected_checkpoint": "multilingual",+ "named_language_accuracy": 0.0,+ "named_language_correct": 0,+ "named_language_guesses": {+ "en": 1,+ "undecided": 2973+ },+ "route_accuracy": 0.998319,+ "route_correct": 2969,+ "routed_to": {+ "english": 5,+ "multilingual": 2969+ }+ },+ "named_language_accuracy_delta": 0.0,+ "route_accuracy_delta": 0.0+ },+ "vi": {+ "after": {+ "examples": 2974,+ "expected_checkpoint": "multilingual",+ "named_language_accuracy": 0.0,+ "named_language_correct": 0,+ "named_language_guesses": {+ "en": 7,+ "es": 24,+ "fr": 32,+ "it": 44,+ "pt": 39,+ "undecided": 2828+ },+ "route_accuracy": 0.956288,+ "route_correct": 2844,+ "routed_to": {+ "english": 130,+ "multilingual": 2844+ }+ },+ "before": {+ "examples": 2974,+ "expected_checkpoint": "multilingual",+ "named_language_accuracy": 0.0,+ "named_language_correct": 0,+ "named_language_guesses": {+ "en": 7,+ "es": 24,+ "fr": 32,+ "it": 44,+ "pt": 39,+ "undecided": 2828+ },+ "route_accuracy": 0.956288,+ "route_correct": 2844,+ "routed_to": {+ "english": 130,+ "multilingual": 2844+ }+ },+ "named_language_accuracy_delta": 0.0,+ "route_accuracy_delta": 0.0+ },+ "zh-CN": {+ "after": {+ "examples": 2974,+ "expected_checkpoint": "multilingual",+ "named_language_accuracy": 0.0,+ "named_language_correct": 0,+ "named_language_guesses": {+ "undecided": 2974+ },+ "route_accuracy": 0.999664,+ "route_correct": 2973,+ "routed_to": {+ "english": 1,+ "multilingual": 2973+ }+ },+ "before": {+ "examples": 2974,+ "expected_checkpoint": "multilingual",+ "named_language_accuracy": 0.0,+ "named_language_correct": 0,+ "named_language_guesses": {+ "undecided": 2974+ },+ "route_accuracy": 0.999664,+ "route_correct": 2973,+ "routed_to": {+ "english": 1,+ "multilingual": 2973+ }+ },+ "named_language_accuracy_delta": 0.0,+ "route_accuracy_delta": 0.0+ },+ "zh-TW": {+ "after": {+ "examples": 2974,+ "expected_checkpoint": "multilingual",+ "named_language_accuracy": 0.0,+ "named_language_correct": 0,+ "named_language_guesses": {+ "en": 3,+ "undecided": 2971+ },+ "route_accuracy": 0.995965,+ "route_correct": 2962,+ "routed_to": {+ "english": 12,+ "multilingual": 2962+ }+ },+ "before": {+ "examples": 2974,+ "expected_checkpoint": "multilingual",+ "named_language_accuracy": 0.0,+ "named_language_correct": 0,+ "named_language_guesses": {+ "en": 3,+ "undecided": 2971+ },+ "route_accuracy": 0.995965,+ "route_correct": 2962,+ "routed_to": {+ "english": 12,+ "multilingual": 2962+ }+ },+ "named_language_accuracy_delta": 0.0,+ "route_accuracy_delta": 0.0+ }+ },+ "model_free": true,+ "overall": {+ "after": {+ "examples": 151674,+ "named_language_accuracy": 0.088407,+ "named_language_correct": 13409,+ "route_accuracy": 0.774292,+ "route_correct": 117440+ },+ "before": {+ "examples": 151674,+ "named_language_accuracy": 0.088367,+ "named_language_correct": 13403,+ "route_accuracy": 0.774279,+ "route_correct": 117438+ }+ },+ "revision": "940fd47a81eaa7f2cc7b129674d945d618ac38c2",+ "routing_metric_note": "Router._route decisions are recorded; route accuracy uses expected checkpoint by MASSIVE locale, including Router's configured default for undecided text.",+ "routing_policy": "english locale -> english checkpoint; every other locale -> multilingual",+ "split": "test"+}