Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

feat(lang): detect Swedish with measured MASSIVE results

Daniel Nylanderee499bb09558
diff --git a/laya-ts/src/lang.ts b/laya-ts/src/lang.ts
index 2b16adcc..7da5e962 100644
--- a/laya-ts/src/lang.ts
+++ b/laya-ts/src/lang.ts
@@ -77,6 +77,16 @@ const STOP: Record<string, Set<string>> = {
"delle", "dello", "degli", "agli", "alle", "col"]),
nl: new Set(["het", "een", "van", "is", "op", "te", "dat", "niet", "met", "voor", "zijn", "aan",
"door", "maar", "ook", "worden", "deze", "naar", "wordt"]),
+ // Keep Swedish routing in parity with Python. Distinctive words help ASCII-normalised ticket text;
+ // shared Nordic words are tracked separately so Danish and Norwegian are not named Swedish.
+ sv: new Set(["jag", "är", "och", "inte", "att", "från", "till", "behöver", "får", "skulle", "ska",
+ "vill", "måste", "också", "dessa", "detta", "säger", "upp", "utan", "mitt", "min", "om",
+ "kommer", "här", "två", "vi", "nästa", "gör", "göra", "hjälp", "hjälpa", "mig",
+ "återbetalning", "återbetala", "faktura", "gång", "gånger", "hittar", "inställningen",
+ "inställningarna", "lösenord", "när", "öppnar", "spårningen", "aterbetalning", "aterbetala",
+ "behover", "fel", "ganger", "hjalp", "hjalpa", "installningen", "installningarna", "kraschar",
+ "kvittot", "losenord", "nar", "oppnar", "paket", "skicka", "sparningen", "tva", "uppdaterats",
+ "blivit", "debiterade", "appen"]),
ro: new Set(["și", "să", "este", "sunt", "care", "pentru", "din", "dar", "după", "până", "fără",
"ale", "lui", "în", "fost", "acum", "vreau", "trebuie", "foarte", "acest", "această",
"acesta", "aceasta", "mi", "ți", "vă", "nu"]),
@@ -137,8 +147,19 @@ const SHARED_WORDS: Set<string> = (() => {
for (const [w, n] of counts) if (n > 1) out.add(w);
return out;
})();
+// Danish and Norwegian share many common words with Swedish; they can add score but cannot name sv alone.
+const NORDIC_OVERLAP_WORDS = new Set(["hej", "ja", "nej", "jo", "tack", "mig", "min", "om", "kommer", "får", "skulle", "vi"]);
+for (const word of NORDIC_OVERLAP_WORDS) SHARED_WORDS.add(word);
const EN_ONLY_WORDS = new Set([...STOP["en"]].filter((w) => !SHARED_WORDS.has(w)));
+// Short support fragments need a narrower vocabulary than the general four-word language guess.
+// Generic words such as "fel" and "hjälp" are deliberately omitted from this subset.
+const SHORT_SWEDISH_WORDS = new Set([
+ "åtkomst", "atkomst", "lösenord", "losenord", "fakturan", "betalningen", "inloggningen",
+ "glömt", "glomt", "behöver", "behover", "återbetalning", "aterbetalning", "kvitto", "kvittot",
+ "spårningen", "sparningen", "inställningen", "installningen", "felmeddelande", "abonnemanget",
+]);
+
// JS `\w` is ASCII-only, so Python's `[^\W\d_]` needs the Unicode classes spelled out (Nl/No are in Python's `\w`).
const WORD_RE = /[\p{L}\p{Nl}\p{No}]+/gu;
// Lookbehind for the same reason as the Python side (see laya/lang.py): without it the
@@ -298,8 +319,13 @@ export function latinProfile(text: string): LatinProfile {
}
const diacRate = diac / Math.max(1, lowered.length);
const nonEnglish = diacRate >= NON_EN_DIACRITIC_RATE;
+ const nordicOverlap = words.some((w) => NORDIC_OVERLAP_WORDS.has(w)) &&
+ !words.some((w) => EN_ONLY_WORDS.has(w));
+ if (words.length > 1 && words.length < 4 && words.some((w) => SHORT_SWEDISH_WORDS.has(w))) {
+ return { language: "sv", englishHits: 0, diacriticRate: diacRate, looksNonEnglish: nonEnglish };
+ }
if (words.length < 4) {
- return { language: null, englishHits: 0, diacriticRate: diacRate, looksNonEnglish: nonEnglish };
+ return { language: null, englishHits: 0, diacriticRate: diacRate, looksNonEnglish: nonEnglish || nordicOverlap };
}
const scores: Record<string, number> = {};
for (const [lg, sw] of Object.entries(STOP)) {
@@ -320,12 +346,16 @@ export function latinProfile(text: string): LatinProfile {
let lang: string | null = null;
if (bestLg && best >= Math.max(2, en + 2)) {
lang = bestLg;
+ } else if (bestLg === "sv" && words.includes("inte") && words.includes("kan") &&
+ ["kan", "jag", "vi"].includes(words[0]) && en <= 1) {
+ // Short login requests like "kan inte logga in" contain one English-shaped token (`in`).
+ lang = bestLg;
} else if (bestLg && nonEnglish && best >= Math.max(2, en)) {
lang = bestLg;
} else if (en && (!nonEnglish || englishRescuedByWords(wordSet, diacRate))) {
lang = "en";
}
- return { language: lang, englishHits: en, diacriticRate: diacRate, looksNonEnglish: nonEnglish };
+ return { language: lang, englishHits: en, diacriticRate: diacRate, looksNonEnglish: nonEnglish || (lang === null && nordicOverlap) };
}
export function guessLatinLanguage(text: string): string | null {
diff --git a/laya-ts/tests/lang.test.ts b/laya-ts/tests/lang.test.ts
index a04b2b9e..5b765bb5 100644
--- a/laya-ts/tests/lang.test.ts
+++ b/laya-ts/tests/lang.test.ts
@@ -208,7 +208,7 @@ describe("lang", () => {
["sluk lyset i soveværelset", null],
["kan jeg få en refundering for det dobbelte beløb", null],
["stäng av ljuset i sovrummet", null],
- ["jag vill ha en återbetalning för den dubbla avgiften", null],
+ ["jag vill ha en återbetalning för den dubbla avgiften", "sv"],
["The naïve façade needs a fresh coat of paint", null],
])("does not rescue accented non-English text: %s", (text, language) => {
expect(guessLatinLanguage(text)).toBe(language);
diff --git a/laya/lang.py b/laya/lang.py
index f8f0f0d5..2e8a740f 100644
--- a/laya/lang.py
+++ b/laya/lang.py
@@ -118,6 +118,19 @@ _STOP = {
"delle", "dello", "degli", "agli", "alle", "col"},
"nl": {"het", "een", "van", "is", "op", "te", "dat", "niet", "met", "voor", "zijn", "aan",
"door", "maar", "ook", "worden", "deze", "naar", "wordt"},
+ # Swedish function words and common auxiliaries. Several overlap with English or German
+ # (`i`, `kan`, `har`), so the distinctive words below are what lets Swedish text survive an
+ # ASCII-normalising ticket pipeline without treating one stray Nordic letter as the only hint.
+ "sv": {"jag", "är", "och", "inte", "att", "från", "till", "behöver", "får", "skulle", "ska",
+ "vill", "måste", "också", "dessa", "detta", "säger", "upp", "utan", "mitt", "min", "om",
+ "kommer", "här", "två", "vi", "nästa", "gör", "göra",
+ "hjälp", "hjälpa", "mig", "återbetalning", "återbetala", "faktura", "gång", "gånger",
+ "hittar", "inställningen", "inställningarna", "lösenord", "när", "öppnar", "spårningen",
+ # Common spellings from ticket systems that strip Swedish diacritics.
+ "aterbetalning", "aterbetala", "behover", "fel", "ganger", "hjalp", "hjalpa",
+ "installningen", "installningarna", "kraschar", "kvittot", "losenord", "nar",
+ "oppnar", "paket", "skicka", "sparningen", "tva", "uppdaterats", "blivit", "debiterade",
+ "appen"},
# Romanian words that its Romance neighbours do not share, so adding `ro` cannot steal a
# French/Spanish/Italian/Portuguese state: `la`, `o`, `un`, `de`, `pe`, `ca` are deliberately
# left out for that reason, and the diacritic signal below carries the rest.
@@ -153,6 +166,18 @@ _STOP = {
"hər", "nə", "kimi", "görə", "sonra", "əgər", "eger", "deyil", "lakin", "amma",
"ancaq", "artıq", "artiq", "də", "isə", "həm", "yalnız", "yalniz"},
}
+
+# Short support fragments often consist of only two or three words, so they do not reach the
+# four-token minimum used by the general language guess. These spellings are specific enough to
+# identify Swedish in that narrow case; generic words such as `fel`, `hjälp`, `paket` and `appen`
+# are deliberately not sufficient on their own. Keep ASCII-normalised variants beside the forms
+# users commonly type without Swedish characters.
+_SHORT_SWEDISH_WORDS = {
+ "åtkomst", "atkomst", "lösenord", "losenord", "fakturan", "betalningen", "inloggningen",
+ "glömt", "glomt", "behöver", "behover", "återbetalning", "aterbetalning", "kvitto", "kvittot",
+ "spårningen", "sparningen", "inställningen", "installningen", "felmeddelande",
+ "abonnemanget",
+}
# Letters that ordinary English does not use. This is the signal that catches a Latin-script
# language we hold no stopwords for at all (Romanian, Polish, Czech, Turkish, Baltic, ...),
# which is the difference between routing it to the multilingual checkpoint and silently
@@ -174,6 +199,13 @@ _NON_EN_DIACRITICS = set(
# way), though it still counts toward the total of a language that also matched a word of its own.
_SHARED_WORDS = {w for w in {word for words in _STOP.values() for word in words}
if sum(w in words for words in _STOP.values()) > 1}
+# Danish is not in `_STOP`, but several of its common words also occur in the Swedish list.
+# Do not let these words alone name a Danish sentence as Swedish; they remain useful score hits
+# when another, more distinctive Swedish word is present.
+_NORDIC_OVERLAP_WORDS = {
+ "hej", "ja", "nej", "jo", "tack", "mig", "min", "om", "kommer", "får", "skulle", "vi",
+}
+_SHARED_WORDS.update(_NORDIC_OVERLAP_WORDS)
# English function words no other list holds (`in`, `is`, `as`, `was` are shared with German,
# Dutch and Portuguese). They alone carry the English rescue of `latin_profile`.
_EN_ONLY_WORDS = _STOP["en"] - _SHARED_WORDS
@@ -396,9 +428,10 @@ def latin_profile(text: str) -> Dict[str, object]:
"""Evidence behind the Latin-script language guess.
Returns `language` (may be None when undecided), `english_hits`, `diacritic_rate` and
- `looks_non_english`. `analyse` needs the evidence and not just the verdict, because
- "undecided" and "English" are different answers and only one of them is safe to send to the
- English checkpoint.
+ `looks_non_english`. The latter can be true for non-English diacritics or an overlapping
+ Swedish-Danish marker even when this heuristic cannot name the language. `analyse` needs the
+ evidence and not just the verdict, because "undecided" and "English" are different answers
+ and only one of them is safe to send to the English checkpoint.
A non-English language is only named when it matched at least one word that no other list
claims: shared function words alone (`la`, `e`, `o`) identify no particular language.
@@ -409,9 +442,13 @@ def latin_profile(text: str) -> Dict[str, object]:
diac = sum(1 for ch in lowered if ch in _NON_EN_DIACRITICS)
diac_rate = diac / max(1, len(lowered))
non_english = diac_rate >= NON_EN_DIACRITIC_RATE
+ nordic_overlap = bool(set(words) & _NORDIC_OVERLAP_WORDS) and not bool(set(words) & _EN_ONLY_WORDS)
+ if 1 < len(words) < 4 and set(words) & _SHORT_SWEDISH_WORDS:
+ return {"language": "sv", "english_hits": 0, "diacritic_rate": diac_rate,
+ "looks_non_english": non_english}
if len(words) < 4:
return {"language": None, "english_hits": 0, "diacritic_rate": diac_rate,
- "looks_non_english": non_english}
+ "looks_non_english": non_english or nordic_overlap}
# A collision word counts once however often it repeats; every other word counts its hits.
counts = Counter(words)
@@ -431,6 +468,12 @@ def latin_profile(text: str) -> Dict[str, object]:
if best_lg and best >= max(2, en + 2):
# a non-English language needs a clear margin over English function words
lang = best_lg
+ elif (best_lg == "sv" and "inte" in words and "kan" in words
+ and words[0] in {"kan", "jag", "vi"} and en <= 1):
+ # Short login requests such as "kan inte logga in" carry a distinctive Swedish phrase
+ # but also one English-shaped token (`in`). Do not treat `kan` alone as Swedish: it is
+ # common in Danish and Norwegian too.
+ lang = best_lg
elif best_lg and non_english and best >= max(2, en):
# Needs two hits here too. One shared function word ("para" in Turkish text) named Spanish
# on the strength of the diacritics alone, which is a guess dressed as a detection.
@@ -438,7 +481,7 @@ def latin_profile(text: str) -> Dict[str, object]:
elif en and (not non_english or _english_rescued_by_words(words, diac_rate)):
lang = "en"
return {"language": lang, "english_hits": en, "diacritic_rate": diac_rate,
- "looks_non_english": non_english}
+ "looks_non_english": non_english or (lang is None and nordic_overlap)}
def guess_latin_language(text: str) -> Optional[str]:
@@ -543,8 +586,8 @@ def _analyse_text(text: str) -> Dict[str, object]:
lang = prof_lat["language"]
# Undecided is not English. Treating it as English sent every Latin-script language we hold no
# stopwords for to the checkpoint that cannot read it, silently. When nothing identifies the
- # language, non-English letters are enough to prefer the multilingual checkpoint; text with no
- # such letters (including short English) still goes to the English one.
+ # language, non-English letters or a shared Swedish-Danish marker can still prefer the
+ # multilingual checkpoint; text with neither signal still goes to the English one.
undecided = lang is None
english = lang == "en" or (undecided and not prof_lat["looks_non_english"])
return {"script": "latin", "script_profile": prof, "language": lang,
diff --git a/research/evals/massive_route_comparison.md b/research/evals/massive_route_comparison.md
new file mode 100644
index 00000000..f6fdd1db
--- /dev/null
+++ b/research/evals/massive_route_comparison.md
@@ -0,0 +1,62 @@
+# Swedish routing: MASSIVE full-test comparison
+
+This report compares the `laya.lang` detector on all 51 locale files in the pinned
+`mteb/amazon_massive_intent` test split (dataset revision
+`940fd47a81eaa7f2cc7b129674d945d618ac38c2`). Each locale contributes 2,974 examples,
+for 151,674 examples total. The comparison is model-free: it evaluates the same text
+against the baseline `main` detector and the proposed detector without loading either
+checkpoint.
+
+## Method
+
+The expected route is `english` for the `en` locale and `multilingual` for every other
+locale. The script calls the repository's real `Router._route` for both detectors, so
+all other routing rules stay the same, including the configured `english` default when
+Latin text is undecided. Route accuracy measures only that checkpoint choice. Exact
+named-language accuracy is reported separately: a detector can safely choose
+`multilingual` while leaving the specific language undecided. Results for every locale,
+including guess counts, are in
+[`../results/massive_route_comparison_swedish.json`](../results/massive_route_comparison_swedish.json).
+
+Reproduce from the repository root, passing a baseline and candidate checkout:
+
+```sh
+python research/evals/massive_route_comparison.py \
+ --before-module /path/to/baseline/laya/lang.py \
+ --after-module laya/lang.py \
+ --out research/results/massive_route_comparison_swedish.json
+```
+
+The report records the source Git revisions and pinned dataset revision. The script
+downloads the split from Hugging Face on first run and needs `huggingface_hub` installed.
+
+## Results
+
+| Measure | Baseline | Candidate | Change |
+|---|---:|---:|---:|
+| Overall checkpoint route accuracy | 77.428% | 77.429% | +0.001 pp |
+| Exact named-language accuracy | 8.837% | 8.841% | +0.004 pp |
+| Swedish checkpoint route accuracy | 86.954% | 86.954% | 0.000 pp |
+| Swedish named-language accuracy | 0.000% | 30.296% | +30.296 pp |
+| Danish checkpoint route accuracy | 53.430% | 53.430% | 0.000 pp |
+| Norwegian Bokmål route accuracy | 49.630% | 49.630% | 0.000 pp |
+
+Checkpoint route accuracy changes only for `it` (+0.034 pp) and `pt` (+0.034 pp); no
+locale decreased. The overall route gain is two examples out of 151,674. Named-language
+accuracy also changes slightly for `de`, `it`, `pt`, and `ro`, reflecting shared evidence
+rules rather than locale-specific language work. For the 51 locale rows and exact
+before/after counts, use the JSON artifact rather than rounded percentages here.
+
+## Interpretation and limits
+
+MASSIVE measures intent classification utterances, not customer-support transcripts.
+It supplies a broad per-locale routing check, not evidence that Swedish support
+customers are served well end to end. On the pinned split, the Swedish change improves
+named-language detection but barely changes which checkpoint is selected; the
+configured default already sends most ambiguous examples to English.
+
+Swedish, Danish, and Norwegian share short words. In the candidate results, four
+Norwegian examples are named `sv`; all still route to the same multilingual checkpoint
+as before. The detector does not claim that checkpoint-routing gains prove perfect
+language identification. Broader Swedish support data and a focused false-positive
+review remain useful follow-up work.
diff --git a/research/evals/massive_route_comparison.py b/research/evals/massive_route_comparison.py
new file mode 100644
index 00000000..c848a7ca
--- /dev/null
+++ b/research/evals/massive_route_comparison.py
@@ -0,0 +1,169 @@
+"""Compare language routing by locale on the pinned MASSIVE test split.
+
+This model-free diagnostic reports both checkpoint routing and named-language detection before
+and after a proposed change to ``laya.lang``. It evaluates the full test split for every locale.
+
+Run from the repository root:
+
+ python research/evals/massive_route_comparison.py \
+ --before-module /path/to/baseline/laya/lang.py \
+ --after-module laya/lang.py --out research/results/massive_route_comparison.json
+
+No model weights are loaded. A non-English MASSIVE locale should route to ``multilingual``;
+English should route to ``english``. Named-language accuracy is reported separately because the
+router only needs to choose a readable checkpoint, not identify every language by name.
+"""
+from __future__ import annotations
+
+import argparse
+import gzip
+import importlib.util
+import json
+import subprocess
+import sys
+from collections import Counter
+from pathlib import Path
+from typing import Any, Dict
+
+from huggingface_hub import hf_hub_download, list_repo_files
+
+
+DATASET = "mteb/amazon_massive_intent"
+REVISION = "940fd47a81eaa7f2cc7b129674d945d618ac38c2"
+REPO_ROOT = Path(__file__).resolve().parents[2]
+
+
+def load_detector(path: Path, name: str):
+ """Load a standalone Laya language module from an arbitrary checkout."""
+ spec = importlib.util.spec_from_file_location(name, path)
+ if spec is None or spec.loader is None:
+ raise RuntimeError("cannot load language detector: %s" % path)
+ module = importlib.util.module_from_spec(spec)
+ sys.modules[name] = module
+ spec.loader.exec_module(module)
+ return module
+
+
+def route(detector: Any, router_module: Any, text: str) -> tuple[str, str]:
+ """Use the repository's real Router._route decision path without loading a checkpoint."""
+ router = router_module.Router(default="english")
+ decision = router._route(text)
+ return decision.model, (decision.get("detection") or {}).get("language") or "undecided"
+
+
+def source_revision(path: Path) -> str:
+ """Return the Git revision owning a detector source file."""
+ source = path.resolve()
+ root = next((parent for parent in source.parents if (parent / ".git").exists()), source.parent)
+ try:
+ return subprocess.check_output(
+ ["git", "-C", str(root), "rev-parse", "HEAD"], text=True,
+ stderr=subprocess.DEVNULL,
+ ).strip()
+ except (OSError, subprocess.CalledProcessError):
+ return "unknown"
+
+
+def score_locale(detector: Any, router_module: Any, locale: str, path: Path) -> Dict[str, Any]:
+ routes: Counter[str] = Counter()
+ named_languages: Counter[str] = Counter()
+ named_correct = 0
+ examples = 0
+ with gzip.open(path, "rt", encoding="utf-8") as source:
+ for line in source:
+ row = json.loads(line)
+ model, named_language = route(detector, router_module, row["text"])
+ routes[model] += 1
+ named_languages[named_language] += 1
+ named_correct += named_language == locale
+ examples += 1
+ return {
+ "examples": examples,
+ "route_correct": 0,
+ "route_accuracy": None,
+ "routed_to": dict(sorted(routes.items())),
+ "named_language_correct": named_correct,
+ "named_language_accuracy": round(named_correct / examples, 6) if examples else None,
+ "named_language_guesses": dict(sorted(named_languages.items())),
+ }
+
+
+def main(argv=None) -> int:
+ parser = argparse.ArgumentParser(description=__doc__)
+ parser.add_argument("--before-module", required=True, type=Path,
+ help="path to baseline laya/lang.py")
+ parser.add_argument("--after-module", required=True, type=Path,
+ help="path to candidate laya/lang.py")
+ parser.add_argument("--out", type=Path, help="write the JSON report to this path")
+ args = parser.parse_args(argv)
+
+ files = list_repo_files(DATASET, repo_type="dataset", revision=REVISION)
+ locales = sorted(name[len("test/"):-len(".json.gz")] for name in files
+ if name.startswith("test/") and name.endswith(".json.gz"))
+ if not locales:
+ raise RuntimeError("no per-locale MASSIVE test files at pinned revision")
+ before_path = args.before_module.resolve()
+ after_path = args.after_module.resolve()
+ before = load_detector(before_path, "laya_lang_before")
+ after = load_detector(after_path, "laya_lang_after")
+ # Use this checkout's Router implementation and replace only its pure detector function for
+ # each pass; all checkpoint-selection rules remain the repository code.
+ from laya import router as router_module
+
+ report: Dict[str, Any] = {
+ "dataset": DATASET,
+ "revision": REVISION,
+ "split": "test",
+ "model_free": True,
+ "routing_policy": "english locale -> english checkpoint; every other locale -> multilingual",
+ "routing_metric_note": "Router._route decisions are recorded; route accuracy uses expected checkpoint by MASSIVE locale, including Router's configured default for undecided text.",
+ "before_module": str(before_path),
+ "before_revision": source_revision(before_path),
+ "after_module": str(after_path),
+ "after_revision": source_revision(after_path),
+ "locales": {},
+ "overall": {},
+ }
+ before_total = {"examples": 0, "route_correct": 0, "named_language_correct": 0}
+ after_total = dict(before_total)
+ for locale in locales:
+ local_path = Path(hf_hub_download(
+ repo_id=DATASET,
+ filename="test/%s.json.gz" % locale,
+ repo_type="dataset",
+ revision=REVISION,
+ ))
+ router_module.analyse = before.analyse
+ old = score_locale(before, router_module, locale, local_path)
+ router_module.analyse = after.analyse
+ new = score_locale(after, router_module, locale, local_path)
+ expected_model = "english" if locale == "en" else "multilingual"
+ for item in (old, new):
+ item["expected_checkpoint"] = expected_model
+ item["route_correct"] = item["routed_to"].get(expected_model, 0)
+ item["route_accuracy"] = round(item["route_correct"] / item["examples"], 6)
+ report["locales"][locale] = {"before": old, "after": new,
+ "route_accuracy_delta": round(new["route_accuracy"] - old["route_accuracy"], 6),
+ "named_language_accuracy_delta": round(new["named_language_accuracy"] - old["named_language_accuracy"], 6)}
+ for total, item in ((before_total, old), (after_total, new)):
+ total["examples"] += item["examples"]
+ total["route_correct"] += item["route_correct"]
+ total["named_language_correct"] += item["named_language_correct"]
+
+ for name, total in (("before", before_total), ("after", after_total)):
+ count = total["examples"]
+ report["overall"][name] = {
+ **total,
+ "route_accuracy": round(total["route_correct"] / count, 6) if count else None,
+ "named_language_accuracy": round(total["named_language_correct"] / count, 6) if count else None,
+ }
+ if args.out:
+ args.out.parent.mkdir(parents=True, exist_ok=True)
+ args.out.write_text(json.dumps(report, ensure_ascii=False, indent=2, sort_keys=True) + "\n",
+ encoding="utf-8")
+ print(json.dumps(report, ensure_ascii=False, indent=2, sort_keys=True))
+ return 0
+
+
+if __name__ == "__main__":
+ raise SystemExit(main())
diff --git a/research/results/massive_route_comparison_swedish.json b/research/results/massive_route_comparison_swedish.json
new file mode 100644
index 00000000..20c7862b
--- /dev/null
+++ b/research/results/massive_route_comparison_swedish.json
@@ -0,0 +1,2018 @@
+{
+ "after_module": "/Users/danielnylander/Documents/Codex/2026-09-28/kan/work/laya/laya/lang.py",
+ "after_revision": "75f2a04a80dbf7dee874addc29c77a7539860ccd",
+ "before_module": "/Users/danielnylander/Documents/Codex/2026-09-28/kan/work/laya-split/laya/lang.py",
+ "before_revision": "6d942c92081fbc139e736bbd9ac0023223c29b7f",
+ "dataset": "mteb/amazon_massive_intent",
+ "locales": {
+ "af": {
+ "after": {
+ "examples": 2974,
+ "expected_checkpoint": "multilingual",
+ "named_language_accuracy": 0.0,
+ "named_language_correct": 0,
+ "named_language_guesses": {
+ "de": 122,
+ "en": 734,
+ "fr": 1,
+ "it": 4,
+ "nl": 177,
+ "pt": 25,
+ "undecided": 1911
+ },
+ "route_accuracy": 0.167787,
+ "route_correct": 499,
+ "routed_to": {
+ "english": 2475,
+ "multilingual": 499
+ }
+ },
+ "before": {
+ "examples": 2974,
+ "expected_checkpoint": "multilingual",
+ "named_language_accuracy": 0.0,
+ "named_language_correct": 0,
+ "named_language_guesses": {
+ "de": 122,
+ "en": 734,
+ "fr": 1,
+ "it": 4,
+ "nl": 177,
+ "pt": 25,
+ "undecided": 1911
+ },
+ "route_accuracy": 0.167787,
+ "route_correct": 499,
+ "routed_to": {
+ "english": 2475,
+ "multilingual": 499
+ }
+ },
+ "named_language_accuracy_delta": 0.0,
+ "route_accuracy_delta": 0.0
+ },
+ "am": {
+ "after": {
+ "examples": 2974,
+ "expected_checkpoint": "multilingual",
+ "named_language_accuracy": 0.0,
+ "named_language_correct": 0,
+ "named_language_guesses": {
+ "undecided": 2974
+ },
+ "route_accuracy": 1.0,
+ "route_correct": 2974,
+ "routed_to": {
+ "multilingual": 2974
+ }
+ },
+ "before": {
+ "examples": 2974,
+ "expected_checkpoint": "multilingual",
+ "named_language_accuracy": 0.0,
+ "named_language_correct": 0,
+ "named_language_guesses": {
+ "undecided": 2974
+ },
+ "route_accuracy": 1.0,
+ "route_correct": 2974,
+ "routed_to": {
+ "multilingual": 2974
+ }
+ },
+ "named_language_accuracy_delta": 0.0,
+ "route_accuracy_delta": 0.0
+ },
+ "ar": {
+ "after": {
+ "examples": 2974,
+ "expected_checkpoint": "multilingual",
+ "named_language_accuracy": 0.0,
+ "named_language_correct": 0,
+ "named_language_guesses": {
+ "en": 24,
+ "undecided": 2950
+ },
+ "route_accuracy": 0.96503,
+ "route_correct": 2870,
+ "routed_to": {
+ "english": 104,
+ "multilingual": 2870
+ }
+ },
+ "before": {
+ "examples": 2974,
+ "expected_checkpoint": "multilingual",
+ "named_language_accuracy": 0.0,
+ "named_language_correct": 0,
+ "named_language_guesses": {
+ "en": 24,
+ "undecided": 2950
+ },
+ "route_accuracy": 0.96503,
+ "route_correct": 2870,
+ "routed_to": {
+ "english": 104,
+ "multilingual": 2870
+ }
+ },
+ "named_language_accuracy_delta": 0.0,
+ "route_accuracy_delta": 0.0
+ },
+ "az": {
+ "after": {
+ "examples": 2974,
+ "expected_checkpoint": "multilingual",
+ "named_language_accuracy": 0.138198,
+ "named_language_correct": 411,
+ "named_language_guesses": {
+ "az": 411,
+ "fr": 1,
+ "pt": 1,
+ "undecided": 2561
+ },
+ "route_accuracy": 0.973436,
+ "route_correct": 2895,
+ "routed_to": {
+ "english": 79,
+ "multilingual": 2895
+ }
+ },
+ "before": {
+ "examples": 2974,
+ "expected_checkpoint": "multilingual",
+ "named_language_accuracy": 0.138198,
+ "named_language_correct": 411,
+ "named_language_guesses": {
+ "az": 411,
+ "fr": 1,
+ "pt": 1,
+ "undecided": 2561
+ },
+ "route_accuracy": 0.973436,
+ "route_correct": 2895,
+ "routed_to": {
+ "english": 79,
+ "multilingual": 2895
+ }
+ },
+ "named_language_accuracy_delta": 0.0,
+ "route_accuracy_delta": 0.0
+ },
+ "bn": {
+ "after": {
+ "examples": 2974,
+ "expected_checkpoint": "multilingual",
+ "named_language_accuracy": 0.0,
+ "named_language_correct": 0,
+ "named_language_guesses": {
+ "en": 1,
+ "undecided": 2973
+ },
+ "route_accuracy": 0.998319,
+ "route_correct": 2969,
+ "routed_to": {
+ "english": 5,
+ "multilingual": 2969
+ }
+ },
+ "before": {
+ "examples": 2974,
+ "expected_checkpoint": "multilingual",
+ "named_language_accuracy": 0.0,
+ "named_language_correct": 0,
+ "named_language_guesses": {
+ "en": 1,
+ "undecided": 2973
+ },
+ "route_accuracy": 0.998319,
+ "route_correct": 2969,
+ "routed_to": {
+ "english": 5,
+ "multilingual": 2969
+ }
+ },
+ "named_language_accuracy_delta": 0.0,
+ "route_accuracy_delta": 0.0
+ },
+ "cy": {
+ "after": {
+ "examples": 2974,
+ "expected_checkpoint": "multilingual",
+ "named_language_accuracy": 0.0,
+ "named_language_correct": 0,
+ "named_language_guesses": {
+ "bn": 36,
+ "en": 768,
+ "es": 98,
+ "it": 11,
+ "pt": 16,
+ "undecided": 2045
+ },
+ "route_accuracy": 0.132818,
+ "route_correct": 395,
+ "routed_to": {
+ "english": 2579,
+ "multilingual": 395
+ }
+ },
+ "before": {
+ "examples": 2974,
+ "expected_checkpoint": "multilingual",
+ "named_language_accuracy": 0.0,
+ "named_language_correct": 0,
+ "named_language_guesses": {
+ "bn": 36,
+ "en": 768,
+ "es": 98,
+ "it": 11,
+ "pt": 16,
+ "undecided": 2045
+ },
+ "route_accuracy": 0.132818,
+ "route_correct": 395,
+ "routed_to": {
+ "english": 2579,
+ "multilingual": 395
+ }
+ },
+ "named_language_accuracy_delta": 0.0,
+ "route_accuracy_delta": 0.0
+ },
+ "da": {
+ "after": {
+ "examples": 2974,
+ "expected_checkpoint": "multilingual",
+ "named_language_accuracy": 0.0,
+ "named_language_correct": 0,
+ "named_language_guesses": {
+ "de": 24,
+ "en": 547,
+ "fr": 5,
+ "it": 10,
+ "ro": 1,
+ "undecided": 2387
+ },
+ "route_accuracy": 0.534297,
+ "route_correct": 1589,
+ "routed_to": {
+ "english": 1385,
+ "multilingual": 1589
+ }
+ },
+ "before": {
+ "examples": 2974,
+ "expected_checkpoint": "multilingual",
+ "named_language_accuracy": 0.0,
+ "named_language_correct": 0,
+ "named_language_guesses": {
+ "de": 24,
+ "en": 547,
+ "fr": 5,
+ "it": 10,
+ "ro": 1,
+ "undecided": 2387
+ },
+ "route_accuracy": 0.534297,
+ "route_correct": 1589,
+ "routed_to": {
+ "english": 1385,
+ "multilingual": 1589
+ }
+ },
+ "named_language_accuracy_delta": 0.0,
+ "route_accuracy_delta": 0.0
+ },
+ "de": {
+ "after": {
+ "examples": 2974,
+ "expected_checkpoint": "multilingual",
+ "named_language_accuracy": 0.655346,
+ "named_language_correct": 1949,
+ "named_language_guesses": {
+ "de": 1949,
+ "en": 82,
+ "fr": 3,
+ "pt": 1,
+ "undecided": 939
+ },
+ "route_accuracy": 0.752186,
+ "route_correct": 2237,
+ "routed_to": {
+ "english": 737,
+ "multilingual": 2237
+ }
+ },
+ "before": {
+ "examples": 2974,
+ "expected_checkpoint": "multilingual",
+ "named_language_accuracy": 0.65501,
+ "named_language_correct": 1948,
+ "named_language_guesses": {
+ "de": 1948,
+ "en": 82,
+ "fr": 3,
+ "pt": 1,
+ "undecided": 940
+ },
+ "route_accuracy": 0.752186,
+ "route_correct": 2237,
+ "routed_to": {
+ "english": 737,
+ "multilingual": 2237
+ }
+ },
+ "named_language_accuracy_delta": 0.000336,
+ "route_accuracy_delta": 0.0
+ },
+ "el": {
+ "after": {
+ "examples": 2974,
+ "expected_checkpoint": "multilingual",
+ "named_language_accuracy": 0.0,
+ "named_language_correct": 0,
+ "named_language_guesses": {
+ "undecided": 2974
+ },
+ "route_accuracy": 0.99731,
+ "route_correct": 2966,
+ "routed_to": {
+ "english": 8,
+ "multilingual": 2966
+ }
+ },
+ "before": {
+ "examples": 2974,
+ "expected_checkpoint": "multilingual",
+ "named_language_accuracy": 0.0,
+ "named_language_correct": 0,
+ "named_language_guesses": {
+ "undecided": 2974
+ },
+ "route_accuracy": 0.99731,
+ "route_correct": 2966,
+ "routed_to": {
+ "english": 8,
+ "multilingual": 2966
+ }
+ },
+ "named_language_accuracy_delta": 0.0,
+ "route_accuracy_delta": 0.0
+ },
+ "en": {
+ "after": {
+ "examples": 2974,
+ "expected_checkpoint": "english",
+ "named_language_accuracy": 0.80195,
+ "named_language_correct": 2385,
+ "named_language_guesses": {
+ "en": 2385,
+ "undecided": 589
+ },
+ "route_accuracy": 1.0,
+ "route_correct": 2974,
+ "routed_to": {
+ "english": 2974
+ }
+ },
+ "before": {
+ "examples": 2974,
+ "expected_checkpoint": "english",
+ "named_language_accuracy": 0.80195,
+ "named_language_correct": 2385,
+ "named_language_guesses": {
+ "en": 2385,
+ "undecided": 589
+ },
+ "route_accuracy": 1.0,
+ "route_correct": 2974,
+ "routed_to": {
+ "english": 2974
+ }
+ },
+ "named_language_accuracy_delta": 0.0,
+ "route_accuracy_delta": 0.0
+ },
+ "es": {
+ "after": {
+ "examples": 2974,
+ "expected_checkpoint": "multilingual",
+ "named_language_accuracy": 0.467384,
+ "named_language_correct": 1390,
+ "named_language_guesses": {
+ "en": 6,
+ "es": 1390,
+ "fr": 3,
+ "it": 16,
+ "pt": 18,
+ "undecided": 1541
+ },
+ "route_accuracy": 0.711163,
+ "route_correct": 2115,
+ "routed_to": {
+ "english": 859,
+ "multilingual": 2115
+ }
+ },
+ "before": {
+ "examples": 2974,
+ "expected_checkpoint": "multilingual",
+ "named_language_accuracy": 0.467384,
+ "named_language_correct": 1390,
+ "named_language_guesses": {
+ "en": 6,
+ "es": 1390,
+ "fr": 3,
+ "it": 16,
+ "pt": 18,
+ "undecided": 1541
+ },
+ "route_accuracy": 0.711163,
+ "route_correct": 2115,
+ "routed_to": {
+ "english": 859,
+ "multilingual": 2115
+ }
+ },
+ "named_language_accuracy_delta": 0.0,
+ "route_accuracy_delta": 0.0
+ },
+ "fa": {
+ "after": {
+ "examples": 2974,
+ "expected_checkpoint": "multilingual",
+ "named_language_accuracy": 0.0,
+ "named_language_correct": 0,
+ "named_language_guesses": {
+ "undecided": 2974
+ },
+ "route_accuracy": 0.999664,
+ "route_correct": 2973,
+ "routed_to": {
+ "english": 1,
+ "multilingual": 2973
+ }
+ },
+ "before": {
+ "examples": 2974,
+ "expected_checkpoint": "multilingual",
+ "named_language_accuracy": 0.0,
+ "named_language_correct": 0,
+ "named_language_guesses": {
+ "undecided": 2974
+ },
+ "route_accuracy": 0.999664,
+ "route_correct": 2973,
+ "routed_to": {
+ "english": 1,
+ "multilingual": 2973
+ }
+ },
+ "named_language_accuracy_delta": 0.0,
+ "route_accuracy_delta": 0.0
+ },
+ "fi": {
+ "after": {
+ "examples": 2974,
+ "expected_checkpoint": "multilingual",
+ "named_language_accuracy": 0.0,
+ "named_language_correct": 0,
+ "named_language_guesses": {
+ "en": 97,
+ "undecided": 2877
+ },
+ "route_accuracy": 0.667787,
+ "route_correct": 1986,
+ "routed_to": {
+ "english": 988,
+ "multilingual": 1986
+ }
+ },
+ "before": {
+ "examples": 2974,
+ "expected_checkpoint": "multilingual",
+ "named_language_accuracy": 0.0,
+ "named_language_correct": 0,
+ "named_language_guesses": {
+ "en": 97,
+ "undecided": 2877
+ },
+ "route_accuracy": 0.667787,
+ "route_correct": 1986,
+ "routed_to": {
+ "english": 988,
+ "multilingual": 1986
+ }
+ },
+ "named_language_accuracy_delta": 0.0,
+ "route_accuracy_delta": 0.0
+ },
+ "fr": {
+ "after": {
+ "examples": 2974,
+ "expected_checkpoint": "multilingual",
+ "named_language_accuracy": 0.610962,
+ "named_language_correct": 1817,
+ "named_language_guesses": {
+ "en": 7,
+ "es": 14,
+ "fr": 1817,
+ "it": 7,
+ "pt": 6,
+ "undecided": 1123
+ },
+ "route_accuracy": 0.822125,
+ "route_correct": 2445,
+ "routed_to": {
+ "english": 529,
+ "multilingual": 2445
+ }
+ },
+ "before": {
+ "examples": 2974,
+ "expected_checkpoint": "multilingual",
+ "named_language_accuracy": 0.610962,
+ "named_language_correct": 1817,
+ "named_language_guesses": {
+ "en": 7,
+ "es": 14,
+ "fr": 1817,
+ "it": 7,
+ "pt": 6,
+ "undecided": 1123
+ },
+ "route_accuracy": 0.822125,
+ "route_correct": 2445,
+ "routed_to": {
+ "english": 529,
+ "multilingual": 2445
+ }
+ },
+ "named_language_accuracy_delta": 0.0,
+ "route_accuracy_delta": 0.0
+ },
+ "he": {
+ "after": {
+ "examples": 2974,
+ "expected_checkpoint": "multilingual",
+ "named_language_accuracy": 0.0,
+ "named_language_correct": 0,
+ "named_language_guesses": {
+ "undecided": 2974
+ },
+ "route_accuracy": 0.999664,
+ "route_correct": 2973,
+ "routed_to": {
+ "english": 1,
+ "multilingual": 2973
+ }
+ },
+ "before": {
+ "examples": 2974,
+ "expected_checkpoint": "multilingual",
+ "named_language_accuracy": 0.0,
+ "named_language_correct": 0,
+ "named_language_guesses": {
+ "undecided": 2974
+ },
+ "route_accuracy": 0.999664,
+ "route_correct": 2973,
+ "routed_to": {
+ "english": 1,
+ "multilingual": 2973
+ }
+ },
+ "named_language_accuracy_delta": 0.0,
+ "route_accuracy_delta": 0.0
+ },
+ "hi": {
+ "after": {
+ "examples": 2974,
+ "expected_checkpoint": "multilingual",
+ "named_language_accuracy": 0.0,
+ "named_language_correct": 0,
+ "named_language_guesses": {
+ "en": 5,
+ "it": 1,
+ "undecided": 2968
+ },
+ "route_accuracy": 0.993948,
+ "route_correct": 2956,
+ "routed_to": {
+ "english": 18,
+ "multilingual": 2956
+ }
+ },
+ "before": {
+ "examples": 2974,
+ "expected_checkpoint": "multilingual",
+ "named_language_accuracy": 0.0,
+ "named_language_correct": 0,
+ "named_language_guesses": {
+ "en": 5,
+ "it": 1,
+ "undecided": 2968
+ },
+ "route_accuracy": 0.993948,
+ "route_correct": 2956,
+ "routed_to": {
+ "english": 18,
+ "multilingual": 2956
+ }
+ },
+ "named_language_accuracy_delta": 0.0,
+ "route_accuracy_delta": 0.0
+ },
+ "hu": {
+ "after": {
+ "examples": 2974,
+ "expected_checkpoint": "multilingual",
+ "named_language_accuracy": 0.0,
+ "named_language_correct": 0,
+ "named_language_guesses": {
+ "en": 7,
+ "es": 6,
+ "fr": 1,
+ "it": 15,
+ "nl": 3,
+ "undecided": 2942
+ },
+ "route_accuracy": 0.935777,
+ "route_correct": 2783,
+ "routed_to": {
+ "english": 191,
+ "multilingual": 2783
+ }
+ },
+ "before": {
+ "examples": 2974,
+ "expected_checkpoint": "multilingual",
+ "named_language_accuracy": 0.0,
+ "named_language_correct": 0,
+ "named_language_guesses": {
+ "en": 7,
+ "es": 6,
+ "fr": 1,
+ "it": 15,
+ "nl": 3,
+ "undecided": 2942
+ },
+ "route_accuracy": 0.935777,
+ "route_correct": 2783,
+ "routed_to": {
+ "english": 191,
+ "multilingual": 2783
+ }
+ },
+ "named_language_accuracy_delta": 0.0,
+ "route_accuracy_delta": 0.0
+ },
+ "hy": {
+ "after": {
+ "examples": 2974,
+ "expected_checkpoint": "multilingual",
+ "named_language_accuracy": 0.0,
+ "named_language_correct": 0,
+ "named_language_guesses": {
+ "undecided": 2974
+ },
+ "route_accuracy": 0.999328,
+ "route_correct": 2972,
+ "routed_to": {
+ "english": 2,
+ "multilingual": 2972
+ }
+ },
+ "before": {
+ "examples": 2974,
+ "expected_checkpoint": "multilingual",
+ "named_language_accuracy": 0.0,
+ "named_language_correct": 0,
+ "named_language_guesses": {
+ "undecided": 2974
+ },
+ "route_accuracy": 0.999328,
+ "route_correct": 2972,
+ "routed_to": {
+ "english": 2,
+ "multilingual": 2972
+ }
+ },
+ "named_language_accuracy_delta": 0.0,
+ "route_accuracy_delta": 0.0
+ },
+ "id": {
+ "after": {
+ "examples": 2974,
+ "expected_checkpoint": "multilingual",
+ "named_language_accuracy": 0.0,
+ "named_language_correct": 0,
+ "named_language_guesses": {
+ "en": 43,
+ "es": 1,
+ "it": 21,
+ "undecided": 2909
+ },
+ "route_accuracy": 0.007397,
+ "route_correct": 22,
+ "routed_to": {
+ "english": 2952,
+ "multilingual": 22
+ }
+ },
+ "before": {
+ "examples": 2974,
+ "expected_checkpoint": "multilingual",
+ "named_language_accuracy": 0.0,
+ "named_language_correct": 0,
+ "named_language_guesses": {
+ "en": 43,
+ "es": 1,
+ "it": 21,
+ "undecided": 2909
+ },
+ "route_accuracy": 0.007397,
+ "route_correct": 22,
+ "routed_to": {
+ "english": 2952,
+ "multilingual": 22
+ }
+ },
+ "named_language_accuracy_delta": 0.0,
+ "route_accuracy_delta": 0.0
+ },
+ "is": {
+ "after": {
+ "examples": 2974,
+ "expected_checkpoint": "multilingual",
+ "named_language_accuracy": 0.0,
+ "named_language_correct": 0,
+ "named_language_guesses": {
+ "az": 2,
+ "en": 3,
+ "pt": 7,
+ "sv": 5,
+ "undecided": 2957
+ },
+ "route_accuracy": 0.909213,
+ "route_correct": 2704,
+ "routed_to": {
+ "english": 270,
+ "multilingual": 2704
+ }
+ },
+ "before": {
+ "examples": 2974,
+ "expected_checkpoint": "multilingual",
+ "named_language_accuracy": 0.0,
+ "named_language_correct": 0,
+ "named_language_guesses": {
+ "az": 2,
+ "en": 3,
+ "pt": 7,
+ "sv": 5,
+ "undecided": 2957
+ },
+ "route_accuracy": 0.909213,
+ "route_correct": 2704,
+ "routed_to": {
+ "english": 270,
+ "multilingual": 2704
+ }
+ },
+ "named_language_accuracy_delta": 0.0,
+ "route_accuracy_delta": 0.0
+ },
+ "it": {
+ "after": {
+ "examples": 2974,
+ "expected_checkpoint": "multilingual",
+ "named_language_accuracy": 0.590787,
+ "named_language_correct": 1757,
+ "named_language_guesses": {
+ "en": 159,
+ "es": 19,
+ "fr": 4,
+ "it": 1757,
+ "undecided": 1035
+ },
+ "route_accuracy": 0.635508,
+ "route_correct": 1890,
+ "routed_to": {
+ "english": 1084,
+ "multilingual": 1890
+ }
+ },
+ "before": {
+ "examples": 2974,
+ "expected_checkpoint": "multilingual",
+ "named_language_accuracy": 0.589778,
+ "named_language_correct": 1754,
+ "named_language_guesses": {
+ "en": 159,
+ "es": 20,
+ "fr": 5,
+ "it": 1754,
+ "undecided": 1036
+ },
+ "route_accuracy": 0.635171,
+ "route_correct": 1889,
+ "routed_to": {
+ "english": 1085,
+ "multilingual": 1889
+ }
+ },
+ "named_language_accuracy_delta": 0.001009,
+ "route_accuracy_delta": 0.000337
+ },
+ "ja": {
+ "after": {
+ "examples": 2974,
+ "expected_checkpoint": "multilingual",
+ "named_language_accuracy": 0.0,
+ "named_language_correct": 0,
+ "named_language_guesses": {
+ "undecided": 2974
+ },
+ "route_accuracy": 0.997983,
+ "route_correct": 2968,
+ "routed_to": {
+ "english": 6,
+ "multilingual": 2968
+ }
+ },
+ "before": {
+ "examples": 2974,
+ "expected_checkpoint": "multilingual",
+ "named_language_accuracy": 0.0,
+ "named_language_correct": 0,
+ "named_language_guesses": {
+ "undecided": 2974
+ },
+ "route_accuracy": 0.997983,
+ "route_correct": 2968,
+ "routed_to": {
+ "english": 6,
+ "multilingual": 2968
+ }
+ },
+ "named_language_accuracy_delta": 0.0,
+ "route_accuracy_delta": 0.0
+ },
+ "jv": {
+ "after": {
+ "examples": 2974,
+ "expected_checkpoint": "multilingual",
+ "named_language_accuracy": 0.0,
+ "named_language_correct": 0,
+ "named_language_guesses": {
+ "en": 32,
+ "fr": 1,
+ "it": 18,
+ "undecided": 2923
+ },
+ "route_accuracy": 0.009751,
+ "route_correct": 29,
+ "routed_to": {
+ "english": 2945,
+ "multilingual": 29
+ }
+ },
+ "before": {
+ "examples": 2974,
+ "expected_checkpoint": "multilingual",
+ "named_language_accuracy": 0.0,
+ "named_language_correct": 0,
+ "named_language_guesses": {
+ "en": 32,
+ "fr": 1,
+ "it": 18,
+ "undecided": 2923
+ },
+ "route_accuracy": 0.009751,
+ "route_correct": 29,
+ "routed_to": {
+ "english": 2945,
+ "multilingual": 29
+ }
+ },
+ "named_language_accuracy_delta": 0.0,
+ "route_accuracy_delta": 0.0
+ },
+ "ka": {
+ "after": {
+ "examples": 2974,
+ "expected_checkpoint": "multilingual",
+ "named_language_accuracy": 0.0,
+ "named_language_correct": 0,
+ "named_language_guesses": {
+ "undecided": 2974
+ },
+ "route_accuracy": 0.999664,
+ "route_correct": 2973,
+ "routed_to": {
+ "english": 1,
+ "multilingual": 2973
+ }
+ },
+ "before": {
+ "examples": 2974,
+ "expected_checkpoint": "multilingual",
+ "named_language_accuracy": 0.0,
+ "named_language_correct": 0,
+ "named_language_guesses": {
+ "undecided": 2974
+ },
+ "route_accuracy": 0.999664,
+ "route_correct": 2973,
+ "routed_to": {
+ "english": 1,
+ "multilingual": 2973
+ }
+ },
+ "named_language_accuracy_delta": 0.0,
+ "route_accuracy_delta": 0.0
+ },
+ "km": {
+ "after": {
+ "examples": 2974,
+ "expected_checkpoint": "multilingual",
+ "named_language_accuracy": 0.0,
+ "named_language_correct": 0,
+ "named_language_guesses": {
+ "en": 1,
+ "undecided": 2973
+ },
+ "route_accuracy": 0.992939,
+ "route_correct": 2953,
+ "routed_to": {
+ "english": 21,
+ "multilingual": 2953
+ }
+ },
+ "before": {
+ "examples": 2974,
+ "expected_checkpoint": "multilingual",
+ "named_language_accuracy": 0.0,
+ "named_language_correct": 0,
+ "named_language_guesses": {
+ "en": 1,
+ "undecided": 2973
+ },
+ "route_accuracy": 0.992939,
+ "route_correct": 2953,
+ "routed_to": {
+ "english": 21,
+ "multilingual": 2953
+ }
+ },
+ "named_language_accuracy_delta": 0.0,
+ "route_accuracy_delta": 0.0
+ },
+ "kn": {
+ "after": {
+ "examples": 2974,
+ "expected_checkpoint": "multilingual",
+ "named_language_accuracy": 0.0,
+ "named_language_correct": 0,
+ "named_language_guesses": {
+ "undecided": 2974
+ },
+ "route_accuracy": 0.999328,
+ "route_correct": 2972,
+ "routed_to": {
+ "english": 2,
+ "multilingual": 2972
+ }
+ },
+ "before": {
+ "examples": 2974,
+ "expected_checkpoint": "multilingual",
+ "named_language_accuracy": 0.0,
+ "named_language_correct": 0,
+ "named_language_guesses": {
+ "undecided": 2974
+ },
+ "route_accuracy": 0.999328,
+ "route_correct": 2972,
+ "routed_to": {
+ "english": 2,
+ "multilingual": 2972
+ }
+ },
+ "named_language_accuracy_delta": 0.0,
+ "route_accuracy_delta": 0.0
+ },
+ "ko": {
+ "after": {
+ "examples": 2974,
+ "expected_checkpoint": "multilingual",
+ "named_language_accuracy": 0.0,
+ "named_language_correct": 0,
+ "named_language_guesses": {
+ "en": 1,
+ "undecided": 2973
+ },
+ "route_accuracy": 0.997646,
+ "route_correct": 2967,
+ "routed_to": {
+ "english": 7,
+ "multilingual": 2967
+ }
+ },
+ "before": {
+ "examples": 2974,
+ "expected_checkpoint": "multilingual",
+ "named_language_accuracy": 0.0,
+ "named_language_correct": 0,
+ "named_language_guesses": {
+ "en": 1,
+ "undecided": 2973
+ },
+ "route_accuracy": 0.997646,
+ "route_correct": 2967,
+ "routed_to": {
+ "english": 7,
+ "multilingual": 2967
+ }
+ },
+ "named_language_accuracy_delta": 0.0,
+ "route_accuracy_delta": 0.0
+ },
+ "lv": {
+ "after": {
+ "examples": 2974,
+ "expected_checkpoint": "multilingual",
+ "named_language_accuracy": 0.0,
+ "named_language_correct": 0,
+ "named_language_guesses": {
+ "bn": 2,
+ "en": 3,
+ "es": 23,
+ "pt": 1,
+ "undecided": 2945
+ },
+ "route_accuracy": 0.933759,
+ "route_correct": 2777,
+ "routed_to": {
+ "english": 197,
+ "multilingual": 2777
+ }
+ },
+ "before": {
+ "examples": 2974,
+ "expected_checkpoint": "multilingual",
+ "named_language_accuracy": 0.0,
+ "named_language_correct": 0,
+ "named_language_guesses": {
+ "bn": 2,
+ "en": 3,
+ "es": 23,
+ "pt": 1,
+ "undecided": 2945
+ },
+ "route_accuracy": 0.933759,
+ "route_correct": 2777,
+ "routed_to": {
+ "english": 197,
+ "multilingual": 2777
+ }
+ },
+ "named_language_accuracy_delta": 0.0,
+ "route_accuracy_delta": 0.0
+ },
+ "ml": {
+ "after": {
+ "examples": 2974,
+ "expected_checkpoint": "multilingual",
+ "named_language_accuracy": 0.0,
+ "named_language_correct": 0,
+ "named_language_guesses": {
+ "en": 1,
+ "undecided": 2973
+ },
+ "route_accuracy": 0.999664,
+ "route_correct": 2973,
+ "routed_to": {
+ "english": 1,
+ "multilingual": 2973
+ }
+ },
+ "before": {
+ "examples": 2974,
+ "expected_checkpoint": "multilingual",
+ "named_language_accuracy": 0.0,
+ "named_language_correct": 0,
+ "named_language_guesses": {
+ "en": 1,
+ "undecided": 2973
+ },
+ "route_accuracy": 0.999664,
+ "route_correct": 2973,
+ "routed_to": {
+ "english": 1,
+ "multilingual": 2973
+ }
+ },
+ "named_language_accuracy_delta": 0.0,
+ "route_accuracy_delta": 0.0
+ },
+ "mn": {
+ "after": {
+ "examples": 2974,
+ "expected_checkpoint": "multilingual",
+ "named_language_accuracy": 0.0,
+ "named_language_correct": 0,
+ "named_language_guesses": {
+ "undecided": 2974
+ },
+ "route_accuracy": 0.999328,
+ "route_correct": 2972,
+ "routed_to": {
+ "english": 2,
+ "multilingual": 2972
+ }
+ },
+ "before": {
+ "examples": 2974,
+ "expected_checkpoint": "multilingual",
+ "named_language_accuracy": 0.0,
+ "named_language_correct": 0,
+ "named_language_guesses": {
+ "undecided": 2974
+ },
+ "route_accuracy": 0.999328,
+ "route_correct": 2972,
+ "routed_to": {
+ "english": 2,
+ "multilingual": 2972
+ }
+ },
+ "named_language_accuracy_delta": 0.0,
+ "route_accuracy_delta": 0.0
+ },
+ "ms": {
+ "after": {
+ "examples": 2974,
+ "expected_checkpoint": "multilingual",
+ "named_language_accuracy": 0.0,
+ "named_language_correct": 0,
+ "named_language_guesses": {
+ "en": 30,
+ "it": 12,
+ "undecided": 2932
+ },
+ "route_accuracy": 0.004035,
+ "route_correct": 12,
+ "routed_to": {
+ "english": 2962,
+ "multilingual": 12
+ }
+ },
+ "before": {
+ "examples": 2974,
+ "expected_checkpoint": "multilingual",
+ "named_language_accuracy": 0.0,
+ "named_language_correct": 0,
+ "named_language_guesses": {
+ "en": 30,
+ "it": 12,
+ "undecided": 2932
+ },
+ "route_accuracy": 0.004035,
+ "route_correct": 12,
+ "routed_to": {
+ "english": 2962,
+ "multilingual": 12
+ }
+ },
+ "named_language_accuracy_delta": 0.0,
+ "route_accuracy_delta": 0.0
+ },
+ "my": {
+ "after": {
+ "examples": 2974,
+ "expected_checkpoint": "multilingual",
+ "named_language_accuracy": 0.0,
+ "named_language_correct": 0,
+ "named_language_guesses": {
+ "undecided": 2974
+ },
+ "route_accuracy": 0.998991,
+ "route_correct": 2971,
+ "routed_to": {
+ "english": 3,
+ "multilingual": 2971
+ }
+ },
+ "before": {
+ "examples": 2974,
+ "expected_checkpoint": "multilingual",
+ "named_language_accuracy": 0.0,
+ "named_language_correct": 0,
+ "named_language_guesses": {
+ "undecided": 2974
+ },
+ "route_accuracy": 0.998991,
+ "route_correct": 2971,
+ "routed_to": {
+ "english": 3,
+ "multilingual": 2971
+ }
+ },
+ "named_language_accuracy_delta": 0.0,
+ "route_accuracy_delta": 0.0
+ },
+ "nb": {
+ "after": {
+ "examples": 2974,
+ "expected_checkpoint": "multilingual",
+ "named_language_accuracy": 0.0,
+ "named_language_correct": 0,
+ "named_language_guesses": {
+ "de": 7,
+ "en": 469,
+ "fr": 10,
+ "it": 12,
+ "pt": 1,
+ "sv": 4,
+ "undecided": 2471
+ },
+ "route_accuracy": 0.496301,
+ "route_correct": 1476,
+ "routed_to": {
+ "english": 1498,
+ "multilingual": 1476
+ }
+ },
+ "before": {
+ "examples": 2974,
+ "expected_checkpoint": "multilingual",
+ "named_language_accuracy": 0.0,
+ "named_language_correct": 0,
+ "named_language_guesses": {
+ "de": 7,
+ "en": 469,
+ "fr": 10,
+ "it": 12,
+ "pt": 1,
+ "sv": 4,
+ "undecided": 2471
+ },
+ "route_accuracy": 0.496301,
+ "route_correct": 1476,
+ "routed_to": {
+ "english": 1498,
+ "multilingual": 1476
+ }
+ },
+ "named_language_accuracy_delta": 0.0,
+ "route_accuracy_delta": 0.0
+ },
+ "nl": {
+ "after": {
+ "examples": 2974,
+ "expected_checkpoint": "multilingual",
+ "named_language_accuracy": 0.257566,
+ "named_language_correct": 766,
+ "named_language_guesses": {
+ "de": 3,
+ "en": 674,
+ "fr": 4,
+ "it": 1,
+ "nl": 766,
+ "pt": 6,
+ "undecided": 1520
+ },
+ "route_accuracy": 0.283457,
+ "route_correct": 843,
+ "routed_to": {
+ "english": 2131,
+ "multilingual": 843
+ }
+ },
+ "before": {
+ "examples": 2974,
+ "expected_checkpoint": "multilingual",
+ "named_language_accuracy": 0.257566,
+ "named_language_correct": 766,
+ "named_language_guesses": {
+ "de": 3,
+ "en": 674,
+ "fr": 4,
+ "it": 1,
+ "nl": 766,
+ "pt": 6,
+ "undecided": 1520
+ },
+ "route_accuracy": 0.283457,
+ "route_correct": 843,
+ "routed_to": {
+ "english": 2131,
+ "multilingual": 843
+ }
+ },
+ "named_language_accuracy_delta": 0.0,
+ "route_accuracy_delta": 0.0
+ },
+ "pl": {
+ "after": {
+ "examples": 2974,
+ "expected_checkpoint": "multilingual",
+ "named_language_accuracy": 0.0,
+ "named_language_correct": 0,
+ "named_language_guesses": {
+ "de": 1,
+ "en": 53,
+ "es": 1,
+ "pt": 142,
+ "undecided": 2777
+ },
+ "route_accuracy": 0.813046,
+ "route_correct": 2418,
+ "routed_to": {
+ "english": 556,
+ "multilingual": 2418
+ }
+ },
+ "before": {
+ "examples": 2974,
+ "expected_checkpoint": "multilingual",
+ "named_language_accuracy": 0.0,
+ "named_language_correct": 0,
+ "named_language_guesses": {
+ "de": 1,
+ "en": 53,
+ "es": 1,
+ "pt": 135,
+ "undecided": 2784
+ },
+ "route_accuracy": 0.813046,
+ "route_correct": 2418,
+ "routed_to": {
+ "english": 556,
+ "multilingual": 2418
+ }
+ },
+ "named_language_accuracy_delta": 0.0,
+ "route_accuracy_delta": 0.0
+ },
+ "pt": {
+ "after": {
+ "examples": 2974,
+ "expected_checkpoint": "multilingual",
+ "named_language_accuracy": 0.53228,
+ "named_language_correct": 1583,
+ "named_language_guesses": {
+ "en": 62,
+ "es": 14,
+ "fr": 1,
+ "it": 8,
+ "pt": 1583,
+ "undecided": 1306
+ },
+ "route_accuracy": 0.765636,
+ "route_correct": 2277,
+ "routed_to": {
+ "english": 697,
+ "multilingual": 2277
+ }
+ },
+ "before": {
+ "examples": 2974,
+ "expected_checkpoint": "multilingual",
+ "named_language_accuracy": 0.531944,
+ "named_language_correct": 1582,
+ "named_language_guesses": {
+ "en": 62,
+ "es": 14,
+ "fr": 1,
+ "it": 8,
+ "pt": 1582,
+ "undecided": 1307
+ },
+ "route_accuracy": 0.765299,
+ "route_correct": 2276,
+ "routed_to": {
+ "english": 698,
+ "multilingual": 2276
+ }
+ },
+ "named_language_accuracy_delta": 0.000336,
+ "route_accuracy_delta": 0.000337
+ },
+ "ro": {
+ "after": {
+ "examples": 2974,
+ "expected_checkpoint": "multilingual",
+ "named_language_accuracy": 0.151311,
+ "named_language_correct": 450,
+ "named_language_guesses": {
+ "bn": 2,
+ "en": 221,
+ "es": 10,
+ "fr": 120,
+ "it": 98,
+ "pt": 24,
+ "ro": 450,
+ "undecided": 2049
+ },
+ "route_accuracy": 0.533289,
+ "route_correct": 1586,
+ "routed_to": {
+ "english": 1388,
+ "multilingual": 1586
+ }
+ },
+ "before": {
+ "examples": 2974,
+ "expected_checkpoint": "multilingual",
+ "named_language_accuracy": 0.150975,
+ "named_language_correct": 449,
+ "named_language_guesses": {
+ "bn": 2,
+ "en": 221,
+ "es": 10,
+ "fr": 120,
+ "it": 99,
+ "pt": 24,
+ "ro": 449,
+ "undecided": 2049
+ },
+ "route_accuracy": 0.533289,
+ "route_correct": 1586,
+ "routed_to": {
+ "english": 1388,
+ "multilingual": 1586
+ }
+ },
+ "named_language_accuracy_delta": 0.000336,
+ "route_accuracy_delta": 0.0
+ },
+ "ru": {
+ "after": {
+ "examples": 2974,
+ "expected_checkpoint": "multilingual",
+ "named_language_accuracy": 0.0,
+ "named_language_correct": 0,
+ "named_language_guesses": {
+ "undecided": 2974
+ },
+ "route_accuracy": 0.999328,
+ "route_correct": 2972,
+ "routed_to": {
+ "english": 2,
+ "multilingual": 2972
+ }
+ },
+ "before": {
+ "examples": 2974,
+ "expected_checkpoint": "multilingual",
+ "named_language_accuracy": 0.0,
+ "named_language_correct": 0,
+ "named_language_guesses": {
+ "undecided": 2974
+ },
+ "route_accuracy": 0.999328,
+ "route_correct": 2972,
+ "routed_to": {
+ "english": 2,
+ "multilingual": 2972
+ }
+ },
+ "named_language_accuracy_delta": 0.0,
+ "route_accuracy_delta": 0.0
+ },
+ "sl": {
+ "after": {
+ "examples": 2974,
+ "expected_checkpoint": "multilingual",
+ "named_language_accuracy": 0.0,
+ "named_language_correct": 0,
+ "named_language_guesses": {
+ "en": 85,
+ "es": 2,
+ "fr": 34,
+ "it": 14,
+ "pt": 108,
+ "undecided": 2731
+ },
+ "route_accuracy": 0.573638,
+ "route_correct": 1706,
+ "routed_to": {
+ "english": 1268,
+ "multilingual": 1706
+ }
+ },
+ "before": {
+ "examples": 2974,
+ "expected_checkpoint": "multilingual",
+ "named_language_accuracy": 0.0,
+ "named_language_correct": 0,
+ "named_language_guesses": {
+ "en": 85,
+ "es": 2,
+ "fr": 34,
+ "it": 14,
+ "pt": 108,
+ "undecided": 2731
+ },
+ "route_accuracy": 0.573638,
+ "route_correct": 1706,
+ "routed_to": {
+ "english": 1268,
+ "multilingual": 1706
+ }
+ },
+ "named_language_accuracy_delta": 0.0,
+ "route_accuracy_delta": 0.0
+ },
+ "sq": {
+ "after": {
+ "examples": 2974,
+ "expected_checkpoint": "multilingual",
+ "named_language_accuracy": 0.0,
+ "named_language_correct": 0,
+ "named_language_guesses": {
+ "en": 15,
+ "fr": 10,
+ "it": 94,
+ "nl": 1,
+ "pt": 42,
+ "undecided": 2812
+ },
+ "route_accuracy": 0.914929,
+ "route_correct": 2721,
+ "routed_to": {
+ "english": 253,
+ "multilingual": 2721
+ }
+ },
+ "before": {
+ "examples": 2974,
+ "expected_checkpoint": "multilingual",
+ "named_language_accuracy": 0.0,
+ "named_language_correct": 0,
+ "named_language_guesses": {
+ "en": 15,
+ "fr": 10,
+ "it": 94,
+ "nl": 1,
+ "pt": 39,
+ "undecided": 2815
+ },
+ "route_accuracy": 0.914929,
+ "route_correct": 2721,
+ "routed_to": {
+ "english": 253,
+ "multilingual": 2721
+ }
+ },
+ "named_language_accuracy_delta": 0.0,
+ "route_accuracy_delta": 0.0
+ },
+ "sv": {
+ "after": {
+ "examples": 2974,
+ "expected_checkpoint": "multilingual",
+ "named_language_accuracy": 0.302959,
+ "named_language_correct": 901,
+ "named_language_guesses": {
+ "de": 7,
+ "en": 43,
+ "fr": 4,
+ "ro": 1,
+ "sv": 901,
+ "undecided": 2018
+ },
+ "route_accuracy": 0.869536,
+ "route_correct": 2586,
+ "routed_to": {
+ "english": 388,
+ "multilingual": 2586
+ }
+ },
+ "before": {
+ "examples": 2974,
+ "expected_checkpoint": "multilingual",
+ "named_language_accuracy": 0.302959,
+ "named_language_correct": 901,
+ "named_language_guesses": {
+ "de": 7,
+ "en": 43,
+ "fr": 4,
+ "ro": 1,
+ "sv": 901,
+ "undecided": 2018
+ },
+ "route_accuracy": 0.869536,
+ "route_correct": 2586,
+ "routed_to": {
+ "english": 388,
+ "multilingual": 2586
+ }
+ },
+ "named_language_accuracy_delta": 0.0,
+ "route_accuracy_delta": 0.0
+ },
+ "sw": {
+ "after": {
+ "examples": 2974,
+ "expected_checkpoint": "multilingual",
+ "named_language_accuracy": 0.0,
+ "named_language_correct": 0,
+ "named_language_guesses": {
+ "en": 18,
+ "es": 360,
+ "fr": 6,
+ "pt": 40,
+ "undecided": 2550
+ },
+ "route_accuracy": 0.136516,
+ "route_correct": 406,
+ "routed_to": {
+ "english": 2568,
+ "multilingual": 406
+ }
+ },
+ "before": {
+ "examples": 2974,
+ "expected_checkpoint": "multilingual",
+ "named_language_accuracy": 0.0,
+ "named_language_correct": 0,
+ "named_language_guesses": {
+ "en": 18,
+ "es": 360,
+ "fr": 6,
+ "pt": 40,
+ "undecided": 2550
+ },
+ "route_accuracy": 0.136516,
+ "route_correct": 406,
+ "routed_to": {
+ "english": 2568,
+ "multilingual": 406
+ }
+ },
+ "named_language_accuracy_delta": 0.0,
+ "route_accuracy_delta": 0.0
+ },
+ "ta": {
+ "after": {
+ "examples": 2974,
+ "expected_checkpoint": "multilingual",
+ "named_language_accuracy": 0.0,
+ "named_language_correct": 0,
+ "named_language_guesses": {
+ "undecided": 2974
+ },
+ "route_accuracy": 1.0,
+ "route_correct": 2974,
+ "routed_to": {
+ "multilingual": 2974
+ }
+ },
+ "before": {
+ "examples": 2974,
+ "expected_checkpoint": "multilingual",
+ "named_language_accuracy": 0.0,
+ "named_language_correct": 0,
+ "named_language_guesses": {
+ "undecided": 2974
+ },
+ "route_accuracy": 1.0,
+ "route_correct": 2974,
+ "routed_to": {
+ "multilingual": 2974
+ }
+ },
+ "named_language_accuracy_delta": 0.0,
+ "route_accuracy_delta": 0.0
+ },
+ "te": {
+ "after": {
+ "examples": 2974,
+ "expected_checkpoint": "multilingual",
+ "named_language_accuracy": 0.0,
+ "named_language_correct": 0,
+ "named_language_guesses": {
+ "en": 3,
+ "undecided": 2971
+ },
+ "route_accuracy": 0.998655,
+ "route_correct": 2970,
+ "routed_to": {
+ "english": 4,
+ "multilingual": 2970
+ }
+ },
+ "before": {
+ "examples": 2974,
+ "expected_checkpoint": "multilingual",
+ "named_language_accuracy": 0.0,
+ "named_language_correct": 0,
+ "named_language_guesses": {
+ "en": 3,
+ "undecided": 2971
+ },
+ "route_accuracy": 0.998655,
+ "route_correct": 2970,
+ "routed_to": {
+ "english": 4,
+ "multilingual": 2970
+ }
+ },
+ "named_language_accuracy_delta": 0.0,
+ "route_accuracy_delta": 0.0
+ },
+ "th": {
+ "after": {
+ "examples": 2974,
+ "expected_checkpoint": "multilingual",
+ "named_language_accuracy": 0.0,
+ "named_language_correct": 0,
+ "named_language_guesses": {
+ "undecided": 2974
+ },
+ "route_accuracy": 0.998319,
+ "route_correct": 2969,
+ "routed_to": {
+ "english": 5,
+ "multilingual": 2969
+ }
+ },
+ "before": {
+ "examples": 2974,
+ "expected_checkpoint": "multilingual",
+ "named_language_accuracy": 0.0,
+ "named_language_correct": 0,
+ "named_language_guesses": {
+ "undecided": 2974
+ },
+ "route_accuracy": 0.998319,
+ "route_correct": 2969,
+ "routed_to": {
+ "english": 5,
+ "multilingual": 2969
+ }
+ },
+ "named_language_accuracy_delta": 0.0,
+ "route_accuracy_delta": 0.0
+ },
+ "tl": {
+ "after": {
+ "examples": 2974,
+ "expected_checkpoint": "multilingual",
+ "named_language_accuracy": 0.0,
+ "named_language_correct": 0,
+ "named_language_guesses": {
+ "en": 313,
+ "es": 7,
+ "fr": 257,
+ "pt": 77,
+ "undecided": 2320
+ },
+ "route_accuracy": 0.114997,
+ "route_correct": 342,
+ "routed_to": {
+ "english": 2632,
+ "multilingual": 342
+ }
+ },
+ "before": {
+ "examples": 2974,
+ "expected_checkpoint": "multilingual",
+ "named_language_accuracy": 0.0,
+ "named_language_correct": 0,
+ "named_language_guesses": {
+ "en": 313,
+ "es": 7,
+ "fr": 257,
+ "pt": 77,
+ "undecided": 2320
+ },
+ "route_accuracy": 0.114997,
+ "route_correct": 342,
+ "routed_to": {
+ "english": 2632,
+ "multilingual": 342
+ }
+ },
+ "named_language_accuracy_delta": 0.0,
+ "route_accuracy_delta": 0.0
+ },
+ "tr": {
+ "after": {
+ "examples": 2974,
+ "expected_checkpoint": "multilingual",
+ "named_language_accuracy": 0.0,
+ "named_language_correct": 0,
+ "named_language_guesses": {
+ "az": 191,
+ "en": 17,
+ "es": 4,
+ "it": 13,
+ "undecided": 2749
+ },
+ "route_accuracy": 0.905178,
+ "route_correct": 2692,
+ "routed_to": {
+ "english": 282,
+ "multilingual": 2692
+ }
+ },
+ "before": {
+ "examples": 2974,
+ "expected_checkpoint": "multilingual",
+ "named_language_accuracy": 0.0,
+ "named_language_correct": 0,
+ "named_language_guesses": {
+ "az": 191,
+ "en": 17,
+ "es": 4,
+ "it": 13,
+ "undecided": 2749
+ },
+ "route_accuracy": 0.905178,
+ "route_correct": 2692,
+ "routed_to": {
+ "english": 282,
+ "multilingual": 2692
+ }
+ },
+ "named_language_accuracy_delta": 0.0,
+ "route_accuracy_delta": 0.0
+ },
+ "ur": {
+ "after": {
+ "examples": 2974,
+ "expected_checkpoint": "multilingual",
+ "named_language_accuracy": 0.0,
+ "named_language_correct": 0,
+ "named_language_guesses": {
+ "en": 1,
+ "undecided": 2973
+ },
+ "route_accuracy": 0.998319,
+ "route_correct": 2969,
+ "routed_to": {
+ "english": 5,
+ "multilingual": 2969
+ }
+ },
+ "before": {
+ "examples": 2974,
+ "expected_checkpoint": "multilingual",
+ "named_language_accuracy": 0.0,
+ "named_language_correct": 0,
+ "named_language_guesses": {
+ "en": 1,
+ "undecided": 2973
+ },
+ "route_accuracy": 0.998319,
+ "route_correct": 2969,
+ "routed_to": {
+ "english": 5,
+ "multilingual": 2969
+ }
+ },
+ "named_language_accuracy_delta": 0.0,
+ "route_accuracy_delta": 0.0
+ },
+ "vi": {
+ "after": {
+ "examples": 2974,
+ "expected_checkpoint": "multilingual",
+ "named_language_accuracy": 0.0,
+ "named_language_correct": 0,
+ "named_language_guesses": {
+ "en": 7,
+ "es": 24,
+ "fr": 32,
+ "it": 44,
+ "pt": 39,
+ "undecided": 2828
+ },
+ "route_accuracy": 0.956288,
+ "route_correct": 2844,
+ "routed_to": {
+ "english": 130,
+ "multilingual": 2844
+ }
+ },
+ "before": {
+ "examples": 2974,
+ "expected_checkpoint": "multilingual",
+ "named_language_accuracy": 0.0,
+ "named_language_correct": 0,
+ "named_language_guesses": {
+ "en": 7,
+ "es": 24,
+ "fr": 32,
+ "it": 44,
+ "pt": 39,
+ "undecided": 2828
+ },
+ "route_accuracy": 0.956288,
+ "route_correct": 2844,
+ "routed_to": {
+ "english": 130,
+ "multilingual": 2844
+ }
+ },
+ "named_language_accuracy_delta": 0.0,
+ "route_accuracy_delta": 0.0
+ },
+ "zh-CN": {
+ "after": {
+ "examples": 2974,
+ "expected_checkpoint": "multilingual",
+ "named_language_accuracy": 0.0,
+ "named_language_correct": 0,
+ "named_language_guesses": {
+ "undecided": 2974
+ },
+ "route_accuracy": 0.999664,
+ "route_correct": 2973,
+ "routed_to": {
+ "english": 1,
+ "multilingual": 2973
+ }
+ },
+ "before": {
+ "examples": 2974,
+ "expected_checkpoint": "multilingual",
+ "named_language_accuracy": 0.0,
+ "named_language_correct": 0,
+ "named_language_guesses": {
+ "undecided": 2974
+ },
+ "route_accuracy": 0.999664,
+ "route_correct": 2973,
+ "routed_to": {
+ "english": 1,
+ "multilingual": 2973
+ }
+ },
+ "named_language_accuracy_delta": 0.0,
+ "route_accuracy_delta": 0.0
+ },
+ "zh-TW": {
+ "after": {
+ "examples": 2974,
+ "expected_checkpoint": "multilingual",
+ "named_language_accuracy": 0.0,
+ "named_language_correct": 0,
+ "named_language_guesses": {
+ "en": 3,
+ "undecided": 2971
+ },
+ "route_accuracy": 0.995965,
+ "route_correct": 2962,
+ "routed_to": {
+ "english": 12,
+ "multilingual": 2962
+ }
+ },
+ "before": {
+ "examples": 2974,
+ "expected_checkpoint": "multilingual",
+ "named_language_accuracy": 0.0,
+ "named_language_correct": 0,
+ "named_language_guesses": {
+ "en": 3,
+ "undecided": 2971
+ },
+ "route_accuracy": 0.995965,
+ "route_correct": 2962,
+ "routed_to": {
+ "english": 12,
+ "multilingual": 2962
+ }
+ },
+ "named_language_accuracy_delta": 0.0,
+ "route_accuracy_delta": 0.0
+ }
+ },
+ "model_free": true,
+ "overall": {
+ "after": {
+ "examples": 151674,
+ "named_language_accuracy": 0.088407,
+ "named_language_correct": 13409,
+ "route_accuracy": 0.774292,
+ "route_correct": 117440
+ },
+ "before": {
+ "examples": 151674,
+ "named_language_accuracy": 0.088367,
+ "named_language_correct": 13403,
+ "route_accuracy": 0.774279,
+ "route_correct": 117438
+ }
+ },
+ "revision": "940fd47a81eaa7f2cc7b129674d945d618ac38c2",
+ "routing_metric_note": "Router._route decisions are recorded; route accuracy uses expected checkpoint by MASSIVE locale, including Router's configured default for undecided text.",
+ "routing_policy": "english locale -> english checkpoint; every other locale -> multilingual",
+ "split": "test"
+}