Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

feat(rust): add language detection

kevin-gatimuea36f4351746
diff --git a/laya-rust/src/language_detection.rs b/laya-rust/src/language_detection.rs
new file mode 100644
index 00000000..3b104457
--- /dev/null
+++ b/laya-rust/src/language_detection.rs
@@ -0,0 +1,576 @@
+//! Dependency-free language/script detection used to route between Laya checkpoints. Ports
+//! `laya/lang.py`.
+//!
+//! Routing only needs one decision: *is this English Latin text, or is it something the English
+//! checkpoint cannot read?* Benchmarks on MASSIVE (14 languages) showed the English checkpoint
+//! collapsing to near-random on non-Latin scripts (Hindi 0.100, Korean 0.103, Swahili 0.103, Tamil
+//! 0.113 at 20 options, where random is 0.050), while holding up far better on Latin-script
+//! languages (French 0.487, Spanish 0.480). So the signal that matters most is *script*, and the
+//! secondary signal is whether Latin text is English.
+//!
+//! Script detection is exact. The Latin-script language guess is a stopword/diacritic heuristic
+//! and is explicitly best-effort: pass an explicit model or `lang=` when the language is already
+//! known.
+//!
+//! ## Unicode fidelity
+//!
+//! Python's `str.isalpha()` tests Unicode general category `L*` (`Lu`, `Ll`, `Lt`, `Lm`, `Lo`)
+//! only. Rust's `char::is_alphabetic` also accepts combining marks, which would change
+//! [`script_profile`]'s fractions on marked scripts (Hindi matras, for example), so every
+//! alphabetic test here goes through [`unicode_general_category::get_general_category`] instead.
+//! Python's `[^\W\d_]+` word pattern (`\w` = `isalnum()` + `_`, in Unicode mode) is likewise
+//! reproduced with a general-category check rather than a regex crate: `\w` minus digits (`\d` =
+//! category `Nd` only) minus `_` leaves letters plus `Nl`/`No` (a numeric-letter or other-number
+//! character, but not a plain decimal digit).
+
+use std::collections::HashMap;
+use unicode_general_category::{GeneralCategory, get_general_category};
+
+/// Namespace for language/script detection. See the module docs for the source this ports.
+pub struct LanguageDetection;
+
+/// The full detection result for one piece of state. See [`LanguageDetection::analyse`].
+#[derive(Debug, Clone, PartialEq)]
+pub struct LanguageAnalysis {
+ /// The dominant script: `"latin"`, `"han"`, `"devanagari"`, ... or `"unknown"` when the text
+ /// has no letters at all.
+ pub script: String,
+ /// Fraction of alphabetic characters belonging to each detected script.
+ pub script_profile: HashMap<String, f64>,
+ /// Best-effort Latin-script language guess (an ISO-639-1-ish code), or `None` when undecided
+ /// or when the script is not Latin.
+ pub language: Option<String>,
+ /// Whether the English checkpoint can be expected to read this text.
+ pub is_english: bool,
+ /// Whether [`Self::language`] is `None` because no language could be identified (as opposed
+ /// to being `None` because the script itself is non-Latin).
+ pub language_undecided: bool,
+ /// Fraction of (lowercased) characters that are non-English Latin diacritics.
+ pub diacritic_rate: f64,
+ /// `1.0 - script_profile["latin"]`, or `0.0` when there is no script profile at all.
+ pub non_latin_fraction: f64,
+}
+
+// Unicode blocks that the English (ModernBERT-large, 50k English BPE) checkpoint cannot read.
+// Order matters: `detect_script`/`script_profile` take the first matching range, and a tie
+// between two script counts in `detect_script` is broken by which name was inserted first, which
+// is this table's order (for non-Latin scripts) followed by "latin" last.
+const SCRIPT_RANGES: &[(&str, &[(u32, u32)])] = &[
+ ("greek", &[(0x0370, 0x03FF), (0x1F00, 0x1FFF)]),
+ (
+ "cyrillic",
+ &[(0x0400, 0x052F), (0x2DE0, 0x2DFF), (0xA640, 0xA69F)],
+ ),
+ ("armenian", &[(0x0530, 0x058F)]),
+ ("hebrew", &[(0x0590, 0x05FF)]),
+ (
+ "arabic",
+ &[
+ (0x0600, 0x06FF),
+ (0x0750, 0x077F),
+ (0x08A0, 0x08FF),
+ (0xFB50, 0xFDFF),
+ (0xFE70, 0xFEFF),
+ ],
+ ),
+ ("devanagari", &[(0x0900, 0x097F), (0xA8E0, 0xA8FF)]),
+ ("bengali", &[(0x0980, 0x09FF)]),
+ ("gurmukhi", &[(0x0A00, 0x0A7F)]),
+ ("gujarati", &[(0x0A80, 0x0AFF)]),
+ ("oriya", &[(0x0B00, 0x0B7F)]),
+ ("tamil", &[(0x0B80, 0x0BFF)]),
+ ("telugu", &[(0x0C00, 0x0C7F)]),
+ ("kannada", &[(0x0C80, 0x0CFF)]),
+ ("malayalam", &[(0x0D00, 0x0D7F)]),
+ ("sinhala", &[(0x0D80, 0x0DFF)]),
+ ("thai", &[(0x0E00, 0x0E7F)]),
+ ("lao", &[(0x0E80, 0x0EFF)]),
+ ("tibetan", &[(0x0F00, 0x0FFF)]),
+ ("myanmar", &[(0x1000, 0x109F)]),
+ ("georgian", &[(0x10A0, 0x10FF)]),
+ ("ethiopic", &[(0x1200, 0x137F)]),
+ ("khmer", &[(0x1780, 0x17FF)]),
+ (
+ "hangul",
+ &[(0x1100, 0x11FF), (0x3130, 0x318F), (0xAC00, 0xD7AF)],
+ ),
+ (
+ "kana",
+ &[(0x3040, 0x309F), (0x30A0, 0x30FF), (0x31F0, 0x31FF)],
+ ),
+ (
+ "han",
+ &[(0x3400, 0x4DBF), (0x4E00, 0x9FFF), (0xF900, 0xFAFF)],
+ ),
+];
+
+/// A diacritic rate above this is taken as evidence the text is not English, even when no
+/// stopword list matches it.
+const NON_EN_DIACRITIC_RATE: f64 = 0.02;
+
+/// Letters that ordinary English does not use. This catches a Latin-script language with no
+/// stopword list at all (Romanian, Polish, Czech, Turkish, Baltic, ...), which is the difference
+/// between routing it to the multilingual checkpoint and silently handing it to the English one.
+/// Text is lowercased before matching, so only lowercase forms need listing here.
+const NON_EN_DIACRITICS: &str = concat!(
+ "àâäãáåçéèêëíìîïñóòôöõøúùûüýÿßæœ", // Western European
+ "ăâîșțşţ", // Romanian
+ "ąćęłńśźż", // Polish
+ "čďěňřšťůž", // Czech / Slovak
+ "őű", // Hungarian
+ "ğı", // Turkish
+ "āēģīķļņūž", // Baltic
+ "đ", // Serbo-Croatian / Vietnamese
+);
+
+/// Function words. Latin-script languages overlap heavily (de/la/le/un/e/que), so each hit is
+/// weighted and a margin is required before calling something non-English.
+///
+/// The Romance lists (fr/es/pt/it) deliberately carry the *unaccented* function words as well as
+/// the accented ones. A state that lost its accents -- mail clients, ticket systems and any
+/// pipeline that normalises to ASCII strip them -- keeps no diacritic rate for the non-English
+/// signal to read, so `la`, `un`, `y`, `e`, `et`, `deux` and friends are the only evidence left.
+/// With a list of mostly accented words such a state produced one hit or none, fell under the
+/// two-hit margin below, and was handed to the English checkpoint as undecided-but-not-non-English
+/// text.
+///
+/// Order is load-bearing for `latin_profile`'s tie-break (`_STOP` iteration order: en, fr, de, es,
+/// pt, it, nl, ro), so this stays a `Vec` of `(code, words)` pairs rather than a `HashMap`.
+fn stop_lists() -> &'static [&'static str] {
+ // Order is load-bearing (see the doc comment above): en, fr, de, es, pt, it, nl, ro.
+ &["en", "fr", "de", "es", "pt", "it", "nl", "ro"]
+}
+
+// Slices (not fixed-size arrays) throughout, so splitting a long list across several `const`s
+// for readability never needs a manually counted length.
+const EN_STOP: &[&str] = &[
+ "the", "and", "is", "are", "was", "were", "to", "of", "in", "for", "with", "that", "this",
+ "it", "you", "have", "has", "not", "but", "on", "at", "be", "as", "from", "will", "can",
+ "would", "there", "their", "what", "which", "please", "we", "i",
+];
+
+const FR_STOP: &[&str] = &[
+ "le", "la", "les", "des", "une", "est", "pour", "dans", "que", "qui", "avec", "sur", "pas",
+ "plus", "nous", "vous", "être", "cette", "mais", "sont", "ont", "aux", "ce", "et", "du", "au",
+ "ou", "je", "tu", "il", "elle", "ils", "elles", "mon", "ton", "ma", "ta", "sa", "mes", "tes",
+ "ses", "ces", "deux", "trois", "très", "bien", "tout", "tous", "toute", "fait", "veux", "veut",
+ "peux", "peut", "dois", "doit", "merci", "bonjour", "jour", "jours", "mois", "fois", "quand",
+ "comment", "pourquoi", "alors", "donc",
+];
+
+const DE_STOP: &[&str] = &[
+ "der", "die", "das", "und", "ist", "ein", "eine", "den", "dem", "nicht", "mit", "für", "auf",
+ "von", "zu", "sich", "auch", "werden", "wurde", "haben", "sind", "oder", "aber",
+];
+
+const ES_STOP: &[&str] = &[
+ "el", "los", "las", "que", "por", "con", "para", "una", "es", "se", "del", "como", "pero",
+ "son", "está", "este", "esta", "todo", "más", "muy", "hay", "sus", "la", "un", "y", "al", "lo",
+ "le", "les", "su", "mi", "tu", "nos", "ni", "dos", "tres", "fue", "fueron", "ser", "tiene",
+ "tienen", "tengo", "puede", "pueden", "quiero", "necesito", "hemos", "han", "sobre", "entre",
+ "cuando", "donde", "porque", "aunque", "también", "ya", "eso", "esto", "esa", "ese", "nada",
+ "algo", "aquí", "hoy", "gracias",
+];
+
+const PT_STOP: &[&str] = &[
+ "os", "as", "que", "em", "um", "uma", "para", "com", "não", "é", "se", "do", "da", "dos",
+ "das", "mas", "são", "está", "este", "esta", "muito", "pelo", "pela", "o", "e", "na", "nas",
+ "nos", "ao", "aos", "por", "foi", "era", "ser", "sou", "tem", "tenho", "pode", "podem",
+ "quero", "preciso", "eu", "meu", "minha", "seu", "sua", "isso", "isto", "aqui", "ali", "como",
+ "quando", "onde", "porque", "mais", "já", "ainda", "agora", "hoje", "ontem", "dois", "três",
+ "tudo", "nada", "obrigado", "olá",
+];
+
+const IT_STOP: &[&str] = &[
+ "il", "lo", "gli", "che", "di", "per", "con", "non", "è", "si", "del", "della", "sono",
+ "questo", "questa", "anche", "come", "più", "nella", "alla", "la", "le", "un", "uno", "una",
+ "e", "ed", "o", "da", "su", "tra", "fra", "mi", "ci", "ne", "ho", "hai", "ha", "abbiamo",
+ "avete", "hanno", "era", "stato", "stata", "devo", "deve", "devono", "voglio", "vorrei", "mio",
+ "mia", "tuo", "sua", "quando", "dove", "perche", "molto", "poco", "sempre", "mai", "già",
+ "ancora", "adesso", "oggi", "ieri", "grazie", "ciao", "scusa", "nel", "nell", "negli", "sul",
+ "sulla", "sulle", "dal", "dalla", "dallo", "dagli", "dei", "delle", "dello", "degli", "agli",
+ "alle", "col",
+];
+
+const NL_STOP: &[&str] = &[
+ "het", "een", "van", "is", "op", "te", "dat", "niet", "met", "voor", "zijn", "aan", "door",
+ "maar", "ook", "worden", "deze", "naar", "wordt",
+];
+
+const RO_STOP: &[&str] = &[
+ "și", "să", "este", "sunt", "care", "pentru", "din", "dar", "după", "până", "fără", "ale",
+ "lui", "în", "fost", "acum", "vreau", "trebuie", "foarte", "acest", "această", "acesta",
+ "aceasta", "mi", "ți", "vă", "nu",
+];
+
+fn words_for(code: &str) -> &'static [&'static str] {
+ match code {
+ "en" => EN_STOP,
+ "fr" => FR_STOP,
+ "de" => DE_STOP,
+ "es" => ES_STOP,
+ "pt" => PT_STOP,
+ "it" => IT_STOP,
+ "nl" => NL_STOP,
+ "ro" => RO_STOP,
+ other => unreachable!("no stop list for {other:?}"),
+ }
+}
+
+impl LanguageDetection {
+ /// Whether `ch` is alphabetic under Python's `str.isalpha()` rule: Unicode general category
+ /// `Lu`, `Ll`, `Lt`, `Lm` or `Lo`. Unlike `char::is_alphabetic`, this excludes combining
+ /// marks.
+ fn is_alpha(ch: char) -> bool {
+ matches!(
+ get_general_category(ch),
+ GeneralCategory::UppercaseLetter
+ | GeneralCategory::LowercaseLetter
+ | GeneralCategory::TitlecaseLetter
+ | GeneralCategory::ModifierLetter
+ | GeneralCategory::OtherLetter
+ )
+ }
+
+ /// Whether `ch` matches Python's `[^\W\d_]` character class: a `\w` character (letter, `Nl`,
+ /// `No` or `_`) that is neither a decimal digit (`\d` = category `Nd`) nor `_`. In practice
+ /// this is "a letter, or a numeral that is not a plain decimal digit".
+ fn is_word_not_digit_or_underscore(ch: char) -> bool {
+ if ch == '_' {
+ return false;
+ }
+ matches!(
+ get_general_category(ch),
+ GeneralCategory::UppercaseLetter
+ | GeneralCategory::LowercaseLetter
+ | GeneralCategory::TitlecaseLetter
+ | GeneralCategory::ModifierLetter
+ | GeneralCategory::OtherLetter
+ | GeneralCategory::LetterNumber
+ | GeneralCategory::OtherNumber
+ )
+ }
+
+ /// First script whose range table contains `cp`, if any.
+ fn script_for_codepoint(cp: u32) -> Option<&'static str> {
+ for (name, ranges) in SCRIPT_RANGES {
+ if ranges.iter().any(|&(lo, hi)| cp >= lo && cp <= hi) {
+ return Some(name);
+ }
+ }
+ None
+ }
+
+ /// Whether `cp` falls in the Latin block (Basic Latin through Latin-1 Supplement and the
+ /// Latin Extended blocks below Latin Extended Additional, plus Latin Extended Additional
+ /// itself) that `detect_script`/`script_profile` treat as "latin".
+ fn is_latin_codepoint(cp: u32) -> bool {
+ cp < 0x0250 || (0x1E00..=0x1EFF).contains(&cp)
+ }
+
+ /// Collect the string leaves of a state (str / object / array), so detection sees real
+ /// content. Depths beyond 6 are dropped, and keys are ignored (they are usually English).
+ fn iter_text(state: &serde_json::Value, depth: u32, out: &mut Vec<String>) {
+ if depth > 6 {
+ return;
+ }
+ match state {
+ serde_json::Value::String(s) => out.push(s.clone()),
+ serde_json::Value::Object(map) => {
+ for v in map.values() {
+ Self::iter_text(v, depth + 1, out);
+ }
+ }
+ serde_json::Value::Array(items) => {
+ for v in items {
+ Self::iter_text(v, depth + 1, out);
+ }
+ }
+ _ => {}
+ }
+ }
+
+ /// Flatten a state into the text used for detection (keys are ignored: they are usually
+ /// English). `max_chars` truncates by Unicode scalar value, matching Python's `[:max_chars]`
+ /// string slice.
+ pub fn state_text(state: &serde_json::Value, max_chars: usize) -> String {
+ let mut parts = Vec::new();
+ Self::iter_text(state, 0, &mut parts);
+ parts.join(" ").chars().take(max_chars).collect()
+ }
+
+ /// [`Self::state_text`] with Python's default `max_chars=4000`.
+ pub fn state_text_default(state: &serde_json::Value) -> String {
+ Self::state_text(state, 4000)
+ }
+
+ /// Dominant script of `text`: `"latin"`, `"han"`, `"devanagari"`, ... or `"unknown"` if there
+ /// are no letters.
+ ///
+ /// Ties are broken by first-inserted order: the non-Latin scripts in the order they first
+ /// appear in `text`, then `"latin"` last (it is only ever added to the tally after the scan,
+ /// so it never wins a tie against a non-Latin script present in the same text).
+ pub fn detect_script(text: &str) -> String {
+ let mut counts: Vec<(&'static str, i64)> = Vec::new();
+ let mut latin: i64 = 0;
+ for ch in text.chars() {
+ if !Self::is_alpha(ch) {
+ continue;
+ }
+ let cp = ch as u32;
+ if Self::is_latin_codepoint(cp) {
+ latin += 1;
+ continue;
+ }
+ if let Some(name) = Self::script_for_codepoint(cp) {
+ match counts.iter_mut().find(|(n, _)| *n == name) {
+ Some(entry) => entry.1 += 1,
+ None => counts.push((name, 1)),
+ }
+ }
+ // A letter outside every range and outside Latin (Deseret, astral scripts, ...)
+ // counts toward nothing, matching Python exactly.
+ }
+ counts.push(("latin", latin));
+
+ let total: i64 = counts.iter().map(|(_, c)| c).sum();
+ if total == 0 {
+ return "unknown".to_string();
+ }
+ let mut best = 0usize;
+ for i in 1..counts.len() {
+ if counts[i].1 > counts[best].1 {
+ best = i;
+ }
+ }
+ counts[best].0.to_string()
+ }
+
+ /// Fraction of alphabetic characters belonging to each detected script.
+ pub fn script_profile(text: &str) -> HashMap<String, f64> {
+ let mut counts: HashMap<&'static str, i64> = HashMap::new();
+ counts.insert("latin", 0);
+ for ch in text.chars() {
+ if !Self::is_alpha(ch) {
+ continue;
+ }
+ let cp = ch as u32;
+ if Self::is_latin_codepoint(cp) {
+ *counts.get_mut("latin").unwrap() += 1;
+ continue;
+ }
+ if let Some(name) = Self::script_for_codepoint(cp) {
+ *counts.entry(name).or_insert(0) += 1;
+ }
+ }
+ let total: i64 = counts.values().sum();
+ if total == 0 {
+ return HashMap::new();
+ }
+ counts
+ .into_iter()
+ .filter(|&(_, v)| v != 0)
+ .map(|(k, v)| (k.to_string(), v as f64 / total as f64))
+ .collect()
+ }
+
+ /// Words in `text`, split on runs of Python's `[^\W\d_]` character class and lowercased with
+ /// full Unicode case mapping.
+ fn words(text: &str) -> Vec<String> {
+ let mut words = Vec::new();
+ let mut current = String::new();
+ for ch in text.chars() {
+ if Self::is_word_not_digit_or_underscore(ch) {
+ current.push(ch);
+ } else if !current.is_empty() {
+ words.push(std::mem::take(&mut current).to_lowercase());
+ }
+ }
+ if !current.is_empty() {
+ words.push(current.to_lowercase());
+ }
+ words
+ }
+
+ /// Evidence behind the Latin-script language guess: `language` (may be `None` when
+ /// undecided), `english_hits`, `diacritic_rate` and `looks_non_english`. [`Self::analyse`]
+ /// needs the evidence and not just the verdict, because "undecided" and "English" are
+ /// different answers and only one of them is safe to send to the English checkpoint.
+ ///
+ /// A non-English language is only named when it matched at least one word that no other list
+ /// claims: shared function words alone (`la`, `e`, `o`) identify no particular language.
+ pub fn latin_profile(text: &str) -> LatinProfile {
+ let words = Self::words(text);
+ let lowered = text.to_lowercase();
+ let diac = lowered
+ .chars()
+ .filter(|c| NON_EN_DIACRITICS.contains(*c))
+ .count();
+ let diac_rate = diac as f64 / (lowered.chars().count().max(1) as f64);
+ let non_english = diac_rate >= NON_EN_DIACRITIC_RATE;
+
+ if words.len() < 4 {
+ return LatinProfile {
+ language: None,
+ english_hits: 0,
+ diacritic_rate: diac_rate,
+ looks_non_english: non_english,
+ };
+ }
+
+ let word_set: std::collections::HashSet<&str> = words.iter().map(String::as_str).collect();
+
+ let lists = stop_lists();
+ // Word counted more than once across the stop lists: a word may not name a language by
+ // itself, though it still counts toward that language's score.
+ let mut membership: HashMap<&str, i32> = HashMap::new();
+ for &code in lists {
+ for w in words_for(code) {
+ *membership.entry(w).or_insert(0) += 1;
+ }
+ }
+ let shared_words: std::collections::HashSet<&str> = membership
+ .into_iter()
+ .filter(|&(_, count)| count > 1)
+ .map(|(w, _)| w)
+ .collect();
+
+ let mut scores: HashMap<&str, i64> = HashMap::new();
+ for &code in lists {
+ let set: std::collections::HashSet<&str> = words_for(code).iter().copied().collect();
+ let score = words.iter().filter(|w| set.contains(w.as_str())).count() as i64;
+ scores.insert(code, score);
+ }
+ let en = *scores.get("en").unwrap_or(&0);
+
+ // Only a language that matched at least one word no other list claims may be named.
+ // Evaluated (and the winner picked) in `stop_lists()` order, so ties keep the earlier
+ // language, matching Python's dict iteration + `max()`.
+ let mut best_lg: Option<&str> = None;
+ let mut best: i64 = 0;
+ for &code in lists {
+ if code == "en" {
+ continue;
+ }
+ let list_set: std::collections::HashSet<&str> =
+ words_for(code).iter().copied().collect();
+ let has_own_word = word_set
+ .intersection(&list_set)
+ .any(|w| !shared_words.contains(w));
+ if !has_own_word {
+ continue;
+ }
+ let score = scores[code];
+ if best_lg.is_none() || score > best {
+ best_lg = Some(code);
+ best = score;
+ }
+ }
+
+ let mut language = None;
+ if let Some(lg) = best_lg {
+ if best >= 2i64.max(en + 2) {
+ // A non-English language needs a clear margin over English function words.
+ language = Some(lg.to_string());
+ } else if non_english && best >= 2i64.max(en) {
+ // Needs two hits here too: one shared function word ("para" in Turkish text)
+ // named Spanish on the strength of the diacritics alone, which is a guess dressed
+ // as a detection.
+ language = Some(lg.to_string());
+ }
+ }
+ if language.is_none() && en > 0 && !non_english {
+ language = Some("en".to_string());
+ }
+
+ LatinProfile {
+ language,
+ english_hits: en,
+ diacritic_rate: diac_rate,
+ looks_non_english: non_english,
+ }
+ }
+
+ /// Best-effort language code for Latin-script text, or `None` when undecided.
+ ///
+ /// Scores function-word hits per language and requires the winner to beat English by a
+ /// margin and to have matched at least one word of its own, so ordinary English is never
+ /// misrouted and a word of several languages at once names none of them. Short inputs
+ /// usually return `None` on purpose.
+ pub fn guess_latin_language(text: &str) -> Option<String> {
+ Self::latin_profile(text).language
+ }
+
+ /// Full detection result for a state. See [`LanguageAnalysis`].
+ pub fn analyse(state: &serde_json::Value) -> LanguageAnalysis {
+ let text = Self::state_text_default(state);
+ let prof = Self::script_profile(&text);
+ let script = Self::detect_script(&text);
+ let non_latin = if prof.is_empty() {
+ 0.0
+ } else {
+ crate::calibration::Calibration::round4(1.0 - prof.get("latin").copied().unwrap_or(0.0))
+ };
+
+ if script == "unknown" {
+ return LanguageAnalysis {
+ script: "unknown".to_string(),
+ script_profile: prof,
+ language: None,
+ is_english: true,
+ language_undecided: true,
+ diacritic_rate: 0.0,
+ non_latin_fraction: 0.0,
+ };
+ }
+ if script != "latin" {
+ return LanguageAnalysis {
+ script,
+ script_profile: prof,
+ language: None,
+ is_english: false,
+ language_undecided: true,
+ diacritic_rate: 0.0,
+ non_latin_fraction: non_latin,
+ };
+ }
+
+ let prof_lat = Self::latin_profile(&text);
+ let lang = prof_lat.language.clone();
+ // Undecided is not English. Treating it as English sent every Latin-script language with
+ // no stopword list here to the checkpoint that cannot read it, silently. When nothing
+ // identifies the language, non-English letters are enough to prefer the multilingual
+ // checkpoint; text with no such letters (including short English) still goes to the
+ // English one.
+ let undecided = lang.is_none();
+ let english = lang.as_deref() == Some("en") || (undecided && !prof_lat.looks_non_english);
+
+ LanguageAnalysis {
+ script: "latin".to_string(),
+ script_profile: prof,
+ language: lang,
+ is_english: english,
+ language_undecided: undecided,
+ diacritic_rate: crate::calibration::Calibration::round4(prof_lat.diacritic_rate),
+ non_latin_fraction: non_latin,
+ }
+ }
+
+ /// True when the English checkpoint can be expected to read this state.
+ pub fn is_english(state: &serde_json::Value) -> bool {
+ Self::analyse(state).is_english
+ }
+}
+
+/// The evidence [`LanguageDetection::latin_profile`] gathers for a Latin-script language guess.
+#[derive(Debug, Clone, PartialEq)]
+pub struct LatinProfile {
+ /// The guessed language code, or `None` when undecided.
+ pub language: Option<String>,
+ /// How many English stopword hits the text had.
+ pub english_hits: i64,
+ /// Fraction of (lowercased) characters that are non-English Latin diacritics.
+ pub diacritic_rate: f64,
+ /// Whether the diacritic rate alone is high enough to call the text non-English.
+ pub looks_non_english: bool,
+}
diff --git a/laya-rust/tests/language_detection_tests.rs b/laya-rust/tests/language_detection_tests.rs
new file mode 100644
index 00000000..6743ef89
--- /dev/null
+++ b/laya-rust/tests/language_detection_tests.rs
@@ -0,0 +1,456 @@
+//! Tier 1: language/script detection. Ports the language-detection checks from
+//! `tests/test_router.py` (no model weights needed) plus every entry of the routing golden's
+//! `lang_probe.json`.
+
+mod common;
+
+use common::routing_golden;
+use laya::language_detection::LanguageDetection;
+use serde_json::json;
+use std::collections::HashMap;
+
+fn approx(a: f64, b: f64) {
+ assert!((a - b).abs() < 1e-9, "expected {b}, got {a}");
+}
+
+fn profile_from_json(v: &serde_json::Value) -> HashMap<String, f64> {
+ v.as_object()
+ .expect("script_profile must be an object")
+ .iter()
+ .map(|(k, val)| {
+ (
+ k.clone(),
+ val.as_f64().expect("profile value must be a number"),
+ )
+ })
+ .collect()
+}
+
+// ---------------------------------------------------------------------- named cases (test_router.py)
+
+#[test]
+fn detect_script_matches_named_cases() {
+ let cases: [(&str, &str, &str); 17] = [
+ (
+ "english",
+ "The customer was charged twice and wants a refund.",
+ "latin",
+ ),
+ ("armenian", "Հայերեն", "armenian"),
+ ("armenian uppercase", "ՀԱՅԵՐԵՆ", "armenian"),
+ ("armenian punctuation only", "։֊", "unknown"),
+ (
+ "french",
+ "Le client a été facturé deux fois et demande un remboursement.",
+ "latin",
+ ),
+ (
+ "hindi",
+ "ग्राहक से दो बार शुल्क लिया गया और वह धनवापसी चाहता है।",
+ "devanagari",
+ ),
+ (
+ "japanese",
+ "お客様は二重に請求されたため返金を希望しています。",
+ "kana",
+ ),
+ ("chinese", "客户被重复扣款要求退款", "han"),
+ ("korean", "고객이 두 번 청구되어 환불을 원합니다", "hangul"),
+ (
+ "arabic",
+ "تم خصم المبلغ مرتين من العميل ويريد استرداد الأموال",
+ "arabic",
+ ),
+ ("tamil", "வாடிக்கையாளரிடம் இருமுறை கட்டணம் வசூலிக்கப்பட்டது", "tamil"),
+ (
+ "russian",
+ "С клиента дважды сняли деньги и он хочет возврат",
+ "cyrillic",
+ ),
+ ("thai", "ลูกค้าถูกเรียกเก็บเงินสองครั้งและต้องการเงินคืน", "thai"),
+ (
+ "greek",
+ "Ο πελάτης χρεώθηκε δύο φορές και θέλει επιστροφή χρημάτων",
+ "greek",
+ ),
+ ("hebrew", "הלקוח חויב פעמיים ורוצה החזר כספי", "hebrew"),
+ ("empty", "", "unknown"),
+ ("digits only", "12345 6789", "unknown"),
+ ];
+ for (label, text, want) in cases {
+ assert_eq!(
+ LanguageDetection::detect_script(text),
+ want,
+ "script/{label}"
+ );
+ }
+}
+
+#[test]
+fn is_english_matches_named_cases() {
+ let cases: [(&str, &str, bool); 16] = [
+ (
+ "plain english",
+ "Please refund the duplicate charge on invoice 4411 today.",
+ true,
+ ),
+ ("armenian", "Հայերեն", false),
+ ("english short", "refund me", true),
+ ("hindi", "ग्राहक से दो बार शुल्क लिया गया", false),
+ ("japanese", "お客様は二重に請求されました", false),
+ ("russian", "С клиента дважды сняли деньги", false),
+ (
+ "french long",
+ "Le client a été facturé deux fois et il demande un remboursement pour la \
+ facture qui a été payée le mois dernier avec la carte de crédit",
+ false,
+ ),
+ (
+ "german long",
+ "Der Kunde wurde zweimal belastet und möchte eine Rückerstattung für die \
+ Rechnung die nicht korrekt ist und auch nicht bezahlt wurde",
+ false,
+ ),
+ (
+ "romanian",
+ "Gătește-mi o rețetă de sarmale de post pentru mâine.",
+ false,
+ ),
+ (
+ "romanian invoice",
+ "Am fost taxat de două ori pentru factura din luna martie și vreau banii",
+ false,
+ ),
+ (
+ "polish",
+ "Klient został obciążony dwukrotnie i chce zwrot pieniędzy za fakturę",
+ false,
+ ),
+ (
+ "czech",
+ "Zákazníkovi byla částka účtována dvakrát a žádá o vrácení peněz",
+ false,
+ ),
+ (
+ "turkish",
+ "Müşteriden iki kez ücret alındı ve para iadesi istiyor lütfen yardım",
+ false,
+ ),
+ (
+ "vietnamese",
+ "Khách hàng đã bị thu phí hai lần và muốn được hoàn tiền ngay",
+ false,
+ ),
+ (
+ "english with loanwords",
+ "We visited a cafe in Zurich and the naive assumption about the \
+ invoice was wrong, so please refund the duplicate charge",
+ true,
+ ),
+ ("empty", "", true),
+ ];
+ for (label, text, want) in cases {
+ let v = json!(text);
+ assert_eq!(
+ LanguageDetection::is_english(&v),
+ want,
+ "is_english/{label}"
+ );
+ }
+}
+
+#[test]
+fn latin_profile_undecided_reporting() {
+ let undecided = json!("Müşteriden iki kez ücret alındı ve para iadesi istiyor");
+ let a = LanguageDetection::analyse(&undecided);
+ assert!(a.language_undecided, "latin/undecided is flagged");
+ assert_eq!(a.language, None, "latin/undecided names no language");
+
+ let english = json!("Please refund the duplicate charge on the invoice");
+ assert!(
+ !LanguageDetection::analyse(&english).language_undecided,
+ "latin/english is not undecided"
+ );
+
+ let romanian = json!("Gătește-mi o rețetă de sarmale");
+ assert!(
+ LanguageDetection::analyse(&romanian).diacritic_rate > 0.02,
+ "latin/diacritic rate reported"
+ );
+
+ let plain_english = json!("Please refund the duplicate charge today");
+ approx(
+ LanguageDetection::analyse(&plain_english).diacritic_rate,
+ 0.0,
+ );
+
+ // A 0-0 tie between non-English stopword lists is no evidence for any of them.
+ assert_eq!(
+ LanguageDetection::guess_latin_language("Cât e ora acum la Tokyo"),
+ None,
+ "latin_lang/zero tie invents nothing"
+ );
+
+ // Known limitation, kept visible on purpose (see laya/lang.py and issue #35): short Romanian
+ // with no diacritics and an English function word still reads as English.
+ assert!(LanguageDetection::is_english(&json!(
+ "Care este ora in Tokyo?"
+ )));
+}
+
+#[test]
+fn guess_latin_language_matches_named_cases() {
+ let cases: [(&str, &str, Option<&str>); 5] = [
+ (
+ "english",
+ "The customer was charged twice and wants a refund for this invoice",
+ Some("en"),
+ ),
+ (
+ "french",
+ "Le client a ete facture deux fois et il demande un remboursement pour la facture",
+ Some("fr"),
+ ),
+ (
+ "german",
+ "Der Kunde wurde zweimal belastet und moechte eine Rueckerstattung fuer die Rechnung",
+ Some("de"),
+ ),
+ (
+ "spanish",
+ "El cliente fue cobrado dos veces y quiere que le devuelvan el dinero por la factura",
+ Some("es"),
+ ),
+ ("too short", "refund", None),
+ ];
+ for (label, text, want) in cases {
+ assert_eq!(
+ LanguageDetection::guess_latin_language(text).as_deref(),
+ want,
+ "latin_lang/{label}"
+ );
+ }
+ assert_eq!(
+ LanguageDetection::guess_latin_language(
+ "Please refund the duplicate charge on invoice 4411 today because \
+ we have been waiting for three days and nobody has replied to us"
+ )
+ .as_deref(),
+ Some("en"),
+ "latin_lang/long english stays en"
+ );
+}
+
+#[test]
+fn state_text_flattens_nested_state() {
+ let dict_state = json!({"body": "charged twice", "n": 3});
+ assert!(LanguageDetection::state_text_default(&dict_state).contains("charged twice"));
+
+ let nested = json!({"a": {"b": ["deep"]}});
+ assert!(LanguageDetection::state_text_default(&nested).contains("deep"));
+
+ let list_state = json!(["x", {"y": "z"}]);
+ assert!(LanguageDetection::state_text_default(&list_state).contains("x"));
+
+ assert_eq!(LanguageDetection::state_text_default(&json!(null)), "");
+
+ // Keys must not drive detection: English keys around Hindi content stay non-English.
+ let keyed = json!({"subject": "नमस्ते", "body": "ग्राहक से दो बार शुल्क लिया गया"});
+ assert!(
+ !LanguageDetection::is_english(&keyed),
+ "state_text/keys ignored"
+ );
+}
+
+#[test]
+fn script_profile_named_cases() {
+ let armenian = LanguageDetection::analyse(&json!("Հայերեն"));
+ assert_eq!(armenian.script_profile.len(), 1);
+ approx(*armenian.script_profile.get("armenian").unwrap(), 1.0);
+
+ let mixed = LanguageDetection::analyse(&json!("Հայերեն abc"));
+ approx(mixed.non_latin_fraction, 0.7);
+}
+
+#[test]
+fn plain_ascii_romance_named_cases() {
+ // #172: ASCII-stripped Romance text carries no diacritic rate, so the function-word lists are
+ // the only evidence left.
+ let cases: [(&str, &str); 10] = [
+ (
+ "es",
+ "El pedido llego roto y nadie responde cuando escribo al soporte",
+ ),
+ ("es", "Quiero cancelar mi plan y pedir un reembolso"),
+ ("es", "La factura tiene un error en el importe total"),
+ (
+ "es",
+ "Necesito que me devuelvan el dinero de la compra duplicada",
+ ),
+ (
+ "it",
+ "Il cliente e stato addebitato due volte e vuole un rimborso",
+ ),
+ (
+ "it",
+ "Voglio cancellare il mio abbonamento e chiedere un rimborso",
+ ),
+ ("it", "La fattura contiene un errore nell importo totale"),
+ (
+ "pt",
+ "O cliente foi cobrado duas vezes e quer o dinheiro de volta",
+ ),
+ (
+ "fr",
+ "Le client a ete facture deux fois et demande un remboursement",
+ ),
+ (
+ "fr",
+ "Je ne peux pas acceder a mon compte et j ai besoin d aide",
+ ),
+ ];
+ for (lang, text) in cases {
+ assert_eq!(
+ LanguageDetection::guess_latin_language(text).as_deref(),
+ Some(lang),
+ "latin_lang/plain ascii {lang} {text}"
+ );
+ assert!(
+ !LanguageDetection::is_english(&json!(text)),
+ "is_english/plain ascii {lang} {text}"
+ );
+ }
+
+ for text in [
+ "The customer was charged twice and wants a refund for this invoice",
+ "Please cancel my subscription and refund the duplicate charge today",
+ "The report by Smith et al. shows the de facto standard, e.g. the LA office and Rio",
+ "Our MI5 and UN contacts discussed the DOS attack in LA last month",
+ "No refund was issued, so I am writing to you again about invoice 4411",
+ "no refund no reply",
+ "The son of the director filed a complaint about the duplicate invoice",
+ ] {
+ assert!(
+ LanguageDetection::is_english(&json!(text)),
+ "is_english/romance control {text}"
+ );
+ }
+
+ // A word several lists claim (`la`, `e`, `o`) says "not English" without saying *which*
+ // language, so it may not name one on its own.
+ assert_eq!(
+ LanguageDetection::guess_latin_language("Cât e ora acum la Tokyo"),
+ None,
+ "latin_lang/shared words alone name nothing"
+ );
+ // ...but a distinctive word in the same state is enough to name the language it belongs to.
+ assert_eq!(
+ LanguageDetection::guess_latin_language(
+ "La fattura contiene un errore nell importo totale"
+ )
+ .as_deref(),
+ Some("it")
+ );
+ assert_eq!(
+ LanguageDetection::guess_latin_language("La factura tiene un error en el importe total")
+ .as_deref(),
+ Some("es")
+ );
+}
+
+// ---------------------------------------------------------------------- lang_probe.json
+
+#[test]
+fn lang_probe_matches_every_entry() {
+ let Some(cases) = routing_golden::load_array("lang_probe.json") else {
+ eprintln!("skip: routing golden not found (tests/golden/routing)");
+ return;
+ };
+
+ for case in &cases {
+ let label = case["label"].as_str().unwrap();
+ let state = &case["state"];
+
+ let expected_state_text = case["state_text"].as_str().unwrap();
+ assert_eq!(
+ LanguageDetection::state_text_default(state),
+ expected_state_text,
+ "{label}: state_text"
+ );
+
+ let text = LanguageDetection::state_text_default(state);
+ assert_eq!(
+ LanguageDetection::detect_script(&text),
+ case["detect_script"].as_str().unwrap(),
+ "{label}: detect_script"
+ );
+
+ let expected_profile = profile_from_json(&case["script_profile"]);
+ let actual_profile = LanguageDetection::script_profile(&text);
+ assert_eq!(
+ actual_profile.len(),
+ expected_profile.len(),
+ "{label}: script_profile key count"
+ );
+ for (k, v) in &expected_profile {
+ let got = actual_profile
+ .get(k)
+ .unwrap_or_else(|| panic!("{label}: script_profile missing key {k}"));
+ approx(*got, *v);
+ }
+
+ let lp = &case["latin_profile"];
+ let actual_lp = LanguageDetection::latin_profile(&text);
+ assert_eq!(
+ actual_lp.language.as_deref(),
+ lp["language"].as_str(),
+ "{label}: latin_profile.language"
+ );
+ assert_eq!(
+ actual_lp.english_hits,
+ lp["english_hits"].as_i64().unwrap(),
+ "{label}: latin_profile.english_hits"
+ );
+ approx(
+ actual_lp.diacritic_rate,
+ lp["diacritic_rate"].as_f64().unwrap(),
+ );
+ assert_eq!(
+ actual_lp.looks_non_english,
+ lp["looks_non_english"].as_bool().unwrap(),
+ "{label}: latin_profile.looks_non_english"
+ );
+
+ let a = &case["analyse"];
+ let actual = LanguageDetection::analyse(state);
+ assert_eq!(
+ actual.script,
+ a["script"].as_str().unwrap(),
+ "{label}: analyse.script"
+ );
+ assert_eq!(
+ actual.language.as_deref(),
+ a["language"].as_str(),
+ "{label}: analyse.language"
+ );
+ assert_eq!(
+ actual.is_english,
+ a["is_english"].as_bool().unwrap(),
+ "{label}: analyse.is_english"
+ );
+ assert_eq!(
+ actual.language_undecided,
+ a["language_undecided"].as_bool().unwrap(),
+ "{label}: analyse.language_undecided"
+ );
+ approx(actual.diacritic_rate, a["diacritic_rate"].as_f64().unwrap());
+ approx(
+ actual.non_latin_fraction,
+ a["non_latin_fraction"].as_f64().unwrap(),
+ );
+ }
+ eprintln!(
+ "lang_probe_matches_every_entry: checked {} cases",
+ cases.len()
+ );
+}