Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
feat(rust): add language detection
kevin-gatimuea36f4351746
diff --git a/laya-rust/src/language_detection.rs b/laya-rust/src/language_detection.rsnew file mode 100644index 00000000..3b104457--- /dev/null+++ b/laya-rust/src/language_detection.rs@@ -0,0 +1,576 @@+//! Dependency-free language/script detection used to route between Laya checkpoints. Ports+//! `laya/lang.py`.+//!+//! Routing only needs one decision: *is this English Latin text, or is it something the English+//! checkpoint cannot read?* Benchmarks on MASSIVE (14 languages) showed the English checkpoint+//! collapsing to near-random on non-Latin scripts (Hindi 0.100, Korean 0.103, Swahili 0.103, Tamil+//! 0.113 at 20 options, where random is 0.050), while holding up far better on Latin-script+//! languages (French 0.487, Spanish 0.480). So the signal that matters most is *script*, and the+//! secondary signal is whether Latin text is English.+//!+//! Script detection is exact. The Latin-script language guess is a stopword/diacritic heuristic+//! and is explicitly best-effort: pass an explicit model or `lang=` when the language is already+//! known.+//!+//! ## Unicode fidelity+//!+//! Python's `str.isalpha()` tests Unicode general category `L*` (`Lu`, `Ll`, `Lt`, `Lm`, `Lo`)+//! only. Rust's `char::is_alphabetic` also accepts combining marks, which would change+//! [`script_profile`]'s fractions on marked scripts (Hindi matras, for example), so every+//! alphabetic test here goes through [`unicode_general_category::get_general_category`] instead.+//! Python's `[^\W\d_]+` word pattern (`\w` = `isalnum()` + `_`, in Unicode mode) is likewise+//! reproduced with a general-category check rather than a regex crate: `\w` minus digits (`\d` =+//! category `Nd` only) minus `_` leaves letters plus `Nl`/`No` (a numeric-letter or other-number+//! character, but not a plain decimal digit).++use std::collections::HashMap;+use unicode_general_category::{GeneralCategory, get_general_category};++/// Namespace for language/script detection. See the module docs for the source this ports.+pub struct LanguageDetection;++/// The full detection result for one piece of state. See [`LanguageDetection::analyse`].+#[derive(Debug, Clone, PartialEq)]+pub struct LanguageAnalysis {+ /// The dominant script: `"latin"`, `"han"`, `"devanagari"`, ... or `"unknown"` when the text+ /// has no letters at all.+ pub script: String,+ /// Fraction of alphabetic characters belonging to each detected script.+ pub script_profile: HashMap<String, f64>,+ /// Best-effort Latin-script language guess (an ISO-639-1-ish code), or `None` when undecided+ /// or when the script is not Latin.+ pub language: Option<String>,+ /// Whether the English checkpoint can be expected to read this text.+ pub is_english: bool,+ /// Whether [`Self::language`] is `None` because no language could be identified (as opposed+ /// to being `None` because the script itself is non-Latin).+ pub language_undecided: bool,+ /// Fraction of (lowercased) characters that are non-English Latin diacritics.+ pub diacritic_rate: f64,+ /// `1.0 - script_profile["latin"]`, or `0.0` when there is no script profile at all.+ pub non_latin_fraction: f64,+}++// Unicode blocks that the English (ModernBERT-large, 50k English BPE) checkpoint cannot read.+// Order matters: `detect_script`/`script_profile` take the first matching range, and a tie+// between two script counts in `detect_script` is broken by which name was inserted first, which+// is this table's order (for non-Latin scripts) followed by "latin" last.+const SCRIPT_RANGES: &[(&str, &[(u32, u32)])] = &[+ ("greek", &[(0x0370, 0x03FF), (0x1F00, 0x1FFF)]),+ (+ "cyrillic",+ &[(0x0400, 0x052F), (0x2DE0, 0x2DFF), (0xA640, 0xA69F)],+ ),+ ("armenian", &[(0x0530, 0x058F)]),+ ("hebrew", &[(0x0590, 0x05FF)]),+ (+ "arabic",+ &[+ (0x0600, 0x06FF),+ (0x0750, 0x077F),+ (0x08A0, 0x08FF),+ (0xFB50, 0xFDFF),+ (0xFE70, 0xFEFF),+ ],+ ),+ ("devanagari", &[(0x0900, 0x097F), (0xA8E0, 0xA8FF)]),+ ("bengali", &[(0x0980, 0x09FF)]),+ ("gurmukhi", &[(0x0A00, 0x0A7F)]),+ ("gujarati", &[(0x0A80, 0x0AFF)]),+ ("oriya", &[(0x0B00, 0x0B7F)]),+ ("tamil", &[(0x0B80, 0x0BFF)]),+ ("telugu", &[(0x0C00, 0x0C7F)]),+ ("kannada", &[(0x0C80, 0x0CFF)]),+ ("malayalam", &[(0x0D00, 0x0D7F)]),+ ("sinhala", &[(0x0D80, 0x0DFF)]),+ ("thai", &[(0x0E00, 0x0E7F)]),+ ("lao", &[(0x0E80, 0x0EFF)]),+ ("tibetan", &[(0x0F00, 0x0FFF)]),+ ("myanmar", &[(0x1000, 0x109F)]),+ ("georgian", &[(0x10A0, 0x10FF)]),+ ("ethiopic", &[(0x1200, 0x137F)]),+ ("khmer", &[(0x1780, 0x17FF)]),+ (+ "hangul",+ &[(0x1100, 0x11FF), (0x3130, 0x318F), (0xAC00, 0xD7AF)],+ ),+ (+ "kana",+ &[(0x3040, 0x309F), (0x30A0, 0x30FF), (0x31F0, 0x31FF)],+ ),+ (+ "han",+ &[(0x3400, 0x4DBF), (0x4E00, 0x9FFF), (0xF900, 0xFAFF)],+ ),+];++/// A diacritic rate above this is taken as evidence the text is not English, even when no+/// stopword list matches it.+const NON_EN_DIACRITIC_RATE: f64 = 0.02;++/// Letters that ordinary English does not use. This catches a Latin-script language with no+/// stopword list at all (Romanian, Polish, Czech, Turkish, Baltic, ...), which is the difference+/// between routing it to the multilingual checkpoint and silently handing it to the English one.+/// Text is lowercased before matching, so only lowercase forms need listing here.+const NON_EN_DIACRITICS: &str = concat!(+ "àâäãáåçéèêëíìîïñóòôöõøúùûüýÿßæœ", // Western European+ "ăâîșțşţ", // Romanian+ "ąćęłńśźż", // Polish+ "čďěňřšťůž", // Czech / Slovak+ "őű", // Hungarian+ "ğı", // Turkish+ "āēģīķļņūž", // Baltic+ "đ", // Serbo-Croatian / Vietnamese+);++/// Function words. Latin-script languages overlap heavily (de/la/le/un/e/que), so each hit is+/// weighted and a margin is required before calling something non-English.+///+/// The Romance lists (fr/es/pt/it) deliberately carry the *unaccented* function words as well as+/// the accented ones. A state that lost its accents -- mail clients, ticket systems and any+/// pipeline that normalises to ASCII strip them -- keeps no diacritic rate for the non-English+/// signal to read, so `la`, `un`, `y`, `e`, `et`, `deux` and friends are the only evidence left.+/// With a list of mostly accented words such a state produced one hit or none, fell under the+/// two-hit margin below, and was handed to the English checkpoint as undecided-but-not-non-English+/// text.+///+/// Order is load-bearing for `latin_profile`'s tie-break (`_STOP` iteration order: en, fr, de, es,+/// pt, it, nl, ro), so this stays a `Vec` of `(code, words)` pairs rather than a `HashMap`.+fn stop_lists() -> &'static [&'static str] {+ // Order is load-bearing (see the doc comment above): en, fr, de, es, pt, it, nl, ro.+ &["en", "fr", "de", "es", "pt", "it", "nl", "ro"]+}++// Slices (not fixed-size arrays) throughout, so splitting a long list across several `const`s+// for readability never needs a manually counted length.+const EN_STOP: &[&str] = &[+ "the", "and", "is", "are", "was", "were", "to", "of", "in", "for", "with", "that", "this",+ "it", "you", "have", "has", "not", "but", "on", "at", "be", "as", "from", "will", "can",+ "would", "there", "their", "what", "which", "please", "we", "i",+];++const FR_STOP: &[&str] = &[+ "le", "la", "les", "des", "une", "est", "pour", "dans", "que", "qui", "avec", "sur", "pas",+ "plus", "nous", "vous", "être", "cette", "mais", "sont", "ont", "aux", "ce", "et", "du", "au",+ "ou", "je", "tu", "il", "elle", "ils", "elles", "mon", "ton", "ma", "ta", "sa", "mes", "tes",+ "ses", "ces", "deux", "trois", "très", "bien", "tout", "tous", "toute", "fait", "veux", "veut",+ "peux", "peut", "dois", "doit", "merci", "bonjour", "jour", "jours", "mois", "fois", "quand",+ "comment", "pourquoi", "alors", "donc",+];++const DE_STOP: &[&str] = &[+ "der", "die", "das", "und", "ist", "ein", "eine", "den", "dem", "nicht", "mit", "für", "auf",+ "von", "zu", "sich", "auch", "werden", "wurde", "haben", "sind", "oder", "aber",+];++const ES_STOP: &[&str] = &[+ "el", "los", "las", "que", "por", "con", "para", "una", "es", "se", "del", "como", "pero",+ "son", "está", "este", "esta", "todo", "más", "muy", "hay", "sus", "la", "un", "y", "al", "lo",+ "le", "les", "su", "mi", "tu", "nos", "ni", "dos", "tres", "fue", "fueron", "ser", "tiene",+ "tienen", "tengo", "puede", "pueden", "quiero", "necesito", "hemos", "han", "sobre", "entre",+ "cuando", "donde", "porque", "aunque", "también", "ya", "eso", "esto", "esa", "ese", "nada",+ "algo", "aquí", "hoy", "gracias",+];++const PT_STOP: &[&str] = &[+ "os", "as", "que", "em", "um", "uma", "para", "com", "não", "é", "se", "do", "da", "dos",+ "das", "mas", "são", "está", "este", "esta", "muito", "pelo", "pela", "o", "e", "na", "nas",+ "nos", "ao", "aos", "por", "foi", "era", "ser", "sou", "tem", "tenho", "pode", "podem",+ "quero", "preciso", "eu", "meu", "minha", "seu", "sua", "isso", "isto", "aqui", "ali", "como",+ "quando", "onde", "porque", "mais", "já", "ainda", "agora", "hoje", "ontem", "dois", "três",+ "tudo", "nada", "obrigado", "olá",+];++const IT_STOP: &[&str] = &[+ "il", "lo", "gli", "che", "di", "per", "con", "non", "è", "si", "del", "della", "sono",+ "questo", "questa", "anche", "come", "più", "nella", "alla", "la", "le", "un", "uno", "una",+ "e", "ed", "o", "da", "su", "tra", "fra", "mi", "ci", "ne", "ho", "hai", "ha", "abbiamo",+ "avete", "hanno", "era", "stato", "stata", "devo", "deve", "devono", "voglio", "vorrei", "mio",+ "mia", "tuo", "sua", "quando", "dove", "perche", "molto", "poco", "sempre", "mai", "già",+ "ancora", "adesso", "oggi", "ieri", "grazie", "ciao", "scusa", "nel", "nell", "negli", "sul",+ "sulla", "sulle", "dal", "dalla", "dallo", "dagli", "dei", "delle", "dello", "degli", "agli",+ "alle", "col",+];++const NL_STOP: &[&str] = &[+ "het", "een", "van", "is", "op", "te", "dat", "niet", "met", "voor", "zijn", "aan", "door",+ "maar", "ook", "worden", "deze", "naar", "wordt",+];++const RO_STOP: &[&str] = &[+ "și", "să", "este", "sunt", "care", "pentru", "din", "dar", "după", "până", "fără", "ale",+ "lui", "în", "fost", "acum", "vreau", "trebuie", "foarte", "acest", "această", "acesta",+ "aceasta", "mi", "ți", "vă", "nu",+];++fn words_for(code: &str) -> &'static [&'static str] {+ match code {+ "en" => EN_STOP,+ "fr" => FR_STOP,+ "de" => DE_STOP,+ "es" => ES_STOP,+ "pt" => PT_STOP,+ "it" => IT_STOP,+ "nl" => NL_STOP,+ "ro" => RO_STOP,+ other => unreachable!("no stop list for {other:?}"),+ }+}++impl LanguageDetection {+ /// Whether `ch` is alphabetic under Python's `str.isalpha()` rule: Unicode general category+ /// `Lu`, `Ll`, `Lt`, `Lm` or `Lo`. Unlike `char::is_alphabetic`, this excludes combining+ /// marks.+ fn is_alpha(ch: char) -> bool {+ matches!(+ get_general_category(ch),+ GeneralCategory::UppercaseLetter+ | GeneralCategory::LowercaseLetter+ | GeneralCategory::TitlecaseLetter+ | GeneralCategory::ModifierLetter+ | GeneralCategory::OtherLetter+ )+ }++ /// Whether `ch` matches Python's `[^\W\d_]` character class: a `\w` character (letter, `Nl`,+ /// `No` or `_`) that is neither a decimal digit (`\d` = category `Nd`) nor `_`. In practice+ /// this is "a letter, or a numeral that is not a plain decimal digit".+ fn is_word_not_digit_or_underscore(ch: char) -> bool {+ if ch == '_' {+ return false;+ }+ matches!(+ get_general_category(ch),+ GeneralCategory::UppercaseLetter+ | GeneralCategory::LowercaseLetter+ | GeneralCategory::TitlecaseLetter+ | GeneralCategory::ModifierLetter+ | GeneralCategory::OtherLetter+ | GeneralCategory::LetterNumber+ | GeneralCategory::OtherNumber+ )+ }++ /// First script whose range table contains `cp`, if any.+ fn script_for_codepoint(cp: u32) -> Option<&'static str> {+ for (name, ranges) in SCRIPT_RANGES {+ if ranges.iter().any(|&(lo, hi)| cp >= lo && cp <= hi) {+ return Some(name);+ }+ }+ None+ }++ /// Whether `cp` falls in the Latin block (Basic Latin through Latin-1 Supplement and the+ /// Latin Extended blocks below Latin Extended Additional, plus Latin Extended Additional+ /// itself) that `detect_script`/`script_profile` treat as "latin".+ fn is_latin_codepoint(cp: u32) -> bool {+ cp < 0x0250 || (0x1E00..=0x1EFF).contains(&cp)+ }++ /// Collect the string leaves of a state (str / object / array), so detection sees real+ /// content. Depths beyond 6 are dropped, and keys are ignored (they are usually English).+ fn iter_text(state: &serde_json::Value, depth: u32, out: &mut Vec<String>) {+ if depth > 6 {+ return;+ }+ match state {+ serde_json::Value::String(s) => out.push(s.clone()),+ serde_json::Value::Object(map) => {+ for v in map.values() {+ Self::iter_text(v, depth + 1, out);+ }+ }+ serde_json::Value::Array(items) => {+ for v in items {+ Self::iter_text(v, depth + 1, out);+ }+ }+ _ => {}+ }+ }++ /// Flatten a state into the text used for detection (keys are ignored: they are usually+ /// English). `max_chars` truncates by Unicode scalar value, matching Python's `[:max_chars]`+ /// string slice.+ pub fn state_text(state: &serde_json::Value, max_chars: usize) -> String {+ let mut parts = Vec::new();+ Self::iter_text(state, 0, &mut parts);+ parts.join(" ").chars().take(max_chars).collect()+ }++ /// [`Self::state_text`] with Python's default `max_chars=4000`.+ pub fn state_text_default(state: &serde_json::Value) -> String {+ Self::state_text(state, 4000)+ }++ /// Dominant script of `text`: `"latin"`, `"han"`, `"devanagari"`, ... or `"unknown"` if there+ /// are no letters.+ ///+ /// Ties are broken by first-inserted order: the non-Latin scripts in the order they first+ /// appear in `text`, then `"latin"` last (it is only ever added to the tally after the scan,+ /// so it never wins a tie against a non-Latin script present in the same text).+ pub fn detect_script(text: &str) -> String {+ let mut counts: Vec<(&'static str, i64)> = Vec::new();+ let mut latin: i64 = 0;+ for ch in text.chars() {+ if !Self::is_alpha(ch) {+ continue;+ }+ let cp = ch as u32;+ if Self::is_latin_codepoint(cp) {+ latin += 1;+ continue;+ }+ if let Some(name) = Self::script_for_codepoint(cp) {+ match counts.iter_mut().find(|(n, _)| *n == name) {+ Some(entry) => entry.1 += 1,+ None => counts.push((name, 1)),+ }+ }+ // A letter outside every range and outside Latin (Deseret, astral scripts, ...)+ // counts toward nothing, matching Python exactly.+ }+ counts.push(("latin", latin));++ let total: i64 = counts.iter().map(|(_, c)| c).sum();+ if total == 0 {+ return "unknown".to_string();+ }+ let mut best = 0usize;+ for i in 1..counts.len() {+ if counts[i].1 > counts[best].1 {+ best = i;+ }+ }+ counts[best].0.to_string()+ }++ /// Fraction of alphabetic characters belonging to each detected script.+ pub fn script_profile(text: &str) -> HashMap<String, f64> {+ let mut counts: HashMap<&'static str, i64> = HashMap::new();+ counts.insert("latin", 0);+ for ch in text.chars() {+ if !Self::is_alpha(ch) {+ continue;+ }+ let cp = ch as u32;+ if Self::is_latin_codepoint(cp) {+ *counts.get_mut("latin").unwrap() += 1;+ continue;+ }+ if let Some(name) = Self::script_for_codepoint(cp) {+ *counts.entry(name).or_insert(0) += 1;+ }+ }+ let total: i64 = counts.values().sum();+ if total == 0 {+ return HashMap::new();+ }+ counts+ .into_iter()+ .filter(|&(_, v)| v != 0)+ .map(|(k, v)| (k.to_string(), v as f64 / total as f64))+ .collect()+ }++ /// Words in `text`, split on runs of Python's `[^\W\d_]` character class and lowercased with+ /// full Unicode case mapping.+ fn words(text: &str) -> Vec<String> {+ let mut words = Vec::new();+ let mut current = String::new();+ for ch in text.chars() {+ if Self::is_word_not_digit_or_underscore(ch) {+ current.push(ch);+ } else if !current.is_empty() {+ words.push(std::mem::take(&mut current).to_lowercase());+ }+ }+ if !current.is_empty() {+ words.push(current.to_lowercase());+ }+ words+ }++ /// Evidence behind the Latin-script language guess: `language` (may be `None` when+ /// undecided), `english_hits`, `diacritic_rate` and `looks_non_english`. [`Self::analyse`]+ /// needs the evidence and not just the verdict, because "undecided" and "English" are+ /// different answers and only one of them is safe to send to the English checkpoint.+ ///+ /// A non-English language is only named when it matched at least one word that no other list+ /// claims: shared function words alone (`la`, `e`, `o`) identify no particular language.+ pub fn latin_profile(text: &str) -> LatinProfile {+ let words = Self::words(text);+ let lowered = text.to_lowercase();+ let diac = lowered+ .chars()+ .filter(|c| NON_EN_DIACRITICS.contains(*c))+ .count();+ let diac_rate = diac as f64 / (lowered.chars().count().max(1) as f64);+ let non_english = diac_rate >= NON_EN_DIACRITIC_RATE;++ if words.len() < 4 {+ return LatinProfile {+ language: None,+ english_hits: 0,+ diacritic_rate: diac_rate,+ looks_non_english: non_english,+ };+ }++ let word_set: std::collections::HashSet<&str> = words.iter().map(String::as_str).collect();++ let lists = stop_lists();+ // Word counted more than once across the stop lists: a word may not name a language by+ // itself, though it still counts toward that language's score.+ let mut membership: HashMap<&str, i32> = HashMap::new();+ for &code in lists {+ for w in words_for(code) {+ *membership.entry(w).or_insert(0) += 1;+ }+ }+ let shared_words: std::collections::HashSet<&str> = membership+ .into_iter()+ .filter(|&(_, count)| count > 1)+ .map(|(w, _)| w)+ .collect();++ let mut scores: HashMap<&str, i64> = HashMap::new();+ for &code in lists {+ let set: std::collections::HashSet<&str> = words_for(code).iter().copied().collect();+ let score = words.iter().filter(|w| set.contains(w.as_str())).count() as i64;+ scores.insert(code, score);+ }+ let en = *scores.get("en").unwrap_or(&0);++ // Only a language that matched at least one word no other list claims may be named.+ // Evaluated (and the winner picked) in `stop_lists()` order, so ties keep the earlier+ // language, matching Python's dict iteration + `max()`.+ let mut best_lg: Option<&str> = None;+ let mut best: i64 = 0;+ for &code in lists {+ if code == "en" {+ continue;+ }+ let list_set: std::collections::HashSet<&str> =+ words_for(code).iter().copied().collect();+ let has_own_word = word_set+ .intersection(&list_set)+ .any(|w| !shared_words.contains(w));+ if !has_own_word {+ continue;+ }+ let score = scores[code];+ if best_lg.is_none() || score > best {+ best_lg = Some(code);+ best = score;+ }+ }++ let mut language = None;+ if let Some(lg) = best_lg {+ if best >= 2i64.max(en + 2) {+ // A non-English language needs a clear margin over English function words.+ language = Some(lg.to_string());+ } else if non_english && best >= 2i64.max(en) {+ // Needs two hits here too: one shared function word ("para" in Turkish text)+ // named Spanish on the strength of the diacritics alone, which is a guess dressed+ // as a detection.+ language = Some(lg.to_string());+ }+ }+ if language.is_none() && en > 0 && !non_english {+ language = Some("en".to_string());+ }++ LatinProfile {+ language,+ english_hits: en,+ diacritic_rate: diac_rate,+ looks_non_english: non_english,+ }+ }++ /// Best-effort language code for Latin-script text, or `None` when undecided.+ ///+ /// Scores function-word hits per language and requires the winner to beat English by a+ /// margin and to have matched at least one word of its own, so ordinary English is never+ /// misrouted and a word of several languages at once names none of them. Short inputs+ /// usually return `None` on purpose.+ pub fn guess_latin_language(text: &str) -> Option<String> {+ Self::latin_profile(text).language+ }++ /// Full detection result for a state. See [`LanguageAnalysis`].+ pub fn analyse(state: &serde_json::Value) -> LanguageAnalysis {+ let text = Self::state_text_default(state);+ let prof = Self::script_profile(&text);+ let script = Self::detect_script(&text);+ let non_latin = if prof.is_empty() {+ 0.0+ } else {+ crate::calibration::Calibration::round4(1.0 - prof.get("latin").copied().unwrap_or(0.0))+ };++ if script == "unknown" {+ return LanguageAnalysis {+ script: "unknown".to_string(),+ script_profile: prof,+ language: None,+ is_english: true,+ language_undecided: true,+ diacritic_rate: 0.0,+ non_latin_fraction: 0.0,+ };+ }+ if script != "latin" {+ return LanguageAnalysis {+ script,+ script_profile: prof,+ language: None,+ is_english: false,+ language_undecided: true,+ diacritic_rate: 0.0,+ non_latin_fraction: non_latin,+ };+ }++ let prof_lat = Self::latin_profile(&text);+ let lang = prof_lat.language.clone();+ // Undecided is not English. Treating it as English sent every Latin-script language with+ // no stopword list here to the checkpoint that cannot read it, silently. When nothing+ // identifies the language, non-English letters are enough to prefer the multilingual+ // checkpoint; text with no such letters (including short English) still goes to the+ // English one.+ let undecided = lang.is_none();+ let english = lang.as_deref() == Some("en") || (undecided && !prof_lat.looks_non_english);++ LanguageAnalysis {+ script: "latin".to_string(),+ script_profile: prof,+ language: lang,+ is_english: english,+ language_undecided: undecided,+ diacritic_rate: crate::calibration::Calibration::round4(prof_lat.diacritic_rate),+ non_latin_fraction: non_latin,+ }+ }++ /// True when the English checkpoint can be expected to read this state.+ pub fn is_english(state: &serde_json::Value) -> bool {+ Self::analyse(state).is_english+ }+}++/// The evidence [`LanguageDetection::latin_profile`] gathers for a Latin-script language guess.+#[derive(Debug, Clone, PartialEq)]+pub struct LatinProfile {+ /// The guessed language code, or `None` when undecided.+ pub language: Option<String>,+ /// How many English stopword hits the text had.+ pub english_hits: i64,+ /// Fraction of (lowercased) characters that are non-English Latin diacritics.+ pub diacritic_rate: f64,+ /// Whether the diacritic rate alone is high enough to call the text non-English.+ pub looks_non_english: bool,+}diff --git a/laya-rust/tests/language_detection_tests.rs b/laya-rust/tests/language_detection_tests.rsnew file mode 100644index 00000000..6743ef89--- /dev/null+++ b/laya-rust/tests/language_detection_tests.rs@@ -0,0 +1,456 @@+//! Tier 1: language/script detection. Ports the language-detection checks from+//! `tests/test_router.py` (no model weights needed) plus every entry of the routing golden's+//! `lang_probe.json`.++mod common;++use common::routing_golden;+use laya::language_detection::LanguageDetection;+use serde_json::json;+use std::collections::HashMap;++fn approx(a: f64, b: f64) {+ assert!((a - b).abs() < 1e-9, "expected {b}, got {a}");+}++fn profile_from_json(v: &serde_json::Value) -> HashMap<String, f64> {+ v.as_object()+ .expect("script_profile must be an object")+ .iter()+ .map(|(k, val)| {+ (+ k.clone(),+ val.as_f64().expect("profile value must be a number"),+ )+ })+ .collect()+}++// ---------------------------------------------------------------------- named cases (test_router.py)++#[test]+fn detect_script_matches_named_cases() {+ let cases: [(&str, &str, &str); 17] = [+ (+ "english",+ "The customer was charged twice and wants a refund.",+ "latin",+ ),+ ("armenian", "Հայերեն", "armenian"),+ ("armenian uppercase", "ՀԱՅԵՐԵՆ", "armenian"),+ ("armenian punctuation only", "։֊", "unknown"),+ (+ "french",+ "Le client a été facturé deux fois et demande un remboursement.",+ "latin",+ ),+ (+ "hindi",+ "ग्राहक से दो बार शुल्क लिया गया और वह धनवापसी चाहता है।",+ "devanagari",+ ),+ (+ "japanese",+ "お客様は二重に請求されたため返金を希望しています。",+ "kana",+ ),+ ("chinese", "客户被重复扣款要求退款", "han"),+ ("korean", "고객이 두 번 청구되어 환불을 원합니다", "hangul"),+ (+ "arabic",+ "تم خصم المبلغ مرتين من العميل ويريد استرداد الأموال",+ "arabic",+ ),+ ("tamil", "வாடிக்கையாளரிடம் இருமுறை கட்டணம் வசூலிக்கப்பட்டது", "tamil"),+ (+ "russian",+ "С клиента дважды сняли деньги и он хочет возврат",+ "cyrillic",+ ),+ ("thai", "ลูกค้าถูกเรียกเก็บเงินสองครั้งและต้องการเงินคืน", "thai"),+ (+ "greek",+ "Ο πελάτης χρεώθηκε δύο φορές και θέλει επιστροφή χρημάτων",+ "greek",+ ),+ ("hebrew", "הלקוח חויב פעמיים ורוצה החזר כספי", "hebrew"),+ ("empty", "", "unknown"),+ ("digits only", "12345 6789", "unknown"),+ ];+ for (label, text, want) in cases {+ assert_eq!(+ LanguageDetection::detect_script(text),+ want,+ "script/{label}"+ );+ }+}++#[test]+fn is_english_matches_named_cases() {+ let cases: [(&str, &str, bool); 16] = [+ (+ "plain english",+ "Please refund the duplicate charge on invoice 4411 today.",+ true,+ ),+ ("armenian", "Հայերեն", false),+ ("english short", "refund me", true),+ ("hindi", "ग्राहक से दो बार शुल्क लिया गया", false),+ ("japanese", "お客様は二重に請求されました", false),+ ("russian", "С клиента дважды сняли деньги", false),+ (+ "french long",+ "Le client a été facturé deux fois et il demande un remboursement pour la \+ facture qui a été payée le mois dernier avec la carte de crédit",+ false,+ ),+ (+ "german long",+ "Der Kunde wurde zweimal belastet und möchte eine Rückerstattung für die \+ Rechnung die nicht korrekt ist und auch nicht bezahlt wurde",+ false,+ ),+ (+ "romanian",+ "Gătește-mi o rețetă de sarmale de post pentru mâine.",+ false,+ ),+ (+ "romanian invoice",+ "Am fost taxat de două ori pentru factura din luna martie și vreau banii",+ false,+ ),+ (+ "polish",+ "Klient został obciążony dwukrotnie i chce zwrot pieniędzy za fakturę",+ false,+ ),+ (+ "czech",+ "Zákazníkovi byla částka účtována dvakrát a žádá o vrácení peněz",+ false,+ ),+ (+ "turkish",+ "Müşteriden iki kez ücret alındı ve para iadesi istiyor lütfen yardım",+ false,+ ),+ (+ "vietnamese",+ "Khách hàng đã bị thu phí hai lần và muốn được hoàn tiền ngay",+ false,+ ),+ (+ "english with loanwords",+ "We visited a cafe in Zurich and the naive assumption about the \+ invoice was wrong, so please refund the duplicate charge",+ true,+ ),+ ("empty", "", true),+ ];+ for (label, text, want) in cases {+ let v = json!(text);+ assert_eq!(+ LanguageDetection::is_english(&v),+ want,+ "is_english/{label}"+ );+ }+}++#[test]+fn latin_profile_undecided_reporting() {+ let undecided = json!("Müşteriden iki kez ücret alındı ve para iadesi istiyor");+ let a = LanguageDetection::analyse(&undecided);+ assert!(a.language_undecided, "latin/undecided is flagged");+ assert_eq!(a.language, None, "latin/undecided names no language");++ let english = json!("Please refund the duplicate charge on the invoice");+ assert!(+ !LanguageDetection::analyse(&english).language_undecided,+ "latin/english is not undecided"+ );++ let romanian = json!("Gătește-mi o rețetă de sarmale");+ assert!(+ LanguageDetection::analyse(&romanian).diacritic_rate > 0.02,+ "latin/diacritic rate reported"+ );++ let plain_english = json!("Please refund the duplicate charge today");+ approx(+ LanguageDetection::analyse(&plain_english).diacritic_rate,+ 0.0,+ );++ // A 0-0 tie between non-English stopword lists is no evidence for any of them.+ assert_eq!(+ LanguageDetection::guess_latin_language("Cât e ora acum la Tokyo"),+ None,+ "latin_lang/zero tie invents nothing"+ );++ // Known limitation, kept visible on purpose (see laya/lang.py and issue #35): short Romanian+ // with no diacritics and an English function word still reads as English.+ assert!(LanguageDetection::is_english(&json!(+ "Care este ora in Tokyo?"+ )));+}++#[test]+fn guess_latin_language_matches_named_cases() {+ let cases: [(&str, &str, Option<&str>); 5] = [+ (+ "english",+ "The customer was charged twice and wants a refund for this invoice",+ Some("en"),+ ),+ (+ "french",+ "Le client a ete facture deux fois et il demande un remboursement pour la facture",+ Some("fr"),+ ),+ (+ "german",+ "Der Kunde wurde zweimal belastet und moechte eine Rueckerstattung fuer die Rechnung",+ Some("de"),+ ),+ (+ "spanish",+ "El cliente fue cobrado dos veces y quiere que le devuelvan el dinero por la factura",+ Some("es"),+ ),+ ("too short", "refund", None),+ ];+ for (label, text, want) in cases {+ assert_eq!(+ LanguageDetection::guess_latin_language(text).as_deref(),+ want,+ "latin_lang/{label}"+ );+ }+ assert_eq!(+ LanguageDetection::guess_latin_language(+ "Please refund the duplicate charge on invoice 4411 today because \+ we have been waiting for three days and nobody has replied to us"+ )+ .as_deref(),+ Some("en"),+ "latin_lang/long english stays en"+ );+}++#[test]+fn state_text_flattens_nested_state() {+ let dict_state = json!({"body": "charged twice", "n": 3});+ assert!(LanguageDetection::state_text_default(&dict_state).contains("charged twice"));++ let nested = json!({"a": {"b": ["deep"]}});+ assert!(LanguageDetection::state_text_default(&nested).contains("deep"));++ let list_state = json!(["x", {"y": "z"}]);+ assert!(LanguageDetection::state_text_default(&list_state).contains("x"));++ assert_eq!(LanguageDetection::state_text_default(&json!(null)), "");++ // Keys must not drive detection: English keys around Hindi content stay non-English.+ let keyed = json!({"subject": "नमस्ते", "body": "ग्राहक से दो बार शुल्क लिया गया"});+ assert!(+ !LanguageDetection::is_english(&keyed),+ "state_text/keys ignored"+ );+}++#[test]+fn script_profile_named_cases() {+ let armenian = LanguageDetection::analyse(&json!("Հայերեն"));+ assert_eq!(armenian.script_profile.len(), 1);+ approx(*armenian.script_profile.get("armenian").unwrap(), 1.0);++ let mixed = LanguageDetection::analyse(&json!("Հայերեն abc"));+ approx(mixed.non_latin_fraction, 0.7);+}++#[test]+fn plain_ascii_romance_named_cases() {+ // #172: ASCII-stripped Romance text carries no diacritic rate, so the function-word lists are+ // the only evidence left.+ let cases: [(&str, &str); 10] = [+ (+ "es",+ "El pedido llego roto y nadie responde cuando escribo al soporte",+ ),+ ("es", "Quiero cancelar mi plan y pedir un reembolso"),+ ("es", "La factura tiene un error en el importe total"),+ (+ "es",+ "Necesito que me devuelvan el dinero de la compra duplicada",+ ),+ (+ "it",+ "Il cliente e stato addebitato due volte e vuole un rimborso",+ ),+ (+ "it",+ "Voglio cancellare il mio abbonamento e chiedere un rimborso",+ ),+ ("it", "La fattura contiene un errore nell importo totale"),+ (+ "pt",+ "O cliente foi cobrado duas vezes e quer o dinheiro de volta",+ ),+ (+ "fr",+ "Le client a ete facture deux fois et demande un remboursement",+ ),+ (+ "fr",+ "Je ne peux pas acceder a mon compte et j ai besoin d aide",+ ),+ ];+ for (lang, text) in cases {+ assert_eq!(+ LanguageDetection::guess_latin_language(text).as_deref(),+ Some(lang),+ "latin_lang/plain ascii {lang} {text}"+ );+ assert!(+ !LanguageDetection::is_english(&json!(text)),+ "is_english/plain ascii {lang} {text}"+ );+ }++ for text in [+ "The customer was charged twice and wants a refund for this invoice",+ "Please cancel my subscription and refund the duplicate charge today",+ "The report by Smith et al. shows the de facto standard, e.g. the LA office and Rio",+ "Our MI5 and UN contacts discussed the DOS attack in LA last month",+ "No refund was issued, so I am writing to you again about invoice 4411",+ "no refund no reply",+ "The son of the director filed a complaint about the duplicate invoice",+ ] {+ assert!(+ LanguageDetection::is_english(&json!(text)),+ "is_english/romance control {text}"+ );+ }++ // A word several lists claim (`la`, `e`, `o`) says "not English" without saying *which*+ // language, so it may not name one on its own.+ assert_eq!(+ LanguageDetection::guess_latin_language("Cât e ora acum la Tokyo"),+ None,+ "latin_lang/shared words alone name nothing"+ );+ // ...but a distinctive word in the same state is enough to name the language it belongs to.+ assert_eq!(+ LanguageDetection::guess_latin_language(+ "La fattura contiene un errore nell importo totale"+ )+ .as_deref(),+ Some("it")+ );+ assert_eq!(+ LanguageDetection::guess_latin_language("La factura tiene un error en el importe total")+ .as_deref(),+ Some("es")+ );+}++// ---------------------------------------------------------------------- lang_probe.json++#[test]+fn lang_probe_matches_every_entry() {+ let Some(cases) = routing_golden::load_array("lang_probe.json") else {+ eprintln!("skip: routing golden not found (tests/golden/routing)");+ return;+ };++ for case in &cases {+ let label = case["label"].as_str().unwrap();+ let state = &case["state"];++ let expected_state_text = case["state_text"].as_str().unwrap();+ assert_eq!(+ LanguageDetection::state_text_default(state),+ expected_state_text,+ "{label}: state_text"+ );++ let text = LanguageDetection::state_text_default(state);+ assert_eq!(+ LanguageDetection::detect_script(&text),+ case["detect_script"].as_str().unwrap(),+ "{label}: detect_script"+ );++ let expected_profile = profile_from_json(&case["script_profile"]);+ let actual_profile = LanguageDetection::script_profile(&text);+ assert_eq!(+ actual_profile.len(),+ expected_profile.len(),+ "{label}: script_profile key count"+ );+ for (k, v) in &expected_profile {+ let got = actual_profile+ .get(k)+ .unwrap_or_else(|| panic!("{label}: script_profile missing key {k}"));+ approx(*got, *v);+ }++ let lp = &case["latin_profile"];+ let actual_lp = LanguageDetection::latin_profile(&text);+ assert_eq!(+ actual_lp.language.as_deref(),+ lp["language"].as_str(),+ "{label}: latin_profile.language"+ );+ assert_eq!(+ actual_lp.english_hits,+ lp["english_hits"].as_i64().unwrap(),+ "{label}: latin_profile.english_hits"+ );+ approx(+ actual_lp.diacritic_rate,+ lp["diacritic_rate"].as_f64().unwrap(),+ );+ assert_eq!(+ actual_lp.looks_non_english,+ lp["looks_non_english"].as_bool().unwrap(),+ "{label}: latin_profile.looks_non_english"+ );++ let a = &case["analyse"];+ let actual = LanguageDetection::analyse(state);+ assert_eq!(+ actual.script,+ a["script"].as_str().unwrap(),+ "{label}: analyse.script"+ );+ assert_eq!(+ actual.language.as_deref(),+ a["language"].as_str(),+ "{label}: analyse.language"+ );+ assert_eq!(+ actual.is_english,+ a["is_english"].as_bool().unwrap(),+ "{label}: analyse.is_english"+ );+ assert_eq!(+ actual.language_undecided,+ a["language_undecided"].as_bool().unwrap(),+ "{label}: analyse.language_undecided"+ );+ approx(actual.diacritic_rate, a["diacritic_rate"].as_f64().unwrap());+ approx(+ actual.non_latin_fraction,+ a["non_latin_fraction"].as_f64().unwrap(),+ );+ }+ eprintln!(+ "lang_probe_matches_every_entry: checked {} cases",+ cases.len()+ );+}