Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
fix(rust): follow Python routing and email golden drift
kevin-gatimu3e85cb40dd9b
diff --git a/laya-rust/CHANGELOG.md b/laya-rust/CHANGELOG.mdindex a76e6c61..e1edbaa6 100644--- a/laya-rust/CHANGELOG.md+++ b/laya-rust/CHANGELOG.md@@ -24,6 +24,7 @@ First release: a port of the Laya Python SDK, tested against laya (Python) 0.3.2### Known gaps-- Language detection and email handling follow Python 0.3.6 (`lang.py`, `email.py`).+- Routing, script counting and fused-line email disclaimers follow the regenerated Python 0.4.0+ probes. New mixed-field language heuristics are not yet ported.- Not yet ported from Python 0.3.21: abstention (`min_confidence`), `predict_long` and`lang_temperatures`.diff --git a/laya-rust/README.md b/laya-rust/README.mdindex f53ebb11..1298ca33 100644--- a/laya-rust/README.md+++ b/laya-rust/README.md@@ -647,12 +647,9 @@ Rust port groups the corresponding functions into a unit struct with associated## Known gaps-- **Language detection and email handling still follow Python 0.3.6, not 0.3.21.**- `src/language_detection.rs` (ports `lang.py`) and `src/laya_email.rs` (ports `email.py`) were- deliberately left untouched when the rest of this crate was brought up to 0.3.21, so- `LanguageDetection` and `LayaEmail` reproduce the 0.3.6 behavior of those two Python modules.- Everything else in this crate — sequence building, calibration, and the answer types — tracks- laya 0.3.21. Porting `lang.py` and `email.py` forward is planned as a follow-up.+- **Python 0.4.0's routing defaults, script counting and fused-line disclaimer fixes are ported**+ and checked against freshly regenerated probes. New mixed-field language heuristics remain+ a follow-up; these probes are not a claim of full coverage of every new Python feature.- **Opt-in features added in Python 0.3.21 are not ported yet:** abstention (`min_confidence`),`predict_long`, and per-language temperatures (`lang_temperatures`). They are off by default indiff --git a/laya-rust/src/language_detection.rs b/laya-rust/src/language_detection.rsindex 3b104457..bc7a4c47 100644--- a/laya-rust/src/language_detection.rs+++ b/laya-rust/src/language_detection.rs@@ -221,7 +221,7 @@ impl LanguageDetection {/// Whether `ch` is alphabetic under Python's `str.isalpha()` rule: Unicode general category/// `Lu`, `Ll`, `Lt`, `Lm` or `Lo`. Unlike `char::is_alphabetic`, this excludes combining/// marks.- fn is_alpha(ch: char) -> bool {+ pub(crate) fn is_alpha(ch: char) -> bool {matches!(get_general_category(ch),GeneralCategory::UppercaseLetter@@ -322,14 +322,11 @@ impl LanguageDetection {latin += 1;continue;}- if let Some(name) = Self::script_for_codepoint(cp) {- match counts.iter_mut().find(|(n, _)| *n == name) {- Some(entry) => entry.1 += 1,- None => counts.push((name, 1)),- }+ let name = Self::script_for_codepoint(cp).unwrap_or("other");+ match counts.iter_mut().find(|(n, _)| *n == name) {+ Some(entry) => entry.1 += 1,+ None => counts.push((name, 1)),}- // A letter outside every range and outside Latin (Deseret, astral scripts, ...)- // counts toward nothing, matching Python exactly.}counts.push(("latin", latin));@@ -359,9 +356,8 @@ impl LanguageDetection {*counts.get_mut("latin").unwrap() += 1;continue;}- if let Some(name) = Self::script_for_codepoint(cp) {- *counts.entry(name).or_insert(0) += 1;- }+ let name = Self::script_for_codepoint(cp).unwrap_or("other");+ *counts.entry(name).or_insert(0) += 1;}let total: i64 = counts.values().sum();if total == 0 {@@ -501,16 +497,70 @@ impl LanguageDetection {Self::latin_profile(text).language}+ /// Non-Latin words, excluding one-letter symbols and capitalised proper names.+ fn has_non_latin_words(text: &str) -> bool {+ let mut current = String::new();+ let mut script = None;+ let is_word = |word: &str| {+ word.chars().count() >= 2+ && word.chars().next().is_some_and(|c| !c.is_uppercase())+ };+ for ch in text.chars() {+ if matches!(get_general_category(ch), GeneralCategory::NonspacingMark+ | GeneralCategory::SpacingMark | GeneralCategory::EnclosingMark)+ {+ continue;+ }+ let next = if Self::is_alpha(ch) && !Self::is_latin_codepoint(ch as u32) {+ Self::script_for_codepoint(ch as u32)+ } else {+ None+ };+ if next.is_some() && next == script {+ current.push(ch);+ continue;+ }+ if is_word(¤t) {+ return true;+ }+ current.clear();+ script = next;+ if next.is_some() {+ current.push(ch);+ }+ }+ is_word(¤t)+ }+/// Full detection result for a state. See [`LanguageAnalysis`].pub fn analyse(state: &serde_json::Value) -> LanguageAnalysis {let text = Self::state_text_default(state);let prof = Self::script_profile(&text);- let script = Self::detect_script(&text);+ let mut script = Self::detect_script(&text);let non_latin = if prof.is_empty() {0.0} else {crate::calibration::Calibration::round4(1.0 - prof.get("latin").copied().unwrap_or(0.0))};+ let n_non_latin = (non_latin * text.chars().filter(|&c| Self::is_alpha(c)).count() as f64)+ .round_ties_even();+ if script == "latin" && Self::has_non_latin_words(&text)+ && (non_latin >= 0.2 || (non_latin >= 0.1 && n_non_latin >= 10.0))+ {+ // Preserve Python's first-seen tie order, rather than HashMap iteration order.+ let mut best = 0.0;+ for ch in text.chars().filter(|&c| Self::is_alpha(c)) {+ if Self::is_latin_codepoint(ch as u32) {+ continue;+ }+ let name = Self::script_for_codepoint(ch as u32).unwrap_or("other");+ let fraction = prof.get(name).copied().unwrap_or(0.0);+ if fraction > best {+ script = name.to_string();+ best = fraction;+ }+ }+ }if script == "unknown" {return LanguageAnalysis {diff --git a/laya-rust/src/laya_email.rs b/laya-rust/src/laya_email.rsindex 525faba8..d444a950 100644--- a/laya-rust/src/laya_email.rs+++ b/laya-rust/src/laya_email.rs@@ -143,6 +143,29 @@ impl LayaEmail {.into_iter().map(|p| p.trim().to_string()).filter(|p| !p.is_empty())+ .flat_map(|part| {+ if !DISCLAIMER.is_match(&part) || !part.contains('\n') {+ return vec![part];+ }+ let mut pieces = Vec::new();+ let mut buf = String::new();+ for line in part.lines().map(str::trim).filter(|l| !l.is_empty()) {+ let starts_sentence = line.chars().find(|&c| {+ crate::language_detection::LanguageDetection::is_alpha(c)+ }).is_some_and(char::is_uppercase);+ if !buf.is_empty() && starts_sentence {+ pieces.push(std::mem::take(&mut buf));+ }+ if !buf.is_empty() {+ buf.push(' ');+ }+ buf.push_str(line);+ }+ if !buf.is_empty() {+ pieces.push(buf);+ }+ pieces+ }).collect();parts.into_iter()diff --git a/laya-rust/src/laya_router.rs b/laya-rust/src/laya_router.rsindex afd1452e..ad3ab50b 100644--- a/laya-rust/src/laya_router.rs+++ b/laya-rust/src/laya_router.rs@@ -146,8 +146,8 @@ pub struct LayaRouterOptions {/// evicted. Always at least 1.pub max_loaded: usize,/// Checkpoint used when [`LayaRouter::route`] cannot find any letters in the state at all.- /// Python's own default is English, kept here for parity even though it means an empty state- /// is sent to the checkpoint that cannot read non-Latin scripts.+ /// Also used for undecided Latin text. Multilingual by default, matching Python 0.4.0;+ /// mostly-English deployments may explicitly select English.pub default: LayaCheckpoint,/// Whether a question-id set that exactly matches one of the four typed-decisions workflow/// signatures should automatically route to `TypedDecisions`. Off by default: that checkpoint@@ -163,7 +163,7 @@ impl Default for LayaRouterOptions {Self {engine_options: LayaOptions::default(),max_loaded: 1,- default: LayaCheckpoint::English,+ default: LayaCheckpoint::Multilingual,auto_task_detection: false,preload: false,}@@ -513,6 +513,15 @@ impl LayaRouter {),)}+ } else if det.language_undecided {+ (+ self.default,+ format!(+ "Latin script, language not identified and no non-English letters; \+ using default ({})",+ self.default.subdir()+ ),+ )} else {(LayaCheckpoint::English, "English Latin text".to_string())};diff --git a/laya-rust/tests/laya_email_tests.rs b/laya-rust/tests/laya_email_tests.rsindex f614ad0a..9513fa6a 100644--- a/laya-rust/tests/laya_email_tests.rs+++ b/laya-rust/tests/laya_email_tests.rs@@ -25,7 +25,7 @@ fn inline_footer_keeps_the_request() {&format!("My account is locked\n{DISCLAIMER}\nPlease unlock it."),3000),- "Please unlock it."+ "My account is locked Please unlock it.");assert_eq!(LayaEmail::clean_body(&format!("My account is locked. {DISCLAIMER}"), 3000),