Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

fix(rust): follow Python routing and email golden drift

kevin-gatimu3e85cb40dd9b
diff --git a/laya-rust/CHANGELOG.md b/laya-rust/CHANGELOG.md
index a76e6c61..e1edbaa6 100644
--- a/laya-rust/CHANGELOG.md
+++ b/laya-rust/CHANGELOG.md
@@ -24,6 +24,7 @@ First release: a port of the Laya Python SDK, tested against laya (Python) 0.3.2
### Known gaps
-- Language detection and email handling follow Python 0.3.6 (`lang.py`, `email.py`).
+- Routing, script counting and fused-line email disclaimers follow the regenerated Python 0.4.0
+ probes. New mixed-field language heuristics are not yet ported.
- Not yet ported from Python 0.3.21: abstention (`min_confidence`), `predict_long` and
`lang_temperatures`.
diff --git a/laya-rust/README.md b/laya-rust/README.md
index f53ebb11..1298ca33 100644
--- a/laya-rust/README.md
+++ b/laya-rust/README.md
@@ -647,12 +647,9 @@ Rust port groups the corresponding functions into a unit struct with associated
## Known gaps
-- **Language detection and email handling still follow Python 0.3.6, not 0.3.21.**
- `src/language_detection.rs` (ports `lang.py`) and `src/laya_email.rs` (ports `email.py`) were
- deliberately left untouched when the rest of this crate was brought up to 0.3.21, so
- `LanguageDetection` and `LayaEmail` reproduce the 0.3.6 behavior of those two Python modules.
- Everything else in this crate — sequence building, calibration, and the answer types — tracks
- laya 0.3.21. Porting `lang.py` and `email.py` forward is planned as a follow-up.
+- **Python 0.4.0's routing defaults, script counting and fused-line disclaimer fixes are ported**
+ and checked against freshly regenerated probes. New mixed-field language heuristics remain
+ a follow-up; these probes are not a claim of full coverage of every new Python feature.
- **Opt-in features added in Python 0.3.21 are not ported yet:** abstention (`min_confidence`),
`predict_long`, and per-language temperatures (`lang_temperatures`). They are off by default in
diff --git a/laya-rust/src/language_detection.rs b/laya-rust/src/language_detection.rs
index 3b104457..bc7a4c47 100644
--- a/laya-rust/src/language_detection.rs
+++ b/laya-rust/src/language_detection.rs
@@ -221,7 +221,7 @@ impl LanguageDetection {
/// Whether `ch` is alphabetic under Python's `str.isalpha()` rule: Unicode general category
/// `Lu`, `Ll`, `Lt`, `Lm` or `Lo`. Unlike `char::is_alphabetic`, this excludes combining
/// marks.
- fn is_alpha(ch: char) -> bool {
+ pub(crate) fn is_alpha(ch: char) -> bool {
matches!(
get_general_category(ch),
GeneralCategory::UppercaseLetter
@@ -322,14 +322,11 @@ impl LanguageDetection {
latin += 1;
continue;
}
- if let Some(name) = Self::script_for_codepoint(cp) {
- match counts.iter_mut().find(|(n, _)| *n == name) {
- Some(entry) => entry.1 += 1,
- None => counts.push((name, 1)),
- }
+ let name = Self::script_for_codepoint(cp).unwrap_or("other");
+ match counts.iter_mut().find(|(n, _)| *n == name) {
+ Some(entry) => entry.1 += 1,
+ None => counts.push((name, 1)),
}
- // A letter outside every range and outside Latin (Deseret, astral scripts, ...)
- // counts toward nothing, matching Python exactly.
}
counts.push(("latin", latin));
@@ -359,9 +356,8 @@ impl LanguageDetection {
*counts.get_mut("latin").unwrap() += 1;
continue;
}
- if let Some(name) = Self::script_for_codepoint(cp) {
- *counts.entry(name).or_insert(0) += 1;
- }
+ let name = Self::script_for_codepoint(cp).unwrap_or("other");
+ *counts.entry(name).or_insert(0) += 1;
}
let total: i64 = counts.values().sum();
if total == 0 {
@@ -501,16 +497,70 @@ impl LanguageDetection {
Self::latin_profile(text).language
}
+ /// Non-Latin words, excluding one-letter symbols and capitalised proper names.
+ fn has_non_latin_words(text: &str) -> bool {
+ let mut current = String::new();
+ let mut script = None;
+ let is_word = |word: &str| {
+ word.chars().count() >= 2
+ && word.chars().next().is_some_and(|c| !c.is_uppercase())
+ };
+ for ch in text.chars() {
+ if matches!(get_general_category(ch), GeneralCategory::NonspacingMark
+ | GeneralCategory::SpacingMark | GeneralCategory::EnclosingMark)
+ {
+ continue;
+ }
+ let next = if Self::is_alpha(ch) && !Self::is_latin_codepoint(ch as u32) {
+ Self::script_for_codepoint(ch as u32)
+ } else {
+ None
+ };
+ if next.is_some() && next == script {
+ current.push(ch);
+ continue;
+ }
+ if is_word(&current) {
+ return true;
+ }
+ current.clear();
+ script = next;
+ if next.is_some() {
+ current.push(ch);
+ }
+ }
+ is_word(&current)
+ }
+
/// Full detection result for a state. See [`LanguageAnalysis`].
pub fn analyse(state: &serde_json::Value) -> LanguageAnalysis {
let text = Self::state_text_default(state);
let prof = Self::script_profile(&text);
- let script = Self::detect_script(&text);
+ let mut script = Self::detect_script(&text);
let non_latin = if prof.is_empty() {
0.0
} else {
crate::calibration::Calibration::round4(1.0 - prof.get("latin").copied().unwrap_or(0.0))
};
+ let n_non_latin = (non_latin * text.chars().filter(|&c| Self::is_alpha(c)).count() as f64)
+ .round_ties_even();
+ if script == "latin" && Self::has_non_latin_words(&text)
+ && (non_latin >= 0.2 || (non_latin >= 0.1 && n_non_latin >= 10.0))
+ {
+ // Preserve Python's first-seen tie order, rather than HashMap iteration order.
+ let mut best = 0.0;
+ for ch in text.chars().filter(|&c| Self::is_alpha(c)) {
+ if Self::is_latin_codepoint(ch as u32) {
+ continue;
+ }
+ let name = Self::script_for_codepoint(ch as u32).unwrap_or("other");
+ let fraction = prof.get(name).copied().unwrap_or(0.0);
+ if fraction > best {
+ script = name.to_string();
+ best = fraction;
+ }
+ }
+ }
if script == "unknown" {
return LanguageAnalysis {
diff --git a/laya-rust/src/laya_email.rs b/laya-rust/src/laya_email.rs
index 525faba8..d444a950 100644
--- a/laya-rust/src/laya_email.rs
+++ b/laya-rust/src/laya_email.rs
@@ -143,6 +143,29 @@ impl LayaEmail {
.into_iter()
.map(|p| p.trim().to_string())
.filter(|p| !p.is_empty())
+ .flat_map(|part| {
+ if !DISCLAIMER.is_match(&part) || !part.contains('\n') {
+ return vec![part];
+ }
+ let mut pieces = Vec::new();
+ let mut buf = String::new();
+ for line in part.lines().map(str::trim).filter(|l| !l.is_empty()) {
+ let starts_sentence = line.chars().find(|&c| {
+ crate::language_detection::LanguageDetection::is_alpha(c)
+ }).is_some_and(char::is_uppercase);
+ if !buf.is_empty() && starts_sentence {
+ pieces.push(std::mem::take(&mut buf));
+ }
+ if !buf.is_empty() {
+ buf.push(' ');
+ }
+ buf.push_str(line);
+ }
+ if !buf.is_empty() {
+ pieces.push(buf);
+ }
+ pieces
+ })
.collect();
parts
.into_iter()
diff --git a/laya-rust/src/laya_router.rs b/laya-rust/src/laya_router.rs
index afd1452e..ad3ab50b 100644
--- a/laya-rust/src/laya_router.rs
+++ b/laya-rust/src/laya_router.rs
@@ -146,8 +146,8 @@ pub struct LayaRouterOptions {
/// evicted. Always at least 1.
pub max_loaded: usize,
/// Checkpoint used when [`LayaRouter::route`] cannot find any letters in the state at all.
- /// Python's own default is English, kept here for parity even though it means an empty state
- /// is sent to the checkpoint that cannot read non-Latin scripts.
+ /// Also used for undecided Latin text. Multilingual by default, matching Python 0.4.0;
+ /// mostly-English deployments may explicitly select English.
pub default: LayaCheckpoint,
/// Whether a question-id set that exactly matches one of the four typed-decisions workflow
/// signatures should automatically route to `TypedDecisions`. Off by default: that checkpoint
@@ -163,7 +163,7 @@ impl Default for LayaRouterOptions {
Self {
engine_options: LayaOptions::default(),
max_loaded: 1,
- default: LayaCheckpoint::English,
+ default: LayaCheckpoint::Multilingual,
auto_task_detection: false,
preload: false,
}
@@ -513,6 +513,15 @@ impl LayaRouter {
),
)
}
+ } else if det.language_undecided {
+ (
+ self.default,
+ format!(
+ "Latin script, language not identified and no non-English letters; \
+ using default ({})",
+ self.default.subdir()
+ ),
+ )
} else {
(LayaCheckpoint::English, "English Latin text".to_string())
};
diff --git a/laya-rust/tests/laya_email_tests.rs b/laya-rust/tests/laya_email_tests.rs
index f614ad0a..9513fa6a 100644
--- a/laya-rust/tests/laya_email_tests.rs
+++ b/laya-rust/tests/laya_email_tests.rs
@@ -25,7 +25,7 @@ fn inline_footer_keeps_the_request() {
&format!("My account is locked\n{DISCLAIMER}\nPlease unlock it."),
3000
),
- "Please unlock it."
+ "My account is locked Please unlock it."
);
assert_eq!(
LayaEmail::clean_body(&format!("My account is locked. {DISCLAIMER}"), 3000),