Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

feat(rust): add checkpoint router

kevin-gatimu079518aad69d
diff --git a/laya-rust/src/laya_router.rs b/laya-rust/src/laya_router.rs
new file mode 100644
index 00000000..afd1452e
--- /dev/null
+++ b/laya-rust/src/laya_router.rs
@@ -0,0 +1,589 @@
+//! Routes a request to the Laya checkpoint best suited to it. Ports `laya/router.py`.
+//!
+//! Three checkpoints, measured on a shared benchmark (17,416 questions, one T4, identical
+//! questions per model):
+//!
+//! ```text
+//! english convaiinnovations/laya 421M ModernBERT-large, 512 tokens
+//! multilingual convaiinnovations/laya-multilingual 322M mmBERT-base, 1024 tokens, 100+ langs
+//! typed-decisions convaiinnovations/laya-typed-decisions 421M ModernBERT-large, 1024 tokens,
+//! fine-tuned on the typed-decisions
+//! workflows
+//! ```
+//!
+//! The English checkpoint does not gently degrade off English, it collapses: on 20-option
+//! MASSIVE intent it scores 0.100 on Hindi and 0.103 on Korean, against 0.050 for random
+//! guessing -- and it reports high confidence while doing so. Script detection is therefore the
+//! primary routing signal; see [`crate::language_detection`].
+//!
+//! `typed-decisions` is never selected automatically unless the caller opts in with
+//! [`LayaRouterOptions::auto_task_detection`] or passes `task = Some("typed_decisions")`: it is
+//! fine-tuned on four specific synthetic workflows and should not be a silent default.
+//!
+//! ## Eviction safety
+//!
+//! Python relies on garbage collection: an evicted `Agent` is simply dropped from `self._agents`
+//! and freed once nothing else references it, including an in-flight `system_one` call. Rust has
+//! no GC, so every loaded engine here is held as `Arc<dyn LayaPredictor>`: [`Self::predict`]
+//! clones the `Arc` out from under the router lock before running inference, so an eviction that
+//! happens mid-predict only drops the router's own reference — the engine itself is freed only
+//! once that in-flight predict's clone is also dropped. Inference therefore never runs while
+//! holding the router's lock, matching Python's `Agent.system_one` being called outside
+//! `self._lock`.
+//!
+//! Building an engine (unlike inference) *does* happen under the lock, exactly as Python's
+//! `load()` holds `self._lock` for the whole `Agent(...)` construction: this is what makes
+//! concurrent `Load` of the same checkpoint build the engine exactly once (every later caller
+//! blocks on the lock, then sees the checkpoint already resident).
+
+use crate::answers::LayaResult;
+use crate::error::{LayaError, Result};
+use crate::language_detection::{LanguageAnalysis, LanguageDetection};
+use crate::laya_engine::LayaEngine;
+use crate::laya_options::{LayaCheckpoint, LayaOptions};
+use crate::questions::QuestionSet;
+use serde_json::Value;
+use std::collections::{HashMap, HashSet};
+use std::sync::{Arc, Mutex};
+
+/// Answers typed questions about a piece of state in one forward pass. Implemented by
+/// [`LayaEngine`] and [`LayaRouter`] so `laya_shortlist` (and any other caller) can work over
+/// either without knowing which one it has — the Rust equivalent of Python's `_call_predict`
+/// duck-typing (`getattr(agent, "predict", None) or getattr(agent, "system_one", None)`).
+///
+/// Takes `state: Value` rather than `impl Into<Value>` (unlike [`LayaEngine::predict`]) so the
+/// trait stays object-safe: `LayaRouter` stores its loaded checkpoints as `Arc<dyn
+/// LayaPredictor>`, and a generic method cannot be part of a trait object's vtable. Callers with
+/// an `impl Into<Value>` state can convert with `.into()` at the call site.
+pub trait LayaPredictor: Send + Sync {
+ /// Answer every question in `questions` about `state`.
+ fn predict(&self, state: Value, questions: &QuestionSet) -> Result<LayaResult>;
+}
+
+impl LayaPredictor for LayaEngine {
+ fn predict(&self, state: Value, questions: &QuestionSet) -> Result<LayaResult> {
+ LayaEngine::predict(self, state, questions)
+ }
+}
+
+/// Question-id signatures of the four typed-decisions workflows, used only when
+/// [`LayaRouterOptions::auto_task_detection`] is enabled.
+const TYPED_DECISION_WORKFLOWS: &[(&str, &[&str])] = &[
+ (
+ "agent_trace_observability",
+ &["action", "needs_review", "outcome", "risk", "urgency"],
+ ),
+ (
+ "customer_service",
+ &["action", "category", "churn_risk", "needs_human", "urgency"],
+ ),
+ (
+ "invoice_processing",
+ &[
+ "discrepancy_severity",
+ "disposition",
+ "duplicate",
+ "matches_order",
+ "urgency",
+ ],
+ ),
+ (
+ "security_incidents",
+ &[
+ "credential_compromise",
+ "disposition",
+ "severity",
+ "true_positive",
+ "urgency",
+ ],
+ ),
+];
+
+/// The routing outcome: which checkpoint, why, and what was detected. Serializes into
+/// `LayaResult::routing` (`answers.rs`). Python's `RouteDecision` also carries a `repo` field
+/// (the resolved HuggingFace repo/subfolder string); it is intentionally not ported, since this
+/// crate's [`LayaRouterOptions`] already resolves artifacts per checkpoint through the same
+/// [`crate::model_artifacts::ModelArtifacts`] path every other engine uses, so there is no
+/// separate "which repo string" question to answer here.
+#[derive(Debug, Clone, PartialEq)]
+pub struct RouteDecision {
+ /// The chosen checkpoint.
+ pub model: LayaCheckpoint,
+ /// Human-readable reason, byte-identical to Python's (including `%r`-style quoting and
+ /// `%.0f%%` percentages) so the two are directly comparable in logs.
+ pub reason: String,
+ /// The script/language detection this decision was based on, or `None` when the model was
+ /// chosen without looking at the state at all (an explicit `model`/`task`, or an opted-in
+ /// typed-decisions workflow match).
+ pub detection: Option<LanguageAnalysis>,
+ /// The typed-decisions workflow the question ids matched, if any — populated whenever a
+ /// match exists, even if [`LayaRouterOptions::auto_task_detection`] was off and the match
+ /// therefore did not decide the routing.
+ pub workflow: Option<String>,
+}
+
+impl std::fmt::Display for RouteDecision {
+ /// Matches Python's `RouteDecision.__repr__`.
+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+ write!(
+ f,
+ "RouteDecision(model={}, reason={})",
+ python_repr_str(self.model.subdir()),
+ python_repr_str(&self.reason)
+ )
+ }
+}
+
+/// Configuration for [`LayaRouter`]. `engine_options` is the template every loaded checkpoint's
+/// [`LayaOptions`] is built from (provider, threads, download settings, cache, HF token); its own
+/// `checkpoint` field is ignored, since each load overrides it with the checkpoint being built.
+pub struct LayaRouterOptions {
+ /// Template engine options shared by every checkpoint the router loads. Only the
+ /// `checkpoint` field varies per load; every other field (execution provider, thread counts,
+ /// `allow_download`, cache directory, HF token) is copied as-is.
+ pub engine_options: LayaOptions,
+ /// How many checkpoints may stay resident at once before the least-recently-used one is
+ /// evicted. Always at least 1.
+ pub max_loaded: usize,
+ /// Checkpoint used when [`LayaRouter::route`] cannot find any letters in the state at all.
+ /// Python's own default is English, kept here for parity even though it means an empty state
+ /// is sent to the checkpoint that cannot read non-Latin scripts.
+ pub default: LayaCheckpoint,
+ /// Whether a question-id set that exactly matches one of the four typed-decisions workflow
+ /// signatures should automatically route to `TypedDecisions`. Off by default: that checkpoint
+ /// is fine-tuned on four specific synthetic workflows and should never be a silent default.
+ pub auto_task_detection: bool,
+ /// Build and load every checkpoint immediately, so no request ever pays a cold load. See
+ /// [`LayaRouter::preload`].
+ pub preload: bool,
+}
+
+impl Default for LayaRouterOptions {
+ fn default() -> Self {
+ Self {
+ engine_options: LayaOptions::default(),
+ max_loaded: 1,
+ default: LayaCheckpoint::English,
+ auto_task_detection: false,
+ preload: false,
+ }
+ }
+}
+
+/// Builds the [`LayaPredictor`] for one checkpoint. The production path
+/// ([`LayaRouter::new`]) always builds a real [`LayaEngine`]; tests substitute a factory that
+/// returns a fake predictor with no ONNX weights via [`LayaRouter::with_factory`].
+type Factory =
+ Box<dyn Fn(LayaCheckpoint, &LayaOptions) -> Result<Arc<dyn LayaPredictor>> + Send + Sync>;
+
+struct RouterState {
+ loaded: HashMap<LayaCheckpoint, Arc<dyn LayaPredictor>>,
+ /// Least-recently-used first.
+ order: Vec<LayaCheckpoint>,
+ max_loaded: usize,
+}
+
+/// Lazily loads Laya checkpoints and sends each request to the right one. See the module docs.
+///
+/// ```no_run
+/// use laya::laya_router::{LayaRouter, LayaRouterOptions};
+///
+/// let router = LayaRouter::new(LayaRouterOptions::default())?;
+/// // router.predict(state, &questions, None, None, None)?; // routes, then answers
+/// # Ok::<(), laya::error::LayaError>(())
+/// ```
+pub struct LayaRouter {
+ engine_options: LayaOptions,
+ default: LayaCheckpoint,
+ auto_task_detection: bool,
+ factory: Factory,
+ state: Mutex<RouterState>,
+}
+
+impl LayaRouter {
+ /// Build a router whose checkpoints are real [`LayaEngine`]s, built with
+ /// [`LayaEngine::create`] on first use.
+ pub fn new(options: LayaRouterOptions) -> Result<Self> {
+ Self::with_factory(options, |checkpoint, opts| {
+ let mut opts = opts.clone();
+ opts.checkpoint = checkpoint;
+ let engine = LayaEngine::create(opts)?;
+ Ok(Arc::new(engine) as Arc<dyn LayaPredictor>)
+ })
+ }
+
+ /// A test-only engine-factory seam: builds a router whose checkpoints come from `factory`
+ /// instead of a real [`LayaEngine`], so LRU/attach/preload/unload tests can run with fake
+ /// predictors and no ~1.3-1.7 GB weights. `factory` receives the checkpoint being loaded and
+ /// the (already checkpoint-substituted) engine options template.
+ ///
+ /// `#[doc(hidden)]` rather than crate-private: Rust has no visibility scoped to "this crate's
+ /// own integration tests", and the integration test suite is a separate crate that links
+ /// against this one, so the seam must be `pub` to reach it.
+ #[doc(hidden)]
+ pub fn with_factory(
+ options: LayaRouterOptions,
+ factory: impl Fn(LayaCheckpoint, &LayaOptions) -> Result<Arc<dyn LayaPredictor>>
+ + Send
+ + Sync
+ + 'static,
+ ) -> Result<Self> {
+ let router = Self {
+ engine_options: options.engine_options,
+ default: options.default,
+ auto_task_detection: options.auto_task_detection,
+ factory: Box::new(factory),
+ state: Mutex::new(RouterState {
+ loaded: HashMap::new(),
+ order: Vec::new(),
+ max_loaded: options.max_loaded.max(1),
+ }),
+ };
+ if options.preload {
+ router.preload(None)?;
+ }
+ Ok(router)
+ }
+
+ // ------------------------------------------------------------------ name resolution
+
+ /// Aliases people are likely to type, resolved to the canonical checkpoint name before
+ /// matching. Mirrors Python's `_ALIASES`.
+ fn alias(key: &str) -> Option<&'static str> {
+ match key {
+ "en" | "laya" | "default" => Some("english"),
+ "multi" | "ml" | "laya-multilingual" => Some("multilingual"),
+ "typed" | "typed_decisions" | "laya-typed-decisions" | "decisions" => {
+ Some("typed-decisions")
+ }
+ _ => None,
+ }
+ }
+
+ /// Normalise a model name (case-insensitive, whitespace-trimmed, with aliases resolved) to a
+ /// [`LayaCheckpoint`]. Ports `normalise_name`.
+ pub fn normalise_name(name: &str) -> Result<LayaCheckpoint> {
+ let key = name.trim().to_lowercase();
+ let key = Self::alias(&key).unwrap_or(key.as_str());
+ match key {
+ "english" => Ok(LayaCheckpoint::English),
+ "multilingual" => Ok(LayaCheckpoint::Multilingual),
+ "typed-decisions" => Ok(LayaCheckpoint::TypedDecisions),
+ _ => Err(LayaError::InvalidConfig(format!(
+ "unknown model {}; choose one of \"english\", \"multilingual\", \"typed-decisions\" \
+ (or an alias: \"en\", \"laya\", \"default\", \"multi\", \"ml\", \"laya-multilingual\", \
+ \"typed\", \"typed_decisions\", \"laya-typed-decisions\", \"decisions\")",
+ python_repr_str(name)
+ ))),
+ }
+ }
+
+ /// Name of the typed-decisions workflow whose question ids these are, else `None`. Requires
+ /// an exact id-set match, so an unrelated schema that happens to contain `"urgency"` is never
+ /// captured. Ports `match_typed_decisions_workflow`.
+ pub fn match_typed_decisions_workflow(questions: &QuestionSet) -> Option<String> {
+ let ids: HashSet<&str> = questions.ids().collect();
+ for (workflow, signature) in TYPED_DECISION_WORKFLOWS {
+ let sig: HashSet<&str> = signature.iter().copied().collect();
+ if ids == sig {
+ return Some((*workflow).to_string());
+ }
+ }
+ None
+ }
+
+ // ------------------------------------------------------------------ loading
+
+ /// Return the predictor for `name`, downloading and building it on first use. Concurrent
+ /// callers requesting the same checkpoint share a single build (see the module docs).
+ pub fn load(&self, name: &str) -> Result<Arc<dyn LayaPredictor>> {
+ let key = Self::normalise_name(name)?;
+ self.load_checkpoint(key)
+ }
+
+ fn load_checkpoint(&self, key: LayaCheckpoint) -> Result<Arc<dyn LayaPredictor>> {
+ let mut state = self.state.lock().expect("router state mutex poisoned");
+ if let Some(engine) = state.loaded.get(&key) {
+ let engine = Arc::clone(engine);
+ Self::touch(&mut state, key);
+ return Ok(engine);
+ }
+ // The lock is held across the build itself (see the module docs): a second caller asking
+ // for the same checkpoint blocks here rather than racing a duplicate build.
+ let engine = (self.factory)(key, &self.engine_options)?;
+ state.loaded.insert(key, Arc::clone(&engine));
+ state.order.push(key);
+ Self::evict(&mut state);
+ Ok(engine)
+ }
+
+ fn touch(state: &mut RouterState, key: LayaCheckpoint) {
+ state.order.retain(|&k| k != key);
+ state.order.push(key);
+ }
+
+ fn evict(state: &mut RouterState) {
+ while state.order.len() > state.max_loaded {
+ let victim = state.order.remove(0);
+ state.loaded.remove(&victim);
+ }
+ }
+
+ /// Register an already-built predictor under `name` instead of loading a second copy. Raises
+ /// `max_loaded` to fit it if needed, so attaching never immediately evicts what was just
+ /// attached.
+ pub fn attach(
+ &self,
+ name: &str,
+ predictor: Arc<dyn LayaPredictor>,
+ ) -> Result<Arc<dyn LayaPredictor>> {
+ let key = Self::normalise_name(name)?;
+ let mut state = self.state.lock().expect("router state mutex poisoned");
+ state.loaded.insert(key, Arc::clone(&predictor));
+ Self::touch(&mut state, key);
+ state.max_loaded = state.max_loaded.max(state.loaded.len());
+ Ok(predictor)
+ }
+
+ /// Download and build checkpoints up front so no request ever pays a model load. `names`
+ /// defaults to every checkpoint (`english`, `multilingual`, `typed-decisions`, in that
+ /// order). `max_loaded` is raised first, before any of the loads run, so the loop's own
+ /// incremental loads never trigger a premature eviction of what preload is about to build.
+ ///
+ /// Unlike Python (whose single re-entrant lock covers the whole preload call), each
+ /// checkpoint's load here takes and releases the router's lock separately — `std::sync::
+ /// Mutex` is not re-entrant, so holding it across a loop of `self.load_checkpoint` calls
+ /// (which each need the lock themselves) would deadlock. This does not change the
+ /// observable end state: `load_checkpoint` already treats an already-resident checkpoint as
+ /// a touch rather than a rebuild, so a preload racing another thread's load/attach still
+ /// converges to the same result, just interleaved more finely than Python's.
+ pub fn preload(&self, names: Option<&[&str]>) -> Result<()> {
+ let keys: Vec<LayaCheckpoint> = match names {
+ Some(names) => names
+ .iter()
+ .map(|n| Self::normalise_name(n))
+ .collect::<Result<_>>()?,
+ None => vec![
+ LayaCheckpoint::English,
+ LayaCheckpoint::Multilingual,
+ LayaCheckpoint::TypedDecisions,
+ ],
+ };
+ {
+ let mut state = self.state.lock().expect("router state mutex poisoned");
+ state.max_loaded = state.max_loaded.max(keys.len()).max(state.loaded.len());
+ }
+ for key in keys {
+ self.load_checkpoint(key)?;
+ }
+ Ok(())
+ }
+
+ /// Free one checkpoint, or every checkpoint when `name` is `None`.
+ pub fn unload(&self, name: Option<&str>) -> Result<()> {
+ let mut state = self.state.lock().expect("router state mutex poisoned");
+ match name {
+ None => {
+ state.loaded.clear();
+ state.order.clear();
+ }
+ Some(name) => {
+ let key = Self::normalise_name(name)?;
+ state.loaded.remove(&key);
+ state.order.retain(|&k| k != key);
+ }
+ }
+ Ok(())
+ }
+
+ /// Currently-resident checkpoints, least-recently-used first.
+ pub fn loaded(&self) -> Vec<LayaCheckpoint> {
+ self.state
+ .lock()
+ .expect("router state mutex poisoned")
+ .order
+ .clone()
+ }
+
+ // ------------------------------------------------------------------ routing
+
+ /// Decide which checkpoint to use, without loading or running anything.
+ ///
+ /// Precedence: explicit `model` > explicit `task` > detected workflow (opt-in) > explicit
+ /// `lang` > detected script/language > [`LayaRouterOptions::default`].
+ pub fn route(
+ &self,
+ state: &Value,
+ questions: Option<&QuestionSet>,
+ model: Option<&str>,
+ task: Option<&str>,
+ lang: Option<&str>,
+ ) -> Result<RouteDecision> {
+ if let Some(model) = model {
+ let key = Self::normalise_name(model)?;
+ return Ok(RouteDecision {
+ model: key,
+ reason: format!("explicit model={}", python_repr_str(model)),
+ detection: None,
+ workflow: None,
+ });
+ }
+
+ if let Some(task) = task {
+ let normalized = task.to_lowercase().replace('-', "_");
+ let target = if normalized == "typed_decisions" {
+ "typed-decisions"
+ } else {
+ task
+ };
+ let key = Self::normalise_name(target)?;
+ return Ok(RouteDecision {
+ model: key,
+ reason: format!("explicit task={}", python_repr_str(task)),
+ detection: None,
+ workflow: None,
+ });
+ }
+
+ let workflow = questions.and_then(Self::match_typed_decisions_workflow);
+ if let Some(wf) = &workflow
+ && self.auto_task_detection
+ {
+ return Ok(RouteDecision {
+ model: LayaCheckpoint::TypedDecisions,
+ reason: format!(
+ "question ids match the {} typed-decisions workflow",
+ python_repr_str(wf)
+ ),
+ detection: None,
+ workflow: Some(wf.clone()),
+ });
+ }
+
+ if let Some(lang) = lang {
+ let lowered = lang.to_lowercase();
+ let head = lowered.split('-').next().unwrap_or("");
+ let key = if matches!(head, "en" | "eng" | "english") {
+ LayaCheckpoint::English
+ } else {
+ LayaCheckpoint::Multilingual
+ };
+ return Ok(RouteDecision {
+ model: key,
+ reason: format!("explicit lang={}", python_repr_str(lang)),
+ detection: None,
+ workflow,
+ });
+ }
+
+ let det = LanguageDetection::analyse(state);
+ let (key, reason) = if det.script == "unknown" {
+ (
+ self.default,
+ format!(
+ "no letters detected in state; using default ({})",
+ self.default.subdir()
+ ),
+ )
+ } else if det.script != "latin" {
+ (
+ LayaCheckpoint::Multilingual,
+ format!(
+ "non-Latin script ({}, {:.0}% of letters); the English checkpoint cannot read it",
+ det.script,
+ 100.0 * det.non_latin_fraction
+ ),
+ )
+ } else if !det.is_english {
+ if let Some(lang) = &det.language {
+ (
+ LayaCheckpoint::Multilingual,
+ format!(
+ "Latin script but language looks like {}, not English",
+ python_repr_str(lang)
+ ),
+ )
+ } else {
+ (
+ LayaCheckpoint::Multilingual,
+ format!(
+ "Latin script, language not identified but {:.0}% non-English letters; \
+ not safe for the English checkpoint",
+ 100.0 * det.diacritic_rate
+ ),
+ )
+ }
+ } else {
+ (LayaCheckpoint::English, "English Latin text".to_string())
+ };
+
+ Ok(RouteDecision {
+ model: key,
+ reason,
+ detection: Some(det),
+ workflow,
+ })
+ }
+
+ // ------------------------------------------------------------------ running
+
+ /// Route, then answer every question in one forward pass on the chosen checkpoint. The
+ /// result's [`LayaResult::routing`] records the decision.
+ pub fn predict(
+ &self,
+ state: impl Into<Value>,
+ questions: &QuestionSet,
+ model: Option<&str>,
+ task: Option<&str>,
+ lang: Option<&str>,
+ ) -> Result<LayaResult> {
+ let state = state.into();
+ let decision = self.route(&state, Some(questions), model, task, lang)?;
+ let engine = self.load_checkpoint(decision.model)?;
+ // Inference deliberately runs after the `Arc<dyn LayaPredictor>` clone above and outside
+ // any lock: see the module docs on eviction safety.
+ let result = engine.predict(state, questions)?;
+ Ok(result.with_routing(decision))
+ }
+}
+
+impl LayaPredictor for LayaRouter {
+ /// [`LayaRouter::predict`] with every override left at its default (no explicit model, task
+ /// or lang) — the entry point `laya_shortlist`'s generic `&dyn LayaPredictor` path uses.
+ fn predict(&self, state: Value, questions: &QuestionSet) -> Result<LayaResult> {
+ LayaRouter::predict(self, state, questions, None, None, None)
+ }
+}
+
+/// Python `repr()` for a string: single-quoted unless the string itself contains a `'` and no
+/// `"`, in which case double quotes are used instead; backslash, the chosen quote character, and
+/// the common control characters are escaped. Every model/task/lang name recorded in the routing
+/// golden vectors is plain ASCII, so this covers the cases that matter without attempting a full
+/// `repr()` (which would also need to classify every Unicode character as printable or not).
+fn python_repr_str(s: &str) -> String {
+ let quote = if s.contains('\'') && !s.contains('"') {
+ '"'
+ } else {
+ '\''
+ };
+ let mut out = String::with_capacity(s.len() + 2);
+ out.push(quote);
+ for c in s.chars() {
+ match c {
+ '\\' => out.push_str("\\\\"),
+ c if c == quote => {
+ out.push('\\');
+ out.push(c);
+ }
+ '\n' => out.push_str("\\n"),
+ '\r' => out.push_str("\\r"),
+ '\t' => out.push_str("\\t"),
+ c if (c as u32) < 0x20 || (c as u32) == 0x7f => {
+ out.push_str(&format!("\\x{:02x}", c as u32));
+ }
+ c => out.push(c),
+ }
+ }
+ out.push(quote);
+ out
+}
diff --git a/laya-rust/tests/common/routing_golden.rs b/laya-rust/tests/common/routing_golden.rs
new file mode 100644
index 00000000..48d3b0ef
--- /dev/null
+++ b/laya-rust/tests/common/routing_golden.rs
@@ -0,0 +1,41 @@
+//! Access to the checkpoint-independent routing golden vectors recorded by
+//! `tools/dump_routing_golden.py`: `tests/golden/routing/*.json`.
+//!
+//! Mirrors [`super::golden_data`]'s "read from the source tree, don't copy" approach and its
+//! `LAYA_GOLDEN_DIR` override (the routing directory is `LAYA_GOLDEN_DIR/routing` when that
+//! variable is set, else the same default base directory `golden_data::golden_base_dir` uses).
+
+use serde_json::Value;
+use std::path::PathBuf;
+
+/// The `golden/routing/` directory.
+pub fn routing_dir() -> PathBuf {
+ super::golden_data::golden_base_dir().join("routing")
+}
+
+/// Parse one file under `golden/routing/`, or `None` if the routing golden tree is not present
+/// at all (so Tier 1 tests can print a skip line instead of failing on a checkout that has not
+/// run Phase A's dump script).
+pub fn load(file_name: &str) -> Option<Value> {
+ let path = routing_dir().join(file_name);
+ if !path.is_file() {
+ return None;
+ }
+ let text = std::fs::read_to_string(&path)
+ .unwrap_or_else(|e| panic!("failed to read routing golden file {}: {e}", path.display()));
+ Some(serde_json::from_str(&text).unwrap_or_else(|e| {
+ panic!(
+ "failed to parse routing golden file {}: {e}",
+ path.display()
+ )
+ }))
+}
+
+/// [`load`], expecting a top-level JSON array.
+pub fn load_array(file_name: &str) -> Option<Vec<Value>> {
+ load(file_name).map(|v| {
+ v.as_array()
+ .unwrap_or_else(|| panic!("{file_name} must be a JSON array"))
+ .clone()
+ })
+}
diff --git a/laya-rust/tests/golden/routing/email_probe.json b/laya-rust/tests/golden/routing/email_probe.json
new file mode 100644
index 00000000..e2c1f27a
--- /dev/null
+++ b/laya-rust/tests/golden/routing/email_probe.json
@@ -0,0 +1,395 @@
+{
+ "clean": [
+ {
+ "label": "inline_footer_no_blank_line",
+ "body": "My account is locked.\nThis email is confidential and intended solely for the named addressee.\nPlease unlock it.",
+ "max_chars": 3000,
+ "result": "My account is locked. Please unlock it."
+ },
+ {
+ "label": "inline_footer_no_terminal_punctuation",
+ "body": "My account is locked\nThis email is confidential and intended solely for the named addressee.\nPlease unlock it.",
+ "max_chars": 3000,
+ "result": "Please unlock it."
+ },
+ {
+ "label": "inline_footer_body_never_emptied",
+ "body": "My account is locked. This email is confidential and intended solely for the named addressee.",
+ "max_chars": 3000,
+ "result": "My account is locked."
+ },
+ {
+ "label": "standalone_footer_paragraph_dropped",
+ "body": "My account is locked.\n\nThis email is confidential and intended solely for the named addressee.",
+ "max_chars": 3000,
+ "result": "My account is locked."
+ },
+ {
+ "label": "wrapped_standalone_footer_dropped",
+ "body": "My account is locked.\n\nThis email and any files transmitted with it are\nconfidential and intended solely for the named addressee.",
+ "max_chars": 3000,
+ "result": "My account is locked."
+ },
+ {
+ "label": "received_in_error_footer_dropped",
+ "body": "Please reopen ticket 4411.\n\nIf you have received this message in error, delete it.",
+ "max_chars": 3000,
+ "result": "Please reopen ticket 4411."
+ },
+ {
+ "label": "quoted_history_removed",
+ "body": "Thanks for the update.\nOn Mon, Sep 20, Bob wrote:\n> original text",
+ "max_chars": 3000,
+ "result": "Thanks for the update."
+ },
+ {
+ "label": "signature_block_removed",
+ "body": "Hi team,\nCan you confirm the refund?\nRegards,\nAlice",
+ "max_chars": 3000,
+ "result": "Hi team,\nCan you confirm the refund?"
+ },
+ {
+ "label": "empty_body",
+ "body": "",
+ "max_chars": 3000,
+ "result": ""
+ },
+ {
+ "label": "quote_header_on_wrote_long_name",
+ "body": "Please see below.\nOn Mon, Jan 5, 2026 at 3:00 PM, Bob Smith <[email protected]> wrote:\n> old text here",
+ "max_chars": 3000,
+ "result": "Please see below."
+ },
+ {
+ "label": "quote_header_original_message",
+ "body": "Approved.\n----- Original Message -----\nFrom: [email protected]\nSubject: Re: Invoice",
+ "max_chars": 3000,
+ "result": "Approved."
+ },
+ {
+ "label": "quote_header_forwarded_message",
+ "body": "See below for context.\n---------- Forwarded Message ----------\nFrom: [email protected]",
+ "max_chars": 3000,
+ "result": "See below for context."
+ },
+ {
+ "label": "quote_header_underscores",
+ "body": "My reply is above.\n________________________________\nFrom: [email protected]",
+ "max_chars": 3000,
+ "result": "My reply is above."
+ },
+ {
+ "label": "quote_header_from_line",
+ "body": "Confirmed, thanks.\nFrom: [email protected]\nSubject: Ticket 4411 update\nDate: today",
+ "max_chars": 3000,
+ "result": "Confirmed, thanks."
+ },
+ {
+ "label": "quote_header_only_no_body_before_kept",
+ "body": "On Mon, Jan 5, 2026, Bob wrote:\n> quoted only, no lines before it to break on",
+ "max_chars": 3000,
+ "result": "On Mon, Jan 5, 2026, Bob wrote:"
+ },
+ {
+ "label": "quote_marker_gt_lines_stripped",
+ "body": "My reply.\n> quoted line one\n> quoted line two\nMore of my reply.",
+ "max_chars": 3000,
+ "result": "My reply.\nMore of my reply."
+ },
+ {
+ "label": "signature_dash_dash",
+ "body": "Hi,\nPlease confirm.\n--\nAlice Smith\nSupport Team",
+ "max_chars": 3000,
+ "result": "Hi,\nPlease confirm."
+ },
+ {
+ "label": "signature_best_regards",
+ "body": "Hi,\nCan you confirm the refund?\nBest regards,\nAlice",
+ "max_chars": 3000,
+ "result": "Hi,\nCan you confirm the refund?"
+ },
+ {
+ "label": "signature_kind_regards",
+ "body": "Hi,\nCan you confirm the refund?\nKind regards,\nBob",
+ "max_chars": 3000,
+ "result": "Hi,\nCan you confirm the refund?"
+ },
+ {
+ "label": "signature_warm_regards",
+ "body": "Hi,\nCan you confirm the refund?\nWarm regards,\nCarol",
+ "max_chars": 3000,
+ "result": "Hi,\nCan you confirm the refund?"
+ },
+ {
+ "label": "signature_many_thanks",
+ "body": "Hi,\nCan you confirm the refund?\nMany thanks,\nDave",
+ "max_chars": 3000,
+ "result": "Hi,\nCan you confirm the refund?"
+ },
+ {
+ "label": "signature_thanks",
+ "body": "Hi,\nCan you confirm the refund?\nThanks,\nEve",
+ "max_chars": 3000,
+ "result": "Hi,\nCan you confirm the refund?"
+ },
+ {
+ "label": "signature_thank_you",
+ "body": "Hi,\nCan you confirm the refund?\nThank you,\nFrank",
+ "max_chars": 3000,
+ "result": "Hi,\nCan you confirm the refund?"
+ },
+ {
+ "label": "signature_cheers",
+ "body": "Hi,\nCan you confirm the refund?\nCheers,\nGrace",
+ "max_chars": 3000,
+ "result": "Hi,\nCan you confirm the refund?"
+ },
+ {
+ "label": "signature_sincerely",
+ "body": "Hi,\nCan you confirm the refund?\nSincerely,\nHeidi",
+ "max_chars": 3000,
+ "result": "Hi,\nCan you confirm the refund?"
+ },
+ {
+ "label": "signature_sent_from_iphone",
+ "body": "On the go, will follow up later.\nSent from my iPhone",
+ "max_chars": 3000,
+ "result": "On the go, will follow up later."
+ },
+ {
+ "label": "signature_sent_from_android",
+ "body": "Quick reply here.\nSent from my Android",
+ "max_chars": 3000,
+ "result": "Quick reply here."
+ },
+ {
+ "label": "signature_too_early_kept_as_body",
+ "body": "Thanks,\nfor looking into this.\nMy account is still locked after the reset.\nPlease escalate this to the on-call engineer today.\nRegards,\nIvan",
+ "max_chars": 3000,
+ "result": "Thanks,\nfor looking into this.\nMy account is still locked after the reset.\nPlease escalate this to the on-call engineer today."
+ },
+ {
+ "label": "disclaimer_mid_paragraph_request_survives",
+ "body": "My account is locked.\nThis email is confidential and intended solely for the named addressee.\nPlease unlock it.",
+ "max_chars": 3000,
+ "result": "My account is locked. Please unlock it."
+ },
+ {
+ "label": "disclaimer_two_sentences_only_boilerplate_dropped",
+ "body": "My account is locked. This email is confidential and intended solely for the named addressee. Please help urgently.",
+ "max_chars": 3000,
+ "result": "My account is locked. Please help urgently."
+ },
+ {
+ "label": "disclaimer_across_two_paragraphs_only_its_own_paragraph_affected",
+ "body": "Please reopen ticket 4411 as soon as you can.\n\nThis email and any attachments are confidential and intended solely for the addressee.\n\nIf you have received this message in error, please delete it.\n\nLet me know once it is done.",
+ "max_chars": 3000,
+ "result": "Please reopen ticket 4411 as soon as you can.\n\nLet me know once it is done."
+ },
+ {
+ "label": "disclaimer_received_in_error_variant_2",
+ "body": "Approve the refund please.\n\nIf you received this e-mail in error, notify the sender immediately.",
+ "max_chars": 3000,
+ "result": "Approve the refund please."
+ },
+ {
+ "label": "crlf_line_endings",
+ "body": "My account is locked.\r\nPlease unlock it.\r\nRegards,\r\nAlice",
+ "max_chars": 3000,
+ "result": "My account is locked.\nPlease unlock it."
+ },
+ {
+ "label": "crlf_with_quote_header",
+ "body": "Thanks for the update.\r\nOn Mon, Sep 20, Bob wrote:\r\n> original text",
+ "max_chars": 3000,
+ "result": "Thanks for the update."
+ },
+ {
+ "label": "literal_backslash_n_becomes_newline",
+ "body": "My account is locked.\\nPlease unlock it.\\nRegards,\\nAlice",
+ "max_chars": 3000,
+ "result": "My account is locked.\nPlease unlock it."
+ },
+ {
+ "label": "literal_backslash_n_with_disclaimer",
+ "body": "My account is locked.\\nThis email is confidential and intended solely for the named addressee.\\nPlease unlock it.",
+ "max_chars": 3000,
+ "result": "My account is locked. Please unlock it."
+ },
+ {
+ "label": "mixed_real_and_literal_newlines",
+ "body": "Line one.\nLine two.\\nLine three (literal escape).\nLine four.",
+ "max_chars": 3000,
+ "result": "Line one.\nLine two.\nLine three (literal escape).\nLine four."
+ },
+ {
+ "label": "unicode_nbsp_not_collapsed",
+ "body": "My account is locked.\nPlease unlock it.",
+ "max_chars": 3000,
+ "result": "My account is locked.\nPlease unlock it."
+ },
+ {
+ "label": "unicode_em_space_and_ideographic_space",
+ "body": "My account is locked.\nPlease unlock it.",
+ "max_chars": 3000,
+ "result": "My account is locked.\nPlease unlock it."
+ },
+ {
+ "label": "unicode_whitespace_leading_trailing_stripped",
+ "body": "  My account is locked. \n Please unlock it.  ",
+ "max_chars": 3000,
+ "result": "My account is locked.\n Please unlock it."
+ },
+ {
+ "label": "unicode_zero_width_space_kept",
+ "body": "My​account is locked.\nPlease​unlock it.",
+ "max_chars": 3000,
+ "result": "My​account is locked.\nPlease​unlock it."
+ },
+ {
+ "label": "max_chars_tiny_cuts_mid_word",
+ "body": "Please refund the duplicate charge on invoice 4411 today.",
+ "max_chars": 10,
+ "result": "Please ref"
+ },
+ {
+ "label": "max_chars_default_no_truncation_needed",
+ "body": "Short message that fits easily.",
+ "max_chars": 3000,
+ "result": "Short message that fits easily."
+ },
+ {
+ "label": "max_chars_large_no_op",
+ "body": "Please refund the duplicate charge on invoice 4411 today.",
+ "max_chars": 100000,
+ "result": "Please refund the duplicate charge on invoice 4411 today."
+ },
+ {
+ "label": "max_chars_exact_boundary",
+ "body": "01234567890123456789012345678901234567890123456789",
+ "max_chars": 50,
+ "result": "01234567890123456789012345678901234567890123456789"
+ },
+ {
+ "label": "max_chars_one_over_boundary",
+ "body": "01234567890123456789012345678901234567890123456789X",
+ "max_chars": 50,
+ "result": "01234567890123456789012345678901234567890123456789"
+ },
+ {
+ "label": "kitchen_sink_quote_signature_disclaimer",
+ "body": "Hi team,\n\nMy account was charged twice for the Pro plan this month and the refund has not arrived yet. Could you please look into this today?\n\nBest regards,\nPriya Sharma\nSent from my iPhone\n\nThis email and any files transmitted with it are confidential and intended solely for the use of the individual or entity to whom they are addressed. If you have received this email in error please notify the sender.\n\nOn Tue, Sep 22, 2026 at 9:14 AM, Support <[email protected]> wrote:\n> Hi Priya, thanks for reaching out, we are looking into it.\n> - Support Team\n",
+ "max_chars": 3000,
+ "result": "Hi team,\n\nMy account was charged twice for the Pro plan this month and the refund has not arrived yet. Could you please look into this today?"
+ }
+ ],
+ "state": [
+ {
+ "label": "basic_clean_true",
+ "subject": "Locked out",
+ "body": "My account is locked. This email is confidential and intended solely for the named addressee.",
+ "sender": null,
+ "clean": true,
+ "extra": {},
+ "result": {
+ "subject": "Locked out",
+ "body": "My account is locked."
+ }
+ },
+ {
+ "label": "basic_clean_false_keeps_raw_body",
+ "subject": "Locked out",
+ "body": "My account is locked.\nThis email is confidential and intended solely for the named addressee.",
+ "sender": null,
+ "clean": false,
+ "extra": {},
+ "result": {
+ "subject": "Locked out",
+ "body": "My account is locked.\nThis email is confidential and intended solely for the named addressee."
+ }
+ },
+ {
+ "label": "with_sender",
+ "subject": "Refund request",
+ "body": "Please refund invoice 4411.",
+ "sender": "[email protected]",
+ "clean": true,
+ "extra": {},
+ "result": {
+ "subject": "Refund request",
+ "body": "Please refund invoice 4411.",
+ "from": "[email protected]"
+ }
+ },
+ {
+ "label": "subject_whitespace_stripped",
+ "subject": " Refund request ",
+ "body": "Please help.",
+ "sender": null,
+ "clean": true,
+ "extra": {},
+ "result": {
+ "subject": "Refund request",
+ "body": "Please help."
+ }
+ },
+ {
+ "label": "extra_fields_kept_when_not_none",
+ "subject": "Ticket",
+ "body": "Body text.",
+ "sender": "[email protected]",
+ "clean": true,
+ "extra": {
+ "priority": "high",
+ "ticket_id": 4411
+ },
+ "result": {
+ "subject": "Ticket",
+ "body": "Body text.",
+ "from": "[email protected]",
+ "priority": "high",
+ "ticket_id": 4411
+ }
+ },
+ {
+ "label": "extra_none_values_dropped",
+ "subject": "Ticket",
+ "body": "Body text.",
+ "sender": null,
+ "clean": true,
+ "extra": {
+ "nullable": null,
+ "kept": "value"
+ },
+ "result": {
+ "subject": "Ticket",
+ "body": "Body text.",
+ "kept": "value"
+ }
+ },
+ {
+ "label": "empty_subject_and_body",
+ "subject": "",
+ "body": "",
+ "sender": null,
+ "clean": true,
+ "extra": {},
+ "result": {
+ "subject": "",
+ "body": ""
+ }
+ },
+ {
+ "label": "hindi_subject_and_body",
+ "subject": "सहायता चाहिए",
+ "body": "मुझसे दो बार शुल्क लिया गया, कृपया रिफंड करें।",
+ "sender": null,
+ "clean": true,
+ "extra": {},
+ "result": {
+ "subject": "सहायता चाहिए",
+ "body": "मुझसे दो बार शुल्क लिया गया, कृपया रिफंड करें।"
+ }
+ }
+ ]
+}
diff --git a/laya-rust/tests/golden/routing/hashing_embedder_probe.json b/laya-rust/tests/golden/routing/hashing_embedder_probe.json
new file mode 100644
index 00000000..4f1525d2
--- /dev/null
+++ b/laya-rust/tests/golden/routing/hashing_embedder_probe.json
@@ -0,0 +1,1520 @@
+[
+ {
+ "text": "",
+ "vector": [
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ -2.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0
+ ]
+ },
+ {
+ "text": "a",
+ "vector": [
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ -1.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 1.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ -1.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0
+ ]
+ },
+ {
+ "text": "hello",
+ "vector": [
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ -1.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 1.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ -1.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 1.0,
+ 0.0,
+ -1.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 1.0,
+ -1.0
+ ]
+ },
+ {
+ "text": "Hello World",
+ "vector": [
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 1.0,
+ 0.0,
+ 1.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ -1.0,
+ 0.0,
+ -1.0,
+ 1.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ -1.0,
+ 0.0,
+ 0.0,
+ 1.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ -1.0,
+ 0.0,
+ 0.0,
+ 1.0,
+ 0.0,
+ -1.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ -1.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 1.0,
+ -1.0
+ ]
+ },
+ {
+ "text": "HELLO WORLD",
+ "vector": [
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 1.0,
+ 0.0,
+ 1.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ -1.0,
+ 0.0,
+ -1.0,
+ 1.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ -1.0,
+ 0.0,
+ 0.0,
+ 1.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ -1.0,
+ 0.0,
+ 0.0,
+ 1.0,
+ 0.0,
+ -1.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ -1.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 1.0,
+ -1.0
+ ]
+ },
+ {
+ "text": "café",
+ "vector": [
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ -1.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ -1.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ -1.0,
+ 0.0,
+ 1.0
+ ]
+ },
+ {
+ "text": "naïve résumé Müller",
+ "vector": [
+ 1.0,
+ 0.0,
+ -1.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 1.0,
+ -1.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ -1.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ -1.0,
+ -1.0,
+ 0.0,
+ 0.0,
+ 1.0,
+ 0.0,
+ 0.0,
+ 1.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ -1.0,
+ 0.0,
+ 0.0,
+ -1.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 1.0,
+ 0.0,
+ 0.0,
+ 1.0,
+ 1.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 1.0
+ ]
+ },
+ {
+ "text": "İstanbul",
+ "vector": [
+ 0.0,
+ 0.0,
+ 0.0,
+ 1.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 1.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 2.0,
+ 0.0,
+ 0.0,
+ 1.0,
+ -1.0,
+ -1.0,
+ 1.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ -1.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 1.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 1.0,
+ 0.0,
+ 0.0,
+ 0.0
+ ]
+ },
+ {
+ "text": "ıstanbul",
+ "vector": [
+ 0.0,
+ -1.0,
+ 0.0,
+ 1.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 1.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 2.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ -1.0,
+ -1.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ -1.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 1.0,
+ 0.0,
+ 0.0,
+ 1.0,
+ 0.0,
+ 0.0,
+ 0.0
+ ]
+ },
+ {
+ "text": "我这个月被重复扣款了两次",
+ "vector": [
+ 0.0,
+ 0.0,
+ 0.0,
+ -1.0,
+ 0.0,
+ 0.0,
+ -1.0,
+ 0.0,
+ 0.0,
+ -1.0,
+ -1.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 1.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 1.0,
+ 0.0,
+ 0.0,
+ 1.0,
+ 0.0,
+ -1.0,
+ 0.0,
+ -1.0,
+ 0.0,
+ 1.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 1.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ -1.0,
+ 1.0,
+ 0.0,
+ 1.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0
+ ]
+ },
+ {
+ "text": "こんにちは世界",
+ "vector": [
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ -1.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ -1.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 1.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 1.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ -1.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 2.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 1.0,
+ 1.0,
+ 0.0,
+ 0.0
+ ]
+ },
+ {
+ "text": "안녕하세요",
+ "vector": [
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 1.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ -1.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ -1.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ -1.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ -1.0,
+ 0.0,
+ -1.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ -1.0,
+ 0.0,
+ 0.0
+ ]
+ },
+ {
+ "text": "тест русского текста",
+ "vector": [
+ -1.0,
+ -1.0,
+ -1.0,
+ -1.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ -1.0,
+ 1.0,
+ 0.0,
+ -1.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 1.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 1.0,
+ -1.0,
+ 0.0,
+ 1.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ -1.0,
+ 0.0,
+ -1.0,
+ 0.0,
+ -1.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ -1.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 1.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ -1.0,
+ 1.0,
+ 3.0,
+ 0.0,
+ 0.0,
+ -1.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0
+ ]
+ },
+ {
+ "text": "مرحبا بالعالم",
+ "vector": [
+ -1.0,
+ 0.0,
+ -2.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ -1.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 1.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 1.0,
+ 1.0,
+ 0.0,
+ -1.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 1.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ -2.0,
+ -1.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 1.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0
+ ]
+ },
+ {
+ "text": "नमस्ते दुनिया",
+ "vector": [
+ 0.0,
+ 0.0,
+ 0.0,
+ -1.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 1.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ -1.0,
+ 0.0,
+ 1.0,
+ 0.0,
+ -1.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 1.0,
+ 0.0,
+ 1.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ -1.0,
+ 1.0,
+ 0.0,
+ 1.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ -1.0,
+ 0.0,
+ 0.0,
+ -1.0,
+ 0.0,
+ -1.0,
+ 0.0,
+ -1.0,
+ 0.0,
+ -1.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0
+ ]
+ },
+ {
+ "text": "😀🚀",
+ "vector": [
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 1.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ -1.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ -1.0,
+ 0.0,
+ 0.0,
+ 1.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0
+ ]
+ },
+ {
+ "text": "refund 😀 today",
+ "vector": [
+ -1.0,
+ 0.0,
+ 1.0,
+ 0.0,
+ 0.0,
+ 1.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 1.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 1.0,
+ 0.0,
+ -1.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ -1.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 1.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ -1.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ -1.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0
+ ]
+ },
+ {
+ "text": "𐐀𐐁",
+ "vector": [
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 1.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ -1.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ -1.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ -1.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0
+ ]
+ },
+ {
+ "text": " ",
+ "vector": [
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ -5.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0
+ ]
+ },
+ {
+ "text": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
+ "vector": [
+ 0.0,
+ -1.0,
+ 0.0,
+ 0.0,
+ -1.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 1.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 48.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ -1.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0
+ ]
+ },
+ {
+ "text": "The quick brown fox jumps over the lazy dog",
+ "vector": [
+ 0.0,
+ 2.0,
+ -2.0,
+ 0.0,
+ 1.0,
+ 0.0,
+ -1.0,
+ 0.0,
+ 2.0,
+ -1.0,
+ -1.0,
+ -1.0,
+ 0.0,
+ 0.0,
+ -2.0,
+ -1.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 1.0,
+ 0.0,
+ 0.0,
+ -1.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ -2.0,
+ -1.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ -2.0,
+ 1.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ -1.0,
+ 0.0,
+ 0.0,
+ -1.0,
+ 0.0,
+ 1.0,
+ -1.0,
+ 1.0,
+ -1.0,
+ 0.0,
+ -1.0,
+ -1.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ -1.0,
+ 0.0,
+ 0.0,
+ -2.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0
+ ]
+ },
+ {
+ "text": "refund!! invoice #4411 @ $99.99 (urgent)",
+ "vector": [
+ 0.0,
+ 0.0,
+ -1.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 1.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 1.0,
+ 0.0,
+ 0.0,
+ 1.0,
+ -1.0,
+ 0.0,
+ 0.0,
+ -1.0,
+ 2.0,
+ -1.0,
+ 2.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 1.0,
+ -1.0,
+ 0.0,
+ 0.0,
+ -2.0,
+ -1.0,
+ 0.0,
+ 0.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 1.0,
+ 0.0,
+ -1.0,
+ 0.0,
+ -1.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ 0.0,
+ -1.0,
+ 0.0,
+ 0.0,
+ -1.0,
+ 0.0,
+ -1.0,
+ 0.0,
+ -1.0,
+ 0.0,
+ 0.0,
+ 0.0
+ ]
+ }
+]
diff --git a/laya-rust/tests/golden/routing/lang_probe.json b/laya-rust/tests/golden/routing/lang_probe.json
new file mode 100644
index 00000000..14c14a08
--- /dev/null
+++ b/laya-rust/tests/golden/routing/lang_probe.json
@@ -0,0 +1,3548 @@
+[
+ {
+ "label": "script/english",
+ "state": "The customer was charged twice and wants a refund.",
+ "state_text": "The customer was charged twice and wants a refund.",
+ "detect_script": "latin",
+ "script_profile": {
+ "latin": 1.0
+ },
+ "latin_profile": {
+ "language": "en",
+ "english_hits": 3,
+ "diacritic_rate": 0.0,
+ "looks_non_english": false
+ },
+ "analyse": {
+ "script": "latin",
+ "script_profile": {
+ "latin": 1.0
+ },
+ "language": "en",
+ "is_english": true,
+ "language_undecided": false,
+ "diacritic_rate": 0.0,
+ "non_latin_fraction": 0.0
+ }
+ },
+ {
+ "label": "script/armenian",
+ "state": "Հայերեն",
+ "state_text": "Հայերեն",
+ "detect_script": "armenian",
+ "script_profile": {
+ "armenian": 1.0
+ },
+ "latin_profile": {
+ "language": null,
+ "english_hits": 0,
+ "diacritic_rate": 0.0,
+ "looks_non_english": false
+ },
+ "analyse": {
+ "script": "armenian",
+ "script_profile": {
+ "armenian": 1.0
+ },
+ "language": null,
+ "is_english": false,
+ "language_undecided": true,
+ "diacritic_rate": 0.0,
+ "non_latin_fraction": 1.0
+ }
+ },
+ {
+ "label": "script/armenian_uppercase",
+ "state": "ՀԱՅԵՐԵՆ",
+ "state_text": "ՀԱՅԵՐԵՆ",
+ "detect_script": "armenian",
+ "script_profile": {
+ "armenian": 1.0
+ },
+ "latin_profile": {
+ "language": null,
+ "english_hits": 0,
+ "diacritic_rate": 0.0,
+ "looks_non_english": false
+ },
+ "analyse": {
+ "script": "armenian",
+ "script_profile": {
+ "armenian": 1.0
+ },
+ "language": null,
+ "is_english": false,
+ "language_undecided": true,
+ "diacritic_rate": 0.0,
+ "non_latin_fraction": 1.0
+ }
+ },
+ {
+ "label": "script/armenian_punctuation_only",
+ "state": "։֊",
+ "state_text": "։֊",
+ "detect_script": "unknown",
+ "script_profile": {},
+ "latin_profile": {
+ "language": null,
+ "english_hits": 0,
+ "diacritic_rate": 0.0,
+ "looks_non_english": false
+ },
+ "analyse": {
+ "script": "unknown",
+ "script_profile": {},
+ "language": null,
+ "is_english": true,
+ "language_undecided": true,
+ "diacritic_rate": 0.0,
+ "non_latin_fraction": 0.0
+ }
+ },
+ {
+ "label": "script/french",
+ "state": "Le client a été facturé deux fois et demande un remboursement.",
+ "state_text": "Le client a été facturé deux fois et demande un remboursement.",
+ "detect_script": "latin",
+ "script_profile": {
+ "latin": 1.0
+ },
+ "latin_profile": {
+ "language": "fr",
+ "english_hits": 0,
+ "diacritic_rate": 0.04838709677419355,
+ "looks_non_english": true
+ },
+ "analyse": {
+ "script": "latin",
+ "script_profile": {
+ "latin": 1.0
+ },
+ "language": "fr",
+ "is_english": false,
+ "language_undecided": false,
+ "diacritic_rate": 0.0484,
+ "non_latin_fraction": 0.0
+ }
+ },
+ {
+ "label": "script/hindi",
+ "state": "ग्राहक से दो बार शुल्क लिया गया और वह धनवापसी चाहता है।",
+ "state_text": "ग्राहक से दो बार शुल्क लिया गया और वह धनवापसी चाहता है।",
+ "detect_script": "devanagari",
+ "script_profile": {
+ "devanagari": 1.0
+ },
+ "latin_profile": {
+ "language": null,
+ "english_hits": 0,
+ "diacritic_rate": 0.0,
+ "looks_non_english": false
+ },
+ "analyse": {
+ "script": "devanagari",
+ "script_profile": {
+ "devanagari": 1.0
+ },
+ "language": null,
+ "is_english": false,
+ "language_undecided": true,
+ "diacritic_rate": 0.0,
+ "non_latin_fraction": 1.0
+ }
+ },
+ {
+ "label": "script/japanese",
+ "state": "お客様は二重に請求されたため返金を希望しています。",
+ "state_text": "お客様は二重に請求されたため返金を希望しています。",
+ "detect_script": "kana",
+ "script_profile": {
+ "kana": 0.5833333333333334,
+ "han": 0.4166666666666667
+ },
+ "latin_profile": {
+ "language": null,
+ "english_hits": 0,
+ "diacritic_rate": 0.0,
+ "looks_non_english": false
+ },
+ "analyse": {
+ "script": "kana",
+ "script_profile": {
+ "kana": 0.5833333333333334,
+ "han": 0.4166666666666667
+ },
+ "language": null,
+ "is_english": false,
+ "language_undecided": true,
+ "diacritic_rate": 0.0,
+ "non_latin_fraction": 1.0
+ }
+ },
+ {
+ "label": "script/chinese",
+ "state": "客户被重复扣款要求退款",
+ "state_text": "客户被重复扣款要求退款",
+ "detect_script": "han",
+ "script_profile": {
+ "han": 1.0
+ },
+ "latin_profile": {
+ "language": null,
+ "english_hits": 0,
+ "diacritic_rate": 0.0,
+ "looks_non_english": false
+ },
+ "analyse": {
+ "script": "han",
+ "script_profile": {
+ "han": 1.0
+ },
+ "language": null,
+ "is_english": false,
+ "language_undecided": true,
+ "diacritic_rate": 0.0,
+ "non_latin_fraction": 1.0
+ }
+ },
+ {
+ "label": "script/korean",
+ "state": "고객이 두 번 청구되어 환불을 원합니다",
+ "state_text": "고객이 두 번 청구되어 환불을 원합니다",
+ "detect_script": "hangul",
+ "script_profile": {
+ "hangul": 1.0
+ },
+ "latin_profile": {
+ "language": null,
+ "english_hits": 0,
+ "diacritic_rate": 0.0,
+ "looks_non_english": false
+ },
+ "analyse": {
+ "script": "hangul",
+ "script_profile": {
+ "hangul": 1.0
+ },
+ "language": null,
+ "is_english": false,
+ "language_undecided": true,
+ "diacritic_rate": 0.0,
+ "non_latin_fraction": 1.0
+ }
+ },
+ {
+ "label": "script/arabic",
+ "state": "تم خصم المبلغ مرتين من العميل ويريد استرداد الأموال",
+ "state_text": "تم خصم المبلغ مرتين من العميل ويريد استرداد الأموال",
+ "detect_script": "arabic",
+ "script_profile": {
+ "arabic": 1.0
+ },
+ "latin_profile": {
+ "language": null,
+ "english_hits": 0,
+ "diacritic_rate": 0.0,
+ "looks_non_english": false
+ },
+ "analyse": {
+ "script": "arabic",
+ "script_profile": {
+ "arabic": 1.0
+ },
+ "language": null,
+ "is_english": false,
+ "language_undecided": true,
+ "diacritic_rate": 0.0,
+ "non_latin_fraction": 1.0
+ }
+ },
+ {
+ "label": "script/tamil",
+ "state": "வாடிக்கையாளரிடம் இருமுறை கட்டணம் வசூலிக்கப்பட்டது",
+ "state_text": "வாடிக்கையாளரிடம் இருமுறை கட்டணம் வசூலிக்கப்பட்டது",
+ "detect_script": "tamil",
+ "script_profile": {
+ "tamil": 1.0
+ },
+ "latin_profile": {
+ "language": null,
+ "english_hits": 0,
+ "diacritic_rate": 0.0,
+ "looks_non_english": false
+ },
+ "analyse": {
+ "script": "tamil",
+ "script_profile": {
+ "tamil": 1.0
+ },
+ "language": null,
+ "is_english": false,
+ "language_undecided": true,
+ "diacritic_rate": 0.0,
+ "non_latin_fraction": 1.0
+ }
+ },
+ {
+ "label": "script/russian",
+ "state": "С клиента дважды сняли деньги и он хочет возврат",
+ "state_text": "С клиента дважды сняли деньги и он хочет возврат",
+ "detect_script": "cyrillic",
+ "script_profile": {
+ "cyrillic": 1.0
+ },
+ "latin_profile": {
+ "language": null,
+ "english_hits": 0,
+ "diacritic_rate": 0.0,
+ "looks_non_english": false
+ },
+ "analyse": {
+ "script": "cyrillic",
+ "script_profile": {
+ "cyrillic": 1.0
+ },
+ "language": null,
+ "is_english": false,
+ "language_undecided": true,
+ "diacritic_rate": 0.0,
+ "non_latin_fraction": 1.0
+ }
+ },
+ {
+ "label": "script/thai",
+ "state": "ลูกค้าถูกเรียกเก็บเงินสองครั้งและต้องการเงินคืน",
+ "state_text": "ลูกค้าถูกเรียกเก็บเงินสองครั้งและต้องการเงินคืน",
+ "detect_script": "thai",
+ "script_profile": {
+ "thai": 1.0
+ },
+ "latin_profile": {
+ "language": null,
+ "english_hits": 0,
+ "diacritic_rate": 0.0,
+ "looks_non_english": false
+ },
+ "analyse": {
+ "script": "thai",
+ "script_profile": {
+ "thai": 1.0
+ },
+ "language": null,
+ "is_english": false,
+ "language_undecided": true,
+ "diacritic_rate": 0.0,
+ "non_latin_fraction": 1.0
+ }
+ },
+ {
+ "label": "script/greek",
+ "state": "Ο πελάτης χρεώθηκε δύο φορές και θέλει επιστροφή χρημάτων",
+ "state_text": "Ο πελάτης χρεώθηκε δύο φορές και θέλει επιστροφή χρημάτων",
+ "detect_script": "greek",
+ "script_profile": {
+ "greek": 1.0
+ },
+ "latin_profile": {
+ "language": null,
+ "english_hits": 0,
+ "diacritic_rate": 0.0,
+ "looks_non_english": false
+ },
+ "analyse": {
+ "script": "greek",
+ "script_profile": {
+ "greek": 1.0
+ },
+ "language": null,
+ "is_english": false,
+ "language_undecided": true,
+ "diacritic_rate": 0.0,
+ "non_latin_fraction": 1.0
+ }
+ },
+ {
+ "label": "script/hebrew",
+ "state": "הלקוח חויב פעמיים ורוצה החזר כספי",
+ "state_text": "הלקוח חויב פעמיים ורוצה החזר כספי",
+ "detect_script": "hebrew",
+ "script_profile": {
+ "hebrew": 1.0
+ },
+ "latin_profile": {
+ "language": null,
+ "english_hits": 0,
+ "diacritic_rate": 0.0,
+ "looks_non_english": false
+ },
+ "analyse": {
+ "script": "hebrew",
+ "script_profile": {
+ "hebrew": 1.0
+ },
+ "language": null,
+ "is_english": false,
+ "language_undecided": true,
+ "diacritic_rate": 0.0,
+ "non_latin_fraction": 1.0
+ }
+ },
+ {
+ "label": "script/empty",
+ "state": "",
+ "state_text": "",
+ "detect_script": "unknown",
+ "script_profile": {},
+ "latin_profile": {
+ "language": null,
+ "english_hits": 0,
+ "diacritic_rate": 0.0,
+ "looks_non_english": false
+ },
+ "analyse": {
+ "script": "unknown",
+ "script_profile": {},
+ "language": null,
+ "is_english": true,
+ "language_undecided": true,
+ "diacritic_rate": 0.0,
+ "non_latin_fraction": 0.0
+ }
+ },
+ {
+ "label": "script/digits_only",
+ "state": "12345 6789",
+ "state_text": "12345 6789",
+ "detect_script": "unknown",
+ "script_profile": {},
+ "latin_profile": {
+ "language": null,
+ "english_hits": 0,
+ "diacritic_rate": 0.0,
+ "looks_non_english": false
+ },
+ "analyse": {
+ "script": "unknown",
+ "script_profile": {},
+ "language": null,
+ "is_english": true,
+ "language_undecided": true,
+ "diacritic_rate": 0.0,
+ "non_latin_fraction": 0.0
+ }
+ },
+ {
+ "label": "is_english/plain_english",
+ "state": "Please refund the duplicate charge on invoice 4411 today.",
+ "state_text": "Please refund the duplicate charge on invoice 4411 today.",
+ "detect_script": "latin",
+ "script_profile": {
+ "latin": 1.0
+ },
+ "latin_profile": {
+ "language": "en",
+ "english_hits": 3,
+ "diacritic_rate": 0.0,
+ "looks_non_english": false
+ },
+ "analyse": {
+ "script": "latin",
+ "script_profile": {
+ "latin": 1.0
+ },
+ "language": "en",
+ "is_english": true,
+ "language_undecided": false,
+ "diacritic_rate": 0.0,
+ "non_latin_fraction": 0.0
+ }
+ },
+ {
+ "label": "is_english/armenian",
+ "state": "Հայերեն",
+ "state_text": "Հայերեն",
+ "detect_script": "armenian",
+ "script_profile": {
+ "armenian": 1.0
+ },
+ "latin_profile": {
+ "language": null,
+ "english_hits": 0,
+ "diacritic_rate": 0.0,
+ "looks_non_english": false
+ },
+ "analyse": {
+ "script": "armenian",
+ "script_profile": {
+ "armenian": 1.0
+ },
+ "language": null,
+ "is_english": false,
+ "language_undecided": true,
+ "diacritic_rate": 0.0,
+ "non_latin_fraction": 1.0
+ }
+ },
+ {
+ "label": "is_english/english_short",
+ "state": "refund me",
+ "state_text": "refund me",
+ "detect_script": "latin",
+ "script_profile": {
+ "latin": 1.0
+ },
+ "latin_profile": {
+ "language": null,
+ "english_hits": 0,
+ "diacritic_rate": 0.0,
+ "looks_non_english": false
+ },
+ "analyse": {
+ "script": "latin",
+ "script_profile": {
+ "latin": 1.0
+ },
+ "language": null,
+ "is_english": true,
+ "language_undecided": true,
+ "diacritic_rate": 0.0,
+ "non_latin_fraction": 0.0
+ }
+ },
+ {
+ "label": "is_english/hindi",
+ "state": "ग्राहक से दो बार शुल्क लिया गया",
+ "state_text": "ग्राहक से दो बार शुल्क लिया गया",
+ "detect_script": "devanagari",
+ "script_profile": {
+ "devanagari": 1.0
+ },
+ "latin_profile": {
+ "language": null,
+ "english_hits": 0,
+ "diacritic_rate": 0.0,
+ "looks_non_english": false
+ },
+ "analyse": {
+ "script": "devanagari",
+ "script_profile": {
+ "devanagari": 1.0
+ },
+ "language": null,
+ "is_english": false,
+ "language_undecided": true,
+ "diacritic_rate": 0.0,
+ "non_latin_fraction": 1.0
+ }
+ },
+ {
+ "label": "is_english/japanese",
+ "state": "お客様は二重に請求されました",
+ "state_text": "お客様は二重に請求されました",
+ "detect_script": "kana",
+ "script_profile": {
+ "kana": 0.5714285714285714,
+ "han": 0.42857142857142855
+ },
+ "latin_profile": {
+ "language": null,
+ "english_hits": 0,
+ "diacritic_rate": 0.0,
+ "looks_non_english": false
+ },
+ "analyse": {
+ "script": "kana",
+ "script_profile": {
+ "kana": 0.5714285714285714,
+ "han": 0.42857142857142855
+ },
+ "language": null,
+ "is_english": false,
+ "language_undecided": true,
+ "diacritic_rate": 0.0,
+ "non_latin_fraction": 1.0
+ }
+ },
+ {
+ "label": "is_english/russian",
+ "state": "С клиента дважды сняли деньги",
+ "state_text": "С клиента дважды сняли деньги",
+ "detect_script": "cyrillic",
+ "script_profile": {
+ "cyrillic": 1.0
+ },
+ "latin_profile": {
+ "language": null,
+ "english_hits": 0,
+ "diacritic_rate": 0.0,
+ "looks_non_english": false
+ },
+ "analyse": {
+ "script": "cyrillic",
+ "script_profile": {
+ "cyrillic": 1.0
+ },
+ "language": null,
+ "is_english": false,
+ "language_undecided": true,
+ "diacritic_rate": 0.0,
+ "non_latin_fraction": 1.0
+ }
+ },
+ {
+ "label": "is_english/french_long",
+ "state": "Le client a été facturé deux fois et il demande un remboursement pour la facture qui a été payée le mois dernier avec la carte de crédit",
+ "state_text": "Le client a été facturé deux fois et il demande un remboursement pour la facture qui a été payée le mois dernier avec la carte de crédit",
+ "detect_script": "latin",
+ "script_profile": {
+ "latin": 1.0
+ },
+ "latin_profile": {
+ "language": "fr",
+ "english_hits": 0,
+ "diacritic_rate": 0.051470588235294115,
+ "looks_non_english": true
+ },
+ "analyse": {
+ "script": "latin",
+ "script_profile": {
+ "latin": 1.0
+ },
+ "language": "fr",
+ "is_english": false,
+ "language_undecided": false,
+ "diacritic_rate": 0.0515,
+ "non_latin_fraction": 0.0
+ }
+ },
+ {
+ "label": "is_english/german_long",
+ "state": "Der Kunde wurde zweimal belastet und möchte eine Rückerstattung für die Rechnung die nicht korrekt ist und auch nicht bezahlt wurde",
+ "state_text": "Der Kunde wurde zweimal belastet und möchte eine Rückerstattung für die Rechnung die nicht korrekt ist und auch nicht bezahlt wurde",
+ "detect_script": "latin",
+ "script_profile": {
+ "latin": 1.0
+ },
+ "latin_profile": {
+ "language": "de",
+ "english_hits": 0,
+ "diacritic_rate": 0.022900763358778626,
+ "looks_non_english": true
+ },
+ "analyse": {
+ "script": "latin",
+ "script_profile": {
+ "latin": 1.0
+ },
+ "language": "de",
+ "is_english": false,
+ "language_undecided": false,
+ "diacritic_rate": 0.0229,
+ "non_latin_fraction": 0.0
+ }
+ },
+ {
+ "label": "is_english/romanian",
+ "state": "Gătește-mi o rețetă de sarmale de post pentru mâine.",
+ "state_text": "Gătește-mi o rețetă de sarmale de post pentru mâine.",
+ "detect_script": "latin",
+ "script_profile": {
+ "latin": 1.0
+ },
+ "latin_profile": {
+ "language": "ro",
+ "english_hits": 0,
+ "diacritic_rate": 0.09615384615384616,
+ "looks_non_english": true
+ },
+ "analyse": {
+ "script": "latin",
+ "script_profile": {
+ "latin": 1.0
+ },
+ "language": "ro",
+ "is_english": false,
+ "language_undecided": false,
+ "diacritic_rate": 0.0962,
+ "non_latin_fraction": 0.0
+ }
+ },
+ {
+ "label": "is_english/romanian_invoice",
+ "state": "Am fost taxat de două ori pentru factura din luna martie și vreau banii",
+ "state_text": "Am fost taxat de două ori pentru factura din luna martie și vreau banii",
+ "detect_script": "latin",
+ "script_profile": {
+ "latin": 1.0
+ },
+ "latin_profile": {
+ "language": "ro",
+ "english_hits": 0,
+ "diacritic_rate": 0.028169014084507043,
+ "looks_non_english": true
+ },
+ "analyse": {
+ "script": "latin",
+ "script_profile": {
+ "latin": 1.0
+ },
+ "language": "ro",
+ "is_english": false,
+ "language_undecided": false,
+ "diacritic_rate": 0.0282,
+ "non_latin_fraction": 0.0
+ }
+ },
+ {
+ "label": "is_english/polish",
+ "state": "Klient został obciążony dwukrotnie i chce zwrot pieniędzy za fakturę",
+ "state_text": "Klient został obciążony dwukrotnie i chce zwrot pieniędzy za fakturę",
+ "detect_script": "latin",
+ "script_profile": {
+ "latin": 1.0
+ },
+ "latin_profile": {
+ "language": null,
+ "english_hits": 1,
+ "diacritic_rate": 0.07352941176470588,
+ "looks_non_english": true
+ },
+ "analyse": {
+ "script": "latin",
+ "script_profile": {
+ "latin": 1.0
+ },
+ "language": null,
+ "is_english": false,
+ "language_undecided": true,
+ "diacritic_rate": 0.0735,
+ "non_latin_fraction": 0.0
+ }
+ },
+ {
+ "label": "is_english/czech",
+ "state": "Zákazníkovi byla částka účtována dvakrát a žádá o vrácení peněz",
+ "state_text": "Zákazníkovi byla částka účtována dvakrát a žádá o vrácení peněz",
+ "detect_script": "latin",
+ "script_profile": {
+ "latin": 1.0
+ },
+ "latin_profile": {
+ "language": null,
+ "english_hits": 0,
+ "diacritic_rate": 0.2222222222222222,
+ "looks_non_english": true
+ },
+ "analyse": {
+ "script": "latin",
+ "script_profile": {
+ "latin": 1.0
+ },
+ "language": null,
+ "is_english": false,
+ "language_undecided": true,
+ "diacritic_rate": 0.2222,
+ "non_latin_fraction": 0.0
+ }
+ },
+ {
+ "label": "is_english/turkish",
+ "state": "Müşteriden iki kez ücret alındı ve para iadesi istiyor lütfen yardım",
+ "state_text": "Müşteriden iki kez ücret alındı ve para iadesi istiyor lütfen yardım",
+ "detect_script": "latin",
+ "script_profile": {
+ "latin": 1.0
+ },
+ "latin_profile": {
+ "language": null,
+ "english_hits": 0,
+ "diacritic_rate": 0.10294117647058823,
+ "looks_non_english": true
+ },
+ "analyse": {
+ "script": "latin",
+ "script_profile": {
+ "latin": 1.0
+ },
+ "language": null,
+ "is_english": false,
+ "language_undecided": true,
+ "diacritic_rate": 0.1029,
+ "non_latin_fraction": 0.0
+ }
+ },
+ {
+ "label": "is_english/vietnamese",
+ "state": "Khách hàng đã bị thu phí hai lần và muốn được hoàn tiền ngay",
+ "state_text": "Khách hàng đã bị thu phí hai lần và muốn được hoàn tiền ngay",
+ "detect_script": "latin",
+ "script_profile": {
+ "latin": 1.0
+ },
+ "latin_profile": {
+ "language": null,
+ "english_hits": 0,
+ "diacritic_rate": 0.13333333333333333,
+ "looks_non_english": true
+ },
+ "analyse": {
+ "script": "latin",
+ "script_profile": {
+ "latin": 1.0
+ },
+ "language": null,
+ "is_english": false,
+ "language_undecided": true,
+ "diacritic_rate": 0.1333,
+ "non_latin_fraction": 0.0
+ }
+ },
+ {
+ "label": "is_english/english_with_loanwords",
+ "state": "We visited a cafe in Zurich and the naive assumption about the invoice was wrong, so please refund the duplicate charge",
+ "state_text": "We visited a cafe in Zurich and the naive assumption about the invoice was wrong, so please refund the duplicate charge",
+ "detect_script": "latin",
+ "script_profile": {
+ "latin": 1.0
+ },
+ "latin_profile": {
+ "language": "en",
+ "english_hits": 8,
+ "diacritic_rate": 0.0,
+ "looks_non_english": false
+ },
+ "analyse": {
+ "script": "latin",
+ "script_profile": {
+ "latin": 1.0
+ },
+ "language": "en",
+ "is_english": true,
+ "language_undecided": false,
+ "diacritic_rate": 0.0,
+ "non_latin_fraction": 0.0
+ }
+ },
+ {
+ "label": "is_english/undecided_turkish",
+ "state": "Müşteriden iki kez ücret alındı ve para iadesi istiyor",
+ "state_text": "Müşteriden iki kez ücret alındı ve para iadesi istiyor",
+ "detect_script": "latin",
+ "script_profile": {
+ "latin": 1.0
+ },
+ "latin_profile": {
+ "language": null,
+ "english_hits": 0,
+ "diacritic_rate": 0.09259259259259259,
+ "looks_non_english": true
+ },
+ "analyse": {
+ "script": "latin",
+ "script_profile": {
+ "latin": 1.0
+ },
+ "language": null,
+ "is_english": false,
+ "language_undecided": true,
+ "diacritic_rate": 0.0926,
+ "non_latin_fraction": 0.0
+ }
+ },
+ {
+ "label": "is_english/english_not_undecided",
+ "state": "Please refund the duplicate charge on the invoice",
+ "state_text": "Please refund the duplicate charge on the invoice",
+ "detect_script": "latin",
+ "script_profile": {
+ "latin": 1.0
+ },
+ "latin_profile": {
+ "language": "en",
+ "english_hits": 4,
+ "diacritic_rate": 0.0,
+ "looks_non_english": false
+ },
+ "analyse": {
+ "script": "latin",
+ "script_profile": {
+ "latin": 1.0
+ },
+ "language": "en",
+ "is_english": true,
+ "language_undecided": false,
+ "diacritic_rate": 0.0,
+ "non_latin_fraction": 0.0
+ }
+ },
+ {
+ "label": "is_english/romanian_diacritics",
+ "state": "Gătește-mi o rețetă de sarmale",
+ "state_text": "Gătește-mi o rețetă de sarmale",
+ "detect_script": "latin",
+ "script_profile": {
+ "latin": 1.0
+ },
+ "latin_profile": {
+ "language": null,
+ "english_hits": 0,
+ "diacritic_rate": 0.13333333333333333,
+ "looks_non_english": true
+ },
+ "analyse": {
+ "script": "latin",
+ "script_profile": {
+ "latin": 1.0
+ },
+ "language": null,
+ "is_english": false,
+ "language_undecided": true,
+ "diacritic_rate": 0.1333,
+ "non_latin_fraction": 0.0
+ }
+ },
+ {
+ "label": "is_english/no_diacritics",
+ "state": "Please refund the duplicate charge today",
+ "state_text": "Please refund the duplicate charge today",
+ "detect_script": "latin",
+ "script_profile": {
+ "latin": 1.0
+ },
+ "latin_profile": {
+ "language": "en",
+ "english_hits": 2,
+ "diacritic_rate": 0.0,
+ "looks_non_english": false
+ },
+ "analyse": {
+ "script": "latin",
+ "script_profile": {
+ "latin": 1.0
+ },
+ "language": "en",
+ "is_english": true,
+ "language_undecided": false,
+ "diacritic_rate": 0.0,
+ "non_latin_fraction": 0.0
+ }
+ },
+ {
+ "label": "is_english/zero_tie_romanian",
+ "state": "Cât e ora acum la Tokyo",
+ "state_text": "Cât e ora acum la Tokyo",
+ "detect_script": "latin",
+ "script_profile": {
+ "latin": 1.0
+ },
+ "latin_profile": {
+ "language": null,
+ "english_hits": 0,
+ "diacritic_rate": 0.043478260869565216,
+ "looks_non_english": true
+ },
+ "analyse": {
+ "script": "latin",
+ "script_profile": {
+ "latin": 1.0
+ },
+ "language": null,
+ "is_english": false,
+ "language_undecided": true,
+ "diacritic_rate": 0.0435,
+ "non_latin_fraction": 0.0
+ }
+ },
+ {
+ "label": "is_english/known_gap_romanian_no_diacritics",
+ "state": "Care este ora in Tokyo?",
+ "state_text": "Care este ora in Tokyo?",
+ "detect_script": "latin",
+ "script_profile": {
+ "latin": 1.0
+ },
+ "latin_profile": {
+ "language": "en",
+ "english_hits": 1,
+ "diacritic_rate": 0.0,
+ "looks_non_english": false
+ },
+ "analyse": {
+ "script": "latin",
+ "script_profile": {
+ "latin": 1.0
+ },
+ "language": "en",
+ "is_english": true,
+ "language_undecided": false,
+ "diacritic_rate": 0.0,
+ "non_latin_fraction": 0.0
+ }
+ },
+ {
+ "label": "latin_lang/english",
+ "state": "The customer was charged twice and wants a refund for this invoice",
+ "state_text": "The customer was charged twice and wants a refund for this invoice",
+ "detect_script": "latin",
+ "script_profile": {
+ "latin": 1.0
+ },
+ "latin_profile": {
+ "language": "en",
+ "english_hits": 5,
+ "diacritic_rate": 0.0,
+ "looks_non_english": false
+ },
+ "analyse": {
+ "script": "latin",
+ "script_profile": {
+ "latin": 1.0
+ },
+ "language": "en",
+ "is_english": true,
+ "language_undecided": false,
+ "diacritic_rate": 0.0,
+ "non_latin_fraction": 0.0
+ }
+ },
+ {
+ "label": "latin_lang/french",
+ "state": "Le client a ete facture deux fois et il demande un remboursement pour la facture",
+ "state_text": "Le client a ete facture deux fois et il demande un remboursement pour la facture",
+ "detect_script": "latin",
+ "script_profile": {
+ "latin": 1.0
+ },
+ "latin_profile": {
+ "language": "fr",
+ "english_hits": 0,
+ "diacritic_rate": 0.0,
+ "looks_non_english": false
+ },
+ "analyse": {
+ "script": "latin",
+ "script_profile": {
+ "latin": 1.0
+ },
+ "language": "fr",
+ "is_english": false,
+ "language_undecided": false,
+ "diacritic_rate": 0.0,
+ "non_latin_fraction": 0.0
+ }
+ },
+ {
+ "label": "latin_lang/german",
+ "state": "Der Kunde wurde zweimal belastet und moechte eine Rueckerstattung fuer die Rechnung",
+ "state_text": "Der Kunde wurde zweimal belastet und moechte eine Rueckerstattung fuer die Rechnung",
+ "detect_script": "latin",
+ "script_profile": {
+ "latin": 1.0
+ },
+ "latin_profile": {
+ "language": "de",
+ "english_hits": 0,
+ "diacritic_rate": 0.0,
+ "looks_non_english": false
+ },
+ "analyse": {
+ "script": "latin",
+ "script_profile": {
+ "latin": 1.0
+ },
+ "language": "de",
+ "is_english": false,
+ "language_undecided": false,
+ "diacritic_rate": 0.0,
+ "non_latin_fraction": 0.0
+ }
+ },
+ {
+ "label": "latin_lang/spanish",
+ "state": "El cliente fue cobrado dos veces y quiere que le devuelvan el dinero por la factura",
+ "state_text": "El cliente fue cobrado dos veces y quiere que le devuelvan el dinero por la factura",
+ "detect_script": "latin",
+ "script_profile": {
+ "latin": 1.0
+ },
+ "latin_profile": {
+ "language": "es",
+ "english_hits": 0,
+ "diacritic_rate": 0.0,
+ "looks_non_english": false
+ },
+ "analyse": {
+ "script": "latin",
+ "script_profile": {
+ "latin": 1.0
+ },
+ "language": "es",
+ "is_english": false,
+ "language_undecided": false,
+ "diacritic_rate": 0.0,
+ "non_latin_fraction": 0.0
+ }
+ },
+ {
+ "label": "latin_lang/too_short",
+ "state": "refund",
+ "state_text": "refund",
+ "detect_script": "latin",
+ "script_profile": {
+ "latin": 1.0
+ },
+ "latin_profile": {
+ "language": null,
+ "english_hits": 0,
+ "diacritic_rate": 0.0,
+ "looks_non_english": false
+ },
+ "analyse": {
+ "script": "latin",
+ "script_profile": {
+ "latin": 1.0
+ },
+ "language": null,
+ "is_english": true,
+ "language_undecided": true,
+ "diacritic_rate": 0.0,
+ "non_latin_fraction": 0.0
+ }
+ },
+ {
+ "label": "latin_lang/long_english_stays_en",
+ "state": "Please refund the duplicate charge on invoice 4411 today because we have been waiting for three days and nobody has replied to us",
+ "state_text": "Please refund the duplicate charge on invoice 4411 today because we have been waiting for three days and nobody has replied to us",
+ "detect_script": "latin",
+ "script_profile": {
+ "latin": 1.0
+ },
+ "latin_profile": {
+ "language": "en",
+ "english_hits": 9,
+ "diacritic_rate": 0.0,
+ "looks_non_english": false
+ },
+ "analyse": {
+ "script": "latin",
+ "script_profile": {
+ "latin": 1.0
+ },
+ "language": "en",
+ "is_english": true,
+ "language_undecided": false,
+ "diacritic_rate": 0.0,
+ "non_latin_fraction": 0.0
+ }
+ },
+ {
+ "label": "state_text/dict",
+ "state": {
+ "body": "charged twice",
+ "n": 3
+ },
+ "state_text": "charged twice",
+ "detect_script": "latin",
+ "script_profile": {
+ "latin": 1.0
+ },
+ "latin_profile": {
+ "language": null,
+ "english_hits": 0,
+ "diacritic_rate": 0.0,
+ "looks_non_english": false
+ },
+ "analyse": {
+ "script": "latin",
+ "script_profile": {
+ "latin": 1.0
+ },
+ "language": null,
+ "is_english": true,
+ "language_undecided": true,
+ "diacritic_rate": 0.0,
+ "non_latin_fraction": 0.0
+ }
+ },
+ {
+ "label": "state_text/nested",
+ "state": {
+ "a": {
+ "b": [
+ "deep"
+ ]
+ }
+ },
+ "state_text": "deep",
+ "detect_script": "latin",
+ "script_profile": {
+ "latin": 1.0
+ },
+ "latin_profile": {
+ "language": null,
+ "english_hits": 0,
+ "diacritic_rate": 0.0,
+ "looks_non_english": false
+ },
+ "analyse": {
+ "script": "latin",
+ "script_profile": {
+ "latin": 1.0
+ },
+ "language": null,
+ "is_english": true,
+ "language_undecided": true,
+ "diacritic_rate": 0.0,
+ "non_latin_fraction": 0.0
+ }
+ },
+ {
+ "label": "state_text/list",
+ "state": [
+ "x",
+ {
+ "y": "z"
+ }
+ ],
+ "state_text": "x z",
+ "detect_script": "latin",
+ "script_profile": {
+ "latin": 1.0
+ },
+ "latin_profile": {
+ "language": null,
+ "english_hits": 0,
+ "diacritic_rate": 0.0,
+ "looks_non_english": false
+ },
+ "analyse": {
+ "script": "latin",
+ "script_profile": {
+ "latin": 1.0
+ },
+ "language": null,
+ "is_english": true,
+ "language_undecided": true,
+ "diacritic_rate": 0.0,
+ "non_latin_fraction": 0.0
+ }
+ },
+ {
+ "label": "state_text/none",
+ "state": null,
+ "state_text": "",
+ "detect_script": "unknown",
+ "script_profile": {},
+ "latin_profile": {
+ "language": null,
+ "english_hits": 0,
+ "diacritic_rate": 0.0,
+ "looks_non_english": false
+ },
+ "analyse": {
+ "script": "unknown",
+ "script_profile": {},
+ "language": null,
+ "is_english": true,
+ "language_undecided": true,
+ "diacritic_rate": 0.0,
+ "non_latin_fraction": 0.0
+ }
+ },
+ {
+ "label": "state_text/keys_ignored",
+ "state": {
+ "subject": "नमस्ते",
+ "body": "ग्राहक से दो बार शुल्क लिया गया"
+ },
+ "state_text": "नमस्ते ग्राहक से दो बार शुल्क लिया गया",
+ "detect_script": "devanagari",
+ "script_profile": {
+ "devanagari": 1.0
+ },
+ "latin_profile": {
+ "language": null,
+ "english_hits": 0,
+ "diacritic_rate": 0.0,
+ "looks_non_english": false
+ },
+ "analyse": {
+ "script": "devanagari",
+ "script_profile": {
+ "devanagari": 1.0
+ },
+ "language": null,
+ "is_english": false,
+ "language_undecided": true,
+ "diacritic_rate": 0.0,
+ "non_latin_fraction": 1.0
+ }
+ },
+ {
+ "label": "profile/armenian_pure",
+ "state": "Հայերեն",
+ "state_text": "Հայերեն",
+ "detect_script": "armenian",
+ "script_profile": {
+ "armenian": 1.0
+ },
+ "latin_profile": {
+ "language": null,
+ "english_hits": 0,
+ "diacritic_rate": 0.0,
+ "looks_non_english": false
+ },
+ "analyse": {
+ "script": "armenian",
+ "script_profile": {
+ "armenian": 1.0
+ },
+ "language": null,
+ "is_english": false,
+ "language_undecided": true,
+ "diacritic_rate": 0.0,
+ "non_latin_fraction": 1.0
+ }
+ },
+ {
+ "label": "profile/armenian_mixed_with_latin",
+ "state": "Հայերեն abc",
+ "state_text": "Հայերեն abc",
+ "detect_script": "armenian",
+ "script_profile": {
+ "latin": 0.3,
+ "armenian": 0.7
+ },
+ "latin_profile": {
+ "language": null,
+ "english_hits": 0,
+ "diacritic_rate": 0.0,
+ "looks_non_english": false
+ },
+ "analyse": {
+ "script": "armenian",
+ "script_profile": {
+ "latin": 0.3,
+ "armenian": 0.7
+ },
+ "language": null,
+ "is_english": false,
+ "language_undecided": true,
+ "diacritic_rate": 0.0,
+ "non_latin_fraction": 0.7
+ }
+ },
+ {
+ "label": "plain_ascii/es_1",
+ "state": "El pedido llego roto y nadie responde cuando escribo al soporte",
+ "state_text": "El pedido llego roto y nadie responde cuando escribo al soporte",
+ "detect_script": "latin",
+ "script_profile": {
+ "latin": 1.0
+ },
+ "latin_profile": {
+ "language": "es",
+ "english_hits": 0,
+ "diacritic_rate": 0.0,
+ "looks_non_english": false
+ },
+ "analyse": {
+ "script": "latin",
+ "script_profile": {
+ "latin": 1.0
+ },
+ "language": "es",
+ "is_english": false,
+ "language_undecided": false,
+ "diacritic_rate": 0.0,
+ "non_latin_fraction": 0.0
+ }
+ },
+ {
+ "label": "plain_ascii/es_2",
+ "state": "Quiero cancelar mi plan y pedir un reembolso",
+ "state_text": "Quiero cancelar mi plan y pedir un reembolso",
+ "detect_script": "latin",
+ "script_profile": {
+ "latin": 1.0
+ },
+ "latin_profile": {
+ "language": "es",
+ "english_hits": 0,
+ "diacritic_rate": 0.0,
+ "looks_non_english": false
+ },
+ "analyse": {
+ "script": "latin",
+ "script_profile": {
+ "latin": 1.0
+ },
+ "language": "es",
+ "is_english": false,
+ "language_undecided": false,
+ "diacritic_rate": 0.0,
+ "non_latin_fraction": 0.0
+ }
+ },
+ {
+ "label": "plain_ascii/es_3",
+ "state": "La factura tiene un error en el importe total",
+ "state_text": "La factura tiene un error en el importe total",
+ "detect_script": "latin",
+ "script_profile": {
+ "latin": 1.0
+ },
+ "latin_profile": {
+ "language": "es",
+ "english_hits": 0,
+ "diacritic_rate": 0.0,
+ "looks_non_english": false
+ },
+ "analyse": {
+ "script": "latin",
+ "script_profile": {
+ "latin": 1.0
+ },
+ "language": "es",
+ "is_english": false,
+ "language_undecided": false,
+ "diacritic_rate": 0.0,
+ "non_latin_fraction": 0.0
+ }
+ },
+ {
+ "label": "plain_ascii/es_4",
+ "state": "Necesito que me devuelvan el dinero de la compra duplicada",
+ "state_text": "Necesito que me devuelvan el dinero de la compra duplicada",
+ "detect_script": "latin",
+ "script_profile": {
+ "latin": 1.0
+ },
+ "latin_profile": {
+ "language": "es",
+ "english_hits": 0,
+ "diacritic_rate": 0.0,
+ "looks_non_english": false
+ },
+ "analyse": {
+ "script": "latin",
+ "script_profile": {
+ "latin": 1.0
+ },
+ "language": "es",
+ "is_english": false,
+ "language_undecided": false,
+ "diacritic_rate": 0.0,
+ "non_latin_fraction": 0.0
+ }
+ },
+ {
+ "label": "plain_ascii/it_1",
+ "state": "Il cliente e stato addebitato due volte e vuole un rimborso",
+ "state_text": "Il cliente e stato addebitato due volte e vuole un rimborso",
+ "detect_script": "latin",
+ "script_profile": {
+ "latin": 1.0
+ },
+ "latin_profile": {
+ "language": "it",
+ "english_hits": 0,
+ "diacritic_rate": 0.0,
+ "looks_non_english": false
+ },
+ "analyse": {
+ "script": "latin",
+ "script_profile": {
+ "latin": 1.0
+ },
+ "language": "it",
+ "is_english": false,
+ "language_undecided": false,
+ "diacritic_rate": 0.0,
+ "non_latin_fraction": 0.0
+ }
+ },
+ {
+ "label": "plain_ascii/it_2",
+ "state": "Voglio cancellare il mio abbonamento e chiedere un rimborso",
+ "state_text": "Voglio cancellare il mio abbonamento e chiedere un rimborso",
+ "detect_script": "latin",
+ "script_profile": {
+ "latin": 1.0
+ },
+ "latin_profile": {
+ "language": "it",
+ "english_hits": 0,
+ "diacritic_rate": 0.0,
+ "looks_non_english": false
+ },
+ "analyse": {
+ "script": "latin",
+ "script_profile": {
+ "latin": 1.0
+ },
+ "language": "it",
+ "is_english": false,
+ "language_undecided": false,
+ "diacritic_rate": 0.0,
+ "non_latin_fraction": 0.0
+ }
+ },
+ {
+ "label": "plain_ascii/it_3",
+ "state": "La fattura contiene un errore nell importo totale",
+ "state_text": "La fattura contiene un errore nell importo totale",
+ "detect_script": "latin",
+ "script_profile": {
+ "latin": 1.0
+ },
+ "latin_profile": {
+ "language": "it",
+ "english_hits": 0,
+ "diacritic_rate": 0.0,
+ "looks_non_english": false
+ },
+ "analyse": {
+ "script": "latin",
+ "script_profile": {
+ "latin": 1.0
+ },
+ "language": "it",
+ "is_english": false,
+ "language_undecided": false,
+ "diacritic_rate": 0.0,
+ "non_latin_fraction": 0.0
+ }
+ },
+ {
+ "label": "plain_ascii/pt_1",
+ "state": "O cliente foi cobrado duas vezes e quer o dinheiro de volta",
+ "state_text": "O cliente foi cobrado duas vezes e quer o dinheiro de volta",
+ "detect_script": "latin",
+ "script_profile": {
+ "latin": 1.0
+ },
+ "latin_profile": {
+ "language": "pt",
+ "english_hits": 0,
+ "diacritic_rate": 0.0,
+ "looks_non_english": false
+ },
+ "analyse": {
+ "script": "latin",
+ "script_profile": {
+ "latin": 1.0
+ },
+ "language": "pt",
+ "is_english": false,
+ "language_undecided": false,
+ "diacritic_rate": 0.0,
+ "non_latin_fraction": 0.0
+ }
+ },
+ {
+ "label": "plain_ascii/fr_1",
+ "state": "Le client a ete facture deux fois et demande un remboursement",
+ "state_text": "Le client a ete facture deux fois et demande un remboursement",
+ "detect_script": "latin",
+ "script_profile": {
+ "latin": 1.0
+ },
+ "latin_profile": {
+ "language": "fr",
+ "english_hits": 0,
+ "diacritic_rate": 0.0,
+ "looks_non_english": false
+ },
+ "analyse": {
+ "script": "latin",
+ "script_profile": {
+ "latin": 1.0
+ },
+ "language": "fr",
+ "is_english": false,
+ "language_undecided": false,
+ "diacritic_rate": 0.0,
+ "non_latin_fraction": 0.0
+ }
+ },
+ {
+ "label": "plain_ascii/fr_2",
+ "state": "Je ne peux pas acceder a mon compte et j ai besoin d aide",
+ "state_text": "Je ne peux pas acceder a mon compte et j ai besoin d aide",
+ "detect_script": "latin",
+ "script_profile": {
+ "latin": 1.0
+ },
+ "latin_profile": {
+ "language": "fr",
+ "english_hits": 0,
+ "diacritic_rate": 0.0,
+ "looks_non_english": false
+ },
+ "analyse": {
+ "script": "latin",
+ "script_profile": {
+ "latin": 1.0
+ },
+ "language": "fr",
+ "is_english": false,
+ "language_undecided": false,
+ "diacritic_rate": 0.0,
+ "non_latin_fraction": 0.0
+ }
+ },
+ {
+ "label": "accented/es",
+ "state": "La facturación tiene un error y necesito una corrección urgente",
+ "state_text": "La facturación tiene un error y necesito una corrección urgente",
+ "detect_script": "latin",
+ "script_profile": {
+ "latin": 1.0
+ },
+ "latin_profile": {
+ "language": "es",
+ "english_hits": 0,
+ "diacritic_rate": 0.031746031746031744,
+ "looks_non_english": true
+ },
+ "analyse": {
+ "script": "latin",
+ "script_profile": {
+ "latin": 1.0
+ },
+ "language": "es",
+ "is_english": false,
+ "language_undecided": false,
+ "diacritic_rate": 0.0317,
+ "non_latin_fraction": 0.0
+ }
+ },
+ {
+ "label": "accented/it",
+ "state": "La fattura è sbagliata, devo avere un rimborso per il pagamento",
+ "state_text": "La fattura è sbagliata, devo avere un rimborso per il pagamento",
+ "detect_script": "latin",
+ "script_profile": {
+ "latin": 1.0
+ },
+ "latin_profile": {
+ "language": "it",
+ "english_hits": 0,
+ "diacritic_rate": 0.015873015873015872,
+ "looks_non_english": false
+ },
+ "analyse": {
+ "script": "latin",
+ "script_profile": {
+ "latin": 1.0
+ },
+ "language": "it",
+ "is_english": false,
+ "language_undecided": false,
+ "diacritic_rate": 0.0159,
+ "non_latin_fraction": 0.0
+ }
+ },
+ {
+ "label": "accented/fr",
+ "state": "La commande est arrivée cassée et personne ne répond au support",
+ "state_text": "La commande est arrivée cassée et personne ne répond au support",
+ "detect_script": "latin",
+ "script_profile": {
+ "latin": 1.0
+ },
+ "latin_profile": {
+ "language": "fr",
+ "english_hits": 0,
+ "diacritic_rate": 0.047619047619047616,
+ "looks_non_english": true
+ },
+ "analyse": {
+ "script": "latin",
+ "script_profile": {
+ "latin": 1.0
+ },
+ "language": "fr",
+ "is_english": false,
+ "language_undecided": false,
+ "diacritic_rate": 0.0476,
+ "non_latin_fraction": 0.0
+ }
+ },
+ {
+ "label": "romance_control/1",
+ "state": "The customer was charged twice and wants a refund for this invoice",
+ "state_text": "The customer was charged twice and wants a refund for this invoice",
+ "detect_script": "latin",
+ "script_profile": {
+ "latin": 1.0
+ },
+ "latin_profile": {
+ "language": "en",
+ "english_hits": 5,
+ "diacritic_rate": 0.0,
+ "looks_non_english": false
+ },
+ "analyse": {
+ "script": "latin",
+ "script_profile": {
+ "latin": 1.0
+ },
+ "language": "en",
+ "is_english": true,
+ "language_undecided": false,
+ "diacritic_rate": 0.0,
+ "non_latin_fraction": 0.0
+ }
+ },
+ {
+ "label": "romance_control/2",
+ "state": "Please cancel my subscription and refund the duplicate charge today",
+ "state_text": "Please cancel my subscription and refund the duplicate charge today",
+ "detect_script": "latin",
+ "script_profile": {
+ "latin": 1.0
+ },
+ "latin_profile": {
+ "language": "en",
+ "english_hits": 3,
+ "diacritic_rate": 0.0,
+ "looks_non_english": false
+ },
+ "analyse": {
+ "script": "latin",
+ "script_profile": {
+ "latin": 1.0
+ },
+ "language": "en",
+ "is_english": true,
+ "language_undecided": false,
+ "diacritic_rate": 0.0,
+ "non_latin_fraction": 0.0
+ }
+ },
+ {
+ "label": "romance_control/3",
+ "state": "The report by Smith et al. shows the de facto standard, e.g. the LA office and Rio",
+ "state_text": "The report by Smith et al. shows the de facto standard, e.g. the LA office and Rio",
+ "detect_script": "latin",
+ "script_profile": {
+ "latin": 1.0
+ },
+ "latin_profile": {
+ "language": "en",
+ "english_hits": 4,
+ "diacritic_rate": 0.0,
+ "looks_non_english": false
+ },
+ "analyse": {
+ "script": "latin",
+ "script_profile": {
+ "latin": 1.0
+ },
+ "language": "en",
+ "is_english": true,
+ "language_undecided": false,
+ "diacritic_rate": 0.0,
+ "non_latin_fraction": 0.0
+ }
+ },
+ {
+ "label": "romance_control/4",
+ "state": "Our MI5 and UN contacts discussed the DOS attack in LA last month",
+ "state_text": "Our MI5 and UN contacts discussed the DOS attack in LA last month",
+ "detect_script": "latin",
+ "script_profile": {
+ "latin": 1.0
+ },
+ "latin_profile": {
+ "language": "en",
+ "english_hits": 3,
+ "diacritic_rate": 0.0,
+ "looks_non_english": false
+ },
+ "analyse": {
+ "script": "latin",
+ "script_profile": {
+ "latin": 1.0
+ },
+ "language": "en",
+ "is_english": true,
+ "language_undecided": false,
+ "diacritic_rate": 0.0,
+ "non_latin_fraction": 0.0
+ }
+ },
+ {
+ "label": "romance_control/5",
+ "state": "No refund was issued, so I am writing to you again about invoice 4411",
+ "state_text": "No refund was issued, so I am writing to you again about invoice 4411",
+ "detect_script": "latin",
+ "script_profile": {
+ "latin": 1.0
+ },
+ "latin_profile": {
+ "language": "en",
+ "english_hits": 4,
+ "diacritic_rate": 0.0,
+ "looks_non_english": false
+ },
+ "analyse": {
+ "script": "latin",
+ "script_profile": {
+ "latin": 1.0
+ },
+ "language": "en",
+ "is_english": true,
+ "language_undecided": false,
+ "diacritic_rate": 0.0,
+ "non_latin_fraction": 0.0
+ }
+ },
+ {
+ "label": "romance_control/6",
+ "state": "no refund no reply",
+ "state_text": "no refund no reply",
+ "detect_script": "latin",
+ "script_profile": {
+ "latin": 1.0
+ },
+ "latin_profile": {
+ "language": null,
+ "english_hits": 0,
+ "diacritic_rate": 0.0,
+ "looks_non_english": false
+ },
+ "analyse": {
+ "script": "latin",
+ "script_profile": {
+ "latin": 1.0
+ },
+ "language": null,
+ "is_english": true,
+ "language_undecided": true,
+ "diacritic_rate": 0.0,
+ "non_latin_fraction": 0.0
+ }
+ },
+ {
+ "label": "romance_control/7",
+ "state": "The son of the director filed a complaint about the duplicate invoice",
+ "state_text": "The son of the director filed a complaint about the duplicate invoice",
+ "detect_script": "latin",
+ "script_profile": {
+ "latin": 1.0
+ },
+ "latin_profile": {
+ "language": "en",
+ "english_hits": 4,
+ "diacritic_rate": 0.0,
+ "looks_non_english": false
+ },
+ "analyse": {
+ "script": "latin",
+ "script_profile": {
+ "latin": 1.0
+ },
+ "language": "en",
+ "is_english": true,
+ "language_undecided": false,
+ "diacritic_rate": 0.0,
+ "non_latin_fraction": 0.0
+ }
+ },
+ {
+ "label": "shared_words/names_nothing",
+ "state": "Cât e ora acum la Tokyo",
+ "state_text": "Cât e ora acum la Tokyo",
+ "detect_script": "latin",
+ "script_profile": {
+ "latin": 1.0
+ },
+ "latin_profile": {
+ "language": null,
+ "english_hits": 0,
+ "diacritic_rate": 0.043478260869565216,
+ "looks_non_english": true
+ },
+ "analyse": {
+ "script": "latin",
+ "script_profile": {
+ "latin": 1.0
+ },
+ "language": null,
+ "is_english": false,
+ "language_undecided": true,
+ "diacritic_rate": 0.0435,
+ "non_latin_fraction": 0.0
+ }
+ },
+ {
+ "label": "shared_words/distinctive_names_it",
+ "state": "La fattura contiene un errore nell importo totale",
+ "state_text": "La fattura contiene un errore nell importo totale",
+ "detect_script": "latin",
+ "script_profile": {
+ "latin": 1.0
+ },
+ "latin_profile": {
+ "language": "it",
+ "english_hits": 0,
+ "diacritic_rate": 0.0,
+ "looks_non_english": false
+ },
+ "analyse": {
+ "script": "latin",
+ "script_profile": {
+ "latin": 1.0
+ },
+ "language": "it",
+ "is_english": false,
+ "language_undecided": false,
+ "diacritic_rate": 0.0,
+ "non_latin_fraction": 0.0
+ }
+ },
+ {
+ "label": "shared_words/shared_hits_still_count_es",
+ "state": "La factura tiene un error en el importe total",
+ "state_text": "La factura tiene un error en el importe total",
+ "detect_script": "latin",
+ "script_profile": {
+ "latin": 1.0
+ },
+ "latin_profile": {
+ "language": "es",
+ "english_hits": 0,
+ "diacritic_rate": 0.0,
+ "looks_non_english": false
+ },
+ "analyse": {
+ "script": "latin",
+ "script_profile": {
+ "latin": 1.0
+ },
+ "language": "es",
+ "is_english": false,
+ "language_undecided": false,
+ "diacritic_rate": 0.0,
+ "non_latin_fraction": 0.0
+ }
+ },
+ {
+ "label": "all_scripts/greek",
+ "state": "Γεια σου κόσμε, πώς είσαι σήμερα;",
+ "state_text": "Γεια σου κόσμε, πώς είσαι σήμερα;",
+ "detect_script": "greek",
+ "script_profile": {
+ "greek": 1.0
+ },
+ "latin_profile": {
+ "language": null,
+ "english_hits": 0,
+ "diacritic_rate": 0.0,
+ "looks_non_english": false
+ },
+ "analyse": {
+ "script": "greek",
+ "script_profile": {
+ "greek": 1.0
+ },
+ "language": null,
+ "is_english": false,
+ "language_undecided": true,
+ "diacritic_rate": 0.0,
+ "non_latin_fraction": 1.0
+ }
+ },
+ {
+ "label": "all_scripts/cyrillic",
+ "state": "Привет, как ты себя чувствуешь сегодня?",
+ "state_text": "Привет, как ты себя чувствуешь сегодня?",
+ "detect_script": "cyrillic",
+ "script_profile": {
+ "cyrillic": 1.0
+ },
+ "latin_profile": {
+ "language": null,
+ "english_hits": 0,
+ "diacritic_rate": 0.0,
+ "looks_non_english": false
+ },
+ "analyse": {
+ "script": "cyrillic",
+ "script_profile": {
+ "cyrillic": 1.0
+ },
+ "language": null,
+ "is_english": false,
+ "language_undecided": true,
+ "diacritic_rate": 0.0,
+ "non_latin_fraction": 1.0
+ }
+ },
+ {
+ "label": "all_scripts/armenian",
+ "state": "Բարեւ ձեզ, ինչպե՞ս եք այսօր",
+ "state_text": "Բարեւ ձեզ, ինչպե՞ս եք այսօր",
+ "detect_script": "armenian",
+ "script_profile": {
+ "armenian": 1.0
+ },
+ "latin_profile": {
+ "language": null,
+ "english_hits": 0,
+ "diacritic_rate": 0.0,
+ "looks_non_english": false
+ },
+ "analyse": {
+ "script": "armenian",
+ "script_profile": {
+ "armenian": 1.0
+ },
+ "language": null,
+ "is_english": false,
+ "language_undecided": true,
+ "diacritic_rate": 0.0,
+ "non_latin_fraction": 1.0
+ }
+ },
+ {
+ "label": "all_scripts/hebrew",
+ "state": "שלום, מה שלומך היום",
+ "state_text": "שלום, מה שלומך היום",
+ "detect_script": "hebrew",
+ "script_profile": {
+ "hebrew": 1.0
+ },
+ "latin_profile": {
+ "language": null,
+ "english_hits": 0,
+ "diacritic_rate": 0.0,
+ "looks_non_english": false
+ },
+ "analyse": {
+ "script": "hebrew",
+ "script_profile": {
+ "hebrew": 1.0
+ },
+ "language": null,
+ "is_english": false,
+ "language_undecided": true,
+ "diacritic_rate": 0.0,
+ "non_latin_fraction": 1.0
+ }
+ },
+ {
+ "label": "all_scripts/arabic",
+ "state": "مرحبا كيف حالك اليوم",
+ "state_text": "مرحبا كيف حالك اليوم",
+ "detect_script": "arabic",
+ "script_profile": {
+ "arabic": 1.0
+ },
+ "latin_profile": {
+ "language": null,
+ "english_hits": 0,
+ "diacritic_rate": 0.0,
+ "looks_non_english": false
+ },
+ "analyse": {
+ "script": "arabic",
+ "script_profile": {
+ "arabic": 1.0
+ },
+ "language": null,
+ "is_english": false,
+ "language_undecided": true,
+ "diacritic_rate": 0.0,
+ "non_latin_fraction": 1.0
+ }
+ },
+ {
+ "label": "all_scripts/devanagari",
+ "state": "नमस्ते आप कैसे हैं आज",
+ "state_text": "नमस्ते आप कैसे हैं आज",
+ "detect_script": "devanagari",
+ "script_profile": {
+ "devanagari": 1.0
+ },
+ "latin_profile": {
+ "language": null,
+ "english_hits": 0,
+ "diacritic_rate": 0.0,
+ "looks_non_english": false
+ },
+ "analyse": {
+ "script": "devanagari",
+ "script_profile": {
+ "devanagari": 1.0
+ },
+ "language": null,
+ "is_english": false,
+ "language_undecided": true,
+ "diacritic_rate": 0.0,
+ "non_latin_fraction": 1.0
+ }
+ },
+ {
+ "label": "all_scripts/bengali",
+ "state": "আমি ভালো আছি, আপনি কেমন আছেন",
+ "state_text": "আমি ভালো আছি, আপনি কেমন আছেন",
+ "detect_script": "bengali",
+ "script_profile": {
+ "bengali": 1.0
+ },
+ "latin_profile": {
+ "language": null,
+ "english_hits": 0,
+ "diacritic_rate": 0.0,
+ "looks_non_english": false
+ },
+ "analyse": {
+ "script": "bengali",
+ "script_profile": {
+ "bengali": 1.0
+ },
+ "language": null,
+ "is_english": false,
+ "language_undecided": true,
+ "diacritic_rate": 0.0,
+ "non_latin_fraction": 1.0
+ }
+ },
+ {
+ "label": "all_scripts/gurmukhi",
+ "state": "ਤੁਸੀਂ ਕਿਵੇਂ ਹੋ, ਮੈਂ ਠੀਕ ਹਾਂ",
+ "state_text": "ਤੁਸੀਂ ਕਿਵੇਂ ਹੋ, ਮੈਂ ਠੀਕ ਹਾਂ",
+ "detect_script": "gurmukhi",
+ "script_profile": {
+ "gurmukhi": 1.0
+ },
+ "latin_profile": {
+ "language": null,
+ "english_hits": 0,
+ "diacritic_rate": 0.0,
+ "looks_non_english": false
+ },
+ "analyse": {
+ "script": "gurmukhi",
+ "script_profile": {
+ "gurmukhi": 1.0
+ },
+ "language": null,
+ "is_english": false,
+ "language_undecided": true,
+ "diacritic_rate": 0.0,
+ "non_latin_fraction": 1.0
+ }
+ },
+ {
+ "label": "all_scripts/gujarati",
+ "state": "તમે કેમ છો, હું સારો છું",
+ "state_text": "તમે કેમ છો, હું સારો છું",
+ "detect_script": "gujarati",
+ "script_profile": {
+ "gujarati": 1.0
+ },
+ "latin_profile": {
+ "language": null,
+ "english_hits": 0,
+ "diacritic_rate": 0.0,
+ "looks_non_english": false
+ },
+ "analyse": {
+ "script": "gujarati",
+ "script_profile": {
+ "gujarati": 1.0
+ },
+ "language": null,
+ "is_english": false,
+ "language_undecided": true,
+ "diacritic_rate": 0.0,
+ "non_latin_fraction": 1.0
+ }
+ },
+ {
+ "label": "all_scripts/oriya",
+ "state": "ତୁମେ କେମିତି ଅଛ, ମୁଁ ଭଲ ଅଛି",
+ "state_text": "ତୁମେ କେମିତି ଅଛ, ମୁଁ ଭଲ ଅଛି",
+ "detect_script": "oriya",
+ "script_profile": {
+ "oriya": 1.0
+ },
+ "latin_profile": {
+ "language": null,
+ "english_hits": 0,
+ "diacritic_rate": 0.0,
+ "looks_non_english": false
+ },
+ "analyse": {
+ "script": "oriya",
+ "script_profile": {
+ "oriya": 1.0
+ },
+ "language": null,
+ "is_english": false,
+ "language_undecided": true,
+ "diacritic_rate": 0.0,
+ "non_latin_fraction": 1.0
+ }
+ },
+ {
+ "label": "all_scripts/tamil",
+ "state": "நீங்கள் எப்படி இருக்கிறீர்கள்",
+ "state_text": "நீங்கள் எப்படி இருக்கிறீர்கள்",
+ "detect_script": "tamil",
+ "script_profile": {
+ "tamil": 1.0
+ },
+ "latin_profile": {
+ "language": null,
+ "english_hits": 0,
+ "diacritic_rate": 0.0,
+ "looks_non_english": false
+ },
+ "analyse": {
+ "script": "tamil",
+ "script_profile": {
+ "tamil": 1.0
+ },
+ "language": null,
+ "is_english": false,
+ "language_undecided": true,
+ "diacritic_rate": 0.0,
+ "non_latin_fraction": 1.0
+ }
+ },
+ {
+ "label": "all_scripts/telugu",
+ "state": "మీరు ఎలా ఉన్నారు, నేను బాగున్నాను",
+ "state_text": "మీరు ఎలా ఉన్నారు, నేను బాగున్నాను",
+ "detect_script": "telugu",
+ "script_profile": {
+ "telugu": 1.0
+ },
+ "latin_profile": {
+ "language": null,
+ "english_hits": 0,
+ "diacritic_rate": 0.0,
+ "looks_non_english": false
+ },
+ "analyse": {
+ "script": "telugu",
+ "script_profile": {
+ "telugu": 1.0
+ },
+ "language": null,
+ "is_english": false,
+ "language_undecided": true,
+ "diacritic_rate": 0.0,
+ "non_latin_fraction": 1.0
+ }
+ },
+ {
+ "label": "all_scripts/kannada",
+ "state": "ನೀವು ಹೇಗಿದ್ದೀರಿ, ನಾನು ಚೆನ್ನಾಗಿದ್ದೇನೆ",
+ "state_text": "ನೀವು ಹೇಗಿದ್ದೀರಿ, ನಾನು ಚೆನ್ನಾಗಿದ್ದೇನೆ",
+ "detect_script": "kannada",
+ "script_profile": {
+ "kannada": 1.0
+ },
+ "latin_profile": {
+ "language": null,
+ "english_hits": 0,
+ "diacritic_rate": 0.0,
+ "looks_non_english": false
+ },
+ "analyse": {
+ "script": "kannada",
+ "script_profile": {
+ "kannada": 1.0
+ },
+ "language": null,
+ "is_english": false,
+ "language_undecided": true,
+ "diacritic_rate": 0.0,
+ "non_latin_fraction": 1.0
+ }
+ },
+ {
+ "label": "all_scripts/malayalam",
+ "state": "നിങ്ങൾക്ക് സുഖമാണോ, എനിക്ക് സുഖമാണ്",
+ "state_text": "നിങ്ങൾക്ക് സുഖമാണോ, എനിക്ക് സുഖമാണ്",
+ "detect_script": "malayalam",
+ "script_profile": {
+ "malayalam": 1.0
+ },
+ "latin_profile": {
+ "language": null,
+ "english_hits": 0,
+ "diacritic_rate": 0.0,
+ "looks_non_english": false
+ },
+ "analyse": {
+ "script": "malayalam",
+ "script_profile": {
+ "malayalam": 1.0
+ },
+ "language": null,
+ "is_english": false,
+ "language_undecided": true,
+ "diacritic_rate": 0.0,
+ "non_latin_fraction": 1.0
+ }
+ },
+ {
+ "label": "all_scripts/sinhala",
+ "state": "ඔබට කොහොමද, මට හොඳින්",
+ "state_text": "ඔබට කොහොමද, මට හොඳින්",
+ "detect_script": "sinhala",
+ "script_profile": {
+ "sinhala": 1.0
+ },
+ "latin_profile": {
+ "language": null,
+ "english_hits": 0,
+ "diacritic_rate": 0.0,
+ "looks_non_english": false
+ },
+ "analyse": {
+ "script": "sinhala",
+ "script_profile": {
+ "sinhala": 1.0
+ },
+ "language": null,
+ "is_english": false,
+ "language_undecided": true,
+ "diacritic_rate": 0.0,
+ "non_latin_fraction": 1.0
+ }
+ },
+ {
+ "label": "all_scripts/thai",
+ "state": "คุณเป็นอย่างไรบ้างวันนี้",
+ "state_text": "คุณเป็นอย่างไรบ้างวันนี้",
+ "detect_script": "thai",
+ "script_profile": {
+ "thai": 1.0
+ },
+ "latin_profile": {
+ "language": null,
+ "english_hits": 0,
+ "diacritic_rate": 0.0,
+ "looks_non_english": false
+ },
+ "analyse": {
+ "script": "thai",
+ "script_profile": {
+ "thai": 1.0
+ },
+ "language": null,
+ "is_english": false,
+ "language_undecided": true,
+ "diacritic_rate": 0.0,
+ "non_latin_fraction": 1.0
+ }
+ },
+ {
+ "label": "all_scripts/lao",
+ "state": "ສະບາຍດີ ທ່ານເປັນແນວໃດແດ່",
+ "state_text": "ສະບາຍດີ ທ່ານເປັນແນວໃດແດ່",
+ "detect_script": "lao",
+ "script_profile": {
+ "lao": 1.0
+ },
+ "latin_profile": {
+ "language": null,
+ "english_hits": 0,
+ "diacritic_rate": 0.0,
+ "looks_non_english": false
+ },
+ "analyse": {
+ "script": "lao",
+ "script_profile": {
+ "lao": 1.0
+ },
+ "language": null,
+ "is_english": false,
+ "language_undecided": true,
+ "diacritic_rate": 0.0,
+ "non_latin_fraction": 1.0
+ }
+ },
+ {
+ "label": "all_scripts/tibetan",
+ "state": "བཀྲ་ཤིས་བདེ་ལེགས་ཁྱེད་རང་ག་འདྲ་ཡོད",
+ "state_text": "བཀྲ་ཤིས་བདེ་ལེགས་ཁྱེད་རང་ག་འདྲ་ཡོད",
+ "detect_script": "tibetan",
+ "script_profile": {
+ "tibetan": 1.0
+ },
+ "latin_profile": {
+ "language": null,
+ "english_hits": 0,
+ "diacritic_rate": 0.0,
+ "looks_non_english": false
+ },
+ "analyse": {
+ "script": "tibetan",
+ "script_profile": {
+ "tibetan": 1.0
+ },
+ "language": null,
+ "is_english": false,
+ "language_undecided": true,
+ "diacritic_rate": 0.0,
+ "non_latin_fraction": 1.0
+ }
+ },
+ {
+ "label": "all_scripts/myanmar",
+ "state": "မင်္ဂလာပါ သင်နေကောင်းလား",
+ "state_text": "မင်္ဂလာပါ သင်နေကောင်းလား",
+ "detect_script": "myanmar",
+ "script_profile": {
+ "myanmar": 1.0
+ },
+ "latin_profile": {
+ "language": null,
+ "english_hits": 0,
+ "diacritic_rate": 0.0,
+ "looks_non_english": false
+ },
+ "analyse": {
+ "script": "myanmar",
+ "script_profile": {
+ "myanmar": 1.0
+ },
+ "language": null,
+ "is_english": false,
+ "language_undecided": true,
+ "diacritic_rate": 0.0,
+ "non_latin_fraction": 1.0
+ }
+ },
+ {
+ "label": "all_scripts/georgian",
+ "state": "გამარჯობა როგორ ხარ დღეს",
+ "state_text": "გამარჯობა როგორ ხარ დღეს",
+ "detect_script": "georgian",
+ "script_profile": {
+ "georgian": 1.0
+ },
+ "latin_profile": {
+ "language": null,
+ "english_hits": 0,
+ "diacritic_rate": 0.0,
+ "looks_non_english": false
+ },
+ "analyse": {
+ "script": "georgian",
+ "script_profile": {
+ "georgian": 1.0
+ },
+ "language": null,
+ "is_english": false,
+ "language_undecided": true,
+ "diacritic_rate": 0.0,
+ "non_latin_fraction": 1.0
+ }
+ },
+ {
+ "label": "all_scripts/ethiopic",
+ "state": "ሰላም እንዴት ነህ ዛሬ",
+ "state_text": "ሰላም እንዴት ነህ ዛሬ",
+ "detect_script": "ethiopic",
+ "script_profile": {
+ "ethiopic": 1.0
+ },
+ "latin_profile": {
+ "language": null,
+ "english_hits": 0,
+ "diacritic_rate": 0.0,
+ "looks_non_english": false
+ },
+ "analyse": {
+ "script": "ethiopic",
+ "script_profile": {
+ "ethiopic": 1.0
+ },
+ "language": null,
+ "is_english": false,
+ "language_undecided": true,
+ "diacritic_rate": 0.0,
+ "non_latin_fraction": 1.0
+ }
+ },
+ {
+ "label": "all_scripts/khmer",
+ "state": "សួស្តី តើអ្នកសុខសប្បាយទេ",
+ "state_text": "សួស្តី តើអ្នកសុខសប្បាយទេ",
+ "detect_script": "khmer",
+ "script_profile": {
+ "khmer": 1.0
+ },
+ "latin_profile": {
+ "language": null,
+ "english_hits": 0,
+ "diacritic_rate": 0.0,
+ "looks_non_english": false
+ },
+ "analyse": {
+ "script": "khmer",
+ "script_profile": {
+ "khmer": 1.0
+ },
+ "language": null,
+ "is_english": false,
+ "language_undecided": true,
+ "diacritic_rate": 0.0,
+ "non_latin_fraction": 1.0
+ }
+ },
+ {
+ "label": "all_scripts/hangul",
+ "state": "안녕하세요 오늘 기분이 어때요",
+ "state_text": "안녕하세요 오늘 기분이 어때요",
+ "detect_script": "hangul",
+ "script_profile": {
+ "hangul": 1.0
+ },
+ "latin_profile": {
+ "language": null,
+ "english_hits": 0,
+ "diacritic_rate": 0.0,
+ "looks_non_english": false
+ },
+ "analyse": {
+ "script": "hangul",
+ "script_profile": {
+ "hangul": 1.0
+ },
+ "language": null,
+ "is_english": false,
+ "language_undecided": true,
+ "diacritic_rate": 0.0,
+ "non_latin_fraction": 1.0
+ }
+ },
+ {
+ "label": "all_scripts/kana",
+ "state": "こんにちは、今日は元気ですか",
+ "state_text": "こんにちは、今日は元気ですか",
+ "detect_script": "kana",
+ "script_profile": {
+ "kana": 0.6923076923076923,
+ "han": 0.3076923076923077
+ },
+ "latin_profile": {
+ "language": null,
+ "english_hits": 0,
+ "diacritic_rate": 0.0,
+ "looks_non_english": false
+ },
+ "analyse": {
+ "script": "kana",
+ "script_profile": {
+ "kana": 0.6923076923076923,
+ "han": 0.3076923076923077
+ },
+ "language": null,
+ "is_english": false,
+ "language_undecided": true,
+ "diacritic_rate": 0.0,
+ "non_latin_fraction": 1.0
+ }
+ },
+ {
+ "label": "all_scripts/han",
+ "state": "你好,你今天过得怎么样",
+ "state_text": "你好,你今天过得怎么样",
+ "detect_script": "han",
+ "script_profile": {
+ "han": 1.0
+ },
+ "latin_profile": {
+ "language": null,
+ "english_hits": 0,
+ "diacritic_rate": 0.0,
+ "looks_non_english": false
+ },
+ "analyse": {
+ "script": "han",
+ "script_profile": {
+ "han": 1.0
+ },
+ "language": null,
+ "is_english": false,
+ "language_undecided": true,
+ "diacritic_rate": 0.0,
+ "non_latin_fraction": 1.0
+ }
+ },
+ {
+ "label": "all_scripts/latin",
+ "state": "Hello, how are you feeling today?",
+ "state_text": "Hello, how are you feeling today?",
+ "detect_script": "latin",
+ "script_profile": {
+ "latin": 1.0
+ },
+ "latin_profile": {
+ "language": "en",
+ "english_hits": 2,
+ "diacritic_rate": 0.0,
+ "looks_non_english": false
+ },
+ "analyse": {
+ "script": "latin",
+ "script_profile": {
+ "latin": 1.0
+ },
+ "language": "en",
+ "is_english": true,
+ "language_undecided": false,
+ "diacritic_rate": 0.0,
+ "non_latin_fraction": 0.0
+ }
+ },
+ {
+ "label": "tie/greek_then_cyrillic_1v1",
+ "state": "ΑА",
+ "state_text": "ΑА",
+ "detect_script": "greek",
+ "script_profile": {
+ "greek": 0.5,
+ "cyrillic": 0.5
+ },
+ "latin_profile": {
+ "language": null,
+ "english_hits": 0,
+ "diacritic_rate": 0.0,
+ "looks_non_english": false
+ },
+ "analyse": {
+ "script": "greek",
+ "script_profile": {
+ "greek": 0.5,
+ "cyrillic": 0.5
+ },
+ "language": null,
+ "is_english": false,
+ "language_undecided": true,
+ "diacritic_rate": 0.0,
+ "non_latin_fraction": 1.0
+ }
+ },
+ {
+ "label": "tie/cyrillic_then_greek_1v1",
+ "state": "АΑ",
+ "state_text": "АΑ",
+ "detect_script": "cyrillic",
+ "script_profile": {
+ "cyrillic": 0.5,
+ "greek": 0.5
+ },
+ "latin_profile": {
+ "language": null,
+ "english_hits": 0,
+ "diacritic_rate": 0.0,
+ "looks_non_english": false
+ },
+ "analyse": {
+ "script": "cyrillic",
+ "script_profile": {
+ "cyrillic": 0.5,
+ "greek": 0.5
+ },
+ "language": null,
+ "is_english": false,
+ "language_undecided": true,
+ "diacritic_rate": 0.0,
+ "non_latin_fraction": 1.0
+ }
+ },
+ {
+ "label": "tie/han_then_latin_1v1",
+ "state": "中 a",
+ "state_text": "中 a",
+ "detect_script": "han",
+ "script_profile": {
+ "latin": 0.5,
+ "han": 0.5
+ },
+ "latin_profile": {
+ "language": null,
+ "english_hits": 0,
+ "diacritic_rate": 0.0,
+ "looks_non_english": false
+ },
+ "analyse": {
+ "script": "han",
+ "script_profile": {
+ "latin": 0.5,
+ "han": 0.5
+ },
+ "language": null,
+ "is_english": false,
+ "language_undecided": true,
+ "diacritic_rate": 0.0,
+ "non_latin_fraction": 0.5
+ }
+ },
+ {
+ "label": "tie/latin_then_han_1v1",
+ "state": "a 中",
+ "state_text": "a 中",
+ "detect_script": "han",
+ "script_profile": {
+ "latin": 0.5,
+ "han": 0.5
+ },
+ "latin_profile": {
+ "language": null,
+ "english_hits": 0,
+ "diacritic_rate": 0.0,
+ "looks_non_english": false
+ },
+ "analyse": {
+ "script": "han",
+ "script_profile": {
+ "latin": 0.5,
+ "han": 0.5
+ },
+ "language": null,
+ "is_english": false,
+ "language_undecided": true,
+ "diacritic_rate": 0.0,
+ "non_latin_fraction": 0.5
+ }
+ },
+ {
+ "label": "tie/three_way_greek_cyrillic_armenian",
+ "state": "ΑАԱ",
+ "state_text": "ΑАԱ",
+ "detect_script": "greek",
+ "script_profile": {
+ "greek": 0.3333333333333333,
+ "cyrillic": 0.3333333333333333,
+ "armenian": 0.3333333333333333
+ },
+ "latin_profile": {
+ "language": null,
+ "english_hits": 0,
+ "diacritic_rate": 0.0,
+ "looks_non_english": false
+ },
+ "analyse": {
+ "script": "greek",
+ "script_profile": {
+ "greek": 0.3333333333333333,
+ "cyrillic": 0.3333333333333333,
+ "armenian": 0.3333333333333333
+ },
+ "language": null,
+ "is_english": false,
+ "language_undecided": true,
+ "diacritic_rate": 0.0,
+ "non_latin_fraction": 1.0
+ }
+ },
+ {
+ "label": "mixed/latin_han_arabic",
+ "state": "invoice 中文 فاتورة refund",
+ "state_text": "invoice 中文 فاتورة refund",
+ "detect_script": "latin",
+ "script_profile": {
+ "latin": 0.6190476190476191,
+ "han": 0.09523809523809523,
+ "arabic": 0.2857142857142857
+ },
+ "latin_profile": {
+ "language": null,
+ "english_hits": 0,
+ "diacritic_rate": 0.0,
+ "looks_non_english": false
+ },
+ "analyse": {
+ "script": "latin",
+ "script_profile": {
+ "latin": 0.6190476190476191,
+ "han": 0.09523809523809523,
+ "arabic": 0.2857142857142857
+ },
+ "language": null,
+ "is_english": true,
+ "language_undecided": true,
+ "diacritic_rate": 0.0,
+ "non_latin_fraction": 0.381
+ }
+ },
+ {
+ "label": "mixed/devanagari_latin_majority_devanagari",
+ "state": "ग्राहक से दो बार शुल्क लिया गया refund",
+ "state_text": "ग्राहक से दो बार शुल्क लिया गया refund",
+ "detect_script": "devanagari",
+ "script_profile": {
+ "latin": 0.2857142857142857,
+ "devanagari": 0.7142857142857143
+ },
+ "latin_profile": {
+ "language": null,
+ "english_hits": 0,
+ "diacritic_rate": 0.0,
+ "looks_non_english": false
+ },
+ "analyse": {
+ "script": "devanagari",
+ "script_profile": {
+ "latin": 0.2857142857142857,
+ "devanagari": 0.7142857142857143
+ },
+ "language": null,
+ "is_english": false,
+ "language_undecided": true,
+ "diacritic_rate": 0.0,
+ "non_latin_fraction": 0.7143
+ }
+ },
+ {
+ "label": "mixed/latin_majority_devanagari_minor",
+ "state": "Please refund the customer today ग्राहक",
+ "state_text": "Please refund the customer today ग्राहक",
+ "detect_script": "latin",
+ "script_profile": {
+ "latin": 0.875,
+ "devanagari": 0.125
+ },
+ "latin_profile": {
+ "language": "en",
+ "english_hits": 2,
+ "diacritic_rate": 0.0,
+ "looks_non_english": false
+ },
+ "analyse": {
+ "script": "latin",
+ "script_profile": {
+ "latin": 0.875,
+ "devanagari": 0.125
+ },
+ "language": "en",
+ "is_english": true,
+ "language_undecided": false,
+ "diacritic_rate": 0.0,
+ "non_latin_fraction": 0.125
+ }
+ },
+ {
+ "label": "latin_tie/fr_vs_de",
+ "state": "être cette nicht sich",
+ "state_text": "être cette nicht sich",
+ "detect_script": "latin",
+ "script_profile": {
+ "latin": 1.0
+ },
+ "latin_profile": {
+ "language": "fr",
+ "english_hits": 0,
+ "diacritic_rate": 0.047619047619047616,
+ "looks_non_english": true
+ },
+ "analyse": {
+ "script": "latin",
+ "script_profile": {
+ "latin": 1.0
+ },
+ "language": "fr",
+ "is_english": false,
+ "language_undecided": false,
+ "diacritic_rate": 0.0476,
+ "non_latin_fraction": 0.0
+ }
+ },
+ {
+ "label": "latin_tie/de_vs_es",
+ "state": "nicht sich el los",
+ "state_text": "nicht sich el los",
+ "detect_script": "latin",
+ "script_profile": {
+ "latin": 1.0
+ },
+ "latin_profile": {
+ "language": "de",
+ "english_hits": 0,
+ "diacritic_rate": 0.0,
+ "looks_non_english": false
+ },
+ "analyse": {
+ "script": "latin",
+ "script_profile": {
+ "latin": 1.0
+ },
+ "language": "de",
+ "is_english": false,
+ "language_undecided": false,
+ "diacritic_rate": 0.0,
+ "non_latin_fraction": 0.0
+ }
+ },
+ {
+ "label": "latin_tie/es_vs_pt",
+ "state": "está una dos tres o não está",
+ "state_text": "está una dos tres o não está",
+ "detect_script": "latin",
+ "script_profile": {
+ "latin": 1.0
+ },
+ "latin_profile": {
+ "language": "es",
+ "english_hits": 0,
+ "diacritic_rate": 0.10714285714285714,
+ "looks_non_english": true
+ },
+ "analyse": {
+ "script": "latin",
+ "script_profile": {
+ "latin": 1.0
+ },
+ "language": "es",
+ "is_english": false,
+ "language_undecided": false,
+ "diacritic_rate": 0.1071,
+ "non_latin_fraction": 0.0
+ }
+ },
+ {
+ "label": "latin_tie/pt_vs_it",
+ "state": "não está o della sono",
+ "state_text": "não está o della sono",
+ "detect_script": "latin",
+ "script_profile": {
+ "latin": 1.0
+ },
+ "latin_profile": {
+ "language": "pt",
+ "english_hits": 0,
+ "diacritic_rate": 0.09523809523809523,
+ "looks_non_english": true
+ },
+ "analyse": {
+ "script": "latin",
+ "script_profile": {
+ "latin": 1.0
+ },
+ "language": "pt",
+ "is_english": false,
+ "language_undecided": false,
+ "diacritic_rate": 0.0952,
+ "non_latin_fraction": 0.0
+ }
+ },
+ {
+ "label": "astral/emoji_only",
+ "state": "😀🚀❤️",
+ "state_text": "😀🚀❤️",
+ "detect_script": "unknown",
+ "script_profile": {},
+ "latin_profile": {
+ "language": null,
+ "english_hits": 0,
+ "diacritic_rate": 0.0,
+ "looks_non_english": false
+ },
+ "analyse": {
+ "script": "unknown",
+ "script_profile": {},
+ "language": null,
+ "is_english": true,
+ "language_undecided": true,
+ "diacritic_rate": 0.0,
+ "non_latin_fraction": 0.0
+ }
+ },
+ {
+ "label": "astral/emoji_with_latin",
+ "state": "refund please 😀 today",
+ "state_text": "refund please 😀 today",
+ "detect_script": "latin",
+ "script_profile": {
+ "latin": 1.0
+ },
+ "latin_profile": {
+ "language": null,
+ "english_hits": 0,
+ "diacritic_rate": 0.0,
+ "looks_non_english": false
+ },
+ "analyse": {
+ "script": "latin",
+ "script_profile": {
+ "latin": 1.0
+ },
+ "language": null,
+ "is_english": true,
+ "language_undecided": true,
+ "diacritic_rate": 0.0,
+ "non_latin_fraction": 0.0
+ }
+ },
+ {
+ "label": "astral/deseret_letter_alone",
+ "state": "𐐀𐐁𐐂",
+ "state_text": "𐐀𐐁𐐂",
+ "detect_script": "unknown",
+ "script_profile": {},
+ "latin_profile": {
+ "language": null,
+ "english_hits": 0,
+ "diacritic_rate": 0.0,
+ "looks_non_english": false
+ },
+ "analyse": {
+ "script": "unknown",
+ "script_profile": {},
+ "language": null,
+ "is_english": true,
+ "language_undecided": true,
+ "diacritic_rate": 0.0,
+ "non_latin_fraction": 0.0
+ }
+ },
+ {
+ "label": "astral/deseret_mixed_with_latin",
+ "state": "refund 𐐀 today",
+ "state_text": "refund 𐐀 today",
+ "detect_script": "latin",
+ "script_profile": {
+ "latin": 1.0
+ },
+ "latin_profile": {
+ "language": null,
+ "english_hits": 0,
+ "diacritic_rate": 0.0,
+ "looks_non_english": false
+ },
+ "analyse": {
+ "script": "latin",
+ "script_profile": {
+ "latin": 1.0
+ },
+ "language": null,
+ "is_english": true,
+ "language_undecided": true,
+ "diacritic_rate": 0.0,
+ "non_latin_fraction": 0.0
+ }
+ },
+ {
+ "label": "astral/mathematical_bold_letters",
+ "state": "𝐀𝐁𝐂",
+ "state_text": "𝐀𝐁𝐂",
+ "detect_script": "unknown",
+ "script_profile": {},
+ "latin_profile": {
+ "language": null,
+ "english_hits": 0,
+ "diacritic_rate": 0.0,
+ "looks_non_english": false
+ },
+ "analyse": {
+ "script": "unknown",
+ "script_profile": {},
+ "language": null,
+ "is_english": true,
+ "language_undecided": true,
+ "diacritic_rate": 0.0,
+ "non_latin_fraction": 0.0
+ }
+ },
+ {
+ "label": "combining/devanagari_with_virama",
+ "state": "ग्राहक",
+ "state_text": "ग्राहक",
+ "detect_script": "devanagari",
+ "script_profile": {
+ "devanagari": 1.0
+ },
+ "latin_profile": {
+ "language": null,
+ "english_hits": 0,
+ "diacritic_rate": 0.0,
+ "looks_non_english": false
+ },
+ "analyse": {
+ "script": "devanagari",
+ "script_profile": {
+ "devanagari": 1.0
+ },
+ "language": null,
+ "is_english": false,
+ "language_undecided": true,
+ "diacritic_rate": 0.0,
+ "non_latin_fraction": 1.0
+ }
+ },
+ {
+ "label": "combining/latin_with_combining_acute",
+ "state": "café résumé",
+ "state_text": "café résumé",
+ "detect_script": "latin",
+ "script_profile": {
+ "latin": 1.0
+ },
+ "latin_profile": {
+ "language": null,
+ "english_hits": 0,
+ "diacritic_rate": 0.0,
+ "looks_non_english": false
+ },
+ "analyse": {
+ "script": "latin",
+ "script_profile": {
+ "latin": 1.0
+ },
+ "language": null,
+ "is_english": true,
+ "language_undecided": true,
+ "diacritic_rate": 0.0,
+ "non_latin_fraction": 0.0
+ }
+ },
+ {
+ "label": "turkish_i/capital_i_with_dot",
+ "state": "İstanbul",
+ "state_text": "İstanbul",
+ "detect_script": "latin",
+ "script_profile": {
+ "latin": 1.0
+ },
+ "latin_profile": {
+ "language": null,
+ "english_hits": 0,
+ "diacritic_rate": 0.0,
+ "looks_non_english": false
+ },
+ "analyse": {
+ "script": "latin",
+ "script_profile": {
+ "latin": 1.0
+ },
+ "language": null,
+ "is_english": true,
+ "language_undecided": true,
+ "diacritic_rate": 0.0,
+ "non_latin_fraction": 0.0
+ }
+ },
+ {
+ "label": "turkish_i/mixed_case",
+ "state": "İYİ GÜNLER İstanbul'da",
+ "state_text": "İYİ GÜNLER İstanbul'da",
+ "detect_script": "latin",
+ "script_profile": {
+ "latin": 1.0
+ },
+ "latin_profile": {
+ "language": null,
+ "english_hits": 0,
+ "diacritic_rate": 0.04,
+ "looks_non_english": true
+ },
+ "analyse": {
+ "script": "latin",
+ "script_profile": {
+ "latin": 1.0
+ },
+ "language": null,
+ "is_english": false,
+ "language_undecided": true,
+ "diacritic_rate": 0.04,
+ "non_latin_fraction": 0.0
+ }
+ },
+ {
+ "label": "turkish_i/lowercase_dotless",
+ "state": "ıstanbul",
+ "state_text": "ıstanbul",
+ "detect_script": "latin",
+ "script_profile": {
+ "latin": 1.0
+ },
+ "latin_profile": {
+ "language": null,
+ "english_hits": 0,
+ "diacritic_rate": 0.125,
+ "looks_non_english": true
+ },
+ "analyse": {
+ "script": "latin",
+ "script_profile": {
+ "latin": 1.0
+ },
+ "language": null,
+ "is_english": false,
+ "language_undecided": true,
+ "diacritic_rate": 0.125,
+ "non_latin_fraction": 0.0
+ }
+ },
+ {
+ "label": "numerics/ascii_digits_only",
+ "state": "12345 67890",
+ "state_text": "12345 67890",
+ "detect_script": "unknown",
+ "script_profile": {},
+ "latin_profile": {
+ "language": null,
+ "english_hits": 0,
+ "diacritic_rate": 0.0,
+ "looks_non_english": false
+ },
+ "analyse": {
+ "script": "unknown",
+ "script_profile": {},
+ "language": null,
+ "is_english": true,
+ "language_undecided": true,
+ "diacritic_rate": 0.0,
+ "non_latin_fraction": 0.0
+ }
+ },
+ {
+ "label": "numerics/roman_numeral_nl",
+ "state": "chapter Ⅷ begins",
+ "state_text": "chapter Ⅷ begins",
+ "detect_script": "latin",
+ "script_profile": {
+ "latin": 1.0
+ },
+ "latin_profile": {
+ "language": null,
+ "english_hits": 0,
+ "diacritic_rate": 0.0,
+ "looks_non_english": false
+ },
+ "analyse": {
+ "script": "latin",
+ "script_profile": {
+ "latin": 1.0
+ },
+ "language": null,
+ "is_english": true,
+ "language_undecided": true,
+ "diacritic_rate": 0.0,
+ "non_latin_fraction": 0.0
+ }
+ },
+ {
+ "label": "numerics/fullwidth_digits",
+ "state": "123 refund",
+ "state_text": "123 refund",
+ "detect_script": "latin",
+ "script_profile": {
+ "latin": 1.0
+ },
+ "latin_profile": {
+ "language": null,
+ "english_hits": 0,
+ "diacritic_rate": 0.0,
+ "looks_non_english": false
+ },
+ "analyse": {
+ "script": "latin",
+ "script_profile": {
+ "latin": 1.0
+ },
+ "language": null,
+ "is_english": true,
+ "language_undecided": true,
+ "diacritic_rate": 0.0,
+ "non_latin_fraction": 0.0
+ }
+ },
+ {
+ "label": "numerics/mixed_digits_and_words",
+ "state": "invoice 4411 was paid twice on 2024-01-05",
+ "state_text": "invoice 4411 was paid twice on 2024-01-05",
+ "detect_script": "latin",
+ "script_profile": {
+ "latin": 1.0
+ },
+ "latin_profile": {
+ "language": "en",
+ "english_hits": 2,
+ "diacritic_rate": 0.0,
+ "looks_non_english": false
+ },
+ "analyse": {
+ "script": "latin",
+ "script_profile": {
+ "latin": 1.0
+ },
+ "language": "en",
+ "is_english": true,
+ "language_undecided": false,
+ "diacritic_rate": 0.0,
+ "non_latin_fraction": 0.0
+ }
+ },
+ {
+ "label": "nesting/depth_6_kept",
+ "state": {
+ "a": {
+ "b": {
+ "c": {
+ "d": {
+ "e": {
+ "f": "visible at depth 6"
+ }
+ }
+ }
+ }
+ }
+ },
+ "state_text": "visible at depth 6",
+ "detect_script": "latin",
+ "script_profile": {
+ "latin": 1.0
+ },
+ "latin_profile": {
+ "language": null,
+ "english_hits": 0,
+ "diacritic_rate": 0.0,
+ "looks_non_english": false
+ },
+ "analyse": {
+ "script": "latin",
+ "script_profile": {
+ "latin": 1.0
+ },
+ "language": null,
+ "is_english": true,
+ "language_undecided": true,
+ "diacritic_rate": 0.0,
+ "non_latin_fraction": 0.0
+ }
+ },
+ {
+ "label": "nesting/depth_7_dropped",
+ "state": {
+ "a": {
+ "b": {
+ "c": {
+ "d": {
+ "e": {
+ "f": {
+ "g": "invisible past depth 6"
+ }
+ }
+ }
+ }
+ }
+ }
+ },
+ "state_text": "",
+ "detect_script": "unknown",
+ "script_profile": {},
+ "latin_profile": {
+ "language": null,
+ "english_hits": 0,
+ "diacritic_rate": 0.0,
+ "looks_non_english": false
+ },
+ "analyse": {
+ "script": "unknown",
+ "script_profile": {},
+ "language": null,
+ "is_english": true,
+ "language_undecided": true,
+ "diacritic_rate": 0.0,
+ "non_latin_fraction": 0.0
+ }
+ },
+ {
+ "label": "nesting/list_depth_7_dropped",
+ "state": [
+ [
+ [
+ [
+ [
+ [
+ [
+ "invisible list depth 7"
+ ]
+ ]
+ ]
+ ]
+ ]
+ ]
+ ],
+ "state_text": "",
+ "detect_script": "unknown",
+ "script_profile": {},
+ "latin_profile": {
+ "language": null,
+ "english_hits": 0,
+ "diacritic_rate": 0.0,
+ "looks_non_english": false
+ },
+ "analyse": {
+ "script": "unknown",
+ "script_profile": {},
+ "language": null,
+ "is_english": true,
+ "language_undecided": true,
+ "diacritic_rate": 0.0,
+ "non_latin_fraction": 0.0
+ }
+ },
+ {
+ "label": "nesting/mixed_depth_boundary",
+ "state": {
+ "a": {
+ "b": {
+ "c": {
+ "d": {
+ "e": [
+ "kept depth 6",
+ {
+ "f": "dropped depth 7"
+ }
+ ]
+ }
+ }
+ }
+ }
+ },
+ "state_text": "kept depth 6",
+ "detect_script": "latin",
+ "script_profile": {
+ "latin": 1.0
+ },
+ "latin_profile": {
+ "language": null,
+ "english_hits": 0,
+ "diacritic_rate": 0.0,
+ "looks_non_english": false
+ },
+ "analyse": {
+ "script": "latin",
+ "script_profile": {
+ "latin": 1.0
+ },
+ "language": null,
+ "is_english": true,
+ "language_undecided": true,
+ "diacritic_rate": 0.0,
+ "non_latin_fraction": 0.0
+ }
+ },
+ {
+ "label": "long/ascii_over_4000",
+ "state": "refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. ",
+ "state_text": "refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund please. refund ple",
+ "detect_script": "latin",
+ "script_profile": {
+ "latin": 1.0
+ },
+ "latin_profile": {
+ "language": "en",
+ "english_hits": 266,
+ "diacritic_rate": 0.0,
+ "looks_non_english": false
+ },
+ "analyse": {
+ "script": "latin",
+ "script_profile": {
+ "latin": 1.0
+ },
+ "language": "en",
+ "is_english": true,
+ "language_undecided": false,
+ "diacritic_rate": 0.0,
+ "non_latin_fraction": 0.0
+ }
+ },
+ {
+ "label": "long/multibyte_over_4000",
+ "state": "退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 ",
+ "state_text": "退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 退款请求 ",
+ "detect_script": "han",
+ "script_profile": {
+ "han": 1.0
+ },
+ "latin_profile": {
+ "language": null,
+ "english_hits": 0,
+ "diacritic_rate": 0.0,
+ "looks_non_english": false
+ },
+ "analyse": {
+ "script": "han",
+ "script_profile": {
+ "han": 1.0
+ },
+ "language": null,
+ "is_english": false,
+ "language_undecided": true,
+ "diacritic_rate": 0.0,
+ "non_latin_fraction": 1.0
+ }
+ },
+ {
+ "label": "long/mixed_script_over_4000_boundary_near_switch",
+ "state": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa客户被重复扣款要求退款客户被重复扣款要求退款客户被重复扣款要求退款客户被重复扣款要求退款客户被重复扣款要求退款",
+ "state_text": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa客户被重复扣款要求退",
+ "detect_script": "latin",
+ "script_profile": {
+ "latin": 0.9975,
+ "han": 0.0025
+ },
+ "latin_profile": {
+ "language": null,
+ "english_hits": 0,
+ "diacritic_rate": 0.0,
+ "looks_non_english": false
+ },
+ "analyse": {
+ "script": "latin",
+ "script_profile": {
+ "latin": 0.9975,
+ "han": 0.0025
+ },
+ "language": null,
+ "is_english": true,
+ "language_undecided": true,
+ "diacritic_rate": 0.0,
+ "non_latin_fraction": 0.0025
+ }
+ },
+ {
+ "label": "long/english_over_1000",
+ "state": "Please refund the duplicate charge on invoice 4411 today. Please refund the duplicate charge on invoice 4411 today. Please refund the duplicate charge on invoice 4411 today. Please refund the duplicate charge on invoice 4411 today. Please refund the duplicate charge on invoice 4411 today. Please refund the duplicate charge on invoice 4411 today. Please refund the duplicate charge on invoice 4411 today. Please refund the duplicate charge on invoice 4411 today. Please refund the duplicate charge on invoice 4411 today. Please refund the duplicate charge on invoice 4411 today. Please refund the duplicate charge on invoice 4411 today. Please refund the duplicate charge on invoice 4411 today. Please refund the duplicate charge on invoice 4411 today. Please refund the duplicate charge on invoice 4411 today. Please refund the duplicate charge on invoice 4411 today. Please refund the duplicate charge on invoice 4411 today. Please refund the duplicate charge on invoice 4411 today. Please refund the duplicate charge on invoice 4411 today. Please refund the duplicate charge on invoice 4411 today. Please refund the duplicate charge on invoice 4411 today. ",
+ "state_text": "Please refund the duplicate charge on invoice 4411 today. Please refund the duplicate charge on invoice 4411 today. Please refund the duplicate charge on invoice 4411 today. Please refund the duplicate charge on invoice 4411 today. Please refund the duplicate charge on invoice 4411 today. Please refund the duplicate charge on invoice 4411 today. Please refund the duplicate charge on invoice 4411 today. Please refund the duplicate charge on invoice 4411 today. Please refund the duplicate charge on invoice 4411 today. Please refund the duplicate charge on invoice 4411 today. Please refund the duplicate charge on invoice 4411 today. Please refund the duplicate charge on invoice 4411 today. Please refund the duplicate charge on invoice 4411 today. Please refund the duplicate charge on invoice 4411 today. Please refund the duplicate charge on invoice 4411 today. Please refund the duplicate charge on invoice 4411 today. Please refund the duplicate charge on invoice 4411 today. Please refund the duplicate charge on invoice 4411 today. Please refund the duplicate charge on invoice 4411 today. Please refund the duplicate charge on invoice 4411 today. ",
+ "detect_script": "latin",
+ "script_profile": {
+ "latin": 1.0
+ },
+ "latin_profile": {
+ "language": "en",
+ "english_hits": 60,
+ "diacritic_rate": 0.0,
+ "looks_non_english": false
+ },
+ "analyse": {
+ "script": "latin",
+ "script_profile": {
+ "latin": 1.0
+ },
+ "language": "en",
+ "is_english": true,
+ "language_undecided": false,
+ "diacritic_rate": 0.0,
+ "non_latin_fraction": 0.0
+ }
+ }
+]
diff --git a/laya-rust/tests/golden/routing/meta.json b/laya-rust/tests/golden/routing/meta.json
new file mode 100644
index 00000000..5cf9a90b
--- /dev/null
+++ b/laya-rust/tests/golden/routing/meta.json
@@ -0,0 +1,17 @@
+{
+ "generator": "tools/dump_routing_golden.py",
+ "laya_version": "0.3.6",
+ "default_shortlist_k": 20,
+ "notes": {
+ "route_probe": "RouteDecision is serialized as {model, reason, detection, workflow}; 'repo' is intentionally omitted (not part of the ported RouteDecision).",
+ "shortlist_probe": "Non-finite floats (NaN, +Inf, -Inf) inside a case's 'vectors' matrix are encoded as the JSON strings \"nan\", \"inf\", \"-inf\" because JSON has no native literal for them; a case with vectors=null is a k>=n passthrough case where embed_fn must never be called."
+ },
+ "counts": {
+ "lang_probe": 134,
+ "route_probe": 72,
+ "email_probe_clean": 47,
+ "email_probe_state": 8,
+ "shortlist_probe": 22,
+ "hashing_embedder_probe": 22
+ }
+}
diff --git a/laya-rust/tests/golden/routing/route_probe.json b/laya-rust/tests/golden/routing/route_probe.json
new file mode 100644
index 00000000..6d284165
--- /dev/null
+++ b/laya-rust/tests/golden/routing/route_probe.json
@@ -0,0 +1,1947 @@
+[
+ {
+ "label": "english_text",
+ "router_kwargs": {},
+ "route_kwargs": {
+ "state": {
+ "body": "I was charged twice, please refund."
+ },
+ "questions": {
+ "dept": {
+ "type": "choice",
+ "instructions": "Which team?",
+ "criteria": {
+ "billing": null,
+ "tech": null
+ }
+ }
+ }
+ },
+ "decision": {
+ "model": "english",
+ "reason": "English Latin text",
+ "detection": {
+ "script": "latin",
+ "script_profile": {
+ "latin": 1.0
+ },
+ "language": "en",
+ "is_english": true,
+ "language_undecided": false,
+ "diacritic_rate": 0.0,
+ "non_latin_fraction": 0.0
+ },
+ "workflow": null
+ }
+ },
+ {
+ "label": "armenian_text",
+ "router_kwargs": {},
+ "route_kwargs": {
+ "state": {
+ "body": "Հայերեն"
+ },
+ "questions": {
+ "dept": {
+ "type": "choice",
+ "instructions": "Which team?",
+ "criteria": {
+ "billing": null,
+ "tech": null
+ }
+ }
+ }
+ },
+ "decision": {
+ "model": "multilingual",
+ "reason": "non-Latin script (armenian, 100% of letters); the English checkpoint cannot read it",
+ "detection": {
+ "script": "armenian",
+ "script_profile": {
+ "armenian": 1.0
+ },
+ "language": null,
+ "is_english": false,
+ "language_undecided": true,
+ "diacritic_rate": 0.0,
+ "non_latin_fraction": 1.0
+ },
+ "workflow": null
+ }
+ },
+ {
+ "label": "armenian_explicit_override",
+ "router_kwargs": {},
+ "route_kwargs": {
+ "state": {
+ "body": "Հայերեն"
+ },
+ "questions": {
+ "dept": {
+ "type": "choice",
+ "instructions": "Which team?",
+ "criteria": {
+ "billing": null,
+ "tech": null
+ }
+ }
+ },
+ "model": "english"
+ },
+ "decision": {
+ "model": "english",
+ "reason": "explicit model='english'",
+ "detection": null,
+ "workflow": null
+ }
+ },
+ {
+ "label": "hindi_text",
+ "router_kwargs": {},
+ "route_kwargs": {
+ "state": {
+ "body": "मुझसे दो बार शुल्क लिया गया"
+ },
+ "questions": {
+ "dept": {
+ "type": "choice",
+ "instructions": "Which team?",
+ "criteria": {
+ "billing": null,
+ "tech": null
+ }
+ }
+ }
+ },
+ "decision": {
+ "model": "multilingual",
+ "reason": "non-Latin script (devanagari, 100% of letters); the English checkpoint cannot read it",
+ "detection": {
+ "script": "devanagari",
+ "script_profile": {
+ "devanagari": 1.0
+ },
+ "language": null,
+ "is_english": false,
+ "language_undecided": true,
+ "diacritic_rate": 0.0,
+ "non_latin_fraction": 1.0
+ },
+ "workflow": null
+ }
+ },
+ {
+ "label": "japanese_text",
+ "router_kwargs": {},
+ "route_kwargs": {
+ "state": {
+ "body": "二重に請求されました"
+ },
+ "questions": {
+ "dept": {
+ "type": "choice",
+ "instructions": "Which team?",
+ "criteria": {
+ "billing": null,
+ "tech": null
+ }
+ }
+ }
+ },
+ "decision": {
+ "model": "multilingual",
+ "reason": "non-Latin script (kana, 100% of letters); the English checkpoint cannot read it",
+ "detection": {
+ "script": "kana",
+ "script_profile": {
+ "han": 0.4,
+ "kana": 0.6
+ },
+ "language": null,
+ "is_english": false,
+ "language_undecided": true,
+ "diacritic_rate": 0.0,
+ "non_latin_fraction": 1.0
+ },
+ "workflow": null
+ }
+ },
+ {
+ "label": "korean_text",
+ "router_kwargs": {},
+ "route_kwargs": {
+ "state": {
+ "body": "두 번 청구되었습니다"
+ },
+ "questions": {
+ "dept": {
+ "type": "choice",
+ "instructions": "Which team?",
+ "criteria": {
+ "billing": null,
+ "tech": null
+ }
+ }
+ }
+ },
+ "decision": {
+ "model": "multilingual",
+ "reason": "non-Latin script (hangul, 100% of letters); the English checkpoint cannot read it",
+ "detection": {
+ "script": "hangul",
+ "script_profile": {
+ "hangul": 1.0
+ },
+ "language": null,
+ "is_english": false,
+ "language_undecided": true,
+ "diacritic_rate": 0.0,
+ "non_latin_fraction": 1.0
+ },
+ "workflow": null
+ }
+ },
+ {
+ "label": "arabic_text",
+ "router_kwargs": {},
+ "route_kwargs": {
+ "state": {
+ "body": "تم خصم المبلغ مرتين"
+ },
+ "questions": {
+ "dept": {
+ "type": "choice",
+ "instructions": "Which team?",
+ "criteria": {
+ "billing": null,
+ "tech": null
+ }
+ }
+ }
+ },
+ "decision": {
+ "model": "multilingual",
+ "reason": "non-Latin script (arabic, 100% of letters); the English checkpoint cannot read it",
+ "detection": {
+ "script": "arabic",
+ "script_profile": {
+ "arabic": 1.0
+ },
+ "language": null,
+ "is_english": false,
+ "language_undecided": true,
+ "diacritic_rate": 0.0,
+ "non_latin_fraction": 1.0
+ },
+ "workflow": null
+ }
+ },
+ {
+ "label": "german_text",
+ "router_kwargs": {},
+ "route_kwargs": {
+ "state": {
+ "body": "Der Kunde wurde zweimal belastet und moechte eine Rueckerstattung fuer die Rechnung die nicht korrekt ist"
+ },
+ "questions": {
+ "dept": {
+ "type": "choice",
+ "instructions": "Which team?",
+ "criteria": {
+ "billing": null,
+ "tech": null
+ }
+ }
+ }
+ },
+ "decision": {
+ "model": "multilingual",
+ "reason": "Latin script but language looks like 'de', not English",
+ "detection": {
+ "script": "latin",
+ "script_profile": {
+ "latin": 1.0
+ },
+ "language": "de",
+ "is_english": false,
+ "language_undecided": false,
+ "diacritic_rate": 0.0,
+ "non_latin_fraction": 0.0
+ },
+ "workflow": null
+ }
+ },
+ {
+ "label": "explicit_model",
+ "router_kwargs": {},
+ "route_kwargs": {
+ "state": {
+ "body": "anything"
+ },
+ "questions": {
+ "dept": {
+ "type": "choice",
+ "instructions": "Which team?",
+ "criteria": {
+ "billing": null,
+ "tech": null
+ }
+ }
+ },
+ "model": "multilingual"
+ },
+ "decision": {
+ "model": "multilingual",
+ "reason": "explicit model='multilingual'",
+ "detection": null,
+ "workflow": null
+ }
+ },
+ {
+ "label": "explicit_model_overrides_script",
+ "router_kwargs": {},
+ "route_kwargs": {
+ "state": {
+ "body": "मुझसे दो बार"
+ },
+ "questions": {
+ "dept": {
+ "type": "choice",
+ "instructions": "Which team?",
+ "criteria": {
+ "billing": null,
+ "tech": null
+ }
+ }
+ },
+ "model": "english"
+ },
+ "decision": {
+ "model": "english",
+ "reason": "explicit model='english'",
+ "detection": null,
+ "workflow": null
+ }
+ },
+ {
+ "label": "explicit_task",
+ "router_kwargs": {},
+ "route_kwargs": {
+ "state": {
+ "body": "x"
+ },
+ "questions": {
+ "dept": {
+ "type": "choice",
+ "instructions": "Which team?",
+ "criteria": {
+ "billing": null,
+ "tech": null
+ }
+ }
+ },
+ "task": "typed_decisions"
+ },
+ "decision": {
+ "model": "typed-decisions",
+ "reason": "explicit task='typed_decisions'",
+ "detection": null,
+ "workflow": null
+ }
+ },
+ {
+ "label": "explicit_lang_en",
+ "router_kwargs": {},
+ "route_kwargs": {
+ "state": {
+ "body": "मुझसे दो बार"
+ },
+ "questions": {
+ "dept": {
+ "type": "choice",
+ "instructions": "Which team?",
+ "criteria": {
+ "billing": null,
+ "tech": null
+ }
+ }
+ },
+ "lang": "en"
+ },
+ "decision": {
+ "model": "english",
+ "reason": "explicit lang='en'",
+ "detection": null,
+ "workflow": null
+ }
+ },
+ {
+ "label": "explicit_lang_de",
+ "router_kwargs": {},
+ "route_kwargs": {
+ "state": {
+ "body": "hello there"
+ },
+ "questions": {
+ "dept": {
+ "type": "choice",
+ "instructions": "Which team?",
+ "criteria": {
+ "billing": null,
+ "tech": null
+ }
+ }
+ },
+ "lang": "de"
+ },
+ "decision": {
+ "model": "multilingual",
+ "reason": "explicit lang='de'",
+ "detection": null,
+ "workflow": null
+ }
+ },
+ {
+ "label": "explicit_lang_eng_alias",
+ "router_kwargs": {},
+ "route_kwargs": {
+ "state": {
+ "body": "hello"
+ },
+ "questions": {
+ "dept": {
+ "type": "choice",
+ "instructions": "Which team?",
+ "criteria": {
+ "billing": null,
+ "tech": null
+ }
+ }
+ },
+ "lang": "eng"
+ },
+ "decision": {
+ "model": "english",
+ "reason": "explicit lang='eng'",
+ "detection": null,
+ "workflow": null
+ }
+ },
+ {
+ "label": "explicit_lang_english_word",
+ "router_kwargs": {},
+ "route_kwargs": {
+ "state": {
+ "body": "hello"
+ },
+ "questions": {
+ "dept": {
+ "type": "choice",
+ "instructions": "Which team?",
+ "criteria": {
+ "billing": null,
+ "tech": null
+ }
+ }
+ },
+ "lang": "English"
+ },
+ "decision": {
+ "model": "english",
+ "reason": "explicit lang='English'",
+ "detection": null,
+ "workflow": null
+ }
+ },
+ {
+ "label": "explicit_lang_en_us",
+ "router_kwargs": {},
+ "route_kwargs": {
+ "state": {
+ "body": "hello"
+ },
+ "questions": {
+ "dept": {
+ "type": "choice",
+ "instructions": "Which team?",
+ "criteria": {
+ "billing": null,
+ "tech": null
+ }
+ }
+ },
+ "lang": "en-US"
+ },
+ "decision": {
+ "model": "english",
+ "reason": "explicit lang='en-US'",
+ "detection": null,
+ "workflow": null
+ }
+ },
+ {
+ "label": "explicit_lang_fr",
+ "router_kwargs": {},
+ "route_kwargs": {
+ "state": {
+ "body": "hello"
+ },
+ "questions": {
+ "dept": {
+ "type": "choice",
+ "instructions": "Which team?",
+ "criteria": {
+ "billing": null,
+ "tech": null
+ }
+ }
+ },
+ "lang": "fr"
+ },
+ "decision": {
+ "model": "multilingual",
+ "reason": "explicit lang='fr'",
+ "detection": null,
+ "workflow": null
+ }
+ },
+ {
+ "label": "td_workflow_auto_off",
+ "router_kwargs": {},
+ "route_kwargs": {
+ "state": {
+ "body": "I was charged twice"
+ },
+ "questions": {
+ "action": {
+ "type": "noul",
+ "instructions": "x"
+ },
+ "category": {
+ "type": "noul",
+ "instructions": "x"
+ },
+ "churn_risk": {
+ "type": "noul",
+ "instructions": "x"
+ },
+ "needs_human": {
+ "type": "noul",
+ "instructions": "x"
+ },
+ "urgency": {
+ "type": "noul",
+ "instructions": "x"
+ }
+ }
+ },
+ "decision": {
+ "model": "english",
+ "reason": "English Latin text",
+ "detection": {
+ "script": "latin",
+ "script_profile": {
+ "latin": 1.0
+ },
+ "language": "en",
+ "is_english": true,
+ "language_undecided": false,
+ "diacritic_rate": 0.0,
+ "non_latin_fraction": 0.0
+ },
+ "workflow": "customer_service"
+ }
+ },
+ {
+ "label": "empty_state",
+ "router_kwargs": {},
+ "route_kwargs": {
+ "state": {},
+ "questions": {
+ "dept": {
+ "type": "choice",
+ "instructions": "Which team?",
+ "criteria": {
+ "billing": null,
+ "tech": null
+ }
+ }
+ }
+ },
+ "decision": {
+ "model": "english",
+ "reason": "no letters detected in state; using default (english)",
+ "detection": {
+ "script": "unknown",
+ "script_profile": {},
+ "language": null,
+ "is_english": true,
+ "language_undecided": true,
+ "diacritic_rate": 0.0,
+ "non_latin_fraction": 0.0
+ },
+ "workflow": null
+ }
+ },
+ {
+ "label": "none_state",
+ "router_kwargs": {},
+ "route_kwargs": {
+ "state": null,
+ "questions": {
+ "dept": {
+ "type": "choice",
+ "instructions": "Which team?",
+ "criteria": {
+ "billing": null,
+ "tech": null
+ }
+ }
+ }
+ },
+ "decision": {
+ "model": "english",
+ "reason": "no letters detected in state; using default (english)",
+ "detection": {
+ "script": "unknown",
+ "script_profile": {},
+ "language": null,
+ "is_english": true,
+ "language_undecided": true,
+ "diacritic_rate": 0.0,
+ "non_latin_fraction": 0.0
+ },
+ "workflow": null
+ }
+ },
+ {
+ "label": "no_questions_none",
+ "router_kwargs": {},
+ "route_kwargs": {
+ "state": {
+ "body": "मुझसे दो बार"
+ },
+ "questions": null
+ },
+ "decision": {
+ "model": "multilingual",
+ "reason": "non-Latin script (devanagari, 100% of letters); the English checkpoint cannot read it",
+ "detection": {
+ "script": "devanagari",
+ "script_profile": {
+ "devanagari": 1.0
+ },
+ "language": null,
+ "is_english": false,
+ "language_undecided": true,
+ "diacritic_rate": 0.0,
+ "non_latin_fraction": 1.0
+ },
+ "workflow": null
+ }
+ },
+ {
+ "label": "td_agent_trace_observability_auto_on",
+ "router_kwargs": {
+ "auto_task_detection": true
+ },
+ "route_kwargs": {
+ "state": {
+ "body": "trace"
+ },
+ "questions": {
+ "action": {
+ "type": "noul",
+ "instructions": "x"
+ },
+ "needs_review": {
+ "type": "noul",
+ "instructions": "x"
+ },
+ "outcome": {
+ "type": "noul",
+ "instructions": "x"
+ },
+ "risk": {
+ "type": "noul",
+ "instructions": "x"
+ },
+ "urgency": {
+ "type": "noul",
+ "instructions": "x"
+ }
+ }
+ },
+ "decision": {
+ "model": "typed-decisions",
+ "reason": "question ids match the 'agent_trace_observability' typed-decisions workflow",
+ "detection": null,
+ "workflow": "agent_trace_observability"
+ }
+ },
+ {
+ "label": "td_customer_service_auto_on",
+ "router_kwargs": {
+ "auto_task_detection": true
+ },
+ "route_kwargs": {
+ "state": {
+ "body": "I was charged twice"
+ },
+ "questions": {
+ "action": {
+ "type": "noul",
+ "instructions": "x"
+ },
+ "category": {
+ "type": "noul",
+ "instructions": "x"
+ },
+ "churn_risk": {
+ "type": "noul",
+ "instructions": "x"
+ },
+ "needs_human": {
+ "type": "noul",
+ "instructions": "x"
+ },
+ "urgency": {
+ "type": "noul",
+ "instructions": "x"
+ }
+ }
+ },
+ "decision": {
+ "model": "typed-decisions",
+ "reason": "question ids match the 'customer_service' typed-decisions workflow",
+ "detection": null,
+ "workflow": "customer_service"
+ }
+ },
+ {
+ "label": "td_invoice_processing_auto_on",
+ "router_kwargs": {
+ "auto_task_detection": true
+ },
+ "route_kwargs": {
+ "state": {
+ "body": "invoice mismatch"
+ },
+ "questions": {
+ "discrepancy_severity": {
+ "type": "noul",
+ "instructions": "x"
+ },
+ "disposition": {
+ "type": "noul",
+ "instructions": "x"
+ },
+ "duplicate": {
+ "type": "noul",
+ "instructions": "x"
+ },
+ "matches_order": {
+ "type": "noul",
+ "instructions": "x"
+ },
+ "urgency": {
+ "type": "noul",
+ "instructions": "x"
+ }
+ }
+ },
+ "decision": {
+ "model": "typed-decisions",
+ "reason": "question ids match the 'invoice_processing' typed-decisions workflow",
+ "detection": null,
+ "workflow": "invoice_processing"
+ }
+ },
+ {
+ "label": "td_security_incidents_auto_on",
+ "router_kwargs": {
+ "auto_task_detection": true
+ },
+ "route_kwargs": {
+ "state": {
+ "body": "possible breach"
+ },
+ "questions": {
+ "credential_compromise": {
+ "type": "noul",
+ "instructions": "x"
+ },
+ "disposition": {
+ "type": "noul",
+ "instructions": "x"
+ },
+ "severity": {
+ "type": "noul",
+ "instructions": "x"
+ },
+ "true_positive": {
+ "type": "noul",
+ "instructions": "x"
+ },
+ "urgency": {
+ "type": "noul",
+ "instructions": "x"
+ }
+ }
+ },
+ "decision": {
+ "model": "typed-decisions",
+ "reason": "question ids match the 'security_incidents' typed-decisions workflow",
+ "detection": null,
+ "workflow": "security_incidents"
+ }
+ },
+ {
+ "label": "td_auto_on_but_generic_questions",
+ "router_kwargs": {
+ "auto_task_detection": true
+ },
+ "route_kwargs": {
+ "state": {
+ "body": "I was charged twice"
+ },
+ "questions": {
+ "dept": {
+ "type": "choice",
+ "instructions": "Which team?",
+ "criteria": {
+ "billing": null,
+ "tech": null
+ }
+ }
+ }
+ },
+ "decision": {
+ "model": "english",
+ "reason": "English Latin text",
+ "detection": {
+ "script": "latin",
+ "script_profile": {
+ "latin": 1.0
+ },
+ "language": "en",
+ "is_english": true,
+ "language_undecided": false,
+ "diacritic_rate": 0.0,
+ "non_latin_fraction": 0.0
+ },
+ "workflow": null
+ }
+ },
+ {
+ "label": "td_explicit_beats_workflow",
+ "router_kwargs": {
+ "auto_task_detection": true
+ },
+ "route_kwargs": {
+ "state": {
+ "body": "x"
+ },
+ "questions": {
+ "action": {
+ "type": "noul",
+ "instructions": "x"
+ },
+ "category": {
+ "type": "noul",
+ "instructions": "x"
+ },
+ "churn_risk": {
+ "type": "noul",
+ "instructions": "x"
+ },
+ "needs_human": {
+ "type": "noul",
+ "instructions": "x"
+ },
+ "urgency": {
+ "type": "noul",
+ "instructions": "x"
+ }
+ },
+ "model": "multilingual"
+ },
+ "decision": {
+ "model": "multilingual",
+ "reason": "explicit model='multilingual'",
+ "detection": null,
+ "workflow": null
+ }
+ },
+ {
+ "label": "td_near_miss_partial_overlap",
+ "router_kwargs": {
+ "auto_task_detection": true
+ },
+ "route_kwargs": {
+ "state": {
+ "body": "x"
+ },
+ "questions": {
+ "urgency": {
+ "type": "noul",
+ "instructions": "x"
+ },
+ "category": {
+ "type": "noul",
+ "instructions": "x"
+ }
+ }
+ },
+ "decision": {
+ "model": "english",
+ "reason": "English Latin text",
+ "detection": {
+ "script": "latin",
+ "script_profile": {
+ "latin": 1.0
+ },
+ "language": null,
+ "is_english": true,
+ "language_undecided": true,
+ "diacritic_rate": 0.0,
+ "non_latin_fraction": 0.0
+ },
+ "workflow": null
+ }
+ },
+ {
+ "label": "td_near_miss_superset",
+ "router_kwargs": {
+ "auto_task_detection": true
+ },
+ "route_kwargs": {
+ "state": {
+ "body": "x"
+ },
+ "questions": {
+ "action": {
+ "type": "noul",
+ "instructions": "x"
+ },
+ "category": {
+ "type": "noul",
+ "instructions": "x"
+ },
+ "churn_risk": {
+ "type": "noul",
+ "instructions": "x"
+ },
+ "needs_human": {
+ "type": "noul",
+ "instructions": "x"
+ },
+ "urgency": {
+ "type": "noul",
+ "instructions": "x"
+ },
+ "extra_field": {
+ "type": "noul",
+ "instructions": "x"
+ }
+ }
+ },
+ "decision": {
+ "model": "english",
+ "reason": "English Latin text",
+ "detection": {
+ "script": "latin",
+ "script_profile": {
+ "latin": 1.0
+ },
+ "language": null,
+ "is_english": true,
+ "language_undecided": true,
+ "diacritic_rate": 0.0,
+ "non_latin_fraction": 0.0
+ },
+ "workflow": null
+ }
+ },
+ {
+ "label": "td_near_miss_subset",
+ "router_kwargs": {
+ "auto_task_detection": true
+ },
+ "route_kwargs": {
+ "state": {
+ "body": "x"
+ },
+ "questions": {
+ "discrepancy_severity": {
+ "type": "noul",
+ "instructions": "x"
+ },
+ "disposition": {
+ "type": "noul",
+ "instructions": "x"
+ },
+ "duplicate": {
+ "type": "noul",
+ "instructions": "x"
+ },
+ "matches_order": {
+ "type": "noul",
+ "instructions": "x"
+ }
+ }
+ },
+ "decision": {
+ "model": "english",
+ "reason": "English Latin text",
+ "detection": {
+ "script": "latin",
+ "script_profile": {
+ "latin": 1.0
+ },
+ "language": null,
+ "is_english": true,
+ "language_undecided": true,
+ "diacritic_rate": 0.0,
+ "non_latin_fraction": 0.0
+ },
+ "workflow": null
+ }
+ },
+ {
+ "label": "td_near_miss_one_id_swapped",
+ "router_kwargs": {
+ "auto_task_detection": true
+ },
+ "route_kwargs": {
+ "state": {
+ "body": "x"
+ },
+ "questions": {
+ "credential_compromise": {
+ "type": "noul",
+ "instructions": "x"
+ },
+ "disposition": {
+ "type": "noul",
+ "instructions": "x"
+ },
+ "severity": {
+ "type": "noul",
+ "instructions": "x"
+ },
+ "true_positive": {
+ "type": "noul",
+ "instructions": "x"
+ },
+ "not_a_real_field": {
+ "type": "noul",
+ "instructions": "x"
+ }
+ }
+ },
+ "decision": {
+ "model": "english",
+ "reason": "English Latin text",
+ "detection": {
+ "script": "latin",
+ "script_profile": {
+ "latin": 1.0
+ },
+ "language": null,
+ "is_english": true,
+ "language_undecided": true,
+ "diacritic_rate": 0.0,
+ "non_latin_fraction": 0.0
+ },
+ "workflow": null
+ }
+ },
+ {
+ "label": "unknown_latin_romanian",
+ "router_kwargs": {},
+ "route_kwargs": {
+ "state": "Gătește-mi o rețetă de sarmale de post pentru mâine."
+ },
+ "decision": {
+ "model": "multilingual",
+ "reason": "Latin script but language looks like 'ro', not English",
+ "detection": {
+ "script": "latin",
+ "script_profile": {
+ "latin": 1.0
+ },
+ "language": "ro",
+ "is_english": false,
+ "language_undecided": false,
+ "diacritic_rate": 0.0962,
+ "non_latin_fraction": 0.0
+ },
+ "workflow": null
+ }
+ },
+ {
+ "label": "unknown_latin_romanian_agent_request",
+ "router_kwargs": {},
+ "route_kwargs": {
+ "state": "Exportă APK-ul pentru Android și pune-l pe Drive ca să-l instalez."
+ },
+ "decision": {
+ "model": "multilingual",
+ "reason": "Latin script but language looks like 'ro', not English",
+ "detection": {
+ "script": "latin",
+ "script_profile": {
+ "latin": 1.0
+ },
+ "language": "ro",
+ "is_english": false,
+ "language_undecided": false,
+ "diacritic_rate": 0.0455,
+ "non_latin_fraction": 0.0
+ },
+ "workflow": null
+ }
+ },
+ {
+ "label": "unknown_latin_polish",
+ "router_kwargs": {},
+ "route_kwargs": {
+ "state": "Klient został obciążony dwukrotnie i chce zwrot pieniędzy za fakturę"
+ },
+ "decision": {
+ "model": "multilingual",
+ "reason": "Latin script, language not identified but 7% non-English letters; not safe for the English checkpoint",
+ "detection": {
+ "script": "latin",
+ "script_profile": {
+ "latin": 1.0
+ },
+ "language": null,
+ "is_english": false,
+ "language_undecided": true,
+ "diacritic_rate": 0.0735,
+ "non_latin_fraction": 0.0
+ },
+ "workflow": null
+ }
+ },
+ {
+ "label": "unknown_latin_turkish",
+ "router_kwargs": {},
+ "route_kwargs": {
+ "state": "Müşteriden iki kez ücret alındı ve para iadesi istiyor lütfen yardım"
+ },
+ "decision": {
+ "model": "multilingual",
+ "reason": "Latin script, language not identified but 10% non-English letters; not safe for the English checkpoint",
+ "detection": {
+ "script": "latin",
+ "script_profile": {
+ "latin": 1.0
+ },
+ "language": null,
+ "is_english": false,
+ "language_undecided": true,
+ "diacritic_rate": 0.1029,
+ "non_latin_fraction": 0.0
+ },
+ "workflow": null
+ }
+ },
+ {
+ "label": "unknown_latin_undecided_reason",
+ "router_kwargs": {},
+ "route_kwargs": {
+ "state": "Müşteriden iki kez ücret alındı ve para iadesi istiyor"
+ },
+ "decision": {
+ "model": "multilingual",
+ "reason": "Latin script, language not identified but 9% non-English letters; not safe for the English checkpoint",
+ "detection": {
+ "script": "latin",
+ "script_profile": {
+ "latin": 1.0
+ },
+ "language": null,
+ "is_english": false,
+ "language_undecided": true,
+ "diacritic_rate": 0.0926,
+ "non_latin_fraction": 0.0
+ },
+ "workflow": null
+ }
+ },
+ {
+ "label": "english_still_english",
+ "router_kwargs": {},
+ "route_kwargs": {
+ "state": "Please refund the duplicate charge on invoice 4411 today."
+ },
+ "decision": {
+ "model": "english",
+ "reason": "English Latin text",
+ "detection": {
+ "script": "latin",
+ "script_profile": {
+ "latin": 1.0
+ },
+ "language": "en",
+ "is_english": true,
+ "language_undecided": false,
+ "diacritic_rate": 0.0,
+ "non_latin_fraction": 0.0
+ },
+ "workflow": null
+ }
+ },
+ {
+ "label": "short_english_still_english",
+ "router_kwargs": {},
+ "route_kwargs": {
+ "state": "refund me"
+ },
+ "decision": {
+ "model": "english",
+ "reason": "English Latin text",
+ "detection": {
+ "script": "latin",
+ "script_profile": {
+ "latin": 1.0
+ },
+ "language": null,
+ "is_english": true,
+ "language_undecided": true,
+ "diacritic_rate": 0.0,
+ "non_latin_fraction": 0.0
+ },
+ "workflow": null
+ }
+ },
+ {
+ "label": "plain_ascii_es_1",
+ "router_kwargs": {},
+ "route_kwargs": {
+ "state": "El pedido llego roto y nadie responde cuando escribo al soporte"
+ },
+ "decision": {
+ "model": "multilingual",
+ "reason": "Latin script but language looks like 'es', not English",
+ "detection": {
+ "script": "latin",
+ "script_profile": {
+ "latin": 1.0
+ },
+ "language": "es",
+ "is_english": false,
+ "language_undecided": false,
+ "diacritic_rate": 0.0,
+ "non_latin_fraction": 0.0
+ },
+ "workflow": null
+ }
+ },
+ {
+ "label": "plain_ascii_it_1",
+ "router_kwargs": {},
+ "route_kwargs": {
+ "state": "Il cliente e stato addebitato due volte e vuole un rimborso"
+ },
+ "decision": {
+ "model": "multilingual",
+ "reason": "Latin script but language looks like 'it', not English",
+ "detection": {
+ "script": "latin",
+ "script_profile": {
+ "latin": 1.0
+ },
+ "language": "it",
+ "is_english": false,
+ "language_undecided": false,
+ "diacritic_rate": 0.0,
+ "non_latin_fraction": 0.0
+ },
+ "workflow": null
+ }
+ },
+ {
+ "label": "plain_ascii_pt_1",
+ "router_kwargs": {},
+ "route_kwargs": {
+ "state": "O cliente foi cobrado duas vezes e quer o dinheiro de volta"
+ },
+ "decision": {
+ "model": "multilingual",
+ "reason": "Latin script but language looks like 'pt', not English",
+ "detection": {
+ "script": "latin",
+ "script_profile": {
+ "latin": 1.0
+ },
+ "language": "pt",
+ "is_english": false,
+ "language_undecided": false,
+ "diacritic_rate": 0.0,
+ "non_latin_fraction": 0.0
+ },
+ "workflow": null
+ }
+ },
+ {
+ "label": "plain_ascii_fr_1",
+ "router_kwargs": {},
+ "route_kwargs": {
+ "state": "Le client a ete facture deux fois et demande un remboursement"
+ },
+ "decision": {
+ "model": "multilingual",
+ "reason": "Latin script but language looks like 'fr', not English",
+ "detection": {
+ "script": "latin",
+ "script_profile": {
+ "latin": 1.0
+ },
+ "language": "fr",
+ "is_english": false,
+ "language_undecided": false,
+ "diacritic_rate": 0.0,
+ "non_latin_fraction": 0.0
+ },
+ "workflow": null
+ }
+ },
+ {
+ "label": "accented_es",
+ "router_kwargs": {},
+ "route_kwargs": {
+ "state": "La facturación tiene un error y necesito una corrección urgente"
+ },
+ "decision": {
+ "model": "multilingual",
+ "reason": "Latin script but language looks like 'es', not English",
+ "detection": {
+ "script": "latin",
+ "script_profile": {
+ "latin": 1.0
+ },
+ "language": "es",
+ "is_english": false,
+ "language_undecided": false,
+ "diacritic_rate": 0.0317,
+ "non_latin_fraction": 0.0
+ },
+ "workflow": null
+ }
+ },
+ {
+ "label": "accented_it",
+ "router_kwargs": {},
+ "route_kwargs": {
+ "state": "La fattura è sbagliata, devo avere un rimborso per il pagamento"
+ },
+ "decision": {
+ "model": "multilingual",
+ "reason": "Latin script but language looks like 'it', not English",
+ "detection": {
+ "script": "latin",
+ "script_profile": {
+ "latin": 1.0
+ },
+ "language": "it",
+ "is_english": false,
+ "language_undecided": false,
+ "diacritic_rate": 0.0159,
+ "non_latin_fraction": 0.0
+ },
+ "workflow": null
+ }
+ },
+ {
+ "label": "accented_fr",
+ "router_kwargs": {},
+ "route_kwargs": {
+ "state": "La commande est arrivée cassée et personne ne répond au support"
+ },
+ "decision": {
+ "model": "multilingual",
+ "reason": "Latin script but language looks like 'fr', not English",
+ "detection": {
+ "script": "latin",
+ "script_profile": {
+ "latin": 1.0
+ },
+ "language": "fr",
+ "is_english": false,
+ "language_undecided": false,
+ "diacritic_rate": 0.0476,
+ "non_latin_fraction": 0.0
+ },
+ "workflow": null
+ }
+ },
+ {
+ "label": "romance_control_1",
+ "router_kwargs": {},
+ "route_kwargs": {
+ "state": "The customer was charged twice and wants a refund for this invoice"
+ },
+ "decision": {
+ "model": "english",
+ "reason": "English Latin text",
+ "detection": {
+ "script": "latin",
+ "script_profile": {
+ "latin": 1.0
+ },
+ "language": "en",
+ "is_english": true,
+ "language_undecided": false,
+ "diacritic_rate": 0.0,
+ "non_latin_fraction": 0.0
+ },
+ "workflow": null
+ }
+ },
+ {
+ "label": "romance_control_2",
+ "router_kwargs": {},
+ "route_kwargs": {
+ "state": "Our MI5 and UN contacts discussed the DOS attack in LA last month"
+ },
+ "decision": {
+ "model": "english",
+ "reason": "English Latin text",
+ "detection": {
+ "script": "latin",
+ "script_profile": {
+ "latin": 1.0
+ },
+ "language": "en",
+ "is_english": true,
+ "language_undecided": false,
+ "diacritic_rate": 0.0,
+ "non_latin_fraction": 0.0
+ },
+ "workflow": null
+ }
+ },
+ {
+ "label": "shared_words_still_multilingual",
+ "router_kwargs": {},
+ "route_kwargs": {
+ "state": "Cât e ora acum la Tokyo"
+ },
+ "decision": {
+ "model": "multilingual",
+ "reason": "Latin script, language not identified but 4% non-English letters; not safe for the English checkpoint",
+ "detection": {
+ "script": "latin",
+ "script_profile": {
+ "latin": 1.0
+ },
+ "language": null,
+ "is_english": false,
+ "language_undecided": true,
+ "diacritic_rate": 0.0435,
+ "non_latin_fraction": 0.0
+ },
+ "workflow": null
+ }
+ },
+ {
+ "label": "custom_default_multilingual",
+ "router_kwargs": {
+ "default": "multilingual"
+ },
+ "route_kwargs": {
+ "state": "12345"
+ },
+ "decision": {
+ "model": "multilingual",
+ "reason": "no letters detected in state; using default (multilingual)",
+ "detection": {
+ "script": "unknown",
+ "script_profile": {},
+ "language": null,
+ "is_english": true,
+ "language_undecided": true,
+ "diacritic_rate": 0.0,
+ "non_latin_fraction": 0.0
+ },
+ "workflow": null
+ }
+ },
+ {
+ "label": "custom_default_typed_decisions",
+ "router_kwargs": {
+ "default": "typed-decisions"
+ },
+ "route_kwargs": {
+ "state": ""
+ },
+ "decision": {
+ "model": "typed-decisions",
+ "reason": "no letters detected in state; using default (typed-decisions)",
+ "detection": {
+ "script": "unknown",
+ "script_profile": {},
+ "language": null,
+ "is_english": true,
+ "language_undecided": true,
+ "diacritic_rate": 0.0,
+ "non_latin_fraction": 0.0
+ },
+ "workflow": null
+ }
+ },
+ {
+ "label": "custom_default_english_script_unknown",
+ "router_kwargs": {
+ "default": "english"
+ },
+ "route_kwargs": {
+ "state": "12345 6789"
+ },
+ "decision": {
+ "model": "english",
+ "reason": "no letters detected in state; using default (english)",
+ "detection": {
+ "script": "unknown",
+ "script_profile": {},
+ "language": null,
+ "is_english": true,
+ "language_undecided": true,
+ "diacritic_rate": 0.0,
+ "non_latin_fraction": 0.0
+ },
+ "workflow": null
+ }
+ },
+ {
+ "label": "bundle_default_hindi",
+ "router_kwargs": {},
+ "route_kwargs": {
+ "state": {
+ "m": "मुझसे दो बार"
+ },
+ "questions": {
+ "dept": {
+ "type": "choice",
+ "instructions": "Which team?",
+ "criteria": {
+ "billing": null,
+ "tech": null
+ }
+ }
+ }
+ },
+ "decision": {
+ "model": "multilingual",
+ "reason": "non-Latin script (devanagari, 100% of letters); the English checkpoint cannot read it",
+ "detection": {
+ "script": "devanagari",
+ "script_profile": {
+ "devanagari": 1.0
+ },
+ "language": null,
+ "is_english": false,
+ "language_undecided": true,
+ "diacritic_rate": 0.0,
+ "non_latin_fraction": 1.0
+ },
+ "workflow": null
+ }
+ },
+ {
+ "label": "standalone_opt_in_hindi",
+ "router_kwargs": {
+ "standalone_repos": true
+ },
+ "route_kwargs": {
+ "state": {
+ "m": "मुझसे दो बार"
+ },
+ "questions": {
+ "dept": {
+ "type": "choice",
+ "instructions": "Which team?",
+ "criteria": {
+ "billing": null,
+ "tech": null
+ }
+ }
+ }
+ },
+ "decision": {
+ "model": "multilingual",
+ "reason": "non-Latin script (devanagari, 100% of letters); the English checkpoint cannot read it",
+ "detection": {
+ "script": "devanagari",
+ "script_profile": {
+ "devanagari": 1.0
+ },
+ "language": null,
+ "is_english": false,
+ "language_undecided": true,
+ "diacritic_rate": 0.0,
+ "non_latin_fraction": 1.0
+ },
+ "workflow": null
+ }
+ },
+ {
+ "label": "standalone_english_unchanged",
+ "router_kwargs": {
+ "standalone_repos": true
+ },
+ "route_kwargs": {
+ "state": {
+ "m": "I was charged twice"
+ },
+ "questions": {
+ "dept": {
+ "type": "choice",
+ "instructions": "Which team?",
+ "criteria": {
+ "billing": null,
+ "tech": null
+ }
+ }
+ }
+ },
+ "decision": {
+ "model": "english",
+ "reason": "English Latin text",
+ "detection": {
+ "script": "latin",
+ "script_profile": {
+ "latin": 1.0
+ },
+ "language": "en",
+ "is_english": true,
+ "language_undecided": false,
+ "diacritic_rate": 0.0,
+ "non_latin_fraction": 0.0
+ },
+ "workflow": null
+ }
+ },
+ {
+ "label": "local_path_override",
+ "router_kwargs": {
+ "models": {
+ "english": "/tmp/en",
+ "multilingual": "/tmp/ml"
+ }
+ },
+ "route_kwargs": {
+ "state": {
+ "m": "मुझसे दो बार"
+ },
+ "questions": {
+ "dept": {
+ "type": "choice",
+ "instructions": "Which team?",
+ "criteria": {
+ "billing": null,
+ "tech": null
+ }
+ }
+ }
+ },
+ "decision": {
+ "model": "multilingual",
+ "reason": "non-Latin script (devanagari, 100% of letters); the English checkpoint cannot read it",
+ "detection": {
+ "script": "devanagari",
+ "script_profile": {
+ "devanagari": 1.0
+ },
+ "language": null,
+ "is_english": false,
+ "language_undecided": true,
+ "diacritic_rate": 0.0,
+ "non_latin_fraction": 1.0
+ },
+ "workflow": null
+ }
+ },
+ {
+ "label": "alias_en",
+ "router_kwargs": {},
+ "route_kwargs": {
+ "state": "x",
+ "model": "en"
+ },
+ "decision": {
+ "model": "english",
+ "reason": "explicit model='en'",
+ "detection": null,
+ "workflow": null
+ }
+ },
+ {
+ "label": "alias_laya",
+ "router_kwargs": {},
+ "route_kwargs": {
+ "state": "x",
+ "model": "laya"
+ },
+ "decision": {
+ "model": "english",
+ "reason": "explicit model='laya'",
+ "detection": null,
+ "workflow": null
+ }
+ },
+ {
+ "label": "alias_default",
+ "router_kwargs": {},
+ "route_kwargs": {
+ "state": "x",
+ "model": "default"
+ },
+ "decision": {
+ "model": "english",
+ "reason": "explicit model='default'",
+ "detection": null,
+ "workflow": null
+ }
+ },
+ {
+ "label": "alias_multi",
+ "router_kwargs": {},
+ "route_kwargs": {
+ "state": "x",
+ "model": "multi"
+ },
+ "decision": {
+ "model": "multilingual",
+ "reason": "explicit model='multi'",
+ "detection": null,
+ "workflow": null
+ }
+ },
+ {
+ "label": "alias_ml",
+ "router_kwargs": {},
+ "route_kwargs": {
+ "state": "x",
+ "model": "ml"
+ },
+ "decision": {
+ "model": "multilingual",
+ "reason": "explicit model='ml'",
+ "detection": null,
+ "workflow": null
+ }
+ },
+ {
+ "label": "alias_laya_multilingual",
+ "router_kwargs": {},
+ "route_kwargs": {
+ "state": "x",
+ "model": "laya-multilingual"
+ },
+ "decision": {
+ "model": "multilingual",
+ "reason": "explicit model='laya-multilingual'",
+ "detection": null,
+ "workflow": null
+ }
+ },
+ {
+ "label": "alias_typed",
+ "router_kwargs": {},
+ "route_kwargs": {
+ "state": "x",
+ "model": "typed"
+ },
+ "decision": {
+ "model": "typed-decisions",
+ "reason": "explicit model='typed'",
+ "detection": null,
+ "workflow": null
+ }
+ },
+ {
+ "label": "alias_typed_decisions",
+ "router_kwargs": {},
+ "route_kwargs": {
+ "state": "x",
+ "model": "typed_decisions"
+ },
+ "decision": {
+ "model": "typed-decisions",
+ "reason": "explicit model='typed_decisions'",
+ "detection": null,
+ "workflow": null
+ }
+ },
+ {
+ "label": "alias_laya_typed_decisions",
+ "router_kwargs": {},
+ "route_kwargs": {
+ "state": "x",
+ "model": "laya-typed-decisions"
+ },
+ "decision": {
+ "model": "typed-decisions",
+ "reason": "explicit model='laya-typed-decisions'",
+ "detection": null,
+ "workflow": null
+ }
+ },
+ {
+ "label": "alias_decisions",
+ "router_kwargs": {},
+ "route_kwargs": {
+ "state": "x",
+ "model": "decisions"
+ },
+ "decision": {
+ "model": "typed-decisions",
+ "reason": "explicit model='decisions'",
+ "detection": null,
+ "workflow": null
+ }
+ },
+ {
+ "label": "alias_uppercase_ML",
+ "router_kwargs": {},
+ "route_kwargs": {
+ "state": "x",
+ "model": "ML"
+ },
+ "decision": {
+ "model": "multilingual",
+ "reason": "explicit model='ML'",
+ "detection": null,
+ "workflow": null
+ }
+ },
+ {
+ "label": "alias_mixed_case_English",
+ "router_kwargs": {},
+ "route_kwargs": {
+ "state": "x",
+ "model": "English"
+ },
+ "decision": {
+ "model": "english",
+ "reason": "explicit model='English'",
+ "detection": null,
+ "workflow": null
+ }
+ },
+ {
+ "label": "alias_via_task_typed_decisions_with_dash",
+ "router_kwargs": {},
+ "route_kwargs": {
+ "state": "x",
+ "task": "typed-decisions"
+ },
+ "decision": {
+ "model": "typed-decisions",
+ "reason": "explicit task='typed-decisions'",
+ "detection": null,
+ "workflow": null
+ }
+ },
+ {
+ "label": "alias_via_task_typed_decisions_with_underscore",
+ "router_kwargs": {},
+ "route_kwargs": {
+ "state": "x",
+ "task": "typed_decisions"
+ },
+ "decision": {
+ "model": "typed-decisions",
+ "reason": "explicit task='typed_decisions'",
+ "detection": null,
+ "workflow": null
+ }
+ },
+ {
+ "label": "payload_shape_hindi",
+ "router_kwargs": {},
+ "route_kwargs": {
+ "state": {
+ "body": "मुझसे दो बार शुल्क लिया गया"
+ },
+ "questions": {
+ "dept": {
+ "type": "choice",
+ "instructions": "Which team?",
+ "criteria": {
+ "billing": null,
+ "tech": null
+ }
+ }
+ }
+ },
+ "decision": {
+ "model": "multilingual",
+ "reason": "non-Latin script (devanagari, 100% of letters); the English checkpoint cannot read it",
+ "detection": {
+ "script": "devanagari",
+ "script_profile": {
+ "devanagari": 1.0
+ },
+ "language": null,
+ "is_english": false,
+ "language_undecided": true,
+ "diacritic_rate": 0.0,
+ "non_latin_fraction": 1.0
+ },
+ "workflow": null
+ }
+ },
+ {
+ "label": "payload_shape_armenian_non_latin_percent",
+ "router_kwargs": {},
+ "route_kwargs": {
+ "state": "Հայերեն abc"
+ },
+ "decision": {
+ "model": "multilingual",
+ "reason": "non-Latin script (armenian, 70% of letters); the English checkpoint cannot read it",
+ "detection": {
+ "script": "armenian",
+ "script_profile": {
+ "latin": 0.3,
+ "armenian": 0.7
+ },
+ "language": null,
+ "is_english": false,
+ "language_undecided": true,
+ "diacritic_rate": 0.0,
+ "non_latin_fraction": 0.7
+ },
+ "workflow": null
+ }
+ },
+ {
+ "label": "payload_shape_unidentified_latin_percent",
+ "router_kwargs": {},
+ "route_kwargs": {
+ "state": "Müşteriden iki kez ücret alındı ve para iadesi istiyor"
+ },
+ "decision": {
+ "model": "multilingual",
+ "reason": "Latin script, language not identified but 9% non-English letters; not safe for the English checkpoint",
+ "detection": {
+ "script": "latin",
+ "script_profile": {
+ "latin": 1.0
+ },
+ "language": null,
+ "is_english": false,
+ "language_undecided": true,
+ "diacritic_rate": 0.0926,
+ "non_latin_fraction": 0.0
+ },
+ "workflow": null
+ }
+ }
+]
diff --git a/laya-rust/tests/golden/routing/sample_inputs.json b/laya-rust/tests/golden/routing/sample_inputs.json
new file mode 100644
index 00000000..c3cb120f
--- /dev/null
+++ b/laya-rust/tests/golden/routing/sample_inputs.json
@@ -0,0 +1,508 @@
+{
+ "detection_states": [
+ {
+ "label": "english",
+ "state": "The customer was charged twice this month and wants a refund.",
+ "analyse": {
+ "script": "latin",
+ "script_profile": {
+ "latin": 1.0
+ },
+ "language": "en",
+ "is_english": true,
+ "language_undecided": false,
+ "diacritic_rate": 0.0,
+ "non_latin_fraction": 0.0
+ }
+ },
+ {
+ "label": "spanish_accent_stripped",
+ "state": "El cliente fue cobrado dos veces y quiere que le devuelvan el dinero",
+ "analyse": {
+ "script": "latin",
+ "script_profile": {
+ "latin": 1.0
+ },
+ "language": "es",
+ "is_english": false,
+ "language_undecided": false,
+ "diacritic_rate": 0.0,
+ "non_latin_fraction": 0.0
+ }
+ },
+ {
+ "label": "french",
+ "state": "Le client a été facturé deux fois et demande un remboursement.",
+ "analyse": {
+ "script": "latin",
+ "script_profile": {
+ "latin": 1.0
+ },
+ "language": "fr",
+ "is_english": false,
+ "language_undecided": false,
+ "diacritic_rate": 0.0484,
+ "non_latin_fraction": 0.0
+ }
+ },
+ {
+ "label": "romanian",
+ "state": "Am fost taxat de două ori pentru factura din luna martie și vreau banii",
+ "analyse": {
+ "script": "latin",
+ "script_profile": {
+ "latin": 1.0
+ },
+ "language": "ro",
+ "is_english": false,
+ "language_undecided": false,
+ "diacritic_rate": 0.0282,
+ "non_latin_fraction": 0.0
+ }
+ },
+ {
+ "label": "hindi",
+ "state": "ग्राहक से दो बार शुल्क लिया गया और वह धनवापसी चाहता है।",
+ "analyse": {
+ "script": "devanagari",
+ "script_profile": {
+ "devanagari": 1.0
+ },
+ "language": null,
+ "is_english": false,
+ "language_undecided": true,
+ "diacritic_rate": 0.0,
+ "non_latin_fraction": 1.0
+ }
+ },
+ {
+ "label": "arabic",
+ "state": "تم خصم المبلغ مرتين من العميل ويريد استرداد الأموال",
+ "analyse": {
+ "script": "arabic",
+ "script_profile": {
+ "arabic": 1.0
+ },
+ "language": null,
+ "is_english": false,
+ "language_undecided": true,
+ "diacritic_rate": 0.0,
+ "non_latin_fraction": 1.0
+ }
+ },
+ {
+ "label": "japanese",
+ "state": "お客様は二重に請求されたため返金を希望しています。",
+ "analyse": {
+ "script": "kana",
+ "script_profile": {
+ "kana": 0.5833333333333334,
+ "han": 0.4166666666666667
+ },
+ "language": null,
+ "is_english": false,
+ "language_undecided": true,
+ "diacritic_rate": 0.0,
+ "non_latin_fraction": 1.0
+ }
+ },
+ {
+ "label": "dict_state",
+ "state": {
+ "subject": "Double charge",
+ "body": "मुझसे दो बार शुल्क लिया गया"
+ },
+ "analyse": {
+ "script": "devanagari",
+ "script_profile": {
+ "latin": 0.48,
+ "devanagari": 0.52
+ },
+ "language": null,
+ "is_english": false,
+ "language_undecided": true,
+ "diacritic_rate": 0.0,
+ "non_latin_fraction": 0.52
+ }
+ }
+ ],
+ "routing_decisions": [
+ {
+ "label": "auto_english",
+ "router_kwargs": {},
+ "route_kwargs": {
+ "state": "Please refund the duplicate charge on invoice 4411 today."
+ },
+ "decision": {
+ "model": "english",
+ "reason": "English Latin text",
+ "detection": {
+ "script": "latin",
+ "script_profile": {
+ "latin": 1.0
+ },
+ "language": "en",
+ "is_english": true,
+ "language_undecided": false,
+ "diacritic_rate": 0.0,
+ "non_latin_fraction": 0.0
+ },
+ "workflow": null
+ }
+ },
+ {
+ "label": "auto_hindi",
+ "router_kwargs": {},
+ "route_kwargs": {
+ "state": "मुझसे दो बार शुल्क लिया गया, कृपया रिफंड करें।"
+ },
+ "decision": {
+ "model": "multilingual",
+ "reason": "non-Latin script (devanagari, 100% of letters); the English checkpoint cannot read it",
+ "detection": {
+ "script": "devanagari",
+ "script_profile": {
+ "devanagari": 1.0
+ },
+ "language": null,
+ "is_english": false,
+ "language_undecided": true,
+ "diacritic_rate": 0.0,
+ "non_latin_fraction": 1.0
+ },
+ "workflow": null
+ }
+ },
+ {
+ "label": "auto_unknown_latin_romanian",
+ "router_kwargs": {},
+ "route_kwargs": {
+ "state": "Gătește-mi o rețetă de sarmale de post pentru mâine."
+ },
+ "decision": {
+ "model": "multilingual",
+ "reason": "Latin script but language looks like 'ro', not English",
+ "detection": {
+ "script": "latin",
+ "script_profile": {
+ "latin": 1.0
+ },
+ "language": "ro",
+ "is_english": false,
+ "language_undecided": false,
+ "diacritic_rate": 0.0962,
+ "non_latin_fraction": 0.0
+ },
+ "workflow": null
+ }
+ },
+ {
+ "label": "explicit_model_multilingual",
+ "router_kwargs": {},
+ "route_kwargs": {
+ "state": "anything at all",
+ "model": "multilingual"
+ },
+ "decision": {
+ "model": "multilingual",
+ "reason": "explicit model='multilingual'",
+ "detection": null,
+ "workflow": null
+ }
+ },
+ {
+ "label": "explicit_lang_de",
+ "router_kwargs": {},
+ "route_kwargs": {
+ "state": "hello there",
+ "lang": "de"
+ },
+ "decision": {
+ "model": "multilingual",
+ "reason": "explicit lang='de'",
+ "detection": null,
+ "workflow": null
+ }
+ },
+ {
+ "label": "workflow_opt_in_customer_service",
+ "router_kwargs": {
+ "auto_task_detection": true
+ },
+ "route_kwargs": {
+ "state": {
+ "body": "I was charged twice"
+ },
+ "questions": {
+ "action": {
+ "type": "noul",
+ "instructions": "x"
+ },
+ "category": {
+ "type": "noul",
+ "instructions": "x"
+ },
+ "churn_risk": {
+ "type": "noul",
+ "instructions": "x"
+ },
+ "needs_human": {
+ "type": "noul",
+ "instructions": "x"
+ },
+ "urgency": {
+ "type": "noul",
+ "instructions": "x"
+ }
+ }
+ },
+ "decision": {
+ "model": "typed-decisions",
+ "reason": "question ids match the 'customer_service' typed-decisions workflow",
+ "detection": null,
+ "workflow": "customer_service"
+ }
+ },
+ {
+ "label": "workflow_opt_in_off_by_default",
+ "router_kwargs": {},
+ "route_kwargs": {
+ "state": {
+ "body": "I was charged twice"
+ },
+ "questions": {
+ "action": {
+ "type": "noul",
+ "instructions": "x"
+ },
+ "category": {
+ "type": "noul",
+ "instructions": "x"
+ },
+ "churn_risk": {
+ "type": "noul",
+ "instructions": "x"
+ },
+ "needs_human": {
+ "type": "noul",
+ "instructions": "x"
+ },
+ "urgency": {
+ "type": "noul",
+ "instructions": "x"
+ }
+ }
+ },
+ "decision": {
+ "model": "english",
+ "reason": "English Latin text",
+ "detection": {
+ "script": "latin",
+ "script_profile": {
+ "latin": 1.0
+ },
+ "language": "en",
+ "is_english": true,
+ "language_undecided": false,
+ "diacritic_rate": 0.0,
+ "non_latin_fraction": 0.0
+ },
+ "workflow": "customer_service"
+ }
+ }
+ ],
+ "support_email": {
+ "questions": {
+ "department": {
+ "type": "choice",
+ "instructions": "Which department should handle this request?",
+ "criteria": {
+ "billing": "invoices, payments, refunds",
+ "technical": "bugs, outages, system errors",
+ "sales": "pricing, new contracts",
+ "other": "everything else"
+ }
+ },
+ "urgency": {
+ "type": "score",
+ "instructions": "How urgent is this request?",
+ "criteria": [
+ "not urgent",
+ "soon",
+ "critical deadline or blocking issue"
+ ]
+ },
+ "churn_risk": {
+ "type": "noul",
+ "instructions": "Does the user threaten to cancel or leave?"
+ },
+ "refund_requested": {
+ "type": "noul",
+ "instructions": "Does the user explicitly request a refund?"
+ }
+ },
+ "english_state": {
+ "from": "[email protected]",
+ "subject": "Duplicate charge on invoice #4411",
+ "body": "Hi, we were billed twice for March. Please refund the duplicate today or we will cancel our plan."
+ },
+ "hindi_state": {
+ "from": "[email protected]",
+ "subject": "चालान में दोहरा शुल्क",
+ "body": "मार्च का बिल दो बार लिया गया, कृपया रिफंड करें।"
+ },
+ "english_routing": {
+ "model": "english",
+ "reason": "English Latin text",
+ "detection": {
+ "script": "latin",
+ "script_profile": {
+ "latin": 1.0
+ },
+ "language": "en",
+ "is_english": true,
+ "language_undecided": false,
+ "diacritic_rate": 0.0,
+ "non_latin_fraction": 0.0
+ },
+ "workflow": null
+ },
+ "hindi_routing": {
+ "model": "multilingual",
+ "reason": "non-Latin script (devanagari, 74% of letters); the English checkpoint cannot read it",
+ "detection": {
+ "script": "devanagari",
+ "script_profile": {
+ "latin": 0.2619047619047619,
+ "devanagari": 0.7380952380952381
+ },
+ "language": null,
+ "is_english": false,
+ "language_undecided": true,
+ "diacritic_rate": 0.0,
+ "non_latin_fraction": 0.7381
+ },
+ "workflow": null
+ }
+ },
+ "raw_email": {
+ "subject": "Re: Refund for invoice 4411",
+ "sender": "[email protected]",
+ "body": "Hi team,\n\nI was charged twice for the Pro plan this month and the second charge still has not been refunded. This is the third time I have written about it -- please resolve this today or I will need to cancel the account.\n\nBest regards,\nPriya Sharma\nSent from my iPhone\n\nThis email and any files transmitted with it are confidential and intended solely for the use of the individual or entity to whom they are addressed. If you have received this email in error please notify the sender.\n\nOn Tue, Sep 22, 2026 at 9:14 AM, Support <[email protected]> wrote:\n> Hi Priya, thanks for reaching out, we are looking into it.\n> - Support Team\n",
+ "cleaned_body": "Hi team,\n\nI was charged twice for the Pro plan this month and the second charge still has not been refunded. This is the third time I have written about it -- please resolve this today or I will need to cancel the account.",
+ "state": {
+ "subject": "Re: Refund for invoice 4411",
+ "body": "Hi team,\n\nI was charged twice for the Pro plan this month and the second charge still has not been refunded. This is the third time I have written about it -- please resolve this today or I will need to cancel the account.",
+ "from": "[email protected]"
+ },
+ "routing": {
+ "model": "english",
+ "reason": "English Latin text",
+ "detection": {
+ "script": "latin",
+ "script_profile": {
+ "latin": 1.0
+ },
+ "language": "en",
+ "is_english": true,
+ "language_undecided": false,
+ "diacritic_rate": 0.0,
+ "non_latin_fraction": 0.0
+ },
+ "workflow": null
+ }
+ },
+ "banking_shortlist": {
+ "state": "I tried to withdraw cash from the ATM but it kept the card and gave me no money.",
+ "instructions": "What is the customer's banking intent?",
+ "criteria": {
+ "activate_my_card": "turn on a newly received card",
+ "age_limit": "minimum or maximum age to use the service",
+ "apple_pay_or_google_pay": "adding the card to a mobile wallet",
+ "atm_support": "which ATMs the card works at",
+ "automatic_top_up": "automatically topping up the balance",
+ "balance_not_updated_after_bank_transfer": "a bank transfer is missing from the balance",
+ "balance_not_updated_after_cheque_or_cash_deposit": "a deposit is missing from the balance",
+ "beneficiary_not_allowed": "cannot add a payment recipient",
+ "cancel_transfer": "wants to cancel a transfer already sent",
+ "card_about_to_expire": "the card is close to its expiry date",
+ "card_acceptance": "where the card is accepted",
+ "card_arrival": "asking when a new card will arrive",
+ "card_delivery_estimate": "asking how long delivery will take",
+ "card_linking": "linking a card to the account",
+ "card_not_working": "the card is declined or not working",
+ "card_payment_fee_charged": "an unexpected fee was charged on a card payment",
+ "card_payment_not_recognised": "a card payment the customer does not recognise",
+ "card_payment_wrong_exchange_rate": "the exchange rate used was wrong",
+ "card_swallowed": "the ATM kept the card",
+ "cash_withdrawal_charge": "an unexpected charge for a cash withdrawal",
+ "cash_withdrawal_not_recognised": "a cash withdrawal the customer does not recognise",
+ "change_pin": "wants to change the PIN",
+ "compromised_card": "the card may have been compromised",
+ "contactless_not_working": "contactless payment is not working",
+ "country_support": "which countries the service supports",
+ "declined_card_payment": "a card payment was declined",
+ "declined_cash_withdrawal": "a cash withdrawal was declined",
+ "declined_transfer": "a transfer was declined",
+ "direct_debit_payment_not_recognised": "a direct debit the customer does not recognise",
+ "disposable_card_limits": "limits on a disposable virtual card",
+ "edit_personal_details": "wants to edit name, address or other personal details",
+ "exchange_charge": "a fee charged for currency exchange",
+ "exchange_rate": "asking what the current exchange rate is",
+ "exchange_via_app": "exchanging currency inside the app",
+ "extra_charge_on_statement": "an unexplained extra charge on the statement",
+ "failed_transfer": "a transfer failed",
+ "fiat_currency_support": "which regular currencies are supported",
+ "get_disposable_virtual_card": "wants a disposable virtual card",
+ "get_physical_card": "wants a physical card",
+ "getting_spare_card": "wants a spare or backup card"
+ },
+ "k": 20,
+ "n": 40,
+ "passthrough": false,
+ "labels": [
+ "card_swallowed",
+ "compromised_card",
+ "cash_withdrawal_not_recognised",
+ "card_payment_not_recognised",
+ "automatic_top_up",
+ "card_linking",
+ "atm_support",
+ "direct_debit_payment_not_recognised",
+ "card_payment_wrong_exchange_rate",
+ "balance_not_updated_after_bank_transfer",
+ "card_not_working",
+ "declined_card_payment",
+ "card_payment_fee_charged",
+ "card_acceptance",
+ "exchange_rate",
+ "card_delivery_estimate",
+ "activate_my_card",
+ "balance_not_updated_after_cheque_or_cash_deposit",
+ "card_about_to_expire",
+ "country_support"
+ ],
+ "scores": [
+ 0.5368900200020207,
+ 0.5229341330248302,
+ 0.4926179352570128,
+ 0.4672093110356841,
+ 0.4630985024104475,
+ 0.4370994625491302,
+ 0.37826231739982075,
+ 0.36425922895521706,
+ 0.347170429282197,
+ 0.3320328885206982,
+ 0.3300546962105677,
+ 0.32050294323210465,
+ 0.3154110908433172,
+ 0.29833907794225273,
+ 0.29153747948292086,
+ 0.2862182802596687,
+ 0.2750998671998696,
+ 0.2737391010624727,
+ 0.2667574502450347,
+ 0.2631287408127802
+ ]
+ }
+}
diff --git a/laya-rust/tests/golden/routing/shortlist_probe.json b/laya-rust/tests/golden/routing/shortlist_probe.json
new file mode 100644
index 00000000..67273532
--- /dev/null
+++ b/laya-rust/tests/golden/routing/shortlist_probe.json
@@ -0,0 +1,790 @@
+[
+ {
+ "label": "tie_cosine_keeps_earlier_label",
+ "state": "pay me",
+ "criteria": {
+ "alpha": null,
+ "beta": "",
+ "gamma": "mid",
+ "delta": "same"
+ },
+ "k": 2,
+ "instructions": null,
+ "vectors": [
+ [
+ 1.0,
+ 0.0
+ ],
+ [
+ 1.0,
+ 0.0
+ ],
+ [
+ 0.0,
+ 1.0
+ ],
+ [
+ 0.6,
+ 0.8
+ ],
+ [
+ 1.0,
+ 0.0
+ ]
+ ],
+ "labels": [
+ "alpha",
+ "delta"
+ ],
+ "scores": [
+ 1.0,
+ 1.0
+ ],
+ "passthrough": false,
+ "n": 4
+ },
+ {
+ "label": "k1_is_earliest_max",
+ "state": "pay me",
+ "criteria": {
+ "alpha": null,
+ "beta": "",
+ "gamma": "mid",
+ "delta": "same"
+ },
+ "k": 1,
+ "instructions": null,
+ "vectors": [
+ [
+ 1.0,
+ 0.0
+ ],
+ [
+ 1.0,
+ 0.0
+ ],
+ [
+ 0.0,
+ 1.0
+ ],
+ [
+ 0.6,
+ 0.8
+ ],
+ [
+ 1.0,
+ 0.0
+ ]
+ ],
+ "labels": [
+ "alpha"
+ ],
+ "scores": [
+ 1.0
+ ],
+ "passthrough": false,
+ "n": 4
+ },
+ {
+ "label": "k3_appends_next_cosine",
+ "state": "pay me",
+ "criteria": {
+ "alpha": null,
+ "beta": "",
+ "gamma": "mid",
+ "delta": "same"
+ },
+ "k": 3,
+ "instructions": null,
+ "vectors": [
+ [
+ 1.0,
+ 0.0
+ ],
+ [
+ 1.0,
+ 0.0
+ ],
+ [
+ 0.0,
+ 1.0
+ ],
+ [
+ 0.6,
+ 0.8
+ ],
+ [
+ 1.0,
+ 0.0
+ ]
+ ],
+ "labels": [
+ "alpha",
+ "delta",
+ "gamma"
+ ],
+ "scores": [
+ 1.0,
+ 1.0,
+ 0.6
+ ],
+ "passthrough": false,
+ "n": 4
+ },
+ {
+ "label": "zero_query_keeps_original_order",
+ "state": "pay me",
+ "criteria": {
+ "alpha": null,
+ "beta": "",
+ "gamma": "mid",
+ "delta": "same"
+ },
+ "k": 2,
+ "instructions": null,
+ "vectors": [
+ [
+ 0.0,
+ 0.0
+ ],
+ [
+ 1.0,
+ 0.0
+ ],
+ [
+ 0.0,
+ 1.0
+ ],
+ [
+ 0.6,
+ 0.8
+ ],
+ [
+ 3.0,
+ 4.0
+ ]
+ ],
+ "labels": [
+ "alpha",
+ "beta"
+ ],
+ "scores": [
+ 0.0,
+ 0.0
+ ],
+ "passthrough": false,
+ "n": 4
+ },
+ {
+ "label": "zero_option_vector_loses",
+ "state": "pay me",
+ "criteria": {
+ "alpha": null,
+ "beta": null,
+ "gamma": null
+ },
+ "k": 2,
+ "instructions": null,
+ "vectors": [
+ [
+ 1.0,
+ 0.0
+ ],
+ [
+ 0.0,
+ 0.0
+ ],
+ [
+ 1.0,
+ 0.0
+ ],
+ [
+ 0.0,
+ 1.0
+ ]
+ ],
+ "labels": [
+ "beta",
+ "alpha"
+ ],
+ "scores": [
+ 1.0,
+ 0.0
+ ],
+ "passthrough": false,
+ "n": 3
+ },
+ {
+ "label": "nan_vector_sorts_behind_finite_match",
+ "state": "pay me",
+ "criteria": {
+ "alpha": null,
+ "beta": null
+ },
+ "k": 1,
+ "instructions": null,
+ "vectors": [
+ [
+ 1.0,
+ 0.0
+ ],
+ [
+ "nan",
+ "nan"
+ ],
+ [
+ 1.0,
+ 0.0
+ ]
+ ],
+ "labels": [
+ "beta"
+ ],
+ "scores": [
+ 1.0
+ ],
+ "passthrough": false,
+ "n": 2
+ },
+ {
+ "label": "positive_inf_and_negative_inf_both_zeroed",
+ "state": "pay me",
+ "criteria": {
+ "alpha": null,
+ "beta": null,
+ "gamma": null
+ },
+ "k": 2,
+ "instructions": null,
+ "vectors": [
+ [
+ 1.0,
+ 0.0
+ ],
+ [
+ "inf",
+ "inf"
+ ],
+ [
+ "-inf",
+ 0.0
+ ],
+ [
+ 1.0,
+ 0.0
+ ]
+ ],
+ "labels": [
+ "gamma",
+ "alpha"
+ ],
+ "scores": [
+ 1.0,
+ 0.0
+ ],
+ "passthrough": false,
+ "n": 3
+ },
+ {
+ "label": "nan_component_in_query_only_zeroes_that_component",
+ "state": "pay me",
+ "criteria": {
+ "alpha": null,
+ "beta": null
+ },
+ "k": 1,
+ "instructions": null,
+ "vectors": [
+ [
+ "nan",
+ 1.0
+ ],
+ [
+ 1.0,
+ 0.0
+ ],
+ [
+ 0.0,
+ 1.0
+ ]
+ ],
+ "labels": [
+ "beta"
+ ],
+ "scores": [
+ 1.0
+ ],
+ "passthrough": false,
+ "n": 2
+ },
+ {
+ "label": "fully_nan_query_falls_back_to_order",
+ "state": "pay me",
+ "criteria": {
+ "alpha": null,
+ "beta": null
+ },
+ "k": 1,
+ "instructions": null,
+ "vectors": [
+ [
+ "nan",
+ "nan"
+ ],
+ [
+ 1.0,
+ 0.0
+ ],
+ [
+ 0.0,
+ 1.0
+ ]
+ ],
+ "labels": [
+ "alpha"
+ ],
+ "scores": [
+ 0.0
+ ],
+ "passthrough": false,
+ "n": 2
+ },
+ {
+ "label": "k_equals_n_passthrough",
+ "state": "pay me",
+ "criteria": {
+ "alpha": null,
+ "beta": "",
+ "gamma": "mid",
+ "delta": "same"
+ },
+ "k": 4,
+ "instructions": null,
+ "vectors": null,
+ "labels": [
+ "alpha",
+ "beta",
+ "gamma",
+ "delta"
+ ],
+ "scores": null,
+ "passthrough": true,
+ "n": 4
+ },
+ {
+ "label": "k_greater_than_n_passthrough",
+ "state": "pay me",
+ "criteria": {
+ "alpha": null,
+ "beta": "",
+ "gamma": "mid",
+ "delta": "same"
+ },
+ "k": 20,
+ "instructions": null,
+ "vectors": null,
+ "labels": [
+ "alpha",
+ "beta",
+ "gamma",
+ "delta"
+ ],
+ "scores": null,
+ "passthrough": true,
+ "n": 4
+ },
+ {
+ "label": "k_equals_n_single_option_passthrough",
+ "state": "pay me",
+ "criteria": {
+ "solo": "the only choice"
+ },
+ "k": 1,
+ "instructions": null,
+ "vectors": null,
+ "labels": [
+ "solo"
+ ],
+ "scores": null,
+ "passthrough": true,
+ "n": 1
+ },
+ {
+ "label": "list_criteria_topk",
+ "state": "pay me",
+ "criteria": [
+ "alpha",
+ "beta",
+ "gamma"
+ ],
+ "k": 2,
+ "instructions": "Classify",
+ "vectors": [
+ [
+ 0.0,
+ 1.0
+ ],
+ [
+ 1.0,
+ 0.0
+ ],
+ [
+ 0.0,
+ 1.0
+ ],
+ [
+ 0.0,
+ 0.2
+ ]
+ ],
+ "labels": [
+ "beta",
+ "gamma"
+ ],
+ "scores": [
+ 1.0,
+ 1.0
+ ],
+ "passthrough": false,
+ "n": 3
+ },
+ {
+ "label": "list_criteria_passthrough",
+ "state": "hello",
+ "criteria": [
+ "alpha",
+ "beta",
+ "gamma"
+ ],
+ "k": 5,
+ "instructions": "Which?",
+ "vectors": null,
+ "labels": [
+ "alpha",
+ "beta",
+ "gamma"
+ ],
+ "scores": null,
+ "passthrough": true,
+ "n": 3
+ },
+ {
+ "label": "dict_state_query",
+ "state": {
+ "text": "hi"
+ },
+ "criteria": [
+ "alpha",
+ "beta"
+ ],
+ "k": 1,
+ "instructions": "Classify",
+ "vectors": [
+ [
+ 1.0,
+ 0.0
+ ],
+ [
+ 1.0,
+ 0.0
+ ],
+ [
+ 0.0,
+ 1.0
+ ]
+ ],
+ "labels": [
+ "alpha"
+ ],
+ "scores": [
+ 1.0
+ ],
+ "passthrough": false,
+ "n": 2
+ },
+ {
+ "label": "zero_and_false_criteria_values",
+ "state": "pay me",
+ "criteria": {
+ "zero": 0,
+ "no": false,
+ "bare": null,
+ "named": {
+ "desc": "payments"
+ }
+ },
+ "k": 4,
+ "instructions": null,
+ "vectors": null,
+ "labels": [
+ "zero",
+ "no",
+ "bare",
+ "named"
+ ],
+ "scores": null,
+ "passthrough": true,
+ "n": 4
+ },
+ {
+ "label": "zero_and_false_criteria_ranked",
+ "state": "pay me",
+ "criteria": {
+ "zero": 0,
+ "no": false,
+ "bare": null,
+ "named": {
+ "desc": "payments"
+ }
+ },
+ "k": 2,
+ "instructions": null,
+ "vectors": [
+ [
+ 1.0,
+ 0.0
+ ],
+ [
+ 1.0,
+ 0.0
+ ],
+ [
+ 0.0,
+ 1.0
+ ],
+ [
+ 0.9,
+ 0.1
+ ],
+ [
+ 0.0,
+ 1.0
+ ]
+ ],
+ "labels": [
+ "zero",
+ "bare"
+ ],
+ "scores": [
+ 1.0,
+ 0.9938837346736189
+ ],
+ "passthrough": false,
+ "n": 4
+ },
+ {
+ "label": "dict_instructions",
+ "state": "pay me",
+ "criteria": {
+ "alpha": null,
+ "beta": null
+ },
+ "k": 1,
+ "instructions": {
+ "task": "route",
+ "hint": "Müller & co <urgent>"
+ },
+ "vectors": [
+ [
+ 1.0,
+ 0.0
+ ],
+ [
+ 1.0,
+ 0.0
+ ],
+ [
+ 0.0,
+ 1.0
+ ]
+ ],
+ "labels": [
+ "alpha"
+ ],
+ "scores": [
+ 1.0
+ ],
+ "passthrough": false,
+ "n": 2
+ },
+ {
+ "label": "list_instructions",
+ "state": "pay me",
+ "criteria": {
+ "alpha": null,
+ "beta": null
+ },
+ "k": 1,
+ "instructions": [
+ "step one",
+ "step two",
+ 3
+ ],
+ "vectors": [
+ [
+ 0.0,
+ 1.0
+ ],
+ [
+ 0.0,
+ 1.0
+ ],
+ [
+ 1.0,
+ 0.0
+ ]
+ ],
+ "labels": [
+ "alpha"
+ ],
+ "scores": [
+ 1.0
+ ],
+ "passthrough": false,
+ "n": 2
+ },
+ {
+ "label": "numeric_instructions",
+ "state": "pay me",
+ "criteria": {
+ "alpha": null,
+ "beta": null
+ },
+ "k": 1,
+ "instructions": 42,
+ "vectors": [
+ [
+ 1.0,
+ 0.0
+ ],
+ [
+ 1.0,
+ 0.0
+ ],
+ [
+ 0.0,
+ 1.0
+ ]
+ ],
+ "labels": [
+ "alpha"
+ ],
+ "scores": [
+ 1.0
+ ],
+ "passthrough": false,
+ "n": 2
+ },
+ {
+ "label": "empty_string_instructions_same_as_none",
+ "state": "pay me",
+ "criteria": {
+ "alpha": null,
+ "beta": null
+ },
+ "k": 1,
+ "instructions": "",
+ "vectors": [
+ [
+ 1.0,
+ 0.0
+ ],
+ [
+ 1.0,
+ 0.0
+ ],
+ [
+ 0.0,
+ 1.0
+ ]
+ ],
+ "labels": [
+ "alpha"
+ ],
+ "scores": [
+ 1.0
+ ],
+ "passthrough": false,
+ "n": 2
+ },
+ {
+ "label": "many_options_k5_of_10",
+ "state": "I need help exporting invoices to a spreadsheet",
+ "criteria": {
+ "topic_00": "detailed description number 0 for this support topic",
+ "topic_01": "detailed description number 1 for this support topic",
+ "topic_02": "detailed description number 2 for this support topic",
+ "topic_03": "detailed description number 3 for this support topic",
+ "topic_04": "detailed description number 4 for this support topic",
+ "topic_05": "detailed description number 5 for this support topic",
+ "topic_06": "detailed description number 6 for this support topic",
+ "topic_07": "detailed description number 7 for this support topic",
+ "topic_08": "detailed description number 8 for this support topic",
+ "topic_09": "detailed description number 9 for this support topic"
+ },
+ "k": 5,
+ "instructions": "Pick the single best matching category.",
+ "vectors": [
+ [
+ 1.0,
+ 0.0
+ ],
+ [
+ 0.0,
+ 1.0
+ ],
+ [
+ 1.0,
+ 2.0
+ ],
+ [
+ 2.0,
+ 0.0
+ ],
+ [
+ 0.0,
+ 1.0
+ ],
+ [
+ 1.0,
+ 2.0
+ ],
+ [
+ 2.0,
+ 0.0
+ ],
+ [
+ 0.0,
+ 1.0
+ ],
+ [
+ 1.0,
+ 2.0
+ ],
+ [
+ 2.0,
+ 0.0
+ ],
+ [
+ 0.0,
+ 1.0
+ ]
+ ],
+ "labels": [
+ "topic_02",
+ "topic_05",
+ "topic_08",
+ "topic_01",
+ "topic_04"
+ ],
+ "scores": [
+ 1.0,
+ 1.0,
+ 1.0,
+ 0.4472135954999579,
+ 0.4472135954999579
+ ],
+ "passthrough": false,
+ "n": 10
+ }
+]
diff --git a/laya-rust/tests/laya_router_tests.rs b/laya-rust/tests/laya_router_tests.rs
new file mode 100644
index 00000000..c7658754
--- /dev/null
+++ b/laya-rust/tests/laya_router_tests.rs
@@ -0,0 +1,514 @@
+//! Tier 1: the checkpoint router. Ports the routing/LRU/attach/preload/unload/thread-safety
+//! checks from `tests/test_router.py` (everything except the `repo`-carrying assertions: this
+//! port's `RouteDecision` intentionally drops that field — see `laya_router.rs`'s doc comment)
+//! plus every entry of the routing golden's `route_probe.json`.
+
+mod common;
+
+use common::golden_data;
+use common::routing_golden;
+use laya::answers::{LayaResult, Usage};
+use laya::error::Result;
+use laya::laya_options::LayaCheckpoint;
+use laya::laya_router::{LayaPredictor, LayaRouter, LayaRouterOptions};
+use laya::questions::QuestionSet;
+use serde_json::{Value, json};
+use std::collections::HashMap;
+use std::sync::{Arc, Mutex};
+
+/// A predictor with no ONNX weights: records its own checkpoint name and, optionally, sleeps
+/// before answering (to widen a race window in the concurrency tests). Ports Python's `_Stub`.
+struct FakePredictor {
+ name: &'static str,
+ delay: std::time::Duration,
+}
+
+impl LayaPredictor for FakePredictor {
+ fn predict(&self, _state: Value, questions: &QuestionSet) -> Result<LayaResult> {
+ if !self.delay.is_zero() {
+ std::thread::sleep(self.delay);
+ }
+ Ok(LayaResult::new(
+ self.name,
+ questions.ids().map(String::from).collect(),
+ HashMap::new(),
+ Usage::default(),
+ ))
+ }
+}
+
+fn checkpoint_name(c: LayaCheckpoint) -> &'static str {
+ c.subdir()
+}
+
+/// A router whose checkpoints are [`FakePredictor`]s, so LRU/attach/preload/unload tests need no
+/// ONNX weights. `build_log` records the checkpoint (and build order) each factory call built, so
+/// tests can assert exactly how many times — and for which checkpoints — the factory ran.
+fn fake_router(max_loaded: usize, build_log: Arc<Mutex<Vec<LayaCheckpoint>>>) -> LayaRouter {
+ fake_router_with_delay(max_loaded, build_log, std::time::Duration::ZERO)
+}
+
+fn fake_router_with_delay(
+ max_loaded: usize,
+ build_log: Arc<Mutex<Vec<LayaCheckpoint>>>,
+ delay: std::time::Duration,
+) -> LayaRouter {
+ let options = LayaRouterOptions {
+ max_loaded,
+ ..LayaRouterOptions::default()
+ };
+ LayaRouter::with_factory(options, move |checkpoint, _opts| {
+ build_log.lock().unwrap().push(checkpoint);
+ Ok(Arc::new(FakePredictor {
+ name: checkpoint_name(checkpoint),
+ delay,
+ }) as Arc<dyn LayaPredictor>)
+ })
+ .expect("fake router construction never fails (preload is off)")
+}
+
+// ---------------------------------------------------------------------- name normalisation
+
+#[test]
+fn normalise_name_resolves_aliases() {
+ let cases = [
+ ("en", LayaCheckpoint::English),
+ ("laya", LayaCheckpoint::English),
+ ("multi", LayaCheckpoint::Multilingual),
+ ("ML", LayaCheckpoint::Multilingual),
+ ("typed", LayaCheckpoint::TypedDecisions),
+ ("typed_decisions", LayaCheckpoint::TypedDecisions),
+ ("English", LayaCheckpoint::English),
+ ("laya-typed-decisions", LayaCheckpoint::TypedDecisions),
+ ("laya-multilingual", LayaCheckpoint::Multilingual),
+ ("default", LayaCheckpoint::English),
+ ("decisions", LayaCheckpoint::TypedDecisions),
+ ];
+ for (alias, want) in cases {
+ assert_eq!(
+ LayaRouter::normalise_name(alias).unwrap(),
+ want,
+ "alias/{alias}"
+ );
+ }
+ assert!(
+ LayaRouter::normalise_name("nope").is_err(),
+ "alias/unknown raises"
+ );
+}
+
+// ---------------------------------------------------------------------- workflow signatures
+
+#[test]
+fn workflow_signatures_match_exactly() {
+ let td: [(&str, &[&str]); 4] = [
+ (
+ "agent_trace_observability",
+ &["action", "needs_review", "outcome", "risk", "urgency"],
+ ),
+ (
+ "customer_service",
+ &["action", "category", "churn_risk", "needs_human", "urgency"],
+ ),
+ (
+ "invoice_processing",
+ &[
+ "discrepancy_severity",
+ "disposition",
+ "duplicate",
+ "matches_order",
+ "urgency",
+ ],
+ ),
+ (
+ "security_incidents",
+ &[
+ "credential_compromise",
+ "disposition",
+ "severity",
+ "true_positive",
+ "urgency",
+ ],
+ ),
+ ];
+ for (workflow, ids) in td {
+ let questions = noul_question_set(ids);
+ assert_eq!(
+ LayaRouter::match_typed_decisions_workflow(&questions).as_deref(),
+ Some(workflow),
+ "workflow/{workflow}"
+ );
+ }
+
+ let partial = noul_question_set(&["urgency", "category"]);
+ assert_eq!(
+ LayaRouter::match_typed_decisions_workflow(&partial),
+ None,
+ "workflow/partial overlap"
+ );
+
+ let superset = noul_question_set(&[
+ "action",
+ "category",
+ "churn_risk",
+ "needs_human",
+ "urgency",
+ "extra",
+ ]);
+ assert_eq!(
+ LayaRouter::match_typed_decisions_workflow(&superset),
+ None,
+ "workflow/superset"
+ );
+
+ let empty = QuestionSet::new();
+ assert_eq!(
+ LayaRouter::match_typed_decisions_workflow(&empty),
+ None,
+ "workflow/empty"
+ );
+}
+
+fn noul_question_set(ids: &[&str]) -> QuestionSet {
+ let mut set = QuestionSet::new();
+ for id in ids {
+ set = set
+ .with(
+ *id,
+ laya::questions::Question::noul("x", Value::Null, Value::Null),
+ )
+ .unwrap();
+ }
+ set
+}
+
+// ---------------------------------------------------------------------- LRU bookkeeping
+
+#[test]
+fn lru_cap_one_keeps_newest() {
+ let log = Arc::new(Mutex::new(Vec::new()));
+ let router = fake_router(1, log);
+ router.load("english").unwrap();
+ router.load("multilingual").unwrap();
+ assert_eq!(
+ router.loaded(),
+ vec![LayaCheckpoint::Multilingual],
+ "lru/cap 1 keeps newest"
+ );
+}
+
+#[test]
+fn lru_cap_two_evicts_oldest() {
+ let log = Arc::new(Mutex::new(Vec::new()));
+ let router = fake_router(2, log);
+ router.load("english").unwrap();
+ router.load("multilingual").unwrap();
+ router.load("typed-decisions").unwrap();
+ assert_eq!(
+ router.loaded(),
+ vec![LayaCheckpoint::Multilingual, LayaCheckpoint::TypedDecisions],
+ "lru/cap 2 evicts oldest"
+ );
+}
+
+#[test]
+fn lru_touch_protects_from_eviction() {
+ let log = Arc::new(Mutex::new(Vec::new()));
+ let router = fake_router(2, log);
+ router.load("english").unwrap();
+ router.load("multilingual").unwrap();
+ router.load("english").unwrap(); // touch english
+ router.load("typed-decisions").unwrap();
+ let mut loaded = router.loaded();
+ loaded.sort_by_key(|c| checkpoint_name(*c));
+ assert_eq!(
+ loaded,
+ vec![LayaCheckpoint::English, LayaCheckpoint::TypedDecisions],
+ "lru/touch protects"
+ );
+}
+
+#[test]
+fn unload_one_and_all() {
+ let log = Arc::new(Mutex::new(Vec::new()));
+ let router = fake_router(2, log);
+ router.load("english").unwrap();
+ router.load("multilingual").unwrap();
+ router.unload(Some("english")).unwrap();
+ assert!(
+ !router.loaded().contains(&LayaCheckpoint::English),
+ "lru/unload one"
+ );
+ router.unload(None).unwrap();
+ assert!(router.loaded().is_empty(), "lru/unload all");
+}
+
+// ---------------------------------------------------------------------- attach
+
+#[test]
+fn attach_registers_and_protects_from_eviction() {
+ let log = Arc::new(Mutex::new(Vec::new()));
+ // Python's equivalent test starts from `max_loaded=1` and then explicitly bumps
+ // `max_loaded` to 2 before loading a second checkpoint (`attach` alone only guarantees room
+ // for what is already attached, not for a load that has not happened yet). This router is
+ // simply constructed with room for two from the start, since this port has no public
+ // `max_loaded` setter to mutate after construction.
+ let router = fake_router(2, log);
+ let sentinel: Arc<dyn LayaPredictor> = Arc::new(FakePredictor {
+ name: "already-built",
+ delay: std::time::Duration::ZERO,
+ });
+ let attached = router.attach("english", Arc::clone(&sentinel)).unwrap();
+ assert!(Arc::ptr_eq(&attached, &sentinel));
+ assert!(
+ router.loaded().contains(&LayaCheckpoint::English),
+ "attach/counts as resident"
+ );
+
+ // A later load of a second checkpoint must not evict the attached one.
+ router.load("multilingual").unwrap();
+ let mut loaded = router.loaded();
+ loaded.sort_by_key(|c| checkpoint_name(*c));
+ assert_eq!(
+ loaded,
+ vec![LayaCheckpoint::English, LayaCheckpoint::Multilingual],
+ "attach/survives a later load"
+ );
+
+ // Still the exact same object attach() was given, not a rebuilt one.
+ let engine = router.load("english").unwrap();
+ assert!(
+ Arc::ptr_eq(&engine, &sentinel),
+ "attach/still the same object"
+ );
+}
+
+#[test]
+fn attach_accepts_aliases() {
+ let log = Arc::new(Mutex::new(Vec::new()));
+ let router = fake_router(1, log);
+ let sentinel: Arc<dyn LayaPredictor> = Arc::new(FakePredictor {
+ name: "x",
+ delay: std::time::Duration::ZERO,
+ });
+ assert!(
+ router.attach("en", sentinel).is_ok(),
+ "attach/accepts aliases"
+ );
+}
+
+// ---------------------------------------------------------------------- preload
+
+#[test]
+fn preload_all_three_stay_resident_and_raises_max_loaded() {
+ let log = Arc::new(Mutex::new(Vec::new()));
+ let router = fake_router(1, log);
+ router.preload(None).unwrap();
+ let mut loaded = router.loaded();
+ loaded.sort_by_key(|c| checkpoint_name(*c));
+ assert_eq!(
+ loaded,
+ vec![
+ LayaCheckpoint::English,
+ LayaCheckpoint::Multilingual,
+ LayaCheckpoint::TypedDecisions
+ ],
+ "preload/all three stay resident"
+ );
+}
+
+#[test]
+fn preload_subset_stays_resident_and_touch_does_not_evict() {
+ let log = Arc::new(Mutex::new(Vec::new()));
+ let router = fake_router(1, log.clone());
+ router.preload(Some(&["english", "multilingual"])).unwrap();
+ let mut loaded = router.loaded();
+ loaded.sort_by_key(|c| checkpoint_name(*c));
+ assert_eq!(
+ loaded,
+ vec![LayaCheckpoint::English, LayaCheckpoint::Multilingual],
+ "preload/subset stays resident"
+ );
+ let builds_before = log.lock().unwrap().len();
+ // Routing to an already-resident checkpoint must not evict anything, and must not rebuild.
+ router.load("english").unwrap();
+ assert_eq!(
+ log.lock().unwrap().len(),
+ builds_before,
+ "preload/touch does not evict or rebuild"
+ );
+ let mut loaded = router.loaded();
+ loaded.sort_by_key(|c| checkpoint_name(*c));
+ assert_eq!(
+ loaded,
+ vec![LayaCheckpoint::English, LayaCheckpoint::Multilingual]
+ );
+}
+
+// ---------------------------------------------------------------------- thread safety
+
+#[test]
+fn concurrent_load_of_the_same_checkpoint_builds_once() {
+ let log = Arc::new(Mutex::new(Vec::new()));
+ let router = Arc::new(fake_router_with_delay(
+ 1,
+ log.clone(),
+ std::time::Duration::from_millis(20),
+ ));
+
+ let handles: Vec<_> = (0..8)
+ .map(|_| {
+ let router = Arc::clone(&router);
+ std::thread::spawn(move || router.load("english").unwrap())
+ })
+ .collect();
+ let results: Vec<_> = handles.into_iter().map(|h| h.join().unwrap()).collect();
+
+ // Every caller got the same object.
+ for r in &results[1..] {
+ assert!(
+ Arc::ptr_eq(&results[0], r),
+ "threads/8 concurrent loads share one predictor"
+ );
+ }
+ assert_eq!(
+ log.lock().unwrap().len(),
+ 1,
+ "threads/predictor constructed exactly once"
+ );
+ assert_eq!(
+ router.loaded(),
+ vec![LayaCheckpoint::English],
+ "threads/LRU view stays consistent"
+ );
+}
+
+#[test]
+fn predict_survives_eviction_of_its_own_checkpoint() {
+ // max_loaded=1 with a slow predict on "english": while it is running, another thread loads
+ // "multilingual", which evicts "english" from the router's map. The in-flight predict must
+ // still complete successfully against the engine it already captured a clone of.
+ let log = Arc::new(Mutex::new(Vec::new()));
+ let router = Arc::new(fake_router_with_delay(
+ 1,
+ log,
+ std::time::Duration::from_millis(100),
+ ));
+ router.load("english").unwrap();
+
+ let predicting = {
+ let router = Arc::clone(&router);
+ std::thread::spawn(move || {
+ let mut qs = QuestionSet::new();
+ qs = qs
+ .with(
+ "q",
+ laya::questions::Question::noul("x", Value::Null, Value::Null),
+ )
+ .unwrap();
+ router.predict(json!("hello"), &qs, Some("english"), None, None)
+ })
+ };
+
+ // Give the predict a moment to start, then evict "english" by loading a second checkpoint.
+ std::thread::sleep(std::time::Duration::from_millis(20));
+ router.load("multilingual").unwrap();
+ assert!(
+ !router.loaded().contains(&LayaCheckpoint::English),
+ "english should have been evicted by the load above"
+ );
+
+ let result = predicting
+ .join()
+ .unwrap()
+ .expect("in-flight predict must still succeed");
+ assert_eq!(result.model(), "english");
+}
+
+// ---------------------------------------------------------------------- route_probe.json
+
+fn router_from_kwargs(kwargs: &Value) -> LayaRouter {
+ let mut options = LayaRouterOptions::default();
+ if let Some(default) = kwargs.get("default").and_then(Value::as_str) {
+ options.default = LayaRouter::normalise_name(default).unwrap();
+ }
+ if let Some(auto) = kwargs.get("auto_task_detection").and_then(Value::as_bool) {
+ options.auto_task_detection = auto;
+ }
+ // `models` / `standalone_repos` router_kwargs only affect the `repo` field Python's
+ // RouteDecision carries; this port's RouteDecision has no `repo` field (see laya_router.rs),
+ // so those two knobs have no observable effect on `model`/`reason`/`detection`/`workflow` and
+ // are intentionally not threaded through here.
+ LayaRouter::new(options).expect("router construction never fails (preload is off)")
+}
+
+#[test]
+fn route_probe_matches_every_entry() {
+ let Some(cases) = routing_golden::load_array("route_probe.json") else {
+ eprintln!("skip: routing golden not found (tests/golden/routing)");
+ return;
+ };
+
+ for case in &cases {
+ let label = case["label"].as_str().unwrap();
+ let router = router_from_kwargs(&case["router_kwargs"]);
+ let route_kwargs = &case["route_kwargs"];
+ let state = &route_kwargs["state"];
+ let questions = route_kwargs
+ .get("questions")
+ .filter(|q| !q.is_null())
+ .map(|q| golden_data::build_questions(q).unwrap());
+ let model = route_kwargs.get("model").and_then(Value::as_str);
+ let task = route_kwargs.get("task").and_then(Value::as_str);
+ let lang = route_kwargs.get("lang").and_then(Value::as_str);
+
+ let decision = router
+ .route(state, questions.as_ref(), model, task, lang)
+ .unwrap_or_else(|e| panic!("{label}: route failed: {e}"));
+
+ let expected = &case["decision"];
+ assert_eq!(
+ checkpoint_name(decision.model),
+ expected["model"].as_str().unwrap(),
+ "{label}: model"
+ );
+ assert_eq!(
+ decision.reason,
+ expected["reason"].as_str().unwrap(),
+ "{label}: reason"
+ );
+ assert_eq!(
+ decision.workflow.as_deref(),
+ expected["workflow"].as_str(),
+ "{label}: workflow"
+ );
+
+ match (&decision.detection, expected.get("detection")) {
+ (None, Some(Value::Null)) | (None, None) => {}
+ (Some(got), Some(want)) if !want.is_null() => {
+ assert_eq!(
+ got.script,
+ want["script"].as_str().unwrap(),
+ "{label}: detection.script"
+ );
+ assert_eq!(
+ got.language.as_deref(),
+ want["language"].as_str(),
+ "{label}: detection.language"
+ );
+ assert_eq!(
+ got.is_english,
+ want["is_english"].as_bool().unwrap(),
+ "{label}: detection.is_english"
+ );
+ }
+ (got, want) => {
+ panic!("{label}: detection presence mismatch: got {got:?}, want {want:?}")
+ }
+ }
+ }
+ eprintln!(
+ "route_probe_matches_every_entry: checked {} cases",
+ cases.len()
+ );
+}
diff --git a/laya-rust/tests/router_end_to_end_tests.rs b/laya-rust/tests/router_end_to_end_tests.rs
new file mode 100644
index 00000000..c198a4bb
--- /dev/null
+++ b/laya-rust/tests/router_end_to_end_tests.rs
@@ -0,0 +1,169 @@
+//! Tier 2 (needs ONNX artifacts): the router driving real engines end to end.
+//!
+//! Run the full suite with:
+//! `LAYA_ONNX_ROOT=<parent-of-checkpoints> cargo test --release -- --test-threads=1`
+//! `--test-threads=1` matters here even more than in `predict_parity_tests.rs`: the router itself
+//! can hold up to two ~1.3-1.7 GB engines resident (English + Multilingual, per the plan's
+//! `max_loaded=2` router-predict section), so two of *this* file's tests running at once could
+//! hold four engines between them.
+
+mod common;
+
+use common::{golden_data::CheckpointGoldenData, result_compare, routing_golden, test_artifacts};
+use laya::laya_engine::LayaEngine;
+use laya::laya_options::LayaCheckpoint;
+use laya::laya_router::{LayaPredictor, LayaRouter, LayaRouterOptions};
+use std::sync::Arc;
+
+fn have_english_and_multilingual() -> bool {
+ [LayaCheckpoint::English, LayaCheckpoint::Multilingual]
+ .into_iter()
+ .all(|c| test_artifacts::has_model(c) && test_artifacts::has_tokenizer(c))
+}
+
+/// Builds each engine from the directory `test_artifacts` found (the repo walk-up included), not the
+/// router's default resolution, which only checks `LAYA_ONNX_ROOT` and the cache: otherwise the
+/// skip check above can pass while the router still fails to find the artifacts.
+fn new_router(max_loaded: usize) -> LayaRouter {
+ LayaRouter::with_factory(
+ LayaRouterOptions {
+ max_loaded,
+ ..LayaRouterOptions::default()
+ },
+ |checkpoint, opts| {
+ let mut opts = opts.clone();
+ opts.checkpoint = checkpoint;
+ opts.model_directory = test_artifacts::locate(checkpoint);
+ Ok(Arc::new(LayaEngine::create(opts)?) as Arc<dyn LayaPredictor>)
+ },
+ )
+ .expect("router construction")
+}
+
+#[test]
+fn router_predict_routes_and_matches_the_recorded_checkpoint_result() {
+ if !have_english_and_multilingual() {
+ eprintln!("skip: need both the english and multilingual ONNX artifacts");
+ return;
+ }
+
+ let router = new_router(2);
+
+ // English state -> english checkpoint, compared against english's own recorded golden.
+ let english_golden = CheckpointGoldenData::for_checkpoint(LayaCheckpoint::English.subdir());
+ if !english_golden.available() {
+ eprintln!("skip: no golden data for 'english'");
+ return;
+ }
+ let english_case = english_golden.load("case_sample_app_english.json");
+ let state = english_case["state"].clone();
+ let questions = common::golden_data::build_questions(&english_case["questions"])
+ .expect("build english questions");
+
+ let result = router
+ .predict(state, &questions, None, None, None)
+ .expect("router predict (english state)");
+ let routing = result.routing().expect("routing must be populated");
+ assert_eq!(
+ routing.model,
+ LayaCheckpoint::English,
+ "english state routes to english"
+ );
+ result_compare::assert_result_matches(
+ "router/sample_app_english",
+ &english_case["result"],
+ &result,
+ );
+
+ // Hindi state -> multilingual checkpoint, compared against multilingual's own recorded golden.
+ let multilingual_golden =
+ CheckpointGoldenData::for_checkpoint(LayaCheckpoint::Multilingual.subdir());
+ if !multilingual_golden.available() {
+ eprintln!("skip: no golden data for 'multilingual'");
+ return;
+ }
+ let hindi_case = multilingual_golden.load("case_sample_app_hindi.json");
+ let state = hindi_case["state"].clone();
+ let questions = common::golden_data::build_questions(&hindi_case["questions"])
+ .expect("build hindi questions");
+
+ let result = router
+ .predict(state, &questions, None, None, None)
+ .expect("router predict (hindi state)");
+ let routing = result.routing().expect("routing must be populated");
+ assert_eq!(
+ routing.model,
+ LayaCheckpoint::Multilingual,
+ "hindi state routes to multilingual"
+ );
+ result_compare::assert_result_matches(
+ "router/sample_app_hindi",
+ &hindi_case["result"],
+ &result,
+ );
+
+ // Both `sample_inputs.json` (Phase A's combined-sample fixture) and the recorded cases must
+ // agree that these are the states the sample uses, as a light cross-check that nothing has
+ // drifted between the two.
+ if let Some(sample_inputs) = routing_golden::load("sample_inputs.json") {
+ let _ = sample_inputs; // presence-only check: the file loads and parses.
+ }
+}
+
+#[test]
+fn router_predict_is_correct_under_eight_thread_concurrency_with_max_loaded_one() {
+ if !have_english_and_multilingual() {
+ eprintln!("skip: need both the english and multilingual ONNX artifacts");
+ return;
+ }
+ let english_golden = CheckpointGoldenData::for_checkpoint(LayaCheckpoint::English.subdir());
+ let multilingual_golden =
+ CheckpointGoldenData::for_checkpoint(LayaCheckpoint::Multilingual.subdir());
+ if !english_golden.available() || !multilingual_golden.available() {
+ eprintln!("skip: golden data missing for english or multilingual");
+ return;
+ }
+
+ let english_case = english_golden.load("case_sample_app_english.json");
+ let hindi_case = multilingual_golden.load("case_sample_app_hindi.json");
+
+ let router = Arc::new(new_router(1)); // max_loaded 1 forces an eviction on every alternation
+
+ // 4 threads on the English state, 4 on the Hindi state, all launched together so `max_loaded
+ // = 1` forces genuine evictions under load rather than settling onto one checkpoint.
+ let handles: Vec<_> = (0..8)
+ .map(|i| {
+ let router = Arc::clone(&router);
+ let (case, expected_model) = if i % 2 == 0 {
+ (english_case.clone(), LayaCheckpoint::English)
+ } else {
+ (hindi_case.clone(), LayaCheckpoint::Multilingual)
+ };
+ std::thread::spawn(move || {
+ let state = case["state"].clone();
+ let questions = common::golden_data::build_questions(&case["questions"])
+ .expect("build questions");
+ let result = router
+ .predict(state, &questions, None, None, None)
+ .expect("concurrent router predict");
+ let routing = result.routing().expect("routing must be populated");
+ assert_eq!(
+ routing.model, expected_model,
+ "thread {i}: routed to the expected checkpoint"
+ );
+ (case, result)
+ })
+ })
+ .collect();
+
+ for (i, handle) in handles.into_iter().enumerate() {
+ let (case, result) = handle.join().unwrap_or_else(|e| {
+ panic!("thread {i} panicked (a disposed-engine use would panic or corrupt here): {e:?}")
+ });
+ result_compare::assert_result_matches(
+ &format!("router/concurrent[{i}]"),
+ &case["result"],
+ &result,
+ );
+ }
+}
diff --git a/laya-rust/tools/dump_routing_golden.py b/laya-rust/tools/dump_routing_golden.py
new file mode 100644
index 00000000..b5fb126d
--- /dev/null
+++ b/laya-rust/tools/dump_routing_golden.py
@@ -0,0 +1,404 @@
+"""Record golden vectors for the router/language/email/shortlist port (Phase A).
+
+Pure Python, no model weights and no ONNX session: everything this script touches --
+`laya.lang`, `laya.router.Router.route` (never `.load`/`.predict`), `laya.email` and
+`laya.shortlist.shortlist_choice` -- is dependency-free Python plus NumPy. It reads its input
+corpus from `laya-rust/tools/routing_cases.py` and writes one JSON file per probe to
+`laya-rust/tests/golden/routing/`:
+
+ lang_probe.json lang.analyse / detect_script / script_profile /
+ latin_profile / state_text, for every LANG_STATES entry.
+ route_probe.json Router.route() decisions (model, reason, detection,
+ workflow -- `repo` is intentionally omitted: the ported
+ RouteDecision has no Repo field) for every ROUTE_CASES entry.
+ email_probe.json clean_email_body and email_state for every EMAIL_CLEAN_CASES /
+ EMAIL_STATE_CASES entry.
+ shortlist_probe.json shortlist_choice(labels, scores, passthrough) for every
+ SHORTLIST_CASES entry, with the exact positional embedding
+ matrix each case used (so a port needs no embedder to check
+ its ranking/cosine/tie-break logic against this file). NaN/Inf
+ values in a matrix are written as the strings "nan"/"inf"/
+ "-inf" (see meta.json) because JSON has no literal for them.
+ hashing_embedder_probe.json Exact laya-rust/tools/hashing_embedder.embed() vectors for a fixed probe
+ list of strings, including non-ASCII and astral input.
+ sample_inputs.json The combined routing sample's fixed inputs (detection states,
+ routing-decision examples, support questions/emails, a raw
+ email, and the banking shortlist demo), plus the
+ Python-computed outputs a sample or test can check itself
+ against with no model loaded.
+ meta.json Generator/version info and per-file entry counts.
+
+Usage:
+ laya-rust/tools/dump_routing_golden.py # writes routing/ next to the checkpoint goldens
+ laya-rust/tools/dump_routing_golden.py --force # overwrite files whose content differs
+ laya-rust/tools/dump_routing_golden.py --out DIR # write elsewhere (rare; testing only)
+
+Refuses to overwrite an existing golden file whose content would change unless `--force` is
+given -- a content diff, not a mtime check, so re-running with no corpus changes is always a
+silent no-op.
+"""
+import argparse
+import json
+import math
+import os
+import sys
+
+TOOLS = os.path.dirname(os.path.abspath(__file__))
+REPO = os.path.dirname(os.path.dirname(TOOLS))
+sys.path.insert(0, REPO)
+sys.path.insert(0, TOOLS)
+
+from routing_cases import ( # noqa: E402
+ EMAIL_CLEAN_CASES,
+ EMAIL_STATE_CASES,
+ LANG_STATES,
+ ROUTE_CASES,
+ SAMPLE_BANKING_CRITERIA,
+ SAMPLE_BANKING_INSTRUCTIONS,
+ SAMPLE_BANKING_STATE,
+ SAMPLE_DETECTION_STATES,
+ SAMPLE_RAW_EMAIL_BODY,
+ SAMPLE_RAW_EMAIL_SENDER,
+ SAMPLE_RAW_EMAIL_SUBJECT,
+ SAMPLE_ROUTING_DECISIONS,
+ SAMPLE_SUPPORT_EMAIL_EN,
+ SAMPLE_SUPPORT_EMAIL_HI,
+ SAMPLE_SUPPORT_QUESTIONS,
+ SHORTLIST_CASES,
+)
+
+import hashing_embedder # noqa: E402
+
+from laya.email import clean_email_body, email_state # noqa: E402
+from laya.lang import analyse, detect_script, latin_profile, script_profile, state_text # noqa: E402
+from laya.router import Router # noqa: E402
+from laya.shortlist import DEFAULT_SHORTLIST_K, shortlist_choice # noqa: E402
+
+OUT_DIR_DEFAULT = os.path.join(REPO, "laya-rust", "tests", "golden", "routing")
+
+# ~20+ strings for hashing_embedder_probe.json: ASCII, accents, full case mapping, CJK,
+# Cyrillic, Arabic, Devanagari, astral (emoji and a script with no astral-BMP equivalent),
+# whitespace-only, and a long repeated run to exercise bucket collisions.
+HASHING_EMBEDDER_PROBE_STRINGS = [
+ "",
+ "a",
+ "hello",
+ "Hello World",
+ "HELLO WORLD",
+ "café",
+ "naïve résumé Müller",
+ "İstanbul",
+ "ıstanbul",
+ "我这个月被重复扣款了两次",
+ "こんにちは世界",
+ "안녕하세요",
+ "тест русского текста",
+ "مرحبا بالعالم",
+ "नमस्ते दुनिया",
+ "\U0001F600\U0001F680",
+ "refund \U0001F600 today",
+ "\U00010400\U00010401",
+ " ",
+ "a" * 50,
+ "The quick brown fox jumps over the lazy dog",
+ "refund!! invoice #4411 @ $99.99 (urgent)",
+]
+
+
+def _encode_floats(value):
+ """Recursively replace non-finite floats with the sentinel strings "nan"/"inf"/"-inf",
+ since JSON has no literal for them (Python's json module would otherwise emit the
+ non-standard bareword `NaN`/`Infinity`, which most other languages' JSON parsers reject).
+ Everything else passes through unchanged."""
+ if isinstance(value, float):
+ if math.isnan(value):
+ return "nan"
+ if math.isinf(value):
+ return "inf" if value > 0 else "-inf"
+ return value
+ if isinstance(value, list):
+ return [_encode_floats(v) for v in value]
+ if isinstance(value, dict):
+ return {k: _encode_floats(v) for k, v in value.items()}
+ return value
+
+
+def dump(obj, path: str, force: bool) -> str:
+ """Write `obj` as pretty JSON to `path`. Refuses to overwrite a file whose content would
+ change unless `force` is set. Returns "written", "unchanged" or "refused"."""
+ new_text = json.dumps(obj, ensure_ascii=False, indent=2, sort_keys=False, allow_nan=False) + "\n"
+ if os.path.exists(path):
+ with open(path, encoding="utf-8", newline="\n") as f:
+ old_text = f.read()
+ if old_text == new_text:
+ return "unchanged"
+ if not force:
+ raise SystemExit(
+ "refusing to overwrite %s: its content would change and --force was not given.\n"
+ "Re-run with --force if this change is intended." % path
+ )
+ os.makedirs(os.path.dirname(path), exist_ok=True)
+ with open(path, "w", encoding="utf-8", newline="\n") as f:
+ f.write(new_text)
+ print(" wrote %-28s %8.1f KB" % (os.path.basename(path), os.path.getsize(path) / 1024.0))
+ return "written"
+
+
+# ============================================================================ lang_probe
+def build_lang_probe():
+ out = []
+ for label, state in LANG_STATES:
+ text = state_text(state)
+ out.append({
+ "label": label,
+ "state": state,
+ "state_text": text,
+ "detect_script": detect_script(text),
+ "script_profile": script_profile(text),
+ "latin_profile": latin_profile(text),
+ "analyse": analyse(state),
+ })
+ return out
+
+
+# ============================================================================ route_probe
+def _decision_payload(decision):
+ """RouteDecision -> {model, reason, detection, workflow}. `repo` is intentionally
+ dropped: the ported RouteDecision type has no Repo field (see the porting plan)."""
+ return {
+ "model": decision["model"],
+ "reason": decision["reason"],
+ "detection": decision["detection"],
+ "workflow": decision["workflow"],
+ }
+
+
+def build_route_probe():
+ out = []
+ for label, router_kwargs, route_kwargs in ROUTE_CASES:
+ router = Router(**router_kwargs)
+ decision = router.route(**route_kwargs)
+ out.append({
+ "label": label,
+ "router_kwargs": router_kwargs,
+ "route_kwargs": route_kwargs,
+ "decision": _decision_payload(decision),
+ })
+ return out
+
+
+# ============================================================================ email_probe
+def build_email_probe():
+ clean = []
+ for label, body, max_chars in EMAIL_CLEAN_CASES:
+ clean.append({
+ "label": label,
+ "body": body,
+ "max_chars": max_chars,
+ "result": clean_email_body(body, max_chars),
+ })
+ state = []
+ for label, subject, body, sender, clean_flag, extra in EMAIL_STATE_CASES:
+ state.append({
+ "label": label,
+ "subject": subject,
+ "body": body,
+ "sender": sender,
+ "clean": clean_flag,
+ "extra": extra,
+ "result": email_state(subject, body, sender, clean_flag, **extra),
+ })
+ return {"clean": clean, "state": state}
+
+
+# ============================================================================ shortlist_probe
+class _Boom:
+ """embed_fn stand-in for a passthrough case: raising proves shortlist_choice never calls
+ it when k >= n, exactly like laya-rust/tools/routing_cases.py's SHORTLIST_CASES expects."""
+
+ def __call__(self, texts):
+ raise AssertionError("embed_fn must not be called when k >= n (passthrough)")
+
+
+def _matrix_embed_fn(vectors):
+ import numpy as np
+
+ def embed_fn(texts):
+ if len(texts) != len(vectors):
+ raise AssertionError(
+ "case's embed_fn called with %d texts, expected %d (vectors length)"
+ % (len(texts), len(vectors))
+ )
+ return np.array(vectors, dtype=np.float64)
+
+ return embed_fn
+
+
+def build_shortlist_probe():
+ out = []
+ for case in SHORTLIST_CASES:
+ vectors = case["vectors"]
+ embed_fn = _Boom() if vectors is None else _matrix_embed_fn(vectors)
+ labels, scores, passthrough, n = _rank_with_metadata(
+ case["state"], case["criteria"], embed_fn, case["k"], case["instructions"]
+ )
+ out.append({
+ "label": case["label"],
+ "state": case["state"],
+ "criteria": case["criteria"],
+ "k": case["k"],
+ "instructions": case["instructions"],
+ "vectors": _encode_floats(vectors) if vectors is not None else None,
+ "labels": labels,
+ "scores": scores,
+ "passthrough": passthrough,
+ "n": n,
+ })
+ return out
+
+
+def _rank_with_metadata(state, criteria, embed_fn, k, instructions):
+ """shortlist_choice only returns labels; re-derive passthrough/n/scores the same way
+ predict_shortlist does, by calling the private _rank directly (still the real code path,
+ just the one that also reports the metadata the golden needs)."""
+ from laya.shortlist import _rank
+
+ labels, scores, passthrough, n = _rank(state, criteria, embed_fn, k, instructions)
+ # Cross-check against the public function so the golden can never silently diverge from
+ # what a caller of shortlist_choice actually gets.
+ public_labels = shortlist_choice(state, criteria, embed_fn, k=k, instructions=instructions)
+ if list(labels) != list(public_labels):
+ raise SystemExit(
+ "internal inconsistency: _rank labels %r != shortlist_choice labels %r for case"
+ % (labels, public_labels)
+ )
+ return list(labels), scores, passthrough, n
+
+
+# ============================================================================ hashing_embedder_probe
+def build_hashing_embedder_probe():
+ vecs = hashing_embedder.embed(HASHING_EMBEDDER_PROBE_STRINGS)
+ return [
+ {"text": s, "vector": [float(x) for x in vecs[i]]}
+ for i, s in enumerate(HASHING_EMBEDDER_PROBE_STRINGS)
+ ]
+
+
+# ============================================================================ sample_inputs
+def build_sample_inputs():
+ detection = []
+ for label, state in SAMPLE_DETECTION_STATES:
+ detection.append({"label": label, "state": state, "analyse": analyse(state)})
+
+ routing = []
+ for label, router_kwargs, route_kwargs in SAMPLE_ROUTING_DECISIONS:
+ router = Router(**router_kwargs)
+ decision = router.route(**route_kwargs)
+ routing.append({
+ "label": label,
+ "router_kwargs": router_kwargs,
+ "route_kwargs": route_kwargs,
+ "decision": _decision_payload(decision),
+ })
+
+ support_email = {
+ "questions": SAMPLE_SUPPORT_QUESTIONS,
+ "english_state": SAMPLE_SUPPORT_EMAIL_EN,
+ "hindi_state": SAMPLE_SUPPORT_EMAIL_HI,
+ "english_routing": _decision_payload(Router().route(SAMPLE_SUPPORT_EMAIL_EN, SAMPLE_SUPPORT_QUESTIONS)),
+ "hindi_routing": _decision_payload(Router().route(SAMPLE_SUPPORT_EMAIL_HI, SAMPLE_SUPPORT_QUESTIONS)),
+ }
+
+ cleaned_body = clean_email_body(SAMPLE_RAW_EMAIL_BODY)
+ raw_email = {
+ "subject": SAMPLE_RAW_EMAIL_SUBJECT,
+ "sender": SAMPLE_RAW_EMAIL_SENDER,
+ "body": SAMPLE_RAW_EMAIL_BODY,
+ "cleaned_body": cleaned_body,
+ "state": email_state(SAMPLE_RAW_EMAIL_SUBJECT, SAMPLE_RAW_EMAIL_BODY, SAMPLE_RAW_EMAIL_SENDER),
+ "routing": _decision_payload(Router().route(
+ email_state(SAMPLE_RAW_EMAIL_SUBJECT, SAMPLE_RAW_EMAIL_BODY, SAMPLE_RAW_EMAIL_SENDER)
+ )),
+ }
+
+ banking_k = 20
+ banking_labels, banking_scores, banking_passthrough, banking_n = _rank_with_metadata(
+ SAMPLE_BANKING_STATE, SAMPLE_BANKING_CRITERIA, hashing_embedder.embed,
+ banking_k, SAMPLE_BANKING_INSTRUCTIONS,
+ )
+ banking = {
+ "state": SAMPLE_BANKING_STATE,
+ "instructions": SAMPLE_BANKING_INSTRUCTIONS,
+ "criteria": SAMPLE_BANKING_CRITERIA,
+ "k": banking_k,
+ "n": banking_n,
+ "passthrough": banking_passthrough,
+ "labels": banking_labels,
+ "scores": banking_scores,
+ }
+
+ return {
+ "detection_states": detection,
+ "routing_decisions": routing,
+ "support_email": support_email,
+ "raw_email": raw_email,
+ "banking_shortlist": banking,
+ }
+
+
+def main() -> int:
+ ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
+ ap.add_argument("--out", default=OUT_DIR_DEFAULT, help="output directory (default: %s)" % OUT_DIR_DEFAULT)
+ ap.add_argument("--force", action="store_true", help="overwrite files whose content would change")
+ args = ap.parse_args()
+
+ out_dir = os.path.abspath(args.out)
+
+ lang_probe = build_lang_probe()
+ route_probe = build_route_probe()
+ email_probe = build_email_probe()
+ shortlist_probe = build_shortlist_probe()
+ hashing_probe = build_hashing_embedder_probe()
+ sample_inputs = build_sample_inputs()
+
+ meta = {
+ "generator": "tools/dump_routing_golden.py",
+ "laya_version": __import__("laya").__version__,
+ "default_shortlist_k": DEFAULT_SHORTLIST_K,
+ "notes": {
+ "route_probe": "RouteDecision is serialized as {model, reason, detection, workflow}; "
+ "'repo' is intentionally omitted (not part of the ported RouteDecision).",
+ "shortlist_probe": "Non-finite floats (NaN, +Inf, -Inf) inside a case's 'vectors' matrix "
+ "are encoded as the JSON strings \"nan\", \"inf\", \"-inf\" because JSON "
+ "has no native literal for them; a case with vectors=null is a k>=n "
+ "passthrough case where embed_fn must never be called.",
+ },
+ "counts": {
+ "lang_probe": len(lang_probe),
+ "route_probe": len(route_probe),
+ "email_probe_clean": len(email_probe["clean"]),
+ "email_probe_state": len(email_probe["state"]),
+ "shortlist_probe": len(shortlist_probe),
+ "hashing_embedder_probe": len(hashing_probe),
+ },
+ }
+
+ results = {}
+ results["lang_probe.json"] = dump(lang_probe, os.path.join(out_dir, "lang_probe.json"), args.force)
+ results["route_probe.json"] = dump(route_probe, os.path.join(out_dir, "route_probe.json"), args.force)
+ results["email_probe.json"] = dump(email_probe, os.path.join(out_dir, "email_probe.json"), args.force)
+ results["shortlist_probe.json"] = dump(shortlist_probe, os.path.join(out_dir, "shortlist_probe.json"), args.force)
+ results["hashing_embedder_probe.json"] = dump(
+ hashing_probe, os.path.join(out_dir, "hashing_embedder_probe.json"), args.force
+ )
+ results["sample_inputs.json"] = dump(sample_inputs, os.path.join(out_dir, "sample_inputs.json"), args.force)
+ results["meta.json"] = dump(meta, os.path.join(out_dir, "meta.json"), args.force)
+
+ print("\nrouting golden vectors -> %s" % out_dir)
+ for name, status in results.items():
+ print(" %-28s %s" % (name, status))
+ return 0
+
+
+if __name__ == "__main__":
+ raise SystemExit(main())
diff --git a/laya-rust/tools/routing_cases.py b/laya-rust/tools/routing_cases.py
new file mode 100644
index 00000000..a9a3ed34
--- /dev/null
+++ b/laya-rust/tools/routing_cases.py
@@ -0,0 +1,771 @@
+"""Probe corpus for the routing/language/email/shortlist golden vectors.
+
+Consumed only by `laya-rust/tools/dump_routing_golden.py`, which runs every entry through the real
+`laya.lang` / `laya.router` / `laya.email` / `laya.shortlist` code paths and records
+whatever Python actually produces. Nothing in this module is itself a golden value -- it is
+the *input* side only, chosen to hit the parity pitfalls documented in the porting plan
+(Unicode code-point iteration, `str.lower()`/`isalpha()` semantics, `max()` tie order,
+`round()` half-even, quote/signature/disclaimer regexes, cosine edge cases).
+
+Four sections, one per probe file:
+ LANG_STATES -> lang_probe.json (lang.analyse / detect_script / script_profile /
+ latin_profile / state_text)
+ ROUTE_CASES -> route_probe.json (router.Router.route)
+ EMAIL_CASES -> email_probe.json (email.clean_email_body / email_state)
+ SHORTLIST_CASES -> shortlist_probe.json (shortlist.shortlist_choice)
+
+Plus the single source of truth for the combined routing sample's own fixed inputs
+(SAMPLE_* below), so the Rust sample can be diffed against a Python-recorded golden.
+"""
+
+# ============================================================================ lang_probe
+#
+# label -> state, fed to `lang.analyse(state)` and `lang.state_text(state)` directly, and to
+# `detect_script`/`script_profile`/`latin_profile` via `state_text(state)` (dump script's job,
+# not this module's -- these three take plain text, not a full state).
+
+LANG_STATES = [
+ # ---- straight from tests/test_router.py SCRIPTS (script detection) ----
+ ("script/english", "The customer was charged twice and wants a refund."),
+ ("script/armenian", "Հայերեն"),
+ ("script/armenian_uppercase", "ՀԱՅԵՐԵՆ"),
+ ("script/armenian_punctuation_only", "։֊"),
+ ("script/french", "Le client a été facturé deux fois et demande un remboursement."),
+ ("script/hindi", "ग्राहक से दो बार शुल्क लिया गया और वह धनवापसी चाहता है।"),
+ ("script/japanese", "お客様は二重に請求されたため返金を希望しています。"),
+ ("script/chinese", "客户被重复扣款要求退款"),
+ ("script/korean", "고객이 두 번 청구되어 환불을 원합니다"),
+ ("script/arabic", "تم خصم المبلغ مرتين من العميل ويريد استرداد الأموال"),
+ ("script/tamil", "வாடிக்கையாளரிடம் இருமுறை கட்டணம் வசூலிக்கப்பட்டது"),
+ ("script/russian", "С клиента дважды сняли деньги и он хочет возврат"),
+ ("script/thai", "ลูกค้าถูกเรียกเก็บเงินสองครั้งและต้องการเงินคืน"),
+ ("script/greek", "Ο πελάτης χρεώθηκε δύο φορές και θέλει επιστροφή χρημάτων"),
+ ("script/hebrew", "הלקוח חויב פעמיים ורוצה החזר כספי"),
+ ("script/empty", ""),
+ ("script/digits_only", "12345 6789"),
+
+ # ---- straight from tests/test_router.py is_english checks ----
+ ("is_english/plain_english", "Please refund the duplicate charge on invoice 4411 today."),
+ ("is_english/armenian", "Հայերեն"),
+ ("is_english/english_short", "refund me"),
+ ("is_english/hindi", "ग्राहक से दो बार शुल्क लिया गया"),
+ ("is_english/japanese", "お客様は二重に請求されました"),
+ ("is_english/russian", "С клиента дважды сняли деньги"),
+ ("is_english/french_long", "Le client a été facturé deux fois et il demande un remboursement pour la "
+ "facture qui a été payée le mois dernier avec la carte de crédit"),
+ ("is_english/german_long", "Der Kunde wurde zweimal belastet und möchte eine Rückerstattung für die "
+ "Rechnung die nicht korrekt ist und auch nicht bezahlt wurde"),
+ ("is_english/romanian", "Gătește-mi o rețetă de sarmale de post pentru mâine."),
+ ("is_english/romanian_invoice", "Am fost taxat de două ori pentru factura din luna martie și vreau banii"),
+ ("is_english/polish", "Klient został obciążony dwukrotnie i chce zwrot pieniędzy za fakturę"),
+ ("is_english/czech", "Zákazníkovi byla částka účtována dvakrát a žádá o vrácení peněz"),
+ ("is_english/turkish", "Müşteriden iki kez ücret alındı ve para iadesi istiyor lütfen yardım"),
+ ("is_english/vietnamese", "Khách hàng đã bị thu phí hai lần và muốn được hoàn tiền ngay"),
+ ("is_english/english_with_loanwords", "We visited a cafe in Zurich and the naive assumption about the "
+ "invoice was wrong, so please refund the duplicate charge"),
+ ("is_english/undecided_turkish", "Müşteriden iki kez ücret alındı ve para iadesi istiyor"),
+ ("is_english/english_not_undecided", "Please refund the duplicate charge on the invoice"),
+ ("is_english/romanian_diacritics", "Gătește-mi o rețetă de sarmale"),
+ ("is_english/no_diacritics", "Please refund the duplicate charge today"),
+ ("is_english/zero_tie_romanian", "Cât e ora acum la Tokyo"),
+ ("is_english/known_gap_romanian_no_diacritics", "Care este ora in Tokyo?"),
+
+ # ---- latin language guess ----
+ ("latin_lang/english", "The customer was charged twice and wants a refund for this invoice"),
+ ("latin_lang/french", "Le client a ete facture deux fois et il demande un remboursement pour la facture"),
+ ("latin_lang/german", "Der Kunde wurde zweimal belastet und moechte eine Rueckerstattung fuer die Rechnung"),
+ ("latin_lang/spanish", "El cliente fue cobrado dos veces y quiere que le devuelvan el dinero por la factura"),
+ ("latin_lang/too_short", "refund"),
+ ("latin_lang/long_english_stays_en",
+ "Please refund the duplicate charge on invoice 4411 today because we have been waiting "
+ "for three days and nobody has replied to us"),
+
+ # ---- state flattening ----
+ ("state_text/dict", {"body": "charged twice", "n": 3}),
+ ("state_text/nested", {"a": {"b": ["deep"]}}),
+ ("state_text/list", ["x", {"y": "z"}]),
+ ("state_text/none", None),
+ ("state_text/keys_ignored", {"subject": "नमस्ते", "body": "ग्राहक से दो बार शुल्क लिया गया"}),
+
+ # ---- workflow/profile edge cases from tests ----
+ ("profile/armenian_pure", "Հայերեն"),
+ ("profile/armenian_mixed_with_latin", "Հայերեն abc"),
+
+ # ---- plain-ASCII Romance (#172): accent-stripped states must still route non-English ----
+ ("plain_ascii/es_1", "El pedido llego roto y nadie responde cuando escribo al soporte"),
+ ("plain_ascii/es_2", "Quiero cancelar mi plan y pedir un reembolso"),
+ ("plain_ascii/es_3", "La factura tiene un error en el importe total"),
+ ("plain_ascii/es_4", "Necesito que me devuelvan el dinero de la compra duplicada"),
+ ("plain_ascii/it_1", "Il cliente e stato addebitato due volte e vuole un rimborso"),
+ ("plain_ascii/it_2", "Voglio cancellare il mio abbonamento e chiedere un rimborso"),
+ ("plain_ascii/it_3", "La fattura contiene un errore nell importo totale"),
+ ("plain_ascii/pt_1", "O cliente foi cobrado duas vezes e quer o dinheiro de volta"),
+ ("plain_ascii/fr_1", "Le client a ete facture deux fois et demande un remboursement"),
+ ("plain_ascii/fr_2", "Je ne peux pas acceder a mon compte et j ai besoin d aide"),
+ ("accented/es", "La facturación tiene un error y necesito una corrección urgente"),
+ ("accented/it", "La fattura è sbagliata, devo avere un rimborso per il pagamento"),
+ ("accented/fr", "La commande est arrivée cassée et personne ne répond au support"),
+ ("romance_control/1", "The customer was charged twice and wants a refund for this invoice"),
+ ("romance_control/2", "Please cancel my subscription and refund the duplicate charge today"),
+ ("romance_control/3", "The report by Smith et al. shows the de facto standard, e.g. the LA office and Rio"),
+ ("romance_control/4", "Our MI5 and UN contacts discussed the DOS attack in LA last month"),
+ ("romance_control/5", "No refund was issued, so I am writing to you again about invoice 4411"),
+ ("romance_control/6", "no refund no reply"),
+ ("romance_control/7", "The son of the director filed a complaint about the duplicate invoice"),
+ ("shared_words/names_nothing", "Cât e ora acum la Tokyo"),
+ ("shared_words/distinctive_names_it", "La fattura contiene un errore nell importo totale"),
+ ("shared_words/shared_hits_still_count_es", "La factura tiene un error en el importe total"),
+
+ # ---- all 25 non-Latin scripts recognised by _SCRIPT_RANGES, plus a couple more Latin ----
+ ("all_scripts/greek", "Γεια σου κόσμε, πώς είσαι σήμερα;"),
+ ("all_scripts/cyrillic", "Привет, как ты себя чувствуешь сегодня?"),
+ ("all_scripts/armenian", "Բարեւ ձեզ, ինչպե՞ս եք այսօր"),
+ ("all_scripts/hebrew", "שלום, מה שלומך היום"),
+ ("all_scripts/arabic", "مرحبا كيف حالك اليوم"),
+ ("all_scripts/devanagari", "नमस्ते आप कैसे हैं आज"),
+ ("all_scripts/bengali", "আমি ভালো আছি, আপনি কেমন আছেন"),
+ ("all_scripts/gurmukhi", "ਤੁਸੀਂ ਕਿਵੇਂ ਹੋ, ਮੈਂ ਠੀਕ ਹਾਂ"),
+ ("all_scripts/gujarati", "તમે કેમ છો, હું સારો છું"),
+ ("all_scripts/oriya", "ତୁମେ କେମିତି ଅଛ, ମୁଁ ଭଲ ଅଛି"),
+ ("all_scripts/tamil", "நீங்கள் எப்படி இருக்கிறீர்கள்"),
+ ("all_scripts/telugu", "మీరు ఎలా ఉన్నారు, నేను బాగున్నాను"),
+ ("all_scripts/kannada", "ನೀವು ಹೇಗಿದ್ದೀರಿ, ನಾನು ಚೆನ್ನಾಗಿದ್ದೇನೆ"),
+ ("all_scripts/malayalam", "നിങ്ങൾക്ക് സുഖമാണോ, എനിക്ക് സുഖമാണ്"),
+ ("all_scripts/sinhala", "ඔබට කොහොමද, මට හොඳින්"),
+ ("all_scripts/thai", "คุณเป็นอย่างไรบ้างวันนี้"),
+ ("all_scripts/lao", "ສະບາຍດີ ທ່ານເປັນແນວໃດແດ່"),
+ ("all_scripts/tibetan", "བཀྲ་ཤིས་བདེ་ལེགས་ཁྱེད་རང་ག་འདྲ་ཡོད"),
+ ("all_scripts/myanmar", "မင်္ဂလာပါ သင်နေကောင်းလား"),
+ ("all_scripts/georgian", "გამარჯობა როგორ ხარ დღეს"),
+ ("all_scripts/ethiopic", "ሰላም እንዴት ነህ ዛሬ"),
+ ("all_scripts/khmer", "សួស្តី តើអ្នកសុខសប្បាយទេ"),
+ ("all_scripts/hangul", "안녕하세요 오늘 기분이 어때요"),
+ ("all_scripts/kana", "こんにちは、今日は元気ですか"),
+ ("all_scripts/han", "你好,你今天过得怎么样"),
+ ("all_scripts/latin", "Hello, how are you feeling today?"),
+
+ # ---- mixed scripts / ties (max() tie order: first-inserted key wins; latin is always
+ # inserted last, so it never wins a tie against a script that appeared) ----
+ ("tie/greek_then_cyrillic_1v1", "ΑА"), # expect "greek" (first seen)
+ ("tie/cyrillic_then_greek_1v1", "АΑ"), # expect "cyrillic" (first seen)
+ ("tie/han_then_latin_1v1", "中 a"), # expect "han" (latin inserted last)
+ ("tie/latin_then_han_1v1", "a 中"), # expect "han" still (order in text ignored for latin)
+ ("tie/three_way_greek_cyrillic_armenian", "ΑАԱ"), # 1-1-1, expect "greek" (first)
+ ("mixed/latin_han_arabic", "invoice 中文 فاتورة refund"),
+ ("mixed/devanagari_latin_majority_devanagari", "ग्राहक से दो बार शुल्क लिया गया refund"),
+ ("mixed/latin_majority_devanagari_minor", "Please refund the customer today ग्राहक"),
+
+ # ---- latin_profile ties: _STOP dict order is en, fr, de, es, pt, it, nl, ro; a tied
+ # score picks the first-iterated language among those with real (non-shared) evidence ----
+ ("latin_tie/fr_vs_de", "être cette nicht sich"), # expect "fr" (fr before de)
+ ("latin_tie/de_vs_es", "nicht sich el los"), # expect "de" (de before es)
+ ("latin_tie/es_vs_pt", "está una dos tres o não está"),
+ ("latin_tie/pt_vs_it", "não está o della sono"),
+
+ # ---- astral characters (outside the BMP): emoji is not alphabetic and is dropped from
+ # every count; a letter outside every _SCRIPT_RANGES bucket (Deseret, astral) is
+ # alphabetic but also silently dropped from counts (neither latin nor any script) ----
+ ("astral/emoji_only", "\U0001F600\U0001F680\U00002764\U0000FE0F"),
+ ("astral/emoji_with_latin", "refund please \U0001F600 today"),
+ ("astral/deseret_letter_alone", "\U00010400\U00010401\U00010402"), # Deseret, no matching script bucket
+ ("astral/deseret_mixed_with_latin", "refund \U00010400 today"),
+ ("astral/mathematical_bold_letters", "\U0001D400\U0001D401\U0001D402"), # Mathematical Bold A B C
+
+ # ---- combining marks: Python str.isalpha() is Unicode category L* only, so a combining
+ # mark (Mn) like the Devanagari virama is NOT counted as alphabetic; a port whose
+ # "is letter" check includes marks (e.g. Rust char::is_alphabetic) will disagree ----
+ ("combining/devanagari_with_virama", "ग्राहक"), # contains U+094D VIRAMA (Mn)
+ ("combining/latin_with_combining_acute", "café résumé"), # decomposed é
+
+ # ---- Turkish dotted capital I: str.lower() maps İ -> 2 code points ("i" + combining dot) ----
+ ("turkish_i/capital_i_with_dot", "İstanbul"),
+ ("turkish_i/mixed_case", "İYİ GÜNLER İstanbul'da"),
+ ("turkish_i/lowercase_dotless", "ıstanbul"),
+
+ # ---- digits / numerics: `\w` (used by `_WORD`) = isalnum() + '_', which is letters and
+ # numeric No/Nl categories, not marks; digits (Nd) are explicitly excluded by `\d` ----
+ ("numerics/ascii_digits_only", "12345 67890"),
+ ("numerics/roman_numeral_nl", "chapter Ⅷ begins"), # U+2167 ROMAN NUMERAL EIGHT (Nl)
+ ("numerics/fullwidth_digits", "123 refund"), # fullwidth 123 (Nd)
+ ("numerics/mixed_digits_and_words", "invoice 4411 was paid twice on 2024-01-05"),
+
+ # ---- nested dict/list states deeper than the _iter_text depth-6 cutoff ----
+ ("nesting/depth_6_kept", {"a": {"b": {"c": {"d": {"e": {"f": "visible at depth 6"}}}}}}),
+ ("nesting/depth_7_dropped", {"a": {"b": {"c": {"d": {"e": {"f": {"g": "invisible past depth 6"}}}}}}}),
+ ("nesting/list_depth_7_dropped", [[[[[[["invisible list depth 7"]]]]]]]),
+ ("nesting/mixed_depth_boundary",
+ {"a": {"b": {"c": {"d": {"e": ["kept depth 6", {"f": "dropped depth 7"}]}}}}}),
+
+ # ---- >4000-char inputs: state_text's default max_chars=4000 slice is per code point ----
+ ("long/ascii_over_4000",
+ "refund please. " * 400), # 16 * 400 = 6400 chars
+ ("long/multibyte_over_4000",
+ "退款请求 " * 1000), # 5 chars * 1000 = 5000 code points, multi-byte in UTF-8
+ ("long/mixed_script_over_4000_boundary_near_switch",
+ ("a" * 3990) + "客户被重复扣款要求退款" * 5), # ascii run crosses the 4000 cut into Han
+
+ # ---- English control long enough to stay decisively English despite length ----
+ ("long/english_over_1000",
+ "Please refund the duplicate charge on invoice 4411 today. " * 20),
+]
+
+
+# ============================================================================ route_probe
+#
+# Each entry: (label, router_kwargs, route_kwargs). `router_kwargs` builds the `Router`
+# (state that survives across the whole case: `default`, `auto_task_detection`,
+# `standalone_repos`, `models` override); `route_kwargs` is passed straight to `.route(...)`
+# (state, questions, model=, task=, lang=). Router() with no models loads nothing -- `.route`
+# never touches the network or the filesystem.
+
+Q_GENERIC = {"dept": {"type": "choice", "instructions": "Which team?",
+ "criteria": {"billing": None, "tech": None}}}
+
+_TD_IDS = {
+ "agent_trace_observability": ["action", "needs_review", "outcome", "risk", "urgency"],
+ "customer_service": ["action", "category", "churn_risk", "needs_human", "urgency"],
+ "invoice_processing": ["discrepancy_severity", "disposition", "duplicate", "matches_order", "urgency"],
+ "security_incidents": ["credential_compromise", "disposition", "severity", "true_positive", "urgency"],
+}
+
+
+def _td_questions(ids):
+ return {i: {"type": "noul", "instructions": "x"} for i in ids}
+
+
+ROUTE_CASES = [
+ # ---- default router, precedence ladder ----
+ ("english_text", {}, {"state": {"body": "I was charged twice, please refund."}, "questions": Q_GENERIC}),
+ ("armenian_text", {}, {"state": {"body": "Հայերեն"}, "questions": Q_GENERIC}),
+ ("armenian_explicit_override", {}, {"state": {"body": "Հայերեն"}, "questions": Q_GENERIC, "model": "english"}),
+ ("hindi_text", {}, {"state": {"body": "मुझसे दो बार शुल्क लिया गया"}, "questions": Q_GENERIC}),
+ ("japanese_text", {}, {"state": {"body": "二重に請求されました"}, "questions": Q_GENERIC}),
+ ("korean_text", {}, {"state": {"body": "두 번 청구되었습니다"}, "questions": Q_GENERIC}),
+ ("arabic_text", {}, {"state": {"body": "تم خصم المبلغ مرتين"}, "questions": Q_GENERIC}),
+ ("german_text", {}, {"state": {"body": "Der Kunde wurde zweimal belastet und moechte eine Rueckerstattung "
+ "fuer die Rechnung die nicht korrekt ist"}, "questions": Q_GENERIC}),
+ ("explicit_model", {}, {"state": {"body": "anything"}, "questions": Q_GENERIC, "model": "multilingual"}),
+ ("explicit_model_overrides_script", {}, {"state": {"body": "मुझसे दो बार"}, "questions": Q_GENERIC, "model": "english"}),
+ ("explicit_task", {}, {"state": {"body": "x"}, "questions": Q_GENERIC, "task": "typed_decisions"}),
+ ("explicit_lang_en", {}, {"state": {"body": "मुझसे दो बार"}, "questions": Q_GENERIC, "lang": "en"}),
+ ("explicit_lang_de", {}, {"state": {"body": "hello there"}, "questions": Q_GENERIC, "lang": "de"}),
+ ("explicit_lang_eng_alias", {}, {"state": {"body": "hello"}, "questions": Q_GENERIC, "lang": "eng"}),
+ ("explicit_lang_english_word", {}, {"state": {"body": "hello"}, "questions": Q_GENERIC, "lang": "English"}),
+ ("explicit_lang_en_us", {}, {"state": {"body": "hello"}, "questions": Q_GENERIC, "lang": "en-US"}),
+ ("explicit_lang_fr", {}, {"state": {"body": "hello"}, "questions": Q_GENERIC, "lang": "fr"}),
+ ("td_workflow_auto_off", {}, {"state": {"body": "I was charged twice"}, "questions": _td_questions(_TD_IDS["customer_service"])}),
+ ("empty_state", {}, {"state": {}, "questions": Q_GENERIC}),
+ ("none_state", {}, {"state": None, "questions": Q_GENERIC}),
+ ("no_questions_none", {}, {"state": {"body": "मुझसे दो बार"}, "questions": None}),
+
+ # ---- auto_task_detection opt-in, all four workflow signatures ----
+ ("td_agent_trace_observability_auto_on", {"auto_task_detection": True},
+ {"state": {"body": "trace"}, "questions": _td_questions(_TD_IDS["agent_trace_observability"])}),
+ ("td_customer_service_auto_on", {"auto_task_detection": True},
+ {"state": {"body": "I was charged twice"}, "questions": _td_questions(_TD_IDS["customer_service"])}),
+ ("td_invoice_processing_auto_on", {"auto_task_detection": True},
+ {"state": {"body": "invoice mismatch"}, "questions": _td_questions(_TD_IDS["invoice_processing"])}),
+ ("td_security_incidents_auto_on", {"auto_task_detection": True},
+ {"state": {"body": "possible breach"}, "questions": _td_questions(_TD_IDS["security_incidents"])}),
+ ("td_auto_on_but_generic_questions", {"auto_task_detection": True},
+ {"state": {"body": "I was charged twice"}, "questions": Q_GENERIC}),
+ ("td_explicit_beats_workflow", {"auto_task_detection": True},
+ {"state": {"body": "x"}, "questions": _td_questions(_TD_IDS["customer_service"]), "model": "multilingual"}),
+ # near-miss id sets: must NOT match any workflow even with auto_task_detection on
+ ("td_near_miss_partial_overlap", {"auto_task_detection": True},
+ {"state": {"body": "x"}, "questions": {"urgency": {"type": "noul", "instructions": "x"},
+ "category": {"type": "noul", "instructions": "x"}}}),
+ ("td_near_miss_superset", {"auto_task_detection": True},
+ {"state": {"body": "x"}, "questions": _td_questions(_TD_IDS["customer_service"] + ["extra_field"])}),
+ ("td_near_miss_subset", {"auto_task_detection": True},
+ {"state": {"body": "x"}, "questions": _td_questions(_TD_IDS["invoice_processing"][:-1])}),
+ ("td_near_miss_one_id_swapped", {"auto_task_detection": True},
+ {"state": {"body": "x"},
+ "questions": _td_questions(_TD_IDS["security_incidents"][:-1] + ["not_a_real_field"])}),
+
+ # ---- unknown-Latin routing (#35): scripts with no stopword list, on the diacritic signal ----
+ ("unknown_latin_romanian", {}, {"state": "Gătește-mi o rețetă de sarmale de post pentru mâine."}),
+ ("unknown_latin_romanian_agent_request", {}, {"state": "Exportă APK-ul pentru Android și pune-l pe Drive ca să-l instalez."}),
+ ("unknown_latin_polish", {}, {"state": "Klient został obciążony dwukrotnie i chce zwrot pieniędzy za fakturę"}),
+ ("unknown_latin_turkish", {}, {"state": "Müşteriden iki kez ücret alındı ve para iadesi istiyor lütfen yardım"}),
+ ("unknown_latin_undecided_reason", {}, {"state": "Müşteriden iki kez ücret alındı ve para iadesi istiyor"}),
+ ("english_still_english", {}, {"state": "Please refund the duplicate charge on invoice 4411 today."}),
+ ("short_english_still_english", {}, {"state": "refund me"}),
+
+ # ---- plain-ASCII Romance / accented forms (#172) ----
+ ("plain_ascii_es_1", {}, {"state": "El pedido llego roto y nadie responde cuando escribo al soporte"}),
+ ("plain_ascii_it_1", {}, {"state": "Il cliente e stato addebitato due volte e vuole un rimborso"}),
+ ("plain_ascii_pt_1", {}, {"state": "O cliente foi cobrado duas vezes e quer o dinheiro de volta"}),
+ ("plain_ascii_fr_1", {}, {"state": "Le client a ete facture deux fois et demande un remboursement"}),
+ ("accented_es", {}, {"state": "La facturación tiene un error y necesito una corrección urgente"}),
+ ("accented_it", {}, {"state": "La fattura è sbagliata, devo avere un rimborso per il pagamento"}),
+ ("accented_fr", {}, {"state": "La commande est arrivée cassée et personne ne répond au support"}),
+ ("romance_control_1", {}, {"state": "The customer was charged twice and wants a refund for this invoice"}),
+ ("romance_control_2", {}, {"state": "Our MI5 and UN contacts discussed the DOS attack in LA last month"}),
+ ("shared_words_still_multilingual", {}, {"state": "Cât e ora acum la Tokyo"}),
+
+ # ---- custom default ----
+ ("custom_default_multilingual", {"default": "multilingual"}, {"state": "12345"}),
+ ("custom_default_typed_decisions", {"default": "typed-decisions"}, {"state": ""}),
+ ("custom_default_english_script_unknown", {"default": "english"}, {"state": "12345 6789"}),
+
+ # ---- bundle vs standalone repos, local override ----
+ ("bundle_default_hindi", {}, {"state": {"m": "मुझसे दो बार"}, "questions": Q_GENERIC}),
+ ("standalone_opt_in_hindi", {"standalone_repos": True}, {"state": {"m": "मुझसे दो बार"}, "questions": Q_GENERIC}),
+ ("standalone_english_unchanged", {"standalone_repos": True}, {"state": {"m": "I was charged twice"}, "questions": Q_GENERIC}),
+ ("local_path_override", {"models": {"english": "/tmp/en", "multilingual": "/tmp/ml"}},
+ {"state": {"m": "मुझसे दो बार"}, "questions": Q_GENERIC}),
+
+ # ---- every alias, as an explicit `model=` (normalise_name feeds straight into RouteDecision) ----
+ ("alias_en", {}, {"state": "x", "model": "en"}),
+ ("alias_laya", {}, {"state": "x", "model": "laya"}),
+ ("alias_default", {}, {"state": "x", "model": "default"}),
+ ("alias_multi", {}, {"state": "x", "model": "multi"}),
+ ("alias_ml", {}, {"state": "x", "model": "ml"}),
+ ("alias_laya_multilingual", {}, {"state": "x", "model": "laya-multilingual"}),
+ ("alias_typed", {}, {"state": "x", "model": "typed"}),
+ ("alias_typed_decisions", {}, {"state": "x", "model": "typed_decisions"}),
+ ("alias_laya_typed_decisions", {}, {"state": "x", "model": "laya-typed-decisions"}),
+ ("alias_decisions", {}, {"state": "x", "model": "decisions"}),
+ ("alias_uppercase_ML", {}, {"state": "x", "model": "ML"}),
+ ("alias_mixed_case_English", {}, {"state": "x", "model": "English"}),
+ ("alias_via_task_typed_decisions_with_dash", {}, {"state": "x", "task": "typed-decisions"}),
+ ("alias_via_task_typed_decisions_with_underscore", {}, {"state": "x", "task": "typed_decisions"}),
+
+ # ---- decision payload shape (repo string, reason text, detection dict) ----
+ ("payload_shape_hindi", {}, {"state": {"body": "मुझसे दो बार शुल्क लिया गया"}, "questions": Q_GENERIC}),
+ ("payload_shape_armenian_non_latin_percent", {}, {"state": "Հայերեն abc"}),
+ ("payload_shape_unidentified_latin_percent", {}, {"state": "Müşteriden iki kez ücret alındı ve para iadesi istiyor"}),
+]
+
+
+# ============================================================================ email_probe
+
+_DISCLAIMER_TEXT = "This email is confidential and intended solely for the named addressee."
+
+EMAIL_CLEAN_CASES = [
+ # ---- straight from tests/test_email.py ----
+ ("inline_footer_no_blank_line", "My account is locked.\n%s\nPlease unlock it." % _DISCLAIMER_TEXT, 3000),
+ ("inline_footer_no_terminal_punctuation", "My account is locked\n%s\nPlease unlock it." % _DISCLAIMER_TEXT, 3000),
+ ("inline_footer_body_never_emptied", "My account is locked. %s" % _DISCLAIMER_TEXT, 3000),
+ ("standalone_footer_paragraph_dropped", "My account is locked.\n\n%s" % _DISCLAIMER_TEXT, 3000),
+ ("wrapped_standalone_footer_dropped",
+ "My account is locked.\n\nThis email and any files transmitted with it are\n"
+ "confidential and intended solely for the named addressee.", 3000),
+ ("received_in_error_footer_dropped",
+ "Please reopen ticket 4411.\n\nIf you have received this message in error, delete it.", 3000),
+ ("quoted_history_removed", "Thanks for the update.\nOn Mon, Sep 20, Bob wrote:\n> original text", 3000),
+ ("signature_block_removed", "Hi team,\nCan you confirm the refund?\nRegards,\nAlice", 3000),
+ ("empty_body", "", 3000),
+
+ # ---- quote header variants (each _QUOTE_HEADERS pattern) ----
+ ("quote_header_on_wrote_long_name",
+ "Please see below.\nOn Mon, Jan 5, 2026 at 3:00 PM, Bob Smith <[email protected]> wrote:\n> old text here", 3000),
+ ("quote_header_original_message",
+ "Approved.\n----- Original Message -----\nFrom: [email protected]\nSubject: Re: Invoice", 3000),
+ ("quote_header_forwarded_message",
+ "See below for context.\n---------- Forwarded Message ----------\nFrom: [email protected]", 3000),
+ ("quote_header_underscores",
+ "My reply is above.\n________________________________\nFrom: [email protected]", 3000),
+ ("quote_header_from_line",
+ "Confirmed, thanks.\nFrom: [email protected]\nSubject: Ticket 4411 update\nDate: today", 3000),
+ ("quote_header_only_no_body_before_kept",
+ "On Mon, Jan 5, 2026, Bob wrote:\n> quoted only, no lines before it to break on", 3000),
+ ("quote_marker_gt_lines_stripped",
+ "My reply.\n> quoted line one\n> quoted line two\nMore of my reply.", 3000),
+
+ # ---- signature marker variants, at various positions ----
+ ("signature_dash_dash", "Hi,\nPlease confirm.\n--\nAlice Smith\nSupport Team", 3000),
+ ("signature_best_regards", "Hi,\nCan you confirm the refund?\nBest regards,\nAlice", 3000),
+ ("signature_kind_regards", "Hi,\nCan you confirm the refund?\nKind regards,\nBob", 3000),
+ ("signature_warm_regards", "Hi,\nCan you confirm the refund?\nWarm regards,\nCarol", 3000),
+ ("signature_many_thanks", "Hi,\nCan you confirm the refund?\nMany thanks,\nDave", 3000),
+ ("signature_thanks", "Hi,\nCan you confirm the refund?\nThanks,\nEve", 3000),
+ ("signature_thank_you", "Hi,\nCan you confirm the refund?\nThank you,\nFrank", 3000),
+ ("signature_cheers", "Hi,\nCan you confirm the refund?\nCheers,\nGrace", 3000),
+ ("signature_sincerely", "Hi,\nCan you confirm the refund?\nSincerely,\nHeidi", 3000),
+ ("signature_sent_from_iphone", "On the go, will follow up later.\nSent from my iPhone", 3000),
+ ("signature_sent_from_android", "Quick reply here.\nSent from my Android", 3000),
+ ("signature_too_early_kept_as_body",
+ # A "Thanks," line inside the first 60% of a short body is not treated as a signature
+ # cutoff by the search window (range starts at max(1, int(len(lines)*0.6), len(lines)-8)));
+ # this pins that boundary behaviour rather than assuming it.
+ "Thanks,\nfor looking into this.\nMy account is still locked after the reset.\n"
+ "Please escalate this to the on-call engineer today.\nRegards,\nIvan", 3000),
+
+ # ---- disclaimers inside and across paragraphs ----
+ ("disclaimer_mid_paragraph_request_survives",
+ "My account is locked.\n%s\nPlease unlock it." % _DISCLAIMER_TEXT, 3000),
+ ("disclaimer_two_sentences_only_boilerplate_dropped",
+ "My account is locked. %s Please help urgently." % _DISCLAIMER_TEXT, 3000),
+ ("disclaimer_across_two_paragraphs_only_its_own_paragraph_affected",
+ "Please reopen ticket 4411 as soon as you can.\n\n"
+ "This email and any attachments are confidential and intended solely for the addressee.\n\n"
+ "If you have received this message in error, please delete it.\n\n"
+ "Let me know once it is done.", 3000),
+ ("disclaimer_received_in_error_variant_2",
+ "Approve the refund please.\n\nIf you received this e-mail in error, notify the sender immediately.", 3000),
+
+ # ---- CRLF and literal backslash-n ----
+ ("crlf_line_endings", "My account is locked.\r\nPlease unlock it.\r\nRegards,\r\nAlice", 3000),
+ ("crlf_with_quote_header", "Thanks for the update.\r\nOn Mon, Sep 20, Bob wrote:\r\n> original text", 3000),
+ ("literal_backslash_n_becomes_newline",
+ "My account is locked.\\nPlease unlock it.\\nRegards,\\nAlice", 3000),
+ ("literal_backslash_n_with_disclaimer",
+ "My account is locked.\\n%s\\nPlease unlock it." % _DISCLAIMER_TEXT, 3000),
+ ("mixed_real_and_literal_newlines",
+ "Line one.\nLine two.\\nLine three (literal escape).\nLine four.", 3000),
+
+ # ---- Unicode whitespace: `[ \\t]+` in clean_email_body only collapses ASCII space/tab, not
+ # general Unicode whitespace; `.strip()` on paragraphs strips all Unicode whitespace ----
+ ("unicode_nbsp_not_collapsed",
+ "My account is locked.\nPlease unlock it.", 3000),
+ ("unicode_em_space_and_ideographic_space",
+ "My account is locked.\nPlease unlock it.", 3000),
+ ("unicode_whitespace_leading_trailing_stripped",
+ "  My account is locked. \n Please unlock it.  ", 3000),
+ ("unicode_zero_width_space_kept",
+ "My​account is locked.\nPlease​unlock it.", 3000),
+
+ # ---- max_chars variants (default 3000, and non-default) ----
+ ("max_chars_tiny_cuts_mid_word", "Please refund the duplicate charge on invoice 4411 today.", 10),
+ ("max_chars_default_no_truncation_needed", "Short message that fits easily.", 3000),
+ ("max_chars_large_no_op", "Please refund the duplicate charge on invoice 4411 today.", 100000),
+ ("max_chars_exact_boundary",
+ "0123456789" * 5, 50), # exactly 50 chars input, max_chars=50 -> no truncation
+ ("max_chars_one_over_boundary",
+ "0123456789" * 5 + "X", 50), # 51 chars, max_chars=50 -> drop last char
+
+ # ---- long, multi-paragraph, everything at once (kitchen sink) ----
+ ("kitchen_sink_quote_signature_disclaimer",
+ "Hi team,\n\n"
+ "My account was charged twice for the Pro plan this month and the refund has not "
+ "arrived yet. Could you please look into this today?\n\n"
+ "Best regards,\n"
+ "Priya Sharma\n"
+ "Sent from my iPhone\n\n"
+ "This email and any files transmitted with it are confidential and intended solely "
+ "for the use of the individual or entity to whom they are addressed. If you have "
+ "received this email in error please notify the sender.\n\n"
+ "On Tue, Sep 22, 2026 at 9:14 AM, Support <[email protected]> wrote:\n"
+ "> Hi Priya, thanks for reaching out, we are looking into it.\n"
+ "> - Support Team\n", 3000),
+]
+
+EMAIL_STATE_CASES = [
+ # (label, subject, body, sender, clean, extra)
+ ("basic_clean_true", "Locked out", "My account is locked. %s" % _DISCLAIMER_TEXT, None, True, {}),
+ ("basic_clean_false_keeps_raw_body", "Locked out", "My account is locked.\n%s" % _DISCLAIMER_TEXT, None, False, {}),
+ ("with_sender", "Refund request", "Please refund invoice 4411.", "[email protected]", True, {}),
+ ("subject_whitespace_stripped", " Refund request ", "Please help.", None, True, {}),
+ ("extra_fields_kept_when_not_none", "Ticket", "Body text.", "[email protected]", True, {"priority": "high", "ticket_id": 4411}),
+ ("extra_none_values_dropped", "Ticket", "Body text.", None, True, {"nullable": None, "kept": "value"}),
+ ("empty_subject_and_body", "", "", None, True, {}),
+ ("hindi_subject_and_body", "सहायता चाहिए", "मुझसे दो बार शुल्क लिया गया, कृपया रिफंड करें।", None, True, {}),
+]
+
+
+# ============================================================================ shortlist_probe
+#
+# Each entry supplies a positional embedding matrix directly (query vector first, then one
+# vector per criteria item in order) rather than a text-keyed table, so the golden dump needs
+# no embedder at all: `embed_fn` for these cases is just "return this matrix, in this order",
+# and the dump script asserts it is called with exactly `len(vectors)` texts when `vectors`
+# is not None. A case with `vectors=None` asserts k >= n passthrough: embed_fn must not be
+# called at all (checked by the dump script raising if it is).
+
+SHORTLIST_CASES = [
+ # ---- deterministic top-k, ties keep the earlier label ----
+ {
+ "label": "tie_cosine_keeps_earlier_label",
+ "state": "pay me", "criteria": {"alpha": None, "beta": "", "gamma": "mid", "delta": "same"},
+ "k": 2, "instructions": None,
+ "vectors": [[1.0, 0.0], [1.0, 0.0], [0.0, 1.0], [0.6, 0.8], [1.0, 0.0]],
+ },
+ {
+ "label": "k1_is_earliest_max",
+ "state": "pay me", "criteria": {"alpha": None, "beta": "", "gamma": "mid", "delta": "same"},
+ "k": 1, "instructions": None,
+ "vectors": [[1.0, 0.0], [1.0, 0.0], [0.0, 1.0], [0.6, 0.8], [1.0, 0.0]],
+ },
+ {
+ "label": "k3_appends_next_cosine",
+ "state": "pay me", "criteria": {"alpha": None, "beta": "", "gamma": "mid", "delta": "same"},
+ "k": 3, "instructions": None,
+ "vectors": [[1.0, 0.0], [1.0, 0.0], [0.0, 1.0], [0.6, 0.8], [1.0, 0.0]],
+ },
+ # ---- zero query vector: every cosine is 0, earliest labels win ----
+ {
+ "label": "zero_query_keeps_original_order",
+ "state": "pay me", "criteria": {"alpha": None, "beta": "", "gamma": "mid", "delta": "same"},
+ "k": 2, "instructions": None,
+ "vectors": [[0.0, 0.0], [1.0, 0.0], [0.0, 1.0], [0.6, 0.8], [3.0, 4.0]],
+ },
+ # ---- zero option vector loses to a real match ----
+ {
+ "label": "zero_option_vector_loses",
+ "state": "pay me", "criteria": {"alpha": None, "beta": None, "gamma": None},
+ "k": 2, "instructions": None,
+ "vectors": [[1.0, 0.0], [0.0, 0.0], [1.0, 0.0], [0.0, 1.0]],
+ },
+ # ---- NaN / inf: non-finite embeddings become 0 (np.nan_to_num), so they never outrank a
+ # finite match, but inf-vs-inf or nan-vs-nan still tie at 0 and fall back to order ----
+ {
+ "label": "nan_vector_sorts_behind_finite_match",
+ "state": "pay me", "criteria": {"alpha": None, "beta": None},
+ "k": 1, "instructions": None,
+ "vectors": [[1.0, 0.0], [float("nan"), float("nan")], [1.0, 0.0]],
+ },
+ {
+ "label": "positive_inf_and_negative_inf_both_zeroed",
+ "state": "pay me", "criteria": {"alpha": None, "beta": None, "gamma": None},
+ "k": 2, "instructions": None,
+ "vectors": [[1.0, 0.0], [float("inf"), float("inf")], [float("-inf"), 0.0], [1.0, 0.0]],
+ },
+ {
+ # nan_to_num zeroes only the NaN component, not the whole vector: [nan, 1.0] becomes
+ # [0.0, 1.0], a real unit vector, so this is NOT the same as a zero query -- it still
+ # picks a winner (beta) rather than falling back to insertion order.
+ "label": "nan_component_in_query_only_zeroes_that_component",
+ "state": "pay me", "criteria": {"alpha": None, "beta": None},
+ "k": 1, "instructions": None,
+ "vectors": [[float("nan"), 1.0], [1.0, 0.0], [0.0, 1.0]],
+ },
+ {
+ # Every component of the query is NaN, so nan_to_num zeroes the whole vector: this IS
+ # a true zero query, and falls back to insertion order like zero_query_keeps_original_order.
+ "label": "fully_nan_query_falls_back_to_order",
+ "state": "pay me", "criteria": {"alpha": None, "beta": None},
+ "k": 1, "instructions": None,
+ "vectors": [[float("nan"), float("nan")], [1.0, 0.0], [0.0, 1.0]],
+ },
+ # ---- k >= n passthrough: embed_fn must not be called ----
+ {
+ "label": "k_equals_n_passthrough",
+ "state": "pay me", "criteria": {"alpha": None, "beta": "", "gamma": "mid", "delta": "same"},
+ "k": 4, "instructions": None, "vectors": None,
+ },
+ {
+ "label": "k_greater_than_n_passthrough",
+ "state": "pay me", "criteria": {"alpha": None, "beta": "", "gamma": "mid", "delta": "same"},
+ "k": 20, "instructions": None, "vectors": None,
+ },
+ {
+ "label": "k_equals_n_single_option_passthrough",
+ "state": "pay me", "criteria": {"solo": "the only choice"},
+ "k": 1, "instructions": None, "vectors": None,
+ },
+ # ---- list criteria (positional labels, value=None each) ----
+ {
+ "label": "list_criteria_topk",
+ "state": "pay me", "criteria": ["alpha", "beta", "gamma"],
+ "k": 2, "instructions": "Classify",
+ "vectors": [[0.0, 1.0], [1.0, 0.0], [0.0, 1.0], [0.0, 0.2]],
+ },
+ {
+ "label": "list_criteria_passthrough",
+ "state": "hello", "criteria": ["alpha", "beta", "gamma"],
+ "k": 5, "instructions": "Which?", "vectors": None,
+ },
+ # ---- dict state serialisation through query text (informational only: matrix is positional) ----
+ {
+ "label": "dict_state_query",
+ "state": {"text": "hi"}, "criteria": ["alpha", "beta"],
+ "k": 1, "instructions": "Classify",
+ "vectors": [[1.0, 0.0], [1.0, 0.0], [0.0, 1.0]],
+ },
+ # ---- 0 and False are real criterion values (not "no description") ----
+ {
+ "label": "zero_and_false_criteria_values",
+ "state": "pay me", "criteria": {"zero": 0, "no": False, "bare": None, "named": {"desc": "payments"}},
+ "k": 4, "instructions": None, "vectors": None, # k == n -> passthrough, but exercises render text
+ },
+ {
+ "label": "zero_and_false_criteria_ranked",
+ "state": "pay me", "criteria": {"zero": 0, "no": False, "bare": None, "named": {"desc": "payments"}},
+ "k": 2, "instructions": None,
+ "vectors": [[1.0, 0.0], [1.0, 0.0], [0.0, 1.0], [0.9, 0.1], [0.0, 1.0]],
+ },
+ # ---- non-string instructions: dict / list / number, run through json.dumps(ensure_ascii=False) ----
+ {
+ "label": "dict_instructions",
+ "state": "pay me", "criteria": {"alpha": None, "beta": None},
+ "k": 1, "instructions": {"task": "route", "hint": "Müller & co <urgent>"},
+ "vectors": [[1.0, 0.0], [1.0, 0.0], [0.0, 1.0]],
+ },
+ {
+ "label": "list_instructions",
+ "state": "pay me", "criteria": {"alpha": None, "beta": None},
+ "k": 1, "instructions": ["step one", "step two", 3],
+ "vectors": [[0.0, 1.0], [0.0, 1.0], [1.0, 0.0]],
+ },
+ {
+ "label": "numeric_instructions",
+ "state": "pay me", "criteria": {"alpha": None, "beta": None},
+ "k": 1, "instructions": 42,
+ "vectors": [[1.0, 0.0], [1.0, 0.0], [0.0, 1.0]],
+ },
+ {
+ "label": "empty_string_instructions_same_as_none",
+ "state": "pay me", "criteria": {"alpha": None, "beta": None},
+ "k": 1, "instructions": "",
+ "vectors": [[1.0, 0.0], [1.0, 0.0], [0.0, 1.0]],
+ },
+ # ---- many options, mid-range k (mirrors the checkpoint end-to-end case's shape at n=40) ----
+ {
+ "label": "many_options_k5_of_10",
+ "state": "I need help exporting invoices to a spreadsheet",
+ "criteria": {"topic_%02d" % i: "detailed description number %d for this support topic" % i for i in range(10)},
+ "k": 5, "instructions": "Pick the single best matching category.",
+ "vectors": [[1.0, 0.0]] + [[float(i % 3), float((i + 1) % 3)] for i in range(10)],
+ },
+]
+
+
+# ============================================================================ sample inputs
+#
+# Single source of truth for the combined routing sample (Laya.Routing / laya-routing), so
+# both languages' samples read the same fixed content and their stdout can be diffed.
+
+# Section 1: ~8 detection states (English, accent-stripped Spanish, French, Romanian, Hindi,
+# Arabic, Japanese, a dict state).
+SAMPLE_DETECTION_STATES = [
+ ("english", "The customer was charged twice this month and wants a refund."),
+ ("spanish_accent_stripped", "El cliente fue cobrado dos veces y quiere que le devuelvan el dinero"),
+ ("french", "Le client a été facturé deux fois et demande un remboursement."),
+ ("romanian", "Am fost taxat de două ori pentru factura din luna martie și vreau banii"),
+ ("hindi", "ग्राहक से दो बार शुल्क लिया गया और वह धनवापसी चाहता है।"),
+ ("arabic", "تم خصم المبلغ مرتين من العميل ويريد استرداد الأموال"),
+ ("japanese", "お客様は二重に請求されたため返金を希望しています。"),
+ ("dict_state", {"subject": "Double charge", "body": "मुझसे दो बार शुल्क लिया गया"}),
+]
+
+# Section 2: routing-decision examples, including explicit model/lang and opt-in workflow
+# detection, with no engine loaded (Route only).
+SAMPLE_ROUTING_DECISIONS = [
+ ("auto_english", {}, {"state": "Please refund the duplicate charge on invoice 4411 today."}),
+ ("auto_hindi", {}, {"state": "मुझसे दो बार शुल्क लिया गया, कृपया रिफंड करें।"}),
+ ("auto_unknown_latin_romanian", {}, {"state": "Gătește-mi o rețetă de sarmale de post pentru mâine."}),
+ ("explicit_model_multilingual", {}, {"state": "anything at all", "model": "multilingual"}),
+ ("explicit_lang_de", {}, {"state": "hello there", "lang": "de"}),
+ ("workflow_opt_in_customer_service", {"auto_task_detection": True},
+ {"state": {"body": "I was charged twice"},
+ "questions": _td_questions(_TD_IDS["customer_service"])}),
+ ("workflow_opt_in_off_by_default", {},
+ {"state": {"body": "I was charged twice"},
+ "questions": _td_questions(_TD_IDS["customer_service"])}),
+]
+
+# Section 3: an English and a Hindi support email, same support questions, run through
+# Router.predict (engine loaded, MaxLoaded=2).
+SAMPLE_SUPPORT_QUESTIONS = {
+ "department": {
+ "type": "choice",
+ "instructions": "Which department should handle this request?",
+ "criteria": {
+ "billing": "invoices, payments, refunds",
+ "technical": "bugs, outages, system errors",
+ "sales": "pricing, new contracts",
+ "other": "everything else",
+ },
+ },
+ "urgency": {
+ "type": "score",
+ "instructions": "How urgent is this request?",
+ "criteria": ["not urgent", "soon", "critical deadline or blocking issue"],
+ },
+ "churn_risk": {
+ "type": "noul",
+ "instructions": "Does the user threaten to cancel or leave?",
+ },
+ "refund_requested": {
+ "type": "noul",
+ "instructions": "Does the user explicitly request a refund?",
+ },
+}
+
+SAMPLE_SUPPORT_EMAIL_EN = {
+ "from": "[email protected]",
+ "subject": "Duplicate charge on invoice #4411",
+ "body": "Hi, we were billed twice for March. Please refund the duplicate today "
+ "or we will cancel our plan.",
+}
+
+# Written as escapes deliberately (see laya-rust/tools/sample_cases.py's note on the same pitfall): a
+# hand-retyped Devanagari string is an easy place to silently swap one similar-looking
+# character for another and produce a "parity bug" that is really a transcription error.
+SAMPLE_SUPPORT_EMAIL_HI = {
+ "from": "[email protected]",
+ "subject": "चालान में दोहरा "
+ "शुल्क",
+ "body": "मार्च का बिल "
+ "दो बार लिया गया, "
+ "कृपया रिफंड करें।",
+}
+
+# Section 4: a raw email with quoted history, signature and disclaimer, run through
+# LayaEmail.CleanBody -> LayaEmail.State -> LayaPresets.Email() through the router.
+SAMPLE_RAW_EMAIL_SUBJECT = "Re: Refund for invoice 4411"
+SAMPLE_RAW_EMAIL_SENDER = "[email protected]"
+SAMPLE_RAW_EMAIL_BODY = (
+ "Hi team,\n\n"
+ "I was charged twice for the Pro plan this month and the second charge still has "
+ "not been refunded. This is the third time I have written about it -- please "
+ "resolve this today or I will need to cancel the account.\n\n"
+ "Best regards,\n"
+ "Priya Sharma\n"
+ "Sent from my iPhone\n\n"
+ "This email and any files transmitted with it are confidential and intended "
+ "solely for the use of the individual or entity to whom they are addressed. If "
+ "you have received this email in error please notify the sender.\n\n"
+ "On Tue, Sep 22, 2026 at 9:14 AM, Support <[email protected]> wrote:\n"
+ "> Hi Priya, thanks for reaching out, we are looking into it.\n"
+ "> - Support Team\n"
+)
+
+# Section 5: a 40-intent banking choice question through LayaShortlist.Predict with the
+# hashing embedder (demo only).
+SAMPLE_BANKING_STATE = "I tried to withdraw cash from the ATM but it kept the card and gave me no money."
+SAMPLE_BANKING_INSTRUCTIONS = "What is the customer's banking intent?"
+SAMPLE_BANKING_CRITERIA = {
+ "activate_my_card": "turn on a newly received card",
+ "age_limit": "minimum or maximum age to use the service",
+ "apple_pay_or_google_pay": "adding the card to a mobile wallet",
+ "atm_support": "which ATMs the card works at",
+ "automatic_top_up": "automatically topping up the balance",
+ "balance_not_updated_after_bank_transfer": "a bank transfer is missing from the balance",
+ "balance_not_updated_after_cheque_or_cash_deposit": "a deposit is missing from the balance",
+ "beneficiary_not_allowed": "cannot add a payment recipient",
+ "cancel_transfer": "wants to cancel a transfer already sent",
+ "card_about_to_expire": "the card is close to its expiry date",
+ "card_acceptance": "where the card is accepted",
+ "card_arrival": "asking when a new card will arrive",
+ "card_delivery_estimate": "asking how long delivery will take",
+ "card_linking": "linking a card to the account",
+ "card_not_working": "the card is declined or not working",
+ "card_payment_fee_charged": "an unexpected fee was charged on a card payment",
+ "card_payment_not_recognised": "a card payment the customer does not recognise",
+ "card_payment_wrong_exchange_rate": "the exchange rate used was wrong",
+ "card_swallowed": "the ATM kept the card",
+ "cash_withdrawal_charge": "an unexpected charge for a cash withdrawal",
+ "cash_withdrawal_not_recognised": "a cash withdrawal the customer does not recognise",
+ "change_pin": "wants to change the PIN",
+ "compromised_card": "the card may have been compromised",
+ "contactless_not_working": "contactless payment is not working",
+ "country_support": "which countries the service supports",
+ "declined_card_payment": "a card payment was declined",
+ "declined_cash_withdrawal": "a cash withdrawal was declined",
+ "declined_transfer": "a transfer was declined",
+ "direct_debit_payment_not_recognised": "a direct debit the customer does not recognise",
+ "disposable_card_limits": "limits on a disposable virtual card",
+ "edit_personal_details": "wants to edit name, address or other personal details",
+ "exchange_charge": "a fee charged for currency exchange",
+ "exchange_rate": "asking what the current exchange rate is",
+ "exchange_via_app": "exchanging currency inside the app",
+ "extra_charge_on_statement": "an unexplained extra charge on the statement",
+ "failed_transfer": "a transfer failed",
+ "fiat_currency_support": "which regular currencies are supported",
+ "get_disposable_virtual_card": "wants a disposable virtual card",
+ "get_physical_card": "wants a physical card",
+ "getting_spare_card": "wants a spare or backup card",
+}