Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
feat(rust): add Python-compatible JSON dialects
kevin-gatimu29fc34ea3d41
diff --git a/laya-rust/src/python_json.rs b/laya-rust/src/python_json.rsnew file mode 100644index 00000000..677b7618--- /dev/null+++ b/laya-rust/src/python_json.rs@@ -0,0 +1,222 @@+//! A hand-written serializer that byte-for-byte reproduces Python's `json.dumps` output for the+//! three call-site dialects Laya uses, including Python's float `repr` rules.+//!+//! `serde_json`'s own writer cannot be reused here: it uses compact separators, HTML-escapes+//! nothing (good) but also formats floats differently from Python (`2.0` -> `2.0` is fine, but the+//! exponent thresholds and digit counts differ), and has no `ensure_ascii` knob.++use serde_json::Value;++/// Namespace for the three `json.dumps` dialects Laya uses. A unit struct rather than free+/// functions so the Rust API reads the same way the ported `PythonJson` static class does.+pub struct PythonJson;++impl PythonJson {+ /// Matches Python's `serialize_state`: a string passes through verbatim; everything else+ /// (including `null`, which becomes `"null"`) is serialized with `ensure_ascii=False`.+ pub fn state(value: &Value) -> String {+ if let Value::String(s) = value {+ return s.clone();+ }+ Self::dumps(value, false)+ }++ /// Matches Python's `render_criterion`: a string passes through verbatim; anything else is+ /// serialized with `ensure_ascii=False`. (`default=str` has no Rust equivalent because+ /// `Value` only ever holds JSON-representable data to begin with; see the crate README.)+ pub fn criterion(value: &Value) -> String {+ if let Value::String(s) = value {+ return s.clone();+ }+ Self::dumps(value, false)+ }++ /// Matches Python's `_to_internal` instructions path: a string passes through verbatim;+ /// anything else is serialized with `ensure_ascii=False`, the same escaping as [`Self::state`]+ /// and [`Self::criterion`]. Laya 0.3.6 escaped non-ASCII here (`ensure_ascii=True`); 0.3.21+ /// switched to `ensure_ascii=False` because the tokenizer then read the escaped `\uXXXX` text+ /// back as literal characters instead of the real ones — on the English checkpoint one German+ /// question answered differently as a dict than as the identical plain string.+ pub fn instructions(value: &Value) -> String {+ if let Value::String(s) = value {+ return s.clone();+ }+ Self::dumps(value, false)+ }++ /// The fourth dialect: `laya/shortlist.py`'s `_query_text` builds the shortlist's embedding+ /// query as `"%s\n%s" % (instructions, serialize_state(state))`, where a non-string+ /// `instructions` value is first replaced by `json.dumps(instructions, ensure_ascii=False)`.+ /// That is byte-for-byte the same rule as [`Self::state`] (string passes through verbatim,+ /// everything else becomes `ensure_ascii=False` JSON) applied to the instructions value+ /// instead of the state value — a separate named entry point because it is a distinct call+ /// site in `laya_shortlist.rs`, not because the formatting differs.+ pub fn shortlist_instructions(value: &Value) -> String {+ Self::state(value)+ }++ /// The underlying serializer: `json.dumps(value, ensure_ascii=escape_non_ascii,+ /// separators=(", ", ": "))`. Strings are still quoted and escaped here (unlike the three+ /// dialect entry points above, which special-case a top-level string to pass through+ /// unquoted).+ fn dumps(value: &Value, escape_non_ascii: bool) -> String {+ let mut out = String::new();+ Self::write_value(&mut out, value, escape_non_ascii);+ out+ }++ fn write_value(out: &mut String, value: &Value, escape_non_ascii: bool) {+ match value {+ Value::Null => out.push_str("null"),+ Value::Bool(b) => out.push_str(if *b { "true" } else { "false" }),+ Value::String(s) => Self::write_string(out, s, escape_non_ascii),+ Value::Number(n) => Self::write_number(out, n),+ Value::Array(items) => {+ out.push('[');+ for (i, item) in items.iter().enumerate() {+ if i > 0 {+ out.push_str(", ");+ }+ Self::write_value(out, item, escape_non_ascii);+ }+ out.push(']');+ }+ Value::Object(map) => {+ out.push('{');+ for (i, (k, v)) in map.iter().enumerate() {+ if i > 0 {+ out.push_str(", ");+ }+ Self::write_string(out, k, escape_non_ascii);+ out.push_str(": ");+ Self::write_value(out, v, escape_non_ascii);+ }+ out.push('}');+ }+ }+ }++ fn write_number(out: &mut String, n: &serde_json::Number) {+ if n.is_f64() {+ // A JSON literal with a `.` or exponent always parses to Number::Float, so this is+ // exactly Python's int/float distinction (see json-dialects.md "Integer preservation").+ out.push_str(&Self::repr(+ n.as_f64().expect("is_f64 implies as_f64 succeeds"),+ ));+ } else {+ // Integers (positive or negative) render as their plain decimal digits, same as Python.+ out.push_str(&n.to_string());+ }+ }++ fn write_string(out: &mut String, s: &str, escape_non_ascii: bool) {+ out.push('"');+ for c in s.chars() {+ match c {+ '"' => out.push_str("\\\""),+ '\\' => out.push_str("\\\\"),+ '\u{8}' => out.push_str("\\b"),+ '\u{c}' => out.push_str("\\f"),+ '\n' => out.push_str("\\n"),+ '\r' => out.push_str("\\r"),+ '\t' => out.push_str("\\t"),+ c if (c as u32) < 0x20 => {+ out.push_str(&format!("\\u{:04x}", c as u32));+ }+ c if escape_non_ascii && (c as u32) > 0x7f => {+ let cp = c as u32;+ if cp > 0xFFFF {+ // Astral character: emit as a UTF-16 surrogate pair, matching Python.+ let v = cp - 0x10000;+ let high = 0xD800 + (v >> 10);+ let low = 0xDC00 + (v & 0x3FF);+ out.push_str(&format!("\\u{high:04x}\\u{low:04x}"));+ } else {+ out.push_str(&format!("\\u{cp:04x}"));+ }+ }+ c => out.push(c),+ }+ }+ out.push('"');+ }++ /// Formats an `f64` exactly as Python's `repr(float)`: shortest round-trip decimal digits,+ /// always containing a `.` or `e`, exponent notation when the decimal exponent is `>= 16` or+ /// `<= -5`. Ports `PythonJson.Repr(double)`.+ pub fn repr(value: f64) -> String {+ if value.is_nan() {+ return "NaN".to_string();+ }+ if value.is_infinite() {+ return if value > 0.0 {+ "Infinity".to_string()+ } else {+ "-Infinity".to_string()+ };+ }+ if value == 0.0 {+ return if value.is_sign_negative() {+ "-0.0".to_string()+ } else {+ "0.0".to_string()+ };+ }++ let neg = value.is_sign_negative();+ let abs = value.abs();++ // Rust's `{:e}` for f64 already gives the shortest decimal that round-trips (same+ // algorithm family Display uses), normalized to exactly one digit before the point — e.g.+ // "3.333e-1", "1e0", "5e-324". That single-leading-digit form is exactly what the+ // reformatting below assumes, so there is no need to search for the decimal point or+ // exponent marker location separately.+ let raw = format!("{abs:e}");+ let e_pos = raw.find('e').expect("LowerExp output always contains 'e'");+ let mantissa = &raw[..e_pos];+ let exp10: i32 = raw[e_pos + 1..]+ .parse()+ .expect("LowerExp exponent is always a valid integer");++ let mut digits: String = mantissa.chars().filter(|&c| c != '.').collect();+ while digits.len() > 1 && digits.ends_with('0') {+ digits.pop();+ }++ let prefix = if neg { "-" } else { "" };++ if exp10 >= 16 || exp10 <= -5 {+ let mantissa_str = if digits.len() == 1 {+ digits.clone()+ } else {+ format!("{}.{}", &digits[..1], &digits[1..])+ };+ let exp_str = if exp10 >= 0 {+ format!("e+{exp10:02}")+ } else {+ format!("e-{:02}", -exp10)+ };+ format!("{prefix}{mantissa_str}{exp_str}")+ } else if exp10 >= 0 {+ // Fixed notation, |value| >= 1.+ let int_digit_count = (exp10 + 1) as usize;+ if digits.len() <= int_digit_count {+ let mut s = digits.clone();+ while s.len() < int_digit_count {+ s.push('0');+ }+ format!("{prefix}{s}.0")+ } else {+ format!(+ "{prefix}{}.{}",+ &digits[..int_digit_count],+ &digits[int_digit_count..]+ )+ }+ } else {+ // Fixed notation, 0 < |value| < 1.+ let leading_zeros = (-exp10 - 1) as usize;+ format!("{prefix}0.{}{digits}", "0".repeat(leading_zeros))+ }+ }+}diff --git a/laya-rust/tests/python_json_tests.rs b/laya-rust/tests/python_json_tests.rsnew file mode 100644index 00000000..640fc23b--- /dev/null+++ b/laya-rust/tests/python_json_tests.rs@@ -0,0 +1,183 @@+//! Tier 1: verifies that [`PythonJson`] reproduces Python's `json.dumps` byte-for-byte.++mod common;++use common::golden_data::CheckpointGoldenData;+use laya::python_json::PythonJson;+use serde_json::{Value, json};++/// Every checkpoint's `json_probe.json` is checked: the three dialects do+/// not depend on the checkpoint, but this also verifies all three golden directories are+/// discoverable and mutually consistent.+const CHECKPOINTS: [&str; 3] = ["multilingual", "english", "typed-decisions"];++#[test]+fn all_dialects_match_golden_for_every_checkpoint() {+ let mut checked_any = false;++ for checkpoint in CHECKPOINTS {+ let data = CheckpointGoldenData::for_checkpoint(checkpoint);+ if !data.available() {+ eprintln!(+ "skip: no golden data for '{checkpoint}' (set LAYA_GOLDEN_DIR to override the default location)"+ );+ continue;+ }+ checked_any = true;++ let probe = data.load("json_probe.json");+ let entries = probe+ .as_array()+ .expect("json_probe.json must be a JSON array");+ assert!(+ !entries.is_empty(),+ "{checkpoint}'s json_probe.json is empty"+ );++ for (i, entry) in entries.iter().enumerate() {+ let value = entry.get("value").cloned().unwrap_or(Value::Null);++ assert_dialect(+ checkpoint,+ i,+ "criterion",+ entry.get("criterion_dialect").and_then(Value::as_str),+ &PythonJson::criterion(&value),+ );+ assert_dialect(+ checkpoint,+ i,+ "instructions",+ entry.get("instructions_dialect").and_then(Value::as_str),+ &PythonJson::instructions(&value),+ );+ assert_dialect(+ checkpoint,+ i,+ "state",+ entry.get("state_dialect").and_then(Value::as_str),+ &PythonJson::state(&value),+ );+ }+ }++ if !checked_any {+ eprintln!("skip: no golden data found for any checkpoint");+ }+}++fn assert_dialect(+ checkpoint: &str,+ index: usize,+ dialect: &str,+ expected: Option<&str>,+ actual: &str,+) {+ assert_eq!(+ expected,+ Some(actual),+ "{checkpoint} json_probe[{index}] {dialect} dialect mismatch"+ );+}++// ── State(null) ─────────────────────────────────────────────────────────────++#[test]+fn state_with_null_returns_json_null() {+ // Python: serialize_state(None) -> json.dumps(None) -> "null".+ assert_eq!("null", PythonJson::state(&Value::Null));+}++// ── Repr-specific unit tests ──────────────────────────────────────────────────++#[test]+fn repr_matches_python() {+ let cases: &[(f64, &str)] = &[+ (f64::NAN, "NaN"),+ (f64::INFINITY, "Infinity"),+ (f64::NEG_INFINITY, "-Infinity"),+ (0.0, "0.0"),+ (1.0, "1.0"),+ (2.0, "2.0"),+ (-0.125, "-0.125"),+ (0.5, "0.5"),+ (1e21, "1e+21"),+ (1e-7, "1e-07"),+ (0.3333333333333333, "0.3333333333333333"),+ (1e15, "1000000000000000.0"),+ (1e16, "1e+16"),+ (1e17, "1e+17"),+ (0.0001, "0.0001"),+ (1e-5, "1e-05"),+ (123456789012345.6, "123456789012345.6"),+ (1.7976931348623157e308, "1.7976931348623157e+308"),+ (5e-324, "5e-324"),+ ];+ for &(input, expected) in cases {+ assert_eq!(expected, PythonJson::repr(input), "repr({input})");+ }+}++#[test]+fn repr_negative_zero() {+ // -0.0 needs a separate test because a float literal cannot distinguish it from 0.0 in a table.+ assert_eq!("-0.0", PythonJson::repr(-0.0));+}++// ── Escaping unit tests ────────────────────────────────────────────────────────++#[test]+fn ensure_ascii_false_keeps_literal_unicode() {+ let value = json!({"a": "naïve"});+ assert_eq!("{\"a\": \"naïve\"}", PythonJson::criterion(&value));+}++#[test]+fn instructions_no_longer_escapes_non_ascii_since_0_3_21() {+ // laya 0.3.21 changed `_to_internal`'s instructions path from `ensure_ascii=True` to+ // `ensure_ascii=False` (see `PythonJson::instructions`'s doc comment), matching `criterion`+ // and `state`. ü = U+00FC; 😀 = U+1F600 must now come through literal, not as `\uXXXX`.+ let value = json!({"a": "naïve", "b": "😀"});+ assert_eq!(+ "{\"a\": \"naïve\", \"b\": \"😀\"}",+ PythonJson::instructions(&value)+ );+}++#[test]+fn html_chars_not_escaped() {+ // Strings pass through verbatim.+ let value = Value::String("<tag> & 'quoted'".to_string());+ assert_eq!("<tag> & 'quoted'", PythonJson::criterion(&value));+}++#[test]+fn slash_not_escaped() {+ let value = json!({"url": "a/b"});+ assert_eq!("{\"url\": \"a/b\"}", PythonJson::criterion(&value));+}++#[test]+fn control_chars_escaped() {+ let value = json!({"x": "\u{1}\u{1f}"});+ assert_eq!("{\"x\": \"\\u0001\\u001f\"}", PythonJson::criterion(&value));+}++// ── Integer preservation ───────────────────────────────────────────────────────++#[test]+fn integers_never_gain_a_decimal_point() {+ // A JSON literal `3` must render as "3", not "3.0" — the whole reason serde_json's+ // int/float-preserving `Value::Number` was chosen over routing everything through f64.+ let value = json!({"open_tickets": 3});+ assert_eq!("{\"open_tickets\": 3}", PythonJson::state(&value));++ let value = json!({"open_tickets": 3.0});+ assert_eq!("{\"open_tickets\": 3.0}", PythonJson::state(&value));+}++#[test]+fn large_integer_within_i64_stays_exact() {+ let value = json!(1234567890123456789i64);+ assert_eq!("1234567890123456789", PythonJson::state(&value));+}