Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

feat(rust): add Python-compatible JSON dialects

kevin-gatimu29fc34ea3d41
diff --git a/laya-rust/src/python_json.rs b/laya-rust/src/python_json.rs
new file mode 100644
index 00000000..677b7618
--- /dev/null
+++ b/laya-rust/src/python_json.rs
@@ -0,0 +1,222 @@
+//! A hand-written serializer that byte-for-byte reproduces Python's `json.dumps` output for the
+//! three call-site dialects Laya uses, including Python's float `repr` rules.
+//!
+//! `serde_json`'s own writer cannot be reused here: it uses compact separators, HTML-escapes
+//! nothing (good) but also formats floats differently from Python (`2.0` -> `2.0` is fine, but the
+//! exponent thresholds and digit counts differ), and has no `ensure_ascii` knob.
+
+use serde_json::Value;
+
+/// Namespace for the three `json.dumps` dialects Laya uses. A unit struct rather than free
+/// functions so the Rust API reads the same way the ported `PythonJson` static class does.
+pub struct PythonJson;
+
+impl PythonJson {
+ /// Matches Python's `serialize_state`: a string passes through verbatim; everything else
+ /// (including `null`, which becomes `"null"`) is serialized with `ensure_ascii=False`.
+ pub fn state(value: &Value) -> String {
+ if let Value::String(s) = value {
+ return s.clone();
+ }
+ Self::dumps(value, false)
+ }
+
+ /// Matches Python's `render_criterion`: a string passes through verbatim; anything else is
+ /// serialized with `ensure_ascii=False`. (`default=str` has no Rust equivalent because
+ /// `Value` only ever holds JSON-representable data to begin with; see the crate README.)
+ pub fn criterion(value: &Value) -> String {
+ if let Value::String(s) = value {
+ return s.clone();
+ }
+ Self::dumps(value, false)
+ }
+
+ /// Matches Python's `_to_internal` instructions path: a string passes through verbatim;
+ /// anything else is serialized with `ensure_ascii=False`, the same escaping as [`Self::state`]
+ /// and [`Self::criterion`]. Laya 0.3.6 escaped non-ASCII here (`ensure_ascii=True`); 0.3.21
+ /// switched to `ensure_ascii=False` because the tokenizer then read the escaped `\uXXXX` text
+ /// back as literal characters instead of the real ones — on the English checkpoint one German
+ /// question answered differently as a dict than as the identical plain string.
+ pub fn instructions(value: &Value) -> String {
+ if let Value::String(s) = value {
+ return s.clone();
+ }
+ Self::dumps(value, false)
+ }
+
+ /// The fourth dialect: `laya/shortlist.py`'s `_query_text` builds the shortlist's embedding
+ /// query as `"%s\n%s" % (instructions, serialize_state(state))`, where a non-string
+ /// `instructions` value is first replaced by `json.dumps(instructions, ensure_ascii=False)`.
+ /// That is byte-for-byte the same rule as [`Self::state`] (string passes through verbatim,
+ /// everything else becomes `ensure_ascii=False` JSON) applied to the instructions value
+ /// instead of the state value — a separate named entry point because it is a distinct call
+ /// site in `laya_shortlist.rs`, not because the formatting differs.
+ pub fn shortlist_instructions(value: &Value) -> String {
+ Self::state(value)
+ }
+
+ /// The underlying serializer: `json.dumps(value, ensure_ascii=escape_non_ascii,
+ /// separators=(", ", ": "))`. Strings are still quoted and escaped here (unlike the three
+ /// dialect entry points above, which special-case a top-level string to pass through
+ /// unquoted).
+ fn dumps(value: &Value, escape_non_ascii: bool) -> String {
+ let mut out = String::new();
+ Self::write_value(&mut out, value, escape_non_ascii);
+ out
+ }
+
+ fn write_value(out: &mut String, value: &Value, escape_non_ascii: bool) {
+ match value {
+ Value::Null => out.push_str("null"),
+ Value::Bool(b) => out.push_str(if *b { "true" } else { "false" }),
+ Value::String(s) => Self::write_string(out, s, escape_non_ascii),
+ Value::Number(n) => Self::write_number(out, n),
+ Value::Array(items) => {
+ out.push('[');
+ for (i, item) in items.iter().enumerate() {
+ if i > 0 {
+ out.push_str(", ");
+ }
+ Self::write_value(out, item, escape_non_ascii);
+ }
+ out.push(']');
+ }
+ Value::Object(map) => {
+ out.push('{');
+ for (i, (k, v)) in map.iter().enumerate() {
+ if i > 0 {
+ out.push_str(", ");
+ }
+ Self::write_string(out, k, escape_non_ascii);
+ out.push_str(": ");
+ Self::write_value(out, v, escape_non_ascii);
+ }
+ out.push('}');
+ }
+ }
+ }
+
+ fn write_number(out: &mut String, n: &serde_json::Number) {
+ if n.is_f64() {
+ // A JSON literal with a `.` or exponent always parses to Number::Float, so this is
+ // exactly Python's int/float distinction (see json-dialects.md "Integer preservation").
+ out.push_str(&Self::repr(
+ n.as_f64().expect("is_f64 implies as_f64 succeeds"),
+ ));
+ } else {
+ // Integers (positive or negative) render as their plain decimal digits, same as Python.
+ out.push_str(&n.to_string());
+ }
+ }
+
+ fn write_string(out: &mut String, s: &str, escape_non_ascii: bool) {
+ out.push('"');
+ for c in s.chars() {
+ match c {
+ '"' => out.push_str("\\\""),
+ '\\' => out.push_str("\\\\"),
+ '\u{8}' => out.push_str("\\b"),
+ '\u{c}' => out.push_str("\\f"),
+ '\n' => out.push_str("\\n"),
+ '\r' => out.push_str("\\r"),
+ '\t' => out.push_str("\\t"),
+ c if (c as u32) < 0x20 => {
+ out.push_str(&format!("\\u{:04x}", c as u32));
+ }
+ c if escape_non_ascii && (c as u32) > 0x7f => {
+ let cp = c as u32;
+ if cp > 0xFFFF {
+ // Astral character: emit as a UTF-16 surrogate pair, matching Python.
+ let v = cp - 0x10000;
+ let high = 0xD800 + (v >> 10);
+ let low = 0xDC00 + (v & 0x3FF);
+ out.push_str(&format!("\\u{high:04x}\\u{low:04x}"));
+ } else {
+ out.push_str(&format!("\\u{cp:04x}"));
+ }
+ }
+ c => out.push(c),
+ }
+ }
+ out.push('"');
+ }
+
+ /// Formats an `f64` exactly as Python's `repr(float)`: shortest round-trip decimal digits,
+ /// always containing a `.` or `e`, exponent notation when the decimal exponent is `>= 16` or
+ /// `<= -5`. Ports `PythonJson.Repr(double)`.
+ pub fn repr(value: f64) -> String {
+ if value.is_nan() {
+ return "NaN".to_string();
+ }
+ if value.is_infinite() {
+ return if value > 0.0 {
+ "Infinity".to_string()
+ } else {
+ "-Infinity".to_string()
+ };
+ }
+ if value == 0.0 {
+ return if value.is_sign_negative() {
+ "-0.0".to_string()
+ } else {
+ "0.0".to_string()
+ };
+ }
+
+ let neg = value.is_sign_negative();
+ let abs = value.abs();
+
+ // Rust's `{:e}` for f64 already gives the shortest decimal that round-trips (same
+ // algorithm family Display uses), normalized to exactly one digit before the point — e.g.
+ // "3.333e-1", "1e0", "5e-324". That single-leading-digit form is exactly what the
+ // reformatting below assumes, so there is no need to search for the decimal point or
+ // exponent marker location separately.
+ let raw = format!("{abs:e}");
+ let e_pos = raw.find('e').expect("LowerExp output always contains 'e'");
+ let mantissa = &raw[..e_pos];
+ let exp10: i32 = raw[e_pos + 1..]
+ .parse()
+ .expect("LowerExp exponent is always a valid integer");
+
+ let mut digits: String = mantissa.chars().filter(|&c| c != '.').collect();
+ while digits.len() > 1 && digits.ends_with('0') {
+ digits.pop();
+ }
+
+ let prefix = if neg { "-" } else { "" };
+
+ if exp10 >= 16 || exp10 <= -5 {
+ let mantissa_str = if digits.len() == 1 {
+ digits.clone()
+ } else {
+ format!("{}.{}", &digits[..1], &digits[1..])
+ };
+ let exp_str = if exp10 >= 0 {
+ format!("e+{exp10:02}")
+ } else {
+ format!("e-{:02}", -exp10)
+ };
+ format!("{prefix}{mantissa_str}{exp_str}")
+ } else if exp10 >= 0 {
+ // Fixed notation, |value| >= 1.
+ let int_digit_count = (exp10 + 1) as usize;
+ if digits.len() <= int_digit_count {
+ let mut s = digits.clone();
+ while s.len() < int_digit_count {
+ s.push('0');
+ }
+ format!("{prefix}{s}.0")
+ } else {
+ format!(
+ "{prefix}{}.{}",
+ &digits[..int_digit_count],
+ &digits[int_digit_count..]
+ )
+ }
+ } else {
+ // Fixed notation, 0 < |value| < 1.
+ let leading_zeros = (-exp10 - 1) as usize;
+ format!("{prefix}0.{}{digits}", "0".repeat(leading_zeros))
+ }
+ }
+}
diff --git a/laya-rust/tests/python_json_tests.rs b/laya-rust/tests/python_json_tests.rs
new file mode 100644
index 00000000..640fc23b
--- /dev/null
+++ b/laya-rust/tests/python_json_tests.rs
@@ -0,0 +1,183 @@
+//! Tier 1: verifies that [`PythonJson`] reproduces Python's `json.dumps` byte-for-byte.
+
+mod common;
+
+use common::golden_data::CheckpointGoldenData;
+use laya::python_json::PythonJson;
+use serde_json::{Value, json};
+
+/// Every checkpoint's `json_probe.json` is checked: the three dialects do
+/// not depend on the checkpoint, but this also verifies all three golden directories are
+/// discoverable and mutually consistent.
+const CHECKPOINTS: [&str; 3] = ["multilingual", "english", "typed-decisions"];
+
+#[test]
+fn all_dialects_match_golden_for_every_checkpoint() {
+ let mut checked_any = false;
+
+ for checkpoint in CHECKPOINTS {
+ let data = CheckpointGoldenData::for_checkpoint(checkpoint);
+ if !data.available() {
+ eprintln!(
+ "skip: no golden data for '{checkpoint}' (set LAYA_GOLDEN_DIR to override the default location)"
+ );
+ continue;
+ }
+ checked_any = true;
+
+ let probe = data.load("json_probe.json");
+ let entries = probe
+ .as_array()
+ .expect("json_probe.json must be a JSON array");
+ assert!(
+ !entries.is_empty(),
+ "{checkpoint}'s json_probe.json is empty"
+ );
+
+ for (i, entry) in entries.iter().enumerate() {
+ let value = entry.get("value").cloned().unwrap_or(Value::Null);
+
+ assert_dialect(
+ checkpoint,
+ i,
+ "criterion",
+ entry.get("criterion_dialect").and_then(Value::as_str),
+ &PythonJson::criterion(&value),
+ );
+ assert_dialect(
+ checkpoint,
+ i,
+ "instructions",
+ entry.get("instructions_dialect").and_then(Value::as_str),
+ &PythonJson::instructions(&value),
+ );
+ assert_dialect(
+ checkpoint,
+ i,
+ "state",
+ entry.get("state_dialect").and_then(Value::as_str),
+ &PythonJson::state(&value),
+ );
+ }
+ }
+
+ if !checked_any {
+ eprintln!("skip: no golden data found for any checkpoint");
+ }
+}
+
+fn assert_dialect(
+ checkpoint: &str,
+ index: usize,
+ dialect: &str,
+ expected: Option<&str>,
+ actual: &str,
+) {
+ assert_eq!(
+ expected,
+ Some(actual),
+ "{checkpoint} json_probe[{index}] {dialect} dialect mismatch"
+ );
+}
+
+// ── State(null) ─────────────────────────────────────────────────────────────
+
+#[test]
+fn state_with_null_returns_json_null() {
+ // Python: serialize_state(None) -> json.dumps(None) -> "null".
+ assert_eq!("null", PythonJson::state(&Value::Null));
+}
+
+// ── Repr-specific unit tests ──────────────────────────────────────────────────
+
+#[test]
+fn repr_matches_python() {
+ let cases: &[(f64, &str)] = &[
+ (f64::NAN, "NaN"),
+ (f64::INFINITY, "Infinity"),
+ (f64::NEG_INFINITY, "-Infinity"),
+ (0.0, "0.0"),
+ (1.0, "1.0"),
+ (2.0, "2.0"),
+ (-0.125, "-0.125"),
+ (0.5, "0.5"),
+ (1e21, "1e+21"),
+ (1e-7, "1e-07"),
+ (0.3333333333333333, "0.3333333333333333"),
+ (1e15, "1000000000000000.0"),
+ (1e16, "1e+16"),
+ (1e17, "1e+17"),
+ (0.0001, "0.0001"),
+ (1e-5, "1e-05"),
+ (123456789012345.6, "123456789012345.6"),
+ (1.7976931348623157e308, "1.7976931348623157e+308"),
+ (5e-324, "5e-324"),
+ ];
+ for &(input, expected) in cases {
+ assert_eq!(expected, PythonJson::repr(input), "repr({input})");
+ }
+}
+
+#[test]
+fn repr_negative_zero() {
+ // -0.0 needs a separate test because a float literal cannot distinguish it from 0.0 in a table.
+ assert_eq!("-0.0", PythonJson::repr(-0.0));
+}
+
+// ── Escaping unit tests ────────────────────────────────────────────────────────
+
+#[test]
+fn ensure_ascii_false_keeps_literal_unicode() {
+ let value = json!({"a": "naïve"});
+ assert_eq!("{\"a\": \"naïve\"}", PythonJson::criterion(&value));
+}
+
+#[test]
+fn instructions_no_longer_escapes_non_ascii_since_0_3_21() {
+ // laya 0.3.21 changed `_to_internal`'s instructions path from `ensure_ascii=True` to
+ // `ensure_ascii=False` (see `PythonJson::instructions`'s doc comment), matching `criterion`
+ // and `state`. ü = U+00FC; 😀 = U+1F600 must now come through literal, not as `\uXXXX`.
+ let value = json!({"a": "naïve", "b": "😀"});
+ assert_eq!(
+ "{\"a\": \"naïve\", \"b\": \"😀\"}",
+ PythonJson::instructions(&value)
+ );
+}
+
+#[test]
+fn html_chars_not_escaped() {
+ // Strings pass through verbatim.
+ let value = Value::String("<tag> & 'quoted'".to_string());
+ assert_eq!("<tag> & 'quoted'", PythonJson::criterion(&value));
+}
+
+#[test]
+fn slash_not_escaped() {
+ let value = json!({"url": "a/b"});
+ assert_eq!("{\"url\": \"a/b\"}", PythonJson::criterion(&value));
+}
+
+#[test]
+fn control_chars_escaped() {
+ let value = json!({"x": "\u{1}\u{1f}"});
+ assert_eq!("{\"x\": \"\\u0001\\u001f\"}", PythonJson::criterion(&value));
+}
+
+// ── Integer preservation ───────────────────────────────────────────────────────
+
+#[test]
+fn integers_never_gain_a_decimal_point() {
+ // A JSON literal `3` must render as "3", not "3.0" — the whole reason serde_json's
+ // int/float-preserving `Value::Number` was chosen over routing everything through f64.
+ let value = json!({"open_tickets": 3});
+ assert_eq!("{\"open_tickets\": 3}", PythonJson::state(&value));
+
+ let value = json!({"open_tickets": 3.0});
+ assert_eq!("{\"open_tickets\": 3.0}", PythonJson::state(&value));
+}
+
+#[test]
+fn large_integer_within_i64_stays_exact() {
+ let value = json!(1234567890123456789i64);
+ assert_eq!("1234567890123456789", PythonJson::state(&value));
+}