Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

feat(rust): add typed questions, sequence builder and collator

kevin-gatimucd39e6b61e9b
diff --git a/laya-rust/src/collator.rs b/laya-rust/src/collator.rs
new file mode 100644
index 00000000..74c3dbdb
--- /dev/null
+++ b/laya-rust/src/collator.rs
@@ -0,0 +1,120 @@
+//! Pads tokenized questions into one batch. Ports `collate_items` from `laya/common.py`. The
+//! padded tensors match the input shapes the ONNX graph expects.
+
+use crate::error::{LayaError, Result};
+use crate::questions::QuestionType;
+
+/// One question, tokenized and ready to be batched.
+#[derive(Debug, Clone)]
+pub struct SequenceItem {
+ /// The token sequence.
+ pub ids: Vec<u32>,
+ /// One position per option, pointing at that option's `[MASK]`.
+ pub markers: Vec<usize>,
+ /// Which question type, fed to the model's type embedding.
+ pub qtype: QuestionType,
+}
+
+impl SequenceItem {
+ /// Build a sequence item from its parts.
+ pub fn new(ids: Vec<u32>, markers: Vec<usize>, qtype: QuestionType) -> Self {
+ Self {
+ ids,
+ markers,
+ qtype,
+ }
+ }
+}
+
+/// A padded batch, laid out as the row-major tensors the ONNX graph takes.
+#[derive(Debug, Clone)]
+pub struct CollatedBatch {
+ /// Rows in the batch: one per question.
+ pub count: usize,
+ /// Padded sequence length, the longest row.
+ pub sequence_length: usize,
+ /// Padded marker-axis width.
+ pub marker_count: usize,
+ /// `[count, sequence_length]`, padded with the pad id.
+ pub input_ids: Vec<i64>,
+ /// `[count, sequence_length]`, 1 on real tokens and 0 on padding.
+ pub attention_mask: Vec<i64>,
+ /// `[count, marker_count]`, 0 in padding columns.
+ pub marker_pos: Vec<i64>,
+ /// `[count, marker_count]`, false in padding columns.
+ pub marker_mask: Vec<bool>,
+ /// `[count]`, the question type per row.
+ pub qtype: Vec<i64>,
+ /// The real (unpadded) option count per row — how many logits to read from that row.
+ pub marker_counts: Vec<usize>,
+ /// Non-padding tokens across the batch, which is what Python reports as input tokens.
+ pub input_tokens: usize,
+}
+
+/// Namespace for batch collation. See the module docs for the source this ports.
+pub struct Collator;
+
+impl Collator {
+ /// The marker axis is never narrower than this.
+ ///
+ /// The tracer baked `TopK k=2` into the exported graph: Python chooses the top-2 branch at
+ /// runtime and the export only ever saw two or more markers, so a batch whose questions all
+ /// have a single option makes TopK fail outright. Padding to two columns with `marker_mask =
+ /// false` is exactly equivalent, not merely close: the model's `masked_fill` puts `-1e4` in
+ /// the pad column, which underflows softmax to 0.0, so `p == [1.0, 0.0]` — precisely the
+ /// `[top1, 0.0]` Python substitutes. Reading only [`CollatedBatch::marker_counts`] logits per
+ /// row keeps the pad column out of the answer.
+ pub const MIN_MARKERS: usize = 2;
+
+ /// Pad a batch of tokenized questions. Returns an error for an empty batch rather than
+ /// producing a zero-row tensor the ONNX graph would reject anyway.
+ pub fn collate(items: &[SequenceItem], pad_id: u32) -> Result<CollatedBatch> {
+ if items.is_empty() {
+ return Err(LayaError::InvalidQuestion("nothing to collate".to_string()));
+ }
+
+ let n = items.len();
+ let length = items.iter().map(|it| it.ids.len()).max().unwrap_or(0);
+ let marker_count =
+ Self::MIN_MARKERS.max(items.iter().map(|it| it.markers.len()).max().unwrap_or(0));
+
+ let mut input_ids = vec![i64::from(pad_id); n * length];
+ let mut attention_mask = vec![0i64; n * length];
+ let mut marker_pos = vec![0i64; n * marker_count];
+ let mut marker_mask = vec![false; n * marker_count];
+ let mut qtype = vec![0i64; n];
+ let mut marker_counts = vec![0usize; n];
+ let mut input_tokens = 0usize;
+
+ for (i, item) in items.iter().enumerate() {
+ let row = i * length;
+ for (j, &id) in item.ids.iter().enumerate() {
+ input_ids[row + j] = i64::from(id);
+ attention_mask[row + j] = 1;
+ }
+ input_tokens += item.ids.len();
+
+ let mrow = i * marker_count;
+ for (j, &pos) in item.markers.iter().enumerate() {
+ marker_pos[mrow + j] = pos as i64;
+ marker_mask[mrow + j] = true;
+ }
+
+ qtype[i] = item.qtype as i64;
+ marker_counts[i] = item.markers.len();
+ }
+
+ Ok(CollatedBatch {
+ count: n,
+ sequence_length: length,
+ marker_count,
+ input_ids,
+ attention_mask,
+ marker_pos,
+ marker_mask,
+ qtype,
+ marker_counts,
+ input_tokens,
+ })
+ }
+}
diff --git a/laya-rust/src/questions.rs b/laya-rust/src/questions.rs
new file mode 100644
index 00000000..3e763cbc
--- /dev/null
+++ b/laya-rust/src/questions.rs
@@ -0,0 +1,329 @@
+//! Typed questions and the insertion-ordered set they are asked in.
+//!
+//! Ports `laya/presets.py`'s question shape. Order is load-bearing throughout this module: batch
+//! row order follows `QuestionSet` order, and choice label order decides which logit each
+//! label's probability comes from (see `sequence-construction.md`).
+
+use crate::error::{LayaError, Result};
+use indexmap::IndexMap;
+use serde_json::Value;
+use std::collections::HashSet;
+
+/// The three question types Laya answers. Values match Python's `QTYPES` and the `qtype` tensor
+/// encoding (`model-contract.md`).
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
+pub enum QuestionType {
+ /// Pick one label from a set of named options.
+ Choice = 0,
+ /// Place the state on an ordered scale; the answer is an expected level.
+ Score = 1,
+ /// Does the statement hold? The answer is a probability.
+ Noul = 2,
+}
+
+// The Python type name (`SequenceBuilder::type_name`) and the `qtype` tensor's integer encoding
+// (a plain `qtype as i64` cast, since the discriminants above already match Python's `QTYPES`)
+// live in `sequence_builder.rs` alongside the rest of that logic, rather than here, keeping this
+// enum bare.
+
+impl TryFrom<i64> for QuestionType {
+ type Error = LayaError;
+
+ /// The question type for a `qtype` tensor value. Rust has no implicit enum-from-integer cast,
+ /// so this is the explicit equivalent.
+ fn try_from(index: i64) -> Result<Self> {
+ match index {
+ 0 => Ok(QuestionType::Choice),
+ 1 => Ok(QuestionType::Score),
+ 2 => Ok(QuestionType::Noul),
+ other => Err(LayaError::InvalidQuestion(format!(
+ "unknown qtype index {other}"
+ ))),
+ }
+ }
+}
+
+/// A [`QuestionType::Choice`] question: ordered, named options.
+#[derive(Debug, Clone)]
+pub struct ChoiceQuestion {
+ instructions: Value,
+ options: Vec<(String, Value)>,
+}
+
+impl ChoiceQuestion {
+ /// The prompt: a string verbatim, or anything else rendered through
+ /// [`crate::python_json::instructions`].
+ pub fn instructions(&self) -> &Value {
+ &self.instructions
+ }
+
+ /// The options in label order. Index N here is the label for logit N.
+ pub fn options(&self) -> &[(String, Value)] {
+ &self.options
+ }
+
+ /// The labels, in order.
+ pub fn labels(&self) -> impl Iterator<Item = &str> {
+ self.options.iter().map(|(label, _)| label.as_str())
+ }
+}
+
+/// A [`QuestionType::Score`] question: an ordered scale, lowest level first.
+#[derive(Debug, Clone)]
+pub struct ScoreQuestion {
+ instructions: Value,
+ levels: Vec<Value>,
+}
+
+impl ScoreQuestion {
+ /// The prompt. See [`ChoiceQuestion::instructions`].
+ pub fn instructions(&self) -> &Value {
+ &self.instructions
+ }
+
+ /// The levels, lowest first. The answer is an expectation over these indices.
+ pub fn levels(&self) -> &[Value] {
+ &self.levels
+ }
+}
+
+/// A [`QuestionType::Noul`] question: does the statement hold?
+#[derive(Debug, Clone)]
+pub struct NoulQuestion {
+ instructions: Value,
+ if_false: Value,
+ if_true: Value,
+}
+
+impl NoulQuestion {
+ /// The prompt. See [`ChoiceQuestion::instructions`].
+ pub fn instructions(&self) -> &Value {
+ &self.instructions
+ }
+
+ /// Wording for the false option, or `Value::Null` for the default.
+ pub fn if_false(&self) -> &Value {
+ &self.if_false
+ }
+
+ /// Wording for the true option, or `Value::Null` for the default.
+ pub fn if_true(&self) -> &Value {
+ &self.if_true
+ }
+}
+
+/// One typed question. Construct with [`Question::choice`], [`Question::choice_labels`],
+/// [`Question::score`] or [`Question::noul`].
+#[derive(Debug, Clone)]
+pub enum Question {
+ /// See [`ChoiceQuestion`].
+ Choice(ChoiceQuestion),
+ /// See [`ScoreQuestion`].
+ Score(ScoreQuestion),
+ /// See [`NoulQuestion`].
+ Noul(NoulQuestion),
+}
+
+impl Question {
+ /// Which of the three question types this is.
+ pub fn question_type(&self) -> QuestionType {
+ match self {
+ Question::Choice(_) => QuestionType::Choice,
+ Question::Score(_) => QuestionType::Score,
+ Question::Noul(_) => QuestionType::Noul,
+ }
+ }
+
+ /// The prompt. A string is used verbatim; anything else is serialized with
+ /// [`crate::python_json::instructions`], matching Python's `_to_internal`.
+ pub fn instructions(&self) -> &Value {
+ match self {
+ Question::Choice(c) => c.instructions(),
+ Question::Score(s) => s.instructions(),
+ Question::Noul(n) => n.instructions(),
+ }
+ }
+
+ /// This question as a choice question.
+ pub fn as_choice(&self) -> Option<&ChoiceQuestion> {
+ match self {
+ Question::Choice(c) => Some(c),
+ _ => None,
+ }
+ }
+
+ /// This question as a score question.
+ pub fn as_score(&self) -> Option<&ScoreQuestion> {
+ match self {
+ Question::Score(s) => Some(s),
+ _ => None,
+ }
+ }
+
+ /// This question as a noul question.
+ pub fn as_noul(&self) -> Option<&NoulQuestion> {
+ match self {
+ Question::Noul(n) => Some(n),
+ _ => None,
+ }
+ }
+
+ /// A choice question over ordered `(label, description)` options. A `Value::Null` description
+ /// means "no description"; note that `0` and `false` are real descriptions and are rendered
+ /// (see [`crate::python_json::is_blank_criterion`]).
+ ///
+ /// Rejects duplicate labels and an empty option list: Python would silently collapse
+ /// duplicate keys into one option, changing the option count and so every marker position, so
+ /// this refuses rather than silently answering a different question.
+ pub fn choice<I, L, D>(instructions: impl Into<Value>, options: I) -> Result<Question>
+ where
+ I: IntoIterator<Item = (L, D)>,
+ L: Into<String>,
+ D: Into<Value>,
+ {
+ let mut list = Vec::new();
+ let mut seen = HashSet::new();
+ for (label, description) in options {
+ let label = label.into();
+ if !seen.insert(label.clone()) {
+ return Err(LayaError::InvalidQuestion(format!(
+ "duplicate choice label '{label}'"
+ )));
+ }
+ list.push((label, description.into()));
+ }
+ if list.is_empty() {
+ return Err(LayaError::InvalidQuestion(
+ "a choice question needs at least one option".to_string(),
+ ));
+ }
+ Ok(Question::Choice(ChoiceQuestion {
+ instructions: instructions.into(),
+ options: list,
+ }))
+ }
+
+ /// A choice question over bare labels, with no descriptions.
+ pub fn choice_labels<I, L>(instructions: impl Into<Value>, labels: I) -> Result<Question>
+ where
+ I: IntoIterator<Item = L>,
+ L: Into<String>,
+ {
+ Question::choice(
+ instructions,
+ labels.into_iter().map(|label| (label, Value::Null)),
+ )
+ }
+
+ /// A score question over ordered levels, lowest first. Rejects an empty level list.
+ pub fn score<I, D>(instructions: impl Into<Value>, levels: I) -> Result<Question>
+ where
+ I: IntoIterator<Item = D>,
+ D: Into<Value>,
+ {
+ let list: Vec<Value> = levels.into_iter().map(Into::into).collect();
+ if list.is_empty() {
+ return Err(LayaError::InvalidQuestion(
+ "a score question needs at least one level".to_string(),
+ ));
+ }
+ Ok(Question::Score(ScoreQuestion {
+ instructions: instructions.into(),
+ levels: list,
+ }))
+ }
+
+ /// A yes/no question. Supplying `if_false` / `if_true` (e.g. `Some("on fire")`, or `None::<Value>`
+ /// for the default) replaces the default "no, the statement does not hold" / "yes, the
+ /// statement holds" wording.
+ pub fn noul(
+ instructions: impl Into<Value>,
+ if_false: impl Into<Value>,
+ if_true: impl Into<Value>,
+ ) -> Question {
+ Question::Noul(NoulQuestion {
+ instructions: instructions.into(),
+ if_false: if_false.into(),
+ if_true: if_true.into(),
+ })
+ }
+}
+
+/// An insertion-ordered set of question id to [`Question`], answered in one batch.
+///
+/// Deliberately backed by [`IndexMap`] rather than a hash map: the batch row order must match the
+/// order the caller wrote the questions in, because that is what Python's `dict` guarantees and
+/// what the answer order is compared against.
+#[derive(Debug, Clone, Default)]
+pub struct QuestionSet {
+ map: IndexMap<String, Question>,
+}
+
+impl QuestionSet {
+ /// An empty set.
+ pub fn new() -> Self {
+ Self {
+ map: IndexMap::new(),
+ }
+ }
+
+ /// How many questions are in the set.
+ pub fn len(&self) -> usize {
+ self.map.len()
+ }
+
+ /// Whether the set has no questions.
+ pub fn is_empty(&self) -> bool {
+ self.map.is_empty()
+ }
+
+ /// Insert a question by id. Returns an error if the id is already present, rather than
+ /// silently overwriting it and shifting every subsequent row's meaning.
+ pub fn insert(&mut self, id: impl Into<String>, question: Question) -> Result<()> {
+ let id = id.into();
+ if self.map.contains_key(&id) {
+ return Err(LayaError::InvalidQuestion(format!(
+ "duplicate question id '{id}'"
+ )));
+ }
+ self.map.insert(id, question);
+ Ok(())
+ }
+
+ /// Builder-style [`insert`](Self::insert): `QuestionSet::new().with("a", q1)?.with("b", q2)?`.
+ pub fn with(mut self, id: impl Into<String>, question: Question) -> Result<Self> {
+ self.insert(id, question)?;
+ Ok(self)
+ }
+
+ /// The question with this id, if any.
+ pub fn get(&self, id: &str) -> Option<&Question> {
+ self.map.get(id)
+ }
+
+ /// Whether the set contains this id.
+ pub fn contains_id(&self, id: &str) -> bool {
+ self.map.contains_key(id)
+ }
+
+ /// The question ids, in insertion order.
+ pub fn ids(&self) -> impl Iterator<Item = &str> {
+ self.map.keys().map(String::as_str)
+ }
+
+ /// Every `(id, question)` pair, in insertion order.
+ pub fn iter(&self) -> impl Iterator<Item = (&str, &Question)> {
+ self.map.iter().map(|(id, q)| (id.as_str(), q))
+ }
+}
+
+impl std::ops::Index<&str> for QuestionSet {
+ type Output = Question;
+
+ /// Panics if no question carries this id; use [`QuestionSet::get`] for a fallible lookup.
+ fn index(&self, id: &str) -> &Question {
+ self.map
+ .get(id)
+ .unwrap_or_else(|| panic!("no question with id '{id}'"))
+ }
+}
diff --git a/laya-rust/src/sequence_builder.rs b/laya-rust/src/sequence_builder.rs
new file mode 100644
index 00000000..9f1e8bc7
--- /dev/null
+++ b/laya-rust/src/sequence_builder.rs
@@ -0,0 +1,206 @@
+//! Builds the typed token sequence one question is answered from. Ports `build_sequence`,
+//! `render_options` and `render_criterion` from `laya/common.py`, step by step.
+//!
+//! The layout is `[CLS] <type> question: instructions [SEP] [MASK] opt0 [MASK] opt1 ... [SEP]
+//! state [SEP]`. The scorer reads the hidden state at each `[MASK]`, so a marker position that is
+//! off by one token attaches an option's probability to the wrong place in the sequence and the
+//! model answers a different question without anything failing. Every offset here is therefore
+//! load-bearing.
+
+use crate::error::Result;
+use crate::python_json::PythonJson;
+use crate::questions::Question;
+use crate::tokenization::LayaTokenizer;
+use serde_json::Value;
+
+/// Namespace for sequence construction. See the module docs for the source this ports.
+pub struct SequenceBuilder;
+
+impl SequenceBuilder {
+ /// One option fragment is capped at this many tokens, before the marker is prepended.
+ const MAX_OPTION_TOKENS: usize = 48;
+
+ /// The head is never squeezed below this, even when the options have eaten the budget.
+ const MIN_HEAD_TOKENS: usize = 8;
+
+ /// The Python type name, which appears verbatim in the prompt text.
+ pub fn type_name(qtype: crate::questions::QuestionType) -> &'static str {
+ use crate::questions::QuestionType;
+ match qtype {
+ QuestionType::Choice => "choice",
+ QuestionType::Score => "score",
+ QuestionType::Noul => "noul",
+ }
+ }
+
+ /// The instruction text as it reaches the tokenizer: a string verbatim, anything else through
+ /// the `json.dumps` dialect that keeps non-ASCII characters literal (`ensure_ascii=False`).
+ /// Ports `Agent._to_internal` (laya 0.3.21; 0.3.6 escaped non-ASCII here instead).
+ pub fn render_instructions(question: &Question) -> String {
+ match question.instructions() {
+ Value::String(s) => s.clone(),
+ other => PythonJson::instructions(other),
+ }
+ }
+
+ /// The option texts in label-index order. Index N is the option the Nth logit scores, so this
+ /// order is what binds a probability to a label.
+ pub fn render_options(question: &Question) -> Vec<String> {
+ match question {
+ Question::Choice(c) => c
+ .options()
+ .iter()
+ .map(|(label, description)| {
+ if Self::is_blank_criterion(description) {
+ label.clone()
+ } else {
+ format!("{label}: {}", PythonJson::criterion(description))
+ }
+ })
+ .collect(),
+ Question::Score(s) => s
+ .levels()
+ .iter()
+ .enumerate()
+ .map(|(i, level)| format!("level {i}: {}", PythonJson::criterion(level)))
+ .collect(),
+ Question::Noul(n) => {
+ let false_text = if Self::is_blank_criterion(n.if_false()) {
+ "no, the statement does not hold".to_string()
+ } else {
+ PythonJson::criterion(n.if_false())
+ };
+ let true_text = if Self::is_blank_criterion(n.if_true()) {
+ "yes, the statement holds".to_string()
+ } else {
+ PythonJson::criterion(n.if_true())
+ };
+ vec![format!("false: {false_text}"), format!("true: {true_text}")]
+ }
+ }
+ }
+
+ /// Whether a criterion value means "no description". Python tests `v is None or v == ""`, so
+ /// only null and the empty string qualify: `0` and `false` are real descriptions and get
+ /// rendered.
+ fn is_blank_criterion(value: &Value) -> bool {
+ matches!(value, Value::Null) || matches!(value, Value::String(s) if s.is_empty())
+ }
+
+ /// A literal mask token anywhere in user input would forge a marker the scorer then reads as
+ /// an option, so every fragment has it replaced with a space before encoding.
+ fn scrub(text: &str, mask_token: &str) -> String {
+ if mask_token.is_empty() {
+ text.to_string()
+ } else {
+ text.replace(mask_token, " ")
+ }
+ }
+
+ /// Python's `//`: floor division, rounding toward negative infinity. `b` is always `>= 1` at
+ /// every call site here, so this is equivalent to (and implemented with) `div_euclid`.
+ fn floor_div(a: i64, b: i64) -> i64 {
+ a.div_euclid(b)
+ }
+
+ /// Build the token sequence and marker positions for one question.
+ ///
+ /// `tokenizer` must encode without special tokens. `state` is the value being judged;
+ /// `max_len` is the total sequence budget; `head_max_len` is the budget for instructions plus
+ /// option fragments; `truncate_left` keeps the end of an over-long state rather than its
+ /// start. `LayaEngine::predict_value` sets this to `true` exactly when `state` is a JSON
+ /// array, matching `Agent._encode_state`'s `truncate_left = isinstance(state, list)` (laya
+ /// 0.3.21): a conversation-turn list is serialized newest-last, so right-truncating it would
+ /// drop the newest turn instead of the oldest one.
+ ///
+ /// Returns the token ids and one marker position per option. Markers landing at or past
+ /// `max_len` are dropped — a short marker list means options did not fit, which the caller
+ /// must treat as an error rather than answering a truncated question; this function itself
+ /// never errors on that condition, matching `SequenceBuilder.Build` exactly.
+ pub fn build(
+ tokenizer: &dyn LayaTokenizer,
+ state: &Value,
+ question: &Question,
+ max_len: usize,
+ head_max_len: usize,
+ truncate_left: bool,
+ ) -> Result<(Vec<u32>, Vec<usize>)> {
+ let special = tokenizer.special();
+ let mask_token = special.mask_token.as_str();
+
+ let options = Self::render_options(question);
+ let instructions = Self::scrub(&Self::render_instructions(question), mask_token);
+ let header_text = format!(
+ "{} question: {instructions}",
+ Self::type_name(question.question_type())
+ );
+ let mut header_ids: Vec<u32> = tokenizer.encode(&header_text)?;
+
+ let mut option_ids: Vec<Vec<u32>> = Vec::with_capacity(options.len());
+ for option in &options {
+ let encoded = tokenizer.encode(&format!(" {}", Self::scrub(option, mask_token)))?;
+ let take = Self::MAX_OPTION_TOKENS.min(encoded.len());
+ let mut fragment = Vec::with_capacity(take + 1);
+ fragment.push(special.mask_id);
+ fragment.extend_from_slice(&encoded[..take]);
+ option_ids.push(fragment);
+ }
+
+ let option_len_sum =
+ |opts: &[Vec<u32>]| -> i64 { opts.iter().map(|o| o.len() as i64).sum() };
+
+ let mut opt_budget = head_max_len as i64 - option_len_sum(&option_ids);
+ if opt_budget < 16 {
+ let per = 4i64.max(Self::floor_div(
+ head_max_len as i64 - 16,
+ 1i64.max(option_ids.len() as i64),
+ )) as usize;
+ for fragment in &mut option_ids {
+ if fragment.len() > per {
+ fragment.truncate(per);
+ }
+ }
+ opt_budget = head_max_len as i64 - option_len_sum(&option_ids);
+ }
+
+ let head_take = (Self::MIN_HEAD_TOKENS as i64).max(opt_budget) as usize;
+ if header_ids.len() > head_take {
+ header_ids.truncate(head_take);
+ }
+
+ let mut ids: Vec<u32> = Vec::with_capacity(max_len);
+ ids.push(special.cls_id);
+ ids.extend_from_slice(&header_ids);
+ ids.push(special.sep_id);
+
+ let mut markers = Vec::with_capacity(option_ids.len());
+ for fragment in &option_ids {
+ markers.push(ids.len());
+ ids.extend_from_slice(fragment);
+ }
+ ids.push(special.sep_id);
+
+ // -1 reserves the final SEP that closes the whole sequence, appended below.
+ let room = (max_len as i64 - ids.len() as i64 - 1).max(0) as usize;
+ let state_text = Self::scrub(&PythonJson::state(state), mask_token);
+ let state_ids = tokenizer.encode(&state_text)?;
+ let sliced: &[u32] = if truncate_left {
+ // Python's `state_ids[max(0, len(state_ids) - room):]` (laya 0.3.21): newest tokens
+ // kept, and with no room left none of the state (0.3.6's `st[-room:]` kept all of
+ // it). `LayaEngine::predict_value` reaches this branch for a JSON-array state (a
+ // conversation-turn list).
+ &state_ids[state_ids.len().saturating_sub(room)..]
+ } else {
+ &state_ids[..room.min(state_ids.len())]
+ };
+ ids.extend_from_slice(sliced);
+ ids.push(special.sep_id);
+
+ if ids.len() > max_len {
+ ids.truncate(max_len);
+ }
+ markers.retain(|&m| m < max_len);
+
+ Ok((ids, markers))
+ }
+}
diff --git a/laya-rust/tests/collator_tests.rs b/laya-rust/tests/collator_tests.rs
new file mode 100644
index 00000000..b1a84dbe
--- /dev/null
+++ b/laya-rust/tests/collator_tests.rs
@@ -0,0 +1,233 @@
+//! Tier 1: batch padding, which needs no artifacts.
+
+mod common;
+
+use common::golden_data::{self, CheckpointGoldenData};
+use laya::collator::{Collator, SequenceItem};
+use laya::questions::QuestionType;
+
+fn item(length: usize, markers: usize, qtype: QuestionType) -> SequenceItem {
+ SequenceItem::new(
+ (10..10 + length as u32).collect(),
+ (1..=markers).collect(),
+ qtype,
+ )
+}
+
+#[test]
+fn pads_to_the_longest_row_and_marks_attention_accordingly() {
+ let batch = Collator::collate(
+ &[
+ item(5, 2, QuestionType::Choice),
+ item(3, 2, QuestionType::Choice),
+ ],
+ 0,
+ )
+ .unwrap();
+
+ assert_eq!(2, batch.count);
+ assert_eq!(5, batch.sequence_length);
+ assert_eq!(vec![1, 1, 1, 1, 1, 1, 1, 1, 0, 0], batch.attention_mask);
+ // The short row's tail is the pad id, and attention is 0 there.
+ assert_eq!(0, batch.input_ids[8]);
+ assert_eq!(0, batch.input_ids[9]);
+}
+
+#[test]
+fn fills_with_the_given_pad_id_rather_than_zero() {
+ let batch = Collator::collate(
+ &[
+ item(3, 2, QuestionType::Choice),
+ item(1, 2, QuestionType::Choice),
+ ],
+ 77,
+ )
+ .unwrap();
+ assert_eq!(77, batch.input_ids[4]);
+ assert_eq!(77, batch.input_ids[5]);
+ assert_eq!(0, batch.attention_mask[4]);
+}
+
+#[test]
+fn counts_only_real_tokens_as_input_tokens() {
+ let batch = Collator::collate(
+ &[
+ item(5, 2, QuestionType::Choice),
+ item(3, 2, QuestionType::Choice),
+ ],
+ 0,
+ )
+ .unwrap();
+ assert_eq!(8, batch.input_tokens);
+}
+
+#[test]
+fn widens_the_marker_axis_to_two_for_single_option_batches() {
+ // The exported graph has TopK k=2 baked in: a one-column marker axis makes it fail outright.
+ let batch = Collator::collate(&[item(6, 1, QuestionType::Choice)], 0).unwrap();
+
+ assert_eq!(Collator::MIN_MARKERS, batch.marker_count);
+ assert_eq!(vec![1, 0], batch.marker_pos);
+ assert_eq!(vec![true, false], batch.marker_mask);
+ // The real option count is kept separately, so the pad column never reaches an answer.
+ assert_eq!(vec![1], batch.marker_counts);
+}
+
+#[test]
+fn pads_the_marker_axis_to_the_widest_question() {
+ let batch = Collator::collate(
+ &[
+ item(6, 4, QuestionType::Choice),
+ item(6, 2, QuestionType::Choice),
+ ],
+ 0,
+ )
+ .unwrap();
+
+ assert_eq!(4, batch.marker_count);
+ assert_eq!(vec![1, 2, 3, 4, 1, 2, 0, 0], batch.marker_pos);
+ assert_eq!(
+ vec![true, true, true, true, true, true, false, false],
+ batch.marker_mask
+ );
+ assert_eq!(vec![4, 2], batch.marker_counts);
+}
+
+#[test]
+fn carries_the_question_type_per_row_using_pythons_numbering() {
+ let batch = Collator::collate(
+ &[
+ item(4, 2, QuestionType::Noul),
+ item(4, 2, QuestionType::Choice),
+ item(4, 2, QuestionType::Score),
+ ],
+ 0,
+ )
+ .unwrap();
+
+ assert_eq!(vec![2, 0, 1], batch.qtype);
+}
+
+#[test]
+fn refuses_an_empty_batch() {
+ assert!(Collator::collate(&[], 0).is_err());
+}
+
+#[test]
+fn reproduces_the_recorded_batch_tensors() {
+ let data = CheckpointGoldenData::for_checkpoint("multilingual");
+ if !data.available() {
+ eprintln!("skip: no golden data for 'multilingual'");
+ return;
+ }
+
+ let pad_id = data.meta()["special_tokens"]["pad_id"].as_u64().unwrap() as u32;
+
+ for case in data.success_cases() {
+ let root = data.load_case(&case);
+ let expected = &root["batch"];
+
+ // Rebuild the batch from the recorded per-question ids, so this checks the collator alone
+ // and stays meaningful without a tokenizer present.
+ let items: Vec<SequenceItem> = root["per_question"]
+ .as_array()
+ .unwrap()
+ .iter()
+ .map(|q| {
+ let ids: Vec<u32> = golden_data::ints(&q["input_ids"])
+ .into_iter()
+ .map(|v| v as u32)
+ .collect();
+ let markers = golden_data::indices(&q["markers"]);
+ let qtype = QuestionType::try_from(q["qtype"].as_i64().unwrap())
+ .expect("valid qtype in golden data");
+ SequenceItem::new(ids, markers, qtype)
+ })
+ .collect();
+
+ let widest_markers = items.iter().map(|it| it.markers.len()).max().unwrap_or(0);
+ let batch = Collator::collate(&items, pad_id).unwrap_or_else(|e| {
+ panic!("case '{case}': collate failed: {e}");
+ });
+
+ assert_matrix(
+ &expected["input_ids"],
+ &batch.input_ids,
+ batch.sequence_length,
+ &case,
+ );
+ assert_matrix(
+ &expected["attention_mask"],
+ &batch.attention_mask,
+ batch.sequence_length,
+ &case,
+ );
+ assert_matrix(
+ &expected["marker_pos"],
+ &batch.marker_pos,
+ batch.marker_count,
+ &case,
+ );
+
+ let expected_qtype = golden_data::ints(&expected["qtype"]);
+ assert_eq!(expected_qtype, batch.qtype, "case '{case}': qtype mismatch");
+
+ let expected_mask = expected["marker_mask"].as_array().unwrap();
+ assert_eq!(
+ expected_mask.len(),
+ batch.count,
+ "case '{case}': marker_mask row count"
+ );
+ for (r, row) in expected_mask.iter().enumerate() {
+ let row = row.as_array().unwrap();
+ for (c, expected) in row.iter().enumerate().take(batch.marker_count) {
+ assert_eq!(
+ expected.as_bool().unwrap(),
+ batch.marker_mask[r * batch.marker_count + c],
+ "case '{case}': marker_mask[{r}][{c}]"
+ );
+ }
+ }
+
+ let expected_input_tokens =
+ root["result"]["usage"]["input_tokens"].as_u64().unwrap() as usize;
+ assert_eq!(
+ expected_input_tokens, batch.input_tokens,
+ "case '{case}': input_tokens"
+ );
+
+ // marker_pos above already pinned the padded width; this checks the flag the dump script
+ // set, so a future golden that stops exercising the TopK edge case is noticed.
+ let expected_padded = root["marker_axis_padded"].as_bool().unwrap();
+ assert_eq!(
+ expected_padded,
+ batch.marker_count > widest_markers,
+ "case '{case}': marker_axis_padded flag"
+ );
+ }
+}
+
+fn assert_matrix(
+ expected: &serde_json::Value,
+ actual: &[i64],
+ cols: usize,
+ case: &golden_data::GoldenCaseInfo,
+) {
+ let rows = expected.as_array().unwrap();
+ assert_eq!(
+ rows.len() * cols,
+ actual.len(),
+ "case '{case}': matrix size"
+ );
+ for (r, row) in rows.iter().enumerate() {
+ let row = row.as_array().unwrap();
+ assert_eq!(cols, row.len(), "case '{case}': row {r} width");
+ for (c, value) in row.iter().enumerate() {
+ assert_eq!(
+ value.as_i64().unwrap(),
+ actual[r * cols + c],
+ "case '{case}': [{r}][{c}]"
+ );
+ }
+ }
+}
diff --git a/laya-rust/tests/golden_questions_tests.rs b/laya-rust/tests/golden_questions_tests.rs
new file mode 100644
index 00000000..c0938c51
--- /dev/null
+++ b/laya-rust/tests/golden_questions_tests.rs
@@ -0,0 +1,74 @@
+//! Verifies that rebuilding a golden case's `questions` into a [`QuestionSet`] and rendering it
+//! with [`SequenceBuilder`] reproduces exactly what the recorded `per_question` entries show —
+//! i.e. that [`Question`] construction, [`SequenceBuilder::render_options`] and
+//! [`SequenceBuilder::render_instructions`] agree with the real Python `_to_internal` /
+//! `render_options` pipeline, not just with each other. Also checks `serialized_state` against
+//! [`PythonJson::state`]. This is a Rust-specific addition.
+
+mod common;
+
+use common::golden_data::{self, CheckpointGoldenData};
+use laya::python_json::PythonJson;
+use laya::sequence_builder::SequenceBuilder;
+use serde_json::Value;
+
+const CHECKPOINTS: [&str; 3] = ["multilingual", "english", "typed-decisions"];
+
+#[test]
+fn golden_questions_render_exactly_as_recorded() {
+ let mut checked_any = false;
+
+ for checkpoint in CHECKPOINTS {
+ let data = CheckpointGoldenData::for_checkpoint(checkpoint);
+ if !data.available() {
+ eprintln!("skip: no golden data for '{checkpoint}'");
+ continue;
+ }
+
+ for case in data.success_cases() {
+ checked_any = true;
+ let root = data.load_case(&case);
+
+ let question_set = golden_data::build_questions(&root["questions"])
+ .unwrap_or_else(|e| panic!("{checkpoint}/{case}: failed to build questions: {e}"));
+
+ for per_question in root["per_question"].as_array().unwrap() {
+ let id = per_question["id"].as_str().unwrap();
+ let question = question_set
+ .get(id)
+ .unwrap_or_else(|| panic!("{checkpoint}/{case}: missing question '{id}'"));
+
+ let expected_options: Vec<&str> = per_question["options"]
+ .as_array()
+ .unwrap()
+ .iter()
+ .map(|v| v.as_str().unwrap())
+ .collect();
+ let actual_options = SequenceBuilder::render_options(question);
+ assert_eq!(
+ expected_options, actual_options,
+ "{checkpoint}/{case}: question '{id}' options"
+ );
+
+ let expected_instructions = per_question["instructions"].as_str().unwrap();
+ let actual_instructions = SequenceBuilder::render_instructions(question);
+ assert_eq!(
+ expected_instructions, actual_instructions,
+ "{checkpoint}/{case}: question '{id}' instructions"
+ );
+ }
+
+ if let Some(expected_state) = root.get("serialized_state").and_then(Value::as_str) {
+ let actual_state = PythonJson::state(&root["state"]);
+ assert_eq!(
+ expected_state, actual_state,
+ "{checkpoint}/{case}: serialized_state vs PythonJson::state"
+ );
+ }
+ }
+ }
+
+ if !checked_any {
+ eprintln!("skip: no golden data found for any checkpoint");
+ }
+}
diff --git a/laya-rust/tests/sequence_builder_tests.rs b/laya-rust/tests/sequence_builder_tests.rs
new file mode 100644
index 00000000..64d6e22b
--- /dev/null
+++ b/laya-rust/tests/sequence_builder_tests.rs
@@ -0,0 +1,300 @@
+//! Tier 1: sequence layout and the option budget, using [`StubTokenizer`] so no artifacts are
+//! needed. Real token ids are a Phase 2 tokenizer-parity concern.
+
+mod common;
+
+use common::stub_tokenizer::StubTokenizer;
+use laya::questions::{Question, QuestionType};
+use laya::sequence_builder::SequenceBuilder;
+use laya::tokenization::LayaTokenizer;
+use serde_json::{Value, json};
+
+#[test]
+fn lays_out_cls_head_sep_options_sep_state_sep() {
+ let tok = StubTokenizer::new();
+ let q = Question::choice("pick one", [("a", Value::Null), ("b", Value::Null)]).unwrap();
+ let (ids, markers) =
+ SequenceBuilder::build(&tok, &json!("state text"), &q, 64, 32, false).unwrap();
+
+ assert_eq!(tok.special().cls_id, ids[0]);
+ assert_eq!(tok.special().sep_id, *ids.last().unwrap());
+ // Each marker is a mask token, and the two options are adjacent fragments.
+ assert_eq!(2, markers.len());
+ for &m in &markers {
+ assert_eq!(tok.special().mask_id, ids[m]);
+ }
+ assert!(markers[1] > markers[0]);
+ // A SEP closes the head, immediately before the first marker.
+ assert_eq!(tok.special().sep_id, ids[markers[0] - 1]);
+}
+
+#[test]
+fn marker_positions_point_at_consecutive_option_fragments() {
+ let tok = StubTokenizer::new();
+ // One word per option keeps the arithmetic checkable: mask + 1 word = 2 ids per fragment.
+ let q = Question::choice(
+ "h",
+ [
+ ("alpha", Value::Null),
+ ("beta", Value::Null),
+ ("gamma", Value::Null),
+ ],
+ )
+ .unwrap();
+ let (ids, markers) = SequenceBuilder::build(&tok, &json!("s"), &q, 64, 32, false).unwrap();
+
+ assert_eq!(3, markers.len());
+ assert_eq!(markers[0] + 2, markers[1]);
+ assert_eq!(markers[1] + 2, markers[2]);
+ assert_eq!(tok.special().sep_id, ids[markers[2] + 2]);
+}
+
+#[test]
+fn puts_the_type_name_in_the_head_text() {
+ let tok = StubTokenizer::new();
+ let q = Question::noul("does it hold", Value::Null, Value::Null);
+ SequenceBuilder::build(&tok, &json!("s"), &q, 64, 32, false).unwrap();
+ assert_eq!("noul question: does it hold", tok.encoded()[0]);
+}
+
+#[test]
+fn type_names_match_python() {
+ assert_eq!("choice", SequenceBuilder::type_name(QuestionType::Choice));
+ assert_eq!("score", SequenceBuilder::type_name(QuestionType::Score));
+ assert_eq!("noul", SequenceBuilder::type_name(QuestionType::Noul));
+}
+
+#[test]
+fn option_fragments_are_encoded_with_a_leading_space() {
+ let tok = StubTokenizer::new();
+ let q = Question::choice("h", [("a", Value::Null)]).unwrap();
+ SequenceBuilder::build(&tok, &json!("s"), &q, 64, 32, false).unwrap();
+ // Python encodes " " + option, which the Metaspace pre-tokenizer turns into a word boundary.
+ assert!(tok.encoded().iter().any(|s| s == " a"));
+}
+
+#[test]
+fn renders_options_the_way_python_does() {
+ assert_eq!(
+ vec!["a: first".to_string(), "b".to_string()],
+ SequenceBuilder::render_options(
+ &Question::choice("h", [("a", json!("first")), ("b", Value::Null)]).unwrap()
+ )
+ );
+
+ assert_eq!(
+ vec!["level 0: low".to_string(), "level 1: high".to_string()],
+ SequenceBuilder::render_options(&Question::score("h", ["low", "high"]).unwrap())
+ );
+
+ assert_eq!(
+ vec![
+ "false: no, the statement does not hold".to_string(),
+ "true: yes, the statement holds".to_string()
+ ],
+ SequenceBuilder::render_options(&Question::noul("h", Value::Null, Value::Null))
+ );
+
+ assert_eq!(
+ vec!["false: all fine".to_string(), "true: on fire".to_string()],
+ SequenceBuilder::render_options(&Question::noul("h", "all fine", "on fire"))
+ );
+}
+
+#[test]
+fn treats_only_null_and_empty_string_as_a_missing_description() {
+ // Python tests `v is None or v == ""`, so 0 and false are real descriptions.
+ let q = Question::choice(
+ "h",
+ [
+ ("zero", json!(0)),
+ ("no", json!(false)),
+ ("blank", json!("")),
+ ("none", Value::Null),
+ ],
+ )
+ .unwrap();
+ assert_eq!(
+ vec![
+ "zero: 0".to_string(),
+ "no: false".to_string(),
+ "blank".to_string(),
+ "none".to_string()
+ ],
+ SequenceBuilder::render_options(&q)
+ );
+}
+
+#[test]
+fn scrubs_a_mask_token_out_of_every_fragment() {
+ let tok = StubTokenizer::new();
+ let mask = &tok.special().mask_token;
+ // A literal <mask> in user input would forge a marker the scorer then reads as an option.
+ let q = Question::choice(
+ format!("is this {mask} safe"),
+ [(format!("a{mask}b"), Value::Null)],
+ )
+ .unwrap();
+ SequenceBuilder::build(
+ &tok,
+ &json!(format!("state {mask} here")),
+ &q,
+ 64,
+ 32,
+ false,
+ )
+ .unwrap();
+
+ for text in tok.encoded() {
+ assert!(
+ !text.contains(mask.as_str()),
+ "encoded text still contains the mask token: {text:?}"
+ );
+ }
+}
+
+#[test]
+fn truncates_long_options_to_forty_eight_tokens() {
+ let tok = StubTokenizer::new();
+ let words: Vec<String> = (0..120).map(|i| format!("w{i}")).collect();
+ let q = Question::choice("h", [("opt", json!(words.join(" ")))]).unwrap();
+ let (ids, markers) = SequenceBuilder::build(&tok, &json!("s"), &q, 512, 256, false).unwrap();
+
+ // The only fragment is mask + at most 48 option tokens, then the closing SEP. "opt: " adds two
+ // words, so 48 is the cap that bites rather than the option running out.
+ assert_eq!(1, markers.len());
+ let fragment_end = ids[markers[0]..]
+ .iter()
+ .position(|&id| id == tok.special().sep_id)
+ .unwrap()
+ + markers[0];
+ assert_eq!(markers[0] + 49, fragment_end);
+}
+
+#[test]
+fn shrinks_options_when_they_would_eat_the_whole_head_budget() {
+ let tok = StubTokenizer::new();
+ // 20 options of ~10 tokens each against a 96-token head budget forces the `opt_budget < 16`
+ // branch, which caps every fragment at `(head_max_len - 16) // n`.
+ let options: Vec<(String, Value)> = (0..20)
+ .map(|i| {
+ let desc = (0..10)
+ .map(|j| format!("d{i}_{j}"))
+ .collect::<Vec<_>>()
+ .join(" ");
+ (format!("opt{i}"), json!(desc))
+ })
+ .collect();
+ let q = Question::choice("head words here", options).unwrap();
+ let (ids, markers) = SequenceBuilder::build(&tok, &json!("s"), &q, 512, 96, false).unwrap();
+
+ assert_eq!(20, markers.len());
+ // Mirrors the production formula `max(4, (head_max_len - 16) // n)`; with these numbers it
+ // happens to always pick the 4-token floor.
+ #[allow(clippy::unnecessary_min_or_max)]
+ let per = 4.max((96 - 16) / 20);
+ for i in 1..markers.len() {
+ let width = markers[i] - markers[i - 1];
+ assert!(
+ width <= per,
+ "fragment {} is {width} tokens, above the {per}-token cap",
+ i - 1
+ );
+ }
+ assert_eq!(tok.special().cls_id, ids[0]);
+}
+
+#[test]
+fn keeps_at_least_eight_head_tokens_when_options_dominate() {
+ let tok = StubTokenizer::new();
+ let options: Vec<(String, Value)> = (0..30)
+ .map(|i| (format!("o{i}"), json!("a b c d e f g h")))
+ .collect();
+ let q = Question::choice("one two three four five six seven eight nine ten", options).unwrap();
+ let (ids, markers) = SequenceBuilder::build(&tok, &json!("s"), &q, 512, 64, false).unwrap();
+
+ // CLS + head + SEP, so the first marker cannot be earlier than position 2.
+ assert!(markers[0] >= 2);
+ assert_eq!(tok.special().sep_id, ids[markers[0] - 1]);
+}
+
+#[test]
+fn truncates_the_state_to_fit_max_len() {
+ let tok = StubTokenizer::new();
+ let state: Vec<String> = (0..500).map(|i| format!("s{i}")).collect();
+ let q = Question::noul("h", Value::Null, Value::Null);
+ let (ids, markers) =
+ SequenceBuilder::build(&tok, &json!(state.join(" ")), &q, 64, 32, false).unwrap();
+
+ assert_eq!(64, ids.len());
+ assert_eq!(tok.special().sep_id, *ids.last().unwrap());
+ assert_eq!(2, markers.len());
+}
+
+#[test]
+fn keeps_the_state_tail_when_truncating_left() {
+ let tok = StubTokenizer::new();
+ let state: Vec<String> = (0..200).map(|i| format!("s{i}")).collect();
+ let state = json!(state.join(" "));
+ let q = Question::noul("h", Value::Null, Value::Null);
+ let (head_ids, head_markers) = SequenceBuilder::build(&tok, &state, &q, 64, 32, false).unwrap();
+ let (tail_ids, tail_markers) = SequenceBuilder::build(&tok, &state, &q, 64, 32, true).unwrap();
+
+ assert_eq!(head_ids.len(), tail_ids.len());
+ assert_ne!(head_ids, tail_ids);
+ // Both keep the same prefix up to the state, and differ from there on.
+ assert_eq!(head_markers, tail_markers);
+}
+
+#[test]
+fn left_truncation_with_no_room_drops_the_whole_state() {
+ // laya 0.3.21 slices `state_ids[max(0, len - room):]`, so with room == 0 no state token
+ // survives. 0.3.6's `st[-room:]` kept the whole state there (`x[-0:]` is all of `x`).
+ let tok = StubTokenizer::new();
+ let q = Question::noul("h", Value::Null, Value::Null);
+ let (empty_ids, _) = SequenceBuilder::build(&tok, &json!(""), &q, 512, 32, false).unwrap();
+ let max_len = empty_ids.len();
+
+ let state: Vec<String> = (0..50).map(|i| format!("s{i}")).collect();
+ let (ids, _) =
+ SequenceBuilder::build(&tok, &json!(state.join(" ")), &q, max_len, 32, true).unwrap();
+
+ assert_eq!(empty_ids, ids);
+}
+
+#[test]
+fn drops_markers_that_would_land_past_max_len() {
+ let tok = StubTokenizer::new();
+ // 40 options cannot fit a 32-token sequence, so markers are lost. A later `predict` phase must
+ // treat a short marker list as an error rather than answering a question missing options.
+ let options: Vec<(String, Value)> = (0..40).map(|i| (format!("o{i}"), json!("desc"))).collect();
+ let option_count = options.len();
+ let q = Question::choice("h", options).unwrap();
+ let (ids, markers) = SequenceBuilder::build(&tok, &json!("s"), &q, 32, 256, false).unwrap();
+
+ assert_eq!(32, ids.len());
+ assert!(markers.len() < option_count);
+ for &m in &markers {
+ assert!(m < 32);
+ }
+}
+
+#[test]
+fn renders_non_string_instructions_with_literal_unicode() {
+ // laya 0.3.21: json.dumps(..., ensure_ascii=False), so non-ASCII stays literal rather than
+ // coming out as \uXXXX (0.3.6's behavior, which the tokenizer then read back as escape text).
+ let q = Question::noul(json!({"hint": "Müller"}), Value::Null, Value::Null);
+ let rendered = SequenceBuilder::render_instructions(&q);
+
+ assert!(rendered.contains('ü'));
+ assert!(!rendered.contains("\\u"));
+}
+
+#[test]
+fn passes_string_instructions_through_untouched() {
+ let q = Question::noul("Müller & co <urgent>", Value::Null, Value::Null);
+ assert_eq!(
+ "Müller & co <urgent>",
+ SequenceBuilder::render_instructions(&q)
+ );
+}