Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
feat(rust): add typed questions, sequence builder and collator
kevin-gatimucd39e6b61e9b
diff --git a/laya-rust/src/collator.rs b/laya-rust/src/collator.rsnew file mode 100644index 00000000..74c3dbdb--- /dev/null+++ b/laya-rust/src/collator.rs@@ -0,0 +1,120 @@+//! Pads tokenized questions into one batch. Ports `collate_items` from `laya/common.py`. The+//! padded tensors match the input shapes the ONNX graph expects.++use crate::error::{LayaError, Result};+use crate::questions::QuestionType;++/// One question, tokenized and ready to be batched.+#[derive(Debug, Clone)]+pub struct SequenceItem {+ /// The token sequence.+ pub ids: Vec<u32>,+ /// One position per option, pointing at that option's `[MASK]`.+ pub markers: Vec<usize>,+ /// Which question type, fed to the model's type embedding.+ pub qtype: QuestionType,+}++impl SequenceItem {+ /// Build a sequence item from its parts.+ pub fn new(ids: Vec<u32>, markers: Vec<usize>, qtype: QuestionType) -> Self {+ Self {+ ids,+ markers,+ qtype,+ }+ }+}++/// A padded batch, laid out as the row-major tensors the ONNX graph takes.+#[derive(Debug, Clone)]+pub struct CollatedBatch {+ /// Rows in the batch: one per question.+ pub count: usize,+ /// Padded sequence length, the longest row.+ pub sequence_length: usize,+ /// Padded marker-axis width.+ pub marker_count: usize,+ /// `[count, sequence_length]`, padded with the pad id.+ pub input_ids: Vec<i64>,+ /// `[count, sequence_length]`, 1 on real tokens and 0 on padding.+ pub attention_mask: Vec<i64>,+ /// `[count, marker_count]`, 0 in padding columns.+ pub marker_pos: Vec<i64>,+ /// `[count, marker_count]`, false in padding columns.+ pub marker_mask: Vec<bool>,+ /// `[count]`, the question type per row.+ pub qtype: Vec<i64>,+ /// The real (unpadded) option count per row — how many logits to read from that row.+ pub marker_counts: Vec<usize>,+ /// Non-padding tokens across the batch, which is what Python reports as input tokens.+ pub input_tokens: usize,+}++/// Namespace for batch collation. See the module docs for the source this ports.+pub struct Collator;++impl Collator {+ /// The marker axis is never narrower than this.+ ///+ /// The tracer baked `TopK k=2` into the exported graph: Python chooses the top-2 branch at+ /// runtime and the export only ever saw two or more markers, so a batch whose questions all+ /// have a single option makes TopK fail outright. Padding to two columns with `marker_mask =+ /// false` is exactly equivalent, not merely close: the model's `masked_fill` puts `-1e4` in+ /// the pad column, which underflows softmax to 0.0, so `p == [1.0, 0.0]` — precisely the+ /// `[top1, 0.0]` Python substitutes. Reading only [`CollatedBatch::marker_counts`] logits per+ /// row keeps the pad column out of the answer.+ pub const MIN_MARKERS: usize = 2;++ /// Pad a batch of tokenized questions. Returns an error for an empty batch rather than+ /// producing a zero-row tensor the ONNX graph would reject anyway.+ pub fn collate(items: &[SequenceItem], pad_id: u32) -> Result<CollatedBatch> {+ if items.is_empty() {+ return Err(LayaError::InvalidQuestion("nothing to collate".to_string()));+ }++ let n = items.len();+ let length = items.iter().map(|it| it.ids.len()).max().unwrap_or(0);+ let marker_count =+ Self::MIN_MARKERS.max(items.iter().map(|it| it.markers.len()).max().unwrap_or(0));++ let mut input_ids = vec![i64::from(pad_id); n * length];+ let mut attention_mask = vec![0i64; n * length];+ let mut marker_pos = vec![0i64; n * marker_count];+ let mut marker_mask = vec![false; n * marker_count];+ let mut qtype = vec![0i64; n];+ let mut marker_counts = vec![0usize; n];+ let mut input_tokens = 0usize;++ for (i, item) in items.iter().enumerate() {+ let row = i * length;+ for (j, &id) in item.ids.iter().enumerate() {+ input_ids[row + j] = i64::from(id);+ attention_mask[row + j] = 1;+ }+ input_tokens += item.ids.len();++ let mrow = i * marker_count;+ for (j, &pos) in item.markers.iter().enumerate() {+ marker_pos[mrow + j] = pos as i64;+ marker_mask[mrow + j] = true;+ }++ qtype[i] = item.qtype as i64;+ marker_counts[i] = item.markers.len();+ }++ Ok(CollatedBatch {+ count: n,+ sequence_length: length,+ marker_count,+ input_ids,+ attention_mask,+ marker_pos,+ marker_mask,+ qtype,+ marker_counts,+ input_tokens,+ })+ }+}diff --git a/laya-rust/src/questions.rs b/laya-rust/src/questions.rsnew file mode 100644index 00000000..3e763cbc--- /dev/null+++ b/laya-rust/src/questions.rs@@ -0,0 +1,329 @@+//! Typed questions and the insertion-ordered set they are asked in.+//!+//! Ports `laya/presets.py`'s question shape. Order is load-bearing throughout this module: batch+//! row order follows `QuestionSet` order, and choice label order decides which logit each+//! label's probability comes from (see `sequence-construction.md`).++use crate::error::{LayaError, Result};+use indexmap::IndexMap;+use serde_json::Value;+use std::collections::HashSet;++/// The three question types Laya answers. Values match Python's `QTYPES` and the `qtype` tensor+/// encoding (`model-contract.md`).+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]+pub enum QuestionType {+ /// Pick one label from a set of named options.+ Choice = 0,+ /// Place the state on an ordered scale; the answer is an expected level.+ Score = 1,+ /// Does the statement hold? The answer is a probability.+ Noul = 2,+}++// The Python type name (`SequenceBuilder::type_name`) and the `qtype` tensor's integer encoding+// (a plain `qtype as i64` cast, since the discriminants above already match Python's `QTYPES`)+// live in `sequence_builder.rs` alongside the rest of that logic, rather than here, keeping this+// enum bare.++impl TryFrom<i64> for QuestionType {+ type Error = LayaError;++ /// The question type for a `qtype` tensor value. Rust has no implicit enum-from-integer cast,+ /// so this is the explicit equivalent.+ fn try_from(index: i64) -> Result<Self> {+ match index {+ 0 => Ok(QuestionType::Choice),+ 1 => Ok(QuestionType::Score),+ 2 => Ok(QuestionType::Noul),+ other => Err(LayaError::InvalidQuestion(format!(+ "unknown qtype index {other}"+ ))),+ }+ }+}++/// A [`QuestionType::Choice`] question: ordered, named options.+#[derive(Debug, Clone)]+pub struct ChoiceQuestion {+ instructions: Value,+ options: Vec<(String, Value)>,+}++impl ChoiceQuestion {+ /// The prompt: a string verbatim, or anything else rendered through+ /// [`crate::python_json::instructions`].+ pub fn instructions(&self) -> &Value {+ &self.instructions+ }++ /// The options in label order. Index N here is the label for logit N.+ pub fn options(&self) -> &[(String, Value)] {+ &self.options+ }++ /// The labels, in order.+ pub fn labels(&self) -> impl Iterator<Item = &str> {+ self.options.iter().map(|(label, _)| label.as_str())+ }+}++/// A [`QuestionType::Score`] question: an ordered scale, lowest level first.+#[derive(Debug, Clone)]+pub struct ScoreQuestion {+ instructions: Value,+ levels: Vec<Value>,+}++impl ScoreQuestion {+ /// The prompt. See [`ChoiceQuestion::instructions`].+ pub fn instructions(&self) -> &Value {+ &self.instructions+ }++ /// The levels, lowest first. The answer is an expectation over these indices.+ pub fn levels(&self) -> &[Value] {+ &self.levels+ }+}++/// A [`QuestionType::Noul`] question: does the statement hold?+#[derive(Debug, Clone)]+pub struct NoulQuestion {+ instructions: Value,+ if_false: Value,+ if_true: Value,+}++impl NoulQuestion {+ /// The prompt. See [`ChoiceQuestion::instructions`].+ pub fn instructions(&self) -> &Value {+ &self.instructions+ }++ /// Wording for the false option, or `Value::Null` for the default.+ pub fn if_false(&self) -> &Value {+ &self.if_false+ }++ /// Wording for the true option, or `Value::Null` for the default.+ pub fn if_true(&self) -> &Value {+ &self.if_true+ }+}++/// One typed question. Construct with [`Question::choice`], [`Question::choice_labels`],+/// [`Question::score`] or [`Question::noul`].+#[derive(Debug, Clone)]+pub enum Question {+ /// See [`ChoiceQuestion`].+ Choice(ChoiceQuestion),+ /// See [`ScoreQuestion`].+ Score(ScoreQuestion),+ /// See [`NoulQuestion`].+ Noul(NoulQuestion),+}++impl Question {+ /// Which of the three question types this is.+ pub fn question_type(&self) -> QuestionType {+ match self {+ Question::Choice(_) => QuestionType::Choice,+ Question::Score(_) => QuestionType::Score,+ Question::Noul(_) => QuestionType::Noul,+ }+ }++ /// The prompt. A string is used verbatim; anything else is serialized with+ /// [`crate::python_json::instructions`], matching Python's `_to_internal`.+ pub fn instructions(&self) -> &Value {+ match self {+ Question::Choice(c) => c.instructions(),+ Question::Score(s) => s.instructions(),+ Question::Noul(n) => n.instructions(),+ }+ }++ /// This question as a choice question.+ pub fn as_choice(&self) -> Option<&ChoiceQuestion> {+ match self {+ Question::Choice(c) => Some(c),+ _ => None,+ }+ }++ /// This question as a score question.+ pub fn as_score(&self) -> Option<&ScoreQuestion> {+ match self {+ Question::Score(s) => Some(s),+ _ => None,+ }+ }++ /// This question as a noul question.+ pub fn as_noul(&self) -> Option<&NoulQuestion> {+ match self {+ Question::Noul(n) => Some(n),+ _ => None,+ }+ }++ /// A choice question over ordered `(label, description)` options. A `Value::Null` description+ /// means "no description"; note that `0` and `false` are real descriptions and are rendered+ /// (see [`crate::python_json::is_blank_criterion`]).+ ///+ /// Rejects duplicate labels and an empty option list: Python would silently collapse+ /// duplicate keys into one option, changing the option count and so every marker position, so+ /// this refuses rather than silently answering a different question.+ pub fn choice<I, L, D>(instructions: impl Into<Value>, options: I) -> Result<Question>+ where+ I: IntoIterator<Item = (L, D)>,+ L: Into<String>,+ D: Into<Value>,+ {+ let mut list = Vec::new();+ let mut seen = HashSet::new();+ for (label, description) in options {+ let label = label.into();+ if !seen.insert(label.clone()) {+ return Err(LayaError::InvalidQuestion(format!(+ "duplicate choice label '{label}'"+ )));+ }+ list.push((label, description.into()));+ }+ if list.is_empty() {+ return Err(LayaError::InvalidQuestion(+ "a choice question needs at least one option".to_string(),+ ));+ }+ Ok(Question::Choice(ChoiceQuestion {+ instructions: instructions.into(),+ options: list,+ }))+ }++ /// A choice question over bare labels, with no descriptions.+ pub fn choice_labels<I, L>(instructions: impl Into<Value>, labels: I) -> Result<Question>+ where+ I: IntoIterator<Item = L>,+ L: Into<String>,+ {+ Question::choice(+ instructions,+ labels.into_iter().map(|label| (label, Value::Null)),+ )+ }++ /// A score question over ordered levels, lowest first. Rejects an empty level list.+ pub fn score<I, D>(instructions: impl Into<Value>, levels: I) -> Result<Question>+ where+ I: IntoIterator<Item = D>,+ D: Into<Value>,+ {+ let list: Vec<Value> = levels.into_iter().map(Into::into).collect();+ if list.is_empty() {+ return Err(LayaError::InvalidQuestion(+ "a score question needs at least one level".to_string(),+ ));+ }+ Ok(Question::Score(ScoreQuestion {+ instructions: instructions.into(),+ levels: list,+ }))+ }++ /// A yes/no question. Supplying `if_false` / `if_true` (e.g. `Some("on fire")`, or `None::<Value>`+ /// for the default) replaces the default "no, the statement does not hold" / "yes, the+ /// statement holds" wording.+ pub fn noul(+ instructions: impl Into<Value>,+ if_false: impl Into<Value>,+ if_true: impl Into<Value>,+ ) -> Question {+ Question::Noul(NoulQuestion {+ instructions: instructions.into(),+ if_false: if_false.into(),+ if_true: if_true.into(),+ })+ }+}++/// An insertion-ordered set of question id to [`Question`], answered in one batch.+///+/// Deliberately backed by [`IndexMap`] rather than a hash map: the batch row order must match the+/// order the caller wrote the questions in, because that is what Python's `dict` guarantees and+/// what the answer order is compared against.+#[derive(Debug, Clone, Default)]+pub struct QuestionSet {+ map: IndexMap<String, Question>,+}++impl QuestionSet {+ /// An empty set.+ pub fn new() -> Self {+ Self {+ map: IndexMap::new(),+ }+ }++ /// How many questions are in the set.+ pub fn len(&self) -> usize {+ self.map.len()+ }++ /// Whether the set has no questions.+ pub fn is_empty(&self) -> bool {+ self.map.is_empty()+ }++ /// Insert a question by id. Returns an error if the id is already present, rather than+ /// silently overwriting it and shifting every subsequent row's meaning.+ pub fn insert(&mut self, id: impl Into<String>, question: Question) -> Result<()> {+ let id = id.into();+ if self.map.contains_key(&id) {+ return Err(LayaError::InvalidQuestion(format!(+ "duplicate question id '{id}'"+ )));+ }+ self.map.insert(id, question);+ Ok(())+ }++ /// Builder-style [`insert`](Self::insert): `QuestionSet::new().with("a", q1)?.with("b", q2)?`.+ pub fn with(mut self, id: impl Into<String>, question: Question) -> Result<Self> {+ self.insert(id, question)?;+ Ok(self)+ }++ /// The question with this id, if any.+ pub fn get(&self, id: &str) -> Option<&Question> {+ self.map.get(id)+ }++ /// Whether the set contains this id.+ pub fn contains_id(&self, id: &str) -> bool {+ self.map.contains_key(id)+ }++ /// The question ids, in insertion order.+ pub fn ids(&self) -> impl Iterator<Item = &str> {+ self.map.keys().map(String::as_str)+ }++ /// Every `(id, question)` pair, in insertion order.+ pub fn iter(&self) -> impl Iterator<Item = (&str, &Question)> {+ self.map.iter().map(|(id, q)| (id.as_str(), q))+ }+}++impl std::ops::Index<&str> for QuestionSet {+ type Output = Question;++ /// Panics if no question carries this id; use [`QuestionSet::get`] for a fallible lookup.+ fn index(&self, id: &str) -> &Question {+ self.map+ .get(id)+ .unwrap_or_else(|| panic!("no question with id '{id}'"))+ }+}diff --git a/laya-rust/src/sequence_builder.rs b/laya-rust/src/sequence_builder.rsnew file mode 100644index 00000000..9f1e8bc7--- /dev/null+++ b/laya-rust/src/sequence_builder.rs@@ -0,0 +1,206 @@+//! Builds the typed token sequence one question is answered from. Ports `build_sequence`,+//! `render_options` and `render_criterion` from `laya/common.py`, step by step.+//!+//! The layout is `[CLS] <type> question: instructions [SEP] [MASK] opt0 [MASK] opt1 ... [SEP]+//! state [SEP]`. The scorer reads the hidden state at each `[MASK]`, so a marker position that is+//! off by one token attaches an option's probability to the wrong place in the sequence and the+//! model answers a different question without anything failing. Every offset here is therefore+//! load-bearing.++use crate::error::Result;+use crate::python_json::PythonJson;+use crate::questions::Question;+use crate::tokenization::LayaTokenizer;+use serde_json::Value;++/// Namespace for sequence construction. See the module docs for the source this ports.+pub struct SequenceBuilder;++impl SequenceBuilder {+ /// One option fragment is capped at this many tokens, before the marker is prepended.+ const MAX_OPTION_TOKENS: usize = 48;++ /// The head is never squeezed below this, even when the options have eaten the budget.+ const MIN_HEAD_TOKENS: usize = 8;++ /// The Python type name, which appears verbatim in the prompt text.+ pub fn type_name(qtype: crate::questions::QuestionType) -> &'static str {+ use crate::questions::QuestionType;+ match qtype {+ QuestionType::Choice => "choice",+ QuestionType::Score => "score",+ QuestionType::Noul => "noul",+ }+ }++ /// The instruction text as it reaches the tokenizer: a string verbatim, anything else through+ /// the `json.dumps` dialect that keeps non-ASCII characters literal (`ensure_ascii=False`).+ /// Ports `Agent._to_internal` (laya 0.3.21; 0.3.6 escaped non-ASCII here instead).+ pub fn render_instructions(question: &Question) -> String {+ match question.instructions() {+ Value::String(s) => s.clone(),+ other => PythonJson::instructions(other),+ }+ }++ /// The option texts in label-index order. Index N is the option the Nth logit scores, so this+ /// order is what binds a probability to a label.+ pub fn render_options(question: &Question) -> Vec<String> {+ match question {+ Question::Choice(c) => c+ .options()+ .iter()+ .map(|(label, description)| {+ if Self::is_blank_criterion(description) {+ label.clone()+ } else {+ format!("{label}: {}", PythonJson::criterion(description))+ }+ })+ .collect(),+ Question::Score(s) => s+ .levels()+ .iter()+ .enumerate()+ .map(|(i, level)| format!("level {i}: {}", PythonJson::criterion(level)))+ .collect(),+ Question::Noul(n) => {+ let false_text = if Self::is_blank_criterion(n.if_false()) {+ "no, the statement does not hold".to_string()+ } else {+ PythonJson::criterion(n.if_false())+ };+ let true_text = if Self::is_blank_criterion(n.if_true()) {+ "yes, the statement holds".to_string()+ } else {+ PythonJson::criterion(n.if_true())+ };+ vec![format!("false: {false_text}"), format!("true: {true_text}")]+ }+ }+ }++ /// Whether a criterion value means "no description". Python tests `v is None or v == ""`, so+ /// only null and the empty string qualify: `0` and `false` are real descriptions and get+ /// rendered.+ fn is_blank_criterion(value: &Value) -> bool {+ matches!(value, Value::Null) || matches!(value, Value::String(s) if s.is_empty())+ }++ /// A literal mask token anywhere in user input would forge a marker the scorer then reads as+ /// an option, so every fragment has it replaced with a space before encoding.+ fn scrub(text: &str, mask_token: &str) -> String {+ if mask_token.is_empty() {+ text.to_string()+ } else {+ text.replace(mask_token, " ")+ }+ }++ /// Python's `//`: floor division, rounding toward negative infinity. `b` is always `>= 1` at+ /// every call site here, so this is equivalent to (and implemented with) `div_euclid`.+ fn floor_div(a: i64, b: i64) -> i64 {+ a.div_euclid(b)+ }++ /// Build the token sequence and marker positions for one question.+ ///+ /// `tokenizer` must encode without special tokens. `state` is the value being judged;+ /// `max_len` is the total sequence budget; `head_max_len` is the budget for instructions plus+ /// option fragments; `truncate_left` keeps the end of an over-long state rather than its+ /// start. `LayaEngine::predict_value` sets this to `true` exactly when `state` is a JSON+ /// array, matching `Agent._encode_state`'s `truncate_left = isinstance(state, list)` (laya+ /// 0.3.21): a conversation-turn list is serialized newest-last, so right-truncating it would+ /// drop the newest turn instead of the oldest one.+ ///+ /// Returns the token ids and one marker position per option. Markers landing at or past+ /// `max_len` are dropped — a short marker list means options did not fit, which the caller+ /// must treat as an error rather than answering a truncated question; this function itself+ /// never errors on that condition, matching `SequenceBuilder.Build` exactly.+ pub fn build(+ tokenizer: &dyn LayaTokenizer,+ state: &Value,+ question: &Question,+ max_len: usize,+ head_max_len: usize,+ truncate_left: bool,+ ) -> Result<(Vec<u32>, Vec<usize>)> {+ let special = tokenizer.special();+ let mask_token = special.mask_token.as_str();++ let options = Self::render_options(question);+ let instructions = Self::scrub(&Self::render_instructions(question), mask_token);+ let header_text = format!(+ "{} question: {instructions}",+ Self::type_name(question.question_type())+ );+ let mut header_ids: Vec<u32> = tokenizer.encode(&header_text)?;++ let mut option_ids: Vec<Vec<u32>> = Vec::with_capacity(options.len());+ for option in &options {+ let encoded = tokenizer.encode(&format!(" {}", Self::scrub(option, mask_token)))?;+ let take = Self::MAX_OPTION_TOKENS.min(encoded.len());+ let mut fragment = Vec::with_capacity(take + 1);+ fragment.push(special.mask_id);+ fragment.extend_from_slice(&encoded[..take]);+ option_ids.push(fragment);+ }++ let option_len_sum =+ |opts: &[Vec<u32>]| -> i64 { opts.iter().map(|o| o.len() as i64).sum() };++ let mut opt_budget = head_max_len as i64 - option_len_sum(&option_ids);+ if opt_budget < 16 {+ let per = 4i64.max(Self::floor_div(+ head_max_len as i64 - 16,+ 1i64.max(option_ids.len() as i64),+ )) as usize;+ for fragment in &mut option_ids {+ if fragment.len() > per {+ fragment.truncate(per);+ }+ }+ opt_budget = head_max_len as i64 - option_len_sum(&option_ids);+ }++ let head_take = (Self::MIN_HEAD_TOKENS as i64).max(opt_budget) as usize;+ if header_ids.len() > head_take {+ header_ids.truncate(head_take);+ }++ let mut ids: Vec<u32> = Vec::with_capacity(max_len);+ ids.push(special.cls_id);+ ids.extend_from_slice(&header_ids);+ ids.push(special.sep_id);++ let mut markers = Vec::with_capacity(option_ids.len());+ for fragment in &option_ids {+ markers.push(ids.len());+ ids.extend_from_slice(fragment);+ }+ ids.push(special.sep_id);++ // -1 reserves the final SEP that closes the whole sequence, appended below.+ let room = (max_len as i64 - ids.len() as i64 - 1).max(0) as usize;+ let state_text = Self::scrub(&PythonJson::state(state), mask_token);+ let state_ids = tokenizer.encode(&state_text)?;+ let sliced: &[u32] = if truncate_left {+ // Python's `state_ids[max(0, len(state_ids) - room):]` (laya 0.3.21): newest tokens+ // kept, and with no room left none of the state (0.3.6's `st[-room:]` kept all of+ // it). `LayaEngine::predict_value` reaches this branch for a JSON-array state (a+ // conversation-turn list).+ &state_ids[state_ids.len().saturating_sub(room)..]+ } else {+ &state_ids[..room.min(state_ids.len())]+ };+ ids.extend_from_slice(sliced);+ ids.push(special.sep_id);++ if ids.len() > max_len {+ ids.truncate(max_len);+ }+ markers.retain(|&m| m < max_len);++ Ok((ids, markers))+ }+}diff --git a/laya-rust/tests/collator_tests.rs b/laya-rust/tests/collator_tests.rsnew file mode 100644index 00000000..b1a84dbe--- /dev/null+++ b/laya-rust/tests/collator_tests.rs@@ -0,0 +1,233 @@+//! Tier 1: batch padding, which needs no artifacts.++mod common;++use common::golden_data::{self, CheckpointGoldenData};+use laya::collator::{Collator, SequenceItem};+use laya::questions::QuestionType;++fn item(length: usize, markers: usize, qtype: QuestionType) -> SequenceItem {+ SequenceItem::new(+ (10..10 + length as u32).collect(),+ (1..=markers).collect(),+ qtype,+ )+}++#[test]+fn pads_to_the_longest_row_and_marks_attention_accordingly() {+ let batch = Collator::collate(+ &[+ item(5, 2, QuestionType::Choice),+ item(3, 2, QuestionType::Choice),+ ],+ 0,+ )+ .unwrap();++ assert_eq!(2, batch.count);+ assert_eq!(5, batch.sequence_length);+ assert_eq!(vec![1, 1, 1, 1, 1, 1, 1, 1, 0, 0], batch.attention_mask);+ // The short row's tail is the pad id, and attention is 0 there.+ assert_eq!(0, batch.input_ids[8]);+ assert_eq!(0, batch.input_ids[9]);+}++#[test]+fn fills_with_the_given_pad_id_rather_than_zero() {+ let batch = Collator::collate(+ &[+ item(3, 2, QuestionType::Choice),+ item(1, 2, QuestionType::Choice),+ ],+ 77,+ )+ .unwrap();+ assert_eq!(77, batch.input_ids[4]);+ assert_eq!(77, batch.input_ids[5]);+ assert_eq!(0, batch.attention_mask[4]);+}++#[test]+fn counts_only_real_tokens_as_input_tokens() {+ let batch = Collator::collate(+ &[+ item(5, 2, QuestionType::Choice),+ item(3, 2, QuestionType::Choice),+ ],+ 0,+ )+ .unwrap();+ assert_eq!(8, batch.input_tokens);+}++#[test]+fn widens_the_marker_axis_to_two_for_single_option_batches() {+ // The exported graph has TopK k=2 baked in: a one-column marker axis makes it fail outright.+ let batch = Collator::collate(&[item(6, 1, QuestionType::Choice)], 0).unwrap();++ assert_eq!(Collator::MIN_MARKERS, batch.marker_count);+ assert_eq!(vec![1, 0], batch.marker_pos);+ assert_eq!(vec![true, false], batch.marker_mask);+ // The real option count is kept separately, so the pad column never reaches an answer.+ assert_eq!(vec![1], batch.marker_counts);+}++#[test]+fn pads_the_marker_axis_to_the_widest_question() {+ let batch = Collator::collate(+ &[+ item(6, 4, QuestionType::Choice),+ item(6, 2, QuestionType::Choice),+ ],+ 0,+ )+ .unwrap();++ assert_eq!(4, batch.marker_count);+ assert_eq!(vec![1, 2, 3, 4, 1, 2, 0, 0], batch.marker_pos);+ assert_eq!(+ vec![true, true, true, true, true, true, false, false],+ batch.marker_mask+ );+ assert_eq!(vec![4, 2], batch.marker_counts);+}++#[test]+fn carries_the_question_type_per_row_using_pythons_numbering() {+ let batch = Collator::collate(+ &[+ item(4, 2, QuestionType::Noul),+ item(4, 2, QuestionType::Choice),+ item(4, 2, QuestionType::Score),+ ],+ 0,+ )+ .unwrap();++ assert_eq!(vec![2, 0, 1], batch.qtype);+}++#[test]+fn refuses_an_empty_batch() {+ assert!(Collator::collate(&[], 0).is_err());+}++#[test]+fn reproduces_the_recorded_batch_tensors() {+ let data = CheckpointGoldenData::for_checkpoint("multilingual");+ if !data.available() {+ eprintln!("skip: no golden data for 'multilingual'");+ return;+ }++ let pad_id = data.meta()["special_tokens"]["pad_id"].as_u64().unwrap() as u32;++ for case in data.success_cases() {+ let root = data.load_case(&case);+ let expected = &root["batch"];++ // Rebuild the batch from the recorded per-question ids, so this checks the collator alone+ // and stays meaningful without a tokenizer present.+ let items: Vec<SequenceItem> = root["per_question"]+ .as_array()+ .unwrap()+ .iter()+ .map(|q| {+ let ids: Vec<u32> = golden_data::ints(&q["input_ids"])+ .into_iter()+ .map(|v| v as u32)+ .collect();+ let markers = golden_data::indices(&q["markers"]);+ let qtype = QuestionType::try_from(q["qtype"].as_i64().unwrap())+ .expect("valid qtype in golden data");+ SequenceItem::new(ids, markers, qtype)+ })+ .collect();++ let widest_markers = items.iter().map(|it| it.markers.len()).max().unwrap_or(0);+ let batch = Collator::collate(&items, pad_id).unwrap_or_else(|e| {+ panic!("case '{case}': collate failed: {e}");+ });++ assert_matrix(+ &expected["input_ids"],+ &batch.input_ids,+ batch.sequence_length,+ &case,+ );+ assert_matrix(+ &expected["attention_mask"],+ &batch.attention_mask,+ batch.sequence_length,+ &case,+ );+ assert_matrix(+ &expected["marker_pos"],+ &batch.marker_pos,+ batch.marker_count,+ &case,+ );++ let expected_qtype = golden_data::ints(&expected["qtype"]);+ assert_eq!(expected_qtype, batch.qtype, "case '{case}': qtype mismatch");++ let expected_mask = expected["marker_mask"].as_array().unwrap();+ assert_eq!(+ expected_mask.len(),+ batch.count,+ "case '{case}': marker_mask row count"+ );+ for (r, row) in expected_mask.iter().enumerate() {+ let row = row.as_array().unwrap();+ for (c, expected) in row.iter().enumerate().take(batch.marker_count) {+ assert_eq!(+ expected.as_bool().unwrap(),+ batch.marker_mask[r * batch.marker_count + c],+ "case '{case}': marker_mask[{r}][{c}]"+ );+ }+ }++ let expected_input_tokens =+ root["result"]["usage"]["input_tokens"].as_u64().unwrap() as usize;+ assert_eq!(+ expected_input_tokens, batch.input_tokens,+ "case '{case}': input_tokens"+ );++ // marker_pos above already pinned the padded width; this checks the flag the dump script+ // set, so a future golden that stops exercising the TopK edge case is noticed.+ let expected_padded = root["marker_axis_padded"].as_bool().unwrap();+ assert_eq!(+ expected_padded,+ batch.marker_count > widest_markers,+ "case '{case}': marker_axis_padded flag"+ );+ }+}++fn assert_matrix(+ expected: &serde_json::Value,+ actual: &[i64],+ cols: usize,+ case: &golden_data::GoldenCaseInfo,+) {+ let rows = expected.as_array().unwrap();+ assert_eq!(+ rows.len() * cols,+ actual.len(),+ "case '{case}': matrix size"+ );+ for (r, row) in rows.iter().enumerate() {+ let row = row.as_array().unwrap();+ assert_eq!(cols, row.len(), "case '{case}': row {r} width");+ for (c, value) in row.iter().enumerate() {+ assert_eq!(+ value.as_i64().unwrap(),+ actual[r * cols + c],+ "case '{case}': [{r}][{c}]"+ );+ }+ }+}diff --git a/laya-rust/tests/golden_questions_tests.rs b/laya-rust/tests/golden_questions_tests.rsnew file mode 100644index 00000000..c0938c51--- /dev/null+++ b/laya-rust/tests/golden_questions_tests.rs@@ -0,0 +1,74 @@+//! Verifies that rebuilding a golden case's `questions` into a [`QuestionSet`] and rendering it+//! with [`SequenceBuilder`] reproduces exactly what the recorded `per_question` entries show —+//! i.e. that [`Question`] construction, [`SequenceBuilder::render_options`] and+//! [`SequenceBuilder::render_instructions`] agree with the real Python `_to_internal` /+//! `render_options` pipeline, not just with each other. Also checks `serialized_state` against+//! [`PythonJson::state`]. This is a Rust-specific addition.++mod common;++use common::golden_data::{self, CheckpointGoldenData};+use laya::python_json::PythonJson;+use laya::sequence_builder::SequenceBuilder;+use serde_json::Value;++const CHECKPOINTS: [&str; 3] = ["multilingual", "english", "typed-decisions"];++#[test]+fn golden_questions_render_exactly_as_recorded() {+ let mut checked_any = false;++ for checkpoint in CHECKPOINTS {+ let data = CheckpointGoldenData::for_checkpoint(checkpoint);+ if !data.available() {+ eprintln!("skip: no golden data for '{checkpoint}'");+ continue;+ }++ for case in data.success_cases() {+ checked_any = true;+ let root = data.load_case(&case);++ let question_set = golden_data::build_questions(&root["questions"])+ .unwrap_or_else(|e| panic!("{checkpoint}/{case}: failed to build questions: {e}"));++ for per_question in root["per_question"].as_array().unwrap() {+ let id = per_question["id"].as_str().unwrap();+ let question = question_set+ .get(id)+ .unwrap_or_else(|| panic!("{checkpoint}/{case}: missing question '{id}'"));++ let expected_options: Vec<&str> = per_question["options"]+ .as_array()+ .unwrap()+ .iter()+ .map(|v| v.as_str().unwrap())+ .collect();+ let actual_options = SequenceBuilder::render_options(question);+ assert_eq!(+ expected_options, actual_options,+ "{checkpoint}/{case}: question '{id}' options"+ );++ let expected_instructions = per_question["instructions"].as_str().unwrap();+ let actual_instructions = SequenceBuilder::render_instructions(question);+ assert_eq!(+ expected_instructions, actual_instructions,+ "{checkpoint}/{case}: question '{id}' instructions"+ );+ }++ if let Some(expected_state) = root.get("serialized_state").and_then(Value::as_str) {+ let actual_state = PythonJson::state(&root["state"]);+ assert_eq!(+ expected_state, actual_state,+ "{checkpoint}/{case}: serialized_state vs PythonJson::state"+ );+ }+ }+ }++ if !checked_any {+ eprintln!("skip: no golden data found for any checkpoint");+ }+}diff --git a/laya-rust/tests/sequence_builder_tests.rs b/laya-rust/tests/sequence_builder_tests.rsnew file mode 100644index 00000000..64d6e22b--- /dev/null+++ b/laya-rust/tests/sequence_builder_tests.rs@@ -0,0 +1,300 @@+//! Tier 1: sequence layout and the option budget, using [`StubTokenizer`] so no artifacts are+//! needed. Real token ids are a Phase 2 tokenizer-parity concern.++mod common;++use common::stub_tokenizer::StubTokenizer;+use laya::questions::{Question, QuestionType};+use laya::sequence_builder::SequenceBuilder;+use laya::tokenization::LayaTokenizer;+use serde_json::{Value, json};++#[test]+fn lays_out_cls_head_sep_options_sep_state_sep() {+ let tok = StubTokenizer::new();+ let q = Question::choice("pick one", [("a", Value::Null), ("b", Value::Null)]).unwrap();+ let (ids, markers) =+ SequenceBuilder::build(&tok, &json!("state text"), &q, 64, 32, false).unwrap();++ assert_eq!(tok.special().cls_id, ids[0]);+ assert_eq!(tok.special().sep_id, *ids.last().unwrap());+ // Each marker is a mask token, and the two options are adjacent fragments.+ assert_eq!(2, markers.len());+ for &m in &markers {+ assert_eq!(tok.special().mask_id, ids[m]);+ }+ assert!(markers[1] > markers[0]);+ // A SEP closes the head, immediately before the first marker.+ assert_eq!(tok.special().sep_id, ids[markers[0] - 1]);+}++#[test]+fn marker_positions_point_at_consecutive_option_fragments() {+ let tok = StubTokenizer::new();+ // One word per option keeps the arithmetic checkable: mask + 1 word = 2 ids per fragment.+ let q = Question::choice(+ "h",+ [+ ("alpha", Value::Null),+ ("beta", Value::Null),+ ("gamma", Value::Null),+ ],+ )+ .unwrap();+ let (ids, markers) = SequenceBuilder::build(&tok, &json!("s"), &q, 64, 32, false).unwrap();++ assert_eq!(3, markers.len());+ assert_eq!(markers[0] + 2, markers[1]);+ assert_eq!(markers[1] + 2, markers[2]);+ assert_eq!(tok.special().sep_id, ids[markers[2] + 2]);+}++#[test]+fn puts_the_type_name_in_the_head_text() {+ let tok = StubTokenizer::new();+ let q = Question::noul("does it hold", Value::Null, Value::Null);+ SequenceBuilder::build(&tok, &json!("s"), &q, 64, 32, false).unwrap();+ assert_eq!("noul question: does it hold", tok.encoded()[0]);+}++#[test]+fn type_names_match_python() {+ assert_eq!("choice", SequenceBuilder::type_name(QuestionType::Choice));+ assert_eq!("score", SequenceBuilder::type_name(QuestionType::Score));+ assert_eq!("noul", SequenceBuilder::type_name(QuestionType::Noul));+}++#[test]+fn option_fragments_are_encoded_with_a_leading_space() {+ let tok = StubTokenizer::new();+ let q = Question::choice("h", [("a", Value::Null)]).unwrap();+ SequenceBuilder::build(&tok, &json!("s"), &q, 64, 32, false).unwrap();+ // Python encodes " " + option, which the Metaspace pre-tokenizer turns into a word boundary.+ assert!(tok.encoded().iter().any(|s| s == " a"));+}++#[test]+fn renders_options_the_way_python_does() {+ assert_eq!(+ vec!["a: first".to_string(), "b".to_string()],+ SequenceBuilder::render_options(+ &Question::choice("h", [("a", json!("first")), ("b", Value::Null)]).unwrap()+ )+ );++ assert_eq!(+ vec!["level 0: low".to_string(), "level 1: high".to_string()],+ SequenceBuilder::render_options(&Question::score("h", ["low", "high"]).unwrap())+ );++ assert_eq!(+ vec![+ "false: no, the statement does not hold".to_string(),+ "true: yes, the statement holds".to_string()+ ],+ SequenceBuilder::render_options(&Question::noul("h", Value::Null, Value::Null))+ );++ assert_eq!(+ vec!["false: all fine".to_string(), "true: on fire".to_string()],+ SequenceBuilder::render_options(&Question::noul("h", "all fine", "on fire"))+ );+}++#[test]+fn treats_only_null_and_empty_string_as_a_missing_description() {+ // Python tests `v is None or v == ""`, so 0 and false are real descriptions.+ let q = Question::choice(+ "h",+ [+ ("zero", json!(0)),+ ("no", json!(false)),+ ("blank", json!("")),+ ("none", Value::Null),+ ],+ )+ .unwrap();+ assert_eq!(+ vec![+ "zero: 0".to_string(),+ "no: false".to_string(),+ "blank".to_string(),+ "none".to_string()+ ],+ SequenceBuilder::render_options(&q)+ );+}++#[test]+fn scrubs_a_mask_token_out_of_every_fragment() {+ let tok = StubTokenizer::new();+ let mask = &tok.special().mask_token;+ // A literal <mask> in user input would forge a marker the scorer then reads as an option.+ let q = Question::choice(+ format!("is this {mask} safe"),+ [(format!("a{mask}b"), Value::Null)],+ )+ .unwrap();+ SequenceBuilder::build(+ &tok,+ &json!(format!("state {mask} here")),+ &q,+ 64,+ 32,+ false,+ )+ .unwrap();++ for text in tok.encoded() {+ assert!(+ !text.contains(mask.as_str()),+ "encoded text still contains the mask token: {text:?}"+ );+ }+}++#[test]+fn truncates_long_options_to_forty_eight_tokens() {+ let tok = StubTokenizer::new();+ let words: Vec<String> = (0..120).map(|i| format!("w{i}")).collect();+ let q = Question::choice("h", [("opt", json!(words.join(" ")))]).unwrap();+ let (ids, markers) = SequenceBuilder::build(&tok, &json!("s"), &q, 512, 256, false).unwrap();++ // The only fragment is mask + at most 48 option tokens, then the closing SEP. "opt: " adds two+ // words, so 48 is the cap that bites rather than the option running out.+ assert_eq!(1, markers.len());+ let fragment_end = ids[markers[0]..]+ .iter()+ .position(|&id| id == tok.special().sep_id)+ .unwrap()+ + markers[0];+ assert_eq!(markers[0] + 49, fragment_end);+}++#[test]+fn shrinks_options_when_they_would_eat_the_whole_head_budget() {+ let tok = StubTokenizer::new();+ // 20 options of ~10 tokens each against a 96-token head budget forces the `opt_budget < 16`+ // branch, which caps every fragment at `(head_max_len - 16) // n`.+ let options: Vec<(String, Value)> = (0..20)+ .map(|i| {+ let desc = (0..10)+ .map(|j| format!("d{i}_{j}"))+ .collect::<Vec<_>>()+ .join(" ");+ (format!("opt{i}"), json!(desc))+ })+ .collect();+ let q = Question::choice("head words here", options).unwrap();+ let (ids, markers) = SequenceBuilder::build(&tok, &json!("s"), &q, 512, 96, false).unwrap();++ assert_eq!(20, markers.len());+ // Mirrors the production formula `max(4, (head_max_len - 16) // n)`; with these numbers it+ // happens to always pick the 4-token floor.+ #[allow(clippy::unnecessary_min_or_max)]+ let per = 4.max((96 - 16) / 20);+ for i in 1..markers.len() {+ let width = markers[i] - markers[i - 1];+ assert!(+ width <= per,+ "fragment {} is {width} tokens, above the {per}-token cap",+ i - 1+ );+ }+ assert_eq!(tok.special().cls_id, ids[0]);+}++#[test]+fn keeps_at_least_eight_head_tokens_when_options_dominate() {+ let tok = StubTokenizer::new();+ let options: Vec<(String, Value)> = (0..30)+ .map(|i| (format!("o{i}"), json!("a b c d e f g h")))+ .collect();+ let q = Question::choice("one two three four five six seven eight nine ten", options).unwrap();+ let (ids, markers) = SequenceBuilder::build(&tok, &json!("s"), &q, 512, 64, false).unwrap();++ // CLS + head + SEP, so the first marker cannot be earlier than position 2.+ assert!(markers[0] >= 2);+ assert_eq!(tok.special().sep_id, ids[markers[0] - 1]);+}++#[test]+fn truncates_the_state_to_fit_max_len() {+ let tok = StubTokenizer::new();+ let state: Vec<String> = (0..500).map(|i| format!("s{i}")).collect();+ let q = Question::noul("h", Value::Null, Value::Null);+ let (ids, markers) =+ SequenceBuilder::build(&tok, &json!(state.join(" ")), &q, 64, 32, false).unwrap();++ assert_eq!(64, ids.len());+ assert_eq!(tok.special().sep_id, *ids.last().unwrap());+ assert_eq!(2, markers.len());+}++#[test]+fn keeps_the_state_tail_when_truncating_left() {+ let tok = StubTokenizer::new();+ let state: Vec<String> = (0..200).map(|i| format!("s{i}")).collect();+ let state = json!(state.join(" "));+ let q = Question::noul("h", Value::Null, Value::Null);+ let (head_ids, head_markers) = SequenceBuilder::build(&tok, &state, &q, 64, 32, false).unwrap();+ let (tail_ids, tail_markers) = SequenceBuilder::build(&tok, &state, &q, 64, 32, true).unwrap();++ assert_eq!(head_ids.len(), tail_ids.len());+ assert_ne!(head_ids, tail_ids);+ // Both keep the same prefix up to the state, and differ from there on.+ assert_eq!(head_markers, tail_markers);+}++#[test]+fn left_truncation_with_no_room_drops_the_whole_state() {+ // laya 0.3.21 slices `state_ids[max(0, len - room):]`, so with room == 0 no state token+ // survives. 0.3.6's `st[-room:]` kept the whole state there (`x[-0:]` is all of `x`).+ let tok = StubTokenizer::new();+ let q = Question::noul("h", Value::Null, Value::Null);+ let (empty_ids, _) = SequenceBuilder::build(&tok, &json!(""), &q, 512, 32, false).unwrap();+ let max_len = empty_ids.len();++ let state: Vec<String> = (0..50).map(|i| format!("s{i}")).collect();+ let (ids, _) =+ SequenceBuilder::build(&tok, &json!(state.join(" ")), &q, max_len, 32, true).unwrap();++ assert_eq!(empty_ids, ids);+}++#[test]+fn drops_markers_that_would_land_past_max_len() {+ let tok = StubTokenizer::new();+ // 40 options cannot fit a 32-token sequence, so markers are lost. A later `predict` phase must+ // treat a short marker list as an error rather than answering a question missing options.+ let options: Vec<(String, Value)> = (0..40).map(|i| (format!("o{i}"), json!("desc"))).collect();+ let option_count = options.len();+ let q = Question::choice("h", options).unwrap();+ let (ids, markers) = SequenceBuilder::build(&tok, &json!("s"), &q, 32, 256, false).unwrap();++ assert_eq!(32, ids.len());+ assert!(markers.len() < option_count);+ for &m in &markers {+ assert!(m < 32);+ }+}++#[test]+fn renders_non_string_instructions_with_literal_unicode() {+ // laya 0.3.21: json.dumps(..., ensure_ascii=False), so non-ASCII stays literal rather than+ // coming out as \uXXXX (0.3.6's behavior, which the tokenizer then read back as escape text).+ let q = Question::noul(json!({"hint": "Müller"}), Value::Null, Value::Null);+ let rendered = SequenceBuilder::render_instructions(&q);++ assert!(rendered.contains('ü'));+ assert!(!rendered.contains("\\u"));+}++#[test]+fn passes_string_instructions_through_untouched() {+ let q = Question::noul("Müller & co <urgent>", Value::Null, Value::Null);+ assert_eq!(+ "Müller & co <urgent>",+ SequenceBuilder::render_instructions(&q)+ );+}