Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
//! The plan behind `LayaEngine::predict_batch`: how many states go through one forward pass, when
//! states are reordered to pad less, and how results come back in input order. Ports
//! `ONNXAgent._infer_batch` (`laya/onnx_agent.py`), whose `batch_size` and `sort_by_length`
//! handling is the same contract as `Agent.predict_batch`.
//!
//! The driver here is generic over what an encoded state is, so the chunking, the sort windows
//! and the order restoration are testable without a model: the engine supplies the tokenizing and
//! the forward pass.
//!
//! Python's per-call hooks are not part of this port. [`BatchOptions`] is `#[non_exhaustive]` so a
//! field can be added without a signature change.
use crate::confidence::MinConfidence;
use crate::error::{LayaError, Result};
use std::ops::Range;
/// How many batches of states the `sort_by_length` lookahead spans. Bounds the tokenized
/// lookahead independently of the input size, as Python does.
const SORT_WINDOW_BATCHES: usize = 8;
/// Options for `LayaEngine::predict_batch` and `LayaRouter::predict_batch`. Every field is
/// optional; `Default` gives Python's defaults. Build one with the chained setters.
///
/// The per-call hooks, which Python's `predict_batch` also accepts, are not part of this port.
#[derive(Debug, Clone, Default, PartialEq)]
#[non_exhaustive]
pub struct BatchOptions {
/// Most states per forward pass. `None` or `0` sends them all in one pass. Python treats a
/// `batch_size` of `None`, `0` or any negative number as unset; a negative size cannot be
/// written as a `usize`, so `0` stands for all of them here. Set it to bound peak memory when
/// batching many or long states.
pub batch_size: Option<usize>,
/// Group similarly sized encoded states within windows of eight batches, so each batch pads
/// to a shorter maximum. Takes effect only with a `batch_size` greater than one and smaller
/// than the number of states; otherwise it does nothing. Results keep input order. Changing
/// the batch shapes can slightly change floating-point predictions near decision thresholds.
pub sort_by_length: bool,
/// Opt-in abstention threshold on `answer_confidence` (#361), applied batch-wide to every
/// answer of every result; it plays no part in how states are grouped or batched. See
/// [`crate::PredictOptions::min_confidence`] for what it reports.
pub min_confidence: Option<MinConfidence>,
/// Selects an entry of [`crate::LayaOptions::lang_temperatures`] for every state of the call;
/// see [`crate::PredictOptions::lang`]. `LayaRouter::predict_batch` refuses it
/// (`LayaError::NotSupported`): a request names its own `lang`, as in Python.
pub lang: Option<String>,
/// Override the checkpoint's `max_len` for every state of the call; see
/// [`crate::PredictOptions::max_len`]. `LayaRouter::predict_batch` refuses it: a request names
/// its own.
pub max_len: Option<i64>,
/// Override the checkpoint's `head_max_len` for every state of the call; see
/// [`crate::PredictOptions::head_max_len`]. Refused by `LayaRouter::predict_batch`.
pub head_max_len: Option<i64>,
}
impl BatchOptions {
/// Python's defaults: one pass for every state, input order.
pub fn new() -> Self {
Self::default()
}
/// Set [`Self::batch_size`].
#[must_use]
pub fn batch_size(mut self, batch_size: usize) -> Self {
self.batch_size = Some(batch_size);
self
}
/// Set [`Self::sort_by_length`].
#[must_use]
pub fn sort_by_length(mut self, sort_by_length: bool) -> Self {
self.sort_by_length = sort_by_length;
self
}
/// Set [`Self::min_confidence`]. Build the value with [`MinConfidence::scalar`] or
/// [`MinConfidence::map`].
#[must_use]
pub fn min_confidence(mut self, min_confidence: MinConfidence) -> Self {
self.min_confidence = Some(min_confidence);
self
}
/// Set [`Self::lang`].
#[must_use]
pub fn lang(mut self, lang: impl Into<String>) -> Self {
self.lang = Some(lang.into());
self
}
/// Set [`Self::max_len`].
#[must_use]
pub fn max_len(mut self, max_len: i64) -> Self {
self.max_len = Some(max_len);
self
}
/// Set [`Self::head_max_len`].
#[must_use]
pub fn head_max_len(mut self, head_max_len: i64) -> Self {
self.head_max_len = Some(head_max_len);
self
}
/// States per forward pass for `count` states: `batch_size` when it is positive, else all.
fn chunk(&self, count: usize) -> usize {
match self.batch_size {
Some(size) if size > 0 => size,
_ => count,
}
}
/// Whether `sort_by_length` reorders `count` states: `1 < chunk < count`.
pub(crate) fn reorders(&self, count: usize) -> bool {
let chunk = self.chunk(count);
self.sort_by_length && 1 < chunk && chunk < count
}
}
/// Encodes, orders, chunks and runs `count` states, returning one result per state in input order.
///
/// * `encode` turns a range of input indices into their encoded states (all of a window at once,
/// so a tokenizing error surfaces before any forward pass of that window, as in Python);
/// * `row_len` is a state's longest encoded question row, the sort key;
/// * `infer` runs one chunk of encoded states and returns one result per state of the chunk.
///
/// Without a reordering `sort_by_length` the windows are the chunks themselves. With one, each
/// window spans eight chunks, its states are stably sorted by `row_len`, and the sorted states are
/// cut into chunks.
pub(crate) fn run<E, R>(
count: usize,
options: &BatchOptions,
mut encode: impl FnMut(Range<usize>) -> Result<Vec<E>>,
row_len: impl Fn(&E) -> usize,
mut infer: impl FnMut(Vec<E>) -> Result<Vec<R>>,
) -> Result<Vec<R>> {
if count == 0 {
return Ok(Vec::new());
}
let chunk = options.chunk(count);
let reorder = options.reorders(count);
let window = if reorder {
chunk.saturating_mul(SORT_WINDOW_BATCHES)
} else {
chunk
};
let mut results = Vec::with_capacity(count);
for start in (0..count).step_by(window) {
let end = start.saturating_add(window).min(count);
let mut slots: Vec<Option<E>> = encode(start..end)?.into_iter().map(Some).collect();
let mut order: Vec<usize> = (0..slots.len()).collect();
if reorder {
order.sort_by_key(|&i| slots[i].as_ref().map_or(0, &row_len));
}
let mut window_results: Vec<Option<R>> = Vec::with_capacity(slots.len());
window_results.resize_with(slots.len(), || None);
for indices in order.chunks(chunk) {
let batch: Vec<E> = indices.iter().filter_map(|&i| slots[i].take()).collect();
let outputs = infer(batch)?;
if outputs.len() != indices.len() {
return Err(internal(format!(
"a forward pass returned {} results for {} states",
outputs.len(),
indices.len()
)));
}
for (&index, output) in indices.iter().zip(outputs) {
window_results[index] = Some(output);
}
}
for result in window_results {
results.push(result.ok_or_else(|| internal("a state was left unanswered".into()))?);
}
}
Ok(results)
}
fn internal(message: String) -> LayaError {
LayaError::Onnx(format!("internal error: {message}"))
}
#[cfg(test)]
mod tests {
use super::*;
/// An encoded state: its input index and its longest row.
type Fake = (usize, usize);
/// Runs the driver over states whose row lengths are `lens`, recording each forward pass as
/// the input indices it carried. The result for a state is its own index.
fn drive(lens: &[usize], options: &BatchOptions) -> (Vec<usize>, Vec<Vec<usize>>) {
let mut passes = Vec::new();
let results = run(
lens.len(),
options,
|range| Ok(range.map(|i| (i, lens[i])).collect::<Vec<Fake>>()),
|state| state.1,
|batch| {
passes.push(batch.iter().map(|s| s.0).collect::<Vec<_>>());
Ok(batch.into_iter().map(|s| s.0).collect())
},
)
.unwrap();
(results, passes)
}
fn sized(batch_size: usize, sort: bool) -> BatchOptions {
BatchOptions::new()
.batch_size(batch_size)
.sort_by_length(sort)
}
#[test]
fn no_batch_size_sends_every_state_in_one_pass() {
let (results, passes) = drive(&[5, 4, 3], &BatchOptions::new());
assert_eq!(results, [0, 1, 2]);
assert_eq!(passes, [vec![0, 1, 2]]);
}
#[test]
fn zero_batch_size_is_unset_as_python_treats_it() {
let (_, passes) = drive(&[5, 4, 3], &sized(0, true));
assert_eq!(passes, [vec![0, 1, 2]]);
}
#[test]
fn batch_size_cuts_chunks_in_input_order() {
let (results, passes) = drive(&[1, 2, 3, 4, 5], &sized(2, false));
assert_eq!(results, [0, 1, 2, 3, 4]);
assert_eq!(passes, [vec![0, 1], vec![2, 3], vec![4]]);
}
#[test]
fn sorting_groups_similar_lengths_and_restores_input_order() {
let (results, passes) = drive(&[9, 1, 8, 2, 7], &sized(2, true));
assert_eq!(results, [0, 1, 2, 3, 4]);
assert_eq!(passes, [vec![1, 3], vec![4, 2], vec![0]]);
}
#[test]
fn sorting_is_stable_for_equal_lengths() {
let (_, passes) = drive(&[3, 3, 3, 1, 3], &sized(2, true));
assert_eq!(passes, [vec![3, 0], vec![1, 2], vec![4]]);
}
#[test]
fn sorting_needs_a_batch_size_above_one_and_below_the_state_count() {
let lens = [9, 1, 8, 2];
// No batch size: one pass, unsorted.
assert_eq!(drive(&lens, &sized(0, true)).1, [vec![0, 1, 2, 3]]);
// A batch of one: nothing to group.
assert_eq!(
drive(&lens, &sized(1, true)).1,
[vec![0], vec![1], vec![2], vec![3]]
);
// A batch as large as the input, or larger: one pass, unsorted.
assert_eq!(drive(&lens, &sized(4, true)).1, [vec![0, 1, 2, 3]]);
assert_eq!(drive(&lens, &sized(10, true)).1, [vec![0, 1, 2, 3]]);
// Just under the count sorts.
assert_eq!(drive(&lens, &sized(3, true)).1, [vec![1, 3, 2], vec![0]]);
}
#[test]
fn reorders_follows_the_python_gate() {
assert!(sized(2, true).reorders(3));
assert!(!sized(2, false).reorders(3));
assert!(!sized(2, true).reorders(2));
assert!(!sized(1, true).reorders(3));
assert!(!BatchOptions::new().sort_by_length(true).reorders(3));
}
#[test]
fn sort_windows_span_eight_batches() {
// Batch size 2, so a window is 16 states. Descending lengths: the first window sorts only
// among its own 16 states, the next one among the remaining 4.
let lens: Vec<usize> = (0..20).rev().collect();
let (results, passes) = drive(&lens, &sized(2, true));
assert_eq!(results, (0..20).collect::<Vec<_>>());
assert_eq!(passes[0], [15, 14]);
assert_eq!(passes[7], [1, 0]);
assert_eq!(passes[8], [19, 18]);
assert_eq!(passes[9], [17, 16]);
assert_eq!(passes.len(), 10);
}
#[test]
fn unsorted_windows_are_the_chunks() {
let (_, passes) = drive(&[3, 2, 1], &sized(2, false));
assert_eq!(passes, [vec![0, 1], vec![2]]);
}
#[test]
fn no_states_run_nothing() {
let (results, passes) = drive(&[], &sized(2, true));
assert!(results.is_empty());
assert!(passes.is_empty());
}
#[test]
fn an_encode_error_stops_before_any_pass_of_its_window() {
let mut passes = 0;
let outcome: Result<Vec<usize>> = run(
3,
&BatchOptions::new(),
|_| Err(LayaError::InvalidQuestion("boom".into())),
|_: &usize| 0,
|batch: Vec<usize>| {
passes += 1;
Ok(batch)
},
);
assert!(matches!(outcome, Err(LayaError::InvalidQuestion(_))));
assert_eq!(passes, 0);
}
#[test]
fn a_pass_returning_the_wrong_count_is_an_error_not_a_panic() {
let outcome: Result<Vec<usize>> = run(
2,
&BatchOptions::new(),
|range| Ok(range.collect::<Vec<usize>>()),
|_: &usize| 0,
|_| Ok(vec![0usize]),
);
assert!(outcome.is_err());
}
#[test]
fn options_take_chained_setters() {
let options = BatchOptions::new().batch_size(4).sort_by_length(true);
assert_eq!(options.batch_size, Some(4));
assert!(options.sort_by_length);
assert_eq!(BatchOptions::default(), BatchOptions::new());
}
// ------------------------------------------------------------------ recorded session runs
/// Where the per-checkpoint goldens live: `LAYA_GOLDEN_DIR`, else `tests/golden`.
fn golden_root() -> std::path::PathBuf {
match std::env::var("LAYA_GOLDEN_DIR") {
Ok(dir) if !dir.trim().is_empty() => dir.into(),
_ => std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/golden"),
}
}
/// The fused artifact directory for a checkpoint, found as the integration tests find it:
/// `LAYA_ONNX_DIR` (multilingual only), `LAYA_ONNX_ROOT`, then `onnx/<checkpoint>` up the tree.
fn artifact_dir(checkpoint: crate::LayaCheckpoint) -> Option<std::path::PathBuf> {
use std::path::{Path, PathBuf};
let complete = |dir: &Path| {
dir.join("rl_agent_config.json").is_file()
&& dir.join("model.onnx").is_file()
&& dir.join("model.onnx.data").is_file()
&& dir.join("tokenizer").join("tokenizer.json").is_file()
};
let name = checkpoint.subdir();
if checkpoint == crate::LayaCheckpoint::Multilingual
&& let Ok(dir) = std::env::var("LAYA_ONNX_DIR")
&& !dir.trim().is_empty()
{
let dir = PathBuf::from(dir);
return complete(&dir).then_some(dir);
}
if let Ok(root) = std::env::var("LAYA_ONNX_ROOT")
&& !root.trim().is_empty()
{
let dir = PathBuf::from(root).join(name);
if complete(&dir) {
return Some(dir);
}
}
let mut dir = Some(Path::new(env!("CARGO_MANIFEST_DIR")).to_path_buf());
while let Some(d) = dir {
let candidate = d.join("onnx").join(name);
if complete(&candidate) {
return Some(candidate);
}
dir = d.parent().map(Path::to_path_buf);
}
None
}
/// The questions of a recorded case, from its `questions` definition (choice, score, noul).
fn recorded_questions(definition: &serde_json::Value) -> crate::QuestionSet {
use crate::{Question, QuestionSet};
use serde_json::Value;
let mut set = QuestionSet::new();
for (id, def) in definition.as_object().expect("questions object") {
let instructions = def["instructions"].clone();
let criteria = def.get("criteria");
let question = match def["type"].as_str().expect("question type") {
"choice" => {
let options: Vec<(String, Value)> = match criteria.expect("criteria") {
Value::Array(labels) => labels
.iter()
.map(|l| (l.as_str().expect("label").to_string(), Value::Null))
.collect(),
Value::Object(map) => {
map.iter().map(|(k, v)| (k.clone(), v.clone())).collect()
}
other => panic!("{id}: unusable criteria {other}"),
};
Question::choice(instructions, options).expect("choice question")
}
"score" => {
let levels = criteria.and_then(Value::as_array).expect("levels").clone();
Question::score(instructions, levels).expect("score question")
}
"noul" => {
let side = |key: &str| {
criteria
.and_then(|c| c.get(key))
.cloned()
.unwrap_or(Value::Null)
};
Question::noul(instructions, side("false"), side("true"))
}
other => panic!("{id}: unknown question type {other}"),
};
set.insert(id.clone(), question).expect("unique ids");
}
set
}
fn recorded_options(options: &serde_json::Value) -> BatchOptions {
BatchOptions {
// Python treats a batch_size that is zero or negative as unset.
batch_size: options
.get("batch_size")
.and_then(serde_json::Value::as_i64)
.and_then(|size| usize::try_from(size).ok())
.filter(|&size| size > 0),
sort_by_length: options
.get("sort_by_length")
.and_then(serde_json::Value::as_bool)
.unwrap_or(false),
..BatchOptions::default()
}
}
/// Python's own session-run shapes against this port's chunk planning, over this port's own
/// token counts: each state's width (its longest question row) must equal Python's
/// `state_widths`, and the planned runs, `[rows, width]` per forward pass, must be the ones
/// Python made. Needs the tokenizer and the batch goldens, not a forward pass; skips like the
/// integration tests when either is missing.
/// Splits `LAYA_TEST_CHECKPOINTS` as `tests/common/test_artifacts.rs` does and panics on a name
/// that is not a checkpoint subdirectory.
fn selected_checkpoint_names(raw: &str) -> Vec<String> {
let names: Vec<String> = raw
.split(|c: char| c.is_whitespace() || c == ',')
.filter(|name| !name.is_empty())
.map(str::to_owned)
.collect();
let known = [
crate::LayaCheckpoint::English.subdir(),
crate::LayaCheckpoint::Multilingual.subdir(),
crate::LayaCheckpoint::TypedDecisions.subdir(),
];
if let Some(unknown) = names.iter().find(|n| !known.contains(&n.as_str())) {
panic!("unknown LAYA_TEST_CHECKPOINTS entry '{unknown}' in '{raw}'");
}
names
}
#[test]
fn checkpoint_selection_accepts_known_names() {
assert!(selected_checkpoint_names("").is_empty());
assert_eq!(
selected_checkpoint_names(" english, typed-decisions "),
["english", "typed-decisions"]
);
}
#[test]
#[should_panic(expected = "unknown LAYA_TEST_CHECKPOINTS entry 'typo' in 'english typo'")]
fn checkpoint_selection_rejects_an_unknown_name() {
selected_checkpoint_names("english typo");
}
#[test]
fn plans_the_recorded_session_runs() {
use crate::laya_config::LayaConfig;
use crate::model_artifacts::ModelArtifacts;
use crate::sequence_builder::SequenceBuilder;
use crate::tokenization::HfTokenizer;
use serde_json::Value;
// `LAYA_TEST_CHECKPOINTS` (space- or comma-separated subdirectory names) narrows this to the
// checkpoints a parity job owns; unset runs all. Same semantics as `tests/common`, including
// failing on an unknown name, so a library-only run cannot pass by ignoring a typo.
let selected =
selected_checkpoint_names(&std::env::var("LAYA_TEST_CHECKPOINTS").unwrap_or_default());
let selected: Vec<&str> = selected.iter().map(String::as_str).collect();
let mut checked_any = false;
for checkpoint in [
crate::LayaCheckpoint::Multilingual,
crate::LayaCheckpoint::English,
crate::LayaCheckpoint::TypedDecisions,
]
.into_iter()
.filter(|c| selected.is_empty() || selected.contains(&c.subdir()))
{
let Some(dir) = artifact_dir(checkpoint) else {
continue;
};
let golden_dir = golden_root().join(checkpoint.subdir());
let Ok(index) = std::fs::read_to_string(golden_dir.join("index.json")) else {
eprintln!("skip: no golden data for '{}'", checkpoint.subdir());
continue;
};
let index: Value = serde_json::from_str(&index).expect("index.json");
let artifacts =
ModelArtifacts::from_directory(&dir, Some(checkpoint)).expect("artifacts");
let config = LayaConfig::load(&artifacts.config_path).expect("config");
let tokenizer = HfTokenizer::from_file_with_config(
&artifacts.tokenizer_path,
artifacts.tokenizer_config_path.as_deref(),
)
.expect("tokenizer");
for entry in index.as_array().expect("index array") {
let is_batch = entry["kind"].as_str() == Some("batch");
if !is_batch || entry["expects_error"].as_bool() == Some(true) {
continue;
}
let text =
std::fs::read_to_string(golden_dir.join(entry["file"].as_str().unwrap()))
.expect("case file");
let root: Value = serde_json::from_str(&text).expect("case json");
let case = format!(
"{}/{}",
checkpoint.subdir(),
entry["name"].as_str().unwrap()
);
let states = root["states"].as_array().expect("states");
let questions = recorded_questions(&root["questions"]);
if states.is_empty() || questions.is_empty() {
continue;
}
checked_any = true;
// A case that passes its own token budget was recorded with it
// (Python: predict_batch(max_len=...)); a bool is the int 0 or 1.
let budget = |name: &str, fallback: usize| -> i64 {
match &root["options"][name] {
Value::Bool(b) => i64::from(*b),
Value::Number(n) => n.as_i64().expect("integer budget"),
_ => i64::try_from(fallback).expect("config budget"),
}
};
let max_len = budget("max_len", config.max_len);
let head_max_len = budget("head_max_len", config.head_max_len);
let widths: Vec<usize> = states
.iter()
.map(|state| {
questions
.iter()
.map(|(_, question)| {
SequenceBuilder::build_with_overrides(
&tokenizer,
state,
question,
max_len,
head_max_len,
matches!(state, Value::Array(_)),
)
.expect("sequence")
.0
.len()
})
.max()
.unwrap_or(0)
})
.collect();
let recorded_widths: Vec<usize> = root["state_widths"]
.as_array()
.expect("state_widths")
.iter()
.map(|w| w.as_u64().expect("width") as usize)
.collect();
assert_eq!(recorded_widths, widths, "{case}: state widths");
let mut runs: Vec<[usize; 2]> = Vec::new();
run(
states.len(),
&recorded_options(&root["options"]),
|range| Ok(range.map(|i| widths[i]).collect::<Vec<usize>>()),
|width: &usize| *width,
|chunk: Vec<usize>| {
runs.push([
chunk.len() * questions.len(),
chunk.iter().copied().max().unwrap_or(0),
]);
Ok(chunk)
},
)
.expect("plan");
let recorded_runs: Vec<[usize; 2]> = root["runs"]
.as_array()
.expect("runs")
.iter()
.map(|r| {
let r = r.as_array().expect("run");
[
r[0].as_u64().unwrap() as usize,
r[1].as_u64().unwrap() as usize,
]
})
.collect();
assert_eq!(recorded_runs, runs, "{case}: planned session runs");
}
}
if !checked_any {
eprintln!("skip: no ONNX tokenizer + batch golden data found for any checkpoint");
}
}
}