Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
//! Token-level parity against Python for every checkpoint, needing `tokenizer/tokenizer.json` but
//! not the model weights.
//!
//! This is where a port like this most plausibly goes wrong: a divergence of one token shifts
//! every marker after it, and the model then answers a different question without anything
//! failing. These tests compare against ids recorded from the real HuggingFace tokenizer, so they
//! fail loudly instead. Each test loops over all three checkpoints and loads the tokenizer inline,
//! so missing artifacts for one checkpoint skip only that checkpoint's cases — the tokenizer alone
//! is a few hundred KB, so (unlike the Tier 2 tests) there is no memory reason to avoid loading
//! all three at once.
mod common;
use common::golden_data::{self, CheckpointGoldenData};
use common::test_artifacts;
use laya::laya_options::LayaCheckpoint;
use laya::python_json::PythonJson;
use laya::sequence_builder::SequenceBuilder;
use laya::tokenization::{HfTokenizer, LayaTokenizer};
use serde_json::Value;
const CHECKPOINTS: [LayaCheckpoint; 3] = [
LayaCheckpoint::Multilingual,
LayaCheckpoint::English,
LayaCheckpoint::TypedDecisions,
];
/// The checkpoints this run covers (all, or those `LAYA_TEST_CHECKPOINTS` selects).
fn checkpoints() -> Vec<LayaCheckpoint> {
CHECKPOINTS
.into_iter()
.filter(|c| test_artifacts::is_selected(*c))
.collect()
}
/// Loads the tokenizer for `checkpoint`, or prints a skip line and returns `None` when
/// `tokenizer/tokenizer.json` is not available locally.
fn load_tokenizer(checkpoint: LayaCheckpoint) -> Option<HfTokenizer> {
if !test_artifacts::has_tokenizer(checkpoint) {
eprintln!(
"skip: no tokenizer.json for '{}'; set LAYA_ONNX_ROOT or LAYA_ONNX_DIR",
checkpoint.subdir()
);
return None;
}
let dir = test_artifacts::locate(checkpoint).unwrap_or_else(|| {
panic!(
"has_tokenizer was true but locate() found nothing for '{}'",
checkpoint.subdir()
)
});
let path = dir.join("tokenizer").join("tokenizer.json");
Some(HfTokenizer::from_file(&path).unwrap_or_else(|e| {
panic!(
"failed to load tokenizer for '{}' at {}: {e}",
checkpoint.subdir(),
path.display()
)
}))
}
fn to_u32(values: Vec<i64>) -> Vec<u32> {
values.into_iter().map(|v| v as u32).collect()
}
#[test]
fn special_token_ids_match_the_recorded_values() {
for checkpoint in checkpoints() {
let Some(tok) = load_tokenizer(checkpoint) else {
continue;
};
let golden = CheckpointGoldenData::for_checkpoint(checkpoint.subdir());
if !golden.available() {
eprintln!(
"skip: no golden data for '{}'; run tools/dump_golden.py",
checkpoint.subdir()
);
continue;
}
let meta = golden.meta();
let expected = &meta["special_tokens"];
let special = tok.special();
let name = checkpoint.subdir();
assert_eq!(
expected["pad_id"].as_u64().unwrap() as u32,
special.pad_id,
"{name}: pad_id"
);
assert_eq!(
expected["cls_id"].as_u64().unwrap() as u32,
special.cls_id,
"{name}: cls_id"
);
assert_eq!(
expected["sep_id"].as_u64().unwrap() as u32,
special.sep_id,
"{name}: sep_id"
);
assert_eq!(
expected["mask_id"].as_u64().unwrap() as u32,
special.mask_id,
"{name}: mask_id"
);
assert_eq!(
expected["unk_id"].as_u64().unwrap() as u32,
special.unk_id,
"{name}: unk_id"
);
assert_eq!(
expected["mask_token"].as_str().unwrap(),
special.mask_token,
"{name}: mask_token"
);
}
}
#[test]
fn encodes_every_probe_string_exactly_as_python_does() {
for checkpoint in checkpoints() {
let Some(tok) = load_tokenizer(checkpoint) else {
continue;
};
let golden = CheckpointGoldenData::for_checkpoint(checkpoint.subdir());
if !golden.available() {
eprintln!(
"skip: no golden data for '{}'; run tools/dump_golden.py",
checkpoint.subdir()
);
continue;
}
let probes = golden.load("tokenizer_probe.json");
let mut failures = Vec::new();
for probe in probes
.as_array()
.expect("tokenizer_probe.json must be an array")
{
let text = probe["text"].as_str().expect("probe entry needs 'text'");
let expected = to_u32(golden_data::ints(&probe["input_ids"]));
let actual = tok
.encode(text)
.unwrap_or_else(|e| panic!("{}: encode {text:?} failed: {e}", checkpoint.subdir()));
if expected != actual {
failures.push(format!(
" {text:?}\n python: {expected:?}\n rust: {actual:?}"
));
}
}
assert!(
failures.is_empty(),
"{}: {} probe string(s) tokenized differently:\n{}",
checkpoint.subdir(),
failures.len(),
failures.join("\n")
);
}
}
#[test]
fn adds_no_special_tokens_of_its_own() {
// The shipped tokenizer.json has a TemplateProcessing post-processor that wraps every encode.
// Left in place it would inject the checkpoint's CLS and SEP ids into every fragment, shifting
// all marker positions. HfTokenizer must never invoke it.
for checkpoint in checkpoints() {
let Some(tok) = load_tokenizer(checkpoint) else {
continue;
};
let ids = tok.encode("hello").unwrap();
assert_ne!(
tok.special().cls_id,
ids[0],
"{}: leading id looks like an injected CLS",
checkpoint.subdir()
);
assert_ne!(
tok.special().sep_id,
*ids.last().unwrap(),
"{}: trailing id looks like an injected SEP",
checkpoint.subdir()
);
}
}
#[test]
fn encodes_the_empty_string_to_nothing() {
for checkpoint in checkpoints() {
let Some(tok) = load_tokenizer(checkpoint) else {
continue;
};
assert!(
tok.encode("").unwrap().is_empty(),
"{}: empty string should encode to nothing",
checkpoint.subdir()
);
}
}
#[test]
fn builds_the_recorded_sequence_for_every_question() {
for checkpoint in checkpoints() {
let Some(tok) = load_tokenizer(checkpoint) else {
continue;
};
let golden = CheckpointGoldenData::for_checkpoint(checkpoint.subdir());
if !golden.available() {
continue;
}
let meta = golden.meta();
let config_max_len = meta["max_len"].as_i64().unwrap();
let config_head_max_len = meta["head_max_len"].as_i64().unwrap();
for case in golden.success_cases() {
let root = golden.load_case(&case);
let state = root["state"].clone();
// A case that passes its own token budget was recorded with it
// (Python: system_one(max_len=...)); a bool is the int 0 or 1.
let budget = |name: &str, fallback: i64| match &root["options"][name] {
Value::Bool(b) => i64::from(*b),
Value::Number(n) => n.as_i64().expect("integer budget"),
_ => fallback,
};
let max_len = budget("max_len", config_max_len);
let head_max_len = budget("head_max_len", config_head_max_len);
let questions = golden_data::build_questions(&root["questions"]).unwrap_or_else(|e| {
panic!(
"{}/{case}: failed to build questions: {e}",
checkpoint.subdir()
)
});
for recorded in root["per_question"]
.as_array()
.expect("case must have 'per_question'")
{
let id = recorded["id"]
.as_str()
.expect("per_question entry needs 'id'");
let question = questions.get(id).unwrap_or_else(|| {
panic!("{}/{case}: missing question '{id}'", checkpoint.subdir())
});
// The rendered option strings come first: they are the readable form of a
// divergence, so a mismatch here explains a mismatch in the ids below.
let expected_options: Vec<&str> = recorded["options"]
.as_array()
.unwrap()
.iter()
.map(|v| v.as_str().unwrap())
.collect();
assert_eq!(
expected_options,
SequenceBuilder::render_options(question),
"{}/{case}/{id}: options",
checkpoint.subdir()
);
assert_eq!(
recorded["instructions"].as_str().unwrap(),
SequenceBuilder::render_instructions(question),
"{}/{case}/{id}: instructions",
checkpoint.subdir()
);
let (ids, markers) = SequenceBuilder::build_signed(
&tok,
&state,
question,
max_len,
head_max_len,
false,
)
.unwrap_or_else(|e| {
panic!("{}/{case}/{id}: build failed: {e}", checkpoint.subdir())
});
assert_eq!(
golden_data::indices(&recorded["markers"]),
markers,
"{}/{case}/{id}: markers",
checkpoint.subdir()
);
assert_eq!(
to_u32(golden_data::ints(&recorded["input_ids"])),
ids,
"{}/{case}/{id}: input_ids",
checkpoint.subdir()
);
}
}
}
}
#[test]
fn serializes_state_exactly_as_python_does() {
// State serialization is checkpoint-independent (it is the same JSON serializer for all), but
// exercising it across checkpoints gives coverage over diverse state shapes in the golden
// corpus of every checkpoint.
for checkpoint in checkpoints() {
let golden = CheckpointGoldenData::for_checkpoint(checkpoint.subdir());
if !golden.available() {
continue;
}
for case in golden.index() {
let root = golden.load_case(&case);
if let Some(expected) = root.get("serialized_state").and_then(Value::as_str) {
let actual = PythonJson::state(&root["state"]);
assert_eq!(
expected,
actual,
"{}/{case}: serialized_state vs PythonJson::state",
checkpoint.subdir()
);
}
}
}
}