Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

11.8 KB
Raw Blame
//! Tier 1: sequence layout and the option budget, using [`StubTokenizer`] so no artifacts are
//! needed. Real token ids are a Phase 2 tokenizer-parity concern.

mod common;

use common::stub_tokenizer::StubTokenizer;
use laya::questions::{Question, QuestionType};
use laya::sequence_builder::SequenceBuilder;
use laya::tokenization::LayaTokenizer;
use serde_json::{Value, json};

#[test]
fn lays_out_cls_head_sep_options_sep_state_sep() {
    let tok = StubTokenizer::new();
    let q = Question::choice("pick one", [("a", Value::Null), ("b", Value::Null)]).unwrap();
    let (ids, markers) =
        SequenceBuilder::build(&tok, &json!("state text"), &q, 64, 32, false).unwrap();

    assert_eq!(tok.special().cls_id, ids[0]);
    assert_eq!(tok.special().sep_id, *ids.last().unwrap());
    // Each marker is a mask token, and the two options are adjacent fragments.
    assert_eq!(2, markers.len());
    for &m in &markers {
        assert_eq!(tok.special().mask_id, ids[m]);
    }
    assert!(markers[1] > markers[0]);
    // A SEP closes the head, immediately before the first marker.
    assert_eq!(tok.special().sep_id, ids[markers[0] - 1]);
}

#[test]
fn marker_positions_point_at_consecutive_option_fragments() {
    let tok = StubTokenizer::new();
    // One word per option keeps the arithmetic checkable: mask + 1 word = 2 ids per fragment.
    let q = Question::choice(
        "h",
        [
            ("alpha", Value::Null),
            ("beta", Value::Null),
            ("gamma", Value::Null),
        ],
    )
    .unwrap();
    let (ids, markers) = SequenceBuilder::build(&tok, &json!("s"), &q, 64, 32, false).unwrap();

    assert_eq!(3, markers.len());
    assert_eq!(markers[0] + 2, markers[1]);
    assert_eq!(markers[1] + 2, markers[2]);
    assert_eq!(tok.special().sep_id, ids[markers[2] + 2]);
}

#[test]
fn puts_the_type_name_in_the_head_text() {
    let tok = StubTokenizer::new();
    let q = Question::noul("does it hold", Value::Null, Value::Null);
    SequenceBuilder::build(&tok, &json!("s"), &q, 64, 32, false).unwrap();
    assert_eq!("noul question: does it hold", tok.encoded()[0]);
}

#[test]
fn type_names_match_python() {
    assert_eq!("choice", SequenceBuilder::type_name(QuestionType::Choice));
    assert_eq!("score", SequenceBuilder::type_name(QuestionType::Score));
    assert_eq!("noul", SequenceBuilder::type_name(QuestionType::Noul));
}

#[test]
fn option_fragments_are_encoded_with_a_leading_space() {
    let tok = StubTokenizer::new();
    let q = Question::choice("h", [("a", Value::Null)]).unwrap();
    SequenceBuilder::build(&tok, &json!("s"), &q, 64, 32, false).unwrap();
    // Python encodes " " + option, which the Metaspace pre-tokenizer turns into a word boundary.
    assert!(tok.encoded().iter().any(|s| s == " a"));
}

#[test]
fn renders_options_the_way_python_does() {
    assert_eq!(
        vec!["a: first".to_string(), "b".to_string()],
        SequenceBuilder::render_options(
            &Question::choice("h", [("a", json!("first")), ("b", Value::Null)]).unwrap()
        )
    );

    assert_eq!(
        vec!["level 0: low".to_string(), "level 1: high".to_string()],
        SequenceBuilder::render_options(&Question::score("h", ["low", "high"]).unwrap())
    );

    assert_eq!(
        vec![
            "false: no, the statement does not hold".to_string(),
            "true: yes, the statement holds".to_string()
        ],
        SequenceBuilder::render_options(&Question::noul("h", Value::Null, Value::Null))
    );

    assert_eq!(
        vec!["false: all fine".to_string(), "true: on fire".to_string()],
        SequenceBuilder::render_options(&Question::noul("h", "all fine", "on fire"))
    );
}

#[test]
fn treats_only_null_and_empty_string_as_a_missing_description() {
    // Python tests `v is None or v == ""`, so 0 and false are real descriptions.
    let q = Question::choice(
        "h",
        [
            ("zero", json!(0)),
            ("no", json!(false)),
            ("blank", json!("")),
            ("none", Value::Null),
        ],
    )
    .unwrap();
    assert_eq!(
        vec![
            "zero: 0".to_string(),
            "no: false".to_string(),
            "blank".to_string(),
            "none".to_string()
        ],
        SequenceBuilder::render_options(&q)
    );
}

#[test]
fn scrubs_a_mask_token_out_of_every_fragment() {
    let tok = StubTokenizer::new();
    let mask = &tok.special().mask_token;
    // A literal <mask> in user input would forge a marker the scorer then reads as an option.
    let q = Question::choice(
        format!("is this {mask} safe"),
        [(format!("a{mask}b"), Value::Null)],
    )
    .unwrap();
    SequenceBuilder::build(
        &tok,
        &json!(format!("state {mask} here")),
        &q,
        64,
        32,
        false,
    )
    .unwrap();

    for text in tok.encoded() {
        assert!(
            !text.contains(mask.as_str()),
            "encoded text still contains the mask token: {text:?}"
        );
    }
}

#[test]
fn truncates_long_options_to_forty_eight_tokens() {
    let tok = StubTokenizer::new();
    let words: Vec<String> = (0..120).map(|i| format!("w{i}")).collect();
    let q = Question::choice("h", [("opt", json!(words.join(" ")))]).unwrap();
    let (ids, markers) = SequenceBuilder::build(&tok, &json!("s"), &q, 512, 256, false).unwrap();

    // The only fragment is mask + at most 48 option tokens, then the closing SEP. "opt: " adds two
    // words, so 48 is the cap that bites rather than the option running out.
    assert_eq!(1, markers.len());
    let fragment_end = ids[markers[0]..]
        .iter()
        .position(|&id| id == tok.special().sep_id)
        .unwrap()
        + markers[0];
    assert_eq!(markers[0] + 49, fragment_end);
}

#[test]
fn shrinks_options_when_they_would_eat_the_whole_head_budget() {
    let tok = StubTokenizer::new();
    // 20 options of ~10 tokens each against a 96-token head budget forces the `opt_budget < 16`
    // branch, which caps every fragment at `(head_max_len - 16) // n`.
    let options: Vec<(String, Value)> = (0..20)
        .map(|i| {
            let desc = (0..10)
                .map(|j| format!("d{i}_{j}"))
                .collect::<Vec<_>>()
                .join(" ");
            (format!("opt{i}"), json!(desc))
        })
        .collect();
    let q = Question::choice("head words here", options).unwrap();
    let (ids, markers) = SequenceBuilder::build(&tok, &json!("s"), &q, 512, 96, false).unwrap();

    assert_eq!(20, markers.len());
    // Mirrors the production formula `max(4, (head_max_len - 16) // n)`; with these numbers it
    // happens to always pick the 4-token floor.
    #[allow(clippy::unnecessary_min_or_max)]
    let per = 4.max((96 - 16) / 20);
    for i in 1..markers.len() {
        let width = markers[i] - markers[i - 1];
        assert!(
            width <= per,
            "fragment {} is {width} tokens, above the {per}-token cap",
            i - 1
        );
    }
    assert_eq!(tok.special().cls_id, ids[0]);
}

#[test]
fn keeps_at_least_eight_head_tokens_when_options_dominate() {
    let tok = StubTokenizer::new();
    let options: Vec<(String, Value)> = (0..30)
        .map(|i| (format!("o{i}"), json!("a b c d e f g h")))
        .collect();
    let q = Question::choice("one two three four five six seven eight nine ten", options).unwrap();
    let (ids, markers) = SequenceBuilder::build(&tok, &json!("s"), &q, 512, 64, false).unwrap();

    // CLS + head + SEP, so the first marker cannot be earlier than position 2.
    assert!(markers[0] >= 2);
    assert_eq!(tok.special().sep_id, ids[markers[0] - 1]);
}

#[test]
fn truncates_the_state_to_fit_max_len() {
    let tok = StubTokenizer::new();
    let state: Vec<String> = (0..500).map(|i| format!("s{i}")).collect();
    let q = Question::noul("h", Value::Null, Value::Null);
    let (ids, markers) =
        SequenceBuilder::build(&tok, &json!(state.join(" ")), &q, 64, 32, false).unwrap();

    assert_eq!(64, ids.len());
    assert_eq!(tok.special().sep_id, *ids.last().unwrap());
    assert_eq!(2, markers.len());
}

#[test]
fn keeps_the_state_tail_when_truncating_left() {
    let tok = StubTokenizer::new();
    let state: Vec<String> = (0..200).map(|i| format!("s{i}")).collect();
    let state = json!(state.join(" "));
    let q = Question::noul("h", Value::Null, Value::Null);
    let (head_ids, head_markers) = SequenceBuilder::build(&tok, &state, &q, 64, 32, false).unwrap();
    let (tail_ids, tail_markers) = SequenceBuilder::build(&tok, &state, &q, 64, 32, true).unwrap();

    assert_eq!(head_ids.len(), tail_ids.len());
    assert_ne!(head_ids, tail_ids);
    // Both keep the same prefix up to the state, and differ from there on.
    assert_eq!(head_markers, tail_markers);
}

#[test]
fn left_truncation_with_no_room_drops_the_whole_state() {
    // laya 0.3.21 slices `state_ids[max(0, len - room):]`, so with room == 0 no state token
    // survives. 0.3.6's `st[-room:]` kept the whole state there (`x[-0:]` is all of `x`).
    let tok = StubTokenizer::new();
    let q = Question::noul("h", Value::Null, Value::Null);
    let (empty_ids, _) = SequenceBuilder::build(&tok, &json!(""), &q, 512, 32, false).unwrap();
    let max_len = empty_ids.len();

    let state: Vec<String> = (0..50).map(|i| format!("s{i}")).collect();
    let (ids, _) =
        SequenceBuilder::build(&tok, &json!(state.join(" ")), &q, max_len, 32, true).unwrap();

    assert_eq!(empty_ids, ids);
}

#[test]
fn drops_markers_that_would_land_past_max_len() {
    let tok = StubTokenizer::new();
    // 40 options cannot fit a 32-token sequence, so markers are lost. A later `predict` phase must
    // treat a short marker list as an error rather than answering a question missing options.
    let options: Vec<(String, Value)> = (0..40).map(|i| (format!("o{i}"), json!("desc"))).collect();
    let option_count = options.len();
    let q = Question::choice("h", options).unwrap();
    let (ids, markers) = SequenceBuilder::build(&tok, &json!("s"), &q, 32, 256, false).unwrap();

    assert_eq!(32, ids.len());
    assert!(markers.len() < option_count);
    for &m in &markers {
        assert!(m < 32);
    }
}

#[test]
fn renders_non_string_instructions_with_literal_unicode() {
    // laya 0.3.21: json.dumps(..., ensure_ascii=False), so non-ASCII stays literal rather than
    // coming out as \uXXXX (0.3.6's behavior, which the tokenizer then read back as escape text).
    let q = Question::noul(json!({"hint": "Müller"}), Value::Null, Value::Null);
    let rendered = SequenceBuilder::render_instructions(&q);

    assert!(rendered.contains('ü'));
    assert!(!rendered.contains("\\u"));
}

#[test]
fn passes_string_instructions_through_untouched() {
    let q = Question::noul("Müller & co <urgent>", Value::Null, Value::Null);
    assert_eq!(
        "Müller & co <urgent>",
        SequenceBuilder::render_instructions(&q)
    );
}

// Python accepts any int for `max_len` / `head_max_len` (slicing by a huge int is fine) and builds
// the same sequence as with the usual bounds when nothing is truncated (checked with
// `common.build_sequence` for 2**61, 2**63 - 1, 2**64 - 1 and 10**30). The buffer must not be
// sized from the bound: 2**61 + 2 elements of u32 is a capacity overflow, and `usize::MAX + 2`
// overflows outright.
#[test]
fn a_huge_head_or_sequence_bound_builds_the_same_sequence_without_panicking() {
    let tok = StubTokenizer::new();
    let q = Question::choice("pick one", [("a", Value::Null), ("b", Value::Null)]).unwrap();
    let state = json!("state text");
    let base = SequenceBuilder::build(&tok, &state, &q, 512, 192, false).unwrap();
    for huge in [1usize << 61, i64::MAX as usize, 1usize << 63, usize::MAX] {
        for (max_len, head_max_len) in [(512, huge), (huge, 192), (huge, huge)] {
            let built =
                SequenceBuilder::build(&tok, &state, &q, max_len, head_max_len, false).unwrap();
            assert_eq!(base, built, "max_len={max_len} head_max_len={head_max_len}");
        }
    }
}