Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

21.8 KB
Raw Blame
//! `predict_long`: the model-free pieces (router capability check, tokenizer trait default) and the
//! model-backed parity of every recorded `"kind": "long"` golden case.
//!
//! The goldens come from the genuine `ONNXAgent.predict_long` over the same ONNX graph. Besides the
//! answers, window attribution and usage, the cases record the tokenized state length, the
//! effective window/stride/room and each window's decoded text, which pin the token offsets, the
//! decoder's special-token retention and the budget arithmetic independently of rounded scores.
//!
//! Run with `LAYA_ONNX_ROOT` (and `LAYA_ONNX_SPLIT_ROOT` for the split layout) set and
//! `--test-threads=1`, as for the other model-backed tests.

mod common;

use common::engines;
use common::golden_data::{self, CheckpointGoldenData};
use common::result_compare::assert_answers_match;
use common::stub_tokenizer::StubTokenizer;
use laya::answers::{LayaResult, Usage, WindowInfo};
use laya::error::LayaError;
use laya::laya_engine::LayaEngine;
use laya::laya_options::LayaCheckpoint;
use laya::laya_router::{LayaPredictor, LayaRouter, LayaRouterOptions};
use laya::long_prediction::{LongOptions, window_budget};
use laya::python_json::PythonJson;
use laya::questions::{Question, QuestionSet};
use laya::tokenization::{HfTokenizer, LayaTokenizer};
use serde_json::Value;
use std::collections::HashMap;
use std::sync::Arc;

// ---------------------------------------------------------------------- model-free

#[test]
fn a_tokenizer_without_decode_reports_it_instead_of_breaking_the_trait() {
    // `StubTokenizer` implements only the required methods; `decode` is the trait's default.
    let error = StubTokenizer::new().decode(&[1, 2, 3]).unwrap_err();
    assert!(matches!(error, LayaError::Tokenizer(_)), "got {error:?}");
}

struct EncodeOnlyPredictor;

impl LayaPredictor for EncodeOnlyPredictor {
    fn predict(&self, _state: Value, questions: &QuestionSet) -> laya::Result<LayaResult> {
        Ok(LayaResult::new(
            "fake",
            questions.ids().map(String::from).collect(),
            HashMap::new(),
            Usage::default(),
        ))
    }
}

#[test]
fn a_router_never_silently_truncates_through_a_predictor_without_predict_long() {
    let router = LayaRouter::with_factory(LayaRouterOptions::default(), |_, _| {
        Ok(Arc::new(EncodeOnlyPredictor) as Arc<dyn LayaPredictor>)
    })
    .expect("router with a fake factory");
    let questions = QuestionSet::new()
        .with("flag", Question::noul("Is it?", Value::Null, Value::Null))
        .unwrap();

    let error = router
        .predict_long(
            "some state",
            &questions,
            &LongOptions::new(),
            Some("en"),
            None,
            None,
        )
        .unwrap_err();
    assert!(
        matches!(error, LayaError::NotSupported(_)),
        "expected NotSupported, got {error:?}"
    );
    assert!(error.to_string().contains("no predict_long"), "{error}");
}

#[test]
fn long_options_treat_zero_like_unset_and_take_chained_setters() {
    let options = LongOptions::new().window(64).stride(32).batch_size(2);
    assert_eq!(options.window, Some(64));
    assert_eq!(options.stride, Some(32));
    assert_eq!(options.batch_size, Some(2));
    assert_eq!(options.aggregate, None);
    assert_eq!(LongOptions::default(), LongOptions::new());
}

// ---------------------------------------------------------------------- model-backed parity

fn option_usize(options: &Value, name: &str) -> Option<usize> {
    options
        .get(name)
        .and_then(Value::as_u64)
        .map(|v| v as usize)
}

fn long_options(options: &Value) -> LongOptions {
    let mut built = LongOptions::new();
    built.window = option_usize(options, "window");
    built.stride = option_usize(options, "stride");
    built.batch_size = option_usize(options, "batch_size");
    built.aggregate = options
        .get("aggregate")
        .and_then(Value::as_str)
        .map(str::to_string);
    built.lang = golden_data::recorded_lang(options);
    built
}

fn expect_usize(value: &Value, what: &str) -> usize {
    value
        .as_u64()
        .unwrap_or_else(|| panic!("{what}: expected a non-negative integer, got {value}"))
        as usize
}

fn check_long_usage(where_case: &str, want: &Value, got: &LayaResult) {
    assert_eq!(
        expect_usize(&want["input_tokens"], "input_tokens") as u32,
        got.usage().input_tokens,
        "{where_case}: usage.input_tokens"
    );
    assert_eq!(
        expect_usize(&want["output_tokens"], "output_tokens") as u32,
        got.usage().output_tokens,
        "{where_case}: usage.output_tokens"
    );
    let scan = got
        .scan_usage()
        .unwrap_or_else(|| panic!("{where_case}: predict_long must report scan usage"));
    assert_eq!(
        expect_usize(&want["windows"], "windows"),
        scan.windows,
        "{where_case}: usage.windows"
    );

    let Some(state) = got.state_usage() else {
        // Only a call with no questions has no per-state diagnostics, and Python then omits them.
        for key in [
            "state_tokens",
            "state_tokens_dropped",
            "truncated",
            "truncated_questions",
        ] {
            assert!(
                want.get(key).is_none(),
                "{where_case}: usage.{key} missing from result"
            );
        }
        return;
    };
    assert_eq!(
        expect_usize(&want["state_tokens"], "state_tokens"),
        state.state_tokens,
        "{where_case}: usage.state_tokens"
    );
    assert_eq!(
        expect_usize(&want["state_tokens_dropped"], "state_tokens_dropped"),
        state.state_tokens_dropped,
        "{where_case}: usage.state_tokens_dropped"
    );
    // Python reports a bool for one pass and a count of truncated windows for a scan.
    let truncated = match &want["truncated"] {
        Value::Bool(b) => usize::from(*b),
        other => expect_usize(other, "truncated"),
    };
    assert_eq!(
        truncated, scan.truncated_windows,
        "{where_case}: usage.truncated"
    );
    assert_eq!(
        truncated > 0,
        state.truncated,
        "{where_case}: truncated flag"
    );
    let ids: Vec<&str> = want["truncated_questions"]
        .as_array()
        .unwrap()
        .iter()
        .map(|id| id.as_str().unwrap())
        .collect();
    let got_ids: Vec<&str> = state
        .truncated_questions
        .iter()
        .map(String::as_str)
        .collect();
    assert_eq!(ids, got_ids, "{where_case}: usage.truncated_questions");

    let want_options = want.get("options").and_then(Value::as_object);
    assert_eq!(
        want_options.map_or(0, |o| o.len()),
        state.options.len(),
        "{where_case}: usage.options count"
    );
    if let Some(want_options) = want_options {
        let want_ids: Vec<&str> = want_options.keys().map(String::as_str).collect();
        let got_ids: Vec<&str> = state.options.keys().map(String::as_str).collect();
        assert_eq!(want_ids, got_ids, "{where_case}: usage.options ids");
        for (id, expected) in want_options {
            let got = &state.options[id];
            assert_eq!(
                expect_usize(&expected["total"], "total"),
                got.total,
                "{where_case}/{id}: options.total"
            );
            assert_eq!(
                expect_usize(&expected["distinct"], "distinct"),
                got.distinct,
                "{where_case}/{id}: options.distinct"
            );
            assert_eq!(
                expected["tokens_per_option"].as_u64().map(|v| v as usize),
                got.tokens_per_option,
                "{where_case}/{id}: options.tokens_per_option"
            );
        }
    }
}

fn check_windows(where_case: &str, want: &Value, got: &LayaResult) {
    let answers = want["answers"].as_object().unwrap();
    for (id, expected) in answers {
        let answer = got
            .get(id)
            .unwrap_or_else(|| panic!("{where_case}/{id}: missing answer"));
        match expected.get("window") {
            Some(window) => assert_eq!(
                Some(WindowInfo {
                    index: expect_usize(&window["index"], "index"),
                    token_start: expect_usize(&window["token_start"], "token_start"),
                    token_end: expect_usize(&window["token_end"], "token_end"),
                    count: expect_usize(&window["count"], "count"),
                }),
                answer.window(),
                "{where_case}/{id}: window attribution"
            ),
            None => assert_eq!(
                None,
                answer.window(),
                "{where_case}/{id}: single-pass answer must carry no window"
            ),
        }
    }
}

fn check_long_case(engine: &LayaEngine, where_case: &str, root: &Value) {
    let state = root["state"].clone();
    let questions = golden_data::build_questions(&root["questions"])
        .unwrap_or_else(|e| panic!("{where_case}: {e}"));
    let options = &root["options"];
    let got = engine
        .predict_long(state.clone(), &questions, &long_options(options))
        .unwrap_or_else(|e| panic!("{where_case}: predict_long failed: {e}"));
    let expected = &root["result"];

    // The SDK keeps its own model identifier instead of the ONNX agent's suffix.
    assert_eq!("laya-rl-agent-onnx", expected["model"].as_str().unwrap());
    assert_eq!("laya-rl-agent", got.model(), "{where_case}: model");
    check_long_usage(where_case, &expected["usage"], &got);
    assert_answers_match(where_case, expected, &got);
    check_windows(where_case, expected, &got);

    // These comparisons pin token offsets, decoder cleanup and special-token retention
    // independently of model probabilities (a dropped token could otherwise round away).
    let artifacts = engine.artifacts();
    let tokenizer = HfTokenizer::from_file_with_config(
        &artifacts.tokenizer_path,
        artifacts.tokenizer_config_path.as_deref(),
    )
    .unwrap_or_else(|e| panic!("{where_case}: tokenizer: {e}"));
    let mask_token = tokenizer.special().mask_token.clone();
    let state_ids = tokenizer
        .encode(&PythonJson::state(&state).replace(&mask_token, " "))
        .unwrap();
    assert_eq!(
        expect_usize(&root["state_tokens"], "state_tokens"),
        state_ids.len(),
        "{where_case}: state token count"
    );
    let config = engine.config();
    let budget = window_budget(
        &tokenizer,
        &questions,
        config.max_len,
        config.head_max_len,
        option_usize(options, "window").unwrap_or(0),
        option_usize(options, "stride").unwrap_or(0),
    )
    .unwrap_or_else(|e| panic!("{where_case}: window_budget: {e}"));
    assert_eq!(
        expect_usize(&root["effective_window"], "effective_window"),
        budget.window,
        "{where_case}: effective window"
    );
    assert_eq!(
        expect_usize(&root["effective_stride"], "effective_stride"),
        budget.stride,
        "{where_case}: effective stride"
    );
    assert_eq!(
        expect_usize(&root["effective_room"], "effective_room"),
        budget.room,
        "{where_case}: effective room"
    );
    for window in root["windows"].as_array().unwrap() {
        let start = expect_usize(&window["token_start"], "token_start");
        let end = expect_usize(&window["token_end"], "token_end");
        assert_eq!(
            window["text"].as_str().unwrap(),
            tokenizer.decode(&state_ids[start..end]).unwrap(),
            "{where_case}: decoded window {start}..{end}"
        );
    }

    let windows = got.scan_usage().unwrap().windows;
    match root["name"].as_str().unwrap() {
        "long_exact_limit" => {
            assert_eq!(
                budget.window,
                state_ids.len(),
                "{where_case}: state at limit"
            );
            assert_eq!(1, windows, "{where_case}: exact-limit state is one pass");
        }
        "long_room_fast_path" => {
            assert!(
                state_ids.len() > budget.window,
                "{where_case}: state must exceed the default window"
            );
            assert_eq!(budget.room, state_ids.len(), "{where_case}: state at room");
            assert_eq!(1, windows, "{where_case}: room-sized state is one pass");
        }
        _ => {}
    }

    if windows == 1 && !questions.is_empty() {
        // The one-pass shortcut is the ordinary predict path, so it must agree with it exactly,
        // with the same language (Python's predict_long passes lang to system_one).
        let mut one_pass = laya::predict_options::PredictOptions::new();
        if let Some(lang) = root["options"]["lang"].as_str() {
            one_pass = one_pass.lang(lang);
        }
        let plain = engine
            .predict_with_options(state, &questions, &one_pass)
            .expect("plain predict");
        assert_eq!(plain.usage(), got.usage(), "{where_case}: one-pass usage");
        for id in questions.ids() {
            assert_eq!(
                plain[id].confidence(),
                got[id].confidence(),
                "{where_case}/{id}: one-pass confidence"
            );
            assert_eq!(
                plain[id].answer_confidence(),
                got[id].answer_confidence(),
                "{where_case}/{id}: one-pass answer_confidence"
            );
            assert_eq!(
                plain[id].action(),
                got[id].action(),
                "{where_case}/{id}: one-pass action"
            );
        }
    }
}

fn run_success_cases(
    load: fn(LayaCheckpoint) -> Option<LayaEngine>,
    label: &str,
    unavailable: &str,
) {
    let mut checked_any = false;
    for checkpoint in engines::checkpoints() {
        let Some(engine) = load(checkpoint) else {
            continue;
        };
        let golden = CheckpointGoldenData::for_checkpoint(checkpoint.subdir());
        if !golden.available() {
            eprintln!("skip: no golden data for '{}'", checkpoint.subdir());
            continue;
        }
        for case in golden.long_cases() {
            checked_any = true;
            let root = golden.load_case(&case);
            check_long_case(
                &engine,
                &format!("{}/{case}{label}", checkpoint.subdir()),
                &root,
            );
        }
    }
    if !checked_any {
        eprintln!("skip: {unavailable}");
    }
}

#[test]
fn reproduces_the_recorded_long_predictions() {
    run_success_cases(
        engines::load,
        "",
        "no long golden data + artifacts found for any checkpoint",
    );
}

#[test]
fn reproduces_the_recorded_long_predictions_for_the_split_layout() {
    run_success_cases(
        engines::load_split,
        " (split layout)",
        "no split-layout ONNX artifacts + long golden data found for any checkpoint",
    );
}

#[test]
fn rejects_the_same_long_requests_python_rejects() {
    let mut checked_any = false;
    for checkpoint in engines::checkpoints() {
        let Some(engine) = engines::load(checkpoint) else {
            continue;
        };
        let golden = CheckpointGoldenData::for_checkpoint(checkpoint.subdir());
        if !golden.available() {
            continue;
        }
        for case in golden.long_error_cases() {
            checked_any = true;
            let root = golden.load_case(&case);
            let questions = golden_data::build_questions(&root["questions"])
                .unwrap_or_else(|e| panic!("{}/{case}: {e}", checkpoint.subdir()));
            let error = engine
                .predict_long(
                    root["state"].clone(),
                    &questions,
                    &long_options(&root["options"]),
                )
                .expect_err(&format!(
                    "{}/{case}: expected predict_long to fail",
                    checkpoint.subdir()
                ));
            let expected_message = root["error_message"].as_str().unwrap();
            let actual_message = error.to_string();
            assert!(
                actual_message.starts_with(expected_message),
                "{}/{case}: error message mismatch\n  python: {expected_message}\n  rust:   {actual_message}",
                checkpoint.subdir()
            );
            let name = root["name"].as_str().unwrap();
            let right_variant = match name {
                "long_invalid_aggregate" => matches!(error, LayaError::UnsupportedAggregate(_)),
                _ => matches!(error, LayaError::InvalidWindow(_)),
            };
            assert!(
                right_variant,
                "{}/{case}: wrong error variant: {error:?}",
                checkpoint.subdir()
            );
        }
    }
    if !checked_any {
        eprintln!("skip: no long error-case golden data + artifacts found for any checkpoint");
    }
}

// ---------------------------------------------------------------------- deferred config types

/// Hard-links every file under `from` into `to` (the weights are over a gigabyte), copying only
/// files small enough that a copy is harmless when linking is not possible.
fn link_tree(from: &std::path::Path, to: &std::path::Path) {
    std::fs::create_dir_all(to).unwrap();
    for entry in std::fs::read_dir(from).unwrap() {
        let entry = entry.unwrap();
        let (source, target) = (entry.path(), to.join(entry.file_name()));
        if source.is_dir() {
            link_tree(&source, &target);
        } else if std::fs::hard_link(&source, &target).is_err() {
            let size = std::fs::metadata(&source).unwrap().len();
            assert!(
                size < 64 * 1024 * 1024,
                "cannot hard-link {} ({size} bytes) into the scratch checkpoint",
                source.display()
            );
            std::fs::copy(&source, &target).unwrap();
        }
    }
}

struct ScratchDir(std::path::PathBuf);

impl Drop for ScratchDir {
    fn drop(&mut self) {
        let _ = std::fs::remove_dir_all(&self.0);
    }
}

/// Checkpoint configs whose `max_len` / `head_max_len` / `temperature` Python reads loosely. A
/// wrongly typed or negative length is refused when the engine is created, with the `TypeError`
/// text Python raises at the first prediction; a `temperature` list shorter than three loads and
/// fails with Python's `IndexError` for a question type whose index it lacks; a longer one is
/// fine. The messages were produced by Python's `common.build_sequence` and `_decode_answers`.
#[test]
fn config_lengths_and_temperatures_are_refused_or_deferred_as_agreed() {
    if !common::test_artifacts::is_selected(LayaCheckpoint::English) {
        return;
    }
    let Some(source) = common::test_artifacts::locate(LayaCheckpoint::English).filter(|dir| {
        dir.join("model.onnx").is_file() && dir.join("tokenizer").join("tokenizer.json").is_file()
    }) else {
        eprintln!("skip: no ONNX artifacts for 'english'; set LAYA_ONNX_ROOT");
        return;
    };
    let config_text = std::fs::read_to_string(source.join("rl_agent_config.json")).unwrap();
    let questions = QuestionSet::new()
        .with(
            "flag",
            Question::noul("Is it urgent?", Value::Null, Value::Null),
        )
        .unwrap();
    let scratch = ScratchDir(
        std::env::temp_dir().join(format!("laya-deferred-length-{}", std::process::id())),
    );
    let operand =
        |kind: &str| format!("TypeError: unsupported operand type(s) for -: '{kind}' and 'int'");
    let make = |change: Option<(&str, Value)>| {
        let _ = std::fs::remove_dir_all(&scratch.0);
        link_tree(&source, &scratch.0);
        let config_path = scratch.0.join("rl_agent_config.json");
        std::fs::remove_file(&config_path).unwrap();
        let mut config: Value = serde_json::from_str(&config_text).unwrap();
        if let Some((key, value)) = change {
            config[key] = value;
        }
        std::fs::write(&config_path, serde_json::to_string(&config).unwrap()).unwrap();
        LayaEngine::from_directory(&scratch.0)
    };

    // Refused at create / from_directory.
    let refused = [
        (("max_len", Value::from("512")), operand("str")),
        (("head_max_len", Value::Null), operand("NoneType")),
        (
            ("max_len", Value::from(300.5)),
            "TypeError: slice indices must be integers or None or have an __index__ method"
                .to_string(),
        ),
        (
            ("max_len", Value::from(-5)),
            "max_len=-5: must be a non-negative integer".to_string(),
        ),
    ];
    for (change, expected) in refused {
        match make(Some(change)) {
            Err(LayaError::InvalidConfig(got)) => assert_eq!(expected, got),
            Err(other) => panic!("expected InvalidConfig({expected}), got {other:?}"),
            Ok(_) => panic!("expected InvalidConfig({expected}), but the engine loaded"),
        }
    }

    // Loads; the short temperature list fails predict and predict_long (a noul question needs
    // index 2); a longer list and the shipped config predict normally.
    let cases = [
        (
            Some(("temperature", serde_json::json!([1.0]))),
            Some("IndexError: list index out of range"),
        ),
        (
            Some(("temperature", serde_json::json!([1.0, 1.0, 1.0, 9.0]))),
            None,
        ),
        (None, None),
    ];
    for (change, expected) in cases {
        let engine = make(change).expect("these configs load");
        let predicted = engine.predict("Please refund my order.", &questions);
        let scanned =
            engine.predict_long("Please refund my order.", &questions, &LongOptions::new());
        match expected {
            Some(message) => {
                for outcome in [predicted, scanned] {
                    match outcome {
                        Err(LayaError::InvalidConfig(got)) => assert_eq!(message, got),
                        other => panic!("expected InvalidConfig({message}), got {other:?}"),
                    }
                }
            }
            None => {
                predicted.expect("predicts normally");
                scanned.expect("scans normally");
            }
        }
        drop(engine);
    }
}