Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

17.8 KB
Raw Blame
//! Tier 2: the exported graph's dynamic axes, option logits and action probabilities, driven through
//! [`laya::LayaEngine::run`].
//!
//! The export was traced at one shape, so every axis being genuinely dynamic is a property of the
//! artifact rather than something the exporter guarantees. This sweep re-runs the 72 configurations
//! the Python probe covered, then checks the graph outputs against recorded ones, so a bad artifact
//! is caught here rather than as a puzzling answer mismatch elsewhere.
//!
//! Run the full suite (all three checkpoints) with:
//! `LAYA_ONNX_ROOT=<parent-of-checkpoints> cargo test --release -- --test-threads=1`
//! (see `predict_parity_tests.rs`'s module docs for why `--test-threads=1` matters here).

mod common;

use common::engines;
use common::golden_data::{self, CheckpointGoldenData};
use laya::calibration::Calibration;
use laya::collator::{CollatedBatch, Collator, SequenceItem};
use laya::laya_config::LayaConfig;
use laya::laya_options::LayaCheckpoint;
use laya::questions::QuestionType;
use serde_json::Value;

/// Candidate sequence lengths, straddling the multilingual traced length of 93. Each checkpoint's
/// actual sweep is this list filtered to `<= config.max_len`, with `max_len` always included.
const SEQUENCE_LENGTHS: [usize; 12] = [16, 32, 55, 59, 80, 92, 93, 94, 128, 256, 512, 1024];
/// Batch sizes: one row, and more than one.
const BATCH_SIZES: [usize; 2] = [1, 3];
/// Marker counts, from the graph's TopK floor of 2 upwards.
const MARKER_COUNTS: [usize; 3] = [2, 5, 9];

fn shapes() -> impl Iterator<Item = (usize, usize, usize)> {
    SEQUENCE_LENGTHS.into_iter().flat_map(|seq| {
        BATCH_SIZES.into_iter().flat_map(move |batch| {
            MARKER_COUNTS
                .into_iter()
                .map(move |markers| (seq, batch, markers))
        })
    })
}

/// The sequence lengths to sweep for one checkpoint: [`SEQUENCE_LENGTHS`] filtered to
/// `<= max_len`, with `max_len` itself always included.
fn checkpoint_sequence_lengths(checkpoint: LayaCheckpoint) -> Vec<usize> {
    // Derive max_len from the artifact config without loading the ONNX graph. Default to 1024
    // (the largest across all checkpoints) when the config cannot be read; the caller's
    // `engines::load` will then skip the whole test anyway if artifacts are absent.
    let max_len = common::test_artifacts::locate(checkpoint)
        .and_then(|dir| LayaConfig::load(dir.join("rl_agent_config.json")).ok())
        .map(|cfg| cfg.max_len)
        .unwrap_or(1024);

    let mut lens: Vec<usize> = SEQUENCE_LENGTHS
        .into_iter()
        .filter(|&s| s <= max_len)
        .collect();
    if !lens.contains(&max_len) {
        lens.push(max_len);
    }
    lens.sort_unstable();
    lens
}

/// Same as [`checkpoint_sequence_lengths`], but reads `rl_agent_config.json` from the split-layout
/// export directory (`<repo>/onnx-split/<ckpt>`) instead of the fused one — the two exports for the
/// same checkpoint are expected to share the same `max_len`, but this reads it from the directory
/// actually under test rather than assuming that.
fn checkpoint_sequence_lengths_split(checkpoint: LayaCheckpoint) -> Vec<usize> {
    let max_len = common::test_artifacts::locate_split(checkpoint)
        .and_then(|dir| LayaConfig::load(dir.join("rl_agent_config.json")).ok())
        .map(|cfg| cfg.max_len)
        .unwrap_or(1024);

    let mut lens: Vec<usize> = SEQUENCE_LENGTHS
        .into_iter()
        .filter(|&s| s <= max_len)
        .collect();
    if !lens.contains(&max_len) {
        lens.push(max_len);
    }
    lens.sort_unstable();
    lens
}

fn item(seq: usize, markers: usize, qtype: QuestionType) -> SequenceItem {
    // A plausible sequence: CLS, ordinary vocabulary ids, SEP. The ids only have to be in range —
    // this exercises shapes, not meaning.
    let mut ids = vec![0u32; seq];
    if seq > 0 {
        ids[0] = 2;
    }
    for (i, id) in ids
        .iter_mut()
        .enumerate()
        .take(seq.saturating_sub(1))
        .skip(1)
    {
        *id = 100 + (i % 5000) as u32;
    }
    if seq > 0 {
        ids[seq - 1] = 1;
    }
    // Markers sit just after CLS and stay inside the sequence.
    let positions: Vec<usize> = (0..markers).map(|i| 1 + i).collect();
    SequenceItem::new(ids, positions, qtype)
}

fn synthesize(seq: usize, batch: usize, markers: usize) -> CollatedBatch {
    let types = [
        QuestionType::Choice,
        QuestionType::Score,
        QuestionType::Noul,
    ];
    let items: Vec<SequenceItem> = (0..batch)
        .map(|r| item(seq, markers, types[r % types.len()]))
        .collect();
    Collator::collate(&items, 0).expect("collate should not fail for a non-empty batch")
}

/// Reconstructs items from a golden case's recorded `batch` field, dropping the padding the
/// collator will re-add, so this isolates the ONNX run from tokenization: a mismatch is the graph
/// or the tensor marshalling, nothing else.
fn from_recorded(recorded: &Value, pad_id: u32) -> CollatedBatch {
    let attention = recorded["attention_mask"]
        .as_array()
        .expect("attention_mask");
    let input_ids = recorded["input_ids"].as_array().expect("input_ids");
    let marker_pos = recorded["marker_pos"].as_array().expect("marker_pos");
    let marker_mask = recorded["marker_mask"].as_array().expect("marker_mask");
    let qtype = recorded["qtype"].as_array().expect("qtype");

    let mut items = Vec::with_capacity(input_ids.len());
    for r in 0..input_ids.len() {
        let length = attention[r]
            .as_array()
            .unwrap()
            .iter()
            .filter(|v| v.as_i64().unwrap() != 0)
            .count();
        let ids: Vec<u32> = golden_data::ints(&input_ids[r])
            .into_iter()
            .take(length)
            .map(|v| v as u32)
            .collect();

        let positions = golden_data::indices(&marker_pos[r]);
        let mask_row = marker_mask[r].as_array().unwrap();
        let markers: Vec<usize> = positions
            .into_iter()
            .enumerate()
            .filter(|(i, _)| mask_row[*i].as_bool().unwrap())
            .map(|(_, pos)| pos)
            .collect();

        let qt = QuestionType::try_from(qtype[r].as_i64().unwrap()).expect("valid qtype");
        items.push(SequenceItem::new(ids, markers, qt));
    }

    Collator::collate(&items, pad_id).expect("collate")
}

fn assert_close_slices(expected: &[f32], actual: &[f32], tolerance: f64, what: &str) {
    assert_eq!(expected.len(), actual.len(), "{what}: length mismatch");
    let mut worst = 0.0f64;
    let mut at = 0usize;
    for (i, (&e, &a)) in expected.iter().zip(actual.iter()).enumerate() {
        assert!(
            e.is_finite() && a.is_finite(),
            "{what}: nonfinite value at index {i}"
        );
        let diff = (f64::from(e) - f64::from(a)).abs();
        if diff > worst {
            worst = diff;
            at = i;
        }
    }
    eprintln!("{what}: worst diff {worst} at index {at} (tolerance {tolerance})");
    assert!(
        worst <= tolerance,
        "{what}: worst difference {worst} at index {at} (python {}, rust {}), tolerance {tolerance}",
        expected.get(at).copied().unwrap_or(f32::NAN),
        actual.get(at).copied().unwrap_or(f32::NAN)
    );
}

fn assert_act_probabilities(expected: &[f32], actual: &[f32], classes: usize, what: &str) {
    assert!(classes > 0, "{what}: no action classes");
    assert_eq!(expected.len(), actual.len(), "{what}: length mismatch");
    assert_eq!(expected.len() % classes, 0, "{what}: incomplete action row");
    assert!(
        expected.iter().chain(actual).all(|v| v.is_finite()),
        "{what}: nonfinite logits"
    );
    for (row, (e, a)) in expected
        .chunks_exact(classes)
        .zip(actual.chunks_exact(classes))
        .enumerate()
    {
        let e: Vec<f32> = Calibration::softmax(e)
            .into_iter()
            .map(|v| v as f32)
            .collect();
        let a: Vec<f32> = Calibration::softmax(a)
            .into_iter()
            .map(|v| v as f32)
            .collect();
        assert_close_slices(&e, &a, 1e-4, &format!("{what} row {row}"));
    }
}

#[test]
fn saturated_act_logit_drift_preserves_probabilities() {
    assert_act_probabilities(
        &[4807.72, -3937.599],
        &[4807.878, -3937.442],
        2,
        "saturated",
    );
}

#[test]
#[should_panic(expected = "worst difference")]
fn small_act_logit_drift_with_real_signal_is_rejected() {
    assert_act_probabilities(&[0.0, 0.0], &[0.005, 0.0], 2, "signal");
}

#[test]
#[should_panic(expected = "nonfinite logits")]
fn nonfinite_act_logits_are_rejected() {
    assert_act_probabilities(&[0.0, 0.0], &[f32::NAN, 0.0], 2, "invalid");
}

// ── multilingual-only shape tests ────────────────────────────────────────────

#[test]
fn runs_at_every_shape() {
    let Some(engine) = engines::load_selected(LayaCheckpoint::Multilingual) else {
        return;
    };

    for (seq, batch, markers) in shapes() {
        let collated = synthesize(seq, batch, markers);
        let output = engine
            .run(&collated)
            .unwrap_or_else(|e| panic!("seq={seq} batch={batch} markers={markers}: {e}"));

        assert_eq!(
            batch * markers,
            output.logits.len(),
            "seq={seq} batch={batch} markers={markers}: logits length"
        );
        assert_eq!(
            batch * 2,
            output.act_logits.len(),
            "seq={seq} batch={batch} markers={markers}: act_logits length"
        );
        for &v in &output.logits {
            assert!(
                v.is_finite(),
                "non-finite logit at seq={seq}, markers={markers}"
            );
        }
        for &v in &output.act_logits {
            assert!(v.is_finite(), "non-finite act logit at seq={seq}");
        }
    }
}

#[test]
fn reproduces_the_recorded_graph_outputs() {
    let Some(engine) = engines::load_selected(LayaCheckpoint::Multilingual) else {
        return;
    };
    let golden = CheckpointGoldenData::for_checkpoint(LayaCheckpoint::Multilingual.subdir());
    if !golden.available() {
        eprintln!("skip: no golden data for 'multilingual'");
        return;
    }

    let pad_id = golden.meta()["special_tokens"]["pad_id"].as_u64().unwrap() as u32;

    for case in golden.success_cases() {
        let root = golden.load_case(&case);
        // Feed the tensors Python recorded rather than ones rebuilt here, so this isolates the
        // ONNX run from tokenization: a mismatch is the graph or the tensor marshalling, nothing
        // else.
        let batch = from_recorded(&root["batch"], pad_id);
        let output = engine
            .run(&batch)
            .unwrap_or_else(|e| panic!("{case}: run failed: {e}"));

        let mut rows = 0;
        let mut cols = 0;
        let expected_logits = golden_data::matrix(&root["logits"], &mut rows, &mut cols);
        assert_eq!(batch.count, rows, "{case}: logits row count");
        assert_eq!(batch.marker_count, cols, "{case}: logits col count");
        // Logits live around +/-10 (and exactly -1e4 in padded columns), so this is a tight bound.
        assert_close_slices(
            &expected_logits,
            &output.logits,
            2e-3,
            &format!("{case}: logits"),
        );

        // Match the unrounded action signal predict consumes, not CPU-kernel drift
        // in large saturated logits. Keep the raw logits in the goldens for diagnostics.
        let expected_act = golden_data::matrix(&root["act_logits"], &mut rows, &mut cols);
        assert_eq!(batch.count, rows, "{case}: action row count");
        assert_act_probabilities(
            &expected_act,
            &output.act_logits,
            cols,
            &format!("{case}: act_probs"),
        );
    }
}

#[test]
fn keeps_padded_marker_columns_at_the_masked_fill_value() {
    // The model writes -1e4 into columns marker_mask says are padding. Calibration relies on that
    // to keep a padded column out of a distribution, so it is worth pinning on the graph itself.
    let Some(engine) = engines::load_selected(LayaCheckpoint::Multilingual) else {
        return;
    };

    let items = vec![
        item(32, 4, QuestionType::Choice),
        item(32, 2, QuestionType::Choice),
    ];
    let batch = Collator::collate(&items, 0).expect("collate");
    let output = engine.run(&batch).expect("run");

    assert_eq!(4, batch.marker_count);
    let row1 = batch.marker_count; // second row's offset into the flat logits vec
    assert!(
        output.logits[row1 + 2] < -1e3,
        "padded column carried {}",
        output.logits[row1 + 2]
    );
    assert!(
        output.logits[row1 + 3] < -1e3,
        "padded column carried {}",
        output.logits[row1 + 3]
    );
    assert!(output.logits[row1 + 1].is_finite());
}

#[test]
fn runs_a_single_option_question_that_topk_would_otherwise_reject() {
    // The graph has TopK k=2 baked in, so a one-column marker axis fails outright. Collator widens
    // it to two; this is the test that the widening is what actually keeps the graph runnable.
    let Some(engine) = engines::load_selected(LayaCheckpoint::Multilingual) else {
        return;
    };

    let items = vec![item(24, 1, QuestionType::Choice)];
    let batch = Collator::collate(&items, 0).expect("collate");
    assert_eq!(2, batch.marker_count);

    let output = engine.run(&batch).expect("run");
    assert!(output.logits[0].is_finite());
}

// ── all-checkpoint shape test ────────────────────────────────────────────────

#[test]
fn runs_at_every_shape_for_all_checkpoints() {
    for checkpoint in engines::checkpoints() {
        let Some(engine) = engines::load(checkpoint) else {
            continue;
        };

        for seq in checkpoint_sequence_lengths(checkpoint) {
            for batch in BATCH_SIZES {
                for markers in MARKER_COUNTS {
                    let collated = synthesize(seq, batch, markers);
                    let output = engine.run(&collated).unwrap_or_else(|e| {
                        panic!(
                            "{}: seq={seq} batch={batch} markers={markers}: {e}",
                            checkpoint.subdir()
                        )
                    });

                    assert_eq!(
                        batch * markers,
                        output.logits.len(),
                        "{}: seq={seq} batch={batch} markers={markers}: logits length",
                        checkpoint.subdir()
                    );
                    assert_eq!(
                        batch * 2,
                        output.act_logits.len(),
                        "{}: seq={seq} batch={batch} markers={markers}: act_logits length",
                        checkpoint.subdir()
                    );
                    for &v in &output.logits {
                        assert!(
                            v.is_finite(),
                            "{}: non-finite logit: seq={seq} markers={markers}",
                            checkpoint.subdir()
                        );
                    }
                    for &v in &output.act_logits {
                        assert!(
                            v.is_finite(),
                            "{}: non-finite act logit: seq={seq}",
                            checkpoint.subdir()
                        );
                    }
                }
            }
        }
        // `engine` drops here, before the next checkpoint's is loaded.
    }
}

/// Same sweep as [`runs_at_every_shape_for_all_checkpoints`], but against each checkpoint's
/// split-layout export (`<repo>/onnx-split/<ckpt>`, the layout laya-ts's exporter produces:
/// `encoder.onnx` + `head.onnx`) instead of the fused `model.onnx`. Skips per checkpoint exactly
/// like the fused sweep does when that checkpoint's split-layout export is not present locally.
#[test]
fn runs_at_every_shape_for_all_checkpoints_split_layout() {
    for checkpoint in engines::checkpoints() {
        let Some(engine) = engines::load_split(checkpoint) else {
            continue;
        };

        for seq in checkpoint_sequence_lengths_split(checkpoint) {
            for batch in BATCH_SIZES {
                for markers in MARKER_COUNTS {
                    let collated = synthesize(seq, batch, markers);
                    let output = engine.run(&collated).unwrap_or_else(|e| {
                        panic!(
                            "{} (split layout): seq={seq} batch={batch} markers={markers}: {e}",
                            checkpoint.subdir()
                        )
                    });

                    assert_eq!(
                        batch * markers,
                        output.logits.len(),
                        "{} (split layout): seq={seq} batch={batch} markers={markers}: logits length",
                        checkpoint.subdir()
                    );
                    assert_eq!(
                        batch * 2,
                        output.act_logits.len(),
                        "{} (split layout): seq={seq} batch={batch} markers={markers}: act_logits length",
                        checkpoint.subdir()
                    );
                    for &v in &output.logits {
                        assert!(
                            v.is_finite(),
                            "{} (split layout): non-finite logit: seq={seq} markers={markers}",
                            checkpoint.subdir()
                        );
                    }
                    for &v in &output.act_logits {
                        assert!(
                            v.is_finite(),
                            "{} (split layout): non-finite act logit: seq={seq}",
                            checkpoint.subdir()
                        );
                    }
                }
            }
        }
        // `engine` drops here, before the next checkpoint's is loaded.
    }
}