Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

ci(rust): shard the parity lane per checkpoint

kevin-gatimud0783005e9c4
diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml
index daa4efa3..c264017c 100644
--- a/.github/workflows/rust.yml
+++ b/.github/workflows/rust.yml
@@ -25,21 +25,53 @@ env:
jobs:
parity:
- name: build + Python parity (all checkpoints)
+ name: parity (${{ matrix.name }})
runs-on: ubuntu-latest
continue-on-error: true
- # 240, not 120, because the suite replays every golden through real ONNX inference and the
- # case count only goes up. At 120 this job hit the wall and GitHub recorded it as CANCELLED,
- # which reads like a superseded run rather than a budget failure: every test that ran had
- # passed, `predict_long_tests` was killed mid-run and `predict_parity_tests` never started.
- # Measured on a7327676: `predict_batch_tests` alone takes 4142s, 69 minutes, of which the two
- # `reproduces_the_recorded_batch_predictions` replays are ~34 minutes each (the second repeats
- # the whole set against the split layout). The lane was already at 83 minutes on main before
- # those landed, so it was one merge away from this.
+ # Sharded per checkpoint. The replays are serial only because each one loads models, and that
+ # constraint is per checkpoint rather than across them, so one job per checkpoint runs in
+ # parallel and each has its own budget instead of one pot that the next parity PR spends.
#
- # The headroom is not the fix. Sharding the replays across jobs is, since they are serial only
- # because each one loads models, and that is per checkpoint rather than across them.
- timeout-minutes: 240
+ # The limits come from the slow-host estimates in the runtime investigation, with headroom:
+ # english about 61 min, typed-decisions about 69, multilingual about 21, and the router cell
+ # (two engines, a handful of requests) well under that. The same job has run in about 25 and
+ # about 47 minutes depending on the host it landed on, so each limit is roughly 1.4x the
+ # slow-host figure. A limit that is hit is recorded by GitHub as CANCELLED, which reads like
+ # a superseded run rather than a budget failure, so these should not be tight.
+ timeout-minutes: ${{ matrix.timeout }}
+ strategy:
+ fail-fast: false
+ # Two cells at a time, as in dotnet.yml: every cell fetches weights from the same Hugging
+ # Face repo and unauthenticated fetches get throttled when several run together. The
+ # exports are cached per checkpoint below, so a fetch only happens on a cache miss.
+ max-parallel: 2
+ matrix:
+ include:
+ # --workspace: the model-free tests run in every checkpoint cell (seconds), so each
+ # floor is that count plus the cell's own model-backed tests.
+ - name: english
+ checkpoints: english
+ test-args: --workspace
+ timeout: 90
+ min-passed: 350
+ - name: multilingual
+ checkpoints: multilingual
+ test-args: --workspace
+ timeout: 40
+ min-passed: 350
+ - name: typed-decisions
+ checkpoints: typed-decisions
+ test-args: --workspace
+ timeout: 100
+ min-passed: 350
+ # The router end-to-end tests load english and multilingual side by side, so they run
+ # only in a cell that selects both. Only that test binary runs here: the checkpoint
+ # cells above own everything else.
+ - name: router
+ checkpoints: english multilingual
+ test-args: --test router_end_to_end_tests
+ timeout: 30
+ min-passed: 3
steps:
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
- uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
@@ -52,10 +84,59 @@ jobs:
python -m pip install --upgrade pip
pip install "torch==2.14.0" --index-url https://download.pytorch.org/whl/cpu
pip install -r laya-rust/tools/requirements-regen.txt
+ # Identical jobs have run in about 25 and about 47 minutes depending on the host, and the
+ # logs carried no hardware information. Print it so a slow run can be attributed.
+ - name: Runner hardware (diagnostic)
+ run: |
+ nproc
+ lscpu | grep -E 'Model name|^CPU\(s\)|Thread|Core'
- name: Test regeneration cache and skip guard
run: python laya-rust/tools/test_ci_tools.py
- - name: Regenerate all goldens from Python at this commit
- run: python laya-rust/tools/regen_golden.py
+
+ # The exports are the slow part (download, trace, verify). They depend only on the pinned
+ # Hugging Face revision (HF_REVISION in regen_golden.py), the two exporters, the pinned
+ # toolchain and the model definition in laya/common.py, so that is the cache key; the
+ # script re-checks the same inputs itself (a stamp) before it reuses a directory. The
+ # goldens are never cached: every cell re-records its own from Python at this commit.
+ - name: Cache pinned Hugging Face downloads
+ uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
+ with:
+ path: ~/.cache/huggingface/hub
+ key: laya-rust-hf-${{ runner.os }}-${{ matrix.name }}-${{ hashFiles('laya-rust/tools/regen_golden.py', 'laya-dotnet/tools/export_onnx.py', 'laya-ts/scripts/export_onnx.py') }}
+ - name: Cache exported english artifacts
+ if: ${{ contains(matrix.checkpoints, 'english') }}
+ uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
+ with:
+ path: |
+ onnx/english
+ onnx-split/english
+ key: laya-rust-onnx-${{ runner.os }}-english-${{ hashFiles('laya-rust/tools/regen_golden.py', 'laya-dotnet/tools/export_onnx.py', 'laya-ts/scripts/export_onnx.py', 'laya-rust/tools/requirements-regen.txt', 'laya/common.py') }}
+ - name: Cache exported multilingual artifacts
+ if: ${{ contains(matrix.checkpoints, 'multilingual') }}
+ uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
+ with:
+ path: |
+ onnx/multilingual
+ onnx-split/multilingual
+ key: laya-rust-onnx-${{ runner.os }}-multilingual-${{ hashFiles('laya-rust/tools/regen_golden.py', 'laya-dotnet/tools/export_onnx.py', 'laya-ts/scripts/export_onnx.py', 'laya-rust/tools/requirements-regen.txt', 'laya/common.py') }}
+ - name: Cache exported typed-decisions artifacts
+ if: ${{ contains(matrix.checkpoints, 'typed-decisions') }}
+ uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
+ with:
+ path: |
+ onnx/typed-decisions
+ onnx-split/typed-decisions
+ key: laya-rust-onnx-${{ runner.os }}-typed-decisions-${{ hashFiles('laya-rust/tools/regen_golden.py', 'laya-dotnet/tools/export_onnx.py', 'laya-ts/scripts/export_onnx.py', 'laya-rust/tools/requirements-regen.txt', 'laya/common.py') }}
+
+ - name: Export the checkpoints and regenerate their goldens from Python at this commit
+ env:
+ # Scoped to this step: only the step that fetches from the hub gets the credential.
+ # Unset expands to "", which the implicit token path treats as anonymous.
+ HF_TOKEN: ${{ secrets.HF_TOKEN }}
+ run: |
+ args=(--checkpoint routing)
+ for c in ${{ matrix.checkpoints }}; do args+=(--checkpoint "$c"); done
+ python laya-rust/tools/regen_golden.py "${args[@]}"
- name: Golden drift vs. committed (informational)
run: |
git --no-pager diff --stat -- laya-rust/tests/golden
@@ -70,19 +151,30 @@ jobs:
~/.cargo/git
laya-rust/target
key: rust-${{ runner.os }}-${{ hashFiles('laya-rust/Cargo.lock') }}
+ # The fused layout is found by walking up to <repo>/onnx and the split layout to
+ # <repo>/onnx-split, which is where regen_golden.py wrote them. LAYA_TEST_CHECKPOINTS
+ # limits the tests to this cell's checkpoints; a deselected checkpoint is skipped silently,
+ # a selected one with missing artifacts or goldens prints `skip:`, which the guard rejects.
+ # Debug profile on purpose. Measured on english with the same 371 tests passing in both: debug
+ # 1318 s warm, release 1296 s including a cold compile (tests alone 1117 s vs 1304 s). The time is
+ # ONNX Runtime native inference, which the profile does not change, so release buys about 2%.
- name: Cargo tests (serial model loads)
working-directory: laya-rust
shell: bash
+ env:
+ LAYA_TEST_CHECKPOINTS: ${{ matrix.checkpoints }}
run: |
set -o pipefail
- cargo test --workspace --locked -- --nocapture --test-threads=1 2>&1 | tee cargo-test.log
+ cargo test ${{ matrix.test-args }} --locked -- --nocapture --test-threads=1 2>&1 | tee cargo-test.log
+ # The floor is per cell, not one whole-lane number: a cell that loads no checkpoint would
+ # otherwise pass by scoring nothing while the others make up the count.
- name: No test was skipped
if: ${{ success() }}
- run: python laya-rust/tools/check_test_skips.py laya-rust/cargo-test.log --min-passed 200
+ run: python laya-rust/tools/check_test_skips.py laya-rust/cargo-test.log --min-passed ${{ matrix.min-passed }}
- uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
if: ${{ failure() }}
with:
- name: rust-parity-diagnostics
+ name: rust-parity-diagnostics-${{ matrix.name }}
path: |
laya-rust/cargo-test.log
laya-rust/tests/golden
diff --git a/laya-rust/README.md b/laya-rust/README.md
index 91f90c1a..31390147 100644
--- a/laya-rust/README.md
+++ b/laya-rust/README.md
@@ -719,9 +719,12 @@ root looking for `onnx/<checkpoint>/`. Tests whose artifacts are missing print a
return, rather than failing — a machine with no exported checkpoints still gets a green Tier-1 run.
The advisory [Rust parity lane](../.github/workflows/rust.yml) runs on **every push and PR**,
-with no path filter. It regenerates fixtures from the Python `laya/` at that commit using
-[`tools/regen_golden.py`](tools/regen_golden.py), then runs the whole Cargo workspace.
-Goldens are never cached. A `skip:` diagnostic or ignored test makes the lane red, even when
+with no path filter. It is sharded per checkpoint (english, multilingual, typed-decisions, plus
+a router cell for the tests that load english and multilingual together). Each cell regenerates
+its own checkpoint's goldens, and the routing goldens, from the Python `laya/` at that commit
+using [`tools/regen_golden.py`](tools/regen_golden.py), then runs the Cargo tests with
+`LAYA_TEST_CHECKPOINTS` set to its checkpoints and its own `--min-passed` floor.
+Goldens are never cached; the exports are, per checkpoint. A `skip:` diagnostic or ignored test makes the lane red, even when
Cargo counts a missing-model early return as a pass. Job-level `continue-on-error` keeps Python
improvements unblocked; Rust drift needs a follow-up port. This workflow never publishes crates.
The skip guard runs only after successful Cargo tests; regeneration/build failures retain their
@@ -760,6 +763,11 @@ stamp matches the pinned checkpoint, exporters, toolchain and Python model defin
The fused exporter is shared with `laya-dotnet/tools/export_onnx.py`; split exports use
`laya-ts/scripts/export_onnx.py`.
+`LAYA_TEST_CHECKPOINTS` (space- or comma-separated subdirectory names, e.g. `english` or
+`"english multilingual"`) limits the model-backed and golden-backed tests to those checkpoints; a
+deselected checkpoint is skipped silently, a selected one with missing artifacts or goldens prints a
+`skip:` line, and an unknown name panics. Unset runs every checkpoint.
+
Model loads are ~1.3–1.7 GB each; run model-backed tests serially (optionally with `--release`):
```bash
diff --git a/laya-rust/src/batch_prediction.rs b/laya-rust/src/batch_prediction.rs
index c9b8e8db..9fbb97c2 100644
--- a/laya-rust/src/batch_prediction.rs
+++ b/laya-rust/src/batch_prediction.rs
@@ -421,12 +421,22 @@ mod tests {
use crate::tokenization::HfTokenizer;
use serde_json::Value;
+ // `LAYA_TEST_CHECKPOINTS` (space- or comma-separated subdirectory names) narrows this to the
+ // checkpoints a parity job owns; unset runs all. Same semantics as `tests/common`.
+ let selected = std::env::var("LAYA_TEST_CHECKPOINTS").unwrap_or_default();
+ let selected: Vec<&str> = selected
+ .split(|c: char| c.is_whitespace() || c == ',')
+ .filter(|name| !name.is_empty())
+ .collect();
let mut checked_any = false;
for checkpoint in [
crate::LayaCheckpoint::Multilingual,
crate::LayaCheckpoint::English,
crate::LayaCheckpoint::TypedDecisions,
- ] {
+ ]
+ .into_iter()
+ .filter(|c| selected.is_empty() || selected.contains(&c.subdir()))
+ {
let Some(dir) = artifact_dir(checkpoint) else {
continue;
};
diff --git a/laya-rust/tests/calibration_tests.rs b/laya-rust/tests/calibration_tests.rs
index d61b8a51..2c0d1b5c 100644
--- a/laya-rust/tests/calibration_tests.rs
+++ b/laya-rust/tests/calibration_tests.rs
@@ -208,6 +208,9 @@ fn config_falls_back_to_pythons_in_code_defaults() {
#[test]
fn shipped_config_parses_as_recorded() {
+ if !common::test_artifacts::is_selected_subdir("multilingual") {
+ return;
+ }
let data = CheckpointGoldenData::for_checkpoint("multilingual");
if !data.available() {
eprintln!("skip: no golden data for 'multilingual'");
diff --git a/laya-rust/tests/collator_tests.rs b/laya-rust/tests/collator_tests.rs
index b1a84dbe..86ffa3fc 100644
--- a/laya-rust/tests/collator_tests.rs
+++ b/laya-rust/tests/collator_tests.rs
@@ -115,6 +115,9 @@ fn refuses_an_empty_batch() {
#[test]
fn reproduces_the_recorded_batch_tensors() {
+ if !common::test_artifacts::is_selected_subdir("multilingual") {
+ return;
+ }
let data = CheckpointGoldenData::for_checkpoint("multilingual");
if !data.available() {
eprintln!("skip: no golden data for 'multilingual'");
diff --git a/laya-rust/tests/common/engines.rs b/laya-rust/tests/common/engines.rs
index d370b5c2..178be840 100644
--- a/laya-rust/tests/common/engines.rs
+++ b/laya-rust/tests/common/engines.rs
@@ -21,6 +21,25 @@ pub const CHECKPOINTS: [LayaCheckpoint; 3] = [
LayaCheckpoint::TypedDecisions,
];
+/// The checkpoints this run covers: every one, or those `LAYA_TEST_CHECKPOINTS` selects (see
+/// [`test_artifacts::selected_subdirs`]). Loops over checkpoints use this, never [`CHECKPOINTS`]
+/// directly, so a parity job only touches the checkpoints it owns.
+pub fn checkpoints() -> Vec<LayaCheckpoint> {
+ CHECKPOINTS
+ .into_iter()
+ .filter(|c| test_artifacts::is_selected(*c))
+ .collect()
+}
+
+/// Like [`load`], for a test pinned to one checkpoint: returns `None` without a skip line when
+/// that checkpoint is not selected, because another parity job owns it.
+pub fn load_selected(checkpoint: LayaCheckpoint) -> Option<LayaEngine> {
+ if !test_artifacts::is_selected(checkpoint) {
+ return None;
+ }
+ load(checkpoint)
+}
+
/// Loads the engine for `checkpoint`, or prints a skip line and returns `None` when its ONNX
/// artifacts are not available locally. Panics (rather than returning `None`) if the artifacts
/// are present but fail to load — that is a real bug, not a missing-fixture skip.
diff --git a/laya-rust/tests/common/test_artifacts.rs b/laya-rust/tests/common/test_artifacts.rs
index 2c6d26a5..c9403a6a 100644
--- a/laya-rust/tests/common/test_artifacts.rs
+++ b/laya-rust/tests/common/test_artifacts.rs
@@ -4,6 +4,43 @@
use laya::laya_options::LayaCheckpoint;
use std::path::{Path, PathBuf};
+/// Every checkpoint subdirectory name `LAYA_TEST_CHECKPOINTS` may select.
+const KNOWN_SUBDIRS: [&str; 3] = ["multilingual", "english", "typed-decisions"];
+
+/// The checkpoint subdirectories a parity job selected with `LAYA_TEST_CHECKPOINTS` (names
+/// separated by spaces or commas), or `None` when the variable is unset or blank, which selects
+/// every checkpoint, so ordinary local runs are unchanged.
+///
+/// A deselected checkpoint is skipped silently, without a `skip:` line, because another job owns
+/// it. A selected checkpoint whose artifacts or goldens are missing still prints `skip:`, which
+/// `tools/check_test_skips.py` rejects. An unknown name panics rather than selecting nothing, so
+/// a typo cannot turn a job into one that tests no checkpoint.
+pub fn selected_subdirs() -> Option<Vec<String>> {
+ let raw = std::env::var("LAYA_TEST_CHECKPOINTS").unwrap_or_default();
+ let names: Vec<String> = raw
+ .split(|c: char| c.is_whitespace() || c == ',')
+ .filter(|name| !name.is_empty())
+ .map(str::to_owned)
+ .collect();
+ if names.is_empty() {
+ return None;
+ }
+ if let Some(unknown) = names.iter().find(|n| !KNOWN_SUBDIRS.contains(&n.as_str())) {
+ panic!("unknown LAYA_TEST_CHECKPOINTS entry '{unknown}' in '{raw}'");
+ }
+ Some(names)
+}
+
+/// Whether the checkpoint subdirectory `subdir` is selected for this run.
+pub fn is_selected_subdir(subdir: &str) -> bool {
+ selected_subdirs().is_none_or(|names| names.iter().any(|n| n == subdir))
+}
+
+/// Whether `checkpoint` is selected for this run (see [`selected_subdirs`]).
+pub fn is_selected(checkpoint: LayaCheckpoint) -> bool {
+ is_selected_subdir(checkpoint.subdir())
+}
+
/// Resolve the artifact directory for `checkpoint`, or `None` if it cannot be found.
///
/// Resolution order: `LAYA_ONNX_DIR` (multilingual only, backward-compat single-checkpoint env
diff --git a/laya-rust/tests/golden_questions_tests.rs b/laya-rust/tests/golden_questions_tests.rs
index c0938c51..bb191783 100644
--- a/laya-rust/tests/golden_questions_tests.rs
+++ b/laya-rust/tests/golden_questions_tests.rs
@@ -14,11 +14,18 @@ use serde_json::Value;
const CHECKPOINTS: [&str; 3] = ["multilingual", "english", "typed-decisions"];
+/// The checkpoints this run covers (all, or those `LAYA_TEST_CHECKPOINTS` selects).
+fn checkpoints() -> impl Iterator<Item = &'static str> {
+ CHECKPOINTS
+ .into_iter()
+ .filter(|c| common::test_artifacts::is_selected_subdir(c))
+}
+
#[test]
fn golden_questions_render_exactly_as_recorded() {
let mut checked_any = false;
- for checkpoint in CHECKPOINTS {
+ for checkpoint in checkpoints() {
let data = CheckpointGoldenData::for_checkpoint(checkpoint);
if !data.available() {
eprintln!("skip: no golden data for '{checkpoint}'");
diff --git a/laya-rust/tests/min_confidence_tests.rs b/laya-rust/tests/min_confidence_tests.rs
index cd495cca..63245132 100644
--- a/laya-rust/tests/min_confidence_tests.rs
+++ b/laya-rust/tests/min_confidence_tests.rs
@@ -688,7 +688,7 @@ fn probe_jev_confidence_matches_python() {
#[test]
fn the_recorded_min_confidence_cases_reproduce() {
let mut checked_any = false;
- for checkpoint in common::engines::CHECKPOINTS {
+ for checkpoint in common::engines::checkpoints() {
let Some(engine) = common::engines::load(checkpoint) else {
continue;
};
diff --git a/laya-rust/tests/predict_batch_tests.rs b/laya-rust/tests/predict_batch_tests.rs
index 07f49a51..231e35ba 100644
--- a/laya-rust/tests/predict_batch_tests.rs
+++ b/laya-rust/tests/predict_batch_tests.rs
@@ -10,7 +10,7 @@
mod common;
-use common::engines::{self, CHECKPOINTS};
+use common::engines;
use common::golden_data::{self, CheckpointGoldenData};
use common::result_compare::{assert_result_matches, assert_state_usage_matches};
use laya::answers::{LayaResult, Usage};
@@ -504,7 +504,7 @@ fn run_success_cases(
unavailable: &str,
) {
let mut checked_any = false;
- for checkpoint in CHECKPOINTS {
+ for checkpoint in engines::checkpoints() {
let Some(engine) = load(checkpoint) else {
continue;
};
@@ -549,7 +549,7 @@ fn reproduces_the_recorded_batch_predictions_for_the_split_layout() {
#[test]
fn rejects_the_same_batch_requests_python_rejects() {
let mut checked_any = false;
- for checkpoint in CHECKPOINTS {
+ for checkpoint in engines::checkpoints() {
let Some(engine) = engines::load(checkpoint) else {
continue;
};
@@ -619,7 +619,7 @@ fn rejects_the_same_batch_requests_python_rejects() {
#[test]
fn predict_is_a_one_state_batch_for_null_states_and_empty_questions() {
let mut checked_any = false;
- for checkpoint in CHECKPOINTS {
+ for checkpoint in engines::checkpoints() {
let Some(engine) = engines::load(checkpoint) else {
continue;
};
diff --git a/laya-rust/tests/predict_long_tests.rs b/laya-rust/tests/predict_long_tests.rs
index 38cf0033..f1afd46e 100644
--- a/laya-rust/tests/predict_long_tests.rs
+++ b/laya-rust/tests/predict_long_tests.rs
@@ -11,7 +11,7 @@
mod common;
-use common::engines::{self, CHECKPOINTS};
+use common::engines;
use common::golden_data::{self, CheckpointGoldenData};
use common::result_compare::assert_answers_match;
use common::stub_tokenizer::StubTokenizer;
@@ -364,7 +364,7 @@ fn run_success_cases(
unavailable: &str,
) {
let mut checked_any = false;
- for checkpoint in CHECKPOINTS {
+ for checkpoint in engines::checkpoints() {
let Some(engine) = load(checkpoint) else {
continue;
};
@@ -409,7 +409,7 @@ fn reproduces_the_recorded_long_predictions_for_the_split_layout() {
#[test]
fn rejects_the_same_long_requests_python_rejects() {
let mut checked_any = false;
- for checkpoint in CHECKPOINTS {
+ for checkpoint in engines::checkpoints() {
let Some(engine) = engines::load(checkpoint) else {
continue;
};
@@ -494,6 +494,9 @@ impl Drop for ScratchDir {
/// fine. The messages were produced by Python's `common.build_sequence` and `_decode_answers`.
#[test]
fn config_lengths_and_temperatures_are_refused_or_deferred_as_agreed() {
+ if !common::test_artifacts::is_selected(LayaCheckpoint::English) {
+ return;
+ }
let Some(source) = common::test_artifacts::locate(LayaCheckpoint::English).filter(|dir| {
dir.join("model.onnx").is_file() && dir.join("tokenizer").join("tokenizer.json").is_file()
}) else {
diff --git a/laya-rust/tests/predict_parity_tests.rs b/laya-rust/tests/predict_parity_tests.rs
index f126b9c7..34572d0c 100644
--- a/laya-rust/tests/predict_parity_tests.rs
+++ b/laya-rust/tests/predict_parity_tests.rs
@@ -13,7 +13,7 @@
mod common;
-use common::engines::{self, CHECKPOINTS};
+use common::engines;
use common::golden_data::{self, CheckpointGoldenData};
use laya::answers::{Answer, ChoiceAnswer, NoulAnswer, ScoreAnswer};
use laya::error::LayaError;
@@ -51,7 +51,7 @@ fn assert_close(expected: f64, actual: f64, tolerance: f64, what: &str) {
fn reproduces_the_recorded_answers() {
let mut checked_any = false;
- for checkpoint in CHECKPOINTS {
+ for checkpoint in engines::checkpoints() {
let Some(engine) = engines::load(checkpoint) else {
continue;
};
@@ -165,7 +165,7 @@ fn reproduces_the_recorded_answers_for_the_split_layout() {
// machines will not have one, and that is expected, not a failure.
let mut checked_any = false;
- for checkpoint in CHECKPOINTS {
+ for checkpoint in engines::checkpoints() {
let Some(engine) = engines::load_split(checkpoint) else {
continue;
};
@@ -248,7 +248,7 @@ fn reproduces_the_recorded_answers_for_the_split_layout() {
fn rejects_the_same_questions_python_rejects() {
let mut checked_any = false;
- for checkpoint in CHECKPOINTS {
+ for checkpoint in engines::checkpoints() {
let Some(engine) = engines::load(checkpoint) else {
continue;
};
@@ -315,7 +315,7 @@ fn rejects_the_same_questions_python_rejects() {
fn answers_the_same_way_when_questions_are_asked_one_at_a_time() {
// Every question rides in one batch, so a padding or masking mistake could let a question be
// influenced by its neighbours. Asking them singly has to give the same answers.
- let Some(engine) = engines::load(LayaCheckpoint::Multilingual) else {
+ let Some(engine) = engines::load_selected(LayaCheckpoint::Multilingual) else {
return;
};
let golden = CheckpointGoldenData::for_checkpoint(LayaCheckpoint::Multilingual.subdir());
@@ -351,7 +351,7 @@ fn answers_the_same_way_when_questions_are_asked_one_at_a_time() {
fn is_safe_to_call_from_several_threads_at_once() {
// The engine is documented as shareable and a caller will hold it as a singleton, so the ONNX
// session mutex and the tokenizer both have to tolerate concurrent use.
- let Some(engine) = engines::load(LayaCheckpoint::Multilingual) else {
+ let Some(engine) = engines::load_selected(LayaCheckpoint::Multilingual) else {
return;
};
let golden = CheckpointGoldenData::for_checkpoint(LayaCheckpoint::Multilingual.subdir());
@@ -404,7 +404,7 @@ fn warm_predict_latency_is_reported() {
// Not a parity check: reports the warm (post-warm-up) predict latency for the final report,
// the way samples/Laya.Sample's Program.cs does with a Stopwatch. Run with `--nocapture` to
// see the printed line.
- let Some(engine) = engines::load(LayaCheckpoint::Multilingual) else {
+ let Some(engine) = engines::load_selected(LayaCheckpoint::Multilingual) else {
return;
};
let golden = CheckpointGoldenData::for_checkpoint(LayaCheckpoint::Multilingual.subdir());
diff --git a/laya-rust/tests/python_json_tests.rs b/laya-rust/tests/python_json_tests.rs
index 83068d5f..c3ddb9c9 100644
--- a/laya-rust/tests/python_json_tests.rs
+++ b/laya-rust/tests/python_json_tests.rs
@@ -11,11 +11,18 @@ use serde_json::{Value, json};
/// discoverable and mutually consistent.
const CHECKPOINTS: [&str; 3] = ["multilingual", "english", "typed-decisions"];
+/// The checkpoints this run covers (all, or those `LAYA_TEST_CHECKPOINTS` selects).
+fn checkpoints() -> impl Iterator<Item = &'static str> {
+ CHECKPOINTS
+ .into_iter()
+ .filter(|c| common::test_artifacts::is_selected_subdir(c))
+}
+
#[test]
fn all_dialects_match_golden_for_every_checkpoint() {
let mut checked_any = false;
- for checkpoint in CHECKPOINTS {
+ for checkpoint in checkpoints() {
let data = CheckpointGoldenData::for_checkpoint(checkpoint);
if !data.available() {
eprintln!(
@@ -131,7 +138,7 @@ fn repr_matches_python_bit_pattern_sweep() {
let mut comparisons = 0;
let mut mismatches = 0;
let mut reference = None;
- for checkpoint in CHECKPOINTS {
+ for checkpoint in checkpoints() {
let data = CheckpointGoldenData::for_checkpoint(checkpoint);
if !data.available() {
eprintln!("skip: no golden data for '{checkpoint}'");
@@ -167,8 +174,8 @@ fn repr_matches_python_bit_pattern_sweep() {
}
eprintln!("repr differential sweep: {comparisons} comparisons, {mismatches} mismatches");
assert!(
- comparisons >= 6000,
- "all three checkpoint probes must be checked"
+ comparisons >= 2000 * checkpoints().count(),
+ "every selected checkpoint's probe must be checked"
);
assert_eq!(0, mismatches, "float repr must match Python byte-for-byte");
}
diff --git a/laya-rust/tests/router_end_to_end_tests.rs b/laya-rust/tests/router_end_to_end_tests.rs
index 3fc4ba0c..852fe44a 100644
--- a/laya-rust/tests/router_end_to_end_tests.rs
+++ b/laya-rust/tests/router_end_to_end_tests.rs
@@ -21,6 +21,14 @@ fn have_english_and_multilingual() -> bool {
.all(|c| test_artifacts::has_model(c) && test_artifacts::has_tokenizer(c))
}
+/// The router tests load english and multilingual side by side, so only a job that selects both
+/// runs them; any other `LAYA_TEST_CHECKPOINTS` selection skips them silently.
+fn router_selected() -> bool {
+ [LayaCheckpoint::English, LayaCheckpoint::Multilingual]
+ .into_iter()
+ .all(test_artifacts::is_selected)
+}
+
/// Builds each engine from the directory `test_artifacts` found (the repo walk-up included), not the
/// router's default resolution, which only checks `LAYA_ONNX_ROOT` and the cache: otherwise the
/// skip check above can pass while the router still fails to find the artifacts.
@@ -42,6 +50,9 @@ fn new_router(max_loaded: usize) -> LayaRouter {
#[test]
fn router_predict_routes_and_matches_the_recorded_checkpoint_result() {
+ if !router_selected() {
+ return;
+ }
if !have_english_and_multilingual() {
eprintln!("skip: need both the english and multilingual ONNX artifacts");
return;
@@ -141,6 +152,9 @@ fn sample_inputs_match_recorded_model_case_inputs() {
#[test]
fn router_predict_is_correct_under_eight_thread_concurrency_with_max_loaded_one() {
+ if !router_selected() {
+ return;
+ }
if !have_english_and_multilingual() {
eprintln!("skip: need both the english and multilingual ONNX artifacts");
return;
diff --git a/laya-rust/tests/shape_sweep_tests.rs b/laya-rust/tests/shape_sweep_tests.rs
index 52e66671..87085b81 100644
--- a/laya-rust/tests/shape_sweep_tests.rs
+++ b/laya-rust/tests/shape_sweep_tests.rs
@@ -234,7 +234,7 @@ fn nonfinite_act_logits_are_rejected() {
#[test]
fn runs_at_every_shape() {
- let Some(engine) = engines::load(LayaCheckpoint::Multilingual) else {
+ let Some(engine) = engines::load_selected(LayaCheckpoint::Multilingual) else {
return;
};
@@ -268,7 +268,7 @@ fn runs_at_every_shape() {
#[test]
fn reproduces_the_recorded_graph_outputs() {
- let Some(engine) = engines::load(LayaCheckpoint::Multilingual) else {
+ let Some(engine) = engines::load_selected(LayaCheckpoint::Multilingual) else {
return;
};
let golden = CheckpointGoldenData::for_checkpoint(LayaCheckpoint::Multilingual.subdir());
@@ -319,7 +319,7 @@ fn reproduces_the_recorded_graph_outputs() {
fn keeps_padded_marker_columns_at_the_masked_fill_value() {
// The model writes -1e4 into columns marker_mask says are padding. Calibration relies on that
// to keep a padded column out of a distribution, so it is worth pinning on the graph itself.
- let Some(engine) = engines::load(LayaCheckpoint::Multilingual) else {
+ let Some(engine) = engines::load_selected(LayaCheckpoint::Multilingual) else {
return;
};
@@ -349,7 +349,7 @@ fn keeps_padded_marker_columns_at_the_masked_fill_value() {
fn runs_a_single_option_question_that_topk_would_otherwise_reject() {
// The graph has TopK k=2 baked in, so a one-column marker axis fails outright. Collator widens
// it to two; this is the test that the widening is what actually keeps the graph runnable.
- let Some(engine) = engines::load(LayaCheckpoint::Multilingual) else {
+ let Some(engine) = engines::load_selected(LayaCheckpoint::Multilingual) else {
return;
};
@@ -365,7 +365,7 @@ fn runs_a_single_option_question_that_topk_would_otherwise_reject() {
#[test]
fn runs_at_every_shape_for_all_checkpoints() {
- for checkpoint in engines::CHECKPOINTS {
+ for checkpoint in engines::checkpoints() {
let Some(engine) = engines::load(checkpoint) else {
continue;
};
@@ -420,7 +420,7 @@ fn runs_at_every_shape_for_all_checkpoints() {
/// like the fused sweep does when that checkpoint's split-layout export is not present locally.
#[test]
fn runs_at_every_shape_for_all_checkpoints_split_layout() {
- for checkpoint in engines::CHECKPOINTS {
+ for checkpoint in engines::checkpoints() {
let Some(engine) = engines::load_split(checkpoint) else {
continue;
};
diff --git a/laya-rust/tests/shortlist_end_to_end_tests.rs b/laya-rust/tests/shortlist_end_to_end_tests.rs
index 020f3da4..92eef56e 100644
--- a/laya-rust/tests/shortlist_end_to_end_tests.rs
+++ b/laya-rust/tests/shortlist_end_to_end_tests.rs
@@ -16,7 +16,7 @@ use laya::laya_shortlist::LayaShortlist;
fn shortlist_predict_matches_the_recorded_result_for_every_checkpoint() {
let mut checked_any = false;
- for checkpoint in engines::CHECKPOINTS {
+ for checkpoint in engines::checkpoints() {
let Some(engine) = engines::load(checkpoint) else {
continue;
};
@@ -85,6 +85,9 @@ fn shortlist_predict_matches_the_recorded_result_for_every_checkpoint() {
/// still pass the assertions above.
#[test]
fn shortlist_many_options_case_shape_is_forty_reduced_to_twenty() {
+ if !common::test_artifacts::is_selected(LayaCheckpoint::Multilingual) {
+ return;
+ }
let Some(engine) = engines::load(LayaCheckpoint::Multilingual) else {
eprintln!("skip: no multilingual artifacts");
return;
diff --git a/laya-rust/tests/tokenizer_parity_tests.rs b/laya-rust/tests/tokenizer_parity_tests.rs
index a8b57ca0..70e2611e 100644
--- a/laya-rust/tests/tokenizer_parity_tests.rs
+++ b/laya-rust/tests/tokenizer_parity_tests.rs
@@ -25,6 +25,14 @@ const CHECKPOINTS: [LayaCheckpoint; 3] = [
LayaCheckpoint::TypedDecisions,
];
+/// The checkpoints this run covers (all, or those `LAYA_TEST_CHECKPOINTS` selects).
+fn checkpoints() -> Vec<LayaCheckpoint> {
+ CHECKPOINTS
+ .into_iter()
+ .filter(|c| test_artifacts::is_selected(*c))
+ .collect()
+}
+
/// Loads the tokenizer for `checkpoint`, or prints a skip line and returns `None` when
/// `tokenizer/tokenizer.json` is not available locally.
fn load_tokenizer(checkpoint: LayaCheckpoint) -> Option<HfTokenizer> {
@@ -57,7 +65,7 @@ fn to_u32(values: Vec<i64>) -> Vec<u32> {
#[test]
fn special_token_ids_match_the_recorded_values() {
- for checkpoint in CHECKPOINTS {
+ for checkpoint in checkpoints() {
let Some(tok) = load_tokenizer(checkpoint) else {
continue;
};
@@ -110,7 +118,7 @@ fn special_token_ids_match_the_recorded_values() {
#[test]
fn encodes_every_probe_string_exactly_as_python_does() {
- for checkpoint in CHECKPOINTS {
+ for checkpoint in checkpoints() {
let Some(tok) = load_tokenizer(checkpoint) else {
continue;
};
@@ -156,7 +164,7 @@ fn adds_no_special_tokens_of_its_own() {
// The shipped tokenizer.json has a TemplateProcessing post-processor that wraps every encode.
// Left in place it would inject the checkpoint's CLS and SEP ids into every fragment, shifting
// all marker positions. HfTokenizer must never invoke it.
- for checkpoint in CHECKPOINTS {
+ for checkpoint in checkpoints() {
let Some(tok) = load_tokenizer(checkpoint) else {
continue;
};
@@ -178,7 +186,7 @@ fn adds_no_special_tokens_of_its_own() {
#[test]
fn encodes_the_empty_string_to_nothing() {
- for checkpoint in CHECKPOINTS {
+ for checkpoint in checkpoints() {
let Some(tok) = load_tokenizer(checkpoint) else {
continue;
};
@@ -192,7 +200,7 @@ fn encodes_the_empty_string_to_nothing() {
#[test]
fn builds_the_recorded_sequence_for_every_question() {
- for checkpoint in CHECKPOINTS {
+ for checkpoint in checkpoints() {
let Some(tok) = load_tokenizer(checkpoint) else {
continue;
};
@@ -275,7 +283,7 @@ fn serializes_state_exactly_as_python_does() {
// State serialization is checkpoint-independent (it is the same JSON serializer for all), but
// exercising it across checkpoints gives coverage over diverse state shapes in the golden
// corpus of every checkpoint.
- for checkpoint in CHECKPOINTS {
+ for checkpoint in checkpoints() {
let golden = CheckpointGoldenData::for_checkpoint(checkpoint.subdir());
if !golden.available() {
continue;
diff --git a/laya-rust/tools/test_ci_tools.py b/laya-rust/tools/test_ci_tools.py
index 535c4701..b40ed436 100644
--- a/laya-rust/tools/test_ci_tools.py
+++ b/laya-rust/tools/test_ci_tools.py
@@ -56,6 +56,17 @@ class SkipGuardTests(unittest.TestCase):
with self.subTest(text=text):
self.assertEqual(1, self.check(text))
+ def test_shard_floor_is_per_shard_and_selected_checkpoint_skips_fail(self):
+ # Each parity shard passes its own --min-passed. A shard that loaded no checkpoint scores
+ # only the model-free tests, which a whole-lane floor of 200 would have let through if the
+ # other shards had made up the count; its own floor does not.
+ model_free_only = summary(passed=60)
+ self.assertEqual(0, self.check(model_free_only, minimum=60))
+ self.assertEqual(1, self.check(model_free_only, minimum=61))
+ self.assertEqual(1, self.check(
+ "skip: no ONNX artifacts for 'english'; set LAYA_ONNX_ROOT\n" + summary(passed=500),
+ minimum=61))
+
def test_missing_output_fails(self):
with patch("sys.argv", ["guard", "missing.log"]), patch.object(
guard.Path, "read_text", side_effect=FileNotFoundError("missing")