A lightweight WebAssembly runtime that is fast, secure, and standards-compliant
//! Run the wasi-nn tests in `crates/test-programs`.
//!
//! It may be difficult to run to run all tests on all platforms; we check the
//! pre-requisites for each test dynamically (see [`check`]). Using
//! `libtest-mimic` allows us then to dynamically ignore tests that cannot run
//! on the current machine.
//!
//! There are two modes these tests run in:
//! - "ignore if unavailable" mode: if the checks for a test fail (e.g., the
//! backend is not installed, test artifacts cannot download, we're on the
//! wrong platform), the test is ignored.
//! - "fail if unavailable" mode: when the `CI` or `FORCE_WASINN_TEST_CHECK`
//! environment variables are set, any checks that fail cause the test to fail
//! early.
mod check;
mod exec;
use libtest_mimic::{Arguments, Trial};
use std::{borrow::Cow, env};
use test_programs_artifacts::*;
use wasmtime::Result;
use wasmtime_wasi_nn::{Backend, backend};
fn main() -> Result<()> {
tracing_subscriber::fmt::init();
if cfg!(miri) {
return Ok(());
}
// Gather a list of the test-program names.
let mut programs = Vec::new();
macro_rules! add_to_list {
($name:ident) => {
programs.push(stringify!($name));
};
}
foreach_nn!(add_to_list);
// Make ignored tests turn into failures.
let error_on_failed_check =
env::var_os("CI").is_some() || env::var_os("FORCE_WASINN_TEST_CHECK").is_some();
// Inform `libtest-mimic` how to run each test program.
let arguments = Arguments::from_args();
let mut trials = Vec::new();
for program in programs {
// Either ignore the test if it cannot run (i.e., downgrade `Fail` to
// `Ignore`) or preemptively fail it if `error_on_failed_check` is set.
let (run_test, mut check) = check_test_program(program);
if !error_on_failed_check {
check = check.downgrade_failure(); // Downgrade `Fail` to `Ignore`.
}
let should_ignore = check.is_ignore();
if arguments.nocapture && should_ignore {
println!("> ignoring {program}: {}", check.reason());
}
let trial = Trial::test(program, move || {
run_test().map_err(|e| format!("{e:?}").into())
})
.with_ignored_flag(should_ignore);
trials.push(trial);
}
// Run the tests.
libtest_mimic::run(&arguments, trials).exit()
}
/// Return the test program to run and a check that must pass for the test to
/// run.
fn check_test_program(name: &str) -> (fn() -> Result<()>, IgnoreCheck) {
match name {
// Legacy WITX-based tests:
"nn_witx_image_classification_openvino" => (
nn_witx_image_classification_openvino,
IgnoreCheck::for_openvino(),
),
"nn_witx_image_classification_openvino_named" => (
nn_witx_image_classification_openvino_named,
IgnoreCheck::for_openvino(),
),
"nn_witx_image_classification_onnx" => {
(nn_witx_image_classification_onnx, IgnoreCheck::for_onnx())
}
"nn_witx_invalid_input_onnx" => (nn_witx_invalid_input_onnx, IgnoreCheck::for_onnx()),
"nn_witx_image_classification_winml_named" => (
nn_witx_image_classification_winml_named,
IgnoreCheck::for_winml(),
),
"nn_witx_image_classification_pytorch" => (
nn_witx_image_classification_pytorch,
IgnoreCheck::for_pytorch(),
),
// WIT-based tests:
"nn_wit_image_classification_openvino" => (
nn_wit_image_classification_openvino,
IgnoreCheck::for_openvino(),
),
"nn_wit_image_classification_openvino_named" => (
nn_wit_image_classification_openvino_named,
IgnoreCheck::for_openvino(),
),
"nn_wit_image_classification_onnx" => {
(nn_wit_image_classification_onnx, IgnoreCheck::for_onnx())
}
"nn_wit_invalid_input_onnx" => (nn_wit_invalid_input_onnx, IgnoreCheck::for_onnx()),
"nn_wit_image_classification_winml_named" => (
nn_wit_image_classification_winml_named,
IgnoreCheck::for_winml(),
),
"nn_wit_image_classification_pytorch" => (
nn_wit_image_classification_pytorch,
IgnoreCheck::for_pytorch(),
),
_ => panic!("unknown test program: {name} (add to this `match`)"),
}
}
fn nn_witx_image_classification_openvino() -> Result<()> {
check::openvino::is_installed()?;
check::openvino::are_artifacts_available()?;
let backend = Backend::from(backend::openvino::OpenvinoBackend::default());
exec::witx::run(NN_WITX_IMAGE_CLASSIFICATION_OPENVINO, backend, false)
}
fn nn_witx_image_classification_openvino_named() -> Result<()> {
check::openvino::is_installed()?;
check::openvino::are_artifacts_available()?;
let backend = Backend::from(backend::openvino::OpenvinoBackend::default());
exec::witx::run(NN_WITX_IMAGE_CLASSIFICATION_OPENVINO_NAMED, backend, true)
}
#[cfg(feature = "onnx")]
fn nn_witx_image_classification_onnx() -> Result<()> {
check::onnx::are_artifacts_available()?;
let backend = Backend::from(backend::onnx::OnnxBackend::default());
exec::witx::run(NN_WITX_IMAGE_CLASSIFICATION_ONNX, backend, false)
}
#[cfg(not(feature = "onnx"))]
fn nn_witx_image_classification_onnx() -> Result<()> {
wasmtime::bail!("this test requires the `onnx` feature")
}
#[cfg(feature = "onnx")]
fn nn_witx_invalid_input_onnx() -> Result<()> {
check::onnx::are_artifacts_available()?;
let backend = Backend::from(backend::onnx::OnnxBackend::default());
exec::witx::run(NN_WITX_INVALID_INPUT_ONNX, backend, false)
}
#[cfg(not(feature = "onnx"))]
fn nn_witx_invalid_input_onnx() -> Result<()> {
wasmtime::bail!("this test requires the `onnx` feature")
}
#[cfg(all(feature = "winml", target_os = "windows"))]
fn nn_witx_image_classification_winml_named() -> Result<()> {
check::winml::is_available()?;
check::onnx::are_artifacts_available()?;
let backend = Backend::from(backend::winml::WinMLBackend::default());
exec::witx::run(NN_WITX_IMAGE_CLASSIFICATION_ONNX, backend, false)
}
#[cfg(not(all(feature = "winml", target_os = "windows")))]
fn nn_witx_image_classification_winml_named() -> Result<()> {
wasmtime::bail!("this test requires the `winml` feature and only runs on windows")
}
#[cfg(feature = "pytorch")]
fn nn_witx_image_classification_pytorch() -> Result<()> {
check::pytorch::are_artifacts_available()?;
let backend = Backend::from(backend::pytorch::PytorchBackend::default());
exec::witx::run(NN_WITX_IMAGE_CLASSIFICATION_PYTORCH, backend, false)
}
#[cfg(not(feature = "pytorch"))]
fn nn_witx_image_classification_pytorch() -> Result<()> {
wasmtime::bail!("this test requires the `pytorch` feature")
}
fn nn_wit_image_classification_openvino() -> Result<()> {
check::openvino::is_installed()?;
check::openvino::are_artifacts_available()?;
let backend = Backend::from(backend::openvino::OpenvinoBackend::default());
exec::wit::run(
NN_WIT_IMAGE_CLASSIFICATION_OPENVINO_COMPONENT,
backend,
false,
)
}
fn nn_wit_image_classification_openvino_named() -> Result<()> {
check::openvino::is_installed()?;
check::openvino::are_artifacts_available()?;
let backend = Backend::from(backend::openvino::OpenvinoBackend::default());
exec::wit::run(
NN_WIT_IMAGE_CLASSIFICATION_OPENVINO_NAMED_COMPONENT,
backend,
true,
)
}
#[cfg(feature = "onnx")]
fn nn_wit_image_classification_onnx() -> Result<()> {
check::onnx::are_artifacts_available()?;
let backend = Backend::from(backend::onnx::OnnxBackend::default());
exec::wit::run(NN_WIT_IMAGE_CLASSIFICATION_ONNX_COMPONENT, backend, false)
}
#[cfg(not(feature = "onnx"))]
fn nn_wit_image_classification_onnx() -> Result<()> {
wasmtime::bail!("this test requires the `onnx` feature")
}
#[cfg(feature = "onnx")]
fn nn_wit_invalid_input_onnx() -> Result<()> {
check::onnx::are_artifacts_available()?;
let backend = Backend::from(backend::onnx::OnnxBackend::default());
exec::wit::run(NN_WIT_INVALID_INPUT_ONNX_COMPONENT, backend, false)
}
#[cfg(not(feature = "onnx"))]
fn nn_wit_invalid_input_onnx() -> Result<()> {
wasmtime::bail!("this test requires the `onnx` feature")
}
#[cfg(feature = "pytorch")]
fn nn_wit_image_classification_pytorch() -> Result<()> {
check::pytorch::are_artifacts_available()?;
let backend = Backend::from(backend::pytorch::PytorchBackend::default());
exec::wit::run(
NN_WIT_IMAGE_CLASSIFICATION_PYTORCH_COMPONENT,
backend,
false,
)
}
#[cfg(not(feature = "pytorch"))]
fn nn_wit_image_classification_pytorch() -> Result<()> {
wasmtime::bail!("this test requires the `pytorch` feature")
}
#[cfg(all(feature = "winml", target_os = "windows"))]
fn nn_wit_image_classification_winml_named() -> Result<()> {
check::winml::is_available()?;
check::onnx::are_artifacts_available()?;
let backend = Backend::from(backend::winml::WinMLBackend::default());
exec::wit::run(NN_WIT_IMAGE_CLASSIFICATION_ONNX_COMPONENT, backend, false)
}
#[cfg(not(all(feature = "winml", target_os = "windows")))]
fn nn_wit_image_classification_winml_named() -> Result<()> {
wasmtime::bail!("this test requires the `winml` feature and only runs on windows")
}
/// Helper for keeping track of what tests should do when pre-test checks fail.
#[derive(Clone)]
enum IgnoreCheck {
Run,
Ignore(Cow<'static, str>),
Fail(Cow<'static, str>),
}
impl IgnoreCheck {
fn reason(&self) -> &str {
match self {
IgnoreCheck::Run => panic!("cannot get reason for `Run`"),
IgnoreCheck::Ignore(reason) => reason,
IgnoreCheck::Fail(reason) => reason,
}
}
fn downgrade_failure(self) -> Self {
if let IgnoreCheck::Fail(reason) = self {
IgnoreCheck::Ignore(reason)
} else {
self
}
}
fn is_ignore(&self) -> bool {
matches!(self, IgnoreCheck::Ignore(_))
}
}
/// Some pre-test checks for various backends.
impl IgnoreCheck {
fn for_openvino() -> IgnoreCheck {
use IgnoreCheck::*;
if !cfg!(target_arch = "x86_64") {
Fail("requires x86_64".into())
} else if !cfg!(target_os = "linux") && !cfg!(target_os = "windows") {
Fail("requires linux or windows or macos".into())
} else if let Err(e) = check::openvino::is_installed() {
Fail(e.to_string().into())
} else {
Run
}
}
fn for_onnx() -> Self {
use IgnoreCheck::*;
#[cfg(feature = "onnx")]
if !cfg!(target_arch = "x86_64") && !cfg!(target_arch = "aarch64") {
Fail("requires x86_64 or aarch64".into())
} else if !cfg!(target_os = "linux")
&& !cfg!(target_os = "windows")
&& !cfg!(target_os = "macos")
{
Fail("requires linux, windows, or macos".into())
} else {
Run
}
#[cfg(not(feature = "onnx"))]
Ignore("requires the `onnx` feature".into())
}
fn for_pytorch() -> Self {
use IgnoreCheck::*;
#[cfg(feature = "pytorch")]
if !cfg!(target_arch = "x86_64") && !cfg!(target_arch = "aarch64") {
Fail("requires x86_64 or aarch64".into())
} else if !cfg!(target_os = "linux")
&& !cfg!(target_os = "windows")
&& !cfg!(target_os = "macos")
{
Fail("requires linux, windows, or macos".into())
} else {
Run
}
#[cfg(not(feature = "pytorch"))]
Ignore("requires the `pytorch` feature".into())
}
fn for_winml() -> IgnoreCheck {
use IgnoreCheck::*;
#[cfg(all(feature = "winml", target_os = "windows"))]
if !cfg!(target_arch = "x86_64") {
Fail("requires x86_64".into())
} else if !cfg!(target_os = "windows") {
Fail("requires windows".into())
} else if let Err(e) = check::winml::is_available() {
Fail(e.to_string().into())
} else {
Run
}
#[cfg(not(all(feature = "winml", target_os = "windows")))]
Ignore("requires the `winml` feature on windows".into())
}
}