A high-performance inference engine for AI models
Wasi build (#341)
Aleksandr Golokoz4b59367cdd20
diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.ymlindex 835540134..068e19952 100644--- a/.github/actions/setup/action.yml+++ b/.github/actions/setup/action.yml@@ -23,24 +23,33 @@ runs:using: compositesteps:- uses: dtolnay/rust-toolchain@stable+ with:+ targets:+ wasm32-wasip1-threads+- uses: dtolnay/rust-toolchain@nightlywith:components: ${{ inputs.linters == 'true' && 'rustfmt' || '' }}+- if: inputs.linters == 'true'run: |arch -arm64 brew install clang-format cargo-denyshell: bash+- if: inputs.metal == 'true'run: |arch -arm64 brew install cmakexcodebuild -downloadComponent MetalToolchainshell: bash+- if: inputs.nextest == 'true'uses: taiki-e/install-action@nextest+- if: inputs.test_model == 'true'uses: astral-sh/setup-uv@v7with:enable-cache: true+- if: inputs.test_model == 'true'run: |mkdir -p ~/.cache/ci-uzu/modelsdiff --git a/.github/workflows/tests.yml b/.github/workflows/tests.ymlindex 2ac5159d9..80cbe0987 100644--- a/.github/workflows/tests.yml+++ b/.github/workflows/tests.yml@@ -50,21 +50,34 @@ jobs:run: cargo deny checktests:- name: ${{ matrix.backend }} ${{ matrix.tracing && ' (tracing)' || '' }} ${{ matrix.grammar && ' (grammar)' || '' }}+ name: ${{ matrix.backend }}+ ${{ matrix.tracing && ' (tracing)' || '' }}+ ${{ matrix.grammar && ' (grammar)' || '' }}+ ${{ matrix.target != '' && format(' ({0})', matrix.target) || '' }}runs-on: [ self-hosted, macos, mac, arm64 ]strategy:matrix:backend: [ 'cpu', 'metal' ]+ target: [ '' ]tracing: [ false, true ]include:- grammar: true- backend: 'cpu'- grammar: false+ target: 'wasm32-wasip1-threads'tracing: false+ grammar: false+ exclude:+ - backend: 'cpu'+ tracing: trueenv:CARGO_ENCODED_RUSTFLAGS: "-Dwarnings"+ FEATURE_FLAGS: >-+ ${{ matrix.target != '' && format('--target {0}', matrix.target) || '' }}+ ${{ matrix.backend != 'cpu' && format('--features {0}', matrix.backend) || '' }}+ ${{ matrix.grammar && '--features grammar' || '' }}+ ${{ matrix.tracing && '--features tracing' || '' }}steps:- uses: actions/checkout@v6@@ -75,27 +88,15 @@ jobs:nextest: true- name: Build crates- run: |- cargo +stable build --no-default-features \- ${{ matrix.backend != 'cpu' && format('--features {0}', matrix.backend) || '' }} \- ${{ matrix.grammar && '--features grammar' || '' }} \- ${{ matrix.tracing && '--features tracing' || '' }} \+ run: cargo +stable build --no-default-features $FEATURE_FLAGS ${{ matrix.target == 'wasm32-wasip1-threads' && '-p uzu' || '' }}- name: Build benchesif: ${{ matrix.backend != 'cpu' && !matrix.tracing }}- run: |- cargo +nightly bench --no-run --no-default-features \- ${{ matrix.backend != 'cpu' && format('--features {0}', matrix.backend) || '' }} \- ${{ matrix.grammar && '--features grammar' || '' }} \- ${{ matrix.tracing && '--features tracing' || '' }} \+ run: cargo +nightly bench --no-run --no-default-features $FEATURE_FLAGS- name: Run testsif: ${{ matrix.backend != 'cpu' && matrix.tracing && matrix.grammar }}- run: |- cargo +stable nextest run --profile=ci --no-default-features \- ${{ matrix.backend != 'cpu' && format('--features {0}', matrix.backend) || '' }} \- ${{ matrix.grammar && '--features grammar' || '' }} \- ${{ matrix.tracing && '--features tracing' || '' }} \+ run: cargo +stable nextest run --profile=ci --no-default-features $FEATURE_FLAGScompleted:name: completeddiff --git a/Cargo.lock b/Cargo.lockindex 666d7b4b3..b0708e1d5 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -1318,6 +1318,17 @@ dependencies = ["cc",]+[[package]]+name = "fancy-regex"+version = "0.14.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "6e24cb5a94bcae1e5408b0effca5cd7172ea3c5755049c5f3af4cd283a165298"+dependencies = [+ "bit-set",+ "regex-automata",+ "regex-syntax 0.8.10",+]+[[package]]name = "fastrand"version = "2.4.1"@@ -4136,6 +4147,7 @@ dependencies = ["dary_heap","derive_builder","esaxx-rs",+ "fancy-regex","getrandom 0.3.4","indicatif","itertools 0.14.0",@@ -4633,6 +4645,7 @@ dependencies = ["rangemap","regex","rhai",+ "rustix","schemars","serde","serde_json",diff --git a/Cargo.toml b/Cargo.tomlindex 20f06a618..fe5a546e0 100644--- a/Cargo.toml+++ b/Cargo.toml@@ -20,7 +20,7 @@ ndarray = "0.17"serde = { version = "1.0", features = ["derive"] }serde_json = "1.0"thiserror = "2.0"-tokenizers = "0.22"+tokenizers = { version = "0.22", features = ["fancy-regex"], default-features = false }objc2 = "0.6"objc2-foundation = "0.3"metal = { package = "mtl-rs", version = "0.1" }@@ -76,6 +76,7 @@ test-tag = "0.1"proptest = "1.10"criterion = "0.8.2"criterion-macro = "0.4.0"+rustix = "1.1.4"# optimize the build script in debug builds[profile.dev.build-override]diff --git a/crates/uzu/Cargo.toml b/crates/uzu/Cargo.tomlindex d12802f3b..1c913fd7c 100644--- a/crates/uzu/Cargo.toml+++ b/crates/uzu/Cargo.toml@@ -10,7 +10,6 @@ build = "build/main.rs"[dependencies]dsl.workspace = true-xgrammar = { workspace = true, optional = true }half.workspace = truendarray.workspace = trueserde.workspace = true@@ -31,6 +30,7 @@ rand.workspace = truesysinfo.workspace = trueitertools.workspace = trueschemars.workspace = true+rustix.workspace = true# metal on macos requires objc2, bytesize, metal[target.'cfg(any(target_os = "macos", target_os = "ios", target_os = "tvos", target_os = "visionos"))'.dependencies]@@ -39,6 +39,10 @@ objc2-foundation = { workspace = true, optional = true }bytesize = { workspace = true, optional = true }metal = { workspace = true, optional = true }+# grammar on non-wasm requires xgrammar+[target.'cfg(not(target_arch = "wasm32"))'.dependencies]+xgrammar = { workspace = true, optional = true }+[dev-dependencies]test-tag.workspace = trueproptest.workspace = truediff --git a/crates/uzu/build/main.rs b/crates/uzu/build/main.rsindex 0486dbf0f..f3f6b0e06 100644--- a/crates/uzu/build/main.rs+++ b/crates/uzu/build/main.rs@@ -23,6 +23,7 @@ async fn main() -> anyhow::Result<()> {println!("cargo::rerun-if-changed=/var/empty/hack_nonexistent_file_to_always_rerun");}+ let target_arch = env::var("CARGO_CFG_TARGET_ARCH")?;let target_os = env::var("CARGO_CFG_TARGET_OS")?;let metal_backend = cfg!(feature = "metal") && matches!(target_os.as_ref(), "macos" | "ios" | "tvos" | "visionos");@@ -31,7 +32,7 @@ async fn main() -> anyhow::Result<()> {println!("cargo::rustc-cfg=metal_backend");}- let grammar_xgrammar = cfg!(feature = "grammar");+ let grammar_xgrammar = cfg!(feature = "grammar") && target_arch != "wasm32";println!("cargo::rustc-check-cfg=cfg(grammar_xgrammar)");if grammar_xgrammar {println!("cargo::rustc-cfg=grammar_xgrammar");diff --git a/crates/uzu/src/parameters/loader.rs b/crates/uzu/src/parameters/loader.rsindex cbe485766..d5414cb28 100644--- a/crates/uzu/src/parameters/loader.rs+++ b/crates/uzu/src/parameters/loader.rs@@ -1,7 +1,6 @@use std::{collections::{HashMap, hash_map::Keys},fs::File,- os::unix::fs::FileExt,};use thiserror::Error;@@ -11,6 +10,7 @@ use crate::{DataType,array::{Array, ArrayContextExt},backends::common::{Backend, Buffer, Context},+ utils::fs::file_read_exact_at,};pub struct ParameterMetadata {@@ -124,7 +124,7 @@ where});}- self.file.read_exact_at(array.as_bytes_mut(), offset as u64)?;+ file_read_exact_at(self.file, array.as_bytes_mut(), offset as u64)?;Ok(array)}@@ -136,7 +136,7 @@ wheredata_type: &mut DataType,) -> Result<(), ParameterLoaderError<C::Backend>> {let metadata_entry = self.index.get(key).ok_or(ParameterLoaderError::KeyNotFound(key.to_string()))?;- self.file.read_exact_at(buf, metadata_entry.offset as u64)?;+ file_read_exact_at(self.file, buf, metadata_entry.offset as u64)?;*shape = metadata_entry.shape.to_owned();*data_type = metadata_entry.data_type;Ok(())@@ -173,7 +173,8 @@ impl<'file, 'context, 'leaf, C: Context> ParameterLeaf<'file, 'context, 'leaf, Clet mut buffer =self.loader.context.create_buffer(self.metadata.size).map_err(ParameterLoaderError::BackendError)?;buffer.set_label(Some(&format!("parameter_loader_{}", self.key.replace(".", "_"))));- self.loader.file.read_exact_at(+ file_read_exact_at(+ self.loader.file,unsafe { std::slice::from_raw_parts_mut(buffer.cpu_ptr().as_ptr() as *mut u8, self.metadata.size) },self.metadata.offset as u64,)?;diff --git a/crates/uzu/src/parameters/safetensors_metadata.rs b/crates/uzu/src/parameters/safetensors_metadata.rsindex edb0e04df..2a027a470 100644--- a/crates/uzu/src/parameters/safetensors_metadata.rs+++ b/crates/uzu/src/parameters/safetensors_metadata.rs@@ -1,11 +1,11 @@// This code is based on the safetensors implementation: https://docs.rs/safetensors/latest/src/safetensors/tensor.rs.html-use std::{collections::HashMap, fs::File, os::unix::fs::FileExt};+use std::{collections::HashMap, fs::File};use serde::{Deserialize, Serialize};use thiserror::Error;-use crate::DataType;+use crate::{DataType, utils::fs::file_read_exact_at};#[derive(Debug, Error)]pub enum HeaderLoadingError {@@ -114,7 +114,7 @@ const MAX_HEADER_SIZE: usize = 100_000_000;pub fn read_metadata(file: &File) -> Result<(usize, HashMetadata), HeaderLoadingError> {let mut header_buffer = [0u8; size_of::<u64>()];- file.read_exact_at(&mut header_buffer, 0).map_err(|_| HeaderLoadingError::HeaderTooSmall)?;+ file_read_exact_at(file, &mut header_buffer, 0).map_err(|_| HeaderLoadingError::HeaderTooSmall)?;let metadata_size: usize =u64::from_le_bytes(header_buffer).try_into().map_err(|_| HeaderLoadingError::HeaderTooLarge)?;if metadata_size > MAX_HEADER_SIZE {@@ -123,7 +123,7 @@ pub fn read_metadata(file: &File) -> Result<(usize, HashMetadata), HeaderLoadinglet stop = metadata_size.checked_add(8).ok_or(HeaderLoadingError::InvalidHeaderLength)?;let mut json_buffer: Box<[u8]> = core::iter::repeat(0).take(stop - size_of::<u64>()).collect();- file.read_exact_at(&mut json_buffer, 8).map_err(|_| HeaderLoadingError::InvalidHeader)?;+ file_read_exact_at(file, &mut json_buffer, 8).map_err(|_| HeaderLoadingError::InvalidHeader)?;let string = core::str::from_utf8(&json_buffer).map_err(|_| HeaderLoadingError::InvalidHeader)?;let metadata: HashMetadata =serde_json::from_str(string).map_err(|_| HeaderLoadingError::InvalidHeaderDeserialization)?;diff --git a/crates/uzu/src/utils/fs.rs b/crates/uzu/src/utils/fs.rsnew file mode 100644index 000000000..dc5a30ab5--- /dev/null+++ b/crates/uzu/src/utils/fs.rs@@ -0,0 +1,22 @@+use std::{fs::File, io};++/// Positional `read_exact` built on `rustix::io::pread`, so the same code+/// path works on Unix and WASI without pulling in `std::os::*::fs::FileExt`.+pub fn file_read_exact_at(+ file: &File,+ mut buf: &mut [u8],+ mut offset: u64,+) -> io::Result<()> {+ while !buf.is_empty() {+ match rustix::io::pread(file, &mut *buf, offset) {+ Ok(0) => return Err(io::ErrorKind::UnexpectedEof.into()),+ Ok(n) => {+ buf = &mut buf[n..];+ offset += n as u64;+ },+ Err(rustix::io::Errno::INTR) => continue,+ Err(e) => return Err(e.into()),+ }+ }+ Ok(())+}diff --git a/crates/uzu/src/utils/mod.rs b/crates/uzu/src/utils/mod.rsindex 530a66f61..a36794f08 100644--- a/crates/uzu/src/utils/mod.rs+++ b/crates/uzu/src/utils/mod.rs@@ -1,4 +1,5 @@pub mod env_utils;+pub mod fs;#[cfg(metal_backend)]pub mod model_size;pub mod pointers;diff --git a/rust-toolchain.toml b/rust-toolchain.tomlindex 310a35a4c..cff150a3d 100644--- a/rust-toolchain.toml+++ b/rust-toolchain.toml@@ -2,4 +2,12 @@channel = "nightly"profile = "default"components = ["rust-src", "rust-analyzer"]-targets = ["aarch64-apple-ios", "aarch64-apple-ios-sim", "aarch64-apple-darwin", "aarch64-unknown-linux-gnu", "aarch64-linux-android", "x86_64-unknown-linux-gnu"]+targets = [+ "aarch64-apple-darwin",+ "aarch64-apple-ios",+ "aarch64-apple-ios-sim",+ "aarch64-linux-android",+ "aarch64-unknown-linux-gnu",+ "wasm32-wasip1-threads",+ "x86_64-unknown-linux-gnu"+]