Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

Wasi build (#341)

Aleksandr Golokoz4b59367cdd20
diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml
index 835540134..068e19952 100644
--- a/.github/actions/setup/action.yml
+++ b/.github/actions/setup/action.yml
@@ -23,24 +23,33 @@ runs:
using: composite
steps:
- uses: dtolnay/rust-toolchain@stable
+ with:
+ targets:
+ wasm32-wasip1-threads
+
- uses: dtolnay/rust-toolchain@nightly
with:
components: ${{ inputs.linters == 'true' && 'rustfmt' || '' }}
+
- if: inputs.linters == 'true'
run: |
arch -arm64 brew install clang-format cargo-deny
shell: bash
+
- if: inputs.metal == 'true'
run: |
arch -arm64 brew install cmake
xcodebuild -downloadComponent MetalToolchain
shell: bash
+
- if: inputs.nextest == 'true'
uses: taiki-e/install-action@nextest
+
- if: inputs.test_model == 'true'
uses: astral-sh/setup-uv@v7
with:
enable-cache: true
+
- if: inputs.test_model == 'true'
run: |
mkdir -p ~/.cache/ci-uzu/models
diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
index 2ac5159d9..80cbe0987 100644
--- a/.github/workflows/tests.yml
+++ b/.github/workflows/tests.yml
@@ -50,21 +50,34 @@ jobs:
run: cargo deny check
tests:
- name: ${{ matrix.backend }} ${{ matrix.tracing && ' (tracing)' || '' }} ${{ matrix.grammar && ' (grammar)' || '' }}
+ name: ${{ matrix.backend }}
+ ${{ matrix.tracing && ' (tracing)' || '' }}
+ ${{ matrix.grammar && ' (grammar)' || '' }}
+ ${{ matrix.target != '' && format(' ({0})', matrix.target) || '' }}
runs-on: [ self-hosted, macos, mac, arm64 ]
strategy:
matrix:
backend: [ 'cpu', 'metal' ]
+ target: [ '' ]
tracing: [ false, true ]
include:
- grammar: true
- backend: 'cpu'
- grammar: false
+ target: 'wasm32-wasip1-threads'
tracing: false
+ grammar: false
+ exclude:
+ - backend: 'cpu'
+ tracing: true
env:
CARGO_ENCODED_RUSTFLAGS: "-Dwarnings"
+ FEATURE_FLAGS: >-
+ ${{ matrix.target != '' && format('--target {0}', matrix.target) || '' }}
+ ${{ matrix.backend != 'cpu' && format('--features {0}', matrix.backend) || '' }}
+ ${{ matrix.grammar && '--features grammar' || '' }}
+ ${{ matrix.tracing && '--features tracing' || '' }}
steps:
- uses: actions/checkout@v6
@@ -75,27 +88,15 @@ jobs:
nextest: true
- name: Build crates
- run: |
- cargo +stable build --no-default-features \
- ${{ matrix.backend != 'cpu' && format('--features {0}', matrix.backend) || '' }} \
- ${{ matrix.grammar && '--features grammar' || '' }} \
- ${{ matrix.tracing && '--features tracing' || '' }} \
+ run: cargo +stable build --no-default-features $FEATURE_FLAGS ${{ matrix.target == 'wasm32-wasip1-threads' && '-p uzu' || '' }}
- name: Build benches
if: ${{ matrix.backend != 'cpu' && !matrix.tracing }}
- run: |
- cargo +nightly bench --no-run --no-default-features \
- ${{ matrix.backend != 'cpu' && format('--features {0}', matrix.backend) || '' }} \
- ${{ matrix.grammar && '--features grammar' || '' }} \
- ${{ matrix.tracing && '--features tracing' || '' }} \
+ run: cargo +nightly bench --no-run --no-default-features $FEATURE_FLAGS
- name: Run tests
if: ${{ matrix.backend != 'cpu' && matrix.tracing && matrix.grammar }}
- run: |
- cargo +stable nextest run --profile=ci --no-default-features \
- ${{ matrix.backend != 'cpu' && format('--features {0}', matrix.backend) || '' }} \
- ${{ matrix.grammar && '--features grammar' || '' }} \
- ${{ matrix.tracing && '--features tracing' || '' }} \
+ run: cargo +stable nextest run --profile=ci --no-default-features $FEATURE_FLAGS
completed:
name: completed
diff --git a/Cargo.lock b/Cargo.lock
index 666d7b4b3..b0708e1d5 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -1318,6 +1318,17 @@ dependencies = [
"cc",
]
+[[package]]
+name = "fancy-regex"
+version = "0.14.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6e24cb5a94bcae1e5408b0effca5cd7172ea3c5755049c5f3af4cd283a165298"
+dependencies = [
+ "bit-set",
+ "regex-automata",
+ "regex-syntax 0.8.10",
+]
+
[[package]]
name = "fastrand"
version = "2.4.1"
@@ -4136,6 +4147,7 @@ dependencies = [
"dary_heap",
"derive_builder",
"esaxx-rs",
+ "fancy-regex",
"getrandom 0.3.4",
"indicatif",
"itertools 0.14.0",
@@ -4633,6 +4645,7 @@ dependencies = [
"rangemap",
"regex",
"rhai",
+ "rustix",
"schemars",
"serde",
"serde_json",
diff --git a/Cargo.toml b/Cargo.toml
index 20f06a618..fe5a546e0 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -20,7 +20,7 @@ ndarray = "0.17"
serde = { version = "1.0", features = ["derive"] }
serde_json = "1.0"
thiserror = "2.0"
-tokenizers = "0.22"
+tokenizers = { version = "0.22", features = ["fancy-regex"], default-features = false }
objc2 = "0.6"
objc2-foundation = "0.3"
metal = { package = "mtl-rs", version = "0.1" }
@@ -76,6 +76,7 @@ test-tag = "0.1"
proptest = "1.10"
criterion = "0.8.2"
criterion-macro = "0.4.0"
+rustix = "1.1.4"
# optimize the build script in debug builds
[profile.dev.build-override]
diff --git a/crates/uzu/Cargo.toml b/crates/uzu/Cargo.toml
index d12802f3b..1c913fd7c 100644
--- a/crates/uzu/Cargo.toml
+++ b/crates/uzu/Cargo.toml
@@ -10,7 +10,6 @@ build = "build/main.rs"
[dependencies]
dsl.workspace = true
-xgrammar = { workspace = true, optional = true }
half.workspace = true
ndarray.workspace = true
serde.workspace = true
@@ -31,6 +30,7 @@ rand.workspace = true
sysinfo.workspace = true
itertools.workspace = true
schemars.workspace = true
+rustix.workspace = true
# metal on macos requires objc2, bytesize, metal
[target.'cfg(any(target_os = "macos", target_os = "ios", target_os = "tvos", target_os = "visionos"))'.dependencies]
@@ -39,6 +39,10 @@ objc2-foundation = { workspace = true, optional = true }
bytesize = { workspace = true, optional = true }
metal = { workspace = true, optional = true }
+# grammar on non-wasm requires xgrammar
+[target.'cfg(not(target_arch = "wasm32"))'.dependencies]
+xgrammar = { workspace = true, optional = true }
+
[dev-dependencies]
test-tag.workspace = true
proptest.workspace = true
diff --git a/crates/uzu/build/main.rs b/crates/uzu/build/main.rs
index 0486dbf0f..f3f6b0e06 100644
--- a/crates/uzu/build/main.rs
+++ b/crates/uzu/build/main.rs
@@ -23,6 +23,7 @@ async fn main() -> anyhow::Result<()> {
println!("cargo::rerun-if-changed=/var/empty/hack_nonexistent_file_to_always_rerun");
}
+ let target_arch = env::var("CARGO_CFG_TARGET_ARCH")?;
let target_os = env::var("CARGO_CFG_TARGET_OS")?;
let metal_backend = cfg!(feature = "metal") && matches!(target_os.as_ref(), "macos" | "ios" | "tvos" | "visionos");
@@ -31,7 +32,7 @@ async fn main() -> anyhow::Result<()> {
println!("cargo::rustc-cfg=metal_backend");
}
- let grammar_xgrammar = cfg!(feature = "grammar");
+ let grammar_xgrammar = cfg!(feature = "grammar") && target_arch != "wasm32";
println!("cargo::rustc-check-cfg=cfg(grammar_xgrammar)");
if grammar_xgrammar {
println!("cargo::rustc-cfg=grammar_xgrammar");
diff --git a/crates/uzu/src/parameters/loader.rs b/crates/uzu/src/parameters/loader.rs
index cbe485766..d5414cb28 100644
--- a/crates/uzu/src/parameters/loader.rs
+++ b/crates/uzu/src/parameters/loader.rs
@@ -1,7 +1,6 @@
use std::{
collections::{HashMap, hash_map::Keys},
fs::File,
- os::unix::fs::FileExt,
};
use thiserror::Error;
@@ -11,6 +10,7 @@ use crate::{
DataType,
array::{Array, ArrayContextExt},
backends::common::{Backend, Buffer, Context},
+ utils::fs::file_read_exact_at,
};
pub struct ParameterMetadata {
@@ -124,7 +124,7 @@ where
});
}
- self.file.read_exact_at(array.as_bytes_mut(), offset as u64)?;
+ file_read_exact_at(self.file, array.as_bytes_mut(), offset as u64)?;
Ok(array)
}
@@ -136,7 +136,7 @@ where
data_type: &mut DataType,
) -> Result<(), ParameterLoaderError<C::Backend>> {
let metadata_entry = self.index.get(key).ok_or(ParameterLoaderError::KeyNotFound(key.to_string()))?;
- self.file.read_exact_at(buf, metadata_entry.offset as u64)?;
+ file_read_exact_at(self.file, buf, metadata_entry.offset as u64)?;
*shape = metadata_entry.shape.to_owned();
*data_type = metadata_entry.data_type;
Ok(())
@@ -173,7 +173,8 @@ impl<'file, 'context, 'leaf, C: Context> ParameterLeaf<'file, 'context, 'leaf, C
let mut buffer =
self.loader.context.create_buffer(self.metadata.size).map_err(ParameterLoaderError::BackendError)?;
buffer.set_label(Some(&format!("parameter_loader_{}", self.key.replace(".", "_"))));
- self.loader.file.read_exact_at(
+ file_read_exact_at(
+ self.loader.file,
unsafe { std::slice::from_raw_parts_mut(buffer.cpu_ptr().as_ptr() as *mut u8, self.metadata.size) },
self.metadata.offset as u64,
)?;
diff --git a/crates/uzu/src/parameters/safetensors_metadata.rs b/crates/uzu/src/parameters/safetensors_metadata.rs
index edb0e04df..2a027a470 100644
--- a/crates/uzu/src/parameters/safetensors_metadata.rs
+++ b/crates/uzu/src/parameters/safetensors_metadata.rs
@@ -1,11 +1,11 @@
// This code is based on the safetensors implementation: https://docs.rs/safetensors/latest/src/safetensors/tensor.rs.html
-use std::{collections::HashMap, fs::File, os::unix::fs::FileExt};
+use std::{collections::HashMap, fs::File};
use serde::{Deserialize, Serialize};
use thiserror::Error;
-use crate::DataType;
+use crate::{DataType, utils::fs::file_read_exact_at};
#[derive(Debug, Error)]
pub enum HeaderLoadingError {
@@ -114,7 +114,7 @@ const MAX_HEADER_SIZE: usize = 100_000_000;
pub fn read_metadata(file: &File) -> Result<(usize, HashMetadata), HeaderLoadingError> {
let mut header_buffer = [0u8; size_of::<u64>()];
- file.read_exact_at(&mut header_buffer, 0).map_err(|_| HeaderLoadingError::HeaderTooSmall)?;
+ file_read_exact_at(file, &mut header_buffer, 0).map_err(|_| HeaderLoadingError::HeaderTooSmall)?;
let metadata_size: usize =
u64::from_le_bytes(header_buffer).try_into().map_err(|_| HeaderLoadingError::HeaderTooLarge)?;
if metadata_size > MAX_HEADER_SIZE {
@@ -123,7 +123,7 @@ pub fn read_metadata(file: &File) -> Result<(usize, HashMetadata), HeaderLoading
let stop = metadata_size.checked_add(8).ok_or(HeaderLoadingError::InvalidHeaderLength)?;
let mut json_buffer: Box<[u8]> = core::iter::repeat(0).take(stop - size_of::<u64>()).collect();
- file.read_exact_at(&mut json_buffer, 8).map_err(|_| HeaderLoadingError::InvalidHeader)?;
+ file_read_exact_at(file, &mut json_buffer, 8).map_err(|_| HeaderLoadingError::InvalidHeader)?;
let string = core::str::from_utf8(&json_buffer).map_err(|_| HeaderLoadingError::InvalidHeader)?;
let metadata: HashMetadata =
serde_json::from_str(string).map_err(|_| HeaderLoadingError::InvalidHeaderDeserialization)?;
diff --git a/crates/uzu/src/utils/fs.rs b/crates/uzu/src/utils/fs.rs
new file mode 100644
index 000000000..dc5a30ab5
--- /dev/null
+++ b/crates/uzu/src/utils/fs.rs
@@ -0,0 +1,22 @@
+use std::{fs::File, io};
+
+/// Positional `read_exact` built on `rustix::io::pread`, so the same code
+/// path works on Unix and WASI without pulling in `std::os::*::fs::FileExt`.
+pub fn file_read_exact_at(
+ file: &File,
+ mut buf: &mut [u8],
+ mut offset: u64,
+) -> io::Result<()> {
+ while !buf.is_empty() {
+ match rustix::io::pread(file, &mut *buf, offset) {
+ Ok(0) => return Err(io::ErrorKind::UnexpectedEof.into()),
+ Ok(n) => {
+ buf = &mut buf[n..];
+ offset += n as u64;
+ },
+ Err(rustix::io::Errno::INTR) => continue,
+ Err(e) => return Err(e.into()),
+ }
+ }
+ Ok(())
+}
diff --git a/crates/uzu/src/utils/mod.rs b/crates/uzu/src/utils/mod.rs
index 530a66f61..a36794f08 100644
--- a/crates/uzu/src/utils/mod.rs
+++ b/crates/uzu/src/utils/mod.rs
@@ -1,4 +1,5 @@
pub mod env_utils;
+pub mod fs;
#[cfg(metal_backend)]
pub mod model_size;
pub mod pointers;
diff --git a/rust-toolchain.toml b/rust-toolchain.toml
index 310a35a4c..cff150a3d 100644
--- a/rust-toolchain.toml
+++ b/rust-toolchain.toml
@@ -2,4 +2,12 @@
channel = "nightly"
profile = "default"
components = ["rust-src", "rust-analyzer"]
-targets = ["aarch64-apple-ios", "aarch64-apple-ios-sim", "aarch64-apple-darwin", "aarch64-unknown-linux-gnu", "aarch64-linux-android", "x86_64-unknown-linux-gnu"]
+targets = [
+ "aarch64-apple-darwin",
+ "aarch64-apple-ios",
+ "aarch64-apple-ios-sim",
+ "aarch64-linux-android",
+ "aarch64-unknown-linux-gnu",
+ "wasm32-wasip1-threads",
+ "x86_64-unknown-linux-gnu"
+]