A high-performance inference engine for AI models
low memory device model restriction
eugenea45f7c16ad47
diff --git a/Cargo.lock b/Cargo.lockindex 9ce5e2260..0bbad5b85 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -816,7 +816,7 @@ dependencies = ["libc","log","rustversion",- "windows",+ "windows 0.48.0",][[package]]@@ -1103,9 +1103,9 @@ checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe"[[package]]name = "libc"-version = "0.2.172"+version = "0.2.174"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "d750af042f7ef4f724306de029d18836c26c1765a54a6a3f094cbd23a7267ffa"+checksum = "1171693293099992e19cddea4e8b849964e9846f4acee11b3948bcc337be8776"[[package]]name = "libloading"@@ -1387,6 +1387,15 @@ dependencies = ["minimal-lexical",]+[[package]]+name = "ntapi"+version = "0.4.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "e8a3895c6391c39d7fe7ebc444a87eb2991b2a0bc718fdabd071eec617fc68e4"+dependencies = [+ "winapi",+]+[[package]]name = "nu-ansi-term"version = "0.46.0"@@ -1495,6 +1504,16 @@ dependencies = ["objc2-core-foundation",]+[[package]]+name = "objc2-io-kit"+version = "0.3.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "71c1c64d6120e51cd86033f67176b1cb66780c2efe34dec55176f77befd93c0a"+dependencies = [+ "libc",+ "objc2-core-foundation",+]+[[package]]name = "object"version = "0.36.7"@@ -2227,6 +2246,20 @@ dependencies = ["unicode-ident",]+[[package]]+name = "sysinfo"+version = "0.36.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "aab138f5c1bb35231de19049060a87977ad23e04f2303e953bc5c2947ac7dec4"+dependencies = [+ "libc",+ "memchr",+ "ntapi",+ "objc2-core-foundation",+ "objc2-io-kit",+ "windows 0.61.3",+]+[[package]]name = "tempfile"version = "3.20.0"@@ -2616,6 +2649,7 @@ dependencies = ["serde","serde_json","serde_json5",+ "sysinfo","thiserror","tokenizers",]@@ -2755,6 +2789,108 @@ dependencies = ["windows-targets 0.48.5",]+[[package]]+name = "windows"+version = "0.61.3"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "9babd3a767a4c1aef6900409f85f5d53ce2544ccdfaa86dad48c91782c6d6893"+dependencies = [+ "windows-collections",+ "windows-core",+ "windows-future",+ "windows-link",+ "windows-numerics",+]++[[package]]+name = "windows-collections"+version = "0.2.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "3beeceb5e5cfd9eb1d76b381630e82c4241ccd0d27f1a39ed41b2760b255c5e8"+dependencies = [+ "windows-core",+]++[[package]]+name = "windows-core"+version = "0.61.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "c0fdd3ddb90610c7638aa2b3a3ab2904fb9e5cdbecc643ddb3647212781c4ae3"+dependencies = [+ "windows-implement",+ "windows-interface",+ "windows-link",+ "windows-result",+ "windows-strings",+]++[[package]]+name = "windows-future"+version = "0.2.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "fc6a41e98427b19fe4b73c550f060b59fa592d7d686537eebf9385621bfbad8e"+dependencies = [+ "windows-core",+ "windows-link",+ "windows-threading",+]++[[package]]+name = "windows-implement"+version = "0.60.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "a47fddd13af08290e67f4acabf4b459f647552718f683a7b415d290ac744a836"+dependencies = [+ "proc-macro2",+ "quote",+ "syn",+]++[[package]]+name = "windows-interface"+version = "0.59.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "bd9211b69f8dcdfa817bfd14bf1c97c9188afa36f4750130fcdf3f400eca9fa8"+dependencies = [+ "proc-macro2",+ "quote",+ "syn",+]++[[package]]+name = "windows-link"+version = "0.1.3"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "5e6ad25900d524eaabdbbb96d20b4311e1e7ae1699af4fb28c17ae66c80d798a"++[[package]]+name = "windows-numerics"+version = "0.2.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "9150af68066c4c5c07ddc0ce30421554771e528bde427614c61038bc2c92c2b1"+dependencies = [+ "windows-core",+ "windows-link",+]++[[package]]+name = "windows-result"+version = "0.3.4"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "56f42bd332cc6c8eac5af113fc0c1fd6a8fd2aa08a0119358686e5160d0586c6"+dependencies = [+ "windows-link",+]++[[package]]+name = "windows-strings"+version = "0.4.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "56e6c93f3a0c3b36176cb1327a4958a0353d5d166c2a35cb268ace15e91d3b57"+dependencies = [+ "windows-link",+]+[[package]]name = "windows-sys"version = "0.48.0"@@ -2813,6 +2949,15 @@ dependencies = ["windows_x86_64_msvc 0.52.6",]+[[package]]+name = "windows-threading"+version = "0.1.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "b66463ad2e0ea3bbf808b7f1d371311c80e115c0b71d60efc142cafbcfb057a6"+dependencies = [+ "windows-link",+]+[[package]]name = "windows_aarch64_gnullvm"version = "0.48.5"diff --git a/uzu/Cargo.toml b/uzu/Cargo.tomlindex 7b5458589..eecc9c5cf 100644--- a/uzu/Cargo.toml+++ b/uzu/Cargo.toml@@ -25,6 +25,7 @@ minijinja = "2.10.2"minijinja-contrib = { version = "2.10.2", features = ["pycompat"] }clap = { version = "4.5.39", features = ["derive"] }rand = "0.9.1"+sysinfo = "0.36.0"[dev-dependencies]is_close = "0.1.3"diff --git a/uzu/src/session/session.rs b/uzu/src/session/session.rsindex e5dba0c80..fc0b422ed 100644--- a/uzu/src/session/session.rs+++ b/uzu/src/session/session.rs@@ -1,4 +1,7 @@-use std::{path::PathBuf, time::Instant};+use std::{+ path::{Path, PathBuf},+ time::Instant,+};use objc2::rc::autoreleasepool;use tokenizers::Tokenizer;@@ -35,13 +38,47 @@ pub struct Session {}impl Session {+ #[cfg(target_os = "ios")]+ fn directory_size(path: &Path) -> std::io::Result<u64> {+ let mut size = 0u64;+ for entry_result in std::fs::read_dir(path)? {+ let entry = entry_result?;+ let metadata = entry.metadata()?;+ if metadata.is_dir() {+ size += Self::directory_size(&entry.path())?;+ } else {+ size += metadata.len();+ }+ }+ Ok(size)+ }++ #[cfg(target_os = "ios")]+ fn assert_model_fits_ram(model_path: &Path) -> Result<(), SessionError> {+ use sysinfo::System;++ let model_size_bytes = Self::directory_size(model_path).unwrap_or(0);++ let mut sys = System::new();+ sys.refresh_memory();++ let allowed_bytes = sys.total_memory() * 60 / 100;++ if model_size_bytes > allowed_bytes {+ Err(SessionError::UnsupportedModel)+ } else {+ Ok(())+ }+ }+pub fn new(model_path: PathBuf) -> Result<Self, SessionError> {- // Load tokenizer JSON+ #[cfg(target_os = "ios")]+ Self::assert_model_fits_ram(&model_path)?;+let tokenizer_path = model_path.join("tokenizer.json");let mut tokenizer = Tokenizer::from_file(&tokenizer_path).map_err(|_| SessionError::UnableToLoadTokenizer)?;- // Load tokenizer configurationlet tokenizer_config =SessionTokenizerConfig::load_and_add_special_tokens_to_tokenizer(model_path.clone(),diff --git a/uzu/src/session/session_error.rs b/uzu/src/session/session_error.rsindex feb489cd9..8cdc3b05a 100644--- a/uzu/src/session/session_error.rs+++ b/uzu/src/session/session_error.rs@@ -12,6 +12,8 @@ pub enum SessionError {UnableToLoadTokenizer,#[error("Unable to load tokenizer configuration")]UnableToLoadTokenizerConfig,+ #[error("Model is too large to fit into available RAM")]+ UnsupportedModel,}impl From<GeneratorError> for SessionError {