Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

low memory device model restriction

eugenea45f7c16ad47
diff --git a/Cargo.lock b/Cargo.lock
index 9ce5e2260..0bbad5b85 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -816,7 +816,7 @@ dependencies = [
"libc",
"log",
"rustversion",
- "windows",
+ "windows 0.48.0",
]
[[package]]
@@ -1103,9 +1103,9 @@ checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe"
[[package]]
name = "libc"
-version = "0.2.172"
+version = "0.2.174"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d750af042f7ef4f724306de029d18836c26c1765a54a6a3f094cbd23a7267ffa"
+checksum = "1171693293099992e19cddea4e8b849964e9846f4acee11b3948bcc337be8776"
[[package]]
name = "libloading"
@@ -1387,6 +1387,15 @@ dependencies = [
"minimal-lexical",
]
+[[package]]
+name = "ntapi"
+version = "0.4.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e8a3895c6391c39d7fe7ebc444a87eb2991b2a0bc718fdabd071eec617fc68e4"
+dependencies = [
+ "winapi",
+]
+
[[package]]
name = "nu-ansi-term"
version = "0.46.0"
@@ -1495,6 +1504,16 @@ dependencies = [
"objc2-core-foundation",
]
+[[package]]
+name = "objc2-io-kit"
+version = "0.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "71c1c64d6120e51cd86033f67176b1cb66780c2efe34dec55176f77befd93c0a"
+dependencies = [
+ "libc",
+ "objc2-core-foundation",
+]
+
[[package]]
name = "object"
version = "0.36.7"
@@ -2227,6 +2246,20 @@ dependencies = [
"unicode-ident",
]
+[[package]]
+name = "sysinfo"
+version = "0.36.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "aab138f5c1bb35231de19049060a87977ad23e04f2303e953bc5c2947ac7dec4"
+dependencies = [
+ "libc",
+ "memchr",
+ "ntapi",
+ "objc2-core-foundation",
+ "objc2-io-kit",
+ "windows 0.61.3",
+]
+
[[package]]
name = "tempfile"
version = "3.20.0"
@@ -2616,6 +2649,7 @@ dependencies = [
"serde",
"serde_json",
"serde_json5",
+ "sysinfo",
"thiserror",
"tokenizers",
]
@@ -2755,6 +2789,108 @@ dependencies = [
"windows-targets 0.48.5",
]
+[[package]]
+name = "windows"
+version = "0.61.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9babd3a767a4c1aef6900409f85f5d53ce2544ccdfaa86dad48c91782c6d6893"
+dependencies = [
+ "windows-collections",
+ "windows-core",
+ "windows-future",
+ "windows-link",
+ "windows-numerics",
+]
+
+[[package]]
+name = "windows-collections"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3beeceb5e5cfd9eb1d76b381630e82c4241ccd0d27f1a39ed41b2760b255c5e8"
+dependencies = [
+ "windows-core",
+]
+
+[[package]]
+name = "windows-core"
+version = "0.61.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c0fdd3ddb90610c7638aa2b3a3ab2904fb9e5cdbecc643ddb3647212781c4ae3"
+dependencies = [
+ "windows-implement",
+ "windows-interface",
+ "windows-link",
+ "windows-result",
+ "windows-strings",
+]
+
+[[package]]
+name = "windows-future"
+version = "0.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fc6a41e98427b19fe4b73c550f060b59fa592d7d686537eebf9385621bfbad8e"
+dependencies = [
+ "windows-core",
+ "windows-link",
+ "windows-threading",
+]
+
+[[package]]
+name = "windows-implement"
+version = "0.60.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a47fddd13af08290e67f4acabf4b459f647552718f683a7b415d290ac744a836"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "windows-interface"
+version = "0.59.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bd9211b69f8dcdfa817bfd14bf1c97c9188afa36f4750130fcdf3f400eca9fa8"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "windows-link"
+version = "0.1.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5e6ad25900d524eaabdbbb96d20b4311e1e7ae1699af4fb28c17ae66c80d798a"
+
+[[package]]
+name = "windows-numerics"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9150af68066c4c5c07ddc0ce30421554771e528bde427614c61038bc2c92c2b1"
+dependencies = [
+ "windows-core",
+ "windows-link",
+]
+
+[[package]]
+name = "windows-result"
+version = "0.3.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "56f42bd332cc6c8eac5af113fc0c1fd6a8fd2aa08a0119358686e5160d0586c6"
+dependencies = [
+ "windows-link",
+]
+
+[[package]]
+name = "windows-strings"
+version = "0.4.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "56e6c93f3a0c3b36176cb1327a4958a0353d5d166c2a35cb268ace15e91d3b57"
+dependencies = [
+ "windows-link",
+]
+
[[package]]
name = "windows-sys"
version = "0.48.0"
@@ -2813,6 +2949,15 @@ dependencies = [
"windows_x86_64_msvc 0.52.6",
]
+[[package]]
+name = "windows-threading"
+version = "0.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b66463ad2e0ea3bbf808b7f1d371311c80e115c0b71d60efc142cafbcfb057a6"
+dependencies = [
+ "windows-link",
+]
+
[[package]]
name = "windows_aarch64_gnullvm"
version = "0.48.5"
diff --git a/uzu/Cargo.toml b/uzu/Cargo.toml
index 7b5458589..eecc9c5cf 100644
--- a/uzu/Cargo.toml
+++ b/uzu/Cargo.toml
@@ -25,6 +25,7 @@ minijinja = "2.10.2"
minijinja-contrib = { version = "2.10.2", features = ["pycompat"] }
clap = { version = "4.5.39", features = ["derive"] }
rand = "0.9.1"
+sysinfo = "0.36.0"
[dev-dependencies]
is_close = "0.1.3"
diff --git a/uzu/src/session/session.rs b/uzu/src/session/session.rs
index e5dba0c80..fc0b422ed 100644
--- a/uzu/src/session/session.rs
+++ b/uzu/src/session/session.rs
@@ -1,4 +1,7 @@
-use std::{path::PathBuf, time::Instant};
+use std::{
+ path::{Path, PathBuf},
+ time::Instant,
+};
use objc2::rc::autoreleasepool;
use tokenizers::Tokenizer;
@@ -35,13 +38,47 @@ pub struct Session {
}
impl Session {
+ #[cfg(target_os = "ios")]
+ fn directory_size(path: &Path) -> std::io::Result<u64> {
+ let mut size = 0u64;
+ for entry_result in std::fs::read_dir(path)? {
+ let entry = entry_result?;
+ let metadata = entry.metadata()?;
+ if metadata.is_dir() {
+ size += Self::directory_size(&entry.path())?;
+ } else {
+ size += metadata.len();
+ }
+ }
+ Ok(size)
+ }
+
+ #[cfg(target_os = "ios")]
+ fn assert_model_fits_ram(model_path: &Path) -> Result<(), SessionError> {
+ use sysinfo::System;
+
+ let model_size_bytes = Self::directory_size(model_path).unwrap_or(0);
+
+ let mut sys = System::new();
+ sys.refresh_memory();
+
+ let allowed_bytes = sys.total_memory() * 60 / 100;
+
+ if model_size_bytes > allowed_bytes {
+ Err(SessionError::UnsupportedModel)
+ } else {
+ Ok(())
+ }
+ }
+
pub fn new(model_path: PathBuf) -> Result<Self, SessionError> {
- // Load tokenizer JSON
+ #[cfg(target_os = "ios")]
+ Self::assert_model_fits_ram(&model_path)?;
+
let tokenizer_path = model_path.join("tokenizer.json");
let mut tokenizer = Tokenizer::from_file(&tokenizer_path)
.map_err(|_| SessionError::UnableToLoadTokenizer)?;
- // Load tokenizer configuration
let tokenizer_config =
SessionTokenizerConfig::load_and_add_special_tokens_to_tokenizer(
model_path.clone(),
diff --git a/uzu/src/session/session_error.rs b/uzu/src/session/session_error.rs
index feb489cd9..8cdc3b05a 100644
--- a/uzu/src/session/session_error.rs
+++ b/uzu/src/session/session_error.rs
@@ -12,6 +12,8 @@ pub enum SessionError {
UnableToLoadTokenizer,
#[error("Unable to load tokenizer configuration")]
UnableToLoadTokenizerConfig,
+ #[error("Model is too large to fit into available RAM")]
+ UnsupportedModel,
}
impl From<GeneratorError> for SessionError {