💚 End-to-end encrypted cloud for everything.
Dart 2faandroidauthye2eeencryptionend-to-end-encryptionfluttergolanggoogle-photosgoogle-photos-alternativeiosopensourcephotosprivacyprivacy-toolstypescriptzero-knowledge
Replace Ensu's direct unsafe Rust code
laurenspriemc13ac2484764
diff --git a/rust/Cargo.lock b/rust/Cargo.lockindex c4127ac674..5b49497860 100644--- a/rust/Cargo.lock+++ b/rust/Cargo.lock@@ -2371,13 +2371,14 @@ dependencies = ["ente-core","ente-ensu","ente-ensu-crypto",+ "filepath","keyring",- "libc","log","notify","rustix","serde","serde_json",+ "sysinfo","tauri","tauri-build","tauri-plugin-dialog",@@ -2565,7 +2566,6 @@ dependencies = ["ente-ensu-crypto","ente-image","llama-cpp-2",- "memmap2","ort","ort-sys","regex",@@ -2574,6 +2574,7 @@ dependencies = ["self_cell","serde","serde_json",+ "sha2 0.11.0","tempfile","thiserror 2.0.18","tokio",@@ -3287,6 +3288,16 @@ dependencies = ["rustc_version",]+[[package]]+name = "filepath"+version = "0.2.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "f81dd23d67f94e07ac941a8da9bd0c2a82d3ea063b7d8a921d1dd05f9052a407"+dependencies = [+ "libc",+ "windows 0.58.0",+]+[[package]]name = "filetime"version = "0.2.27"@@ -4425,7 +4436,7 @@ dependencies = ["js-sys","log","wasm-bindgen",- "windows-core",+ "windows-core 0.62.2",][[package]]@@ -5927,15 +5938,6 @@ version = "2.8.0"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79"-[[package]]-name = "memmap2"-version = "0.9.11"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "d1219ed1b7f229ee7104d281dd01d6802fe28bb6e95d292942c4daacdeb798c0"-dependencies = [- "libc",-]-[[package]]name = "memoffset"version = "0.9.1"@@ -6339,6 +6341,15 @@ dependencies = ["notify",]+[[package]]+name = "ntapi"+version = "0.4.3"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "c3b335231dfd352ffb0f8017f3b6027a4917f7df785ea2143d8af2adc66980ae"+dependencies = [+ "winapi",+]+[[package]]name = "num"version = "0.4.3"@@ -6599,6 +6610,16 @@ dependencies = ["objc2-core-foundation",]+[[package]]+name = "objc2-io-kit"+version = "0.3.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "33fafba39597d6dc1fb709123dfa8289d39406734be322956a69f0931c73bb15"+dependencies = [+ "libc",+ "objc2-core-foundation",+]+[[package]]name = "objc2-io-surface"version = "0.3.2"@@ -9241,6 +9262,20 @@ dependencies = ["syn 2.0.117",]+[[package]]+name = "sysinfo"+version = "0.39.6"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "d2071df9448915b71c4fe6d25deaf1c22f12bd234f01540b77312bb8e41361e6"+dependencies = [+ "libc",+ "memchr",+ "ntapi",+ "objc2-core-foundation",+ "objc2-io-kit",+ "windows 0.62.2",+]+[[package]]name = "system-configuration"version = "0.7.0"@@ -9309,8 +9344,8 @@ dependencies = ["tao-macros","unicode-segmentation","url",- "windows",- "windows-core",+ "windows 0.61.3",+ "windows-core 0.61.2","windows-version","x11-dl",]@@ -9397,7 +9432,7 @@ dependencies = ["webkit2gtk","webview2-com","window-vibrancy",- "windows",+ "windows 0.61.3",][[package]]@@ -9538,7 +9573,7 @@ dependencies = ["tauri-plugin","thiserror 2.0.18","url",- "windows",+ "windows 0.61.3","zbus 5.15.0",]@@ -9623,7 +9658,7 @@ dependencies = ["url","webkit2gtk","webview2-com",- "windows",+ "windows 0.61.3",][[package]]@@ -9648,7 +9683,7 @@ dependencies = ["url","webkit2gtk","webview2-com",- "windows",+ "windows 0.61.3","wry",]@@ -10976,10 +11011,10 @@ checksum = "7130243a7a5b33c54a444e54842e6a9e133de08b5ad7b5861cd8ed9a6a5bc96a"dependencies = ["webview2-com-macros","webview2-com-sys",- "windows",- "windows-core",- "windows-implement",- "windows-interface",+ "windows 0.61.3",+ "windows-core 0.61.2",+ "windows-implement 0.60.2",+ "windows-interface 0.59.3",][[package]]@@ -11000,8 +11035,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "381336cfffd772377d291702245447a5251a2ffa5bad679c99e61bc48bacbf9c"dependencies = ["thiserror 2.0.18",- "windows",- "windows-core",+ "windows 0.61.3",+ "windows-core 0.61.2",][[package]]@@ -11090,17 +11125,39 @@ dependencies = ["num-traits",]+[[package]]+name = "windows"+version = "0.58.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "dd04d41d93c4992d421894c18c8b43496aa748dd4c081bac0dc93eb0489272b6"+dependencies = [+ "windows-core 0.58.0",+ "windows-targets 0.52.6",+]+[[package]]name = "windows"version = "0.61.3"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "9babd3a767a4c1aef6900409f85f5d53ce2544ccdfaa86dad48c91782c6d6893"dependencies = [- "windows-collections",- "windows-core",- "windows-future",+ "windows-collections 0.2.0",+ "windows-core 0.61.2",+ "windows-future 0.2.1","windows-link 0.1.3",- "windows-numerics",+ "windows-numerics 0.2.0",+]++[[package]]+name = "windows"+version = "0.62.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "527fadee13e0c05939a6a05d5bd6eec6cd2e3dbd648b9f8e447c6518133d8580"+dependencies = [+ "windows-collections 0.3.2",+ "windows-core 0.62.2",+ "windows-future 0.3.2",+ "windows-numerics 0.3.1",][[package]]@@ -11109,7 +11166,29 @@ version = "0.2.0"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "3beeceb5e5cfd9eb1d76b381630e82c4241ccd0d27f1a39ed41b2760b255c5e8"dependencies = [- "windows-core",+ "windows-core 0.61.2",+]++[[package]]+name = "windows-collections"+version = "0.3.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "23b2d95af1a8a14a3c7367e1ed4fc9c20e0a26e79551b1454d72583c97cc6610"+dependencies = [+ "windows-core 0.62.2",+]++[[package]]+name = "windows-core"+version = "0.58.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "6ba6d44ec8c2591c134257ce647b7ea6b20335bf6379a27dac5f1641fcf59f99"+dependencies = [+ "windows-implement 0.58.0",+ "windows-interface 0.58.0",+ "windows-result 0.2.0",+ "windows-strings 0.1.0",+ "windows-targets 0.52.6",][[package]]@@ -11118,22 +11197,57 @@ version = "0.61.2"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "c0fdd3ddb90610c7638aa2b3a3ab2904fb9e5cdbecc643ddb3647212781c4ae3"dependencies = [- "windows-implement",- "windows-interface",+ "windows-implement 0.60.2",+ "windows-interface 0.59.3","windows-link 0.1.3","windows-result 0.3.4","windows-strings 0.4.2",]+[[package]]+name = "windows-core"+version = "0.62.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb"+dependencies = [+ "windows-implement 0.60.2",+ "windows-interface 0.59.3",+ "windows-link 0.2.1",+ "windows-result 0.4.1",+ "windows-strings 0.5.1",+]+[[package]]name = "windows-future"version = "0.2.1"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "fc6a41e98427b19fe4b73c550f060b59fa592d7d686537eebf9385621bfbad8e"dependencies = [- "windows-core",+ "windows-core 0.61.2","windows-link 0.1.3",- "windows-threading",+ "windows-threading 0.1.0",+]++[[package]]+name = "windows-future"+version = "0.3.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "e1d6f90251fe18a279739e78025bd6ddc52a7e22f921070ccdc67dde84c605cb"+dependencies = [+ "windows-core 0.62.2",+ "windows-link 0.2.1",+ "windows-threading 0.2.1",+]++[[package]]+name = "windows-implement"+version = "0.58.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "2bbd5b46c938e506ecbce286b6628a02171d56153ba733b6c741fc627ec9579b"+dependencies = [+ "proc-macro2",+ "quote",+ "syn 2.0.117",][[package]]@@ -11147,6 +11261,17 @@ dependencies = ["syn 2.0.117",]+[[package]]+name = "windows-interface"+version = "0.58.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "053c4c462dc91d3b1504c6fe5a726dd15e216ba718e84a0e46a88fbe5ded3515"+dependencies = [+ "proc-macro2",+ "quote",+ "syn 2.0.117",+]+[[package]]name = "windows-interface"version = "0.59.3"@@ -11176,10 +11301,20 @@ version = "0.2.0"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "9150af68066c4c5c07ddc0ce30421554771e528bde427614c61038bc2c92c2b1"dependencies = [- "windows-core",+ "windows-core 0.61.2","windows-link 0.1.3",]+[[package]]+name = "windows-numerics"+version = "0.3.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "6e2e40844ac143cdb44aead537bbf727de9b044e107a0f1220392177d15b0f26"+dependencies = [+ "windows-core 0.62.2",+ "windows-link 0.2.1",+]+[[package]]name = "windows-registry"version = "0.6.1"@@ -11191,6 +11326,15 @@ dependencies = ["windows-strings 0.5.1",]+[[package]]+name = "windows-result"+version = "0.2.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "1d1043d8214f791817bab27572aaa8af63732e11bf84aa21a45a78d6c317ae0e"+dependencies = [+ "windows-targets 0.52.6",+]+[[package]]name = "windows-result"version = "0.3.4"@@ -11209,6 +11353,16 @@ dependencies = ["windows-link 0.2.1",]+[[package]]+name = "windows-strings"+version = "0.1.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "4cd9b125c486025df0eabcb585e62173c6c9eddcec5d117d3b6e8c30e2ee4d10"+dependencies = [+ "windows-result 0.2.0",+ "windows-targets 0.52.6",+]+[[package]]name = "windows-strings"version = "0.4.2"@@ -11353,6 +11507,15 @@ dependencies = ["windows-link 0.1.3",]+[[package]]+name = "windows-threading"+version = "0.2.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "3949bd5b99cafdf1c7ca86b43ca564028dfe27d66958f2470940f73d86d75b37"+dependencies = [+ "windows-link 0.2.1",+]+[[package]]name = "windows-version"version = "0.1.7"@@ -11644,8 +11807,8 @@ dependencies = ["webkit2gtk","webkit2gtk-sys","webview2-com",- "windows",- "windows-core",+ "windows 0.61.3",+ "windows-core 0.61.2","windows-version","x11-dl",]diff --git a/rust/apps/ensu/src-tauri/Cargo.toml b/rust/apps/ensu/src-tauri/Cargo.tomlindex ddbc3584af..ae43146b44 100644--- a/rust/apps/ensu/src-tauri/Cargo.toml+++ b/rust/apps/ensu/src-tauri/Cargo.toml@@ -16,6 +16,7 @@ log.workspace = truenotify = "6.1.1"serde.workspace = trueserde_json.workspace = true+sysinfo = { version = "0.39.6", default-features = false, features = ["system"] }tauri = { version = "2.11.2", features = [] }tauri-plugin-dialog = "2.7.1"tauri-plugin-fs = "2.5.1"@@ -25,11 +26,11 @@ tauri-plugin-updater = "2.10.1"uuid = { workspace = true, features = ["v4"] }[target.'cfg(unix)'.dependencies]-libc = "0.2.0"rustix = { version = "1.1.4", features = ["fs"] }[target.'cfg(windows)'.dependencies]-windows-sys = { version = "0.61.2", features = ["Win32_Foundation", "Win32_Storage_FileSystem", "Win32_System_SystemInformation"] }+filepath = "0.2.0"+windows-sys = { version = "0.61.2", features = ["Win32_Foundation"] }[target.'cfg(any(windows, target_os = "linux"))'.dependencies]tauri-plugin-single-instance = "2.4.4"@@ -37,5 +38,8 @@ tauri-plugin-single-instance = "2.4.4"[features]custom-protocol = ["tauri/custom-protocol"]+[dev-dependencies]+filepath = "0.2.0"+[lints]workspace = truediff --git a/rust/apps/ensu/src-tauri/src/commands/notes/source.rs b/rust/apps/ensu/src-tauri/src/commands/notes/source.rsindex f750ab2d39..03de68423a 100644--- a/rust/apps/ensu/src-tauri/src/commands/notes/source.rs+++ b/rust/apps/ensu/src-tauri/src/commands/notes/source.rs@@ -3,10 +3,6 @@ use std::fs::{self, File, Metadata};use std::io::Read;#[cfg(unix)]use std::os::unix::ffi::OsStrExt;-#[cfg(windows)]-use std::os::windows::ffi::OsStringExt;-#[cfg(windows)]-use std::os::windows::io::AsRawHandle;use std::path::{Component, Path, PathBuf};use ente_ensu::notes::{@@ -19,6 +15,9 @@ use crate::commands::common::ApiError;use super::registry::RegisteredCollection;use super::{NOTES_MAX_SCAN_ENTRIES, remove_owned_entry, source_scan_error, system_time_ms};+#[cfg(any(windows, test))]+mod scoped;+pub(super) fn inventory_source_root(root: &Path,check_for_cancellation: impl FnMut() -> Result<(), ApiError>,@@ -232,59 +231,17 @@ fn scoped_open_error(error: rustix::io::Errno) -> ApiError {}#[cfg(windows)]-fn open_collection_source(root: &Path, _document_id: &str, path: &Path) -> Result<File, ApiError> {- let file = File::open(path).map_err(source_file_error)?;- let final_path = final_windows_handle_path(&file)?;- if !windows_path_is_beneath(&final_path, root) {- return Err(ApiError::new(- "invalid_document",- "Source note escaped its collection",- ));- }- Ok(file)+fn open_collection_source(root: &Path, document_id: &str, _path: &Path) -> Result<File, ApiError> {+ scoped::open(root, document_id).map_err(windows_source_file_error)}-#[cfg(windows)]-#[expect(- unsafe_code,- reason = "Resolve the opened Windows handle to reject path escapes"-)]-fn final_windows_handle_path(file: &File) -> Result<PathBuf, ApiError> {- use std::ffi::OsString;- use windows_sys::Win32::Foundation::HANDLE;- use windows_sys::Win32::Storage::FileSystem::{- FILE_NAME_NORMALIZED, GetFinalPathNameByHandleW, VOLUME_NAME_DOS,- };-- let handle = file.as_raw_handle() as HANDLE;- let flags = FILE_NAME_NORMALIZED | VOLUME_NAME_DOS;- let length = unsafe { GetFinalPathNameByHandleW(handle, std::ptr::null_mut(), 0, flags) };- if length == 0 {- return Err(source_file_error(std::io::Error::last_os_error()));- }- let mut buffer = vec![0_u16; length as usize + 1];- let written = unsafe {- GetFinalPathNameByHandleW(handle, buffer.as_mut_ptr(), buffer.len() as u32, flags)- };- if written == 0 || written as usize >= buffer.len() {- return Err(source_file_error(std::io::Error::last_os_error()));+#[cfg(any(windows, test))]+fn windows_source_file_error(error: std::io::Error) -> ApiError {+ if error.kind() == std::io::ErrorKind::PermissionDenied && error.raw_os_error().is_none() {+ ApiError::new("invalid_document", "Source note path is not allowed")+ } else {+ source_file_error(error)}- Ok(PathBuf::from(OsString::from_wide(- &buffer[..written as usize],- )))-}--#[cfg(windows)]-fn windows_path_is_beneath(path: &Path, root: &Path) -> bool {- let path = path.to_string_lossy().to_ascii_lowercase();- let root = root- .to_string_lossy()- .trim_end_matches(['\\', '/'])- .to_ascii_lowercase();- path == root- || path- .strip_prefix(&root)- .is_some_and(|suffix| suffix.starts_with('\\') || suffix.starts_with('/'))}fn source_file_error(error: std::io::Error) -> ApiError {@@ -399,6 +356,80 @@ mod tests {use super::super::TestDirectory;use super::*;+ #[test]+ fn windows_source_errors_distinguish_io_failures_from_escaped_paths() {+ let denied = std::io::Error::new(std::io::ErrorKind::PermissionDenied, "escaped path");+ assert_eq!(+ windows_source_file_error(denied).name,+ Some("invalid_document")+ );+ for code in [5, 13] {+ let error = std::io::Error::from_raw_os_error(code);+ assert_eq!(windows_source_file_error(error).name, Some("source_io"));+ }+ let missing = std::io::Error::from(std::io::ErrorKind::NotFound);+ assert_eq!(+ windows_source_file_error(missing).name,+ Some("source_changed")+ );+ }++ #[test]+ fn collection_reads_literal_reserved_windows_filenames() {+ let temp = TestDirectory::new();+ let root = fs::canonicalize(temp.path()).unwrap();+ for name in ["con.md", "aux.notes.md", "com¹.md"] {+ fs::write(root.join(name), name).unwrap();+ let (_, bytes, source) = read_collection_source(&root, name).unwrap();+ assert_eq!(bytes, name.as_bytes());+ assert_eq!(source.document_id, name);+ }+ assert_eq!(inventory_source_root(&root, || Ok(())).unwrap().len(), 3);+ }++ #[cfg(windows)]+ #[test]+ fn collection_read_rejects_directory_and_root_junction_swaps() {+ let temp = TestDirectory::new();+ let root = temp.path().join("root");+ let outside = temp.path().join("outside");+ fs::create_dir_all(root.join("nested")).unwrap();+ fs::create_dir(&outside).unwrap();+ fs::write(root.join("nested/note.md"), "inside").unwrap();+ fs::write(outside.join("note.md"), "outside").unwrap();+ let root = fs::canonicalize(root).unwrap();+ assert_eq!(+ read_collection_source(&root, "nested/note.md").unwrap().1,+ b"inside"+ );+ fs::remove_file(root.join("nested/note.md")).unwrap();+ fs::remove_dir(root.join("nested")).unwrap();+ let junction = |path: &Path| {+ assert!(+ std::process::Command::new("cmd")+ .args(["/C", "mklink", "/J"])+ .arg(path)+ .arg(&outside)+ .status()+ .unwrap()+ .success()+ );+ };+ junction(&root.join("nested"));+ assert_eq!(+ read_collection_source(&root, "nested/note.md")+ .unwrap_err()+ .name,+ Some("invalid_document")+ );+ fs::rename(&root, temp.path().join("original-root")).unwrap();+ junction(&root);+ assert_eq!(+ read_collection_source(&root, "note.md").unwrap_err().name,+ Some("invalid_document")+ );+ }+#[cfg(unix)]#[test]fn collection_read_rejects_symlink_swaps() {diff --git a/rust/apps/ensu/src-tauri/src/commands/notes/source/scoped.rs b/rust/apps/ensu/src-tauri/src/commands/notes/source/scoped.rsnew file mode 100644index 0000000000..e977972076--- /dev/null+++ b/rust/apps/ensu/src-tauri/src/commands/notes/source/scoped.rs@@ -0,0 +1,242 @@+use std::fs::File;+use std::io;+use std::path::Path;++use filepath::FilePath;++pub(super) fn open(root: &Path, document_id: &str) -> io::Result<File> {+ if !root.is_absolute() {+ return Err(io::Error::new(+ io::ErrorKind::PermissionDenied,+ "Collection root must be absolute",+ ));+ }+ let mut path = root.to_path_buf();+ for component in document_id.split('/') {+ path.push(component);+ }+ let file = File::open(path)?;+ if !path_is_beneath(&file.path()?, root) {+ return Err(io::Error::new(+ io::ErrorKind::PermissionDenied,+ "Source note escaped its collection",+ ));+ }+ Ok(file)+}++fn path_is_beneath(path: &Path, root: &Path) -> bool {+ let Some((path, root)) = comparison_path(path).zip(comparison_path(root)) else {+ return false;+ };+ let root = root.trim_end_matches('\\');+ path == root+ || path+ .strip_prefix(root)+ .is_some_and(|suffix| suffix.starts_with('\\'))+}++fn comparison_path(path: &Path) -> Option<String> {+ let path = path+ .to_string_lossy()+ .replace('/', "\\")+ .to_ascii_lowercase();+ if path.starts_with(r"\\.\") {+ return None;+ }+ let Some(verbatim) = path.strip_prefix(r"\\?\") else {+ return Some(path);+ };+ if let Some(unc) = verbatim.strip_prefix(r"unc\") {+ return Some(format!(r"\\{unc}"));+ }+ let drive = verbatim.as_bytes().get(..3).is_some_and(|prefix| {+ prefix[0].is_ascii_alphabetic() && prefix[1] == b':' && prefix[2] == b'\\'+ });+ let volume = verbatim+ .split('\\')+ .next()?+ .strip_prefix("volume{")+ .and_then(|volume| volume.strip_suffix('}'))+ .is_some_and(|volume| uuid::Uuid::parse_str(volume).is_ok());+ (drive || volume).then(|| verbatim.to_owned())+}++#[cfg(test)]+mod tests {+ use std::fs;+ use std::io::Read;+ use std::path::{Path, PathBuf};++ use super::{open, path_is_beneath};++ struct Directory(PathBuf);++ impl Directory {+ fn new() -> Self {+ let directory =+ std::env::temp_dir().join(format!("ensu-scoped-{}", uuid::Uuid::new_v4()));+ fs::create_dir_all(directory.join("root/nested")).unwrap();+ fs::create_dir(directory.join("outside")).unwrap();+ fs::write(directory.join("root/nested/note.md"), "inside").unwrap();+ fs::write(directory.join("outside/note.md"), "outside").unwrap();+ Self(fs::canonicalize(directory).unwrap())+ }+ }++ impl Drop for Directory {+ fn drop(&mut self) {+ let _ = fs::remove_dir_all(&self.0);+ }+ }++ #[test]+ fn opens_nested_files_and_rejects_parent_and_absolute_paths() {+ let directory = Directory::new();+ let root = directory.0.join("root");+ let mut contents = String::new();+ open(&root, "nested/note.md")+ .unwrap()+ .read_to_string(&mut contents)+ .unwrap();+ assert_eq!(contents, "inside");+ assert!(open(&root, "../outside/note.md").is_err());+ assert!(open(&root, directory.0.join("outside/note.md").to_str().unwrap()).is_err());+ }++ #[cfg(unix)]+ #[test]+ fn rejects_a_swapped_file_or_directory_symlink() {+ use std::os::unix::fs::symlink;+ let directory = Directory::new();+ let root = directory.0.join("root");+ fs::remove_file(root.join("nested/note.md")).unwrap();+ symlink(+ directory.0.join("outside/note.md"),+ root.join("nested/note.md"),+ )+ .unwrap();+ assert!(open(&root, "nested/note.md").is_err());+ fs::remove_file(root.join("nested/note.md")).unwrap();+ fs::remove_dir(root.join("nested")).unwrap();+ symlink(directory.0.join("outside"), root.join("nested")).unwrap();+ assert!(open(&root, "nested/note.md").is_err());+ fs::rename(&root, directory.0.join("original-root")).unwrap();+ symlink(directory.0.join("outside"), &root).unwrap();+ assert!(open(&root, "note.md").is_err());+ }++ #[cfg(windows)]+ #[test]+ fn rejects_directory_junction_escapes_and_device_names() {+ let directory = Directory::new();+ let root = directory.0.join("root");+ fs::remove_file(root.join("nested/note.md")).unwrap();+ fs::remove_dir(root.join("nested")).unwrap();+ let status = std::process::Command::new("cmd")+ .args(["/C", "mklink", "/J"])+ .arg(root.join("nested"))+ .arg(directory.0.join("outside"))+ .status()+ .unwrap();+ assert!(status.success());+ assert_eq!(+ fs::read_to_string(root.join("nested/note.md")).unwrap(),+ "outside"+ );+ let error = open(&root, "nested/note.md").unwrap_err();+ assert_eq!(error.kind(), std::io::ErrorKind::PermissionDenied);+ assert_eq!(error.raw_os_error(), None);+ for name in ["NUL", "CON", "COM1", "COM¹", "LPT¹"] {+ assert!(open(&root, name).is_err());+ }+ fs::rename(&root, directory.0.join("original-root")).unwrap();+ let status = std::process::Command::new("cmd")+ .args(["/C", "mklink", "/J"])+ .arg(&root)+ .arg(directory.0.join("outside"))+ .status()+ .unwrap();+ assert!(status.success());+ assert_eq!(fs::read_to_string(root.join("note.md")).unwrap(), "outside");+ let error = open(&root, "note.md").unwrap_err();+ assert_eq!(error.kind(), std::io::ErrorKind::PermissionDenied);+ assert_eq!(error.raw_os_error(), None);+ }++ #[test]+ fn supports_unicode_names_and_reads_from_the_opened_handle() {+ let directory = Directory::new();+ let root = directory.0.join("root");+ fs::write(root.join("café.md"), "original").unwrap();+ let mut file = open(&root, "café.md").unwrap();+ fs::rename(root.join("café.md"), root.join("moved.md")).unwrap();+ fs::write(root.join("café.md"), "replacement").unwrap();+ let mut contents = String::new();+ file.read_to_string(&mut contents).unwrap();+ assert_eq!(contents, "original");+ }++ #[test]+ fn supports_paths_longer_than_legacy_windows_limits() {+ let directory = Directory::new();+ let root = directory.0.join("root");+ let relative = format!("{}/note.md", ["long-directory-name"; 16].join("/"));+ let path = root.join(&relative);+ fs::create_dir_all(path.parent().unwrap()).unwrap();+ fs::write(path, "long path").unwrap();+ let mut contents = String::new();+ open(&root, &relative)+ .unwrap()+ .read_to_string(&mut contents)+ .unwrap();+ assert_eq!(contents, "long path");+ }++ #[test]+ fn compares_windows_namespaces_and_path_boundaries() {+ for (path, root) in [+ (r"C:\Notes\note.md", r"\\?\C:\Notes"),+ (r"\\?\C:\Notes\note.md", r"c:\notes\"),+ (r"C:\Notes\café.md", r"\\?\C:\Notes"),+ (+ r"\\server\share\notes\note.md",+ r"\\?\UNC\server\share\notes",+ ),+ (+ r"\\?\UNC\server\share\notes\note.md",+ r"\\server\share\notes",+ ),+ (+ r"Volume{00000000-0000-0000-0000-000000000001}\notes\note.md",+ r"\\?\Volume{00000000-0000-0000-0000-000000000001}\notes",+ ),+ ] {+ assert!(path_is_beneath(Path::new(path), Path::new(root)));+ }+ for (path, root) in [+ (r"C:\Notes-other\note.md", r"\\?\C:\Notes"),+ (r"D:\Notes\note.md", r"\\?\C:\Notes"),+ (r"\\server\share-other\note.md", r"\\?\UNC\server\share"),+ (r"GLOBALROOT\Device\note.md", r"\\?\GLOBALROOT\Device"),+ (r"\\.\C:\Notes\note.md", r"C:\Notes"),+ ] {+ assert!(!path_is_beneath(Path::new(path), Path::new(root)));+ }+ }++ #[test]+ fn reads_literal_reserved_windows_filenames() {+ let directory = Directory::new();+ let root = directory.0.join("root");+ for name in ["con.md", "aux.notes.md", "com¹.md"] {+ fs::write(root.join(name), name).unwrap();+ let mut contents = String::new();+ open(&root, name)+ .unwrap()+ .read_to_string(&mut contents)+ .unwrap();+ assert_eq!(contents, name);+ }+ }+}diff --git a/rust/apps/ensu/src-tauri/src/commands/system.rs b/rust/apps/ensu/src-tauri/src/commands/system.rsindex 10997743f4..573f10fb8e 100644--- a/rust/apps/ensu/src-tauri/src/commands/system.rs+++ b/rust/apps/ensu/src-tauri/src/commands/system.rs@@ -7,44 +7,11 @@ pub struct SystemInfo {total_memory_bytes: Option<u64>,}-#[cfg(unix)]-#[expect(- unsafe_code,- reason = "Read physical memory through the system's sysconf API"-)]fn total_memory_bytes() -> Option<u64> {- let pages = unsafe { libc::sysconf(libc::_SC_PHYS_PAGES) };- let page_size = unsafe { libc::sysconf(libc::_SC_PAGESIZE) };- if pages <= 0 || page_size <= 0 {- return None;- }-- u64::try_from(pages)- .ok()?- .checked_mul(u64::try_from(page_size).ok()?)-}--#[cfg(target_os = "windows")]-#[expect(unsafe_code, reason = "Read physical memory through the Windows API")]-fn total_memory_bytes() -> Option<u64> {- use windows_sys::Win32::System::SystemInformation::{GlobalMemoryStatusEx, MEMORYSTATUSEX};-- let mut status = MEMORYSTATUSEX {- dwLength: std::mem::size_of::<MEMORYSTATUSEX>() as u32,- ..Default::default()- };-- let ok = unsafe { GlobalMemoryStatusEx(&mut status) };- if ok == 0 {- None- } else {- Some(status.ullTotalPhys)- }-}--#[cfg(not(any(unix, target_os = "windows")))]-fn total_memory_bytes() -> Option<u64> {- None+ let mut system = sysinfo::System::new();+ system.refresh_memory_specifics(sysinfo::MemoryRefreshKind::nothing().with_ram());+ let total = system.total_memory();+ (total > 0).then_some(total)}#[tauri::command]diff --git a/rust/apps/ensu/src-tauri/src/main.rs b/rust/apps/ensu/src-tauri/src/main.rsindex bd0a82f84c..8d7513be00 100644--- a/rust/apps/ensu/src-tauri/src/main.rs+++ b/rust/apps/ensu/src-tauri/src/main.rs@@ -1,4 +1,5 @@#![cfg_attr(not(debug_assertions), windows_subsystem = "windows")]+#![forbid(unsafe_code)]use tauri::{Manager, RunEvent, async_runtime};diff --git a/rust/crates/ensu/Cargo.toml b/rust/crates/ensu/Cargo.tomlindex 715b83daff..f5566bfea2 100644--- a/rust/crates/ensu/Cargo.toml+++ b/rust/crates/ensu/Cargo.toml@@ -7,13 +7,13 @@ ente-assets.workspace = trueente-core.workspace = trueente-ensu-crypto.workspace = trueente-image.workspace = true-memmap2 = "0.9.9"regex.workspace = truerubato = { version = "5.0.0", optional = true }rusqlite = { workspace = true, default-features = true, optional = true }self_cell = "1.0.0"serde.workspace = trueserde_json.workspace = true+sha2.workspace = truethiserror.workspace = truetokio = { workspace = true, features = ["rt", "time"] }transcribe-rs = { git = "https://github.com/cjpais/transcribe-rs.git", rev = "efc66111d519a434e28d51d94740008a7ce29360", default-features = false, features = ["onnx", "vad-silero"], optional = true }diff --git a/rust/crates/ensu/src/lib.rs b/rust/crates/ensu/src/lib.rsindex ad8e4cc6f0..514842b6b2 100644--- a/rust/crates/ensu/src/lib.rs+++ b/rust/crates/ensu/src/lib.rs@@ -1,3 +1,5 @@+#![forbid(unsafe_code)]+pub mod config;pub mod conversation;pub mod db;diff --git a/rust/crates/ensu/src/llm/context.rs b/rust/crates/ensu/src/llm/context.rsindex 84effc3c25..d2bf14b340 100644--- a/rust/crates/ensu/src/llm/context.rs+++ b/rust/crates/ensu/src/llm/context.rs@@ -7,11 +7,152 @@ use self_cell::self_cell;use serde::{Deserialize, Serialize};use std::ffi::CString;use std::num::NonZeroU32;+use std::panic::{AssertUnwindSafe, catch_unwind};use std::path::Path;+use std::sync::mpsc;use std::sync::{Arc, Mutex};+use std::time::Instant;+use super::generate::{JobGuard, register_job};use super::model::ModelRef;-use super::{Error, backend, format_error, lock};+use super::worker::Worker;+use super::{+ ChatMessage, ChatRequest, Error, EventSink, GenerationEvent, GenerationSummary, backend,+ cancel, format_error, lock,+};++pub struct Context {+ worker: Worker<LocalContext>,+ context_size: u32,+}++pub type ContextRef = Arc<Context>;++impl Context {+ fn spawn(+ initialize: impl FnOnce() -> Result<LocalContext, Error> + Send + 'static,+ ) -> Result<ContextRef, Error> {+ let worker = Worker::spawn(initialize)?;+ let context_size = worker.call(|context| Ok(context.context_size()))?;+ Ok(Arc::new(Self {+ worker,+ context_size,+ }))+ }++ pub fn new(model: &ModelRef, params: ContextParams) -> Result<ContextRef, Error> {+ let model = model.clone();+ Self::spawn(move || LocalContext::new(&model, params))+ }++ pub fn new_embedding(+ model: &ModelRef,+ params: EmbeddingContextParams,+ ) -> Result<ContextRef, Error> {+ let model = model.clone();+ Self::spawn(move || LocalContext::new_embedding(&model, params))+ }++ pub fn new_knowledge_embedding(+ model: &ModelRef,+ n_threads: Option<i32>,+ ) -> Result<ContextRef, Error> {+ let model = model.clone();+ Self::spawn(move || LocalContext::new_knowledge_embedding(&model, n_threads))+ }++ pub fn context_size(&self) -> u32 {+ self.context_size+ }++ pub fn embed(&self, query: &str) -> Result<Vec<f32>, Error> {+ let query = query.to_owned();+ self.worker.call(move |context| context.embed(&query))+ }++ pub fn embed_document(&self, title: &str, text: &str) -> Result<Vec<f32>, Error> {+ let title = title.to_owned();+ let text = text.to_owned();+ self.worker+ .call(move |context| context.embed_document(&title, &text))+ }++ pub fn measure_text_chat_prompt(&self, request: &ChatRequest) -> Result<usize, Error> {+ let request = request.clone();+ self.worker+ .call(move |context| context.measure_text_chat_prompt(&request))+ }++ pub fn truncate_text_chat_messages(+ &self,+ messages: Vec<ChatMessage>,+ max_tokens: u32,+ ) -> Result<Vec<ChatMessage>, Error> {+ self.worker+ .call(move |context| context.truncate_text_chat_messages(messages, max_tokens))+ }++ pub fn prewarm_multimodal(+ &self,+ mmproj_path: String,+ media_marker: Option<String>,+ ) -> Result<(), Error> {+ self.worker+ .call(move |context| context.prewarm_multimodal(mmproj_path, media_marker))+ }++ pub fn generate_chat_stream(+ &self,+ request: ChatRequest,+ sink: &mut dyn EventSink,+ ) -> Result<GenerationSummary, Error> {+ let (job_id, cancel_flag) = register_job();+ let _job_guard = JobGuard(job_id);+ let start = Instant::now();+ sink.add(GenerationEvent::Text {+ job_id,+ text: String::new(),+ token_id: None,+ });++ let (events, received_events) = mpsc::sync_channel(0);+ let (acknowledge, acknowledgements) = mpsc::sync_channel(0);+ let completed = self.worker.submit(move |context| {+ let mut callback_failed = false;+ let result = context.generate_chat_stream(+ request,+ &mut |event| {+ if events.send(event).is_err() || acknowledgements.recv() != Ok(true) {+ callback_failed = true;+ cancel(job_id);+ }+ },+ job_id,+ cancel_flag,+ start,+ );+ if callback_failed {+ context.invalidate_cache();+ Err(Error::Panicked)+ } else {+ result+ }+ })?;++ let mut callback_failed = false;+ while let Ok(event) = received_events.recv() {+ if !callback_failed {+ callback_failed = catch_unwind(AssertUnwindSafe(|| sink.add(event))).is_err();+ }+ let _ = acknowledge.send(!callback_failed);+ }+ let summary = completed.recv().map_err(|_| Error::Panicked)??;+ sink.add(GenerationEvent::Done {+ summary: summary.clone(),+ });+ Ok(summary)+ }+}#[derive(Debug, Clone, Serialize, Deserialize)]pub struct ContextParams {@@ -59,26 +200,13 @@ struct ContextState {cached_tokens: Vec<LlamaToken>,}-pub struct Context {- state: Mutex<ContextState>,+pub(super) struct LocalContext {mtmd_context: Mutex<Option<CachedMtmdContext>>,+ state: Mutex<ContextState>,embedding_params: Option<EmbeddingContextParams>,}-pub type ContextRef = Arc<Context>;--#[expect(- unsafe_code,- reason = "Retained LLM threading exception; review deferred"-)]-unsafe impl Send for Context {}-#[expect(- unsafe_code,- reason = "Retained LLM threading exception; review deferred"-)]-unsafe impl Sync for Context {}--impl Context {+impl LocalContext {pub fn context_size(&self) -> u32 {lock(&self.state).cell.borrow_dependent().n_ctx()}@@ -88,7 +216,7 @@ impl Context {embedding_params: Option<EmbeddingContextParams>,builder: impl for<'a> FnOnce(&'a ModelRef) -> Result<LlamaContext<'a>, Error>,) -> Result<Self, Error> {- ContextCell::try_new(owner, builder).map(|cell| Context {+ ContextCell::try_new(owner, builder).map(|cell| LocalContext {state: Mutex::new(ContextState {cell,cached_tokens: Vec::new(),@@ -188,8 +316,8 @@ fn mtmd_cache_key_and_params(Ok((key, params))}-impl Context {- pub fn new(model: &ModelRef, params: ContextParams) -> Result<ContextRef, Error> {+impl LocalContext {+ pub fn new(model: &ModelRef, params: ContextParams) -> Result<Self, Error> {let mut context_params = LlamaContextParams::default();if let Some(context_size) = params.context_size {@@ -215,7 +343,7 @@ impl Context {context_params = context_params.with_n_batch(n_batch);}- let context = Context::try_new(Arc::clone(model), None, |model| {+ let context = Self::try_new(Arc::clone(model), None, |model| {let backend = backend()?;model.model()@@ -226,13 +354,10 @@ impl Context {})})?;- Ok(Arc::new(context))+ Ok(context)}- pub fn new_embedding(- model: &ModelRef,- params: EmbeddingContextParams,- ) -> Result<ContextRef, Error> {+ pub fn new_embedding(model: &ModelRef, params: EmbeddingContextParams) -> Result<Self, Error> {let context_size = NonZeroU32::new(params.context_size).ok_or_else(|| Error::InvalidInput("context_size must be > 0".to_string()))?;if params.batch_size == 0 {@@ -269,7 +394,7 @@ impl Context {.with_n_threads_batch(n_threads);}- let context = Context::try_new(Arc::clone(model), Some(params), |model| {+ let context = Self::try_new(Arc::clone(model), Some(params), |model| {let backend = backend()?;model.model()@@ -280,13 +405,13 @@ impl Context {})})?;- Ok(Arc::new(context))+ Ok(context)}pub fn new_knowledge_embedding(model: &ModelRef,n_threads: Option<i32>,- ) -> Result<ContextRef, Error> {+ ) -> Result<Self, Error> {let config = crate::config::knowledge_embedding_config();Self::new_embedding(model,diff --git a/rust/crates/ensu/src/llm/embed.rs b/rust/crates/ensu/src/llm/embed.rsindex 9c6e6e34fa..c81fcd6793 100644--- a/rust/crates/ensu/src/llm/embed.rs+++ b/rust/crates/ensu/src/llm/embed.rs@@ -1,9 +1,10 @@use llama_cpp_2::llama_batch::LlamaBatch;use llama_cpp_2::model::AddBos;-use super::{Context, Error};+use super::Error;+use super::context::LocalContext;-impl Context {+impl LocalContext {pub fn embed(&self, query: &str) -> Result<Vec<f32>, Error> {let params = self.embedding_params().ok_or(Error::Unsupported("Context is not configured for embeddings",diff --git a/rust/crates/ensu/src/llm/generate.rs b/rust/crates/ensu/src/llm/generate.rsindex 0597997dc2..b1f6d07e25 100644--- a/rust/crates/ensu/src/llm/generate.rs+++ b/rust/crates/ensu/src/llm/generate.rs@@ -14,7 +14,7 @@ use std::sync::atomic::{AtomicBool, AtomicI64, Ordering};use std::sync::{Arc, Mutex, OnceLock};use std::time::Instant;-use super::context::Context;+use super::context::LocalContext;use super::event::{EventSink, FinishReason, GenerationEvent, GenerationSummary, JobId};use super::{Error, format_error, lock};@@ -27,7 +27,7 @@ fn cancel_flags() -> &'static Mutex<HashMap<JobId, Arc<AtomicBool>>> {CANCEL_FLAGS.get_or_init(|| Mutex::new(HashMap::new()))}-fn register_job() -> (JobId, Arc<AtomicBool>) {+pub(super) fn register_job() -> (JobId, Arc<AtomicBool>) {let job_id = JOB_COUNTER.fetch_add(1, Ordering::Relaxed);let flag = Arc::new(AtomicBool::new(false));lock(cancel_flags()).insert(job_id, flag.clone());@@ -48,7 +48,7 @@ fn check_cancelled(cancel_flag: &AtomicBool) -> Result<(), Error> {}}-struct JobGuard(JobId);+pub(super) struct JobGuard(pub(super) JobId);impl Drop for JobGuard {fn drop(&mut self) {@@ -476,7 +476,7 @@ fn build_sampler(model: &LlamaModel, request: &SamplingParams) -> Result<LlamaSaOk(LlamaSampler::chain_simple(samplers))}-impl Context {+impl LocalContext {pub fn measure_text_chat_prompt(&self, request: &ChatRequest) -> Result<usize, Error> {if request.image_paths@@ -502,8 +502,11 @@ impl Context {&self,request: ChatRequest,sink: &mut dyn EventSink,+ job_id: JobId,+ cancel_flag: Arc<AtomicBool>,+ start: Instant,) -> Result<GenerationSummary, Error> {- generate_chat_stream(self, request, sink)+ generate_chat_stream(self, request, sink, job_id, cancel_flag, start)}pub fn truncate_text_chat_messages(@@ -561,9 +564,12 @@ impl Context {}fn generate_chat_stream(- context: &Context,+ context: &LocalContext,request: ChatRequest,sink: &mut dyn EventSink,+ job_id: JobId,+ cancel_flag: Arc<AtomicBool>,+ start: Instant,) -> Result<GenerationSummary, Error> {let ChatRequest {messages,@@ -584,16 +590,6 @@ fn generate_chat_stream(grammar,} = request;- let (job_id, cancel_flag) = register_job();- let _job_guard = JobGuard(job_id);- let start = Instant::now();-- sink.add(GenerationEvent::Text {- job_id,- text: String::new(),- token_id: None,- });-let max_tokens = max_tokens.unwrap_or(DEFAULT_GENERATION_MAX_TOKENS);let max_tokens = usize::try_from(max_tokens.max(0)).unwrap_or(0);let stop_sequences = stop_sequences.unwrap_or_default();@@ -843,10 +839,6 @@ fn generate_chat_stream(finish_reason: job.finish_reason,};- sink.add(GenerationEvent::Done {- summary: summary.clone(),- });-Ok(summary)}diff --git a/rust/crates/ensu/src/llm/mod.rs b/rust/crates/ensu/src/llm/mod.rsindex 26ddd5a6b9..885d2d9a96 100644--- a/rust/crates/ensu/src/llm/mod.rs+++ b/rust/crates/ensu/src/llm/mod.rs@@ -4,6 +4,7 @@ mod event;mod generate;mod history;mod model;+mod worker;pub use context::*;pub use event::*;diff --git a/rust/crates/ensu/src/llm/worker.rs b/rust/crates/ensu/src/llm/worker.rsnew file mode 100644index 0000000000..5f6cc03396--- /dev/null+++ b/rust/crates/ensu/src/llm/worker.rs@@ -0,0 +1,88 @@+use std::panic::{AssertUnwindSafe, catch_unwind};+use std::sync::mpsc::{self, Receiver, Sender};+use std::thread::{self, JoinHandle};++use super::Error;++type Task<T> = Box<dyn FnOnce(&T) -> bool + Send>;++pub(super) struct Worker<T> {+ sender: Option<Sender<Task<T>>>,+ thread: Option<JoinHandle<()>>,+}++impl<T: 'static> Worker<T> {+ pub(super) fn spawn(+ initialize: impl FnOnce() -> Result<T, Error> + Send + 'static,+ ) -> Result<Self, Error> {+ let (sender, receiver) = mpsc::channel::<Task<T>>();+ let (ready, initialized) = mpsc::sync_channel(1);+ let thread = thread::Builder::new()+ .name("ensu-llm".to_owned())+ .spawn(move || {+ let state = match initialize() {+ Ok(state) => state,+ Err(error) => {+ let _ = ready.send(Err(error));+ return;+ }+ };+ if ready.send(Ok(())).is_err() {+ return;+ }+ for task in receiver {+ if !task(&state) {+ break;+ }+ }+ })+ .map_err(|error| Error::Llama {+ op: "Failed to start LLM worker",+ message: error.to_string(),+ })?;+ let worker = Self {+ sender: Some(sender),+ thread: Some(thread),+ };+ initialized.recv().map_err(|_| Error::Panicked)??;+ Ok(worker)+ }++ pub(super) fn submit<R: Send + 'static>(+ &self,+ task: impl FnOnce(&T) -> Result<R, Error> + Send + 'static,+ ) -> Result<Receiver<Result<R, Error>>, Error> {+ let (sender, receiver) = mpsc::sync_channel(1);+ self.sender+ .as_ref()+ .ok_or(Error::Panicked)?+ .send(Box::new(move |state| {+ let result = catch_unwind(AssertUnwindSafe(|| task(state)));+ let keep_running = result.is_ok();+ let _ = sender.send(result.unwrap_or(Err(Error::Panicked)));+ keep_running+ }))+ .map_err(|_| Error::Panicked)?;+ Ok(receiver)+ }++ pub(super) fn call<R: Send + 'static>(+ &self,+ task: impl FnOnce(&T) -> Result<R, Error> + Send + 'static,+ ) -> Result<R, Error> {+ self.submit(task)?.recv().map_err(|_| Error::Panicked)?+ }+}++impl<T> Drop for Worker<T> {+ fn drop(&mut self) {+ self.sender.take();+ if let Some(thread) = self.thread.take() {+ let _ = thread.join();+ }+ }+}++#[cfg(test)]+#[path = "worker_tests.rs"]+mod tests;diff --git a/rust/crates/ensu/src/llm/worker_tests.rs b/rust/crates/ensu/src/llm/worker_tests.rsnew file mode 100644index 0000000000..d15911e75a--- /dev/null+++ b/rust/crates/ensu/src/llm/worker_tests.rs@@ -0,0 +1,96 @@+use std::cell::Cell;+use std::rc::Rc;+use std::sync::mpsc;+use std::thread::{self, ThreadId};++use super::Worker;+use crate::llm::Error;++struct State {+ value: Rc<Cell<usize>>,+ owner: ThreadId,+ dropped: mpsc::Sender<ThreadId>,+}++impl Drop for State {+ fn drop(&mut self) {+ let _ = self.dropped.send(thread::current().id());+ }+}++#[test]+fn creates_uses_and_drops_non_send_state_on_one_thread() {+ let (dropped, received) = mpsc::channel();+ let worker = Worker::spawn(move || {+ Ok(State {+ value: Rc::new(Cell::new(0)),+ owner: thread::current().id(),+ dropped,+ })+ })+ .unwrap();+ let owner = worker.call(|state| Ok(state.owner)).unwrap();+ assert_ne!(owner, thread::current().id());+ thread::scope(|scope| {+ for _ in 0..8 {+ scope.spawn(|| {+ for _ in 0..100 {+ worker+ .call(|state| {+ assert_eq!(state.owner, thread::current().id());+ state.value.set(state.value.get() + 1);+ Ok(())+ })+ .unwrap();+ }+ });+ }+ });+ assert_eq!(worker.call(|state| Ok(state.value.get())).unwrap(), 800);+ drop(worker);+ assert_eq!(received.recv().unwrap(), owner);+}++#[test]+fn initialization_failures_and_panics_reach_the_caller() {+ let result = Worker::<()>::spawn(|| Err(Error::InvalidInput("fixture".to_owned())));+ assert!(matches!(result, Err(Error::InvalidInput(_))));+ let result = Worker::<()>::spawn(|| panic!("initialization failed"));+ assert!(matches!(result, Err(Error::Panicked)));+}++#[test]+fn operation_error_keeps_worker_available() {+ let worker = Worker::spawn(|| Ok(Rc::new(Cell::new(7)))).unwrap();+ let result = worker.call::<()>(|_| Err(Error::Cancelled));+ assert!(matches!(result, Err(Error::Cancelled)));+ assert_eq!(worker.call(|state| Ok(state.get())).unwrap(), 7);+}++#[test]+fn panic_closes_pending_requests_and_drops_state_on_worker() {+ let (dropped, received) = mpsc::channel();+ let worker = Worker::spawn(move || {+ Ok(State {+ value: Rc::new(Cell::new(0)),+ owner: thread::current().id(),+ dropped,+ })+ })+ .unwrap();+ let owner = worker.call(|state| Ok(state.owner)).unwrap();+ let (release, released) = mpsc::channel();+ let first = worker+ .submit::<()>(move |_| {+ released.recv().unwrap();+ panic!("operation failed");+ })+ .unwrap();+ let second = worker.submit(|_| Ok(1)).unwrap();+ release.send(()).unwrap();+ assert!(matches!(first.recv().unwrap(), Err(Error::Panicked)));+ assert!(second.recv().is_err());+ assert!(matches!(worker.call(|_| Ok(())), Err(Error::Panicked)));+ drop(worker);+ assert_eq!(received.recv().unwrap(), owner);+}diff --git a/rust/crates/ensu/src/retrieval/index.rs b/rust/crates/ensu/src/retrieval/index.rsindex c420a3f4c1..59ca50bb8c 100644--- a/rust/crates/ensu/src/retrieval/index.rs+++ b/rust/crates/ensu/src/retrieval/index.rs@@ -1,12 +1,12 @@use std::cmp::{Ordering, Reverse};use std::collections::{BTreeMap, BinaryHeap};-use std::fs::{self, File};-use std::io::Read;+use std::fs::{self, File, Metadata};+use std::io::{Read, Seek, SeekFrom};use std::path::Path;-use std::sync::OnceLock;+use std::sync::{Mutex, MutexGuard, OnceLock};-use memmap2::{Mmap, MmapOptions};use serde::Deserialize;+use sha2::{Digest, Sha256};use url::Url;use crate::config::{@@ -18,6 +18,39 @@ use super::{BEGIN_CONTEXT_SENTINEL, END_CONTEXT_SENTINEL, RetrievalError, normalconst MAX_MANIFEST_BYTES: u64 = 1_048_576;const MAX_METADATA_FRAME_BYTES: u64 = 1_048_576;+const READ_BUFFER_BYTES: usize = 1_048_576;++struct IndexFile {+ file: Mutex<File>,+ metadata: Metadata,+}++impl IndexFile {+ fn new(file: File) -> Result<Self, RetrievalError> {+ Ok(Self {+ metadata: file.metadata()?,+ file: Mutex::new(file),+ })+ }++ fn lock(&self) -> Result<MutexGuard<'_, File>, RetrievalError> {+ self.file+ .lock()+ .map_err(|_| RetrievalError::InvalidPack("knowledge file lock poisoned".to_owned()))+ }++ fn verify(&self) -> Result<(), RetrievalError> {+ let current = self.lock()?.metadata()?;+ if current.len() != self.metadata.len()+ || current.modified().ok() != self.metadata.modified().ok()+ {+ return Err(RetrievalError::InvalidPack(+ "knowledge file changed while open".to_owned(),+ ));+ }+ Ok(())+ }+}#[derive(Debug, Clone, PartialEq)]pub struct RetrievalHit {pub locator: super::PassageLocator,@@ -58,6 +91,7 @@ pub struct RetrievalIndex {dataset_identity: String,stable_id: String,revision_sha256: OnceLock<String>,+ revision_lock: Mutex<()>,manifest_bytes: Vec<u8>,count: usize,dim: usize,@@ -65,8 +99,8 @@ pub struct RetrievalIndex {rows_per_block: usize,max_chars: usize,url_template: String,- vectors: Mmap,- metadata: Mmap,+ vectors: IndexFile,+ metadata: IndexFile,offsets: Vec<u64>,}@@ -167,21 +201,13 @@ impl RetrievalIndex {.collect::<Vec<_>>();validate_offsets(&offsets, metadata_len)?;- // Native callers prevent file swaps or deletion while mapped.- #[expect(- unsafe_code,- reason = "File-backed memory mapping requires an unsafe call"- )]- let vectors = unsafe { MmapOptions::new().map(&vector_file)? };- #[expect(- unsafe_code,- reason = "File-backed memory mapping requires an unsafe call"- )]- let metadata = unsafe { MmapOptions::new().map(&metadata_file)? };+ let vectors = IndexFile::new(vector_file)?;+ let metadata = IndexFile::new(metadata_file)?;Ok(Self {stable_id: expected_pack.stable_id.clone(),revision_sha256: OnceLock::new(),+ revision_lock: Mutex::new(()),manifest_bytes,dataset_identity: manifest.dataset,count,@@ -200,15 +226,45 @@ impl RetrievalIndex {&self.dataset_identity}- fn revision_sha256(&self) -> &str {- self.revision_sha256.get_or_init(|| {- let offsets = self- .offsets- .iter()- .flat_map(|offset| offset.to_le_bytes())- .collect::<Vec<_>>();- text_revision_digest(&self.manifest_bytes, &self.metadata, &offsets)- })+ fn revision_sha256(&self) -> Result<&str, RetrievalError> {+ if let Some(digest) = self.revision_sha256.get() {+ return Ok(digest);+ }+ let _initialize = self.revision_lock.lock().map_err(|_| {+ RetrievalError::InvalidPack("knowledge revision lock poisoned".to_owned())+ })?;+ if let Some(digest) = self.revision_sha256.get() {+ return Ok(digest);+ }+ self.metadata.verify()?;+ let mut hash = Sha256::new();+ hash.update(b"ensu-pack-text-revision-v1\0");+ hash.update((self.manifest_bytes.len() as u64).to_le_bytes());+ hash.update(&self.manifest_bytes);+ hash.update(self.metadata.metadata.len().to_le_bytes());+ {+ let mut file = self.metadata.lock()?;+ file.seek(SeekFrom::Start(0))?;+ let mut buffer = vec![0; READ_BUFFER_BYTES];+ let mut remaining = self.metadata.metadata.len();+ while remaining > 0 {+ let count = remaining.min(buffer.len() as u64) as usize;+ file.read_exact(&mut buffer[..count])?;+ hash.update(&buffer[..count]);+ remaining -= count as u64;+ }+ }+ self.metadata.verify()?;+ hash.update((self.offsets.len() as u64 * 8).to_le_bytes());+ for offset in &self.offsets {+ hash.update(offset.to_le_bytes());+ }+ let digest = hash+ .finalize()+ .iter()+ .map(|byte| format!("{byte:02x}"))+ .collect();+ Ok(self.revision_sha256.get_or_init(|| digest))}pub fn search(@@ -242,27 +298,39 @@ impl RetrievalIndex {}let max_hits = max_hits.min(self.count);+ self.vectors.verify()?;+ self.metadata.verify()?;+ let rows_per_read = (READ_BUFFER_BYTES / self.dim).max(1).min(self.count);+ let mut buffer = vec![0; rows_per_read * self.dim];let mut heap = BinaryHeap::<Reverse<RankedRow>>::with_capacity(max_hits);- for row in 0..self.count {- let start = row * self.dim;- let vector = &self.vectors[start..start + self.dim];- let dot = query- .iter()- .zip(vector)- .map(|(query_component, document_component)| {- *query_component * f32::from(*document_component as i8)- })- .sum::<f32>();- let score = dot / self.scale;- if !score.is_finite() || score < threshold {- continue;+ for first_row in (0..self.count).step_by(rows_per_read) {+ let rows = rows_per_read.min(self.count - first_row);+ let bytes = &mut buffer[..rows * self.dim];+ {+ let mut file = self.vectors.lock()?;+ file.seek(SeekFrom::Start((first_row * self.dim) as u64))?;+ file.read_exact(bytes)?;}- let candidate = RankedRow { score, row };- if heap.len() < max_hits {- heap.push(Reverse(candidate));- } else if heap.peek().is_some_and(|worst| candidate > worst.0) {- heap.pop();- heap.push(Reverse(candidate));+ for (index, vector) in bytes.chunks_exact(self.dim).enumerate() {+ let row = first_row + index;+ let dot = query+ .iter()+ .zip(vector)+ .map(|(query_component, document_component)| {+ *query_component * f32::from(*document_component as i8)+ })+ .sum::<f32>();+ let score = dot / self.scale;+ if !score.is_finite() || score < threshold {+ continue;+ }+ let candidate = RankedRow { score, row };+ if heap.len() < max_hits {+ heap.push(Reverse(candidate));+ } else if heap.peek().is_some_and(|worst| candidate > worst.0) {+ heap.pop();+ heap.push(Reverse(candidate));+ }}}@@ -271,7 +339,10 @@ impl RetrievalIndex {.map(|Reverse(candidate)| candidate).collect::<Vec<_>>();ranked.sort_unstable_by(|left, right| right.cmp(left));- self.load_hits(&ranked)+ let hits = self.load_hits(&ranked)?;+ self.vectors.verify()?;+ self.metadata.verify()?;+ Ok(hits)}pub fn reload_passage(@@ -289,13 +360,16 @@ impl RetrievalIndex {let Ok(row) = usize::try_from(*row) else {return Ok(None);};+ self.metadata.verify()?;if dataset_id != &self.stable_id|| row >= self.count- || revision_sha256 != self.revision_sha256()+ || revision_sha256 != self.revision_sha256()?{return Ok(None);}- Ok(self.load_hits(&[RankedRow { score: 0.0, row }])?.pop())+ let hit = self.load_hits(&[RankedRow { score: 0.0, row }])?.pop();+ self.metadata.verify()?;+ Ok(hit)}fn load_hits(&self, ranked: &[RankedRow]) -> Result<Vec<RetrievalHit>, RetrievalError> {@@ -352,7 +426,7 @@ impl RetrievalIndex {hits[rank] = Some(RetrievalHit {locator: super::PassageLocator::EnsuPack {dataset_id: self.stable_id.clone(),- revision_sha256: self.revision_sha256().to_owned(),+ revision_sha256: self.revision_sha256()?.to_owned(),row: selected.row as u64,},score: selected.score,@@ -380,7 +454,9 @@ impl RetrievalIndex {let end = usize::try_from(self.offsets[block + 1]).map_err(|_| {RetrievalError::InvalidPack("metadata frame end is too large".to_string())})?;- let frame = &self.metadata[start..end];+ let mut file = self.metadata.lock()?;+ file.seek(SeekFrom::Start(start as u64))?;+ let frame = (&mut *file).take((end - start) as u64);let decoder = zstd::stream::read::Decoder::new(frame).map_err(|error| RetrievalError::Zstd(error.to_string()))?;let mut capped = decoder.take(MAX_METADATA_FRAME_BYTES + 1);@@ -556,6 +632,7 @@ fn python_quote(value: &str) -> String {encoded}+#[cfg(test)]fn text_revision_digest(manifest: &[u8], metadata: &[u8], offsets: &[u8]) -> String {ente_ensu_crypto::sha256_parts(&[b"ensu-pack-text-revision-v1\0",@@ -679,6 +756,131 @@ pub(super) mod tests {}}+ #[test]+ fn streamed_revision_matches_the_existing_digest_format() {+ let pack = synthetic_pack("simplewiki-test");+ let index = RetrievalIndex::open(&pack.revision, &pack.expected).unwrap();+ let expected = text_revision_digest(+ &fs::read(pack.revision.join(KNOWLEDGE_MANIFEST_FILE)).unwrap(),+ &fs::read(pack.revision.join(KNOWLEDGE_META_FILE)).unwrap(),+ &fs::read(pack.revision.join(KNOWLEDGE_OFFSETS_FILE)).unwrap(),+ );+ assert_eq!(index.revision_sha256().unwrap(), expected);+ assert_eq!(index.revision_sha256().unwrap(), expected);+ }++ #[test]+ fn rejects_truncation_after_open_and_after_caching_a_revision() {+ for name in [KNOWLEDGE_VECTORS_FILE, KNOWLEDGE_META_FILE] {+ let pack = synthetic_pack("simplewiki-test");+ let index = RetrievalIndex::open(&pack.revision, &pack.expected).unwrap();+ let mut query = vec![0.0; 512];+ query[0] = 1.0;+ let hit = index.search(&query, 1, 0.0).unwrap().remove(0);+ fs::write(pack.revision.join(name), []).unwrap();+ assert!(index.search(&query, 1, 0.0).is_err());+ if name == KNOWLEDGE_META_FILE {+ assert!(index.reload_passage(&hit.locator).is_err());+ }+ }+ }++ #[test]+ fn rejects_same_length_metadata_edits() {+ let pack = synthetic_pack("simplewiki-test");+ let index = RetrievalIndex::open(&pack.revision, &pack.expected).unwrap();+ index.revision_sha256().unwrap();+ let path = pack.revision.join(KNOWLEDGE_META_FILE);+ let mut bytes = fs::read(&path).unwrap();+ bytes[0] ^= 1;+ fs::write(&path, bytes).unwrap();+ File::options()+ .write(true)+ .open(path)+ .unwrap()+ .set_modified(std::time::SystemTime::UNIX_EPOCH)+ .unwrap();+ assert!(index.search(&vec![0.0; 512], 1, 0.0).is_err());+ }++ #[test]+ fn chunked_search_matches_a_full_scan_and_supports_concurrent_readers() {+ let pack = synthetic_pack("simplewiki-test");+ let count = 4_101;+ let dim = 512;+ let vectors: Vec<u8> = (0..count * dim)+ .map(|index| ((index * 73 + index / dim * 19) % 255) as u8)+ .collect();+ fs::write(pack.revision.join(KNOWLEDGE_VECTORS_FILE), &vectors).unwrap();+ let mut metadata = Vec::new();+ let mut offsets = vec![0_u64];+ for first in (0..count).step_by(128) {+ let rows = (first..(first + 128).min(count))+ .map(|row| {+ json!({"title":format!("Row {row}"), "text":format!("Passage {row}")})+ .to_string()+ })+ .collect::<Vec<_>>()+ .join("\n");+ metadata.extend(zstd::stream::encode_all(rows.as_bytes(), 1).unwrap());+ offsets.push(metadata.len() as u64);+ }+ fs::write(pack.revision.join(KNOWLEDGE_META_FILE), &metadata).unwrap();+ fs::write(+ pack.revision.join(KNOWLEDGE_OFFSETS_FILE),+ offsets+ .into_iter()+ .flat_map(u64::to_le_bytes)+ .collect::<Vec<_>>(),+ )+ .unwrap();+ let manifest_path = pack.revision.join(KNOWLEDGE_MANIFEST_FILE);+ let mut manifest: serde_json::Value =+ serde_json::from_slice(&fs::read(&manifest_path).unwrap()).unwrap();+ manifest["count"] = json!(count);+ fs::write(manifest_path, serde_json::to_vec(&manifest).unwrap()).unwrap();+ let index = RetrievalIndex::open(&pack.revision, &pack.expected).unwrap();+ std::thread::scope(|scope| {+ for seed in 0..8 {+ let index = &index;+ let vectors = &vectors;+ scope.spawn(move || {+ let query: Vec<f32> = (0..dim)+ .map(|position| ((position * 17 + seed * 31) % 251) as f32 / 251.0)+ .collect();+ let mut expected: Vec<RankedRow> = vectors+ .chunks_exact(dim)+ .enumerate()+ .map(|(row, vector)| RankedRow {+ row,+ score: query+ .iter()+ .zip(vector)+ .map(|(query, value)| *query * f32::from(*value as i8))+ .sum::<f32>()+ / 127.0,+ })+ .filter(|row| row.score >= 0.0)+ .collect();+ expected.sort_unstable_by(|left, right| right.cmp(left));+ expected.truncate(10);+ for _ in 0..3 {+ let hits = index.search(&query, 10, 0.0).unwrap();+ assert_eq!(hits.len(), expected.len());+ for (hit, expected) in hits.iter().zip(&expected) {+ assert_eq!(hit.score, expected.score);+ assert_eq!(hit.title, format!("Row {}", expected.row));+ assert_eq!(+ index.reload_passage(&hit.locator).unwrap().unwrap().text,+ hit.text+ );+ }+ }+ });+ }+ });+ }+#[test]fn passage_reload_preserves_sources_and_rejects_changed_artifacts() {use crate::retrieval::PassageLocator;diff --git a/rust/crates/ensu/tests/llm_worker.rs b/rust/crates/ensu/tests/llm_worker.rsnew file mode 100644index 0000000000..611d407332--- /dev/null+++ b/rust/crates/ensu/tests/llm_worker.rs@@ -0,0 +1,180 @@+#![cfg(test)]++use std::cell::Cell;+use std::rc::Rc;+use std::sync::Arc;+use std::thread;++use ente_ensu::llm::{self, ChatMessage, ChatRequest, Context, ContextParams, GenerationEvent};++fn model() -> llm::ModelRef {+ llm::Model::load(llm::ModelLoadParams {+ model_path: std::env::var("ENSU_TEST_MODEL").unwrap(),+ n_gpu_layers: Some(99),+ use_mmap: None,+ use_mlock: None,+ })+ .unwrap()+}++fn context() -> llm::ContextRef {+ Context::new(+ &model(),+ ContextParams {+ context_size: Some(512),+ n_threads: Some(4),+ n_batch: Some(256),+ },+ )+ .unwrap()+}++fn request() -> ChatRequest {+ ChatRequest {+ messages: vec![ChatMessage {+ role: "user".to_owned(),+ content: "Count from one to five.".to_owned(),+ }],+ temperature: Some(0.0),+ max_tokens: Some(16),+ ..Default::default()+ }+}++#[test]+fn public_context_is_send_and_sync() {+ fn assert_send_sync<T: Send + Sync>() {}+ assert_send_sync::<Context>();+}++#[test]+#[ignore = "requires ENSU_TEST_MODEL"]+fn callbacks_stay_on_caller_and_accept_non_send_state() {+ let context = context();+ let caller = thread::current().id();+ let count = Rc::new(Cell::new(0));+ let mut done = false;+ context+ .generate_chat_stream(request(), &mut |event| {+ assert_eq!(thread::current().id(), caller);+ assert_eq!(context.context_size(), 512);+ count.set(count.get() + 1);+ if let GenerationEvent::Done { .. } = event {+ done = true;+ assert!(context.measure_text_chat_prompt(&request()).unwrap() > 0);+ }+ })+ .unwrap();+ assert!(done && count.get() > 2);+}++#[test]+#[ignore = "requires ENSU_TEST_MODEL"]+fn callback_panic_returns_an_error_and_context_recovers() {+ let context = context();+ let result = context.generate_chat_stream(request(), &mut |event| {+ if let GenerationEvent::Text { text, .. } = event+ && !text.is_empty()+ {+ panic!("callback failure");+ }+ });+ assert!(matches!(result, Err(llm::Error::Panicked)));+ let mut output = String::new();+ context+ .generate_chat_stream(request(), &mut |event| {+ if let GenerationEvent::Text { text, .. } = event {+ output.push_str(&text);+ }+ })+ .unwrap();+ assert!(!output.is_empty());+}++#[test]+#[ignore = "requires ENSU_TEST_MODEL"]+fn cancellation_is_acknowledged_before_another_text_event() {+ let context = context();+ for cancel_immediately in [true, false] {+ let mut cancelled = false;+ let mut done = false;+ let result = context.generate_chat_stream(request(), &mut |event| match event {+ GenerationEvent::Text { job_id, text, .. } => {+ assert!(!cancelled);+ if cancel_immediately || !text.is_empty() {+ llm::cancel(job_id);+ cancelled = true;+ }+ }+ GenerationEvent::Done { .. } => done = true,+ });+ assert!(cancelled && !done);+ assert!(matches!(result, Err(llm::Error::Cancelled)));+ }+}++#[test]+#[ignore = "requires ENSU_TEST_MODEL"]+fn concurrent_generations_preserve_output_and_can_drop_caller_model() {+ let owner = model();+ let context = Context::new(+ &owner,+ ContextParams {+ context_size: Some(512),+ n_threads: Some(4),+ n_batch: Some(256),+ },+ )+ .unwrap();+ drop(owner);+ let mut runs = Vec::new();+ for _ in 0..4 {+ let context = context.clone();+ runs.push(thread::spawn(move || {+ let mut output = String::new();+ context+ .generate_chat_stream(request(), &mut |event| {+ if let GenerationEvent::Text { text, .. } = event {+ output.push_str(&text);+ }+ })+ .unwrap();+ output+ }));+ }+ let outputs: Vec<_> = runs.into_iter().map(|run| run.join().unwrap()).collect();+ assert!(!outputs[0].is_empty());+ assert!(outputs.windows(2).all(|pair| pair[0] == pair[1]));+}++#[test]+#[ignore = "requires ENSU_TEST_EMBED_MODEL"]+fn concurrent_embeddings_match_serial_results() {+ let model = llm::Model::load(llm::ModelLoadParams {+ model_path: std::env::var("ENSU_TEST_EMBED_MODEL").unwrap(),+ n_gpu_layers: Some(99),+ use_mmap: None,+ use_mlock: None,+ })+ .unwrap();+ let context = Context::new_knowledge_embedding(&model, Some(4)).unwrap();+ drop(model);+ let expected = Arc::new(context.embed("How do volcanoes form?").unwrap());+ assert_eq!(expected.len(), 512);+ let mut runs = Vec::new();+ for _ in 0..4 {+ let context = context.clone();+ let expected = expected.clone();+ runs.push(thread::spawn(move || {+ for _ in 0..3 {+ let actual = context.embed("How do volcanoes form?").unwrap();+ for (left, right) in actual.iter().zip(expected.iter()) {+ assert!((left - right).abs() < 1e-5);+ }+ }+ }));+ }+ for run in runs {+ run.join().unwrap();+ }+}