π End-to-end encrypted cloud for everything.
Dart 2faandroidauthye2eeencryptionend-to-end-encryptionfluttergolanggoogle-photosgoogle-photos-alternativeiosopensourcephotosprivacyprivacy-toolstypescriptzero-knowledge
Harden Your Notes indexing and watchers
setalp7bbe263032e4
diff --git a/rust/Cargo.lock b/rust/Cargo.lockindex a22e495af7..e69506f6b2 100644--- a/rust/Cargo.lock+++ b/rust/Cargo.lock@@ -2381,7 +2381,6 @@ dependencies = ["tauri-plugin-fs","tauri-plugin-opener","tauri-plugin-process",- "tauri-plugin-single-instance","tauri-plugin-updater","uuid","windows-sys 0.61.2",@@ -2541,7 +2540,6 @@ dependencies = ["self_cell","serde","serde_json",- "sha2 0.11.0","tempfile","thiserror 2.0.18","tokio",@@ -9457,22 +9455,6 @@ dependencies = ["tauri-plugin",]-[[package]]-name = "tauri-plugin-single-instance"-version = "2.4.4"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "5cd0cb5c412a5071b69bab6a6df1583cbb89460d4a83b6a24769b08d15b6b1e1"-dependencies = [- "serde",- "serde_json",- "tauri",- "thiserror 2.0.18",- "tokio",- "tracing",- "windows-sys 0.60.2",- "zbus 5.15.0",-]-[[package]]name = "tauri-plugin-updater"version = "2.10.1"diff --git a/rust/apps/ensu/src-tauri/Cargo.toml b/rust/apps/ensu/src-tauri/Cargo.tomlindex 7c9aae59b4..261eb359e8 100644--- a/rust/apps/ensu/src-tauri/Cargo.toml+++ b/rust/apps/ensu/src-tauri/Cargo.toml@@ -21,7 +21,6 @@ tauri-plugin-dialog = "2.7.1"tauri-plugin-fs = "2.5.1"tauri-plugin-opener = "2.5.4"tauri-plugin-process = "2.3.1"-tauri-plugin-single-instance = "2.4.4"tauri-plugin-updater = "2.10.1"uuid = { version = "1", features = ["v4"] }diff --git a/rust/apps/ensu/src-tauri/src/commands/knowledge.rs b/rust/apps/ensu/src-tauri/src/commands/knowledge.rsindex a7239645ea..5e51285af8 100644--- a/rust/apps/ensu/src-tauri/src/commands/knowledge.rs+++ b/rust/apps/ensu/src-tauri/src/commands/knowledge.rs@@ -619,10 +619,15 @@ pub async fn knowledge_retrieve(check_cancelled()?;match notes.search_collection(collection_id, &query_embedding) {Ok(collection_hits) => note_hits.extend(collection_hits),- Err(error) => logging::log(- "Knowledge",- format!("Notes search skipped collection={collection_id} error={error}"),- ),+ Err(error) => {+ crate::commands::notes::mark_index_unreadable(&app, collection_id);+ logging::log(+ "Knowledge",+ format!(+ "Notes search skipped and queued for rebuild collection={collection_id} error={error}"+ ),+ );+ }}}}diff --git a/rust/apps/ensu/src-tauri/src/commands/notes.rs b/rust/apps/ensu/src-tauri/src/commands/notes.rsindex 51d01db47d..25facfec4c 100644--- a/rust/apps/ensu/src-tauri/src/commands/notes.rs+++ b/rust/apps/ensu/src-tauri/src/commands/notes.rs@@ -1,7 +1,11 @@use std::collections::{BTreeSet, HashMap};-use std::fs;+use std::fs::{self, File, OpenOptions};#[cfg(unix)]-use std::os::unix::fs::PermissionsExt;+use std::os::fd::AsRawFd;+#[cfg(unix)]+use std::os::unix::fs::{OpenOptionsExt, PermissionsExt};+#[cfg(windows)]+use std::os::windows::fs::OpenOptionsExt;use std::path::{Path, PathBuf};use std::sync::atomic::Ordering;use std::sync::{Arc, Mutex, PoisonError};@@ -19,6 +23,8 @@ use tauri::{AppHandle, Manager, State as TauriState};use tauri_plugin_fs::FsExt;use tauri_plugin_opener::OpenerExt;use uuid::Uuid;+#[cfg(windows)]+use windows_sys::Win32::Foundation::ERROR_SHARING_VIOLATION;use crate::commands::common::ApiError;use crate::commands::llm::ModelDownloadState;@@ -35,14 +41,12 @@ use source::{is_supported_file, read_collection_source, source_root_is_available, validate_new_source_root,};pub use watcher::initialize_for_app;-pub(crate) use watcher::mark_reference_stale;-use watcher::{- emit_state_changed, ensure_collection_watcher, install_collection_watcher,- mark_collection_dirty,-};+use watcher::{emit_state_changed, ensure_collection_watcher, install_collection_watcher};+pub(crate) use watcher::{mark_collection_dirty, mark_reference_stale};const NOTES_DIRECTORY: &str = "notes";const NOTES_INDEX_DIRECTORY: &str = "indexes";+const NOTES_OWNERSHIP_LOCK_FILE: &str = ".owner.lock";const NOTES_EMBEDDING_TITLE_MAX_UTF8_BYTES: usize = 512;const NOTES_QUIET_PERIOD_MS: i64 = 5 * 60 * 1_000;const NOTES_STATE_CHANGED_EVENT: &str = "notes-state-changed";@@ -92,6 +96,7 @@ struct CollectionRuntime {indexed_document_count: u64,last_updated_at_ms: Option<i64>,last_error: Option<String>,+ watcher_error: Option<String>,}impl CollectionRuntime {@@ -103,6 +108,7 @@ impl CollectionRuntime {indexed_document_count: 0,last_updated_at_ms: None,last_error: None,+ watcher_error: None,}}@@ -126,6 +132,7 @@ pub struct NotesCollectionDto {last_updated_at_ms: Option<i64>,update_due_at_ms: Option<i64>,last_error: Option<String>,+ watcher_error: Option<String>,}#[derive(Debug, Serialize)]@@ -219,6 +226,7 @@ impl IndexCache {struct DesktopUpdateState {forced_document_ids: BTreeSet<String>,force_full_hash: bool,+ rebuild_derived_index: bool,quiet_deadline_ms: i64,generation: u64,bypass_quiet_period: bool,@@ -233,7 +241,11 @@ impl DesktopUpdateState {) {self.forced_document_ids.extend(forced_document_ids);self.force_full_hash |= force_full_hash;- self.quiet_deadline_ms = now_ms.saturating_add(NOTES_QUIET_PERIOD_MS);+ self.quiet_deadline_ms = if self.rebuild_derived_index {+ now_ms+ } else {+ now_ms.saturating_add(NOTES_QUIET_PERIOD_MS)+ };self.generation = self.generation.wrapping_add(1);self.bypass_quiet_period = false;}@@ -252,6 +264,17 @@ impl DesktopUpdateState {self.bypass_quiet_period = false;true}++ fn record_unreadable_index(&mut self, now_ms: i64) -> bool {+ if self.rebuild_derived_index {+ return false;+ }+ self.rebuild_derived_index = true;+ self.quiet_deadline_ms = now_ms;+ self.generation = self.generation.wrapping_add(1);+ self.bypass_quiet_period = false;+ true+ }}#[derive(Default)]@@ -271,6 +294,7 @@ pub struct State {watchers: Mutex<HashMap<String, CollectionWatcher>>,watch_work: Mutex<HashMap<String, watcher::WatchWork>>,updates: Mutex<HashMap<String, DesktopUpdateState>>,+ _ownership_lock: File,}#[derive(Clone)]@@ -283,6 +307,8 @@ pub(crate) struct RetrievalHandle {impl State {pub fn new(app_data_dir: PathBuf) -> Result<Self, ApiError> {let notes_directory = app_data_dir.join(NOTES_DIRECTORY);+ ensure_directory(¬es_directory)?;+ let ownership_lock = acquire_notes_ownership(¬es_directory)?;let index_root = notes_directory.join(NOTES_INDEX_DIRECTORY);ensure_directory(&index_root)?;let registry = RegistryStore::open(notes_directory)?;@@ -336,6 +362,7 @@ impl State {watchers: Mutex::new(HashMap::new()),watch_work: Mutex::new(HashMap::new()),updates: Mutex::new(HashMap::new()),+ _ownership_lock: ownership_lock,})}@@ -395,6 +422,21 @@ impl State {.unwrap_or_else(CollectionRuntime::pending)}+ fn set_watcher_error(&self, collection_id: &str, error: Option<&ApiError>) -> bool {+ let mut runtimes = self.runtimes.lock().unwrap_or_else(PoisonError::into_inner);+ let Some(runtime) = runtimes.get_mut(collection_id) else {+ return false;+ };+ let watcher_error = error.map(|_| {+ "Changes cannot be watched automatically. Refresh this folder manually.".to_string()+ });+ if runtime.watcher_error == watcher_error {+ return false;+ }+ runtime.watcher_error = watcher_error;+ true+ }+fn prepare_update(&self,collection_id: &str,@@ -422,7 +464,7 @@ impl State {generation: update.generation,forced_document_ids: update.forced_document_ids.iter().cloned().collect(),force_full_hash: update.force_full_hash,- rebuild_derived_index: false,+ rebuild_derived_index: update.rebuild_derived_index,})}@@ -591,6 +633,35 @@ impl State {}}+pub(crate) fn mark_index_unreadable(app: &AppHandle, collection_id: &str) {+ let state = app.state::<State>();+ if state.registered_collection(collection_id).is_err() {+ return;+ }+ state.evict_cached_index(collection_id);+ {+ let mut updates = state.updates.lock().unwrap_or_else(PoisonError::into_inner);+ updates+ .entry(collection_id.to_string())+ .or_default()+ .record_unreadable_index(now_ms());+ }+ {+ let mut runtimes = state+ .runtimes+ .lock()+ .unwrap_or_else(PoisonError::into_inner);+ let Some(runtime) = runtimes.get_mut(collection_id) else {+ return;+ };+ runtime.status = NotesCollectionStatusDto::Pending;+ runtime.index_available = false;+ runtime.indexing_progress = None;+ runtime.last_error = None;+ }+ emit_state_changed(app, collection_id);+}+impl RetrievalHandle {pub(crate) fn collection_label(&self, collection_id: &str) -> Option<String> {self.registry@@ -659,6 +730,7 @@ fn collection_dto(last_updated_at_ms: runtime.last_updated_at_ms,update_due_at_ms,last_error: runtime.last_error.clone(),+ watcher_error: runtime.watcher_error.clone(),}}@@ -751,13 +823,13 @@ fn register_collection(}notes_state.set_runtime(&collection.id, CollectionRuntime::pending());if let Err(error) = install_collection_watcher(app, &collection) {- notes_state.set_runtime(- &collection.id,- CollectionRuntime {- status: NotesCollectionStatusDto::Error,- last_error: Some(error.message),- ..CollectionRuntime::pending()- },+ notes_state.set_watcher_error(&collection.id, Some(&error));+ crate::logging::log(+ "Notes",+ format!(+ "watcher unavailable collection={} error={}",+ collection.id, error.message+ ),);}let dto = notes_state.collection_dto(&collection.id)?;@@ -906,16 +978,27 @@ pub async fn notes_index_collection(ensure_collection_watcher(&watcher_app, &watcher_collection)}).await- .map_err(|_| ApiError::new("watcher_thread", "Notes watcher task failed"))?;- if let Err(error) = watcher_result {- report_index_error(&app, ¬es_state, &collection_id, None, &error);- return Err(error);+ .unwrap_or_else(|_| Err(ApiError::new("watcher_thread", "Notes watcher task failed")));+ match watcher_result {+ Ok(()) => {+ notes_state.set_watcher_error(&collection_id, None);+ }+ Err(error) => {+ notes_state.set_watcher_error(&collection_id, Some(&error));+ crate::logging::log(+ "Notes",+ format!(+ "watcher unavailable collection={collection_id} error={}",+ error.message+ ),+ );+ }}let had_index = notes_state.has_available_index(&collection_id);let rebuild_derived_index =should_rebuild_derived_index(¬es_state.index_root, &collection_id, force, had_index)?;let mut update = notes_state.prepare_update(&collection_id, force, had_index)?;- update.rebuild_derived_index = rebuild_derived_index;+ update.rebuild_derived_index |= rebuild_derived_index;let cancellation_epoch = llm_state.retrieval_epoch();if let Err(error) = check_cancelled(&cancellation_epoch, retrieval_epoch) {crate::logging::log(@@ -1127,6 +1210,48 @@ fn available_state(app: &AppHandle) -> Result<TauriState<'_, State>, ApiError> {})}+fn acquire_notes_ownership(notes_directory: &Path) -> Result<File, ApiError> {+ let mut options = OpenOptions::new();+ options.read(true).write(true).create(true);+ #[cfg(unix)]+ options.mode(0o600);+ #[cfg(windows)]+ options.share_mode(0);++ let file = options+ .open(notes_directory.join(NOTES_OWNERSHIP_LOCK_FILE))+ .map_err(notes_ownership_open_error)?;++ #[cfg(unix)]+ {+ let result = unsafe { libc::flock(file.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) };+ if result != 0 {+ let error = std::io::Error::last_os_error();+ if error.kind() == std::io::ErrorKind::WouldBlock {+ return Err(notes_in_use_error());+ }+ return Err(io_error(error));+ }+ }++ Ok(file)+}++fn notes_ownership_open_error(error: std::io::Error) -> ApiError {+ #[cfg(windows)]+ if error.raw_os_error() == Some(ERROR_SHARING_VIOLATION as i32) {+ return notes_in_use_error();+ }+ io_error(error)+}++fn notes_in_use_error() -> ApiError {+ ApiError::new(+ "notes_in_use",+ "Your Notes is already open in another Ensu instance",+ )+}+fn ensure_directory(path: &Path) -> Result<(), ApiError> {if !path_exists(path)? {fs::create_dir_all(path).map_err(io_error)?;@@ -1229,6 +1354,59 @@ mod tests {writer.publish(true).unwrap();}+ #[test]+ fn ownership_lock_excludes_another_owner_until_release() {+ let temp = TestDirectory::new();+ let first = acquire_notes_ownership(temp.path()).unwrap();++ let error = acquire_notes_ownership(temp.path()).unwrap_err();+ assert_eq!(error.name, Some("notes_in_use"));+ assert_eq!(+ error.message,+ "Your Notes is already open in another Ensu instance"+ );++ drop(first);+ acquire_notes_ownership(temp.path()).unwrap();+ }++ #[test]+ fn watcher_warning_does_not_replace_ready_index_state() {+ let temp = TestDirectory::new();+ let collection = RegisteredCollection {+ id: COLLECTION_ID.to_string(),+ source_root: temp.path().to_path_buf(),+ };+ let runtime = CollectionRuntime {+ status: NotesCollectionStatusDto::Ready,+ index_available: true,+ indexing_progress: None,+ indexed_document_count: 1,+ last_updated_at_ms: Some(1),+ last_error: None,+ watcher_error: Some("Watcher unavailable".to_string()),+ };++ let dto = collection_dto(&collection, &runtime, None);++ assert_eq!(dto.status, NotesCollectionStatusDto::Ready);+ assert_eq!(dto.watcher_error.as_deref(), Some("Watcher unavailable"));+ assert_eq!(dto.last_error, None);+ }++ #[test]+ fn unreadable_index_requests_one_immediate_derived_rebuild() {+ let mut update = DesktopUpdateState::default();++ assert!(update.record_unreadable_index(42));+ assert!(update.rebuild_derived_index);+ assert_eq!(update.quiet_deadline_ms, 42);+ assert_eq!(update.generation, 1);+ assert!(!update.record_unreadable_index(84));+ assert_eq!(update.quiet_deadline_ms, 42);+ assert_eq!(update.generation, 1);+ }+#[test]fn retrieval_cache_evicts_the_least_recently_used_index_at_its_budget() {let temp = TestDirectory::new();diff --git a/rust/apps/ensu/src-tauri/src/commands/notes/watcher.rs b/rust/apps/ensu/src-tauri/src/commands/notes/watcher.rsindex d4559f368b..405cdab3f1 100644--- a/rust/apps/ensu/src-tauri/src/commands/notes/watcher.rs+++ b/rust/apps/ensu/src-tauri/src/commands/notes/watcher.rs@@ -63,6 +63,15 @@ pub async fn initialize_for_app(app: AppHandle) {.collections.clone();for collection in collections {+ let lifecycle = app.state::<State>().collection_lifecycle(&collection.id);+ let _collection_lifecycle = lifecycle.lock().await;+ if app+ .state::<State>()+ .registered_collection(&collection.id)+ .is_err()+ {+ continue;+ }let install_app = app.clone();let install_collection = collection.clone();let install_result = async_runtime::spawn_blocking(move || {@@ -73,18 +82,21 @@ pub async fn initialize_for_app(app: AppHandle) {Ok(result) => result,Err(_) => Err(ApiError::new("watcher_thread", "Notes watcher task failed")),};- if let Err(error) = install_result {- let state = app.state::<State>();- let mut runtime = state.runtime(&collection.id);- runtime.status = if source_root_is_available(&collection.source_root) {- NotesCollectionStatusDto::Error- } else {- NotesCollectionStatusDto::Unavailable- };- runtime.last_error = Some(error.message);- state.set_runtime(&collection.id, runtime);- emit_state_changed(&app, &collection.id);- continue;+ match install_result {+ Ok(()) => {+ app.state::<State>().set_watcher_error(&collection.id, None);+ }+ Err(error) => {+ let state = app.state::<State>();+ state.set_watcher_error(&collection.id, Some(&error));+ crate::logging::log(+ "Notes",+ format!(+ "watcher unavailable collection={} error={}",+ collection.id, error.message+ ),+ );+ }}inspect_collection_on_startup(app.clone(), collection).await;}@@ -92,8 +104,6 @@ pub async fn initialize_for_app(app: AppHandle) {async fn inspect_collection_on_startup(app: AppHandle, collection: RegisteredCollection) {let index_root = app.state::<State>().index_root.clone();- let lifecycle = app.state::<State>().collection_lifecycle(&collection.id);- let _collection_lifecycle = lifecycle.lock().await;if app.state::<State>().registered_collection(&collection.id)@@ -387,10 +397,19 @@ async fn process_watch_changes(app: AppHandle, collection_id: String, source_roochange};if change.refresh_watches {+ let lifecycle = app.state::<State>().collection_lifecycle(&collection_id);+ let _collection_lifecycle = lifecycle.lock().await;+ if app+ .state::<State>()+ .registered_collection(&collection_id)+ .is_err()+ {+ continue;+ }let refresh_app = app.clone();let refresh_collection_id = collection_id.clone();let refresh_source_root = source_root.clone();- let _ = async_runtime::spawn_blocking(move || {+ let refresh_result = async_runtime::spawn_blocking(move || {refresh_collection_watches(&refresh_app,&refresh_collection_id,@@ -399,6 +418,36 @@ async fn process_watch_changes(app: AppHandle, collection_id: String, source_roo)}).await;+ let refresh_result = match refresh_result {+ Ok(Ok(true)) => Ok(()),+ Ok(Ok(false)) => Err(ApiError::new(+ "watcher",+ "The Notes folder could not be watched",+ )),+ Ok(Err(error)) => Err(error),+ Err(_) => Err(ApiError::new("watcher_thread", "Notes watcher task failed")),+ };+ let state = app.state::<State>();+ match refresh_result {+ Ok(()) => {+ if state.set_watcher_error(&collection_id, None) {+ emit_state_changed(&app, &collection_id);+ }+ }+ Err(error) => {+ let changed = state.set_watcher_error(&collection_id, Some(&error));+ crate::logging::log(+ "Notes",+ format!(+ "watcher refresh failed collection={collection_id} error={}",+ error.message+ ),+ );+ if changed {+ emit_state_changed(&app, &collection_id);+ }+ }+ }}mark_collection_dirty(&app,diff --git a/rust/apps/ensu/src-tauri/src/main.rs b/rust/apps/ensu/src-tauri/src/main.rsindex 4942a7be87..ce750b2646 100644--- a/rust/apps/ensu/src-tauri/src/main.rs+++ b/rust/apps/ensu/src-tauri/src/main.rs@@ -10,12 +10,6 @@ fn main() {logging::log("App", "starting Tauri backend");let app = tauri::Builder::default()- .plugin(tauri_plugin_single_instance::init(|app, _args, _cwd| {- if let Some(window) = app.get_webview_window("main") {- let _ = window.show();- let _ = window.set_focus();- }- })).plugin(tauri_plugin_dialog::init()).plugin(tauri_plugin_fs::init()).plugin(tauri_plugin_opener::init())diff --git a/rust/crates/ensu-crypto/src/lib.rs b/rust/crates/ensu-crypto/src/lib.rsindex 3d2f6b3662..87bbb4fbbc 100644--- a/rust/crates/ensu-crypto/src/lib.rs+++ b/rust/crates/ensu-crypto/src/lib.rs@@ -3,7 +3,7 @@ use ente_core::{crypto::{self, Header, Key, blob},};use hkdf::Hkdf;-use sha2::Sha256;+use sha2::{Digest, Sha256};use thiserror::Error;const ATTACHMENT_KEY_INFO: &[u8] = b"llmchat_attachment_v1";@@ -33,6 +33,10 @@ pub struct EncryptedChatPayload {pub header: String,}+pub fn sha256(bytes: &[u8]) -> [u8; 32] {+ Sha256::digest(bytes).into()+}+pub fn generate_chat_key() -> String {b64::encode(Key::generate().as_bytes())}diff --git a/rust/crates/ensu/Cargo.toml b/rust/crates/ensu/Cargo.tomlindex 16c4a1fbb1..beb23708bd 100644--- a/rust/crates/ensu/Cargo.toml+++ b/rust/crates/ensu/Cargo.toml@@ -14,7 +14,6 @@ rusqlite = { version = "0.38", features = ["bundled"], optional = true }self_cell = "1"serde.workspace = trueserde_json.workspace = true-sha2.workspace = truethiserror.workspace = truetokio = { version = "1", features = ["rt", "time"] }transcribe-rs = { git = "https://github.com/cjpais/transcribe-rs.git", rev = "efc66111d519a434e28d51d94740008a7ce29360", default-features = false, features = ["onnx", "vad-silero"], optional = true }diff --git a/rust/crates/ensu/src/notes.rs b/rust/crates/ensu/src/notes.rsindex 1dfc10df0d..448b6e985c 100644--- a/rust/crates/ensu/src/notes.rs+++ b/rust/crates/ensu/src/notes.rs@@ -8,7 +8,11 @@ mod shard;mod source;mod writer;-use sha2::{Digest, Sha256};+use std::collections::{HashMap, HashSet};+use std::fs;+use std::path::{Path, PathBuf};+use std::sync::{Mutex, OnceLock, PoisonError};+use thiserror::Error;use uuid::Uuid;@@ -49,16 +53,154 @@ pub use indexing::{};pub use reconcile::NotesReconciliationPlan;pub use source::NoteSourceReference;-pub use writer::NotesIndexWriter;+pub use writer::{NotesIndexWriter, ValidatedNotesDocument};pub fn notes_content_revision(bytes: &[u8]) -> String {sha256_hex(bytes)}+type PinnedRevision = ([u8; 32], [u8; 32]);++fn pinned_revision_key(document_storage_id: &str, revision: &str) -> PinnedRevision {+ (+ ente_ensu_crypto::sha256(document_storage_id.as_bytes()),+ ente_ensu_crypto::sha256(revision.as_bytes()),+ )+}++#[derive(Default)]+struct CollectionReaders {+ opening: usize,+ revisions: HashMap<PinnedRevision, usize>,+}++static ACTIVE_COLLECTION_READERS: OnceLock<Mutex<HashMap<PathBuf, CollectionReaders>>> =+ OnceLock::new();++struct NotesCollectionReadPin {+ collection_directory: PathBuf,+ opening: bool,+ revisions: Vec<PinnedRevision>,+}++impl NotesCollectionReadPin {+ fn acquire(collection_directory: &Path) -> Result<Self, NotesError> {+ let collection_directory = fs::canonicalize(collection_directory)?;+ let mut readers = active_collection_readers()+ .lock()+ .unwrap_or_else(PoisonError::into_inner);+ let collection = readers.entry(collection_directory.clone()).or_default();+ collection.opening = collection.opening.checked_add(1).ok_or_else(|| {+ NotesError::InvalidIndex("active Notes reader count overflow".to_string())+ })?;+ Ok(Self {+ collection_directory,+ opening: true,+ revisions: Vec::new(),+ })+ }++ fn collection_directory(&self) -> &Path {+ &self.collection_directory+ }++ fn estimated_heap_bytes(&self) -> usize {+ let pinned_revision_bytes = std::mem::size_of::<PinnedRevision>();+ self.collection_directory+ .as_os_str()+ .len()+ .saturating_add(+ self.revisions+ .capacity()+ .saturating_mul(pinned_revision_bytes),+ )+ .saturating_add(+ self.revisions.len().saturating_mul(+ pinned_revision_bytes+ .saturating_add(std::mem::size_of::<usize>())+ .saturating_add(1),+ ),+ )+ }++ fn pin_revisions(+ &mut self,+ revisions: impl IntoIterator<Item = PinnedRevision>,+ ) -> Result<(), NotesError> {+ let revisions = revisions.into_iter().collect::<HashSet<_>>();+ let mut readers = active_collection_readers()+ .lock()+ .unwrap_or_else(PoisonError::into_inner);+ let collection = readers.get_mut(&self.collection_directory).ok_or_else(|| {+ NotesError::InvalidIndex("active Notes reader registration is missing".to_string())+ })?;+ collection.opening = collection.opening.checked_sub(1).ok_or_else(|| {+ NotesError::InvalidIndex("active Notes reader registration is inconsistent".to_string())+ })?;+ self.opening = false;+ self.revisions.reserve(revisions.len());+ for revision in revisions {+ let reader_count = collection.revisions.entry(revision).or_default();+ *reader_count = reader_count.checked_add(1).ok_or_else(|| {+ NotesError::InvalidIndex("active Notes revision reader count overflow".to_string())+ })?;+ self.revisions.push(revision);+ }+ Ok(())+ }+}++impl Drop for NotesCollectionReadPin {+ fn drop(&mut self) {+ let mut readers = active_collection_readers()+ .lock()+ .unwrap_or_else(PoisonError::into_inner);+ let Some(collection) = readers.get_mut(&self.collection_directory) else {+ return;+ };+ if self.opening {+ collection.opening = collection.opening.saturating_sub(1);+ } else {+ for revision in &self.revisions {+ let remove = collection+ .revisions+ .get_mut(revision)+ .is_some_and(|reader_count| {+ *reader_count = reader_count.saturating_sub(1);+ *reader_count == 0+ });+ if remove {+ collection.revisions.remove(revision);+ }+ }+ }+ if collection.opening == 0 && collection.revisions.is_empty() {+ readers.remove(&self.collection_directory);+ }+ }+}++fn active_collection_readers() -> &'static Mutex<HashMap<PathBuf, CollectionReaders>> {+ ACTIVE_COLLECTION_READERS.get_or_init(|| Mutex::new(HashMap::new()))+}++fn collection_reader_snapshot(collection_directory: &Path) -> (bool, HashSet<PinnedRevision>) {+ let readers = active_collection_readers()+ .lock()+ .unwrap_or_else(PoisonError::into_inner);+ let Some(collection) = readers.get(collection_directory) else {+ return (false, HashSet::new());+ };+ (+ collection.opening > 0,+ collection.revisions.keys().cloned().collect(),+ )+}+fn sha256_hex(bytes: &[u8]) -> String {const HEX: &[u8; 16] = b"0123456789abcdef";let mut encoded = String::with_capacity(64);- for byte in Sha256::digest(bytes) {+ for byte in ente_ensu_crypto::sha256(bytes) {encoded.push(char::from(HEX[usize::from(byte >> 4)]));encoded.push(char::from(HEX[usize::from(byte & 0x0f)]));}diff --git a/rust/crates/ensu/src/notes/index.rs b/rust/crates/ensu/src/notes/index.rsindex 5316c9ef65..ae01dcb2a0 100644--- a/rust/crates/ensu/src/notes/index.rs+++ b/rust/crates/ensu/src/notes/index.rs@@ -1,9 +1,11 @@-use std::collections::BTreeMap;+use std::collections::{BTreeMap, btree_map::Entry};use std::path::Path;-use super::manifest::{collection_directory, load_published_manifest, shard_directory};-use super::shard::{NotesShardIntegrity, load_and_validate_shard};-use super::{NotesError, validate_collection_id};+use super::manifest::{+ collection_directory, document_storage_id, load_published_manifest, shard_directory,+};+use super::shard::{NotesShard, load_and_validate_shard_metadata, load_and_validate_vectors};+use super::{NotesCollectionReadPin, NotesError, pinned_revision_key, validate_collection_id};const RELEVANCE_THRESHOLD: f32 = 0.50;const MAX_HITS_PER_COLLECTION: usize = 5;@@ -20,22 +22,29 @@ pub struct NotesSearchHit {pub text: String,}-struct IndexedChunk {+struct IndexedDocument {document_id: String,revision: String,- title: String,- section: Option<String>,- text: String,+ chunk_count: usize,+ vector_offset: usize,+ shard_sha256: String,+}++#[derive(Clone, Copy)]+struct RankedChunk {+ score: f32,+ document_index: usize,+ chunk_index: usize,vector_offset: usize,}pub struct NotesCollectionIndex {+ read_pin: NotesCollectionReadPin,collection_id: String,- document_count: usize,last_updated_at_ms: Option<i64>,dimension: usize,scale: f32,- chunks: Vec<IndexedChunk>,+ documents: Vec<IndexedDocument>,vectors: Vec<u8>,}@@ -62,8 +71,13 @@ impl NotesCollectionIndex {pub fn open(index_root: impl AsRef<Path>, collection_id: String) -> Result<Self, NotesError> {validate_collection_id(&collection_id)?;let collection_directory = collection_directory(index_root.as_ref(), &collection_id);+ let mut read_pin = NotesCollectionReadPin::acquire(&collection_directory)?;+ let collection_directory = read_pin.collection_directory().to_path_buf();let (manifest, last_updated_at_ms) =load_published_manifest(&collection_directory, &collection_id, true)?;+ read_pin.pin_revisions(manifest.documents.iter().map(|(document_id, document)| {+ pinned_revision_key(&document_storage_id(document_id), &document.revision)+ }))?;let dimension =usize::try_from(manifest.index_contract.retrieval_dimension).map_err(|_| {NotesError::InvalidIndex("retrieval dimension is too large".to_string())@@ -83,65 +97,50 @@ impl NotesCollectionIndex {let vector_bytes = total_chunks.checked_mul(dimension).ok_or_else(|| {NotesError::InvalidIndex("collection vector byte length overflow".to_string())})?;- let document_count = manifest.documents.len();- let mut chunks = Vec::new();- chunks.try_reserve_exact(total_chunks).map_err(|_| {- NotesError::InvalidIndex("collection chunk allocation is too large".to_string())- })?;+ let mut documents = Vec::new();+ documents+ .try_reserve_exact(manifest.documents.len())+ .map_err(|_| {+ NotesError::InvalidIndex("collection document allocation is too large".to_string())+ })?;let mut vectors = Vec::new();vectors.try_reserve_exact(vector_bytes).map_err(|_| {NotesError::InvalidIndex("collection vector allocation is too large".to_string())})?;for (document_id, document) in &manifest.documents {- let loaded = load_and_validate_shard(- &shard_directory(&collection_directory, document_id, &document.revision),- &collection_id,- document_id,- &document.revision,- document.chunk_count as usize,- NotesShardIntegrity {- shard_sha256: &document.shard_sha256,- vectors_sha256: &document.vectors_sha256,- },- None,- )?;- let title = loaded.shard.title;- for (chunk_index, chunk) in loaded.shard.chunks.into_iter().enumerate() {- let start = chunk_index.checked_mul(dimension).ok_or_else(|| {- NotesError::InvalidIndex("shard vector offset overflow".to_string())- })?;- let end = start.checked_add(dimension).ok_or_else(|| {- NotesError::InvalidIndex("shard vector offset overflow".to_string())- })?;- chunks.push(IndexedChunk {- document_id: document_id.clone(),- revision: document.revision.clone(),- title: title.clone(),- section: chunk.section,- text: chunk.text,- vector_offset: vectors.len(),- });- vectors.extend_from_slice(&loaded.vectors[start..end]);- }+ let chunk_count = usize::try_from(document.chunk_count).map_err(|_| {+ NotesError::InvalidIndex("manifest chunk count is too large".to_string())+ })?;+ let directory = shard_directory(&collection_directory, document_id, &document.revision);+ let loaded_vectors =+ load_and_validate_vectors(&directory, chunk_count, &document.vectors_sha256)?;+ documents.push(IndexedDocument {+ document_id: document_id.clone(),+ revision: document.revision.clone(),+ chunk_count,+ vector_offset: vectors.len(),+ shard_sha256: document.shard_sha256.clone(),+ });+ vectors.extend_from_slice(&loaded_vectors);}- if chunks.len() != total_chunks || vectors.len() != vector_bytes {+ if documents.len() != manifest.documents.len() || vectors.len() != vector_bytes {return Err(NotesError::InvalidIndex("loaded collection shape does not match its manifest".to_string(),));}Ok(Self {+ read_pin,collection_id,- document_count,last_updated_at_ms,dimension,scale,- chunks,+ documents,vectors,})}pub fn document_count(&self) -> usize {- self.document_count+ self.documents.len()}pub fn last_updated_at_ms(&self) -> Option<i64> {@@ -151,74 +150,140 @@ impl NotesCollectionIndex {pub fn estimated_heap_bytes(&self) -> usize {self.collection_id.capacity()+ .saturating_add(self.read_pin.estimated_heap_bytes()).saturating_add(- self.chunks+ self.documents.capacity()- .saturating_mul(std::mem::size_of::<IndexedChunk>()),+ .saturating_mul(std::mem::size_of::<IndexedDocument>()),).saturating_add(self.vectors.capacity())- .saturating_add(self.chunks.iter().fold(0_usize, |total, chunk| {+ .saturating_add(self.documents.iter().fold(0_usize, |total, document| {total- .saturating_add(chunk.document_id.capacity())- .saturating_add(chunk.revision.capacity())- .saturating_add(chunk.title.capacity())- .saturating_add(chunk.section.as_ref().map_or(0, String::capacity))- .saturating_add(chunk.text.capacity())+ .saturating_add(document.document_id.capacity())+ .saturating_add(document.revision.capacity())+ .saturating_add(document.shard_sha256.capacity())}))}pub fn search(&self, query: &[f32]) -> Result<Vec<NotesSearchHit>, NotesError> {validate_search(query, self.dimension)?;- let mut ranked = self- .chunks- .iter()- .map(|chunk| {- let vector =- &self.vectors[chunk.vector_offset..chunk.vector_offset + self.dimension];+ let mut ranked = Vec::with_capacity(MAX_HITS_PER_COLLECTION);+ for (document_index, document) in self.documents.iter().enumerate() {+ let mut document_ranked = Vec::with_capacity(MAX_HITS_PER_DOCUMENT);+ for chunk_index in 0..document.chunk_count {+ let vector_offset = document+ .vector_offset+ .checked_add(chunk_index.checked_mul(self.dimension).ok_or_else(|| {+ NotesError::InvalidIndex("shard vector offset overflow".to_string())+ })?)+ .ok_or_else(|| {+ NotesError::InvalidIndex("shard vector offset overflow".to_string())+ })?;+ let vector_end = vector_offset.checked_add(self.dimension).ok_or_else(|| {+ NotesError::InvalidIndex("shard vector offset overflow".to_string())+ })?;+ let vector = self.vectors.get(vector_offset..vector_end).ok_or_else(|| {+ NotesError::InvalidIndex(+ "shard vector range exceeds the collection".to_string(),+ )+ })?;let score = query.iter().zip(vector).map(|(query, stored)| *query * f32::from(*stored as i8)).sum::<f32>()/ self.scale;- (score, chunk)- })- .filter(|(score, _)| score.is_finite() && *score >= RELEVANCE_THRESHOLD)- .collect::<Vec<_>>();- ranked.sort_unstable_by(|(left_score, left), (right_score, right)| {- right_score- .total_cmp(left_score)- .then_with(|| left.document_id.cmp(&right.document_id))- .then_with(|| left.vector_offset.cmp(&right.vector_offset))- });-- let mut document_counts = BTreeMap::<&str, usize>::new();- let mut hits = Vec::with_capacity(MAX_HITS_PER_COLLECTION.min(ranked.len()));- for (score, chunk) in ranked {- let count = document_counts- .entry(chunk.document_id.as_str())- .or_default();- if *count >= MAX_HITS_PER_DOCUMENT {- continue;+ if score.is_finite() && score >= RELEVANCE_THRESHOLD {+ retain_best(+ &mut document_ranked,+ RankedChunk {+ score,+ document_index,+ chunk_index,+ vector_offset,+ },+ MAX_HITS_PER_DOCUMENT,+ &self.documents,+ );+ }+ }+ for candidate in document_ranked {+ retain_best(+ &mut ranked,+ candidate,+ MAX_HITS_PER_COLLECTION,+ &self.documents,+ );}- *count += 1;+ }++ let mut hits = Vec::with_capacity(MAX_HITS_PER_COLLECTION.min(ranked.len()));+ let mut loaded_shards = BTreeMap::<usize, NotesShard>::new();+ for candidate in ranked {+ let document = &self.documents[candidate.document_index];+ let shard = match loaded_shards.entry(candidate.document_index) {+ Entry::Occupied(entry) => entry.into_mut(),+ Entry::Vacant(entry) => {+ let directory = shard_directory(+ self.read_pin.collection_directory(),+ &document.document_id,+ &document.revision,+ );+ entry.insert(load_and_validate_shard_metadata(+ &directory,+ &self.collection_id,+ &document.document_id,+ &document.revision,+ document.chunk_count,+ &document.shard_sha256,+ None,+ )?)+ }+ };+ let chunk = shard.chunks.get(candidate.chunk_index).ok_or_else(|| {+ NotesError::InvalidIndex("selected shard chunk is missing".to_string())+ })?;hits.push(NotesSearchHit {collection_id: self.collection_id.clone(),- document_id: chunk.document_id.clone(),- revision: chunk.revision.clone(),- score,- title: chunk.title.clone(),+ document_id: document.document_id.clone(),+ revision: document.revision.clone(),+ score: candidate.score,+ title: shard.title.clone(),section: chunk.section.clone(),text: chunk.text.clone(),});- if hits.len() == MAX_HITS_PER_COLLECTION {- break;- }}Ok(hits)}}+fn retain_best(+ ranked: &mut Vec<RankedChunk>,+ candidate: RankedChunk,+ limit: usize,+ documents: &[IndexedDocument],+) {+ ranked.push(candidate);+ ranked.sort_unstable_by(|left, right| ranked_order(left, right, documents));+ ranked.truncate(limit);+}++fn ranked_order(+ left: &RankedChunk,+ right: &RankedChunk,+ documents: &[IndexedDocument],+) -> std::cmp::Ordering {+ right+ .score+ .total_cmp(&left.score)+ .then_with(|| {+ documents[left.document_index]+ .document_id+ .cmp(&documents[right.document_index].document_id)+ })+ .then_with(|| left.vector_offset.cmp(&right.vector_offset))+}+fn validate_search(query: &[f32], dimension: usize) -> Result<(), NotesError> {if query.len() != dimension {return Err(NotesError::InvalidInput(format!(@@ -233,3 +298,177 @@ fn validate_search(query: &[f32], dimension: usize) -> Result<(), NotesError> {}Ok(())}++#[cfg(test)]+mod tests {+ use std::fs;++ use super::*;+ use crate::notes::shard::{NOTES_SHARD_FILE, NOTES_VECTORS_FILE};+ use crate::notes::{+ NotesIndexWriter, NotesSourceDocument, PreparedNotesChunk, PreparedNotesDocument,+ notes_content_revision,+ };++ const COLLECTION_ID: &str = "123e4567-e89b-12d3-a456-426614174000";++ fn document(document_id: &str, chunk_count: usize) -> PreparedNotesDocument {+ PreparedNotesDocument {+ document_id: document_id.to_string(),+ revision: notes_content_revision(document_id.as_bytes()),+ title: document_id.to_string(),+ chunks: (0..chunk_count)+ .map(|index| PreparedNotesChunk {+ section: None,+ text: format!("{document_id} chunk {index}"),+ })+ .collect(),+ }+ }++ fn source(document: &PreparedNotesDocument) -> NotesSourceDocument {+ NotesSourceDocument {+ document_id: document.document_id.clone(),+ size: document+ .chunks+ .iter()+ .map(|chunk| chunk.text.len() as u64)+ .sum(),+ modified_at_ms: Some(1),+ }+ }++ fn embedding(axis: usize, score: f32) -> Vec<f32> {+ let mut embedding = vec![0.0; 512];+ embedding[axis] = score;+ embedding+ }++ #[test]+ fn search_loads_metadata_only_for_selected_documents() {+ let temp = tempfile::tempdir().unwrap();+ let first = document("first.md", 1);+ let second = document("second.md", 1);+ let mut writer = NotesIndexWriter::open(temp.path(), COLLECTION_ID.to_string()).unwrap();+ writer+ .commit_document(&first, &[embedding(0, 1.0)], &source(&first))+ .unwrap();+ writer+ .commit_document(&second, &[embedding(1, 1.0)], &source(&second))+ .unwrap();+ writer.publish(true).unwrap();++ let second_shard = shard_directory(+ &collection_directory(temp.path(), COLLECTION_ID),+ &second.document_id,+ &second.revision,+ )+ .join(NOTES_SHARD_FILE);+ fs::write(second_shard, b"corrupt").unwrap();++ let index = NotesCollectionIndex::open(temp.path(), COLLECTION_ID.to_string()).unwrap();+ let hits = index.search(&embedding(0, 1.0)).unwrap();+ assert_eq!(hits.len(), 1);+ assert_eq!(hits[0].document_id, "first.md");+ assert!(matches!(+ index.search(&embedding(1, 1.0)),+ Err(NotesError::InvalidIndex(_) | NotesError::Json(_))+ ));+ }++ #[test]+ fn open_index_keeps_its_shards_alive_until_it_is_dropped() {+ let temp = tempfile::tempdir().unwrap();+ let original = document("note.md", 1);+ let original_directory = shard_directory(+ &collection_directory(temp.path(), COLLECTION_ID),+ &original.document_id,+ &original.revision,+ );+ let mut writer = NotesIndexWriter::open(temp.path(), COLLECTION_ID.to_string()).unwrap();+ writer+ .commit_document(&original, &[embedding(0, 1.0)], &source(&original))+ .unwrap();+ writer.publish(true).unwrap();+ let index = NotesCollectionIndex::open(temp.path(), COLLECTION_ID.to_string()).unwrap();++ let mut replacement = document("note.md", 1);+ replacement.revision = notes_content_revision(b"replacement");+ replacement.chunks[0].text = "replacement chunk".to_string();+ writer+ .commit_document(&replacement, &[embedding(1, 1.0)], &source(&replacement))+ .unwrap();+ writer.publish(true).unwrap();++ assert!(original_directory.is_dir());+ let hits = index.search(&embedding(0, 1.0)).unwrap();+ assert_eq!(hits[0].text, original.chunks[0].text);++ drop(index);+ let replacement_index =+ NotesCollectionIndex::open(temp.path(), COLLECTION_ID.to_string()).unwrap();+ writer.publish(true).unwrap();+ assert!(!original_directory.exists());+ assert_eq!(+ replacement_index.search(&embedding(1, 1.0)).unwrap()[0].text,+ replacement.chunks[0].text+ );+ }++ #[test]+ fn failed_open_releases_its_collection_read_pin() {+ let temp = tempfile::tempdir().unwrap();+ let note = document("note.md", 1);+ let mut writer = NotesIndexWriter::open(temp.path(), COLLECTION_ID.to_string()).unwrap();+ writer+ .commit_document(¬e, &[embedding(0, 1.0)], &source(¬e))+ .unwrap();+ writer.publish(true).unwrap();+ let collection_directory = collection_directory(temp.path(), COLLECTION_ID);+ let vectors_path =+ shard_directory(&collection_directory, ¬e.document_id, ¬e.revision)+ .join(NOTES_VECTORS_FILE);+ fs::write(vectors_path, b"corrupt").unwrap();++ assert!(NotesCollectionIndex::open(temp.path(), COLLECTION_ID.to_string()).is_err());+ let collection_directory = fs::canonicalize(collection_directory).unwrap();+ let (opening, revisions) = crate::notes::collection_reader_snapshot(&collection_directory);+ assert!(!opening);+ assert!(revisions.is_empty());+ }++ #[test]+ fn search_bounds_per_document_and_global_ranking() {+ let temp = tempfile::tempdir().unwrap();+ let dominant = document("a.md", 5);+ let mut writer = NotesIndexWriter::open(temp.path(), COLLECTION_ID.to_string()).unwrap();+ writer+ .commit_document(+ &dominant,+ &[1.0, 0.99, 0.98, 0.97, 0.96]+ .into_iter()+ .map(|score| embedding(0, score))+ .collect::<Vec<_>>(),+ &source(&dominant),+ )+ .unwrap();+ for (document_id, score) in [("b.md", 0.95), ("c.md", 0.94), ("d.md", 0.93)] {+ let document = document(document_id, 1);+ writer+ .commit_document(&document, &[embedding(0, score)], &source(&document))+ .unwrap();+ }+ writer.publish(true).unwrap();++ let index = NotesCollectionIndex::open(temp.path(), COLLECTION_ID.to_string()).unwrap();+ let hits = index.search(&embedding(0, 1.0)).unwrap();++ assert_eq!(hits.len(), MAX_HITS_PER_COLLECTION);+ assert_eq!(+ hits.iter().filter(|hit| hit.document_id == "a.md").count(),+ MAX_HITS_PER_DOCUMENT+ );+ assert_eq!(hits[3].document_id, "b.md");+ assert_eq!(hits[4].document_id, "c.md");+ }+}diff --git a/rust/crates/ensu/src/notes/indexing.rs b/rust/crates/ensu/src/notes/indexing.rsindex 0b29d3cc04..9745b02f18 100644--- a/rust/crates/ensu/src/notes/indexing.rs+++ b/rust/crates/ensu/src/notes/indexing.rs@@ -215,8 +215,8 @@ pub fn index_notes_collection<E>();continue;};- match writer.validate_document(&prepared, &source_metadata) {- Ok(()) => {}+ let validated = match writer.validate_document(&prepared, &source_metadata) {+ Ok(validated) => validated,Err(NotesError::InvalidInput(_)) => {writer.commit_deletions(std::slice::from_ref(document_id))?;processed_requested += 1;@@ -230,7 +230,7 @@ pub fn index_notes_collection<E>(continue;}Err(error) => return Err(error.into()),- }+ };let Some(embeddings) = embed_document(&prepared).map_err(NotesIndexingError::Adapter)?else {writer.commit_deletions(std::slice::from_ref(document_id))?;@@ -267,7 +267,7 @@ pub fn index_notes_collection<E>(processed_requested,);}- writer.commit_document(&prepared, &embeddings, &verified_source)?;+ writer.commit_validated_document(validated, &embeddings)?;processed_requested += 1;progress_document_count += 1;report_indexing_progress(diff --git a/rust/crates/ensu/src/notes/shard.rs b/rust/crates/ensu/src/notes/shard.rsindex 26cc06730a..700bbf5154 100644--- a/rust/crates/ensu/src/notes/shard.rs+++ b/rust/crates/ensu/src/notes/shard.rs@@ -29,11 +29,6 @@ pub(super) struct NotesShardChunk {pub text: String,}-pub(super) struct ValidatedNotesShard {- pub shard: NotesShard,- pub vectors: Vec<u8>,-}-#[derive(Clone, Copy)]pub(super) struct NotesShardIntegrity<'a> {pub shard_sha256: &'a str,@@ -77,15 +72,35 @@ pub(super) fn load_and_validate_shard(expected_chunk_count: usize,expected_integrity: NotesShardIntegrity<'_>,expected_prepared: Option<&PreparedNotesDocument>,-) -> Result<ValidatedNotesShard, NotesError> {- validate_revision(expected_integrity.shard_sha256).map_err(invalid_index_input)?;- validate_revision(expected_integrity.vectors_sha256).map_err(invalid_index_input)?;- let directory_metadata = fs::symlink_metadata(directory)?;- if !directory_metadata.file_type().is_dir() {- return Err(NotesError::InvalidIndex(- "shard path must be a directory".to_string(),- ));- }+) -> Result<(), NotesError> {+ let shard = load_and_validate_shard_metadata(+ directory,+ collection_id,+ expected_document_id,+ expected_revision,+ expected_chunk_count,+ expected_integrity.shard_sha256,+ expected_prepared,+ )?;+ load_and_validate_vectors(+ directory,+ shard.chunks.len(),+ expected_integrity.vectors_sha256,+ )?;+ Ok(())+}++pub(super) fn load_and_validate_shard_metadata(+ directory: &Path,+ collection_id: &str,+ expected_document_id: &str,+ expected_revision: &str,+ expected_chunk_count: usize,+ expected_shard_sha256: &str,+ expected_prepared: Option<&PreparedNotesDocument>,+) -> Result<NotesShard, NotesError> {+ validate_revision(expected_shard_sha256).map_err(invalid_index_input)?;+ validate_shard_directory(directory)?;let shard_path = directory.join(NOTES_SHARD_FILE);let shard_metadata = regular_file_metadata(&shard_path, "shard metadata")?;@@ -96,7 +111,7 @@ pub(super) fn load_and_validate_shard(}let shard_bytes = fs::read(shard_path)?;if shard_bytes.len() as u64 != shard_metadata.len()- || sha256_hex(&shard_bytes) != expected_integrity.shard_sha256+ || sha256_hex(&shard_bytes) != expected_shard_sha256{return Err(NotesError::InvalidIndex("shard metadata digest does not match its manifest".to_string(),@@ -111,14 +126,21 @@ pub(super) fn load_and_validate_shard(expected_chunk_count,expected_prepared,)?;+ Ok(shard)+}+pub(super) fn load_and_validate_vectors(+ directory: &Path,+ expected_chunk_count: usize,+ expected_vectors_sha256: &str,+) -> Result<Vec<u8>, NotesError> {+ validate_revision(expected_vectors_sha256).map_err(invalid_index_input)?;+ validate_shard_directory(directory)?;let vectors_path = directory.join(NOTES_VECTORS_FILE);let vector_metadata = regular_file_metadata(&vectors_path, "shard vectors")?;let vector_dimension = usize::try_from(notes_index_contract().retrieval_dimension).map_err(|_| NotesError::InvalidIndex("vector dimension is too large".to_string()))?;- let expected_vector_bytes = shard- .chunks- .len()+ let expected_vector_bytes = expected_chunk_count.checked_mul(vector_dimension).ok_or_else(|| NotesError::InvalidIndex("shard vector byte length overflow".to_string()))?;let vectors = fs::read(vectors_path)?;@@ -129,12 +151,22 @@ pub(super) fn load_and_validate_shard("shard vector byte length does not match its metadata".to_string(),));}- if sha256_hex(&vectors) != expected_integrity.vectors_sha256 {+ if sha256_hex(&vectors) != expected_vectors_sha256 {return Err(NotesError::InvalidIndex("shard vector digest does not match its manifest".to_string(),));}- Ok(ValidatedNotesShard { shard, vectors })+ Ok(vectors)+}++fn validate_shard_directory(directory: &Path) -> Result<(), NotesError> {+ let directory_metadata = fs::symlink_metadata(directory)?;+ if !directory_metadata.file_type().is_dir() {+ return Err(NotesError::InvalidIndex(+ "shard path must be a directory".to_string(),+ ));+ }+ Ok(())}fn validate_shard(diff --git a/rust/crates/ensu/src/notes/writer.rs b/rust/crates/ensu/src/notes/writer.rsindex 45e1eb2be8..ffd57c7ef0 100644--- a/rust/crates/ensu/src/notes/writer.rs+++ b/rust/crates/ensu/src/notes/writer.rs@@ -20,14 +20,25 @@ use super::shard::{use super::{NOTES_CHUNK_UTF8_BYTES, NOTES_MAX_COLLECTION_CHUNKS, NOTES_MAX_COLLECTION_DOCUMENTS,NOTES_MAX_COLLECTION_SOURCE_BYTES, NotesError, NotesReconciliationPlan, NotesSourceDocument,- PreparedNotesDocument, contains_unsupported_control, is_valid_label, sha256_hex,- validate_collection_id, validate_document_id, validate_revision,+ PreparedNotesDocument, collection_reader_snapshot, contains_unsupported_control,+ is_valid_label, pinned_revision_key, sha256_hex, validate_collection_id, validate_document_id,+ validate_revision,};pub struct NotesIndexWriter {collection_id: String,collection_directory: PathBuf,manifest: NotesManifest,+ source_bytes: u64,+ chunk_count: u64,+}++pub struct ValidatedNotesDocument<'a> {+ collection_id: String,+ prepared: &'a PreparedNotesDocument,+ source_metadata: &'a NotesSourceDocument,+ shard_bytes: Vec<u8>,+ shard_sha256: String,}impl NotesIndexWriter {@@ -121,6 +132,7 @@ impl NotesIndexWriter {ensure_directory(index_root.as_ref())?;let collection_directory = collection_directory(index_root.as_ref(), &collection_id);ensure_directory(&collection_directory)?;+ let collection_directory = fs::canonicalize(collection_directory)?;ensure_directory(&collection_directory.join(NOTES_DOCUMENTS_DIRECTORY))?;recover_manifest_publication(&collection_directory, &collection_id)?;@@ -130,11 +142,14 @@ impl NotesIndexWriter {} else {empty_manifest(&collection_id)};+ let (source_bytes, chunk_count) = manifest_capacity(&manifest)?;let _ = cleanup_unreferenced_shards(&collection_directory, &manifest.documents);Ok(Self {collection_id,collection_directory,manifest,+ source_bytes,+ chunk_count,})}@@ -183,7 +198,29 @@ impl NotesIndexWriter {embeddings: &[Vec<f32>],source_metadata: &NotesSourceDocument,) -> Result<(), NotesError> {- self.validate_document(prepared, source_metadata)?;+ let validated = self.validate_document(prepared, source_metadata)?;+ self.commit_validated_document(validated, embeddings)+ }++ pub fn commit_validated_document(+ &mut self,+ validated: ValidatedNotesDocument<'_>,+ embeddings: &[Vec<f32>],+ ) -> Result<(), NotesError> {+ let ValidatedNotesDocument {+ collection_id,+ prepared,+ source_metadata,+ shard_bytes,+ shard_sha256,+ } = validated;+ if collection_id != self.collection_id {+ return Err(NotesError::InvalidInput(+ "validated document belongs to another collection".to_string(),+ ));+ }+ validate_prepared_document(prepared, source_metadata)?;+ self.validate_document_capacity(prepared, source_metadata)?;if embeddings.len() != prepared.chunks.len() {return Err(NotesError::InvalidInput("embedding count does not match prepared chunks".to_string(),@@ -214,9 +251,6 @@ impl NotesIndexWriter {.clamp(-scale, scale) as i8 as u8}));}- let shard = NotesShard::from_prepared(&self.collection_id, prepared);- let shard_bytes = serialize_shard(&shard)?;- let shard_sha256 = sha256_hex(&shard_bytes);let vectors_sha256 = sha256_hex(&vector_bytes);let final_directory = shard_directory(@@ -307,46 +341,69 @@ impl NotesIndexWriter {self.activate_document(prepared, source_metadata, &shard_sha256, &vectors_sha256)}- pub fn validate_document(+ pub fn validate_document<'a>(+ &self,+ prepared: &'a PreparedNotesDocument,+ source_metadata: &'a NotesSourceDocument,+ ) -> Result<ValidatedNotesDocument<'a>, NotesError> {+ validate_prepared_document(prepared, source_metadata)?;+ self.validate_document_capacity(prepared, source_metadata)?;+ let shard_bytes =+ serialize_shard(&NotesShard::from_prepared(&self.collection_id, prepared))?;+ let shard_sha256 = sha256_hex(&shard_bytes);+ Ok(ValidatedNotesDocument {+ collection_id: self.collection_id.clone(),+ prepared,+ source_metadata,+ shard_bytes,+ shard_sha256,+ })+ }++ fn validate_document_capacity(&self,prepared: &PreparedNotesDocument,source_metadata: &NotesSourceDocument,) -> Result<(), NotesError> {- validate_prepared_document(prepared, source_metadata)?;- let (existing_source_bytes, existing_chunks) = self- .manifest- .documents- .iter()- .filter(|(document_id, _)| *document_id != &prepared.document_id)- .try_fold(- (0_u64, 0_u64),- |(source_total, chunk_total), (_, document)| {- let source_total = source_total- .checked_add(document.source_size)- .ok_or_else(source_content_too_large)?;- let chunk_total = chunk_total- .checked_add(u64::from(document.chunk_count))- .ok_or_else(chunk_count_too_large)?;- Ok::<_, NotesError>((source_total, chunk_total))- },- )?;- let source_bytes = existing_source_bytes- .checked_add(source_metadata.size)+ let existing = self.manifest.documents.get(&prepared.document_id);+ let source_bytes = self+ .source_bytes+ .checked_sub(existing.map_or(0, |document| document.source_size))+ .and_then(|total| total.checked_add(source_metadata.size)).ok_or_else(source_content_too_large)?;- let chunk_count =+ let document_chunks =u64::try_from(prepared.chunks.len()).map_err(|_| chunk_count_too_large())?;- let chunks = existing_chunks- .checked_add(chunk_count)+ let chunks = self+ .chunk_count+ .checked_sub(existing.map_or(0, |document| u64::from(document.chunk_count)))+ .and_then(|total| total.checked_add(document_chunks)).ok_or_else(chunk_count_too_large)?;- ensure_collection_capacity(source_bytes, chunks)?;- serialize_shard(&NotesShard::from_prepared(&self.collection_id, prepared))?;- Ok(())+ ensure_collection_capacity(source_bytes, chunks)}pub fn commit_deletions(&mut self, document_ids: &[String]) -> Result<(), NotesError> {for document_id in document_ids {validate_document_id(document_id)?;+ let Some(document) = self.manifest.documents.get(document_id) else {+ continue;+ };+ let source_bytes = self+ .source_bytes+ .checked_sub(document.source_size)+ .ok_or_else(|| {+ NotesError::InvalidIndex(+ "manifest source byte total is inconsistent".to_string(),+ )+ })?;+ let chunk_count = self+ .chunk_count+ .checked_sub(u64::from(document.chunk_count))+ .ok_or_else(|| {+ NotesError::InvalidIndex("manifest chunk total is inconsistent".to_string())+ })?;self.manifest.documents.remove(document_id);+ self.source_bytes = source_bytes;+ self.chunk_count = chunk_count;}Ok(())}@@ -374,6 +431,18 @@ impl NotesIndexWriter {let chunk_count = u32::try_from(prepared.chunks.len()).map_err(|_| {NotesError::InvalidInput("prepared document has too many chunks".to_string())})?;+ let existing = self.manifest.documents.get(&prepared.document_id);+ let source_bytes = self+ .source_bytes+ .checked_sub(existing.map_or(0, |document| document.source_size))+ .and_then(|total| total.checked_add(source_metadata.size))+ .ok_or_else(source_content_too_large)?;+ let total_chunks = self+ .chunk_count+ .checked_sub(existing.map_or(0, |document| u64::from(document.chunk_count)))+ .and_then(|total| total.checked_add(u64::from(chunk_count)))+ .ok_or_else(chunk_count_too_large)?;+ ensure_collection_capacity(source_bytes, total_chunks)?;self.manifest.documents.insert(prepared.document_id.clone(),NotesManifestDocument {@@ -385,10 +454,32 @@ impl NotesIndexWriter {vectors_sha256: vectors_sha256.to_string(),},);+ self.source_bytes = source_bytes;+ self.chunk_count = total_chunks;Ok(())}}+fn manifest_capacity(manifest: &NotesManifest) -> Result<(u64, u64), NotesError> {+ let (source_bytes, chunks) = manifest.documents.values().try_fold(+ (0_u64, 0_u64),+ |(source_total, chunk_total), document| {+ let source_total = source_total+ .checked_add(document.source_size)+ .ok_or_else(|| {+ NotesError::InvalidIndex("manifest source byte count overflow".to_string())+ })?;+ let chunk_total = chunk_total+ .checked_add(u64::from(document.chunk_count))+ .ok_or_else(|| {+ NotesError::InvalidIndex("manifest chunk count overflow".to_string())+ })?;+ Ok::<_, NotesError>((source_total, chunk_total))+ },+ )?;+ Ok((source_bytes, chunks))+}+fn ensure_collection_capacity(source_bytes: u64, chunks: u64) -> Result<(), NotesError> {if source_bytes > NOTES_MAX_COLLECTION_SOURCE_BYTES {return Err(source_content_too_large());@@ -559,6 +650,10 @@ fn cleanup_unreferenced_shards() -> Result<(), NotesError> {use super::manifest::document_storage_id;+ let (reader_opening, pinned_revisions) = collection_reader_snapshot(collection_directory);+ if reader_opening {+ return Ok(());+ }let keep = documents.iter().map(|(document_id, document)| (document_storage_id(document_id), &document.revision))@@ -567,20 +662,28 @@ fn cleanup_unreferenced_shards(for document_entry in fs::read_dir(&documents_directory)? {let document_entry = document_entry?;let name = document_entry.file_name().to_string_lossy().into_owned();- let Some(active_revision) = keep.get(&name) else {- remove_owned_entry(&document_entry.path())?;- continue;- };if !document_entry.file_type()?.is_dir() {remove_owned_entry(&document_entry.path())?;continue;}for revision_entry in fs::read_dir(document_entry.path())? {let revision_entry = revision_entry?;- if revision_entry.file_name().to_string_lossy() != active_revision.as_str() {+ let revision = revision_entry.file_name().to_string_lossy().into_owned();+ let is_active = keep+ .get(&name)+ .is_some_and(|active_revision| active_revision.as_str() == revision);+ if !is_active && !pinned_revisions.contains(&pinned_revision_key(&name, &revision)) {remove_owned_entry(&revision_entry.path())?;}}+ let document_key = ente_ensu_crypto::sha256(name.as_bytes());+ if !keep.contains_key(&name)+ && !pinned_revisions+ .iter()+ .any(|(document, _)| document == &document_key)+ {+ fs::remove_dir(document_entry.path())?;+ }}Ok(())}@@ -691,6 +794,37 @@ mod tests {));}+ #[test]+ fn maintains_cached_capacity_totals_across_replacement_and_deletion() {+ let temp = tempfile::tempdir().unwrap();+ let first = prepared("first.md", "first");+ let second = prepared("second.md", "second");+ let mut writer = NotesIndexWriter::open(temp.path(), collection()).unwrap();+ writer+ .commit_document(&first, &embeddings(&first, &[1.0]), &metadata(&first, 5))+ .unwrap();+ writer+ .commit_document(&second, &embeddings(&second, &[0.8]), &metadata(&second, 6))+ .unwrap();+ assert_eq!(writer.source_bytes, 11);+ assert_eq!(writer.chunk_count, 2);++ let replacement = prepared("first.md", "replacement");+ writer+ .commit_document(+ &replacement,+ &embeddings(&replacement, &[0.9]),+ &metadata(&replacement, 11),+ )+ .unwrap();+ assert_eq!(writer.source_bytes, 17);+ assert_eq!(writer.chunk_count, 2);++ writer.commit_deletions(&[second.document_id]).unwrap();+ assert_eq!(writer.source_bytes, 11);+ assert_eq!(writer.chunk_count, 1);+ }+#[test]fn rejects_collections_over_capacity_before_embedding() {let inventory = (0..=NOTES_MAX_COLLECTION_DOCUMENTS)diff --git a/web/apps/ensu/package.json b/web/apps/ensu/package.jsonindex 9988398020..ddd4fd1561 100644--- a/web/apps/ensu/package.json+++ b/web/apps/ensu/package.json@@ -2,9 +2,6 @@"name": "ensu","version": "0.0.0","private": true,- "scripts": {- "test": "vitest --run"- },"dependencies": {"@hugeicons/core-free-icons": "1.2.1","@hugeicons/react": "1.1.6",@@ -27,7 +24,6 @@"remark-math": "6.0.0"},"devDependencies": {- "ente-build-config": "*",- "vitest": "4.1.8"+ "ente-build-config": "*"}}diff --git a/web/apps/ensu/src/components/chat/ChatDialogs.tsx b/web/apps/ensu/src/components/chat/ChatDialogs.tsxindex 1d1179677d..70b2fbb6f1 100644--- a/web/apps/ensu/src/components/chat/ChatDialogs.tsx+++ b/web/apps/ensu/src/components/chat/ChatDialogs.tsx@@ -939,7 +939,16 @@ export const ChatDialogs = memo({collection.lastError}</Typography>)}- {canRetry && (+ {collection.watcherError && (+ <Typography+ variant="small"+ sx={{ color: "text.muted" }}+ >+ {collection.watcherError}+ </Typography>+ )}+ {(canRetry ||+ collection.watcherError) && (<Buttonsize="small"onClick={() =>@@ -950,7 +959,9 @@ export const ChatDialogs = memo(}sx={{ alignSelf: "flex-start" }}>- Try again+ {canRetry+ ? "Try again"+ : "Refresh"}</Button>)}</Stack>diff --git a/web/apps/ensu/src/services/notes.ts b/web/apps/ensu/src/services/notes.tsindex e3b1cdd42c..d6edceb580 100644--- a/web/apps/ensu/src/services/notes.ts+++ b/web/apps/ensu/src/services/notes.ts@@ -16,6 +16,7 @@ export interface NotesCollection {lastUpdatedAtMs: number | null;updateDueAtMs: number | null;lastError: string | null;+ watcherError: string | null;}const invokeNotes = async <T>(diff --git a/web/apps/ensu/tests/notes-lifecycle.test.ts b/web/apps/ensu/tests/notes-lifecycle.test.tsdeleted file mode 100644index 8206367461..0000000000--- a/web/apps/ensu/tests/notes-lifecycle.test.ts+++ /dev/null@@ -1,31 +0,0 @@-import { describe, expect, test } from "vitest";-import { NotesLifecycleController } from "../src/services/notes-lifecycle";--describe("NotesLifecycleController", () => {- test("recovers cancelled indexing without duplicating or losing queued work", () => {- const controller = new NotesLifecycleController();-- expect(controller.ensureIndex("notes", false)).toBe("start");- expect(controller.indexActivity("notes")).toBe("starting");- controller.queueIndex("notes", false);- expect(controller.finishIndex("notes")).toEqual({- clearedRemovalWait: false,- });- expect(controller.indexActivity("notes")).toBe("queued");- expect(controller.ensureIndex("notes", false)).toBe("ignored");- expect(controller.drainQueued(true)).toEqual([]);- expect(controller.drainQueued(false)).toEqual([["notes", false]]);-- expect(controller.requestIndex("notes", false, false)).toBe("start");- expect(controller.requestIndex("notes", true, false)).toBe("queued");- controller.finishIndex("notes");- expect(controller.drainQueued(false)).toEqual([["notes", true]]);-- expect(controller.requestIndex("notes", true, false)).toBe("start");- controller.failIndex("notes");- controller.finishIndex("notes");- expect(controller.indexActivity("notes")).toBe("failed");- expect(controller.ensureIndex("notes", false)).toBe("ignored");- expect(controller.requestIndex("notes", true, false)).toBe("start");- });-});diff --git a/web/apps/ensu/tsconfig.json b/web/apps/ensu/tsconfig.jsonindex 6a2a667238..2ff1f734c6 100644--- a/web/apps/ensu/tsconfig.json+++ b/web/apps/ensu/tsconfig.json@@ -9,7 +9,6 @@},"include": ["src",- "tests","next-env.d.ts","next.config.mjs","../../packages/base/global-electron.d.ts",diff --git a/web/package-lock.json b/web/package-lock.jsonindex 7df478b760..4e037b9968 100644--- a/web/package-lock.json+++ b/web/package-lock.json@@ -191,8 +191,7 @@"remark-math": "6.0.0"},"devDependencies": {- "ente-build-config": "*",- "vitest": "4.1.8"+ "ente-build-config": "*"}},"apps/ensu/node_modules/katex": {diff --git a/web/package.json b/web/package.jsonindex 11ea951759..3bee38bffc 100644--- a/web/package.json+++ b/web/package.json@@ -52,7 +52,7 @@"dev:twoof3": "npm exec --workspace twoof3 -- next dev --webpack -p 3012","lint": "concurrently --names 'prettier,eslint,tsc,wasm-imports' \"prettier --check --log-level warn .\" \"npm exec --workspaces -- eslint --max-warnings 0\" \"npm exec --workspaces -- tsc\" \"npm run check:wasm-imports\"","lint:fix": "concurrently --names 'prettier,eslint,tsc' \"prettier --write --log-level warn .\" \"npm exec --workspaces -- eslint --fix --max-warnings 0\" \"npm exec --workspaces -- tsc\"",- "test": "npm run test --workspace ensu && npm run test --workspace ente-core-wasm && npm run test --workspace ente-paste-wasm && npm run test --workspace ente-contacts && npm run test --workspace locker && npm run test --workspace photos"+ "test": "npm run test --workspace ente-core-wasm && npm run test --workspace ente-paste-wasm && npm run test --workspace ente-contacts && npm run test --workspace locker && npm run test --workspace photos"},"overrides": {"@types/react": "19.2.17",