Skip to content

armgit / ente

Public Imported from GitHub
0

πŸ’š End-to-end encrypted cloud for everything.

Harden Your Notes indexing and watchers

setalp7bbe263032e4
diff --git a/rust/Cargo.lock b/rust/Cargo.lock
index a22e495af7..e69506f6b2 100644
--- a/rust/Cargo.lock
+++ b/rust/Cargo.lock
@@ -2381,7 +2381,6 @@ dependencies = [
"tauri-plugin-fs",
"tauri-plugin-opener",
"tauri-plugin-process",
- "tauri-plugin-single-instance",
"tauri-plugin-updater",
"uuid",
"windows-sys 0.61.2",
@@ -2541,7 +2540,6 @@ dependencies = [
"self_cell",
"serde",
"serde_json",
- "sha2 0.11.0",
"tempfile",
"thiserror 2.0.18",
"tokio",
@@ -9457,22 +9455,6 @@ dependencies = [
"tauri-plugin",
]
-[[package]]
-name = "tauri-plugin-single-instance"
-version = "2.4.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5cd0cb5c412a5071b69bab6a6df1583cbb89460d4a83b6a24769b08d15b6b1e1"
-dependencies = [
- "serde",
- "serde_json",
- "tauri",
- "thiserror 2.0.18",
- "tokio",
- "tracing",
- "windows-sys 0.60.2",
- "zbus 5.15.0",
-]
-
[[package]]
name = "tauri-plugin-updater"
version = "2.10.1"
diff --git a/rust/apps/ensu/src-tauri/Cargo.toml b/rust/apps/ensu/src-tauri/Cargo.toml
index 7c9aae59b4..261eb359e8 100644
--- a/rust/apps/ensu/src-tauri/Cargo.toml
+++ b/rust/apps/ensu/src-tauri/Cargo.toml
@@ -21,7 +21,6 @@ tauri-plugin-dialog = "2.7.1"
tauri-plugin-fs = "2.5.1"
tauri-plugin-opener = "2.5.4"
tauri-plugin-process = "2.3.1"
-tauri-plugin-single-instance = "2.4.4"
tauri-plugin-updater = "2.10.1"
uuid = { version = "1", features = ["v4"] }
diff --git a/rust/apps/ensu/src-tauri/src/commands/knowledge.rs b/rust/apps/ensu/src-tauri/src/commands/knowledge.rs
index a7239645ea..5e51285af8 100644
--- a/rust/apps/ensu/src-tauri/src/commands/knowledge.rs
+++ b/rust/apps/ensu/src-tauri/src/commands/knowledge.rs
@@ -619,10 +619,15 @@ pub async fn knowledge_retrieve(
check_cancelled()?;
match notes.search_collection(collection_id, &query_embedding) {
Ok(collection_hits) => note_hits.extend(collection_hits),
- Err(error) => logging::log(
- "Knowledge",
- format!("Notes search skipped collection={collection_id} error={error}"),
- ),
+ Err(error) => {
+ crate::commands::notes::mark_index_unreadable(&app, collection_id);
+ logging::log(
+ "Knowledge",
+ format!(
+ "Notes search skipped and queued for rebuild collection={collection_id} error={error}"
+ ),
+ );
+ }
}
}
}
diff --git a/rust/apps/ensu/src-tauri/src/commands/notes.rs b/rust/apps/ensu/src-tauri/src/commands/notes.rs
index 51d01db47d..25facfec4c 100644
--- a/rust/apps/ensu/src-tauri/src/commands/notes.rs
+++ b/rust/apps/ensu/src-tauri/src/commands/notes.rs
@@ -1,7 +1,11 @@
use std::collections::{BTreeSet, HashMap};
-use std::fs;
+use std::fs::{self, File, OpenOptions};
#[cfg(unix)]
-use std::os::unix::fs::PermissionsExt;
+use std::os::fd::AsRawFd;
+#[cfg(unix)]
+use std::os::unix::fs::{OpenOptionsExt, PermissionsExt};
+#[cfg(windows)]
+use std::os::windows::fs::OpenOptionsExt;
use std::path::{Path, PathBuf};
use std::sync::atomic::Ordering;
use std::sync::{Arc, Mutex, PoisonError};
@@ -19,6 +23,8 @@ use tauri::{AppHandle, Manager, State as TauriState};
use tauri_plugin_fs::FsExt;
use tauri_plugin_opener::OpenerExt;
use uuid::Uuid;
+#[cfg(windows)]
+use windows_sys::Win32::Foundation::ERROR_SHARING_VIOLATION;
use crate::commands::common::ApiError;
use crate::commands::llm::ModelDownloadState;
@@ -35,14 +41,12 @@ use source::{
is_supported_file, read_collection_source, source_root_is_available, validate_new_source_root,
};
pub use watcher::initialize_for_app;
-pub(crate) use watcher::mark_reference_stale;
-use watcher::{
- emit_state_changed, ensure_collection_watcher, install_collection_watcher,
- mark_collection_dirty,
-};
+use watcher::{emit_state_changed, ensure_collection_watcher, install_collection_watcher};
+pub(crate) use watcher::{mark_collection_dirty, mark_reference_stale};
const NOTES_DIRECTORY: &str = "notes";
const NOTES_INDEX_DIRECTORY: &str = "indexes";
+const NOTES_OWNERSHIP_LOCK_FILE: &str = ".owner.lock";
const NOTES_EMBEDDING_TITLE_MAX_UTF8_BYTES: usize = 512;
const NOTES_QUIET_PERIOD_MS: i64 = 5 * 60 * 1_000;
const NOTES_STATE_CHANGED_EVENT: &str = "notes-state-changed";
@@ -92,6 +96,7 @@ struct CollectionRuntime {
indexed_document_count: u64,
last_updated_at_ms: Option<i64>,
last_error: Option<String>,
+ watcher_error: Option<String>,
}
impl CollectionRuntime {
@@ -103,6 +108,7 @@ impl CollectionRuntime {
indexed_document_count: 0,
last_updated_at_ms: None,
last_error: None,
+ watcher_error: None,
}
}
@@ -126,6 +132,7 @@ pub struct NotesCollectionDto {
last_updated_at_ms: Option<i64>,
update_due_at_ms: Option<i64>,
last_error: Option<String>,
+ watcher_error: Option<String>,
}
#[derive(Debug, Serialize)]
@@ -219,6 +226,7 @@ impl IndexCache {
struct DesktopUpdateState {
forced_document_ids: BTreeSet<String>,
force_full_hash: bool,
+ rebuild_derived_index: bool,
quiet_deadline_ms: i64,
generation: u64,
bypass_quiet_period: bool,
@@ -233,7 +241,11 @@ impl DesktopUpdateState {
) {
self.forced_document_ids.extend(forced_document_ids);
self.force_full_hash |= force_full_hash;
- self.quiet_deadline_ms = now_ms.saturating_add(NOTES_QUIET_PERIOD_MS);
+ self.quiet_deadline_ms = if self.rebuild_derived_index {
+ now_ms
+ } else {
+ now_ms.saturating_add(NOTES_QUIET_PERIOD_MS)
+ };
self.generation = self.generation.wrapping_add(1);
self.bypass_quiet_period = false;
}
@@ -252,6 +264,17 @@ impl DesktopUpdateState {
self.bypass_quiet_period = false;
true
}
+
+ fn record_unreadable_index(&mut self, now_ms: i64) -> bool {
+ if self.rebuild_derived_index {
+ return false;
+ }
+ self.rebuild_derived_index = true;
+ self.quiet_deadline_ms = now_ms;
+ self.generation = self.generation.wrapping_add(1);
+ self.bypass_quiet_period = false;
+ true
+ }
}
#[derive(Default)]
@@ -271,6 +294,7 @@ pub struct State {
watchers: Mutex<HashMap<String, CollectionWatcher>>,
watch_work: Mutex<HashMap<String, watcher::WatchWork>>,
updates: Mutex<HashMap<String, DesktopUpdateState>>,
+ _ownership_lock: File,
}
#[derive(Clone)]
@@ -283,6 +307,8 @@ pub(crate) struct RetrievalHandle {
impl State {
pub fn new(app_data_dir: PathBuf) -> Result<Self, ApiError> {
let notes_directory = app_data_dir.join(NOTES_DIRECTORY);
+ ensure_directory(&notes_directory)?;
+ let ownership_lock = acquire_notes_ownership(&notes_directory)?;
let index_root = notes_directory.join(NOTES_INDEX_DIRECTORY);
ensure_directory(&index_root)?;
let registry = RegistryStore::open(notes_directory)?;
@@ -336,6 +362,7 @@ impl State {
watchers: Mutex::new(HashMap::new()),
watch_work: Mutex::new(HashMap::new()),
updates: Mutex::new(HashMap::new()),
+ _ownership_lock: ownership_lock,
})
}
@@ -395,6 +422,21 @@ impl State {
.unwrap_or_else(CollectionRuntime::pending)
}
+ fn set_watcher_error(&self, collection_id: &str, error: Option<&ApiError>) -> bool {
+ let mut runtimes = self.runtimes.lock().unwrap_or_else(PoisonError::into_inner);
+ let Some(runtime) = runtimes.get_mut(collection_id) else {
+ return false;
+ };
+ let watcher_error = error.map(|_| {
+ "Changes cannot be watched automatically. Refresh this folder manually.".to_string()
+ });
+ if runtime.watcher_error == watcher_error {
+ return false;
+ }
+ runtime.watcher_error = watcher_error;
+ true
+ }
+
fn prepare_update(
&self,
collection_id: &str,
@@ -422,7 +464,7 @@ impl State {
generation: update.generation,
forced_document_ids: update.forced_document_ids.iter().cloned().collect(),
force_full_hash: update.force_full_hash,
- rebuild_derived_index: false,
+ rebuild_derived_index: update.rebuild_derived_index,
})
}
@@ -591,6 +633,35 @@ impl State {
}
}
+pub(crate) fn mark_index_unreadable(app: &AppHandle, collection_id: &str) {
+ let state = app.state::<State>();
+ if state.registered_collection(collection_id).is_err() {
+ return;
+ }
+ state.evict_cached_index(collection_id);
+ {
+ let mut updates = state.updates.lock().unwrap_or_else(PoisonError::into_inner);
+ updates
+ .entry(collection_id.to_string())
+ .or_default()
+ .record_unreadable_index(now_ms());
+ }
+ {
+ let mut runtimes = state
+ .runtimes
+ .lock()
+ .unwrap_or_else(PoisonError::into_inner);
+ let Some(runtime) = runtimes.get_mut(collection_id) else {
+ return;
+ };
+ runtime.status = NotesCollectionStatusDto::Pending;
+ runtime.index_available = false;
+ runtime.indexing_progress = None;
+ runtime.last_error = None;
+ }
+ emit_state_changed(app, collection_id);
+}
+
impl RetrievalHandle {
pub(crate) fn collection_label(&self, collection_id: &str) -> Option<String> {
self.registry
@@ -659,6 +730,7 @@ fn collection_dto(
last_updated_at_ms: runtime.last_updated_at_ms,
update_due_at_ms,
last_error: runtime.last_error.clone(),
+ watcher_error: runtime.watcher_error.clone(),
}
}
@@ -751,13 +823,13 @@ fn register_collection(
}
notes_state.set_runtime(&collection.id, CollectionRuntime::pending());
if let Err(error) = install_collection_watcher(app, &collection) {
- notes_state.set_runtime(
- &collection.id,
- CollectionRuntime {
- status: NotesCollectionStatusDto::Error,
- last_error: Some(error.message),
- ..CollectionRuntime::pending()
- },
+ notes_state.set_watcher_error(&collection.id, Some(&error));
+ crate::logging::log(
+ "Notes",
+ format!(
+ "watcher unavailable collection={} error={}",
+ collection.id, error.message
+ ),
);
}
let dto = notes_state.collection_dto(&collection.id)?;
@@ -906,16 +978,27 @@ pub async fn notes_index_collection(
ensure_collection_watcher(&watcher_app, &watcher_collection)
})
.await
- .map_err(|_| ApiError::new("watcher_thread", "Notes watcher task failed"))?;
- if let Err(error) = watcher_result {
- report_index_error(&app, &notes_state, &collection_id, None, &error);
- return Err(error);
+ .unwrap_or_else(|_| Err(ApiError::new("watcher_thread", "Notes watcher task failed")));
+ match watcher_result {
+ Ok(()) => {
+ notes_state.set_watcher_error(&collection_id, None);
+ }
+ Err(error) => {
+ notes_state.set_watcher_error(&collection_id, Some(&error));
+ crate::logging::log(
+ "Notes",
+ format!(
+ "watcher unavailable collection={collection_id} error={}",
+ error.message
+ ),
+ );
+ }
}
let had_index = notes_state.has_available_index(&collection_id);
let rebuild_derived_index =
should_rebuild_derived_index(&notes_state.index_root, &collection_id, force, had_index)?;
let mut update = notes_state.prepare_update(&collection_id, force, had_index)?;
- update.rebuild_derived_index = rebuild_derived_index;
+ update.rebuild_derived_index |= rebuild_derived_index;
let cancellation_epoch = llm_state.retrieval_epoch();
if let Err(error) = check_cancelled(&cancellation_epoch, retrieval_epoch) {
crate::logging::log(
@@ -1127,6 +1210,48 @@ fn available_state(app: &AppHandle) -> Result<TauriState<'_, State>, ApiError> {
})
}
+fn acquire_notes_ownership(notes_directory: &Path) -> Result<File, ApiError> {
+ let mut options = OpenOptions::new();
+ options.read(true).write(true).create(true);
+ #[cfg(unix)]
+ options.mode(0o600);
+ #[cfg(windows)]
+ options.share_mode(0);
+
+ let file = options
+ .open(notes_directory.join(NOTES_OWNERSHIP_LOCK_FILE))
+ .map_err(notes_ownership_open_error)?;
+
+ #[cfg(unix)]
+ {
+ let result = unsafe { libc::flock(file.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) };
+ if result != 0 {
+ let error = std::io::Error::last_os_error();
+ if error.kind() == std::io::ErrorKind::WouldBlock {
+ return Err(notes_in_use_error());
+ }
+ return Err(io_error(error));
+ }
+ }
+
+ Ok(file)
+}
+
+fn notes_ownership_open_error(error: std::io::Error) -> ApiError {
+ #[cfg(windows)]
+ if error.raw_os_error() == Some(ERROR_SHARING_VIOLATION as i32) {
+ return notes_in_use_error();
+ }
+ io_error(error)
+}
+
+fn notes_in_use_error() -> ApiError {
+ ApiError::new(
+ "notes_in_use",
+ "Your Notes is already open in another Ensu instance",
+ )
+}
+
fn ensure_directory(path: &Path) -> Result<(), ApiError> {
if !path_exists(path)? {
fs::create_dir_all(path).map_err(io_error)?;
@@ -1229,6 +1354,59 @@ mod tests {
writer.publish(true).unwrap();
}
+ #[test]
+ fn ownership_lock_excludes_another_owner_until_release() {
+ let temp = TestDirectory::new();
+ let first = acquire_notes_ownership(temp.path()).unwrap();
+
+ let error = acquire_notes_ownership(temp.path()).unwrap_err();
+ assert_eq!(error.name, Some("notes_in_use"));
+ assert_eq!(
+ error.message,
+ "Your Notes is already open in another Ensu instance"
+ );
+
+ drop(first);
+ acquire_notes_ownership(temp.path()).unwrap();
+ }
+
+ #[test]
+ fn watcher_warning_does_not_replace_ready_index_state() {
+ let temp = TestDirectory::new();
+ let collection = RegisteredCollection {
+ id: COLLECTION_ID.to_string(),
+ source_root: temp.path().to_path_buf(),
+ };
+ let runtime = CollectionRuntime {
+ status: NotesCollectionStatusDto::Ready,
+ index_available: true,
+ indexing_progress: None,
+ indexed_document_count: 1,
+ last_updated_at_ms: Some(1),
+ last_error: None,
+ watcher_error: Some("Watcher unavailable".to_string()),
+ };
+
+ let dto = collection_dto(&collection, &runtime, None);
+
+ assert_eq!(dto.status, NotesCollectionStatusDto::Ready);
+ assert_eq!(dto.watcher_error.as_deref(), Some("Watcher unavailable"));
+ assert_eq!(dto.last_error, None);
+ }
+
+ #[test]
+ fn unreadable_index_requests_one_immediate_derived_rebuild() {
+ let mut update = DesktopUpdateState::default();
+
+ assert!(update.record_unreadable_index(42));
+ assert!(update.rebuild_derived_index);
+ assert_eq!(update.quiet_deadline_ms, 42);
+ assert_eq!(update.generation, 1);
+ assert!(!update.record_unreadable_index(84));
+ assert_eq!(update.quiet_deadline_ms, 42);
+ assert_eq!(update.generation, 1);
+ }
+
#[test]
fn retrieval_cache_evicts_the_least_recently_used_index_at_its_budget() {
let temp = TestDirectory::new();
diff --git a/rust/apps/ensu/src-tauri/src/commands/notes/watcher.rs b/rust/apps/ensu/src-tauri/src/commands/notes/watcher.rs
index d4559f368b..405cdab3f1 100644
--- a/rust/apps/ensu/src-tauri/src/commands/notes/watcher.rs
+++ b/rust/apps/ensu/src-tauri/src/commands/notes/watcher.rs
@@ -63,6 +63,15 @@ pub async fn initialize_for_app(app: AppHandle) {
.collections
.clone();
for collection in collections {
+ let lifecycle = app.state::<State>().collection_lifecycle(&collection.id);
+ let _collection_lifecycle = lifecycle.lock().await;
+ if app
+ .state::<State>()
+ .registered_collection(&collection.id)
+ .is_err()
+ {
+ continue;
+ }
let install_app = app.clone();
let install_collection = collection.clone();
let install_result = async_runtime::spawn_blocking(move || {
@@ -73,18 +82,21 @@ pub async fn initialize_for_app(app: AppHandle) {
Ok(result) => result,
Err(_) => Err(ApiError::new("watcher_thread", "Notes watcher task failed")),
};
- if let Err(error) = install_result {
- let state = app.state::<State>();
- let mut runtime = state.runtime(&collection.id);
- runtime.status = if source_root_is_available(&collection.source_root) {
- NotesCollectionStatusDto::Error
- } else {
- NotesCollectionStatusDto::Unavailable
- };
- runtime.last_error = Some(error.message);
- state.set_runtime(&collection.id, runtime);
- emit_state_changed(&app, &collection.id);
- continue;
+ match install_result {
+ Ok(()) => {
+ app.state::<State>().set_watcher_error(&collection.id, None);
+ }
+ Err(error) => {
+ let state = app.state::<State>();
+ state.set_watcher_error(&collection.id, Some(&error));
+ crate::logging::log(
+ "Notes",
+ format!(
+ "watcher unavailable collection={} error={}",
+ collection.id, error.message
+ ),
+ );
+ }
}
inspect_collection_on_startup(app.clone(), collection).await;
}
@@ -92,8 +104,6 @@ pub async fn initialize_for_app(app: AppHandle) {
async fn inspect_collection_on_startup(app: AppHandle, collection: RegisteredCollection) {
let index_root = app.state::<State>().index_root.clone();
- let lifecycle = app.state::<State>().collection_lifecycle(&collection.id);
- let _collection_lifecycle = lifecycle.lock().await;
if app
.state::<State>()
.registered_collection(&collection.id)
@@ -387,10 +397,19 @@ async fn process_watch_changes(app: AppHandle, collection_id: String, source_roo
change
};
if change.refresh_watches {
+ let lifecycle = app.state::<State>().collection_lifecycle(&collection_id);
+ let _collection_lifecycle = lifecycle.lock().await;
+ if app
+ .state::<State>()
+ .registered_collection(&collection_id)
+ .is_err()
+ {
+ continue;
+ }
let refresh_app = app.clone();
let refresh_collection_id = collection_id.clone();
let refresh_source_root = source_root.clone();
- let _ = async_runtime::spawn_blocking(move || {
+ let refresh_result = async_runtime::spawn_blocking(move || {
refresh_collection_watches(
&refresh_app,
&refresh_collection_id,
@@ -399,6 +418,36 @@ async fn process_watch_changes(app: AppHandle, collection_id: String, source_roo
)
})
.await;
+ let refresh_result = match refresh_result {
+ Ok(Ok(true)) => Ok(()),
+ Ok(Ok(false)) => Err(ApiError::new(
+ "watcher",
+ "The Notes folder could not be watched",
+ )),
+ Ok(Err(error)) => Err(error),
+ Err(_) => Err(ApiError::new("watcher_thread", "Notes watcher task failed")),
+ };
+ let state = app.state::<State>();
+ match refresh_result {
+ Ok(()) => {
+ if state.set_watcher_error(&collection_id, None) {
+ emit_state_changed(&app, &collection_id);
+ }
+ }
+ Err(error) => {
+ let changed = state.set_watcher_error(&collection_id, Some(&error));
+ crate::logging::log(
+ "Notes",
+ format!(
+ "watcher refresh failed collection={collection_id} error={}",
+ error.message
+ ),
+ );
+ if changed {
+ emit_state_changed(&app, &collection_id);
+ }
+ }
+ }
}
mark_collection_dirty(
&app,
diff --git a/rust/apps/ensu/src-tauri/src/main.rs b/rust/apps/ensu/src-tauri/src/main.rs
index 4942a7be87..ce750b2646 100644
--- a/rust/apps/ensu/src-tauri/src/main.rs
+++ b/rust/apps/ensu/src-tauri/src/main.rs
@@ -10,12 +10,6 @@ fn main() {
logging::log("App", "starting Tauri backend");
let app = tauri::Builder::default()
- .plugin(tauri_plugin_single_instance::init(|app, _args, _cwd| {
- if let Some(window) = app.get_webview_window("main") {
- let _ = window.show();
- let _ = window.set_focus();
- }
- }))
.plugin(tauri_plugin_dialog::init())
.plugin(tauri_plugin_fs::init())
.plugin(tauri_plugin_opener::init())
diff --git a/rust/crates/ensu-crypto/src/lib.rs b/rust/crates/ensu-crypto/src/lib.rs
index 3d2f6b3662..87bbb4fbbc 100644
--- a/rust/crates/ensu-crypto/src/lib.rs
+++ b/rust/crates/ensu-crypto/src/lib.rs
@@ -3,7 +3,7 @@ use ente_core::{
crypto::{self, Header, Key, blob},
};
use hkdf::Hkdf;
-use sha2::Sha256;
+use sha2::{Digest, Sha256};
use thiserror::Error;
const ATTACHMENT_KEY_INFO: &[u8] = b"llmchat_attachment_v1";
@@ -33,6 +33,10 @@ pub struct EncryptedChatPayload {
pub header: String,
}
+pub fn sha256(bytes: &[u8]) -> [u8; 32] {
+ Sha256::digest(bytes).into()
+}
+
pub fn generate_chat_key() -> String {
b64::encode(Key::generate().as_bytes())
}
diff --git a/rust/crates/ensu/Cargo.toml b/rust/crates/ensu/Cargo.toml
index 16c4a1fbb1..beb23708bd 100644
--- a/rust/crates/ensu/Cargo.toml
+++ b/rust/crates/ensu/Cargo.toml
@@ -14,7 +14,6 @@ rusqlite = { version = "0.38", features = ["bundled"], optional = true }
self_cell = "1"
serde.workspace = true
serde_json.workspace = true
-sha2.workspace = true
thiserror.workspace = true
tokio = { version = "1", features = ["rt", "time"] }
transcribe-rs = { git = "https://github.com/cjpais/transcribe-rs.git", rev = "efc66111d519a434e28d51d94740008a7ce29360", default-features = false, features = ["onnx", "vad-silero"], optional = true }
diff --git a/rust/crates/ensu/src/notes.rs b/rust/crates/ensu/src/notes.rs
index 1dfc10df0d..448b6e985c 100644
--- a/rust/crates/ensu/src/notes.rs
+++ b/rust/crates/ensu/src/notes.rs
@@ -8,7 +8,11 @@ mod shard;
mod source;
mod writer;
-use sha2::{Digest, Sha256};
+use std::collections::{HashMap, HashSet};
+use std::fs;
+use std::path::{Path, PathBuf};
+use std::sync::{Mutex, OnceLock, PoisonError};
+
use thiserror::Error;
use uuid::Uuid;
@@ -49,16 +53,154 @@ pub use indexing::{
};
pub use reconcile::NotesReconciliationPlan;
pub use source::NoteSourceReference;
-pub use writer::NotesIndexWriter;
+pub use writer::{NotesIndexWriter, ValidatedNotesDocument};
pub fn notes_content_revision(bytes: &[u8]) -> String {
sha256_hex(bytes)
}
+type PinnedRevision = ([u8; 32], [u8; 32]);
+
+fn pinned_revision_key(document_storage_id: &str, revision: &str) -> PinnedRevision {
+ (
+ ente_ensu_crypto::sha256(document_storage_id.as_bytes()),
+ ente_ensu_crypto::sha256(revision.as_bytes()),
+ )
+}
+
+#[derive(Default)]
+struct CollectionReaders {
+ opening: usize,
+ revisions: HashMap<PinnedRevision, usize>,
+}
+
+static ACTIVE_COLLECTION_READERS: OnceLock<Mutex<HashMap<PathBuf, CollectionReaders>>> =
+ OnceLock::new();
+
+struct NotesCollectionReadPin {
+ collection_directory: PathBuf,
+ opening: bool,
+ revisions: Vec<PinnedRevision>,
+}
+
+impl NotesCollectionReadPin {
+ fn acquire(collection_directory: &Path) -> Result<Self, NotesError> {
+ let collection_directory = fs::canonicalize(collection_directory)?;
+ let mut readers = active_collection_readers()
+ .lock()
+ .unwrap_or_else(PoisonError::into_inner);
+ let collection = readers.entry(collection_directory.clone()).or_default();
+ collection.opening = collection.opening.checked_add(1).ok_or_else(|| {
+ NotesError::InvalidIndex("active Notes reader count overflow".to_string())
+ })?;
+ Ok(Self {
+ collection_directory,
+ opening: true,
+ revisions: Vec::new(),
+ })
+ }
+
+ fn collection_directory(&self) -> &Path {
+ &self.collection_directory
+ }
+
+ fn estimated_heap_bytes(&self) -> usize {
+ let pinned_revision_bytes = std::mem::size_of::<PinnedRevision>();
+ self.collection_directory
+ .as_os_str()
+ .len()
+ .saturating_add(
+ self.revisions
+ .capacity()
+ .saturating_mul(pinned_revision_bytes),
+ )
+ .saturating_add(
+ self.revisions.len().saturating_mul(
+ pinned_revision_bytes
+ .saturating_add(std::mem::size_of::<usize>())
+ .saturating_add(1),
+ ),
+ )
+ }
+
+ fn pin_revisions(
+ &mut self,
+ revisions: impl IntoIterator<Item = PinnedRevision>,
+ ) -> Result<(), NotesError> {
+ let revisions = revisions.into_iter().collect::<HashSet<_>>();
+ let mut readers = active_collection_readers()
+ .lock()
+ .unwrap_or_else(PoisonError::into_inner);
+ let collection = readers.get_mut(&self.collection_directory).ok_or_else(|| {
+ NotesError::InvalidIndex("active Notes reader registration is missing".to_string())
+ })?;
+ collection.opening = collection.opening.checked_sub(1).ok_or_else(|| {
+ NotesError::InvalidIndex("active Notes reader registration is inconsistent".to_string())
+ })?;
+ self.opening = false;
+ self.revisions.reserve(revisions.len());
+ for revision in revisions {
+ let reader_count = collection.revisions.entry(revision).or_default();
+ *reader_count = reader_count.checked_add(1).ok_or_else(|| {
+ NotesError::InvalidIndex("active Notes revision reader count overflow".to_string())
+ })?;
+ self.revisions.push(revision);
+ }
+ Ok(())
+ }
+}
+
+impl Drop for NotesCollectionReadPin {
+ fn drop(&mut self) {
+ let mut readers = active_collection_readers()
+ .lock()
+ .unwrap_or_else(PoisonError::into_inner);
+ let Some(collection) = readers.get_mut(&self.collection_directory) else {
+ return;
+ };
+ if self.opening {
+ collection.opening = collection.opening.saturating_sub(1);
+ } else {
+ for revision in &self.revisions {
+ let remove = collection
+ .revisions
+ .get_mut(revision)
+ .is_some_and(|reader_count| {
+ *reader_count = reader_count.saturating_sub(1);
+ *reader_count == 0
+ });
+ if remove {
+ collection.revisions.remove(revision);
+ }
+ }
+ }
+ if collection.opening == 0 && collection.revisions.is_empty() {
+ readers.remove(&self.collection_directory);
+ }
+ }
+}
+
+fn active_collection_readers() -> &'static Mutex<HashMap<PathBuf, CollectionReaders>> {
+ ACTIVE_COLLECTION_READERS.get_or_init(|| Mutex::new(HashMap::new()))
+}
+
+fn collection_reader_snapshot(collection_directory: &Path) -> (bool, HashSet<PinnedRevision>) {
+ let readers = active_collection_readers()
+ .lock()
+ .unwrap_or_else(PoisonError::into_inner);
+ let Some(collection) = readers.get(collection_directory) else {
+ return (false, HashSet::new());
+ };
+ (
+ collection.opening > 0,
+ collection.revisions.keys().cloned().collect(),
+ )
+}
+
fn sha256_hex(bytes: &[u8]) -> String {
const HEX: &[u8; 16] = b"0123456789abcdef";
let mut encoded = String::with_capacity(64);
- for byte in Sha256::digest(bytes) {
+ for byte in ente_ensu_crypto::sha256(bytes) {
encoded.push(char::from(HEX[usize::from(byte >> 4)]));
encoded.push(char::from(HEX[usize::from(byte & 0x0f)]));
}
diff --git a/rust/crates/ensu/src/notes/index.rs b/rust/crates/ensu/src/notes/index.rs
index 5316c9ef65..ae01dcb2a0 100644
--- a/rust/crates/ensu/src/notes/index.rs
+++ b/rust/crates/ensu/src/notes/index.rs
@@ -1,9 +1,11 @@
-use std::collections::BTreeMap;
+use std::collections::{BTreeMap, btree_map::Entry};
use std::path::Path;
-use super::manifest::{collection_directory, load_published_manifest, shard_directory};
-use super::shard::{NotesShardIntegrity, load_and_validate_shard};
-use super::{NotesError, validate_collection_id};
+use super::manifest::{
+ collection_directory, document_storage_id, load_published_manifest, shard_directory,
+};
+use super::shard::{NotesShard, load_and_validate_shard_metadata, load_and_validate_vectors};
+use super::{NotesCollectionReadPin, NotesError, pinned_revision_key, validate_collection_id};
const RELEVANCE_THRESHOLD: f32 = 0.50;
const MAX_HITS_PER_COLLECTION: usize = 5;
@@ -20,22 +22,29 @@ pub struct NotesSearchHit {
pub text: String,
}
-struct IndexedChunk {
+struct IndexedDocument {
document_id: String,
revision: String,
- title: String,
- section: Option<String>,
- text: String,
+ chunk_count: usize,
+ vector_offset: usize,
+ shard_sha256: String,
+}
+
+#[derive(Clone, Copy)]
+struct RankedChunk {
+ score: f32,
+ document_index: usize,
+ chunk_index: usize,
vector_offset: usize,
}
pub struct NotesCollectionIndex {
+ read_pin: NotesCollectionReadPin,
collection_id: String,
- document_count: usize,
last_updated_at_ms: Option<i64>,
dimension: usize,
scale: f32,
- chunks: Vec<IndexedChunk>,
+ documents: Vec<IndexedDocument>,
vectors: Vec<u8>,
}
@@ -62,8 +71,13 @@ impl NotesCollectionIndex {
pub fn open(index_root: impl AsRef<Path>, collection_id: String) -> Result<Self, NotesError> {
validate_collection_id(&collection_id)?;
let collection_directory = collection_directory(index_root.as_ref(), &collection_id);
+ let mut read_pin = NotesCollectionReadPin::acquire(&collection_directory)?;
+ let collection_directory = read_pin.collection_directory().to_path_buf();
let (manifest, last_updated_at_ms) =
load_published_manifest(&collection_directory, &collection_id, true)?;
+ read_pin.pin_revisions(manifest.documents.iter().map(|(document_id, document)| {
+ pinned_revision_key(&document_storage_id(document_id), &document.revision)
+ }))?;
let dimension =
usize::try_from(manifest.index_contract.retrieval_dimension).map_err(|_| {
NotesError::InvalidIndex("retrieval dimension is too large".to_string())
@@ -83,65 +97,50 @@ impl NotesCollectionIndex {
let vector_bytes = total_chunks.checked_mul(dimension).ok_or_else(|| {
NotesError::InvalidIndex("collection vector byte length overflow".to_string())
})?;
- let document_count = manifest.documents.len();
- let mut chunks = Vec::new();
- chunks.try_reserve_exact(total_chunks).map_err(|_| {
- NotesError::InvalidIndex("collection chunk allocation is too large".to_string())
- })?;
+ let mut documents = Vec::new();
+ documents
+ .try_reserve_exact(manifest.documents.len())
+ .map_err(|_| {
+ NotesError::InvalidIndex("collection document allocation is too large".to_string())
+ })?;
let mut vectors = Vec::new();
vectors.try_reserve_exact(vector_bytes).map_err(|_| {
NotesError::InvalidIndex("collection vector allocation is too large".to_string())
})?;
for (document_id, document) in &manifest.documents {
- let loaded = load_and_validate_shard(
- &shard_directory(&collection_directory, document_id, &document.revision),
- &collection_id,
- document_id,
- &document.revision,
- document.chunk_count as usize,
- NotesShardIntegrity {
- shard_sha256: &document.shard_sha256,
- vectors_sha256: &document.vectors_sha256,
- },
- None,
- )?;
- let title = loaded.shard.title;
- for (chunk_index, chunk) in loaded.shard.chunks.into_iter().enumerate() {
- let start = chunk_index.checked_mul(dimension).ok_or_else(|| {
- NotesError::InvalidIndex("shard vector offset overflow".to_string())
- })?;
- let end = start.checked_add(dimension).ok_or_else(|| {
- NotesError::InvalidIndex("shard vector offset overflow".to_string())
- })?;
- chunks.push(IndexedChunk {
- document_id: document_id.clone(),
- revision: document.revision.clone(),
- title: title.clone(),
- section: chunk.section,
- text: chunk.text,
- vector_offset: vectors.len(),
- });
- vectors.extend_from_slice(&loaded.vectors[start..end]);
- }
+ let chunk_count = usize::try_from(document.chunk_count).map_err(|_| {
+ NotesError::InvalidIndex("manifest chunk count is too large".to_string())
+ })?;
+ let directory = shard_directory(&collection_directory, document_id, &document.revision);
+ let loaded_vectors =
+ load_and_validate_vectors(&directory, chunk_count, &document.vectors_sha256)?;
+ documents.push(IndexedDocument {
+ document_id: document_id.clone(),
+ revision: document.revision.clone(),
+ chunk_count,
+ vector_offset: vectors.len(),
+ shard_sha256: document.shard_sha256.clone(),
+ });
+ vectors.extend_from_slice(&loaded_vectors);
}
- if chunks.len() != total_chunks || vectors.len() != vector_bytes {
+ if documents.len() != manifest.documents.len() || vectors.len() != vector_bytes {
return Err(NotesError::InvalidIndex(
"loaded collection shape does not match its manifest".to_string(),
));
}
Ok(Self {
+ read_pin,
collection_id,
- document_count,
last_updated_at_ms,
dimension,
scale,
- chunks,
+ documents,
vectors,
})
}
pub fn document_count(&self) -> usize {
- self.document_count
+ self.documents.len()
}
pub fn last_updated_at_ms(&self) -> Option<i64> {
@@ -151,74 +150,140 @@ impl NotesCollectionIndex {
pub fn estimated_heap_bytes(&self) -> usize {
self.collection_id
.capacity()
+ .saturating_add(self.read_pin.estimated_heap_bytes())
.saturating_add(
- self.chunks
+ self.documents
.capacity()
- .saturating_mul(std::mem::size_of::<IndexedChunk>()),
+ .saturating_mul(std::mem::size_of::<IndexedDocument>()),
)
.saturating_add(self.vectors.capacity())
- .saturating_add(self.chunks.iter().fold(0_usize, |total, chunk| {
+ .saturating_add(self.documents.iter().fold(0_usize, |total, document| {
total
- .saturating_add(chunk.document_id.capacity())
- .saturating_add(chunk.revision.capacity())
- .saturating_add(chunk.title.capacity())
- .saturating_add(chunk.section.as_ref().map_or(0, String::capacity))
- .saturating_add(chunk.text.capacity())
+ .saturating_add(document.document_id.capacity())
+ .saturating_add(document.revision.capacity())
+ .saturating_add(document.shard_sha256.capacity())
}))
}
pub fn search(&self, query: &[f32]) -> Result<Vec<NotesSearchHit>, NotesError> {
validate_search(query, self.dimension)?;
- let mut ranked = self
- .chunks
- .iter()
- .map(|chunk| {
- let vector =
- &self.vectors[chunk.vector_offset..chunk.vector_offset + self.dimension];
+ let mut ranked = Vec::with_capacity(MAX_HITS_PER_COLLECTION);
+ for (document_index, document) in self.documents.iter().enumerate() {
+ let mut document_ranked = Vec::with_capacity(MAX_HITS_PER_DOCUMENT);
+ for chunk_index in 0..document.chunk_count {
+ let vector_offset = document
+ .vector_offset
+ .checked_add(chunk_index.checked_mul(self.dimension).ok_or_else(|| {
+ NotesError::InvalidIndex("shard vector offset overflow".to_string())
+ })?)
+ .ok_or_else(|| {
+ NotesError::InvalidIndex("shard vector offset overflow".to_string())
+ })?;
+ let vector_end = vector_offset.checked_add(self.dimension).ok_or_else(|| {
+ NotesError::InvalidIndex("shard vector offset overflow".to_string())
+ })?;
+ let vector = self.vectors.get(vector_offset..vector_end).ok_or_else(|| {
+ NotesError::InvalidIndex(
+ "shard vector range exceeds the collection".to_string(),
+ )
+ })?;
let score = query
.iter()
.zip(vector)
.map(|(query, stored)| *query * f32::from(*stored as i8))
.sum::<f32>()
/ self.scale;
- (score, chunk)
- })
- .filter(|(score, _)| score.is_finite() && *score >= RELEVANCE_THRESHOLD)
- .collect::<Vec<_>>();
- ranked.sort_unstable_by(|(left_score, left), (right_score, right)| {
- right_score
- .total_cmp(left_score)
- .then_with(|| left.document_id.cmp(&right.document_id))
- .then_with(|| left.vector_offset.cmp(&right.vector_offset))
- });
-
- let mut document_counts = BTreeMap::<&str, usize>::new();
- let mut hits = Vec::with_capacity(MAX_HITS_PER_COLLECTION.min(ranked.len()));
- for (score, chunk) in ranked {
- let count = document_counts
- .entry(chunk.document_id.as_str())
- .or_default();
- if *count >= MAX_HITS_PER_DOCUMENT {
- continue;
+ if score.is_finite() && score >= RELEVANCE_THRESHOLD {
+ retain_best(
+ &mut document_ranked,
+ RankedChunk {
+ score,
+ document_index,
+ chunk_index,
+ vector_offset,
+ },
+ MAX_HITS_PER_DOCUMENT,
+ &self.documents,
+ );
+ }
+ }
+ for candidate in document_ranked {
+ retain_best(
+ &mut ranked,
+ candidate,
+ MAX_HITS_PER_COLLECTION,
+ &self.documents,
+ );
}
- *count += 1;
+ }
+
+ let mut hits = Vec::with_capacity(MAX_HITS_PER_COLLECTION.min(ranked.len()));
+ let mut loaded_shards = BTreeMap::<usize, NotesShard>::new();
+ for candidate in ranked {
+ let document = &self.documents[candidate.document_index];
+ let shard = match loaded_shards.entry(candidate.document_index) {
+ Entry::Occupied(entry) => entry.into_mut(),
+ Entry::Vacant(entry) => {
+ let directory = shard_directory(
+ self.read_pin.collection_directory(),
+ &document.document_id,
+ &document.revision,
+ );
+ entry.insert(load_and_validate_shard_metadata(
+ &directory,
+ &self.collection_id,
+ &document.document_id,
+ &document.revision,
+ document.chunk_count,
+ &document.shard_sha256,
+ None,
+ )?)
+ }
+ };
+ let chunk = shard.chunks.get(candidate.chunk_index).ok_or_else(|| {
+ NotesError::InvalidIndex("selected shard chunk is missing".to_string())
+ })?;
hits.push(NotesSearchHit {
collection_id: self.collection_id.clone(),
- document_id: chunk.document_id.clone(),
- revision: chunk.revision.clone(),
- score,
- title: chunk.title.clone(),
+ document_id: document.document_id.clone(),
+ revision: document.revision.clone(),
+ score: candidate.score,
+ title: shard.title.clone(),
section: chunk.section.clone(),
text: chunk.text.clone(),
});
- if hits.len() == MAX_HITS_PER_COLLECTION {
- break;
- }
}
Ok(hits)
}
}
+fn retain_best(
+ ranked: &mut Vec<RankedChunk>,
+ candidate: RankedChunk,
+ limit: usize,
+ documents: &[IndexedDocument],
+) {
+ ranked.push(candidate);
+ ranked.sort_unstable_by(|left, right| ranked_order(left, right, documents));
+ ranked.truncate(limit);
+}
+
+fn ranked_order(
+ left: &RankedChunk,
+ right: &RankedChunk,
+ documents: &[IndexedDocument],
+) -> std::cmp::Ordering {
+ right
+ .score
+ .total_cmp(&left.score)
+ .then_with(|| {
+ documents[left.document_index]
+ .document_id
+ .cmp(&documents[right.document_index].document_id)
+ })
+ .then_with(|| left.vector_offset.cmp(&right.vector_offset))
+}
+
fn validate_search(query: &[f32], dimension: usize) -> Result<(), NotesError> {
if query.len() != dimension {
return Err(NotesError::InvalidInput(format!(
@@ -233,3 +298,177 @@ fn validate_search(query: &[f32], dimension: usize) -> Result<(), NotesError> {
}
Ok(())
}
+
+#[cfg(test)]
+mod tests {
+ use std::fs;
+
+ use super::*;
+ use crate::notes::shard::{NOTES_SHARD_FILE, NOTES_VECTORS_FILE};
+ use crate::notes::{
+ NotesIndexWriter, NotesSourceDocument, PreparedNotesChunk, PreparedNotesDocument,
+ notes_content_revision,
+ };
+
+ const COLLECTION_ID: &str = "123e4567-e89b-12d3-a456-426614174000";
+
+ fn document(document_id: &str, chunk_count: usize) -> PreparedNotesDocument {
+ PreparedNotesDocument {
+ document_id: document_id.to_string(),
+ revision: notes_content_revision(document_id.as_bytes()),
+ title: document_id.to_string(),
+ chunks: (0..chunk_count)
+ .map(|index| PreparedNotesChunk {
+ section: None,
+ text: format!("{document_id} chunk {index}"),
+ })
+ .collect(),
+ }
+ }
+
+ fn source(document: &PreparedNotesDocument) -> NotesSourceDocument {
+ NotesSourceDocument {
+ document_id: document.document_id.clone(),
+ size: document
+ .chunks
+ .iter()
+ .map(|chunk| chunk.text.len() as u64)
+ .sum(),
+ modified_at_ms: Some(1),
+ }
+ }
+
+ fn embedding(axis: usize, score: f32) -> Vec<f32> {
+ let mut embedding = vec![0.0; 512];
+ embedding[axis] = score;
+ embedding
+ }
+
+ #[test]
+ fn search_loads_metadata_only_for_selected_documents() {
+ let temp = tempfile::tempdir().unwrap();
+ let first = document("first.md", 1);
+ let second = document("second.md", 1);
+ let mut writer = NotesIndexWriter::open(temp.path(), COLLECTION_ID.to_string()).unwrap();
+ writer
+ .commit_document(&first, &[embedding(0, 1.0)], &source(&first))
+ .unwrap();
+ writer
+ .commit_document(&second, &[embedding(1, 1.0)], &source(&second))
+ .unwrap();
+ writer.publish(true).unwrap();
+
+ let second_shard = shard_directory(
+ &collection_directory(temp.path(), COLLECTION_ID),
+ &second.document_id,
+ &second.revision,
+ )
+ .join(NOTES_SHARD_FILE);
+ fs::write(second_shard, b"corrupt").unwrap();
+
+ let index = NotesCollectionIndex::open(temp.path(), COLLECTION_ID.to_string()).unwrap();
+ let hits = index.search(&embedding(0, 1.0)).unwrap();
+ assert_eq!(hits.len(), 1);
+ assert_eq!(hits[0].document_id, "first.md");
+ assert!(matches!(
+ index.search(&embedding(1, 1.0)),
+ Err(NotesError::InvalidIndex(_) | NotesError::Json(_))
+ ));
+ }
+
+ #[test]
+ fn open_index_keeps_its_shards_alive_until_it_is_dropped() {
+ let temp = tempfile::tempdir().unwrap();
+ let original = document("note.md", 1);
+ let original_directory = shard_directory(
+ &collection_directory(temp.path(), COLLECTION_ID),
+ &original.document_id,
+ &original.revision,
+ );
+ let mut writer = NotesIndexWriter::open(temp.path(), COLLECTION_ID.to_string()).unwrap();
+ writer
+ .commit_document(&original, &[embedding(0, 1.0)], &source(&original))
+ .unwrap();
+ writer.publish(true).unwrap();
+ let index = NotesCollectionIndex::open(temp.path(), COLLECTION_ID.to_string()).unwrap();
+
+ let mut replacement = document("note.md", 1);
+ replacement.revision = notes_content_revision(b"replacement");
+ replacement.chunks[0].text = "replacement chunk".to_string();
+ writer
+ .commit_document(&replacement, &[embedding(1, 1.0)], &source(&replacement))
+ .unwrap();
+ writer.publish(true).unwrap();
+
+ assert!(original_directory.is_dir());
+ let hits = index.search(&embedding(0, 1.0)).unwrap();
+ assert_eq!(hits[0].text, original.chunks[0].text);
+
+ drop(index);
+ let replacement_index =
+ NotesCollectionIndex::open(temp.path(), COLLECTION_ID.to_string()).unwrap();
+ writer.publish(true).unwrap();
+ assert!(!original_directory.exists());
+ assert_eq!(
+ replacement_index.search(&embedding(1, 1.0)).unwrap()[0].text,
+ replacement.chunks[0].text
+ );
+ }
+
+ #[test]
+ fn failed_open_releases_its_collection_read_pin() {
+ let temp = tempfile::tempdir().unwrap();
+ let note = document("note.md", 1);
+ let mut writer = NotesIndexWriter::open(temp.path(), COLLECTION_ID.to_string()).unwrap();
+ writer
+ .commit_document(&note, &[embedding(0, 1.0)], &source(&note))
+ .unwrap();
+ writer.publish(true).unwrap();
+ let collection_directory = collection_directory(temp.path(), COLLECTION_ID);
+ let vectors_path =
+ shard_directory(&collection_directory, &note.document_id, &note.revision)
+ .join(NOTES_VECTORS_FILE);
+ fs::write(vectors_path, b"corrupt").unwrap();
+
+ assert!(NotesCollectionIndex::open(temp.path(), COLLECTION_ID.to_string()).is_err());
+ let collection_directory = fs::canonicalize(collection_directory).unwrap();
+ let (opening, revisions) = crate::notes::collection_reader_snapshot(&collection_directory);
+ assert!(!opening);
+ assert!(revisions.is_empty());
+ }
+
+ #[test]
+ fn search_bounds_per_document_and_global_ranking() {
+ let temp = tempfile::tempdir().unwrap();
+ let dominant = document("a.md", 5);
+ let mut writer = NotesIndexWriter::open(temp.path(), COLLECTION_ID.to_string()).unwrap();
+ writer
+ .commit_document(
+ &dominant,
+ &[1.0, 0.99, 0.98, 0.97, 0.96]
+ .into_iter()
+ .map(|score| embedding(0, score))
+ .collect::<Vec<_>>(),
+ &source(&dominant),
+ )
+ .unwrap();
+ for (document_id, score) in [("b.md", 0.95), ("c.md", 0.94), ("d.md", 0.93)] {
+ let document = document(document_id, 1);
+ writer
+ .commit_document(&document, &[embedding(0, score)], &source(&document))
+ .unwrap();
+ }
+ writer.publish(true).unwrap();
+
+ let index = NotesCollectionIndex::open(temp.path(), COLLECTION_ID.to_string()).unwrap();
+ let hits = index.search(&embedding(0, 1.0)).unwrap();
+
+ assert_eq!(hits.len(), MAX_HITS_PER_COLLECTION);
+ assert_eq!(
+ hits.iter().filter(|hit| hit.document_id == "a.md").count(),
+ MAX_HITS_PER_DOCUMENT
+ );
+ assert_eq!(hits[3].document_id, "b.md");
+ assert_eq!(hits[4].document_id, "c.md");
+ }
+}
diff --git a/rust/crates/ensu/src/notes/indexing.rs b/rust/crates/ensu/src/notes/indexing.rs
index 0b29d3cc04..9745b02f18 100644
--- a/rust/crates/ensu/src/notes/indexing.rs
+++ b/rust/crates/ensu/src/notes/indexing.rs
@@ -215,8 +215,8 @@ pub fn index_notes_collection<E>(
);
continue;
};
- match writer.validate_document(&prepared, &source_metadata) {
- Ok(()) => {}
+ let validated = match writer.validate_document(&prepared, &source_metadata) {
+ Ok(validated) => validated,
Err(NotesError::InvalidInput(_)) => {
writer.commit_deletions(std::slice::from_ref(document_id))?;
processed_requested += 1;
@@ -230,7 +230,7 @@ pub fn index_notes_collection<E>(
continue;
}
Err(error) => return Err(error.into()),
- }
+ };
let Some(embeddings) = embed_document(&prepared).map_err(NotesIndexingError::Adapter)?
else {
writer.commit_deletions(std::slice::from_ref(document_id))?;
@@ -267,7 +267,7 @@ pub fn index_notes_collection<E>(
processed_requested,
);
}
- writer.commit_document(&prepared, &embeddings, &verified_source)?;
+ writer.commit_validated_document(validated, &embeddings)?;
processed_requested += 1;
progress_document_count += 1;
report_indexing_progress(
diff --git a/rust/crates/ensu/src/notes/shard.rs b/rust/crates/ensu/src/notes/shard.rs
index 26cc06730a..700bbf5154 100644
--- a/rust/crates/ensu/src/notes/shard.rs
+++ b/rust/crates/ensu/src/notes/shard.rs
@@ -29,11 +29,6 @@ pub(super) struct NotesShardChunk {
pub text: String,
}
-pub(super) struct ValidatedNotesShard {
- pub shard: NotesShard,
- pub vectors: Vec<u8>,
-}
-
#[derive(Clone, Copy)]
pub(super) struct NotesShardIntegrity<'a> {
pub shard_sha256: &'a str,
@@ -77,15 +72,35 @@ pub(super) fn load_and_validate_shard(
expected_chunk_count: usize,
expected_integrity: NotesShardIntegrity<'_>,
expected_prepared: Option<&PreparedNotesDocument>,
-) -> Result<ValidatedNotesShard, NotesError> {
- validate_revision(expected_integrity.shard_sha256).map_err(invalid_index_input)?;
- validate_revision(expected_integrity.vectors_sha256).map_err(invalid_index_input)?;
- let directory_metadata = fs::symlink_metadata(directory)?;
- if !directory_metadata.file_type().is_dir() {
- return Err(NotesError::InvalidIndex(
- "shard path must be a directory".to_string(),
- ));
- }
+) -> Result<(), NotesError> {
+ let shard = load_and_validate_shard_metadata(
+ directory,
+ collection_id,
+ expected_document_id,
+ expected_revision,
+ expected_chunk_count,
+ expected_integrity.shard_sha256,
+ expected_prepared,
+ )?;
+ load_and_validate_vectors(
+ directory,
+ shard.chunks.len(),
+ expected_integrity.vectors_sha256,
+ )?;
+ Ok(())
+}
+
+pub(super) fn load_and_validate_shard_metadata(
+ directory: &Path,
+ collection_id: &str,
+ expected_document_id: &str,
+ expected_revision: &str,
+ expected_chunk_count: usize,
+ expected_shard_sha256: &str,
+ expected_prepared: Option<&PreparedNotesDocument>,
+) -> Result<NotesShard, NotesError> {
+ validate_revision(expected_shard_sha256).map_err(invalid_index_input)?;
+ validate_shard_directory(directory)?;
let shard_path = directory.join(NOTES_SHARD_FILE);
let shard_metadata = regular_file_metadata(&shard_path, "shard metadata")?;
@@ -96,7 +111,7 @@ pub(super) fn load_and_validate_shard(
}
let shard_bytes = fs::read(shard_path)?;
if shard_bytes.len() as u64 != shard_metadata.len()
- || sha256_hex(&shard_bytes) != expected_integrity.shard_sha256
+ || sha256_hex(&shard_bytes) != expected_shard_sha256
{
return Err(NotesError::InvalidIndex(
"shard metadata digest does not match its manifest".to_string(),
@@ -111,14 +126,21 @@ pub(super) fn load_and_validate_shard(
expected_chunk_count,
expected_prepared,
)?;
+ Ok(shard)
+}
+pub(super) fn load_and_validate_vectors(
+ directory: &Path,
+ expected_chunk_count: usize,
+ expected_vectors_sha256: &str,
+) -> Result<Vec<u8>, NotesError> {
+ validate_revision(expected_vectors_sha256).map_err(invalid_index_input)?;
+ validate_shard_directory(directory)?;
let vectors_path = directory.join(NOTES_VECTORS_FILE);
let vector_metadata = regular_file_metadata(&vectors_path, "shard vectors")?;
let vector_dimension = usize::try_from(notes_index_contract().retrieval_dimension)
.map_err(|_| NotesError::InvalidIndex("vector dimension is too large".to_string()))?;
- let expected_vector_bytes = shard
- .chunks
- .len()
+ let expected_vector_bytes = expected_chunk_count
.checked_mul(vector_dimension)
.ok_or_else(|| NotesError::InvalidIndex("shard vector byte length overflow".to_string()))?;
let vectors = fs::read(vectors_path)?;
@@ -129,12 +151,22 @@ pub(super) fn load_and_validate_shard(
"shard vector byte length does not match its metadata".to_string(),
));
}
- if sha256_hex(&vectors) != expected_integrity.vectors_sha256 {
+ if sha256_hex(&vectors) != expected_vectors_sha256 {
return Err(NotesError::InvalidIndex(
"shard vector digest does not match its manifest".to_string(),
));
}
- Ok(ValidatedNotesShard { shard, vectors })
+ Ok(vectors)
+}
+
+fn validate_shard_directory(directory: &Path) -> Result<(), NotesError> {
+ let directory_metadata = fs::symlink_metadata(directory)?;
+ if !directory_metadata.file_type().is_dir() {
+ return Err(NotesError::InvalidIndex(
+ "shard path must be a directory".to_string(),
+ ));
+ }
+ Ok(())
}
fn validate_shard(
diff --git a/rust/crates/ensu/src/notes/writer.rs b/rust/crates/ensu/src/notes/writer.rs
index 45e1eb2be8..ffd57c7ef0 100644
--- a/rust/crates/ensu/src/notes/writer.rs
+++ b/rust/crates/ensu/src/notes/writer.rs
@@ -20,14 +20,25 @@ use super::shard::{
use super::{
NOTES_CHUNK_UTF8_BYTES, NOTES_MAX_COLLECTION_CHUNKS, NOTES_MAX_COLLECTION_DOCUMENTS,
NOTES_MAX_COLLECTION_SOURCE_BYTES, NotesError, NotesReconciliationPlan, NotesSourceDocument,
- PreparedNotesDocument, contains_unsupported_control, is_valid_label, sha256_hex,
- validate_collection_id, validate_document_id, validate_revision,
+ PreparedNotesDocument, collection_reader_snapshot, contains_unsupported_control,
+ is_valid_label, pinned_revision_key, sha256_hex, validate_collection_id, validate_document_id,
+ validate_revision,
};
pub struct NotesIndexWriter {
collection_id: String,
collection_directory: PathBuf,
manifest: NotesManifest,
+ source_bytes: u64,
+ chunk_count: u64,
+}
+
+pub struct ValidatedNotesDocument<'a> {
+ collection_id: String,
+ prepared: &'a PreparedNotesDocument,
+ source_metadata: &'a NotesSourceDocument,
+ shard_bytes: Vec<u8>,
+ shard_sha256: String,
}
impl NotesIndexWriter {
@@ -121,6 +132,7 @@ impl NotesIndexWriter {
ensure_directory(index_root.as_ref())?;
let collection_directory = collection_directory(index_root.as_ref(), &collection_id);
ensure_directory(&collection_directory)?;
+ let collection_directory = fs::canonicalize(collection_directory)?;
ensure_directory(&collection_directory.join(NOTES_DOCUMENTS_DIRECTORY))?;
recover_manifest_publication(&collection_directory, &collection_id)?;
@@ -130,11 +142,14 @@ impl NotesIndexWriter {
} else {
empty_manifest(&collection_id)
};
+ let (source_bytes, chunk_count) = manifest_capacity(&manifest)?;
let _ = cleanup_unreferenced_shards(&collection_directory, &manifest.documents);
Ok(Self {
collection_id,
collection_directory,
manifest,
+ source_bytes,
+ chunk_count,
})
}
@@ -183,7 +198,29 @@ impl NotesIndexWriter {
embeddings: &[Vec<f32>],
source_metadata: &NotesSourceDocument,
) -> Result<(), NotesError> {
- self.validate_document(prepared, source_metadata)?;
+ let validated = self.validate_document(prepared, source_metadata)?;
+ self.commit_validated_document(validated, embeddings)
+ }
+
+ pub fn commit_validated_document(
+ &mut self,
+ validated: ValidatedNotesDocument<'_>,
+ embeddings: &[Vec<f32>],
+ ) -> Result<(), NotesError> {
+ let ValidatedNotesDocument {
+ collection_id,
+ prepared,
+ source_metadata,
+ shard_bytes,
+ shard_sha256,
+ } = validated;
+ if collection_id != self.collection_id {
+ return Err(NotesError::InvalidInput(
+ "validated document belongs to another collection".to_string(),
+ ));
+ }
+ validate_prepared_document(prepared, source_metadata)?;
+ self.validate_document_capacity(prepared, source_metadata)?;
if embeddings.len() != prepared.chunks.len() {
return Err(NotesError::InvalidInput(
"embedding count does not match prepared chunks".to_string(),
@@ -214,9 +251,6 @@ impl NotesIndexWriter {
.clamp(-scale, scale) as i8 as u8
}));
}
- let shard = NotesShard::from_prepared(&self.collection_id, prepared);
- let shard_bytes = serialize_shard(&shard)?;
- let shard_sha256 = sha256_hex(&shard_bytes);
let vectors_sha256 = sha256_hex(&vector_bytes);
let final_directory = shard_directory(
@@ -307,46 +341,69 @@ impl NotesIndexWriter {
self.activate_document(prepared, source_metadata, &shard_sha256, &vectors_sha256)
}
- pub fn validate_document(
+ pub fn validate_document<'a>(
+ &self,
+ prepared: &'a PreparedNotesDocument,
+ source_metadata: &'a NotesSourceDocument,
+ ) -> Result<ValidatedNotesDocument<'a>, NotesError> {
+ validate_prepared_document(prepared, source_metadata)?;
+ self.validate_document_capacity(prepared, source_metadata)?;
+ let shard_bytes =
+ serialize_shard(&NotesShard::from_prepared(&self.collection_id, prepared))?;
+ let shard_sha256 = sha256_hex(&shard_bytes);
+ Ok(ValidatedNotesDocument {
+ collection_id: self.collection_id.clone(),
+ prepared,
+ source_metadata,
+ shard_bytes,
+ shard_sha256,
+ })
+ }
+
+ fn validate_document_capacity(
&self,
prepared: &PreparedNotesDocument,
source_metadata: &NotesSourceDocument,
) -> Result<(), NotesError> {
- validate_prepared_document(prepared, source_metadata)?;
- let (existing_source_bytes, existing_chunks) = self
- .manifest
- .documents
- .iter()
- .filter(|(document_id, _)| *document_id != &prepared.document_id)
- .try_fold(
- (0_u64, 0_u64),
- |(source_total, chunk_total), (_, document)| {
- let source_total = source_total
- .checked_add(document.source_size)
- .ok_or_else(source_content_too_large)?;
- let chunk_total = chunk_total
- .checked_add(u64::from(document.chunk_count))
- .ok_or_else(chunk_count_too_large)?;
- Ok::<_, NotesError>((source_total, chunk_total))
- },
- )?;
- let source_bytes = existing_source_bytes
- .checked_add(source_metadata.size)
+ let existing = self.manifest.documents.get(&prepared.document_id);
+ let source_bytes = self
+ .source_bytes
+ .checked_sub(existing.map_or(0, |document| document.source_size))
+ .and_then(|total| total.checked_add(source_metadata.size))
.ok_or_else(source_content_too_large)?;
- let chunk_count =
+ let document_chunks =
u64::try_from(prepared.chunks.len()).map_err(|_| chunk_count_too_large())?;
- let chunks = existing_chunks
- .checked_add(chunk_count)
+ let chunks = self
+ .chunk_count
+ .checked_sub(existing.map_or(0, |document| u64::from(document.chunk_count)))
+ .and_then(|total| total.checked_add(document_chunks))
.ok_or_else(chunk_count_too_large)?;
- ensure_collection_capacity(source_bytes, chunks)?;
- serialize_shard(&NotesShard::from_prepared(&self.collection_id, prepared))?;
- Ok(())
+ ensure_collection_capacity(source_bytes, chunks)
}
pub fn commit_deletions(&mut self, document_ids: &[String]) -> Result<(), NotesError> {
for document_id in document_ids {
validate_document_id(document_id)?;
+ let Some(document) = self.manifest.documents.get(document_id) else {
+ continue;
+ };
+ let source_bytes = self
+ .source_bytes
+ .checked_sub(document.source_size)
+ .ok_or_else(|| {
+ NotesError::InvalidIndex(
+ "manifest source byte total is inconsistent".to_string(),
+ )
+ })?;
+ let chunk_count = self
+ .chunk_count
+ .checked_sub(u64::from(document.chunk_count))
+ .ok_or_else(|| {
+ NotesError::InvalidIndex("manifest chunk total is inconsistent".to_string())
+ })?;
self.manifest.documents.remove(document_id);
+ self.source_bytes = source_bytes;
+ self.chunk_count = chunk_count;
}
Ok(())
}
@@ -374,6 +431,18 @@ impl NotesIndexWriter {
let chunk_count = u32::try_from(prepared.chunks.len()).map_err(|_| {
NotesError::InvalidInput("prepared document has too many chunks".to_string())
})?;
+ let existing = self.manifest.documents.get(&prepared.document_id);
+ let source_bytes = self
+ .source_bytes
+ .checked_sub(existing.map_or(0, |document| document.source_size))
+ .and_then(|total| total.checked_add(source_metadata.size))
+ .ok_or_else(source_content_too_large)?;
+ let total_chunks = self
+ .chunk_count
+ .checked_sub(existing.map_or(0, |document| u64::from(document.chunk_count)))
+ .and_then(|total| total.checked_add(u64::from(chunk_count)))
+ .ok_or_else(chunk_count_too_large)?;
+ ensure_collection_capacity(source_bytes, total_chunks)?;
self.manifest.documents.insert(
prepared.document_id.clone(),
NotesManifestDocument {
@@ -385,10 +454,32 @@ impl NotesIndexWriter {
vectors_sha256: vectors_sha256.to_string(),
},
);
+ self.source_bytes = source_bytes;
+ self.chunk_count = total_chunks;
Ok(())
}
}
+fn manifest_capacity(manifest: &NotesManifest) -> Result<(u64, u64), NotesError> {
+ let (source_bytes, chunks) = manifest.documents.values().try_fold(
+ (0_u64, 0_u64),
+ |(source_total, chunk_total), document| {
+ let source_total = source_total
+ .checked_add(document.source_size)
+ .ok_or_else(|| {
+ NotesError::InvalidIndex("manifest source byte count overflow".to_string())
+ })?;
+ let chunk_total = chunk_total
+ .checked_add(u64::from(document.chunk_count))
+ .ok_or_else(|| {
+ NotesError::InvalidIndex("manifest chunk count overflow".to_string())
+ })?;
+ Ok::<_, NotesError>((source_total, chunk_total))
+ },
+ )?;
+ Ok((source_bytes, chunks))
+}
+
fn ensure_collection_capacity(source_bytes: u64, chunks: u64) -> Result<(), NotesError> {
if source_bytes > NOTES_MAX_COLLECTION_SOURCE_BYTES {
return Err(source_content_too_large());
@@ -559,6 +650,10 @@ fn cleanup_unreferenced_shards(
) -> Result<(), NotesError> {
use super::manifest::document_storage_id;
+ let (reader_opening, pinned_revisions) = collection_reader_snapshot(collection_directory);
+ if reader_opening {
+ return Ok(());
+ }
let keep = documents
.iter()
.map(|(document_id, document)| (document_storage_id(document_id), &document.revision))
@@ -567,20 +662,28 @@ fn cleanup_unreferenced_shards(
for document_entry in fs::read_dir(&documents_directory)? {
let document_entry = document_entry?;
let name = document_entry.file_name().to_string_lossy().into_owned();
- let Some(active_revision) = keep.get(&name) else {
- remove_owned_entry(&document_entry.path())?;
- continue;
- };
if !document_entry.file_type()?.is_dir() {
remove_owned_entry(&document_entry.path())?;
continue;
}
for revision_entry in fs::read_dir(document_entry.path())? {
let revision_entry = revision_entry?;
- if revision_entry.file_name().to_string_lossy() != active_revision.as_str() {
+ let revision = revision_entry.file_name().to_string_lossy().into_owned();
+ let is_active = keep
+ .get(&name)
+ .is_some_and(|active_revision| active_revision.as_str() == revision);
+ if !is_active && !pinned_revisions.contains(&pinned_revision_key(&name, &revision)) {
remove_owned_entry(&revision_entry.path())?;
}
}
+ let document_key = ente_ensu_crypto::sha256(name.as_bytes());
+ if !keep.contains_key(&name)
+ && !pinned_revisions
+ .iter()
+ .any(|(document, _)| document == &document_key)
+ {
+ fs::remove_dir(document_entry.path())?;
+ }
}
Ok(())
}
@@ -691,6 +794,37 @@ mod tests {
));
}
+ #[test]
+ fn maintains_cached_capacity_totals_across_replacement_and_deletion() {
+ let temp = tempfile::tempdir().unwrap();
+ let first = prepared("first.md", "first");
+ let second = prepared("second.md", "second");
+ let mut writer = NotesIndexWriter::open(temp.path(), collection()).unwrap();
+ writer
+ .commit_document(&first, &embeddings(&first, &[1.0]), &metadata(&first, 5))
+ .unwrap();
+ writer
+ .commit_document(&second, &embeddings(&second, &[0.8]), &metadata(&second, 6))
+ .unwrap();
+ assert_eq!(writer.source_bytes, 11);
+ assert_eq!(writer.chunk_count, 2);
+
+ let replacement = prepared("first.md", "replacement");
+ writer
+ .commit_document(
+ &replacement,
+ &embeddings(&replacement, &[0.9]),
+ &metadata(&replacement, 11),
+ )
+ .unwrap();
+ assert_eq!(writer.source_bytes, 17);
+ assert_eq!(writer.chunk_count, 2);
+
+ writer.commit_deletions(&[second.document_id]).unwrap();
+ assert_eq!(writer.source_bytes, 11);
+ assert_eq!(writer.chunk_count, 1);
+ }
+
#[test]
fn rejects_collections_over_capacity_before_embedding() {
let inventory = (0..=NOTES_MAX_COLLECTION_DOCUMENTS)
diff --git a/web/apps/ensu/package.json b/web/apps/ensu/package.json
index 9988398020..ddd4fd1561 100644
--- a/web/apps/ensu/package.json
+++ b/web/apps/ensu/package.json
@@ -2,9 +2,6 @@
"name": "ensu",
"version": "0.0.0",
"private": true,
- "scripts": {
- "test": "vitest --run"
- },
"dependencies": {
"@hugeicons/core-free-icons": "1.2.1",
"@hugeicons/react": "1.1.6",
@@ -27,7 +24,6 @@
"remark-math": "6.0.0"
},
"devDependencies": {
- "ente-build-config": "*",
- "vitest": "4.1.8"
+ "ente-build-config": "*"
}
}
diff --git a/web/apps/ensu/src/components/chat/ChatDialogs.tsx b/web/apps/ensu/src/components/chat/ChatDialogs.tsx
index 1d1179677d..70b2fbb6f1 100644
--- a/web/apps/ensu/src/components/chat/ChatDialogs.tsx
+++ b/web/apps/ensu/src/components/chat/ChatDialogs.tsx
@@ -939,7 +939,16 @@ export const ChatDialogs = memo(
{collection.lastError}
</Typography>
)}
- {canRetry && (
+ {collection.watcherError && (
+ <Typography
+ variant="small"
+ sx={{ color: "text.muted" }}
+ >
+ {collection.watcherError}
+ </Typography>
+ )}
+ {(canRetry ||
+ collection.watcherError) && (
<Button
size="small"
onClick={() =>
@@ -950,7 +959,9 @@ export const ChatDialogs = memo(
}
sx={{ alignSelf: "flex-start" }}
>
- Try again
+ {canRetry
+ ? "Try again"
+ : "Refresh"}
</Button>
)}
</Stack>
diff --git a/web/apps/ensu/src/services/notes.ts b/web/apps/ensu/src/services/notes.ts
index e3b1cdd42c..d6edceb580 100644
--- a/web/apps/ensu/src/services/notes.ts
+++ b/web/apps/ensu/src/services/notes.ts
@@ -16,6 +16,7 @@ export interface NotesCollection {
lastUpdatedAtMs: number | null;
updateDueAtMs: number | null;
lastError: string | null;
+ watcherError: string | null;
}
const invokeNotes = async <T>(
diff --git a/web/apps/ensu/tests/notes-lifecycle.test.ts b/web/apps/ensu/tests/notes-lifecycle.test.ts
deleted file mode 100644
index 8206367461..0000000000
--- a/web/apps/ensu/tests/notes-lifecycle.test.ts
+++ /dev/null
@@ -1,31 +0,0 @@
-import { describe, expect, test } from "vitest";
-import { NotesLifecycleController } from "../src/services/notes-lifecycle";
-
-describe("NotesLifecycleController", () => {
- test("recovers cancelled indexing without duplicating or losing queued work", () => {
- const controller = new NotesLifecycleController();
-
- expect(controller.ensureIndex("notes", false)).toBe("start");
- expect(controller.indexActivity("notes")).toBe("starting");
- controller.queueIndex("notes", false);
- expect(controller.finishIndex("notes")).toEqual({
- clearedRemovalWait: false,
- });
- expect(controller.indexActivity("notes")).toBe("queued");
- expect(controller.ensureIndex("notes", false)).toBe("ignored");
- expect(controller.drainQueued(true)).toEqual([]);
- expect(controller.drainQueued(false)).toEqual([["notes", false]]);
-
- expect(controller.requestIndex("notes", false, false)).toBe("start");
- expect(controller.requestIndex("notes", true, false)).toBe("queued");
- controller.finishIndex("notes");
- expect(controller.drainQueued(false)).toEqual([["notes", true]]);
-
- expect(controller.requestIndex("notes", true, false)).toBe("start");
- controller.failIndex("notes");
- controller.finishIndex("notes");
- expect(controller.indexActivity("notes")).toBe("failed");
- expect(controller.ensureIndex("notes", false)).toBe("ignored");
- expect(controller.requestIndex("notes", true, false)).toBe("start");
- });
-});
diff --git a/web/apps/ensu/tsconfig.json b/web/apps/ensu/tsconfig.json
index 6a2a667238..2ff1f734c6 100644
--- a/web/apps/ensu/tsconfig.json
+++ b/web/apps/ensu/tsconfig.json
@@ -9,7 +9,6 @@
},
"include": [
"src",
- "tests",
"next-env.d.ts",
"next.config.mjs",
"../../packages/base/global-electron.d.ts",
diff --git a/web/package-lock.json b/web/package-lock.json
index 7df478b760..4e037b9968 100644
--- a/web/package-lock.json
+++ b/web/package-lock.json
@@ -191,8 +191,7 @@
"remark-math": "6.0.0"
},
"devDependencies": {
- "ente-build-config": "*",
- "vitest": "4.1.8"
+ "ente-build-config": "*"
}
},
"apps/ensu/node_modules/katex": {
diff --git a/web/package.json b/web/package.json
index 11ea951759..3bee38bffc 100644
--- a/web/package.json
+++ b/web/package.json
@@ -52,7 +52,7 @@
"dev:twoof3": "npm exec --workspace twoof3 -- next dev --webpack -p 3012",
"lint": "concurrently --names 'prettier,eslint,tsc,wasm-imports' \"prettier --check --log-level warn .\" \"npm exec --workspaces -- eslint --max-warnings 0\" \"npm exec --workspaces -- tsc\" \"npm run check:wasm-imports\"",
"lint:fix": "concurrently --names 'prettier,eslint,tsc' \"prettier --write --log-level warn .\" \"npm exec --workspaces -- eslint --fix --max-warnings 0\" \"npm exec --workspaces -- tsc\"",
- "test": "npm run test --workspace ensu && npm run test --workspace ente-core-wasm && npm run test --workspace ente-paste-wasm && npm run test --workspace ente-contacts && npm run test --workspace locker && npm run test --workspace photos"
+ "test": "npm run test --workspace ente-core-wasm && npm run test --workspace ente-paste-wasm && npm run test --workspace ente-contacts && npm run test --workspace locker && npm run test --workspace photos"
},
"overrides": {
"@types/react": "19.2.17",