π End-to-end encrypted cloud for everything.
Dart 2faandroidauthye2eeencryptionend-to-end-encryptionfluttergolanggoogle-photosgoogle-photos-alternativeiosopensourcephotosprivacyprivacy-toolstypescriptzero-knowledge
use std::collections::{BTreeSet, HashMap};
#[cfg(unix)]
use std::fs::TryLockError;
use std::fs::{self, File, OpenOptions};
#[cfg(unix)]
use std::os::unix::fs::{OpenOptionsExt, PermissionsExt};
#[cfg(windows)]
use std::os::windows::fs::OpenOptionsExt;
use std::path::{Path, PathBuf};
use std::sync::atomic::Ordering;
use std::sync::{Arc, Mutex, PoisonError};
use std::time::{Instant, SystemTime, UNIX_EPOCH};
use ente_ensu::model;
use ente_ensu::notes::{
NoteSourceReference, NotesCollectionIndex, NotesError, NotesIndexOutcome, NotesIndexProgress,
cleanup_unreferenced_notes_shards, notes_content_revision, remove_notes_collection,
};
use notify::RecommendedWatcher;
use serde::Serialize;
use tauri::async_runtime;
use tauri::{AppHandle, Manager, State as TauriState};
use tauri_plugin_fs::FsExt;
use tauri_plugin_opener::OpenerExt;
use uuid::Uuid;
#[cfg(windows)]
use windows_sys::Win32::Foundation::ERROR_SHARING_VIOLATION;
use crate::commands::common::ApiError;
use crate::commands::llm::ModelDownloadState;
mod indexing;
mod registry;
mod source;
mod watcher;
use indexing::{index_collection, should_rebuild_derived_index};
use registry::{RegisteredCollection, RegistryStore};
use source::{
canonical_source_root, cleanup_unregistered_indexes, compact_source_location,
is_supported_file, read_collection_source, source_root_is_available, validate_new_source_root,
};
pub use watcher::initialize_for_app;
use watcher::{emit_state_changed, ensure_collection_watcher, install_collection_watcher};
pub(crate) use watcher::{mark_collection_dirty, mark_reference_stale};
const NOTES_DIRECTORY: &str = "notes";
const NOTES_INDEX_DIRECTORY: &str = "indexes";
const NOTES_OWNERSHIP_LOCK_FILE: &str = ".owner.lock";
const NOTES_QUIET_PERIOD_MS: i64 = 5 * 60 * 1_000;
const NOTES_STATE_CHANGED_EVENT: &str = "notes-state-changed";
const NOTES_MAX_SCAN_ENTRIES: usize = 250_000;
const NOTES_MAX_WATCH_DIRECTORIES: usize = 20_000;
const NOTES_INDEX_CACHE_MAX_BYTES: usize = 256 * 1024 * 1024;
#[cfg(test)]
struct TestDirectory(PathBuf);
#[cfg(test)]
impl TestDirectory {
fn new() -> Self {
let path = std::env::temp_dir().join(format!("ensu-notes-{}", Uuid::new_v4()));
fs::create_dir(&path).unwrap();
Self(path)
}
fn path(&self) -> &Path {
&self.0
}
}
#[cfg(test)]
impl Drop for TestDirectory {
fn drop(&mut self) {
let _ = fs::remove_dir_all(&self.0);
}
}
#[derive(Debug, Clone, Copy, Serialize, PartialEq, Eq)]
#[serde(rename_all = "camelCase")]
pub enum NotesCollectionStatusDto {
Indexing,
Updating,
Ready,
Pending,
Unavailable,
Error,
}
#[derive(Debug, Clone)]
struct CollectionRuntime {
status: NotesCollectionStatusDto,
index_available: bool,
indexing_progress: Option<u8>,
indexed_document_count: u64,
last_updated_at_ms: Option<i64>,
last_error: Option<String>,
watcher_error: Option<String>,
}
impl CollectionRuntime {
fn pending() -> Self {
Self {
status: NotesCollectionStatusDto::Pending,
index_available: false,
indexing_progress: None,
indexed_document_count: 0,
last_updated_at_ms: None,
last_error: None,
watcher_error: None,
}
}
fn unavailable() -> Self {
Self {
status: NotesCollectionStatusDto::Unavailable,
last_error: Some("Source folder is unavailable".to_string()),
..Self::pending()
}
}
}
#[derive(Debug, Clone, Serialize)]
#[serde(rename_all = "camelCase")]
pub struct NotesCollectionDto {
id: String,
label: String,
status: NotesCollectionStatusDto,
indexing_progress: Option<u8>,
indexed_document_count: u64,
last_updated_at_ms: Option<i64>,
update_due_at_ms: Option<i64>,
last_error: Option<String>,
watcher_error: Option<String>,
}
#[derive(Debug, Serialize)]
#[serde(rename_all = "camelCase")]
pub struct NotesIndexResultDto {
collection: NotesCollectionDto,
needs_rerun: bool,
}
struct CollectionWatcher {
watcher: RecommendedWatcher,
watched_directories: BTreeSet<PathBuf>,
}
struct CachedIndex {
index: NotesCollectionIndex,
bytes: usize,
last_used: u64,
}
struct IndexCache {
entries: HashMap<String, CachedIndex>,
max_bytes: usize,
total_bytes: usize,
clock: u64,
}
impl IndexCache {
fn new(max_bytes: usize) -> Self {
Self {
entries: HashMap::new(),
max_bytes,
total_bytes: 0,
clock: 0,
}
}
fn get(&mut self, collection_id: &str) -> Option<&NotesCollectionIndex> {
self.clock = self.clock.wrapping_add(1);
let entry = self.entries.get_mut(collection_id)?;
entry.last_used = self.clock;
Some(&entry.index)
}
fn insert(&mut self, collection_id: String, index: NotesCollectionIndex) {
self.remove(&collection_id);
let bytes = index.estimated_heap_bytes();
if bytes > self.max_bytes {
return;
}
while self.total_bytes.saturating_add(bytes) > self.max_bytes {
let Some(oldest) = self
.entries
.iter()
.min_by_key(|(_, entry)| entry.last_used)
.map(|(collection_id, _)| collection_id.clone())
else {
break;
};
self.remove(&oldest);
}
self.clock = self.clock.wrapping_add(1);
self.total_bytes = self.total_bytes.saturating_add(bytes);
self.entries.insert(
collection_id,
CachedIndex {
index,
bytes,
last_used: self.clock,
},
);
}
fn remove(&mut self, collection_id: &str) {
if let Some(entry) = self.entries.remove(collection_id) {
self.total_bytes = self.total_bytes.saturating_sub(entry.bytes);
}
}
fn clear(&mut self) {
self.entries.clear();
self.total_bytes = 0;
}
}
#[derive(Default)]
struct DesktopUpdateState {
forced_document_ids: BTreeSet<String>,
force_full_hash: bool,
rebuild_derived_index: bool,
quiet_deadline_ms: i64,
generation: u64,
bypass_quiet_period: bool,
}
impl DesktopUpdateState {
fn record_change(
&mut self,
now_ms: i64,
forced_document_ids: impl IntoIterator<Item = String>,
force_full_hash: bool,
) {
self.forced_document_ids.extend(forced_document_ids);
self.force_full_hash |= force_full_hash;
self.quiet_deadline_ms = if self.rebuild_derived_index {
now_ms
} else {
now_ms.saturating_add(NOTES_QUIET_PERIOD_MS)
};
self.generation = self.generation.wrapping_add(1);
self.bypass_quiet_period = false;
}
fn record_stale_reference(&mut self, now_ms: i64, document_id: String) -> bool {
let deadline_changed = self.quiet_deadline_ms > now_ms;
let document_changed =
!self.force_full_hash && self.forced_document_ids.insert(document_id);
if !deadline_changed && !document_changed {
return false;
}
if document_changed {
self.generation = self.generation.wrapping_add(1);
}
self.quiet_deadline_ms = now_ms;
self.bypass_quiet_period = false;
true
}
fn record_unreadable_index(&mut self, now_ms: i64) -> bool {
if self.rebuild_derived_index {
return false;
}
self.rebuild_derived_index = true;
self.quiet_deadline_ms = now_ms;
self.generation = self.generation.wrapping_add(1);
self.bypass_quiet_period = false;
true
}
}
#[derive(Default)]
struct UpdateSnapshot {
generation: u64,
forced_document_ids: Vec<String>,
force_full_hash: bool,
rebuild_derived_index: bool,
}
pub struct State {
registry: Arc<Mutex<RegistryStore>>,
runtimes: Arc<Mutex<HashMap<String, CollectionRuntime>>>,
cached_indexes: Arc<Mutex<IndexCache>>,
index_root: PathBuf,
collection_lifecycles: Mutex<HashMap<String, Arc<async_runtime::Mutex<()>>>>,
watchers: Mutex<HashMap<String, CollectionWatcher>>,
watch_work: Mutex<HashMap<String, watcher::WatchWork>>,
updates: Mutex<HashMap<String, DesktopUpdateState>>,
_ownership_lock: File,
}
#[derive(Clone)]
pub(crate) struct RetrievalHandle {
registry: Arc<Mutex<RegistryStore>>,
cached_indexes: Arc<Mutex<IndexCache>>,
index_root: PathBuf,
}
impl State {
pub fn new(app_data_dir: PathBuf) -> Result<Self, ApiError> {
let notes_directory = app_data_dir.join(NOTES_DIRECTORY);
ensure_directory(¬es_directory)?;
let ownership_lock = acquire_notes_ownership(¬es_directory)?;
let index_root = notes_directory.join(NOTES_INDEX_DIRECTORY);
ensure_directory(&index_root)?;
let registry = RegistryStore::open(notes_directory)?;
if registry.incompatible_error.is_none() {
cleanup_unregistered_indexes(&index_root, ®istry.registry.collections);
}
let mut runtimes = HashMap::new();
for collection in ®istry.registry.collections {
let runtime = if !collection.source_root.is_dir() {
CollectionRuntime::unavailable()
} else {
match NotesCollectionIndex::inspect(&index_root, collection.id.clone()) {
Ok(summary) => {
let count = summary.document_count;
if count == 0 {
CollectionRuntime {
status: NotesCollectionStatusDto::Error,
last_error: Some(
"No supported non-empty UTF-8 notes were found".to_string(),
),
..CollectionRuntime::pending()
}
} else {
CollectionRuntime {
status: NotesCollectionStatusDto::Ready,
index_available: true,
indexed_document_count: count as u64,
last_updated_at_ms: summary.last_updated_at_ms,
..CollectionRuntime::pending()
}
}
}
Err(
NotesError::NotReady | NotesError::IncompatibleIndex | NotesError::Io(_),
) => CollectionRuntime::pending(),
Err(_) => CollectionRuntime {
status: NotesCollectionStatusDto::Error,
last_error: Some("The local index needs to be rebuilt".to_string()),
..CollectionRuntime::pending()
},
}
};
runtimes.insert(collection.id.clone(), runtime);
}
Ok(Self {
registry: Arc::new(Mutex::new(registry)),
runtimes: Arc::new(Mutex::new(runtimes)),
cached_indexes: Arc::new(Mutex::new(IndexCache::new(NOTES_INDEX_CACHE_MAX_BYTES))),
index_root,
collection_lifecycles: Mutex::new(HashMap::new()),
watchers: Mutex::new(HashMap::new()),
watch_work: Mutex::new(HashMap::new()),
updates: Mutex::new(HashMap::new()),
_ownership_lock: ownership_lock,
})
}
fn collection_lifecycle(&self, collection_id: &str) -> Arc<async_runtime::Mutex<()>> {
Arc::clone(
self.collection_lifecycles
.lock()
.unwrap_or_else(PoisonError::into_inner)
.entry(collection_id.to_string())
.or_default(),
)
}
fn registered_collection(&self, collection_id: &str) -> Result<RegisteredCollection, ApiError> {
let store = self.registry.lock().unwrap_or_else(PoisonError::into_inner);
store.ensure_compatible()?;
store
.registry
.collections
.iter()
.find(|collection| collection.id == collection_id)
.cloned()
.ok_or_else(|| ApiError::new("not_found", "Notes collection was not found"))
}
fn collection_dto(&self, collection_id: &str) -> Result<NotesCollectionDto, ApiError> {
let collection = self.registered_collection(collection_id)?;
let runtime = self
.runtimes
.lock()
.unwrap_or_else(PoisonError::into_inner)
.get(collection_id)
.cloned()
.unwrap_or_else(CollectionRuntime::pending);
let update_due_at_ms = self
.updates
.lock()
.unwrap_or_else(PoisonError::into_inner)
.get(collection_id)
.map(|update| update.quiet_deadline_ms);
Ok(collection_dto(&collection, &runtime, update_due_at_ms))
}
fn set_runtime(&self, collection_id: &str, runtime: CollectionRuntime) {
self.runtimes
.lock()
.unwrap_or_else(PoisonError::into_inner)
.insert(collection_id.to_string(), runtime);
}
fn runtime(&self, collection_id: &str) -> CollectionRuntime {
self.runtimes
.lock()
.unwrap_or_else(PoisonError::into_inner)
.get(collection_id)
.cloned()
.unwrap_or_else(CollectionRuntime::pending)
}
fn set_watcher_error(&self, collection_id: &str, error: Option<&ApiError>) -> bool {
let mut runtimes = self.runtimes.lock().unwrap_or_else(PoisonError::into_inner);
let Some(runtime) = runtimes.get_mut(collection_id) else {
return false;
};
let watcher_error = error.map(|_| {
"Changes cannot be watched automatically. Refresh this folder manually.".to_string()
});
if runtime.watcher_error == watcher_error {
return false;
}
runtime.watcher_error = watcher_error;
true
}
fn prepare_update(
&self,
collection_id: &str,
force: bool,
wait_for_quiet_period: bool,
) -> Result<UpdateSnapshot, ApiError> {
let mut updates = self.updates.lock().unwrap_or_else(PoisonError::into_inner);
let Some(update) = updates.get_mut(collection_id) else {
return Ok(UpdateSnapshot {
force_full_hash: force,
..UpdateSnapshot::default()
});
};
if force {
update.bypass_quiet_period = true;
}
if !force
&& wait_for_quiet_period
&& !update.bypass_quiet_period
&& update.quiet_deadline_ms > now_ms()
{
return Err(ApiError::new(
"not_due",
"Notes update is waiting for the quiet period",
));
}
Ok(UpdateSnapshot {
generation: update.generation,
forced_document_ids: update.forced_document_ids.iter().cloned().collect(),
force_full_hash: force || update.force_full_hash,
rebuild_derived_index: update.rebuild_derived_index,
})
}
fn cancel_update(&self, collection_id: &str) {
self.updates
.lock()
.unwrap_or_else(PoisonError::into_inner)
.remove(collection_id);
}
fn evict_cached_index(&self, collection_id: &str) {
self.cached_indexes
.lock()
.unwrap_or_else(PoisonError::into_inner)
.remove(collection_id);
}
fn has_available_index(&self, collection_id: &str) -> bool {
self.runtimes
.lock()
.unwrap_or_else(PoisonError::into_inner)
.get(collection_id)
.is_some_and(|runtime| runtime.index_available)
}
fn has_newer_update(&self, collection_id: &str, generation: u64) -> bool {
self.updates
.lock()
.unwrap_or_else(PoisonError::into_inner)
.get(collection_id)
.is_some_and(|update| update.generation != generation)
}
fn retain_unchecked_hints(
&self,
collection_id: &str,
generation: u64,
unchecked_document_ids: &[String],
) {
let mut updates = self.updates.lock().unwrap_or_else(PoisonError::into_inner);
let Some(update) = updates.get_mut(collection_id) else {
return;
};
if update.generation != generation {
return;
}
update.forced_document_ids = unchecked_document_ids.iter().cloned().collect();
update.force_full_hash = false;
}
fn apply_index_outcome(
&self,
collection_id: &str,
update_generation: u64,
outcome: &NotesIndexOutcome,
had_index: bool,
completed_index_updated_at_ms: Option<i64>,
) -> (bool, bool) {
let mut updates = self.updates.lock().unwrap_or_else(PoisonError::into_inner);
let generation_unchanged = updates
.get(collection_id)
.is_none_or(|update| update.generation == update_generation);
let reconciliation_complete = outcome.changed_during_indexing.is_none();
let ready = outcome.ready();
let update_completed = reconciliation_complete && generation_unchanged;
if update_completed {
updates.remove(collection_id);
}
let needs_rerun = !had_index && !generation_unchanged;
let mut runtime = self.runtime(collection_id);
if update_completed {
runtime.index_available = ready;
runtime.last_updated_at_ms = ready.then_some(completed_index_updated_at_ms).flatten();
}
runtime.indexing_progress = needs_rerun.then_some(runtime.indexing_progress).flatten();
runtime.indexed_document_count = outcome.indexed_document_count;
runtime.status = if update_completed && ready {
NotesCollectionStatusDto::Ready
} else if update_completed {
NotesCollectionStatusDto::Error
} else if needs_rerun {
if had_index {
NotesCollectionStatusDto::Updating
} else {
NotesCollectionStatusDto::Indexing
}
} else {
NotesCollectionStatusDto::Pending
};
runtime.last_error = (update_completed && !ready)
.then(|| "No supported non-empty UTF-8 notes were found".to_string());
self.set_runtime(collection_id, runtime);
(needs_rerun, update_completed)
}
fn report_indexing_progress(
app: &AppHandle,
collection_id: &str,
progress: NotesIndexProgress,
) {
let state = app.state::<State>();
let mut runtime = state.runtime(collection_id);
let previous_progress = runtime.indexing_progress;
if !matches!(
runtime.status,
NotesCollectionStatusDto::Indexing | NotesCollectionStatusDto::Updating
) || previous_progress == Some(progress.percentage)
{
return;
}
runtime.indexing_progress = Some(progress.percentage);
runtime.indexed_document_count = progress.indexed_document_count;
state.set_runtime(collection_id, runtime);
if progress.percentage == 100
|| previous_progress.is_none_or(|previous| previous / 10 != progress.percentage / 10)
{
crate::logging::log(
"Notes",
format!(
"index progress collection={collection_id} percent={} processed={} indexed={} total={}",
progress.percentage,
progress.processed_document_count,
progress.indexed_document_count,
progress.total_document_count
),
);
}
emit_state_changed(app, collection_id);
}
pub(crate) fn available_index_collection_ids(&self) -> Vec<String> {
let available_ids = self
.runtimes
.lock()
.unwrap_or_else(PoisonError::into_inner)
.iter()
.filter_map(|(collection_id, runtime)| {
(runtime.index_available
&& !matches!(
runtime.status,
NotesCollectionStatusDto::Unavailable | NotesCollectionStatusDto::Error
))
.then_some(collection_id.clone())
})
.collect::<BTreeSet<_>>();
let collections = self
.registry
.lock()
.unwrap_or_else(PoisonError::into_inner)
.registry
.collections
.clone();
collections
.into_iter()
.filter(|collection| available_ids.contains(&collection.id))
.map(|collection| collection.id)
.collect()
}
pub(crate) fn retrieval_handle(&self) -> RetrievalHandle {
RetrievalHandle {
registry: Arc::clone(&self.registry),
cached_indexes: Arc::clone(&self.cached_indexes),
index_root: self.index_root.clone(),
}
}
}
pub(crate) fn mark_index_unreadable(app: &AppHandle, collection_id: &str) {
let state = app.state::<State>();
if state.registered_collection(collection_id).is_err() {
return;
}
state.evict_cached_index(collection_id);
{
let mut updates = state.updates.lock().unwrap_or_else(PoisonError::into_inner);
updates
.entry(collection_id.to_string())
.or_default()
.record_unreadable_index(now_ms());
}
{
let mut runtimes = state
.runtimes
.lock()
.unwrap_or_else(PoisonError::into_inner);
let Some(runtime) = runtimes.get_mut(collection_id) else {
return;
};
runtime.status = NotesCollectionStatusDto::Pending;
runtime.index_available = false;
runtime.indexing_progress = None;
runtime.last_error = None;
}
emit_state_changed(app, collection_id);
}
impl RetrievalHandle {
pub(crate) fn collection_label(&self, collection_id: &str) -> Option<String> {
self.registry
.lock()
.unwrap_or_else(PoisonError::into_inner)
.registry
.collections
.iter()
.find(|collection| collection.id == collection_id)
.map(|collection| compact_source_location(&collection.source_root))
}
pub(crate) fn search_collection(
&self,
collection_id: &str,
query: &[f32],
) -> Result<Vec<ente_ensu::notes::NotesSearchHit>, NotesError> {
self.with_index(collection_id, |index| index.search(query))
}
pub(crate) fn reload_passage(
&self,
locator: &ente_ensu::notes::NotePassageLocator,
) -> Result<Option<ente_ensu::notes::NotesSearchHit>, NotesError> {
self.with_index(&locator.collection_id, |index| {
index.reload_passage(locator)
})
}
fn with_index<T>(
&self,
collection_id: &str,
read: impl FnOnce(&NotesCollectionIndex) -> Result<T, NotesError>,
) -> Result<T, NotesError> {
if let Some(index) = self
.cached_indexes
.lock()
.unwrap_or_else(PoisonError::into_inner)
.get(collection_id)
{
return read(index);
}
let index = NotesCollectionIndex::open(&self.index_root, collection_id.to_owned())?;
let result = read(&index)?;
self.cached_indexes
.lock()
.unwrap_or_else(PoisonError::into_inner)
.insert(collection_id.to_owned(), index);
Ok(result)
}
pub(crate) fn verify_source_reference(&self, reference: &NoteSourceReference) -> bool {
let collection = self
.registry
.lock()
.unwrap_or_else(PoisonError::into_inner)
.registry
.collections
.iter()
.find(|collection| collection.id == reference.collection_id)
.cloned();
let Some(collection) = collection else {
return false;
};
read_collection_source(&collection.source_root, &reference.document_id)
.map(|(_, bytes, _)| bytes)
.map(|bytes| notes_content_revision(&bytes) == reference.indexed_revision)
.unwrap_or(false)
}
}
fn collection_dto(
collection: &RegisteredCollection,
runtime: &CollectionRuntime,
update_due_at_ms: Option<i64>,
) -> NotesCollectionDto {
NotesCollectionDto {
id: collection.id.clone(),
label: compact_source_location(&collection.source_root),
status: runtime.status,
indexing_progress: runtime.indexing_progress,
indexed_document_count: runtime.indexed_document_count,
last_updated_at_ms: runtime.last_updated_at_ms,
update_due_at_ms,
last_error: runtime.last_error.clone(),
watcher_error: runtime.watcher_error.clone(),
}
}
#[tauri::command]
pub fn notes_list_collections(app: AppHandle) -> Result<Vec<NotesCollectionDto>, ApiError> {
let notes_state = available_state(&app)?;
let store = notes_state
.registry
.lock()
.unwrap_or_else(PoisonError::into_inner);
store.ensure_compatible()?;
let collections = store.registry.collections.clone();
drop(store);
let runtimes = notes_state
.runtimes
.lock()
.unwrap_or_else(PoisonError::into_inner)
.clone();
let updates = notes_state
.updates
.lock()
.unwrap_or_else(PoisonError::into_inner);
Ok(collections
.iter()
.map(|collection| {
collection_dto(
collection,
&runtimes
.get(&collection.id)
.cloned()
.unwrap_or_else(CollectionRuntime::pending),
updates
.get(&collection.id)
.map(|update| update.quiet_deadline_ms),
)
})
.collect())
}
#[tauri::command]
pub async fn notes_add_collection(
app: AppHandle,
source_root: String,
) -> Result<NotesCollectionDto, ApiError> {
async_runtime::spawn_blocking(move || {
let root = canonical_source_root(Path::new(&source_root))?;
if !app.fs_scope().is_allowed(&root) {
return Err(ApiError::new(
"io_scope",
"Notes folder is outside the allowed filesystem scope",
));
}
let notes_state = available_state(&app)?;
register_collection(&app, ¬es_state, root)
})
.await
.map_err(|_| ApiError::new("io_thread", "Notes folder registration task failed"))?
}
fn register_collection(
app: &AppHandle,
notes_state: &State,
root: PathBuf,
) -> Result<NotesCollectionDto, ApiError> {
let notes_directory = notes_state
.index_root
.parent()
.ok_or_else(|| ApiError::new("notes_storage", "Notes storage path is invalid"))?;
let notes_directory = fs::canonicalize(notes_directory)
.map_err(|_| ApiError::new("notes_storage", "Notes storage path is unavailable"))?;
if root.starts_with(¬es_directory) || notes_directory.starts_with(&root) {
return Err(ApiError::new(
"invalid_folder",
"Choose a folder outside Ensu's private Notes storage",
));
}
let collection = RegisteredCollection {
id: Uuid::new_v4().hyphenated().to_string(),
source_root: root,
};
{
let mut store = notes_state
.registry
.lock()
.unwrap_or_else(PoisonError::into_inner);
validate_new_source_root(&store.registry.collections, &collection.source_root)?;
let mut registry = store.registry.clone();
registry.collections.push(collection.clone());
store.replace(registry)?;
}
notes_state.set_runtime(&collection.id, CollectionRuntime::pending());
if let Err(error) = install_collection_watcher(app, &collection) {
notes_state.set_watcher_error(&collection.id, Some(&error));
crate::logging::log(
"Notes",
format!(
"watcher unavailable collection={} error={}",
collection.id, error.message
),
);
}
let dto = notes_state.collection_dto(&collection.id)?;
crate::logging::log(
"Notes",
format!("collection registered collection={}", collection.id),
);
Ok(dto)
}
#[tauri::command]
pub async fn notes_remove_collection(
app: AppHandle,
collection_id: String,
) -> Result<(), ApiError> {
let notes_state = available_state(&app)?;
let lifecycle = notes_state.collection_lifecycle(&collection_id);
let _lifecycle = lifecycle.lock().await;
async_runtime::spawn_blocking(move || {
let notes_state = available_state(&app)?;
remove_collection(¬es_state, &collection_id)
})
.await
.map_err(|_| ApiError::new("io_thread", "Notes folder removal task failed"))?
}
fn remove_collection(notes_state: &State, collection_id: &str) -> Result<(), ApiError> {
let collection = notes_state.registered_collection(collection_id)?;
{
let mut store = notes_state
.registry
.lock()
.unwrap_or_else(PoisonError::into_inner);
let mut registry = store.registry.clone();
registry
.collections
.retain(|candidate| candidate.id != collection_id);
store.replace(registry)?;
}
notes_state
.watchers
.lock()
.unwrap_or_else(PoisonError::into_inner)
.remove(collection_id);
notes_state
.watch_work
.lock()
.unwrap_or_else(PoisonError::into_inner)
.remove(collection_id);
notes_state.cancel_update(collection_id);
notes_state.evict_cached_index(collection_id);
notes_state
.runtimes
.lock()
.unwrap_or_else(PoisonError::into_inner)
.remove(collection_id);
if let Err(error) = remove_notes_collection(¬es_state.index_root, &collection.id) {
crate::logging::log(
"Notes",
format!(
"removed collection but could not clean derived index collection={collection_id} error={error}"
),
);
}
notes_state
.collection_lifecycles
.lock()
.unwrap_or_else(PoisonError::into_inner)
.remove(collection_id);
crate::logging::log(
"Notes",
format!("collection removed collection={collection_id}"),
);
Ok(())
}
fn report_index_error(
app: &AppHandle,
state: &State,
collection_id: &str,
update_generation: Option<u64>,
error: &ApiError,
) {
let cancelled = error.name == Some("cancelled");
let unavailable = matches!(
error.name,
Some("unavailable" | "source_unavailable" | "source_changed")
);
let superseded_file_error = error.name == Some("source_unavailable")
&& update_generation.is_some_and(|generation| {
state.has_newer_update(collection_id, generation)
&& state
.registered_collection(collection_id)
.is_ok_and(|collection| source_root_is_available(&collection.source_root))
});
let mut runtime = state.runtime(collection_id);
if !cancelled {
runtime.indexing_progress = None;
}
runtime.status = if cancelled || superseded_file_error {
NotesCollectionStatusDto::Pending
} else if unavailable {
NotesCollectionStatusDto::Unavailable
} else {
NotesCollectionStatusDto::Error
};
runtime.last_error = (!cancelled && !superseded_file_error).then(|| error.message.clone());
state.set_runtime(collection_id, runtime);
crate::logging::log(
"Notes",
format!(
"index stopped collection={collection_id} reason={}",
error.name.unwrap_or("unknown")
),
);
emit_state_changed(app, collection_id);
}
#[tauri::command]
pub async fn notes_index_collection(
app: AppHandle,
model_state: TauriState<'_, ModelDownloadState>,
llm_state: TauriState<'_, crate::commands::llm::State>,
collection_id: String,
force: bool,
retrieval_epoch: u64,
) -> Result<NotesIndexResultDto, ApiError> {
let request_started_at = Instant::now();
crate::logging::log(
"Notes",
format!("index requested collection={collection_id} force={force}"),
);
let notes_state = available_state(&app)?;
let lifecycle = notes_state.collection_lifecycle(&collection_id);
let _collection_lifecycle = lifecycle.lock().await;
let collection = notes_state.registered_collection(&collection_id)?;
let watcher_app = app.clone();
let watcher_collection = collection.clone();
let watcher_result = async_runtime::spawn_blocking(move || {
ensure_collection_watcher(&watcher_app, &watcher_collection)
})
.await
.unwrap_or_else(|_| Err(ApiError::new("watcher_thread", "Notes watcher task failed")));
match watcher_result {
Ok(()) => {
notes_state.set_watcher_error(&collection_id, None);
}
Err(error) => {
notes_state.set_watcher_error(&collection_id, Some(&error));
crate::logging::log(
"Notes",
format!(
"watcher unavailable collection={collection_id} error={}",
error.message
),
);
}
}
let had_index = notes_state.has_available_index(&collection_id);
let rebuild_derived_index =
should_rebuild_derived_index(¬es_state.index_root, &collection_id, force, had_index)?;
let mut update = notes_state.prepare_update(&collection_id, force, had_index)?;
update.rebuild_derived_index |= rebuild_derived_index;
let cancellation_epoch = llm_state.retrieval_epoch();
if let Err(error) = check_cancelled(&cancellation_epoch, retrieval_epoch) {
crate::logging::log(
"Notes",
format!("index cancelled before start collection={collection_id}"),
);
return Err(error);
}
let _model_lifecycle = llm_state.lifecycle().lock().await;
if let Err(error) = check_cancelled(&cancellation_epoch, retrieval_epoch) {
crate::logging::log(
"Notes",
format!("index cancelled while waiting collection={collection_id}"),
);
return Err(error);
}
crate::commands::llm::replace_state(&llm_state, None, None)?;
let mut runtime = notes_state.runtime(&collection_id);
if !matches!(
runtime.status,
NotesCollectionStatusDto::Indexing | NotesCollectionStatusDto::Updating
) && runtime.indexing_progress.is_none()
{
runtime.indexing_progress = Some(0);
}
runtime.status = if had_index {
NotesCollectionStatusDto::Updating
} else {
NotesCollectionStatusDto::Indexing
};
runtime.last_error = None;
notes_state.set_runtime(&collection_id, runtime);
crate::logging::log(
"Notes",
format!(
"index started collection={collection_id} updating={had_index} rebuild={rebuild_derived_index}"
),
);
emit_state_changed(&app, &collection_id);
let store = model_state.store();
let embedding_asset = model::knowledge_embedding_model_asset();
let embedding_path = model::llm_model_path(&store, &embedding_asset);
let index_root = notes_state.index_root.clone();
let update_generation = update.generation;
let progress_app = app.clone();
let progress_collection_id = collection_id.clone();
let result = async_runtime::spawn_blocking(move || {
index_collection(
&index_root,
&collection,
embedding_path.as_deref(),
&cancellation_epoch,
retrieval_epoch,
&update,
|progress| {
State::report_indexing_progress(&progress_app, &progress_collection_id, progress);
},
)
})
.await
.map_err(|_| ApiError::new("index_thread", "Notes indexing task failed"))
.and_then(|result| result);
let outcome = match result {
Ok(outcome) => outcome,
Err(error) => {
report_index_error(
&app,
¬es_state,
&collection_id,
Some(update_generation),
&error,
);
return Err(error);
}
};
let opened_index = if outcome.ready() {
notes_state.evict_cached_index(&collection_id);
match NotesCollectionIndex::open(¬es_state.index_root, collection_id.clone()) {
Ok(index) => {
let _ = cleanup_unreferenced_notes_shards(¬es_state.index_root, &collection_id);
Some(index)
}
Err(error) => {
let error = notes_error(error);
report_index_error(
&app,
¬es_state,
&collection_id,
Some(update_generation),
&error,
);
return Err(error);
}
}
} else {
None
};
notes_state.retain_unchecked_hints(
&collection_id,
update_generation,
&outcome.unchecked_document_ids,
);
if let Some(document_id) = outcome.changed_during_indexing.clone() {
mark_collection_dirty(&app, &collection_id, [document_id], false);
}
let completed_index_updated_at_ms = opened_index
.as_ref()
.and_then(NotesCollectionIndex::last_updated_at_ms);
let (needs_rerun, update_completed) = notes_state.apply_index_outcome(
&collection_id,
update_generation,
&outcome,
had_index,
completed_index_updated_at_ms,
);
if update_completed {
if let Some(index) = opened_index {
notes_state
.cached_indexes
.lock()
.unwrap_or_else(PoisonError::into_inner)
.insert(collection_id.clone(), index);
} else {
notes_state.evict_cached_index(&collection_id);
let _ = cleanup_unreferenced_notes_shards(¬es_state.index_root, &collection_id);
}
}
emit_state_changed(&app, &collection_id);
let dto = notes_state.collection_dto(&collection_id)?;
crate::logging::log(
"Notes",
format!(
"index completed collection={collection_id} indexed={} needs_rerun={needs_rerun} elapsed_ms={}",
outcome.indexed_document_count,
request_started_at.elapsed().as_millis()
),
);
Ok(NotesIndexResultDto {
collection: dto,
needs_rerun,
})
}
#[tauri::command]
pub async fn notes_open_document(
app: AppHandle,
collection_id: String,
document_id: String,
indexed_revision: String,
) -> Result<(), ApiError> {
let verification_app = app.clone();
let path = async_runtime::spawn_blocking(move || {
let notes_state = available_state(&verification_app)?;
let collection = notes_state.registered_collection(&collection_id)?;
let (path, bytes, _) = read_collection_source(&collection.source_root, &document_id)?;
if !is_supported_file(&path) {
return Err(ApiError::new(
"invalid_document",
"Unsupported note document",
));
}
if notes_content_revision(&bytes) != indexed_revision {
return Err(ApiError::new(
"source_changed",
"Source note changed after this answer was generated",
));
}
Ok(path)
})
.await
.map_err(|_| ApiError::new("io_thread", "Source note verification task failed"))??;
app.opener()
.open_path(path.to_string_lossy().into_owned(), None::<String>)
.map_err(|_| ApiError::new("open_failed", "Source note could not be opened"))
}
fn check_cancelled(
cancellation_epoch: &std::sync::atomic::AtomicU64,
retrieval_epoch: u64,
) -> Result<(), ApiError> {
if cancellation_epoch.load(Ordering::Relaxed) == retrieval_epoch {
Ok(())
} else {
Err(ApiError::new("cancelled", "Notes indexing cancelled"))
}
}
fn notes_error(error: NotesError) -> ApiError {
let (code, message) = match error {
NotesError::InvalidInput(_) => ("invalid_input", "Notes input is invalid"),
NotesError::InvalidIndex(_) => ("invalid_index", "The Notes index is invalid"),
NotesError::IncompatibleIndex => {
("incompatible_index", "The Notes index needs to be rebuilt")
}
NotesError::NotReady => ("not_ready", "The Notes collection is not ready"),
NotesError::CollectionTooLarge(message) => {
return ApiError::new("collection_too_large", message);
}
NotesError::Io(_) => ("io", "The Notes index could not be accessed"),
NotesError::Json(_) => ("json", "The Notes index could not be read"),
};
ApiError::new(code, message)
}
fn available_state(app: &AppHandle) -> Result<TauriState<'_, State>, ApiError> {
app.try_state::<State>().ok_or_else(|| {
ApiError::new(
"notes_unavailable",
"Notes are unavailable because local storage could not be initialized",
)
})
}
fn acquire_notes_ownership(notes_directory: &Path) -> Result<File, ApiError> {
let mut options = OpenOptions::new();
options.read(true).write(true).create(true);
#[cfg(unix)]
options.mode(0o600);
#[cfg(windows)]
options.share_mode(0);
let file = options
.open(notes_directory.join(NOTES_OWNERSHIP_LOCK_FILE))
.map_err(notes_ownership_open_error)?;
#[cfg(unix)]
file.try_lock().map_err(|error| match error {
TryLockError::WouldBlock => notes_in_use_error(),
TryLockError::Error(error) => io_error(error),
})?;
Ok(file)
}
fn notes_ownership_open_error(error: std::io::Error) -> ApiError {
#[cfg(windows)]
if error.raw_os_error() == Some(ERROR_SHARING_VIOLATION as i32) {
return notes_in_use_error();
}
io_error(error)
}
fn notes_in_use_error() -> ApiError {
ApiError::new(
"notes_in_use",
"Your Notes is already open in another Ensu instance",
)
}
fn ensure_directory(path: &Path) -> Result<(), ApiError> {
if !path_exists(path)? {
fs::create_dir_all(path).map_err(io_error)?;
}
if !fs::symlink_metadata(path).map_err(io_error)?.is_dir() {
return Err(ApiError::new(
"invalid_path",
"Notes storage path is invalid",
));
}
#[cfg(unix)]
fs::set_permissions(path, fs::Permissions::from_mode(0o700)).map_err(io_error)?;
Ok(())
}
fn path_exists(path: &Path) -> Result<bool, ApiError> {
match fs::symlink_metadata(path) {
Ok(_) => Ok(true),
Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(false),
Err(error) => Err(io_error(error)),
}
}
fn remove_owned_entry(path: &Path) -> Result<(), ApiError> {
let metadata = fs::symlink_metadata(path).map_err(io_error)?;
if metadata.is_dir() {
fs::remove_dir_all(path).map_err(io_error)
} else {
fs::remove_file(path).map_err(io_error)
}
}
fn io_error(_: std::io::Error) -> ApiError {
ApiError::new("io", "Notes storage operation failed")
}
fn source_scan_error(_: std::io::Error) -> ApiError {
ApiError::new(
"source_unavailable",
"Notes source folder could not be read",
)
}
fn now_ms() -> i64 {
system_time_ms(SystemTime::now()).unwrap_or(0)
}
fn system_time_ms(time: SystemTime) -> Option<i64> {
let millis = time.duration_since(UNIX_EPOCH).ok()?.as_millis();
i64::try_from(millis).ok()
}
pub(crate) fn clear_for_exit(app: &AppHandle) {
let Some(state) = app.try_state::<State>() else {
return;
};
state
.watchers
.lock()
.unwrap_or_else(PoisonError::into_inner)
.clear();
state
.watch_work
.lock()
.unwrap_or_else(PoisonError::into_inner)
.clear();
state
.updates
.lock()
.unwrap_or_else(PoisonError::into_inner)
.clear();
state
.cached_indexes
.lock()
.unwrap_or_else(PoisonError::into_inner)
.clear();
}
#[cfg(test)]
mod tests {
use super::*;
use ente_ensu::notes::{NotesIndexWriter, NotesSourceDocument, prepare_notes_document};
const COLLECTION_ID: &str = "123e4567-e89b-12d3-a456-426614174000";
const SECOND_COLLECTION_ID: &str = "123e4567-e89b-12d3-a456-426614174001";
fn publish_test_index(index_root: &Path, collection_id: &str, document_id: &str) {
let prepared = prepare_notes_document(document_id, b"test note").unwrap();
let mut embedding = vec![0.0; 512];
embedding[0] = 1.0;
let metadata = NotesSourceDocument {
document_id: document_id.to_string(),
size: 9,
modified_at_ms: Some(1),
};
let mut writer = NotesIndexWriter::open(index_root, collection_id.to_string()).unwrap();
writer
.commit_document(&prepared, &[embedding], &metadata)
.unwrap();
writer.publish(true).unwrap();
}
#[test]
fn ownership_lock_excludes_another_owner_until_release() {
let temp = TestDirectory::new();
let first = acquire_notes_ownership(temp.path()).unwrap();
let error = acquire_notes_ownership(temp.path()).unwrap_err();
assert_eq!(error.name, Some("notes_in_use"));
assert_eq!(
error.message,
"Your Notes is already open in another Ensu instance"
);
drop(first);
acquire_notes_ownership(temp.path()).unwrap();
}
#[test]
fn watcher_warning_does_not_replace_ready_index_state() {
let temp = TestDirectory::new();
let collection = RegisteredCollection {
id: COLLECTION_ID.to_string(),
source_root: temp.path().to_path_buf(),
};
let runtime = CollectionRuntime {
status: NotesCollectionStatusDto::Ready,
index_available: true,
indexing_progress: None,
indexed_document_count: 1,
last_updated_at_ms: Some(1),
last_error: None,
watcher_error: Some("Watcher unavailable".to_string()),
};
let dto = collection_dto(&collection, &runtime, None);
assert_eq!(dto.status, NotesCollectionStatusDto::Ready);
assert_eq!(dto.watcher_error.as_deref(), Some("Watcher unavailable"));
assert_eq!(dto.last_error, None);
}
#[test]
fn unreadable_index_requests_one_immediate_derived_rebuild() {
let mut update = DesktopUpdateState::default();
assert!(update.record_unreadable_index(42));
assert!(update.rebuild_derived_index);
assert_eq!(update.quiet_deadline_ms, 42);
assert_eq!(update.generation, 1);
assert!(!update.record_unreadable_index(84));
assert_eq!(update.quiet_deadline_ms, 42);
assert_eq!(update.generation, 1);
}
#[test]
fn retrieval_cache_evicts_the_least_recently_used_index_at_its_budget() {
let temp = TestDirectory::new();
let index_root = temp.path().join("indexes");
publish_test_index(&index_root, COLLECTION_ID, "first.md");
publish_test_index(&index_root, SECOND_COLLECTION_ID, "other.md");
let first = NotesCollectionIndex::open(&index_root, COLLECTION_ID.to_string()).unwrap();
let second =
NotesCollectionIndex::open(&index_root, SECOND_COLLECTION_ID.to_string()).unwrap();
let mut cache = IndexCache::new(first.estimated_heap_bytes());
cache.insert(COLLECTION_ID.to_string(), first);
cache.insert(SECOND_COLLECTION_ID.to_string(), second);
assert!(!cache.entries.contains_key(COLLECTION_ID));
assert!(cache.entries.contains_key(SECOND_COLLECTION_ID));
}
}