π End-to-end encrypted cloud for everything.
Dart 2faandroidauthye2eeencryptionend-to-end-encryptionfluttergolanggoogle-photosgoogle-photos-alternativeiosopensourcephotosprivacyprivacy-toolstypescriptzero-knowledge
[ensu][desktop] Align Notes progress with checkpoints
setalp44b3d3239fe4
diff --git a/rust/apps/ensu/src-tauri/src/commands/notes.rs b/rust/apps/ensu/src-tauri/src/commands/notes.rsindex a9df9026a2..f4006c4fc2 100644--- a/rust/apps/ensu/src-tauri/src/commands/notes.rs+++ b/rust/apps/ensu/src-tauri/src/commands/notes.rs@@ -937,7 +937,9 @@ fn report_index_error(.is_ok_and(|collection| source_root_is_available(&collection.source_root))});let mut runtime = state.runtime(collection_id);- runtime.indexing_progress = None;+ if !cancelled {+ runtime.indexing_progress = None;+ }runtime.status = if cancelled || superseded_file_error {NotesCollectionStatusDto::Pending} else if unavailable {@@ -1024,7 +1026,8 @@ pub async fn notes_index_collection(if !matches!(runtime.status,NotesCollectionStatusDto::Indexing | NotesCollectionStatusDto::Updating- ) {+ ) && runtime.indexing_progress.is_none()+ {runtime.indexing_progress = Some(0);}runtime.status = if had_index {diff --git a/rust/crates/ensu/src/notes/indexing.rs b/rust/crates/ensu/src/notes/indexing.rsindex 9745b02f18..0382fd7c55 100644--- a/rust/crates/ensu/src/notes/indexing.rs+++ b/rust/crates/ensu/src/notes/indexing.rs@@ -3,8 +3,9 @@ use std::path::Path;use super::{NotesError, NotesIndexWriter, NotesSourceDocument, PreparedNotesDocument};-const INITIAL_CHECKPOINT_DOCUMENTS: usize = 64;-const INITIAL_CHECKPOINT_CHUNKS: usize = 128;+const MIN_CHECKPOINT_DOCUMENTS: usize = 8;+const MIN_CHECKPOINT_CHUNKS: usize = 32;+const TARGET_CHECKPOINT_COUNT: usize = 50;#[derive(Debug, Clone, PartialEq, Eq)]pub enum NotesDocumentLoad {@@ -154,18 +155,24 @@ pub fn index_notes_collection<E>(NotesIndexWriter::validate_inventory_capacity(&collection_id, &verified_inventory)?;writer.validate_reconciliation_capacity(&verified_inventory, &prepared_chunk_counts)?;- let mut progress_document_count = source_document_count.saturating_sub(requested.len() as u64);+ let mut checkpointed_document_count =+ source_document_count.saturating_sub(requested.len() as u64);report_indexing_progress(&writer,- progress_document_count,+ checkpointed_document_count,source_document_count,&mut on_progress,);let mut processed_requested = 0_usize;- let mut embedded_documents = 0_usize;- let mut embedded_chunks = 0_usize;- let mut next_document_checkpoint = INITIAL_CHECKPOINT_DOCUMENTS;- let mut next_chunk_checkpoint = INITIAL_CHECKPOINT_CHUNKS;+ let mut documents_since_checkpoint = 0_usize;+ let mut chunks_since_checkpoint = 0_usize;+ let document_checkpoint = checkpoint_interval(requested.len(), MIN_CHECKPOINT_DOCUMENTS);+ let requested_chunk_count = prepared_chunk_counts+ .values()+ .fold(0_usize, |total, count| {+ total.saturating_add(*count as usize)+ });+ let chunk_checkpoint = checkpoint_interval(requested_chunk_count, MIN_CHECKPOINT_CHUNKS);for document_id in &requested {check_for_cancellation().map_err(NotesIndexingError::Adapter)?;let (prepared, source_metadata) =@@ -206,13 +213,6 @@ pub fn index_notes_collection<E>(let Some(prepared) = prepared else {writer.commit_deletions(std::slice::from_ref(document_id))?;processed_requested += 1;- progress_document_count += 1;- report_indexing_progress(- &writer,- progress_document_count,- source_document_count,- &mut on_progress,- );continue;};let validated = match writer.validate_document(&prepared, &source_metadata) {@@ -220,13 +220,6 @@ pub fn index_notes_collection<E>(Err(NotesError::InvalidInput(_)) => {writer.commit_deletions(std::slice::from_ref(document_id))?;processed_requested += 1;- progress_document_count += 1;- report_indexing_progress(- &writer,- progress_document_count,- source_document_count,- &mut on_progress,- );continue;}Err(error) => return Err(error.into()),@@ -235,13 +228,6 @@ pub fn index_notes_collection<E>(else {writer.commit_deletions(std::slice::from_ref(document_id))?;processed_requested += 1;- progress_document_count += 1;- report_indexing_progress(- &writer,- progress_document_count,- source_document_count,- &mut on_progress,- );continue;};check_for_cancellation().map_err(NotesIndexingError::Adapter)?;@@ -269,26 +255,33 @@ pub fn index_notes_collection<E>(}writer.commit_validated_document(validated, &embeddings)?;processed_requested += 1;- progress_document_count += 1;- report_indexing_progress(- &writer,- progress_document_count,- source_document_count,- &mut on_progress,- );- embedded_documents += 1;- embedded_chunks += prepared.chunks.len();+ documents_since_checkpoint += 1;+ chunks_since_checkpoint += prepared.chunks.len();if processed_requested < requested.len()- && (embedded_documents >= next_document_checkpoint- || embedded_chunks >= next_chunk_checkpoint)+ && (documents_since_checkpoint >= document_checkpoint+ || chunks_since_checkpoint >= chunk_checkpoint){writer.publish(index_was_ready)?;- next_document_checkpoint = next_document_checkpoint.saturating_mul(2);- next_chunk_checkpoint = next_chunk_checkpoint.saturating_mul(2);+ checkpointed_document_count =+ checkpointed_document_count.saturating_add(documents_since_checkpoint as u64);+ report_indexing_progress(+ &writer,+ checkpointed_document_count,+ source_document_count,+ &mut on_progress,+ );+ documents_since_checkpoint = 0;+ chunks_since_checkpoint = 0;}}writer.publish(true)?;+ report_indexing_progress(+ &writer,+ source_document_count,+ source_document_count,+ &mut on_progress,+ );Ok(NotesIndexOutcome {changed_during_indexing: None,unchecked_document_ids: Vec::new(),@@ -296,6 +289,10 @@ pub fn index_notes_collection<E>(})}+fn checkpoint_interval(total: usize, minimum: usize) -> usize {+ total.div_ceil(TARGET_CHECKPOINT_COUNT).max(minimum)+}+fn report_indexing_progress(writer: &NotesIndexWriter,processed_document_count: u64,