💚 End-to-end encrypted cloud for everything.
Dart 2faandroidauthye2eeencryptionend-to-end-encryptionfluttergolanggoogle-photosgoogle-photos-alternativeiosopensourcephotosprivacyprivacy-toolstypescriptzero-knowledge
Fix Your Notes indexing lifecycle
setalp87ac1ea18e98
diff --git a/rust/apps/ensu/src-tauri/src/commands/notes.rs b/rust/apps/ensu/src-tauri/src/commands/notes.rsindex 48f85463c4..5804e1fff6 100644--- a/rust/apps/ensu/src-tauri/src/commands/notes.rs+++ b/rust/apps/ensu/src-tauri/src/commands/notes.rs@@ -5,11 +5,11 @@ use std::os::unix::fs::PermissionsExt;use std::path::{Path, PathBuf};use std::sync::atomic::Ordering;use std::sync::{Arc, Mutex, PoisonError};-use std::time::{SystemTime, UNIX_EPOCH};+use std::time::{Instant, SystemTime, UNIX_EPOCH};use ente_ensu::model;use ente_ensu::notes::{- NoteSourceReference, NotesCollectionIndex, NotesError, NotesIndexOutcome,+ NoteSourceReference, NotesCollectionIndex, NotesError, NotesIndexOutcome, NotesIndexProgress,notes_content_revision,};use notify::RecommendedWatcher;@@ -525,18 +525,38 @@ impl State {(needs_rerun, update_completed)}- fn report_indexing_progress(app: &AppHandle, collection_id: &str, progress: u8) {+ fn report_indexing_progress(+ app: &AppHandle,+ collection_id: &str,+ progress: NotesIndexProgress,+ ) {let state = app.state::<State>();let mut runtime = state.runtime(collection_id);+ let previous_progress = runtime.indexing_progress;if !matches!(runtime.status,NotesCollectionStatusDto::Indexing | NotesCollectionStatusDto::Updating- ) || runtime.indexing_progress == Some(progress)+ ) || previous_progress == Some(progress.percentage){return;}- runtime.indexing_progress = Some(progress);+ runtime.indexing_progress = Some(progress.percentage);+ runtime.indexed_document_count = progress.indexed_document_count;state.set_runtime(collection_id, runtime);+ if progress.percentage == 100+ || previous_progress.is_none_or(|previous| previous / 10 != progress.percentage / 10)+ {+ crate::logging::log(+ "Notes",+ format!(+ "index progress collection={collection_id} percent={} processed={} indexed={} total={}",+ progress.percentage,+ progress.processed_document_count,+ progress.indexed_document_count,+ progress.total_document_count+ ),+ );+ }emit_state_changed(app, collection_id);}@@ -747,7 +767,12 @@ fn register_collection(},);}- notes_state.collection_dto(&collection.id)+ let dto = notes_state.collection_dto(&collection.id)?;+ crate::logging::log(+ "Notes",+ format!("collection registered collection={}", collection.id),+ );+ Ok(dto)}#[tauri::command]@@ -817,6 +842,10 @@ fn remove_collection(notes_state: &State, collection_id: &str) -> Result<(), Api.lock().unwrap_or_else(PoisonError::into_inner).remove(collection_id);+ crate::logging::log(+ "Notes",+ format!("collection removed collection={collection_id}"),+ );Ok(())}@@ -850,6 +879,13 @@ fn report_index_error(};runtime.last_error = (!cancelled && !superseded_file_error).then(|| error.message.clone());state.set_runtime(collection_id, runtime);+ crate::logging::log(+ "Notes",+ format!(+ "index stopped collection={collection_id} reason={}",+ error.name.unwrap_or("unknown")+ ),+ );emit_state_changed(app, collection_id);}@@ -862,6 +898,11 @@ pub async fn notes_index_collection(force: bool,retrieval_epoch: u64,) -> Result<NotesIndexResultDto, ApiError> {+ let request_started_at = Instant::now();+ crate::logging::log(+ "Notes",+ format!("index requested collection={collection_id} force={force}"),+ );let notes_state = available_state(&app)?;let lifecycle = notes_state.collection_lifecycle(&collection_id);let _collection_lifecycle = lifecycle.lock().await;@@ -883,9 +924,21 @@ pub async fn notes_index_collection(let mut update = notes_state.prepare_update(&collection_id, force, had_index)?;update.rebuild_derived_index = rebuild_derived_index;let cancellation_epoch = llm_state.retrieval_epoch();- check_cancelled(&cancellation_epoch, retrieval_epoch)?;+ if let Err(error) = check_cancelled(&cancellation_epoch, retrieval_epoch) {+ crate::logging::log(+ "Notes",+ format!("index cancelled before start collection={collection_id}"),+ );+ return Err(error);+ }let _model_lifecycle = llm_state.lifecycle().lock().await;- check_cancelled(&cancellation_epoch, retrieval_epoch)?;+ if let Err(error) = check_cancelled(&cancellation_epoch, retrieval_epoch) {+ crate::logging::log(+ "Notes",+ format!("index cancelled while waiting collection={collection_id}"),+ );+ return Err(error);+ }crate::commands::llm::replace_state(&llm_state, None, None)?;let mut runtime = notes_state.runtime(&collection_id);@@ -902,6 +955,12 @@ pub async fn notes_index_collection(};runtime.last_error = None;notes_state.set_runtime(&collection_id, runtime);+ crate::logging::log(+ "Notes",+ format!(+ "index started collection={collection_id} updating={had_index} rebuild={rebuild_derived_index}"+ ),+ );emit_state_changed(&app, &collection_id);let store = model_state.store();let embedding_asset = model::knowledge_embedding_model_asset();@@ -991,6 +1050,14 @@ pub async fn notes_index_collection(}emit_state_changed(&app, &collection_id);let dto = notes_state.collection_dto(&collection_id)?;+ crate::logging::log(+ "Notes",+ format!(+ "index completed collection={collection_id} indexed={} needs_rerun={needs_rerun} elapsed_ms={}",+ outcome.indexed_document_count,+ request_started_at.elapsed().as_millis()+ ),+ );Ok(NotesIndexResultDto {collection: dto,needs_rerun,diff --git a/rust/apps/ensu/src-tauri/src/commands/notes/indexing.rs b/rust/apps/ensu/src-tauri/src/commands/notes/indexing.rsindex c552686386..17f57b4566 100644--- a/rust/apps/ensu/src-tauri/src/commands/notes/indexing.rs+++ b/rust/apps/ensu/src-tauri/src/commands/notes/indexing.rs@@ -3,8 +3,8 @@ use std::path::Path;use ente_ensu::llm;use ente_ensu::notes::{NotesCollectionIndex, NotesDocumentLoad, NotesError, NotesIndexInput, NotesIndexOutcome,- NotesIndexingError, NotesRevisionStatus, index_notes_collection, notes_content_revision,- prepare_notes_document,+ NotesIndexProgress, NotesIndexingError, NotesRevisionStatus, index_notes_collection,+ notes_content_revision, prepare_notes_document,};use crate::commands::common::ApiError;@@ -25,7 +25,7 @@ pub(super) fn index_collection(cancellation_epoch: &std::sync::atomic::AtomicU64,retrieval_epoch: u64,update: &UpdateSnapshot,- on_progress: impl FnMut(u8),+ on_progress: impl FnMut(NotesIndexProgress),) -> Result<NotesIndexOutcome, ApiError> {check_cancelled(cancellation_epoch, retrieval_epoch)?;let canonical_root = canonical_source_root(&collection.source_root)?;diff --git a/rust/crates/ensu/src/llm/embed.rs b/rust/crates/ensu/src/llm/embed.rsindex 79894adcb2..9c6e6e34fa 100644--- a/rust/crates/ensu/src/llm/embed.rs+++ b/rust/crates/ensu/src/llm/embed.rs@@ -74,8 +74,8 @@ impl Context {message: err.to_string(),})?;}- context.decode(&mut batch).map_err(|err| Error::Llama {- op: "Embedding decode failed",+ context.encode(&mut batch).map_err(|err| Error::Llama {+ op: "Embedding encode failed",message: err.to_string(),})?;let native = context.embeddings_seq_ith(0).map_err(|err| Error::Llama {diff --git a/rust/crates/ensu/src/notes.rs b/rust/crates/ensu/src/notes.rsindex bf31196833..1dfc10df0d 100644--- a/rust/crates/ensu/src/notes.rs+++ b/rust/crates/ensu/src/notes.rs@@ -44,8 +44,8 @@ pub use document::{};pub use index::{NotesCollectionIndex, NotesCollectionIndexSummary, NotesSearchHit};pub use indexing::{- NotesDocumentLoad, NotesIndexInput, NotesIndexOutcome, NotesIndexingError, NotesRevisionStatus,- index_notes_collection,+ NotesDocumentLoad, NotesIndexInput, NotesIndexOutcome, NotesIndexProgress, NotesIndexingError,+ NotesRevisionStatus, index_notes_collection,};pub use reconcile::NotesReconciliationPlan;pub use source::NoteSourceReference;diff --git a/rust/crates/ensu/src/notes/indexing.rs b/rust/crates/ensu/src/notes/indexing.rsindex 42eca251ca..0b29d3cc04 100644--- a/rust/crates/ensu/src/notes/indexing.rs+++ b/rust/crates/ensu/src/notes/indexing.rs@@ -50,6 +50,14 @@ pub struct NotesIndexInput<'a> {pub force_full_hash: bool,}+#[derive(Debug, Clone, Copy, PartialEq, Eq)]+pub struct NotesIndexProgress {+ pub percentage: u8,+ pub processed_document_count: u64,+ pub indexed_document_count: u64,+ pub total_document_count: u64,+}+#[derive(Debug, Clone, PartialEq, Eq)]pub struct NotesIndexOutcome {pub changed_during_indexing: Option<String>,@@ -69,7 +77,7 @@ pub fn index_notes_collection<E>(mut load_document: impl FnMut(&str) -> Result<NotesDocumentLoad, E>,mut embed_document: impl FnMut(&PreparedNotesDocument) -> Result<Option<Vec<Vec<f32>>>, E>,mut verify_revision: impl FnMut(&str, &str) -> Result<NotesRevisionStatus, E>,- mut on_progress: impl FnMut(u8),+ mut on_progress: impl FnMut(NotesIndexProgress),) -> Result<NotesIndexOutcome, NotesIndexingError<E>> {let NotesIndexInput {index_root,@@ -147,10 +155,12 @@ pub fn index_notes_collection<E>(writer.validate_reconciliation_capacity(&verified_inventory, &prepared_chunk_counts)?;let mut progress_document_count = source_document_count.saturating_sub(requested.len() as u64);- on_progress(indexing_progress(+ report_indexing_progress(+ &writer,progress_document_count,source_document_count,- ));+ &mut on_progress,+ );let mut processed_requested = 0_usize;let mut embedded_documents = 0_usize;let mut embedded_chunks = 0_usize;@@ -197,10 +207,12 @@ pub fn index_notes_collection<E>(writer.commit_deletions(std::slice::from_ref(document_id))?;processed_requested += 1;progress_document_count += 1;- on_progress(indexing_progress(+ report_indexing_progress(+ &writer,progress_document_count,source_document_count,- ));+ &mut on_progress,+ );continue;};match writer.validate_document(&prepared, &source_metadata) {@@ -209,10 +221,12 @@ pub fn index_notes_collection<E>(writer.commit_deletions(std::slice::from_ref(document_id))?;processed_requested += 1;progress_document_count += 1;- on_progress(indexing_progress(+ report_indexing_progress(+ &writer,progress_document_count,source_document_count,- ));+ &mut on_progress,+ );continue;}Err(error) => return Err(error.into()),@@ -222,10 +236,12 @@ pub fn index_notes_collection<E>(writer.commit_deletions(std::slice::from_ref(document_id))?;processed_requested += 1;progress_document_count += 1;- on_progress(indexing_progress(+ report_indexing_progress(+ &writer,progress_document_count,source_document_count,- ));+ &mut on_progress,+ );continue;};check_for_cancellation().map_err(NotesIndexingError::Adapter)?;@@ -254,10 +270,12 @@ pub fn index_notes_collection<E>(writer.commit_document(&prepared, &embeddings, &verified_source)?;processed_requested += 1;progress_document_count += 1;- on_progress(indexing_progress(+ report_indexing_progress(+ &writer,progress_document_count,source_document_count,- ));+ &mut on_progress,+ );embedded_documents += 1;embedded_chunks += prepared.chunks.len();if processed_requested < requested.len()@@ -274,10 +292,24 @@ pub fn index_notes_collection<E>(Ok(NotesIndexOutcome {changed_during_indexing: None,unchecked_document_ids: Vec::new(),- indexed_document_count: writer.indexed_document_ids().count() as u64,+ indexed_document_count: writer.indexed_document_count() as u64,})}+fn report_indexing_progress(+ writer: &NotesIndexWriter,+ processed_document_count: u64,+ total_document_count: u64,+ on_progress: &mut impl FnMut(NotesIndexProgress),+) {+ on_progress(NotesIndexProgress {+ percentage: indexing_progress(processed_document_count, total_document_count),+ processed_document_count,+ indexed_document_count: writer.indexed_document_count() as u64,+ total_document_count,+ });+}+fn indexing_progress(processed_document_count: u64, source_document_count: u64) -> u8 {if source_document_count == 0 {return 100;@@ -303,13 +335,13 @@ fn changed_during_indexing_outcome<E>(.skip(processed_requested).cloned().collect(),- indexed_document_count: writer.indexed_document_ids().count() as u64,+ indexed_document_count: writer.indexed_document_count() as u64,})}#[cfg(test)]mod tests {- use std::cell::Cell;+ use std::cell::{Cell, RefCell};use super::*;use crate::notes::{NotesCollectionIndex, prepare_notes_document};@@ -345,7 +377,7 @@ mod tests {let temp = tempfile::tempdir().unwrap();let prepared = prepare_notes_document("note.md", b"note").unwrap();let verified_source = source_at(2);- let progress = Cell::new(0);+ let progress = RefCell::new(Vec::new());let outcome = index_notes_collection::<()>(NotesIndexInput {index_root: temp.path(),@@ -367,12 +399,29 @@ mod tests {source: verified_source.clone(),})},- |value| progress.set(value),+ |value| progress.borrow_mut().push(value),).unwrap();assert!(outcome.ready());- assert_eq!(progress.get(), 100);+ let progress = progress.borrow();+ assert_eq!(+ progress.as_slice(),+ [+ NotesIndexProgress {+ percentage: 0,+ processed_document_count: 0,+ indexed_document_count: 0,+ total_document_count: 1,+ },+ NotesIndexProgress {+ percentage: 100,+ processed_document_count: 1,+ indexed_document_count: 1,+ total_document_count: 1,+ },+ ]+ );let index = NotesCollectionIndex::open(temp.path(), COLLECTION_ID.to_string()).unwrap();assert_eq!(index.document_count(), 1);let writer = NotesIndexWriter::open(temp.path(), COLLECTION_ID.to_string()).unwrap();diff --git a/rust/crates/ensu/src/notes/writer.rs b/rust/crates/ensu/src/notes/writer.rsindex 79803830c2..45e1eb2be8 100644--- a/rust/crates/ensu/src/notes/writer.rs+++ b/rust/crates/ensu/src/notes/writer.rs@@ -146,6 +146,10 @@ impl NotesIndexWriter {self.manifest.documents.keys().map(String::as_str)}+ pub fn indexed_document_count(&self) -> usize {+ self.manifest.documents.len()+ }+pub fn indexed_revision(&self, document_id: &str) -> Option<&str> {self.manifest.documentsdiff --git a/web/apps/ensu/package.json b/web/apps/ensu/package.jsonindex ddd4fd1561..9988398020 100644--- a/web/apps/ensu/package.json+++ b/web/apps/ensu/package.json@@ -2,6 +2,9 @@"name": "ensu","version": "0.0.0","private": true,+ "scripts": {+ "test": "vitest --run"+ },"dependencies": {"@hugeicons/core-free-icons": "1.2.1","@hugeicons/react": "1.1.6",@@ -24,6 +27,7 @@"remark-math": "6.0.0"},"devDependencies": {- "ente-build-config": "*"+ "ente-build-config": "*",+ "vitest": "4.1.8"}}diff --git a/web/apps/ensu/src/components/chat/ChatDialogs.tsx b/web/apps/ensu/src/components/chat/ChatDialogs.tsxindex f51643fe8f..1d1179677d 100644--- a/web/apps/ensu/src/components/chat/ChatDialogs.tsx+++ b/web/apps/ensu/src/components/chat/ChatDialogs.tsx@@ -1,5 +1,5 @@import type { KnowledgePack } from "@/services/knowledge";-import type { NotesCollection } from "@/services/notes";+import type { NotesCollectionView } from "@/services/notes-lifecycle";import { isTauriRuntime as detectTauriAppRuntime } from "@/services/tauri-runtime";import {ArrowLeft01Icon,@@ -153,7 +153,7 @@ export interface ChatDialogsProps {handleDownloadKnowledgePack: (stableId: string) => void;handleCancelKnowledgePackDownload: (stableId: string) => void;handleSetKnowledgePackEnabled: (stableId: string, enabled: boolean) => void;- notesCollections: NotesCollection[];+ notesCollections: NotesCollectionView[];notesCollectionsLoading: boolean;notesCollectionsError: string | null;retryNotesCollections: () => void;@@ -823,16 +823,23 @@ export const ChatDialogs = memo(const indexing =collection.status === "indexing" ||collection.status === "updating";+ const starting =+ collection.activity === "starting";const progress = collection.indexingProgress;const canRetry =- collection.status === "pending" ||collection.status === "error" ||- collection.status === "unavailable";- const retryLabel =- collection.status === "error" ||- collection.status === "unavailable"- ? "Try again"- : "Continue indexing";+ collection.status === "unavailable" ||+ collection.activity === "failed";+ const waitingMessage =+ collection.activity ===+ "waitingForGeneration"+ ? "Indexing will resume after the current response."+ : collection.activity ===+ "waitingForModel"+ ? "Indexing will start when the model is ready."+ : collection.activity === "scheduled"+ ? "Indexing is scheduled after recent changes settle."+ : null;return (<Stackkey={collection.id}@@ -861,13 +868,11 @@ export const ChatDialogs = memo(noWrapsx={{ color: "text.muted" }}>- {- collection.indexedDocumentCount- }{" "}- {collection.indexedDocumentCount ===- 1- ? "note indexed"- : "notes indexed"}+ {starting &&+ collection.indexedDocumentCount ===+ 0+ ? "Preparing notes…"+ : `${collection.indexedDocumentCount} ${collection.indexedDocumentCount === 1 ? "note indexed" : "notes indexed"}`}{collection.lastUpdatedAtMs !=null && (<>@@ -881,7 +886,7 @@ export const ChatDialogs = memo(</Stack><IconButtonaria-label={`Remove ${collection.label}`}- disabled={indexing}+ disabled={indexing || starting}onClick={() =>handleRemoveNotesCollection(collection.id,@@ -917,6 +922,15 @@ export const ChatDialogs = memo(</Typography></Stack>)}+ {starting && <LinearProgress />}+ {waitingMessage && (+ <Typography+ variant="small"+ sx={{ color: "text.muted" }}+ >+ {waitingMessage}+ </Typography>+ )}{collection.lastError && (<Typographyvariant="small"@@ -936,7 +950,7 @@ export const ChatDialogs = memo(}sx={{ alignSelf: "flex-start" }}>- {retryLabel}+ Try again</Button>)}</Stack>diff --git a/web/apps/ensu/src/hooks/use-notes-collections.ts b/web/apps/ensu/src/hooks/use-notes-collections.tsindex 57413e0aa4..7b381b45d8 100644--- a/web/apps/ensu/src/hooks/use-notes-collections.ts+++ b/web/apps/ensu/src/hooks/use-notes-collections.ts@@ -9,10 +9,14 @@ import {selectNotesFolder,type NotesCollection,} from "@/services/notes";-import { NotesLifecycleController } from "@/services/notes-lifecycle";+import {+ NotesLifecycleController,+ type NotesCollectionActivity,+ type NotesCollectionView,+} from "@/services/notes-lifecycle";import { tauriCommandError } from "@/services/tauri-error";import log from "ente-base/log";-import { useCallback, useEffect, useRef, useState } from "react";+import { useCallback, useEffect, useMemo, useRef, useState } from "react";const NOTES_INDEX_RERUN_YIELD_MS = 25;const NOTES_REMOVE_WAIT_ERROR =@@ -45,14 +49,16 @@ export const useNotesCollections = ({null,);const [syncAttempt, setSyncAttempt] = useState(0);+ const [lifecycleRevision, setLifecycleRevision] = useState(0);const controllerRef = useRef<NotesLifecycleController | null>(null);if (!controllerRef.current) {controllerRef.current = new NotesLifecycleController();}const controller = controllerRef.current;- const runIndexRef = useRef<- ((collectionId: string, force?: boolean) => void) | null- >(null);+ const lifecycleChanged = useCallback(+ () => setLifecycleRevision((revision) => revision + 1),+ [],+ );useEffect(() => {controller.activate();@@ -122,15 +128,25 @@ export const useNotesCollections = ({[controller],);- const runIndex = useCallback(- (collectionId: string, force = false) => {- const request = controller.requestIndex(+ const startIndex = useCallback(+ (collectionId: string, force = false, automatic = false) => {+ const blocked =+ !modelReady || isGenerating || isGenerationStarting();+ const request = automatic+ ? controller.ensureIndex(collectionId, blocked)+ : controller.requestIndex(collectionId, force, blocked);+ if (request === "ignored") return;+ lifecycleChanged();+ if (request === "queued") {+ log.info("Your Notes indexing queued", { collectionId });+ return;+ }+ setOperationError(null);+ log.info("Your Notes indexing started", {collectionId,+ automatic,force,- isGenerating || isGenerationStarting(),- );- if (request !== "start") return;- setOperationError(null);+ });void ensureProvider().then((provider) =>@@ -173,7 +189,12 @@ export const useNotesCollections = ({const { name } = tauriCommandError(error);if (name === "cancelled") {controller.queueIndex(collectionId, force);+ log.info(+ "Your Notes indexing queued after cancellation",+ { collectionId },+ );} else if (name !== "not_due") {+ controller.failIndex(collectionId);setOperationError(notesErrorMessage(error));log.error("Failed to index Your Notes", error);}@@ -181,12 +202,8 @@ export const useNotesCollections = ({}).finally(() => {const completion = controller.finishIndex(collectionId);- if (completion.queuedForce !== undefined) {- runIndexRef.current?.(- collectionId,- completion.queuedForce,- );- } else if (completion.clearedRemovalWait) {+ lifecycleChanged();+ if (completion.clearedRemovalWait) {setOperationError((current) =>current === NOTES_REMOVE_WAIT_ERROR? null@@ -200,19 +217,37 @@ export const useNotesCollections = ({ensureProvider,isGenerating,isGenerationStarting,+ lifecycleChanged,+ modelReady,refresh,replaceCollection,],);- runIndexRef.current = runIndex;++ const runIndex = useCallback(+ (collectionId: string, force = false) =>+ startIndex(collectionId, force),+ [startIndex],+ );useEffect(() => {- for (const [collectionId, force] of controller.drainQueued(- isGenerating || isGenerationStarting(),- )) {- runIndex(collectionId, force);+ const blocked = !modelReady || isGenerating || isGenerationStarting();+ const queued = controller.drainQueued(blocked);+ if (queued.length > 0) {+ lifecycleChanged();+ for (const [collectionId, force] of queued) {+ startIndex(collectionId, force);+ }}- }, [controller, isGenerating, isGenerationStarting, runIndex]);+ }, [+ controller,+ isGenerating,+ isGenerationStarting,+ lifecycleChanged,+ lifecycleRevision,+ modelReady,+ startIndex,+ ]);useEffect(() => {if (!isTauriRuntime) return;@@ -275,13 +310,18 @@ export const useNotesCollections = ({for (const collection of collections) {if (collection.status === "pending" &&- collection.updateDueAtMs == null &&- controller.markResumed(collection.id)+ collection.updateDueAtMs == null) {- runIndex(collection.id);+ startIndex(collection.id, false, true);}}- }, [collections, controller, isTauriRuntime, modelReady, runIndex]);+ }, [+ collections,+ isTauriRuntime,+ lifecycleRevision,+ modelReady,+ startIndex,+ ]);const retry = useCallback(() => {setOperationError(null);@@ -324,19 +364,57 @@ export const useNotesCollections = ({log.error("Failed to remove Notes collection", error);void refresh();if (removal.queuedForce !== undefined) {- runIndexRef.current?.(- collectionId,- removal.queuedForce,- );+ startIndex(collectionId, removal.queuedForce);}}});},- [confirmRemoval, controller, refresh],+ [confirmRemoval, controller, refresh, startIndex],+ );++ const collectionViews = useMemo<NotesCollectionView[]>(+ () =>+ collections.map((collection) => {+ if (collection.status !== "pending") {+ return { ...collection, activity: null };+ }+ const lifecycleActivity = controller.indexActivity(+ collection.id,+ );+ let activity: NotesCollectionActivity;+ if (lifecycleActivity === "failed") {+ activity = "failed";+ } else if (lifecycleActivity === "starting") {+ activity = "starting";+ } else if (lifecycleActivity === "queued") {+ activity = !modelReady+ ? "waitingForModel"+ : isGenerating || isGenerationStarting()+ ? "waitingForGeneration"+ : "starting";+ } else if (collection.updateDueAtMs != null) {+ activity = "scheduled";+ } else if (!modelReady) {+ activity = "waitingForModel";+ } else if (isGenerating || isGenerationStarting()) {+ activity = "waitingForGeneration";+ } else {+ activity = "starting";+ }+ return { ...collection, activity };+ }),+ [+ collections,+ controller,+ isGenerating,+ isGenerationStarting,+ lifecycleRevision,+ modelReady,+ ],);return {- collections,+ collections: collectionViews,loading,error: subscriptionError ?? listError ?? operationError,retry,diff --git a/web/apps/ensu/src/services/notes-lifecycle.ts b/web/apps/ensu/src/services/notes-lifecycle.tsindex edb59226b1..031636c1d7 100644--- a/web/apps/ensu/src/services/notes-lifecycle.ts+++ b/web/apps/ensu/src/services/notes-lifecycle.ts@@ -1,10 +1,25 @@+import type { NotesCollection } from "@/services/notes";+type NotesIndexRequest = "start" | "queued" | "ignored";interface NotesIndexCompletion {- queuedForce?: boolean;clearedRemovalWait: boolean;}+type NotesIndexActivity = "starting" | "queued" | "failed" | null;++export type NotesCollectionActivity =+ | "starting"+ | "waitingForGeneration"+ | "waitingForModel"+ | "scheduled"+ | "failed"+ | null;++export interface NotesCollectionView extends NotesCollection {+ activity: NotesCollectionActivity;+}+type NotesRemovalStart =| { kind: "wait" }| { kind: "start"; queuedForce?: boolean };@@ -12,8 +27,8 @@ type NotesRemovalStart =export class NotesLifecycleController {private readonly active = new Set<string>();private readonly queued = new Map<string, boolean>();+ private readonly failed = new Set<string>();private readonly removing = new Set<string>();- private readonly resumed = new Set<string>();private removalWaitCollection: string | null = null;private mutation = 0;private refreshRequest = 0;@@ -25,17 +40,40 @@ export class NotesLifecycleController {blocked: boolean,): NotesIndexRequest {if (this.disposed || this.removing.has(collectionId)) return "ignored";- if (blocked || this.active.has(collectionId)) {+ this.failed.delete(collectionId);+ if (+ blocked ||+ this.active.has(collectionId) ||+ this.queued.has(collectionId)+ ) {this.queueIndex(collectionId, force);return "queued";}- this.resumed.add(collectionId);+ this.active.add(collectionId);+ return "start";+ }++ ensureIndex(collectionId: string, blocked: boolean): NotesIndexRequest {+ if (+ this.disposed ||+ this.removing.has(collectionId) ||+ this.failed.has(collectionId) ||+ this.active.has(collectionId) ||+ this.queued.has(collectionId)+ ) {+ return "ignored";+ }+ if (blocked) {+ this.queueIndex(collectionId, false);+ return "queued";+ }this.active.add(collectionId);return "start";}queueIndex(collectionId: string, force: boolean) {if (this.disposed || this.removing.has(collectionId)) return;+ this.failed.delete(collectionId);this.queued.set(collectionId,force || (this.queued.get(collectionId) ?? false),@@ -52,11 +90,20 @@ export class NotesLifecycleController {finishIndex(collectionId: string): NotesIndexCompletion {this.active.delete(collectionId);- const queuedForce = this.queued.get(collectionId);- this.queued.delete(collectionId);const clearedRemovalWait = this.removalWaitCollection === collectionId;if (clearedRemovalWait) this.removalWaitCollection = null;- return { queuedForce, clearedRemovalWait };+ return { clearedRemovalWait };+ }++ failIndex(collectionId: string) {+ this.queued.delete(collectionId);+ this.failed.add(collectionId);+ }++ indexActivity(collectionId: string): NotesIndexActivity {+ if (this.failed.has(collectionId)) return "failed";+ if (this.queued.has(collectionId)) return "queued";+ return this.active.has(collectionId) ? "starting" : null;}drainQueued(blocked: boolean) {@@ -92,7 +139,7 @@ export class NotesLifecycleController {finishRemoval(collectionId: string) {this.removing.delete(collectionId);- this.resumed.delete(collectionId);+ this.failed.delete(collectionId);if (this.removalWaitCollection === collectionId) {this.removalWaitCollection = null;}@@ -102,12 +149,6 @@ export class NotesLifecycleController {return this.removing.has(collectionId);}- markResumed(collectionId: string) {- if (this.disposed || this.resumed.has(collectionId)) return false;- this.resumed.add(collectionId);- return true;- }-hasActiveJobs() {return this.active.size > 0;}@@ -145,6 +186,7 @@ export class NotesLifecycleController {this.disposed = true;this.active.clear();this.queued.clear();+ this.failed.clear();this.removing.clear();this.cancelRefreshes();}diff --git a/web/apps/ensu/tests/notes-lifecycle.test.ts b/web/apps/ensu/tests/notes-lifecycle.test.tsnew file mode 100644index 0000000000..8206367461--- /dev/null+++ b/web/apps/ensu/tests/notes-lifecycle.test.ts@@ -0,0 +1,31 @@+import { describe, expect, test } from "vitest";+import { NotesLifecycleController } from "../src/services/notes-lifecycle";++describe("NotesLifecycleController", () => {+ test("recovers cancelled indexing without duplicating or losing queued work", () => {+ const controller = new NotesLifecycleController();++ expect(controller.ensureIndex("notes", false)).toBe("start");+ expect(controller.indexActivity("notes")).toBe("starting");+ controller.queueIndex("notes", false);+ expect(controller.finishIndex("notes")).toEqual({+ clearedRemovalWait: false,+ });+ expect(controller.indexActivity("notes")).toBe("queued");+ expect(controller.ensureIndex("notes", false)).toBe("ignored");+ expect(controller.drainQueued(true)).toEqual([]);+ expect(controller.drainQueued(false)).toEqual([["notes", false]]);++ expect(controller.requestIndex("notes", false, false)).toBe("start");+ expect(controller.requestIndex("notes", true, false)).toBe("queued");+ controller.finishIndex("notes");+ expect(controller.drainQueued(false)).toEqual([["notes", true]]);++ expect(controller.requestIndex("notes", true, false)).toBe("start");+ controller.failIndex("notes");+ controller.finishIndex("notes");+ expect(controller.indexActivity("notes")).toBe("failed");+ expect(controller.ensureIndex("notes", false)).toBe("ignored");+ expect(controller.requestIndex("notes", true, false)).toBe("start");+ });+});diff --git a/web/apps/ensu/tsconfig.json b/web/apps/ensu/tsconfig.jsonindex 2ff1f734c6..6a2a667238 100644--- a/web/apps/ensu/tsconfig.json+++ b/web/apps/ensu/tsconfig.json@@ -9,6 +9,7 @@},"include": ["src",+ "tests","next-env.d.ts","next.config.mjs","../../packages/base/global-electron.d.ts",diff --git a/web/package-lock.json b/web/package-lock.jsonindex 4e037b9968..7df478b760 100644--- a/web/package-lock.json+++ b/web/package-lock.json@@ -191,7 +191,8 @@"remark-math": "6.0.0"},"devDependencies": {- "ente-build-config": "*"+ "ente-build-config": "*",+ "vitest": "4.1.8"}},"apps/ensu/node_modules/katex": {diff --git a/web/package.json b/web/package.jsonindex 3bee38bffc..11ea951759 100644--- a/web/package.json+++ b/web/package.json@@ -52,7 +52,7 @@"dev:twoof3": "npm exec --workspace twoof3 -- next dev --webpack -p 3012","lint": "concurrently --names 'prettier,eslint,tsc,wasm-imports' \"prettier --check --log-level warn .\" \"npm exec --workspaces -- eslint --max-warnings 0\" \"npm exec --workspaces -- tsc\" \"npm run check:wasm-imports\"","lint:fix": "concurrently --names 'prettier,eslint,tsc' \"prettier --write --log-level warn .\" \"npm exec --workspaces -- eslint --fix --max-warnings 0\" \"npm exec --workspaces -- tsc\"",- "test": "npm run test --workspace ente-core-wasm && npm run test --workspace ente-paste-wasm && npm run test --workspace ente-contacts && npm run test --workspace locker && npm run test --workspace photos"+ "test": "npm run test --workspace ensu && npm run test --workspace ente-core-wasm && npm run test --workspace ente-paste-wasm && npm run test --workspace ente-contacts && npm run test --workspace locker && npm run test --workspace photos"},"overrides": {"@types/react": "19.2.17",