Skip to content

armgit / ente

Public Imported from GitHub
0

💚 End-to-end encrypted cloud for everything.

Fix Your Notes indexing lifecycle

setalp87ac1ea18e98
diff --git a/rust/apps/ensu/src-tauri/src/commands/notes.rs b/rust/apps/ensu/src-tauri/src/commands/notes.rs
index 48f85463c4..5804e1fff6 100644
--- a/rust/apps/ensu/src-tauri/src/commands/notes.rs
+++ b/rust/apps/ensu/src-tauri/src/commands/notes.rs
@@ -5,11 +5,11 @@ use std::os::unix::fs::PermissionsExt;
use std::path::{Path, PathBuf};
use std::sync::atomic::Ordering;
use std::sync::{Arc, Mutex, PoisonError};
-use std::time::{SystemTime, UNIX_EPOCH};
+use std::time::{Instant, SystemTime, UNIX_EPOCH};
use ente_ensu::model;
use ente_ensu::notes::{
- NoteSourceReference, NotesCollectionIndex, NotesError, NotesIndexOutcome,
+ NoteSourceReference, NotesCollectionIndex, NotesError, NotesIndexOutcome, NotesIndexProgress,
notes_content_revision,
};
use notify::RecommendedWatcher;
@@ -525,18 +525,38 @@ impl State {
(needs_rerun, update_completed)
}
- fn report_indexing_progress(app: &AppHandle, collection_id: &str, progress: u8) {
+ fn report_indexing_progress(
+ app: &AppHandle,
+ collection_id: &str,
+ progress: NotesIndexProgress,
+ ) {
let state = app.state::<State>();
let mut runtime = state.runtime(collection_id);
+ let previous_progress = runtime.indexing_progress;
if !matches!(
runtime.status,
NotesCollectionStatusDto::Indexing | NotesCollectionStatusDto::Updating
- ) || runtime.indexing_progress == Some(progress)
+ ) || previous_progress == Some(progress.percentage)
{
return;
}
- runtime.indexing_progress = Some(progress);
+ runtime.indexing_progress = Some(progress.percentage);
+ runtime.indexed_document_count = progress.indexed_document_count;
state.set_runtime(collection_id, runtime);
+ if progress.percentage == 100
+ || previous_progress.is_none_or(|previous| previous / 10 != progress.percentage / 10)
+ {
+ crate::logging::log(
+ "Notes",
+ format!(
+ "index progress collection={collection_id} percent={} processed={} indexed={} total={}",
+ progress.percentage,
+ progress.processed_document_count,
+ progress.indexed_document_count,
+ progress.total_document_count
+ ),
+ );
+ }
emit_state_changed(app, collection_id);
}
@@ -747,7 +767,12 @@ fn register_collection(
},
);
}
- notes_state.collection_dto(&collection.id)
+ let dto = notes_state.collection_dto(&collection.id)?;
+ crate::logging::log(
+ "Notes",
+ format!("collection registered collection={}", collection.id),
+ );
+ Ok(dto)
}
#[tauri::command]
@@ -817,6 +842,10 @@ fn remove_collection(notes_state: &State, collection_id: &str) -> Result<(), Api
.lock()
.unwrap_or_else(PoisonError::into_inner)
.remove(collection_id);
+ crate::logging::log(
+ "Notes",
+ format!("collection removed collection={collection_id}"),
+ );
Ok(())
}
@@ -850,6 +879,13 @@ fn report_index_error(
};
runtime.last_error = (!cancelled && !superseded_file_error).then(|| error.message.clone());
state.set_runtime(collection_id, runtime);
+ crate::logging::log(
+ "Notes",
+ format!(
+ "index stopped collection={collection_id} reason={}",
+ error.name.unwrap_or("unknown")
+ ),
+ );
emit_state_changed(app, collection_id);
}
@@ -862,6 +898,11 @@ pub async fn notes_index_collection(
force: bool,
retrieval_epoch: u64,
) -> Result<NotesIndexResultDto, ApiError> {
+ let request_started_at = Instant::now();
+ crate::logging::log(
+ "Notes",
+ format!("index requested collection={collection_id} force={force}"),
+ );
let notes_state = available_state(&app)?;
let lifecycle = notes_state.collection_lifecycle(&collection_id);
let _collection_lifecycle = lifecycle.lock().await;
@@ -883,9 +924,21 @@ pub async fn notes_index_collection(
let mut update = notes_state.prepare_update(&collection_id, force, had_index)?;
update.rebuild_derived_index = rebuild_derived_index;
let cancellation_epoch = llm_state.retrieval_epoch();
- check_cancelled(&cancellation_epoch, retrieval_epoch)?;
+ if let Err(error) = check_cancelled(&cancellation_epoch, retrieval_epoch) {
+ crate::logging::log(
+ "Notes",
+ format!("index cancelled before start collection={collection_id}"),
+ );
+ return Err(error);
+ }
let _model_lifecycle = llm_state.lifecycle().lock().await;
- check_cancelled(&cancellation_epoch, retrieval_epoch)?;
+ if let Err(error) = check_cancelled(&cancellation_epoch, retrieval_epoch) {
+ crate::logging::log(
+ "Notes",
+ format!("index cancelled while waiting collection={collection_id}"),
+ );
+ return Err(error);
+ }
crate::commands::llm::replace_state(&llm_state, None, None)?;
let mut runtime = notes_state.runtime(&collection_id);
@@ -902,6 +955,12 @@ pub async fn notes_index_collection(
};
runtime.last_error = None;
notes_state.set_runtime(&collection_id, runtime);
+ crate::logging::log(
+ "Notes",
+ format!(
+ "index started collection={collection_id} updating={had_index} rebuild={rebuild_derived_index}"
+ ),
+ );
emit_state_changed(&app, &collection_id);
let store = model_state.store();
let embedding_asset = model::knowledge_embedding_model_asset();
@@ -991,6 +1050,14 @@ pub async fn notes_index_collection(
}
emit_state_changed(&app, &collection_id);
let dto = notes_state.collection_dto(&collection_id)?;
+ crate::logging::log(
+ "Notes",
+ format!(
+ "index completed collection={collection_id} indexed={} needs_rerun={needs_rerun} elapsed_ms={}",
+ outcome.indexed_document_count,
+ request_started_at.elapsed().as_millis()
+ ),
+ );
Ok(NotesIndexResultDto {
collection: dto,
needs_rerun,
diff --git a/rust/apps/ensu/src-tauri/src/commands/notes/indexing.rs b/rust/apps/ensu/src-tauri/src/commands/notes/indexing.rs
index c552686386..17f57b4566 100644
--- a/rust/apps/ensu/src-tauri/src/commands/notes/indexing.rs
+++ b/rust/apps/ensu/src-tauri/src/commands/notes/indexing.rs
@@ -3,8 +3,8 @@ use std::path::Path;
use ente_ensu::llm;
use ente_ensu::notes::{
NotesCollectionIndex, NotesDocumentLoad, NotesError, NotesIndexInput, NotesIndexOutcome,
- NotesIndexingError, NotesRevisionStatus, index_notes_collection, notes_content_revision,
- prepare_notes_document,
+ NotesIndexProgress, NotesIndexingError, NotesRevisionStatus, index_notes_collection,
+ notes_content_revision, prepare_notes_document,
};
use crate::commands::common::ApiError;
@@ -25,7 +25,7 @@ pub(super) fn index_collection(
cancellation_epoch: &std::sync::atomic::AtomicU64,
retrieval_epoch: u64,
update: &UpdateSnapshot,
- on_progress: impl FnMut(u8),
+ on_progress: impl FnMut(NotesIndexProgress),
) -> Result<NotesIndexOutcome, ApiError> {
check_cancelled(cancellation_epoch, retrieval_epoch)?;
let canonical_root = canonical_source_root(&collection.source_root)?;
diff --git a/rust/crates/ensu/src/llm/embed.rs b/rust/crates/ensu/src/llm/embed.rs
index 79894adcb2..9c6e6e34fa 100644
--- a/rust/crates/ensu/src/llm/embed.rs
+++ b/rust/crates/ensu/src/llm/embed.rs
@@ -74,8 +74,8 @@ impl Context {
message: err.to_string(),
})?;
}
- context.decode(&mut batch).map_err(|err| Error::Llama {
- op: "Embedding decode failed",
+ context.encode(&mut batch).map_err(|err| Error::Llama {
+ op: "Embedding encode failed",
message: err.to_string(),
})?;
let native = context.embeddings_seq_ith(0).map_err(|err| Error::Llama {
diff --git a/rust/crates/ensu/src/notes.rs b/rust/crates/ensu/src/notes.rs
index bf31196833..1dfc10df0d 100644
--- a/rust/crates/ensu/src/notes.rs
+++ b/rust/crates/ensu/src/notes.rs
@@ -44,8 +44,8 @@ pub use document::{
};
pub use index::{NotesCollectionIndex, NotesCollectionIndexSummary, NotesSearchHit};
pub use indexing::{
- NotesDocumentLoad, NotesIndexInput, NotesIndexOutcome, NotesIndexingError, NotesRevisionStatus,
- index_notes_collection,
+ NotesDocumentLoad, NotesIndexInput, NotesIndexOutcome, NotesIndexProgress, NotesIndexingError,
+ NotesRevisionStatus, index_notes_collection,
};
pub use reconcile::NotesReconciliationPlan;
pub use source::NoteSourceReference;
diff --git a/rust/crates/ensu/src/notes/indexing.rs b/rust/crates/ensu/src/notes/indexing.rs
index 42eca251ca..0b29d3cc04 100644
--- a/rust/crates/ensu/src/notes/indexing.rs
+++ b/rust/crates/ensu/src/notes/indexing.rs
@@ -50,6 +50,14 @@ pub struct NotesIndexInput<'a> {
pub force_full_hash: bool,
}
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub struct NotesIndexProgress {
+ pub percentage: u8,
+ pub processed_document_count: u64,
+ pub indexed_document_count: u64,
+ pub total_document_count: u64,
+}
+
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct NotesIndexOutcome {
pub changed_during_indexing: Option<String>,
@@ -69,7 +77,7 @@ pub fn index_notes_collection<E>(
mut load_document: impl FnMut(&str) -> Result<NotesDocumentLoad, E>,
mut embed_document: impl FnMut(&PreparedNotesDocument) -> Result<Option<Vec<Vec<f32>>>, E>,
mut verify_revision: impl FnMut(&str, &str) -> Result<NotesRevisionStatus, E>,
- mut on_progress: impl FnMut(u8),
+ mut on_progress: impl FnMut(NotesIndexProgress),
) -> Result<NotesIndexOutcome, NotesIndexingError<E>> {
let NotesIndexInput {
index_root,
@@ -147,10 +155,12 @@ pub fn index_notes_collection<E>(
writer.validate_reconciliation_capacity(&verified_inventory, &prepared_chunk_counts)?;
let mut progress_document_count = source_document_count.saturating_sub(requested.len() as u64);
- on_progress(indexing_progress(
+ report_indexing_progress(
+ &writer,
progress_document_count,
source_document_count,
- ));
+ &mut on_progress,
+ );
let mut processed_requested = 0_usize;
let mut embedded_documents = 0_usize;
let mut embedded_chunks = 0_usize;
@@ -197,10 +207,12 @@ pub fn index_notes_collection<E>(
writer.commit_deletions(std::slice::from_ref(document_id))?;
processed_requested += 1;
progress_document_count += 1;
- on_progress(indexing_progress(
+ report_indexing_progress(
+ &writer,
progress_document_count,
source_document_count,
- ));
+ &mut on_progress,
+ );
continue;
};
match writer.validate_document(&prepared, &source_metadata) {
@@ -209,10 +221,12 @@ pub fn index_notes_collection<E>(
writer.commit_deletions(std::slice::from_ref(document_id))?;
processed_requested += 1;
progress_document_count += 1;
- on_progress(indexing_progress(
+ report_indexing_progress(
+ &writer,
progress_document_count,
source_document_count,
- ));
+ &mut on_progress,
+ );
continue;
}
Err(error) => return Err(error.into()),
@@ -222,10 +236,12 @@ pub fn index_notes_collection<E>(
writer.commit_deletions(std::slice::from_ref(document_id))?;
processed_requested += 1;
progress_document_count += 1;
- on_progress(indexing_progress(
+ report_indexing_progress(
+ &writer,
progress_document_count,
source_document_count,
- ));
+ &mut on_progress,
+ );
continue;
};
check_for_cancellation().map_err(NotesIndexingError::Adapter)?;
@@ -254,10 +270,12 @@ pub fn index_notes_collection<E>(
writer.commit_document(&prepared, &embeddings, &verified_source)?;
processed_requested += 1;
progress_document_count += 1;
- on_progress(indexing_progress(
+ report_indexing_progress(
+ &writer,
progress_document_count,
source_document_count,
- ));
+ &mut on_progress,
+ );
embedded_documents += 1;
embedded_chunks += prepared.chunks.len();
if processed_requested < requested.len()
@@ -274,10 +292,24 @@ pub fn index_notes_collection<E>(
Ok(NotesIndexOutcome {
changed_during_indexing: None,
unchecked_document_ids: Vec::new(),
- indexed_document_count: writer.indexed_document_ids().count() as u64,
+ indexed_document_count: writer.indexed_document_count() as u64,
})
}
+fn report_indexing_progress(
+ writer: &NotesIndexWriter,
+ processed_document_count: u64,
+ total_document_count: u64,
+ on_progress: &mut impl FnMut(NotesIndexProgress),
+) {
+ on_progress(NotesIndexProgress {
+ percentage: indexing_progress(processed_document_count, total_document_count),
+ processed_document_count,
+ indexed_document_count: writer.indexed_document_count() as u64,
+ total_document_count,
+ });
+}
+
fn indexing_progress(processed_document_count: u64, source_document_count: u64) -> u8 {
if source_document_count == 0 {
return 100;
@@ -303,13 +335,13 @@ fn changed_during_indexing_outcome<E>(
.skip(processed_requested)
.cloned()
.collect(),
- indexed_document_count: writer.indexed_document_ids().count() as u64,
+ indexed_document_count: writer.indexed_document_count() as u64,
})
}
#[cfg(test)]
mod tests {
- use std::cell::Cell;
+ use std::cell::{Cell, RefCell};
use super::*;
use crate::notes::{NotesCollectionIndex, prepare_notes_document};
@@ -345,7 +377,7 @@ mod tests {
let temp = tempfile::tempdir().unwrap();
let prepared = prepare_notes_document("note.md", b"note").unwrap();
let verified_source = source_at(2);
- let progress = Cell::new(0);
+ let progress = RefCell::new(Vec::new());
let outcome = index_notes_collection::<()>(
NotesIndexInput {
index_root: temp.path(),
@@ -367,12 +399,29 @@ mod tests {
source: verified_source.clone(),
})
},
- |value| progress.set(value),
+ |value| progress.borrow_mut().push(value),
)
.unwrap();
assert!(outcome.ready());
- assert_eq!(progress.get(), 100);
+ let progress = progress.borrow();
+ assert_eq!(
+ progress.as_slice(),
+ [
+ NotesIndexProgress {
+ percentage: 0,
+ processed_document_count: 0,
+ indexed_document_count: 0,
+ total_document_count: 1,
+ },
+ NotesIndexProgress {
+ percentage: 100,
+ processed_document_count: 1,
+ indexed_document_count: 1,
+ total_document_count: 1,
+ },
+ ]
+ );
let index = NotesCollectionIndex::open(temp.path(), COLLECTION_ID.to_string()).unwrap();
assert_eq!(index.document_count(), 1);
let writer = NotesIndexWriter::open(temp.path(), COLLECTION_ID.to_string()).unwrap();
diff --git a/rust/crates/ensu/src/notes/writer.rs b/rust/crates/ensu/src/notes/writer.rs
index 79803830c2..45e1eb2be8 100644
--- a/rust/crates/ensu/src/notes/writer.rs
+++ b/rust/crates/ensu/src/notes/writer.rs
@@ -146,6 +146,10 @@ impl NotesIndexWriter {
self.manifest.documents.keys().map(String::as_str)
}
+ pub fn indexed_document_count(&self) -> usize {
+ self.manifest.documents.len()
+ }
+
pub fn indexed_revision(&self, document_id: &str) -> Option<&str> {
self.manifest
.documents
diff --git a/web/apps/ensu/package.json b/web/apps/ensu/package.json
index ddd4fd1561..9988398020 100644
--- a/web/apps/ensu/package.json
+++ b/web/apps/ensu/package.json
@@ -2,6 +2,9 @@
"name": "ensu",
"version": "0.0.0",
"private": true,
+ "scripts": {
+ "test": "vitest --run"
+ },
"dependencies": {
"@hugeicons/core-free-icons": "1.2.1",
"@hugeicons/react": "1.1.6",
@@ -24,6 +27,7 @@
"remark-math": "6.0.0"
},
"devDependencies": {
- "ente-build-config": "*"
+ "ente-build-config": "*",
+ "vitest": "4.1.8"
}
}
diff --git a/web/apps/ensu/src/components/chat/ChatDialogs.tsx b/web/apps/ensu/src/components/chat/ChatDialogs.tsx
index f51643fe8f..1d1179677d 100644
--- a/web/apps/ensu/src/components/chat/ChatDialogs.tsx
+++ b/web/apps/ensu/src/components/chat/ChatDialogs.tsx
@@ -1,5 +1,5 @@
import type { KnowledgePack } from "@/services/knowledge";
-import type { NotesCollection } from "@/services/notes";
+import type { NotesCollectionView } from "@/services/notes-lifecycle";
import { isTauriRuntime as detectTauriAppRuntime } from "@/services/tauri-runtime";
import {
ArrowLeft01Icon,
@@ -153,7 +153,7 @@ export interface ChatDialogsProps {
handleDownloadKnowledgePack: (stableId: string) => void;
handleCancelKnowledgePackDownload: (stableId: string) => void;
handleSetKnowledgePackEnabled: (stableId: string, enabled: boolean) => void;
- notesCollections: NotesCollection[];
+ notesCollections: NotesCollectionView[];
notesCollectionsLoading: boolean;
notesCollectionsError: string | null;
retryNotesCollections: () => void;
@@ -823,16 +823,23 @@ export const ChatDialogs = memo(
const indexing =
collection.status === "indexing" ||
collection.status === "updating";
+ const starting =
+ collection.activity === "starting";
const progress = collection.indexingProgress;
const canRetry =
- collection.status === "pending" ||
collection.status === "error" ||
- collection.status === "unavailable";
- const retryLabel =
- collection.status === "error" ||
- collection.status === "unavailable"
- ? "Try again"
- : "Continue indexing";
+ collection.status === "unavailable" ||
+ collection.activity === "failed";
+ const waitingMessage =
+ collection.activity ===
+ "waitingForGeneration"
+ ? "Indexing will resume after the current response."
+ : collection.activity ===
+ "waitingForModel"
+ ? "Indexing will start when the model is ready."
+ : collection.activity === "scheduled"
+ ? "Indexing is scheduled after recent changes settle."
+ : null;
return (
<Stack
key={collection.id}
@@ -861,13 +868,11 @@ export const ChatDialogs = memo(
noWrap
sx={{ color: "text.muted" }}
>
- {
- collection.indexedDocumentCount
- }{" "}
- {collection.indexedDocumentCount ===
- 1
- ? "note indexed"
- : "notes indexed"}
+ {starting &&
+ collection.indexedDocumentCount ===
+ 0
+ ? "Preparing notes…"
+ : `${collection.indexedDocumentCount} ${collection.indexedDocumentCount === 1 ? "note indexed" : "notes indexed"}`}
{collection.lastUpdatedAtMs !=
null && (
<>
@@ -881,7 +886,7 @@ export const ChatDialogs = memo(
</Stack>
<IconButton
aria-label={`Remove ${collection.label}`}
- disabled={indexing}
+ disabled={indexing || starting}
onClick={() =>
handleRemoveNotesCollection(
collection.id,
@@ -917,6 +922,15 @@ export const ChatDialogs = memo(
</Typography>
</Stack>
)}
+ {starting && <LinearProgress />}
+ {waitingMessage && (
+ <Typography
+ variant="small"
+ sx={{ color: "text.muted" }}
+ >
+ {waitingMessage}
+ </Typography>
+ )}
{collection.lastError && (
<Typography
variant="small"
@@ -936,7 +950,7 @@ export const ChatDialogs = memo(
}
sx={{ alignSelf: "flex-start" }}
>
- {retryLabel}
+ Try again
</Button>
)}
</Stack>
diff --git a/web/apps/ensu/src/hooks/use-notes-collections.ts b/web/apps/ensu/src/hooks/use-notes-collections.ts
index 57413e0aa4..7b381b45d8 100644
--- a/web/apps/ensu/src/hooks/use-notes-collections.ts
+++ b/web/apps/ensu/src/hooks/use-notes-collections.ts
@@ -9,10 +9,14 @@ import {
selectNotesFolder,
type NotesCollection,
} from "@/services/notes";
-import { NotesLifecycleController } from "@/services/notes-lifecycle";
+import {
+ NotesLifecycleController,
+ type NotesCollectionActivity,
+ type NotesCollectionView,
+} from "@/services/notes-lifecycle";
import { tauriCommandError } from "@/services/tauri-error";
import log from "ente-base/log";
-import { useCallback, useEffect, useRef, useState } from "react";
+import { useCallback, useEffect, useMemo, useRef, useState } from "react";
const NOTES_INDEX_RERUN_YIELD_MS = 25;
const NOTES_REMOVE_WAIT_ERROR =
@@ -45,14 +49,16 @@ export const useNotesCollections = ({
null,
);
const [syncAttempt, setSyncAttempt] = useState(0);
+ const [lifecycleRevision, setLifecycleRevision] = useState(0);
const controllerRef = useRef<NotesLifecycleController | null>(null);
if (!controllerRef.current) {
controllerRef.current = new NotesLifecycleController();
}
const controller = controllerRef.current;
- const runIndexRef = useRef<
- ((collectionId: string, force?: boolean) => void) | null
- >(null);
+ const lifecycleChanged = useCallback(
+ () => setLifecycleRevision((revision) => revision + 1),
+ [],
+ );
useEffect(() => {
controller.activate();
@@ -122,15 +128,25 @@ export const useNotesCollections = ({
[controller],
);
- const runIndex = useCallback(
- (collectionId: string, force = false) => {
- const request = controller.requestIndex(
+ const startIndex = useCallback(
+ (collectionId: string, force = false, automatic = false) => {
+ const blocked =
+ !modelReady || isGenerating || isGenerationStarting();
+ const request = automatic
+ ? controller.ensureIndex(collectionId, blocked)
+ : controller.requestIndex(collectionId, force, blocked);
+ if (request === "ignored") return;
+ lifecycleChanged();
+ if (request === "queued") {
+ log.info("Your Notes indexing queued", { collectionId });
+ return;
+ }
+ setOperationError(null);
+ log.info("Your Notes indexing started", {
collectionId,
+ automatic,
force,
- isGenerating || isGenerationStarting(),
- );
- if (request !== "start") return;
- setOperationError(null);
+ });
void ensureProvider()
.then((provider) =>
@@ -173,7 +189,12 @@ export const useNotesCollections = ({
const { name } = tauriCommandError(error);
if (name === "cancelled") {
controller.queueIndex(collectionId, force);
+ log.info(
+ "Your Notes indexing queued after cancellation",
+ { collectionId },
+ );
} else if (name !== "not_due") {
+ controller.failIndex(collectionId);
setOperationError(notesErrorMessage(error));
log.error("Failed to index Your Notes", error);
}
@@ -181,12 +202,8 @@ export const useNotesCollections = ({
})
.finally(() => {
const completion = controller.finishIndex(collectionId);
- if (completion.queuedForce !== undefined) {
- runIndexRef.current?.(
- collectionId,
- completion.queuedForce,
- );
- } else if (completion.clearedRemovalWait) {
+ lifecycleChanged();
+ if (completion.clearedRemovalWait) {
setOperationError((current) =>
current === NOTES_REMOVE_WAIT_ERROR
? null
@@ -200,19 +217,37 @@ export const useNotesCollections = ({
ensureProvider,
isGenerating,
isGenerationStarting,
+ lifecycleChanged,
+ modelReady,
refresh,
replaceCollection,
],
);
- runIndexRef.current = runIndex;
+
+ const runIndex = useCallback(
+ (collectionId: string, force = false) =>
+ startIndex(collectionId, force),
+ [startIndex],
+ );
useEffect(() => {
- for (const [collectionId, force] of controller.drainQueued(
- isGenerating || isGenerationStarting(),
- )) {
- runIndex(collectionId, force);
+ const blocked = !modelReady || isGenerating || isGenerationStarting();
+ const queued = controller.drainQueued(blocked);
+ if (queued.length > 0) {
+ lifecycleChanged();
+ for (const [collectionId, force] of queued) {
+ startIndex(collectionId, force);
+ }
}
- }, [controller, isGenerating, isGenerationStarting, runIndex]);
+ }, [
+ controller,
+ isGenerating,
+ isGenerationStarting,
+ lifecycleChanged,
+ lifecycleRevision,
+ modelReady,
+ startIndex,
+ ]);
useEffect(() => {
if (!isTauriRuntime) return;
@@ -275,13 +310,18 @@ export const useNotesCollections = ({
for (const collection of collections) {
if (
collection.status === "pending" &&
- collection.updateDueAtMs == null &&
- controller.markResumed(collection.id)
+ collection.updateDueAtMs == null
) {
- runIndex(collection.id);
+ startIndex(collection.id, false, true);
}
}
- }, [collections, controller, isTauriRuntime, modelReady, runIndex]);
+ }, [
+ collections,
+ isTauriRuntime,
+ lifecycleRevision,
+ modelReady,
+ startIndex,
+ ]);
const retry = useCallback(() => {
setOperationError(null);
@@ -324,19 +364,57 @@ export const useNotesCollections = ({
log.error("Failed to remove Notes collection", error);
void refresh();
if (removal.queuedForce !== undefined) {
- runIndexRef.current?.(
- collectionId,
- removal.queuedForce,
- );
+ startIndex(collectionId, removal.queuedForce);
}
}
});
},
- [confirmRemoval, controller, refresh],
+ [confirmRemoval, controller, refresh, startIndex],
+ );
+
+ const collectionViews = useMemo<NotesCollectionView[]>(
+ () =>
+ collections.map((collection) => {
+ if (collection.status !== "pending") {
+ return { ...collection, activity: null };
+ }
+ const lifecycleActivity = controller.indexActivity(
+ collection.id,
+ );
+ let activity: NotesCollectionActivity;
+ if (lifecycleActivity === "failed") {
+ activity = "failed";
+ } else if (lifecycleActivity === "starting") {
+ activity = "starting";
+ } else if (lifecycleActivity === "queued") {
+ activity = !modelReady
+ ? "waitingForModel"
+ : isGenerating || isGenerationStarting()
+ ? "waitingForGeneration"
+ : "starting";
+ } else if (collection.updateDueAtMs != null) {
+ activity = "scheduled";
+ } else if (!modelReady) {
+ activity = "waitingForModel";
+ } else if (isGenerating || isGenerationStarting()) {
+ activity = "waitingForGeneration";
+ } else {
+ activity = "starting";
+ }
+ return { ...collection, activity };
+ }),
+ [
+ collections,
+ controller,
+ isGenerating,
+ isGenerationStarting,
+ lifecycleRevision,
+ modelReady,
+ ],
);
return {
- collections,
+ collections: collectionViews,
loading,
error: subscriptionError ?? listError ?? operationError,
retry,
diff --git a/web/apps/ensu/src/services/notes-lifecycle.ts b/web/apps/ensu/src/services/notes-lifecycle.ts
index edb59226b1..031636c1d7 100644
--- a/web/apps/ensu/src/services/notes-lifecycle.ts
+++ b/web/apps/ensu/src/services/notes-lifecycle.ts
@@ -1,10 +1,25 @@
+import type { NotesCollection } from "@/services/notes";
+
type NotesIndexRequest = "start" | "queued" | "ignored";
interface NotesIndexCompletion {
- queuedForce?: boolean;
clearedRemovalWait: boolean;
}
+type NotesIndexActivity = "starting" | "queued" | "failed" | null;
+
+export type NotesCollectionActivity =
+ | "starting"
+ | "waitingForGeneration"
+ | "waitingForModel"
+ | "scheduled"
+ | "failed"
+ | null;
+
+export interface NotesCollectionView extends NotesCollection {
+ activity: NotesCollectionActivity;
+}
+
type NotesRemovalStart =
| { kind: "wait" }
| { kind: "start"; queuedForce?: boolean };
@@ -12,8 +27,8 @@ type NotesRemovalStart =
export class NotesLifecycleController {
private readonly active = new Set<string>();
private readonly queued = new Map<string, boolean>();
+ private readonly failed = new Set<string>();
private readonly removing = new Set<string>();
- private readonly resumed = new Set<string>();
private removalWaitCollection: string | null = null;
private mutation = 0;
private refreshRequest = 0;
@@ -25,17 +40,40 @@ export class NotesLifecycleController {
blocked: boolean,
): NotesIndexRequest {
if (this.disposed || this.removing.has(collectionId)) return "ignored";
- if (blocked || this.active.has(collectionId)) {
+ this.failed.delete(collectionId);
+ if (
+ blocked ||
+ this.active.has(collectionId) ||
+ this.queued.has(collectionId)
+ ) {
this.queueIndex(collectionId, force);
return "queued";
}
- this.resumed.add(collectionId);
+ this.active.add(collectionId);
+ return "start";
+ }
+
+ ensureIndex(collectionId: string, blocked: boolean): NotesIndexRequest {
+ if (
+ this.disposed ||
+ this.removing.has(collectionId) ||
+ this.failed.has(collectionId) ||
+ this.active.has(collectionId) ||
+ this.queued.has(collectionId)
+ ) {
+ return "ignored";
+ }
+ if (blocked) {
+ this.queueIndex(collectionId, false);
+ return "queued";
+ }
this.active.add(collectionId);
return "start";
}
queueIndex(collectionId: string, force: boolean) {
if (this.disposed || this.removing.has(collectionId)) return;
+ this.failed.delete(collectionId);
this.queued.set(
collectionId,
force || (this.queued.get(collectionId) ?? false),
@@ -52,11 +90,20 @@ export class NotesLifecycleController {
finishIndex(collectionId: string): NotesIndexCompletion {
this.active.delete(collectionId);
- const queuedForce = this.queued.get(collectionId);
- this.queued.delete(collectionId);
const clearedRemovalWait = this.removalWaitCollection === collectionId;
if (clearedRemovalWait) this.removalWaitCollection = null;
- return { queuedForce, clearedRemovalWait };
+ return { clearedRemovalWait };
+ }
+
+ failIndex(collectionId: string) {
+ this.queued.delete(collectionId);
+ this.failed.add(collectionId);
+ }
+
+ indexActivity(collectionId: string): NotesIndexActivity {
+ if (this.failed.has(collectionId)) return "failed";
+ if (this.queued.has(collectionId)) return "queued";
+ return this.active.has(collectionId) ? "starting" : null;
}
drainQueued(blocked: boolean) {
@@ -92,7 +139,7 @@ export class NotesLifecycleController {
finishRemoval(collectionId: string) {
this.removing.delete(collectionId);
- this.resumed.delete(collectionId);
+ this.failed.delete(collectionId);
if (this.removalWaitCollection === collectionId) {
this.removalWaitCollection = null;
}
@@ -102,12 +149,6 @@ export class NotesLifecycleController {
return this.removing.has(collectionId);
}
- markResumed(collectionId: string) {
- if (this.disposed || this.resumed.has(collectionId)) return false;
- this.resumed.add(collectionId);
- return true;
- }
-
hasActiveJobs() {
return this.active.size > 0;
}
@@ -145,6 +186,7 @@ export class NotesLifecycleController {
this.disposed = true;
this.active.clear();
this.queued.clear();
+ this.failed.clear();
this.removing.clear();
this.cancelRefreshes();
}
diff --git a/web/apps/ensu/tests/notes-lifecycle.test.ts b/web/apps/ensu/tests/notes-lifecycle.test.ts
new file mode 100644
index 0000000000..8206367461
--- /dev/null
+++ b/web/apps/ensu/tests/notes-lifecycle.test.ts
@@ -0,0 +1,31 @@
+import { describe, expect, test } from "vitest";
+import { NotesLifecycleController } from "../src/services/notes-lifecycle";
+
+describe("NotesLifecycleController", () => {
+ test("recovers cancelled indexing without duplicating or losing queued work", () => {
+ const controller = new NotesLifecycleController();
+
+ expect(controller.ensureIndex("notes", false)).toBe("start");
+ expect(controller.indexActivity("notes")).toBe("starting");
+ controller.queueIndex("notes", false);
+ expect(controller.finishIndex("notes")).toEqual({
+ clearedRemovalWait: false,
+ });
+ expect(controller.indexActivity("notes")).toBe("queued");
+ expect(controller.ensureIndex("notes", false)).toBe("ignored");
+ expect(controller.drainQueued(true)).toEqual([]);
+ expect(controller.drainQueued(false)).toEqual([["notes", false]]);
+
+ expect(controller.requestIndex("notes", false, false)).toBe("start");
+ expect(controller.requestIndex("notes", true, false)).toBe("queued");
+ controller.finishIndex("notes");
+ expect(controller.drainQueued(false)).toEqual([["notes", true]]);
+
+ expect(controller.requestIndex("notes", true, false)).toBe("start");
+ controller.failIndex("notes");
+ controller.finishIndex("notes");
+ expect(controller.indexActivity("notes")).toBe("failed");
+ expect(controller.ensureIndex("notes", false)).toBe("ignored");
+ expect(controller.requestIndex("notes", true, false)).toBe("start");
+ });
+});
diff --git a/web/apps/ensu/tsconfig.json b/web/apps/ensu/tsconfig.json
index 2ff1f734c6..6a2a667238 100644
--- a/web/apps/ensu/tsconfig.json
+++ b/web/apps/ensu/tsconfig.json
@@ -9,6 +9,7 @@
},
"include": [
"src",
+ "tests",
"next-env.d.ts",
"next.config.mjs",
"../../packages/base/global-electron.d.ts",
diff --git a/web/package-lock.json b/web/package-lock.json
index 4e037b9968..7df478b760 100644
--- a/web/package-lock.json
+++ b/web/package-lock.json
@@ -191,7 +191,8 @@
"remark-math": "6.0.0"
},
"devDependencies": {
- "ente-build-config": "*"
+ "ente-build-config": "*",
+ "vitest": "4.1.8"
}
},
"apps/ensu/node_modules/katex": {
diff --git a/web/package.json b/web/package.json
index 3bee38bffc..11ea951759 100644
--- a/web/package.json
+++ b/web/package.json
@@ -52,7 +52,7 @@
"dev:twoof3": "npm exec --workspace twoof3 -- next dev --webpack -p 3012",
"lint": "concurrently --names 'prettier,eslint,tsc,wasm-imports' \"prettier --check --log-level warn .\" \"npm exec --workspaces -- eslint --max-warnings 0\" \"npm exec --workspaces -- tsc\" \"npm run check:wasm-imports\"",
"lint:fix": "concurrently --names 'prettier,eslint,tsc' \"prettier --write --log-level warn .\" \"npm exec --workspaces -- eslint --fix --max-warnings 0\" \"npm exec --workspaces -- tsc\"",
- "test": "npm run test --workspace ente-core-wasm && npm run test --workspace ente-paste-wasm && npm run test --workspace ente-contacts && npm run test --workspace locker && npm run test --workspace photos"
+ "test": "npm run test --workspace ensu && npm run test --workspace ente-core-wasm && npm run test --workspace ente-paste-wasm && npm run test --workspace ente-contacts && npm run test --workspace locker && npm run test --workspace photos"
},
"overrides": {
"@types/react": "19.2.17",