LLM Wiki is a cross-platform desktop application that turns your documents into an organized, interlinked knowledge base — automatically. Instead of traditional RAG (retrieve-and-answer from scratch every time), the LLM incrementally builds and maintains a persistent wiki from your sources。
TypeScript
fix: harden concurrent ingest lifecycle
nash_sufa2652eb8186
diff --git a/src/lib/ingest-queue.test.ts b/src/lib/ingest-queue.test.tsindex ad058a25..10742f5b 100644--- a/src/lib/ingest-queue.test.ts+++ b/src/lib/ingest-queue.test.ts@@ -75,13 +75,14 @@ import {getIngestWorkerLimit,} from "./ingest-queue"import { autoIngest } from "./ingest"-import { readFile, writeFile } from "@/commands/fs"+import { deleteFile, readFile, writeFile } from "@/commands/fs"import { sweepResolvedReviews } from "./sweep-reviews"import { useWikiStore } from "@/stores/wiki-store"const mockAutoIngest = vi.mocked(autoIngest)const mockReadFile = vi.mocked(readFile)const mockWriteFile = vi.mocked(writeFile)+const mockDeleteFile = vi.mocked(deleteFile)const mockSweep = vi.mocked(sweepResolvedReviews)/** Simulate the app having opened `TEST_ID` at `TEST_PATH` so the queue@@ -97,6 +98,8 @@ beforeEach(async () => {mockAutoIngest.mockReset()mockReadFile.mockReset()mockWriteFile.mockReset()+ mockDeleteFile.mockReset()+ mockDeleteFile.mockResolvedValue(undefined as unknown as void)mockSweep.mockReset()mockSweep.mockResolvedValue(0)removePageEmbeddingMock.mockReset()@@ -299,6 +302,63 @@ describe("ingest-queue — concurrent workers", () => {expect(getQueue().find((task) => task.sourcePath === "a.md")?.status).toBe("cancelled")})+ it("does not delete merged pages when cancelling a worker", async () => {+ const finish = createDeferred<string[]>()+ mockAutoIngest.mockImplementation(+ async (_projectPath, _sourcePath, _config, _signal, _folder, onFileWritten) => {+ onFileWritten?.("wiki/concepts/existing.md")+ return finish.promise+ },+ )++ await enqueueIngest(TEST_ID, "cancel-during-write.md")+ await waitFor(() => getQueueSummary().processing === 1)+ const task = getQueue()[0]+ await cancelTask(task.id)++ expect(mockDeleteFile).not.toHaveBeenCalled()+ finish.resolve(["wiki/concepts/existing.md"])+ await flushMicrotasks(10)+ expect(mockDeleteFile).not.toHaveBeenCalled()+ })++ it("does not restart a cancelled task until its previous worker exits", async () => {+ setIngestWorkerLimit(2)+ const firstRun = createDeferred<string[]>()+ mockAutoIngest+ .mockImplementationOnce(() => firstRun.promise)+ .mockResolvedValueOnce(["wiki/sources/retried.md"])++ await enqueueIngest(TEST_ID, "retry-after-cancel.md")+ await waitFor(() => mockAutoIngest.mock.calls.length === 1)+ const task = getQueue()[0]+ await cancelTask(task.id)+ await retryTask(task.id)+ await flushMicrotasks(10)+ expect(mockAutoIngest).toHaveBeenCalledTimes(1)++ firstRun.reject(new Error("aborted"))+ await waitFor(() => mockAutoIngest.mock.calls.length === 2)+ await waitFor(() => getQueue().length === 0)+ })++ it("ignores retry requests for a task that is still processing", async () => {+ setIngestWorkerLimit(2)+ const run = createDeferred<string[]>()+ mockAutoIngest.mockImplementation(() => run.promise)++ await enqueueIngest(TEST_ID, "still-processing.md")+ await waitFor(() => getQueueSummary().processing === 1)+ const task = getQueue()[0]+ await retryTask(task.id)++ expect(task.status).toBe("processing")+ expect(mockAutoIngest).toHaveBeenCalledTimes(1)+ run.resolve(["wiki/sources/still-processing.md"])+ await waitFor(() => getQueue().length === 0)+ expect(mockAutoIngest).toHaveBeenCalledTimes(1)+ })+it("pauses and aborts every active worker after a provider usage limit", async () => {setIngestWorkerLimit(2)const signals: AbortSignal[] = []@@ -896,6 +956,31 @@ describe("ingest-queue — restoreQueue", () => {import { pauseQueue } from "./ingest-queue"describe("ingest-queue — pauseQueue & switch-project survival", () => {+ it("waits for an active commit before completing a project switch", async () => {+ const commit = createDeferred<void>()+ let commitStarted = false+ mockAutoIngest.mockImplementation(+ async (_projectPath, _sourcePath, _config, _signal, _folder, _onFileWritten, options) => {+ await options?.runCommit?.(async () => {+ commitStarted = true+ await commit.promise+ })+ return ["wiki/sources/committed.md"]+ },+ )++ await enqueueIngest(TEST_ID, "committing.md")+ await waitFor(() => commitStarted)+ let pauseFinished = false+ const pause = pauseQueue().then(() => { pauseFinished = true })+ await flushMicrotasks(5)+ expect(pauseFinished).toBe(false)++ commit.resolve()+ await pause+ expect(pauseFinished).toBe(true)+ })+it("pauseQueue persists pending/processing tasks to the paused project's disk", async () => {mockAutoIngest.mockImplementation(() => new Promise(() => {}))await enqueueBatch(TEST_ID, [diff --git a/src/lib/ingest-queue.ts b/src/lib/ingest-queue.tsindex ceb3cf14..f166f5ea 100644--- a/src/lib/ingest-queue.ts+++ b/src/lib/ingest-queue.ts@@ -33,6 +33,8 @@ interface ActiveIngestRun {controller: AbortControllerwrittenFiles: string[]releaseCommitTurn: () => void+ commitActive: boolean+ completion: Promise<void>}const activeRuns = new Map<string, ActiveIngestRun>()@@ -238,12 +240,14 @@ function upsertQueuedIngestTask(}/**- * Delete files written by a cancelled / failed ingest, AND drop the- * matching pages' chunks from LanceDB. Called from the cancel paths- * (cancelTask, cancelAllTasks).+ * Explicitly delete wiki files and their matching LanceDB chunks.+ *+ * Cancellation deliberately does not call this helper: written paths may be+ * pre-existing pages merged in place, and deleting them without snapshots+ * would destroy user data.** Per-file errors are swallowed (best-effort cleanup) — a missing- * file or LanceDB unavailable shouldn't abort the cancel flow.+ * file or LanceDB unavailable shouldn't abort the caller's cleanup flow.* Structural pages (index/log/overview) aren't embedded, so the* cascade no-ops for them.*/@@ -385,13 +389,14 @@ export async function enqueueInactiveProjectBatch(}/**- * Retry a failed task. Only valid for tasks in the active project's- * queue.+ * Retry a failed or cancelled task. Only valid for stopped tasks in the+ * active project's queue.*/export async function retryTask(taskId: string): Promise<void> {const task = queue.find((t) => t.id === taskId)if (!task) returnif (task.projectId !== currentProjectId) return+ if (task.status !== "failed" && task.status !== "cancelled") returnrestoredPausedTaskIds.delete(task.id)task.status = "pending"@@ -479,8 +484,9 @@ export async function movePendingTask(}/**- * Cancel a pending or processing task.- * If processing, aborts the LLM call and cleans up generated files.+ * Cancel a pending or processing task. Processing work is aborted, while any+ * completed atomic page writes are preserved because they may be merges into+ * pre-existing user content.*/export async function cancelTask(taskId: string): Promise<void> {const task = queue.find((t) => t.id === taskId)@@ -492,12 +498,6 @@ export async function cancelTask(taskId: string): Promise<void> {cancelledInFlightTaskIds.add(task.id)const run = activeRuns.get(task.id)run?.controller.abort()- if (run && run.writtenFiles.length > 0) {- await cleanupWrittenFiles(currentProjectPath, run.writtenFiles)- console.log(`[Ingest Queue] Cleaned up ${run.writtenFiles.length} files from cancelled task`)- run.writtenFiles = []- }-}restoredPausedTaskIds.delete(taskId)@@ -531,10 +531,6 @@ export async function cancelTasks(taskIds: readonly string[]): Promise<number> {for (const task of targets) {const run = activeRuns.get(task.id)run?.controller.abort()- if (run && run.writtenFiles.length > 0) {- await cleanupWrittenFiles(currentProjectPath, run.writtenFiles)- run.writtenFiles = []- }}for (const task of targets) {restoredPausedTaskIds.delete(task.id)@@ -583,10 +579,6 @@ export async function discardTasksForSources(for (const task of targets) {const run = activeRuns.get(task.id)run?.controller.abort()- if (run && run.writtenFiles.length > 0) {- await cleanupWrittenFiles(currentProjectPath, run.writtenFiles)- run.writtenFiles = []- }}queue = queue.filter((task) => !targetIds.has(task.id))@@ -609,8 +601,8 @@ export async function clearCompletedTasks(): Promise<void> {/*** Cancel everything that's not finished in the active project's queue:- * aborts the running task (if any), cleans up its partial output, and- * retains every pending + processing item as cancelled so it can be restarted.+ * aborts running tasks and retains every pending + processing item as+ * cancelled so it can be restarted.** Failed tasks are retained so the user can still see / retry them.* Returns the number of tasks removed from the queue.@@ -626,10 +618,6 @@ export async function cancelAllTasks(): Promise<number> {}for (const run of activeRuns.values()) {run.controller.abort()- if (run.writtenFiles.length > 0) {- await cleanupWrittenFiles(currentProjectPath, run.writtenFiles)- run.writtenFiles = []- }}let cancelled = 0@@ -788,7 +776,8 @@ export async function pauseQueue(): Promise<void> {const pausedProjectPath = currentProjectPathqueueEpoch += 1- for (const run of activeRuns.values()) {+ const runs = [...activeRuns.values()]+ for (const run of runs) {run.controller.abort()run.releaseCommitTurn()}@@ -796,9 +785,17 @@ export async function pauseQueue(): Promise<void> {sweepAbortController.abort()sweepAbortController = null}- activeRuns.clear()scheduling = false+ // Preparation is safe to abandon after abort, but a commit may already be+ // inside a filesystem operation that cannot observe AbortSignal. Wait only+ // for those active commits before declaring the project switch complete.+ const committingRuns = runs.filter((run) => run.commitActive)+ if (committingRuns.length > 0) {+ await Promise.allSettled(committingRuns.map((run) => run.completion))+ }+ activeRuns.clear()+// Revert any in-flight processing task back to pending so when the// user returns to this project, the task is re-tried from scratch.for (const task of queue) {@@ -978,10 +975,9 @@ async function runTask(const currentTask = queue.find((candidate) => candidate.id === task.id)if (cancelledInFlightTaskIds.delete(task.id) || currentTask?.status === "cancelled") {- if (run.writtenFiles.length > 0) {- await cleanupWrittenFiles(projectPath, run.writtenFiles)- run.writtenFiles = []- }+ // Written paths may be pre-existing pages that were merged in place.+ // Without a reliable old-content snapshot, deleting them would be data+ // loss. Keep completed atomic writes and let a retry converge via merge.await saveQueue(projectPath)return}@@ -997,18 +993,10 @@ async function runTask(if (currentProjectId !== projectId || run.epoch !== queueEpoch) returnconst currentTask = queue.find((candidate) => candidate.id === task.id)if (cancelledInFlightTaskIds.delete(task.id) || currentTask?.status === "cancelled") {- if (run.writtenFiles.length > 0) {- await cleanupWrittenFiles(projectPath, run.writtenFiles)- run.writtenFiles = []- }await saveQueue(projectPath)return}if (currentTask?.status === "pending") {- if (run.writtenFiles.length > 0) {- await cleanupWrittenFiles(projectPath, run.writtenFiles)- run.writtenFiles = []- }await saveQueue(projectPath)return}@@ -1078,6 +1066,8 @@ async function startTask(projectId: string, task: IngestTask): Promise<boolean>controller: new AbortController(),writtenFiles: [],releaseCommitTurn: reservation.release,+ commitActive: false,+ completion: Promise.resolve(),}restoredPausedTaskIds.delete(task.id)task.status = "processing"@@ -1091,7 +1081,16 @@ async function startTask(projectId: string, task: IngestTask): Promise<boolean>}console.log(`[Ingest Queue] Processing: ${task.sourcePath} (${queue.filter((candidate) => candidate.projectId === projectId && candidate.status === "pending").length} remaining)`)- void runTask(projectId, projectPath, task, run, reservation.runCommit)+ const orderedCommit: typeof reservation.runCommit = (operation) =>+ reservation.runCommit(async () => {+ run.commitActive = true+ try {+ return await operation()+ } finally {+ run.commitActive = false+ }+ })+ run.completion = runTask(projectId, projectPath, task, run, orderedCommit)return true}@@ -1105,6 +1104,7 @@ async function processNext(projectId: string): Promise<void> {const next = queue.find((task) =>task.projectId === projectId &&task.status === "pending" &&+ !activeRuns.has(task.id) &&!restoredPausedTaskIds.has(task.id))if (!next) break@@ -1114,6 +1114,7 @@ async function processNext(projectId: string): Promise<void> {const hasRunnablePending = queue.some((task) =>task.projectId === projectId &&task.status === "pending" &&+ !activeRuns.has(task.id) &&!restoredPausedTaskIds.has(task.id))const hasRestoredPending = queue.some((task) =>@@ -1132,6 +1133,7 @@ async function processNext(projectId: string): Promise<void> {const hasRunnable = queue.some((task) =>task.projectId === projectId &&task.status === "pending" &&+ !activeRuns.has(task.id) &&!restoredPausedTaskIds.has(task.id))const hasAnyPending = queue.some((task) =>