Skip to content

LLM Wiki is a cross-platform desktop application that turns your documents into an organized, interlinked knowledge base — automatically. Instead of traditional RAG (retrieve-and-answer from scratch every time), the LLM incrementally builds and maintains a persistent wiki from your sources。

TypeScript

fix: harden concurrent ingest lifecycle

nash_sufa2652eb8186
diff --git a/src/lib/ingest-queue.test.ts b/src/lib/ingest-queue.test.ts
index ad058a25..10742f5b 100644
--- a/src/lib/ingest-queue.test.ts
+++ b/src/lib/ingest-queue.test.ts
@@ -75,13 +75,14 @@ import {
getIngestWorkerLimit,
} from "./ingest-queue"
import { autoIngest } from "./ingest"
-import { readFile, writeFile } from "@/commands/fs"
+import { deleteFile, readFile, writeFile } from "@/commands/fs"
import { sweepResolvedReviews } from "./sweep-reviews"
import { useWikiStore } from "@/stores/wiki-store"
const mockAutoIngest = vi.mocked(autoIngest)
const mockReadFile = vi.mocked(readFile)
const mockWriteFile = vi.mocked(writeFile)
+const mockDeleteFile = vi.mocked(deleteFile)
const mockSweep = vi.mocked(sweepResolvedReviews)
/** Simulate the app having opened `TEST_ID` at `TEST_PATH` so the queue
@@ -97,6 +98,8 @@ beforeEach(async () => {
mockAutoIngest.mockReset()
mockReadFile.mockReset()
mockWriteFile.mockReset()
+ mockDeleteFile.mockReset()
+ mockDeleteFile.mockResolvedValue(undefined as unknown as void)
mockSweep.mockReset()
mockSweep.mockResolvedValue(0)
removePageEmbeddingMock.mockReset()
@@ -299,6 +302,63 @@ describe("ingest-queue — concurrent workers", () => {
expect(getQueue().find((task) => task.sourcePath === "a.md")?.status).toBe("cancelled")
})
+ it("does not delete merged pages when cancelling a worker", async () => {
+ const finish = createDeferred<string[]>()
+ mockAutoIngest.mockImplementation(
+ async (_projectPath, _sourcePath, _config, _signal, _folder, onFileWritten) => {
+ onFileWritten?.("wiki/concepts/existing.md")
+ return finish.promise
+ },
+ )
+
+ await enqueueIngest(TEST_ID, "cancel-during-write.md")
+ await waitFor(() => getQueueSummary().processing === 1)
+ const task = getQueue()[0]
+ await cancelTask(task.id)
+
+ expect(mockDeleteFile).not.toHaveBeenCalled()
+ finish.resolve(["wiki/concepts/existing.md"])
+ await flushMicrotasks(10)
+ expect(mockDeleteFile).not.toHaveBeenCalled()
+ })
+
+ it("does not restart a cancelled task until its previous worker exits", async () => {
+ setIngestWorkerLimit(2)
+ const firstRun = createDeferred<string[]>()
+ mockAutoIngest
+ .mockImplementationOnce(() => firstRun.promise)
+ .mockResolvedValueOnce(["wiki/sources/retried.md"])
+
+ await enqueueIngest(TEST_ID, "retry-after-cancel.md")
+ await waitFor(() => mockAutoIngest.mock.calls.length === 1)
+ const task = getQueue()[0]
+ await cancelTask(task.id)
+ await retryTask(task.id)
+ await flushMicrotasks(10)
+ expect(mockAutoIngest).toHaveBeenCalledTimes(1)
+
+ firstRun.reject(new Error("aborted"))
+ await waitFor(() => mockAutoIngest.mock.calls.length === 2)
+ await waitFor(() => getQueue().length === 0)
+ })
+
+ it("ignores retry requests for a task that is still processing", async () => {
+ setIngestWorkerLimit(2)
+ const run = createDeferred<string[]>()
+ mockAutoIngest.mockImplementation(() => run.promise)
+
+ await enqueueIngest(TEST_ID, "still-processing.md")
+ await waitFor(() => getQueueSummary().processing === 1)
+ const task = getQueue()[0]
+ await retryTask(task.id)
+
+ expect(task.status).toBe("processing")
+ expect(mockAutoIngest).toHaveBeenCalledTimes(1)
+ run.resolve(["wiki/sources/still-processing.md"])
+ await waitFor(() => getQueue().length === 0)
+ expect(mockAutoIngest).toHaveBeenCalledTimes(1)
+ })
+
it("pauses and aborts every active worker after a provider usage limit", async () => {
setIngestWorkerLimit(2)
const signals: AbortSignal[] = []
@@ -896,6 +956,31 @@ describe("ingest-queue — restoreQueue", () => {
import { pauseQueue } from "./ingest-queue"
describe("ingest-queue — pauseQueue & switch-project survival", () => {
+ it("waits for an active commit before completing a project switch", async () => {
+ const commit = createDeferred<void>()
+ let commitStarted = false
+ mockAutoIngest.mockImplementation(
+ async (_projectPath, _sourcePath, _config, _signal, _folder, _onFileWritten, options) => {
+ await options?.runCommit?.(async () => {
+ commitStarted = true
+ await commit.promise
+ })
+ return ["wiki/sources/committed.md"]
+ },
+ )
+
+ await enqueueIngest(TEST_ID, "committing.md")
+ await waitFor(() => commitStarted)
+ let pauseFinished = false
+ const pause = pauseQueue().then(() => { pauseFinished = true })
+ await flushMicrotasks(5)
+ expect(pauseFinished).toBe(false)
+
+ commit.resolve()
+ await pause
+ expect(pauseFinished).toBe(true)
+ })
+
it("pauseQueue persists pending/processing tasks to the paused project's disk", async () => {
mockAutoIngest.mockImplementation(() => new Promise(() => {}))
await enqueueBatch(TEST_ID, [
diff --git a/src/lib/ingest-queue.ts b/src/lib/ingest-queue.ts
index ceb3cf14..f166f5ea 100644
--- a/src/lib/ingest-queue.ts
+++ b/src/lib/ingest-queue.ts
@@ -33,6 +33,8 @@ interface ActiveIngestRun {
controller: AbortController
writtenFiles: string[]
releaseCommitTurn: () => void
+ commitActive: boolean
+ completion: Promise<void>
}
const activeRuns = new Map<string, ActiveIngestRun>()
@@ -238,12 +240,14 @@ function upsertQueuedIngestTask(
}
/**
- * Delete files written by a cancelled / failed ingest, AND drop the
- * matching pages' chunks from LanceDB. Called from the cancel paths
- * (cancelTask, cancelAllTasks).
+ * Explicitly delete wiki files and their matching LanceDB chunks.
+ *
+ * Cancellation deliberately does not call this helper: written paths may be
+ * pre-existing pages merged in place, and deleting them without snapshots
+ * would destroy user data.
*
* Per-file errors are swallowed (best-effort cleanup) — a missing
- * file or LanceDB unavailable shouldn't abort the cancel flow.
+ * file or LanceDB unavailable shouldn't abort the caller's cleanup flow.
* Structural pages (index/log/overview) aren't embedded, so the
* cascade no-ops for them.
*/
@@ -385,13 +389,14 @@ export async function enqueueInactiveProjectBatch(
}
/**
- * Retry a failed task. Only valid for tasks in the active project's
- * queue.
+ * Retry a failed or cancelled task. Only valid for stopped tasks in the
+ * active project's queue.
*/
export async function retryTask(taskId: string): Promise<void> {
const task = queue.find((t) => t.id === taskId)
if (!task) return
if (task.projectId !== currentProjectId) return
+ if (task.status !== "failed" && task.status !== "cancelled") return
restoredPausedTaskIds.delete(task.id)
task.status = "pending"
@@ -479,8 +484,9 @@ export async function movePendingTask(
}
/**
- * Cancel a pending or processing task.
- * If processing, aborts the LLM call and cleans up generated files.
+ * Cancel a pending or processing task. Processing work is aborted, while any
+ * completed atomic page writes are preserved because they may be merges into
+ * pre-existing user content.
*/
export async function cancelTask(taskId: string): Promise<void> {
const task = queue.find((t) => t.id === taskId)
@@ -492,12 +498,6 @@ export async function cancelTask(taskId: string): Promise<void> {
cancelledInFlightTaskIds.add(task.id)
const run = activeRuns.get(task.id)
run?.controller.abort()
- if (run && run.writtenFiles.length > 0) {
- await cleanupWrittenFiles(currentProjectPath, run.writtenFiles)
- console.log(`[Ingest Queue] Cleaned up ${run.writtenFiles.length} files from cancelled task`)
- run.writtenFiles = []
- }
-
}
restoredPausedTaskIds.delete(taskId)
@@ -531,10 +531,6 @@ export async function cancelTasks(taskIds: readonly string[]): Promise<number> {
for (const task of targets) {
const run = activeRuns.get(task.id)
run?.controller.abort()
- if (run && run.writtenFiles.length > 0) {
- await cleanupWrittenFiles(currentProjectPath, run.writtenFiles)
- run.writtenFiles = []
- }
}
for (const task of targets) {
restoredPausedTaskIds.delete(task.id)
@@ -583,10 +579,6 @@ export async function discardTasksForSources(
for (const task of targets) {
const run = activeRuns.get(task.id)
run?.controller.abort()
- if (run && run.writtenFiles.length > 0) {
- await cleanupWrittenFiles(currentProjectPath, run.writtenFiles)
- run.writtenFiles = []
- }
}
queue = queue.filter((task) => !targetIds.has(task.id))
@@ -609,8 +601,8 @@ export async function clearCompletedTasks(): Promise<void> {
/**
* Cancel everything that's not finished in the active project's queue:
- * aborts the running task (if any), cleans up its partial output, and
- * retains every pending + processing item as cancelled so it can be restarted.
+ * aborts running tasks and retains every pending + processing item as
+ * cancelled so it can be restarted.
*
* Failed tasks are retained so the user can still see / retry them.
* Returns the number of tasks removed from the queue.
@@ -626,10 +618,6 @@ export async function cancelAllTasks(): Promise<number> {
}
for (const run of activeRuns.values()) {
run.controller.abort()
- if (run.writtenFiles.length > 0) {
- await cleanupWrittenFiles(currentProjectPath, run.writtenFiles)
- run.writtenFiles = []
- }
}
let cancelled = 0
@@ -788,7 +776,8 @@ export async function pauseQueue(): Promise<void> {
const pausedProjectPath = currentProjectPath
queueEpoch += 1
- for (const run of activeRuns.values()) {
+ const runs = [...activeRuns.values()]
+ for (const run of runs) {
run.controller.abort()
run.releaseCommitTurn()
}
@@ -796,9 +785,17 @@ export async function pauseQueue(): Promise<void> {
sweepAbortController.abort()
sweepAbortController = null
}
- activeRuns.clear()
scheduling = false
+ // Preparation is safe to abandon after abort, but a commit may already be
+ // inside a filesystem operation that cannot observe AbortSignal. Wait only
+ // for those active commits before declaring the project switch complete.
+ const committingRuns = runs.filter((run) => run.commitActive)
+ if (committingRuns.length > 0) {
+ await Promise.allSettled(committingRuns.map((run) => run.completion))
+ }
+ activeRuns.clear()
+
// Revert any in-flight processing task back to pending so when the
// user returns to this project, the task is re-tried from scratch.
for (const task of queue) {
@@ -978,10 +975,9 @@ async function runTask(
const currentTask = queue.find((candidate) => candidate.id === task.id)
if (cancelledInFlightTaskIds.delete(task.id) || currentTask?.status === "cancelled") {
- if (run.writtenFiles.length > 0) {
- await cleanupWrittenFiles(projectPath, run.writtenFiles)
- run.writtenFiles = []
- }
+ // Written paths may be pre-existing pages that were merged in place.
+ // Without a reliable old-content snapshot, deleting them would be data
+ // loss. Keep completed atomic writes and let a retry converge via merge.
await saveQueue(projectPath)
return
}
@@ -997,18 +993,10 @@ async function runTask(
if (currentProjectId !== projectId || run.epoch !== queueEpoch) return
const currentTask = queue.find((candidate) => candidate.id === task.id)
if (cancelledInFlightTaskIds.delete(task.id) || currentTask?.status === "cancelled") {
- if (run.writtenFiles.length > 0) {
- await cleanupWrittenFiles(projectPath, run.writtenFiles)
- run.writtenFiles = []
- }
await saveQueue(projectPath)
return
}
if (currentTask?.status === "pending") {
- if (run.writtenFiles.length > 0) {
- await cleanupWrittenFiles(projectPath, run.writtenFiles)
- run.writtenFiles = []
- }
await saveQueue(projectPath)
return
}
@@ -1078,6 +1066,8 @@ async function startTask(projectId: string, task: IngestTask): Promise<boolean>
controller: new AbortController(),
writtenFiles: [],
releaseCommitTurn: reservation.release,
+ commitActive: false,
+ completion: Promise.resolve(),
}
restoredPausedTaskIds.delete(task.id)
task.status = "processing"
@@ -1091,7 +1081,16 @@ async function startTask(projectId: string, task: IngestTask): Promise<boolean>
}
console.log(`[Ingest Queue] Processing: ${task.sourcePath} (${queue.filter((candidate) => candidate.projectId === projectId && candidate.status === "pending").length} remaining)`)
- void runTask(projectId, projectPath, task, run, reservation.runCommit)
+ const orderedCommit: typeof reservation.runCommit = (operation) =>
+ reservation.runCommit(async () => {
+ run.commitActive = true
+ try {
+ return await operation()
+ } finally {
+ run.commitActive = false
+ }
+ })
+ run.completion = runTask(projectId, projectPath, task, run, orderedCommit)
return true
}
@@ -1105,6 +1104,7 @@ async function processNext(projectId: string): Promise<void> {
const next = queue.find((task) =>
task.projectId === projectId &&
task.status === "pending" &&
+ !activeRuns.has(task.id) &&
!restoredPausedTaskIds.has(task.id)
)
if (!next) break
@@ -1114,6 +1114,7 @@ async function processNext(projectId: string): Promise<void> {
const hasRunnablePending = queue.some((task) =>
task.projectId === projectId &&
task.status === "pending" &&
+ !activeRuns.has(task.id) &&
!restoredPausedTaskIds.has(task.id)
)
const hasRestoredPending = queue.some((task) =>
@@ -1132,6 +1133,7 @@ async function processNext(projectId: string): Promise<void> {
const hasRunnable = queue.some((task) =>
task.projectId === projectId &&
task.status === "pending" &&
+ !activeRuns.has(task.id) &&
!restoredPausedTaskIds.has(task.id)
)
const hasAnyPending = queue.some((task) =>