Skip to content

LLM Wiki is a cross-platform desktop application that turns your documents into an organized, interlinked knowledge base — automatically. Instead of traditional RAG (retrieve-and-answer from scratch every time), the LLM incrementally builds and maintains a persistent wiki from your sources。

TypeScript

feat: add concurrent ingest worker pool

nash_su2cacd0cca8c0
diff --git a/src/lib/ingest-commit-coordinator.test.ts b/src/lib/ingest-commit-coordinator.test.ts
new file mode 100644
index 00000000..7dc23d53
--- /dev/null
+++ b/src/lib/ingest-commit-coordinator.test.ts
@@ -0,0 +1,56 @@
+import { describe, expect, it } from "vitest"
+import { IngestCommitCoordinator } from "@/lib/ingest-commit-coordinator"
+
+describe("IngestCommitCoordinator", () => {
+ it("commits in reservation order even when preparation finishes out of order", async () => {
+ const coordinator = new IngestCommitCoordinator()
+ const first = coordinator.reserve()
+ const second = coordinator.reserve()
+ const events: string[] = []
+ let finishFirst!: () => void
+ const firstGate = new Promise<void>((resolve) => {
+ finishFirst = resolve
+ })
+
+ const secondCommit = second.runCommit(async () => {
+ events.push("second")
+ })
+ const firstCommit = first.runCommit(async () => {
+ events.push("first:start")
+ await firstGate
+ events.push("first:end")
+ })
+
+ await Promise.resolve()
+ await Promise.resolve()
+ expect(events).toEqual(["first:start"])
+ finishFirst()
+ await Promise.all([firstCommit, secondCommit])
+ expect(events).toEqual(["first:start", "first:end", "second"])
+ })
+
+ it("allows later commits to proceed when an earlier task fails before commit", async () => {
+ const coordinator = new IngestCommitCoordinator()
+ const failedPreparation = coordinator.reserve()
+ const next = coordinator.reserve()
+ const events: string[] = []
+
+ failedPreparation.release()
+ await next.runCommit(async () => {
+ events.push("next")
+ })
+
+ expect(events).toEqual(["next"])
+ })
+
+ it("releases a turn when the commit operation rejects", async () => {
+ const coordinator = new IngestCommitCoordinator()
+ const first = coordinator.reserve()
+ const second = coordinator.reserve()
+
+ await expect(first.runCommit(async () => {
+ throw new Error("write failed")
+ })).rejects.toThrow("write failed")
+ await expect(second.runCommit(async () => "ok")).resolves.toBe("ok")
+ })
+})
diff --git a/src/lib/ingest-commit-coordinator.ts b/src/lib/ingest-commit-coordinator.ts
new file mode 100644
index 00000000..05cfb3eb
--- /dev/null
+++ b/src/lib/ingest-commit-coordinator.ts
@@ -0,0 +1,42 @@
+import type { IngestCommitRunner } from "@/lib/ingest"
+
+interface CommitReservation {
+ runCommit: IngestCommitRunner
+ release: () => void
+}
+
+/**
+ * Reserves commit turns in task-start order while allowing preparation to run
+ * concurrently. A failed or cancelled task must release its unused turn.
+ */
+export class IngestCommitCoordinator {
+ private tail: Promise<void> = Promise.resolve()
+
+ reserve(): CommitReservation {
+ const predecessor = this.tail.catch(() => {})
+ let releaseTurn!: () => void
+ const turnComplete = new Promise<void>((resolve) => {
+ releaseTurn = resolve
+ })
+ this.tail = predecessor.then(() => turnComplete)
+ let released = false
+
+ const release = (): void => {
+ if (released) return
+ released = true
+ releaseTurn()
+ }
+
+ return {
+ runCommit: async (operation) => {
+ await predecessor
+ try {
+ return await operation()
+ } finally {
+ release()
+ }
+ },
+ release,
+ }
+ }
+}
diff --git a/src/lib/ingest-queue.test.ts b/src/lib/ingest-queue.test.ts
index 37093e17..5ec3126d 100644
--- a/src/lib/ingest-queue.test.ts
+++ b/src/lib/ingest-queue.test.ts
@@ -1,5 +1,5 @@
import { describe, it, expect, beforeEach, vi } from "vitest"
-import { flushMicrotasks } from "@/test-helpers/deferred"
+import { createDeferred, flushMicrotasks, waitFor } from "@/test-helpers/deferred"
// Mock autoIngest so tests control success/failure timing.
vi.mock("./ingest", () => ({
@@ -71,6 +71,7 @@ import {
pauseProcessing,
resumeProcessing,
isQueuePaused,
+ setIngestWorkerLimit,
} from "./ingest-queue"
import { autoIngest } from "./ingest"
import { readFile, writeFile } from "@/commands/fs"
@@ -161,6 +162,7 @@ describe("ingest-queue — enqueue & basic processing", () => {
expect.any(AbortSignal),
"scheduled-import",
expect.any(Function),
+ expect.objectContaining({ runCommit: expect.any(Function) }),
)
expect(getQueue()).toHaveLength(0)
})
@@ -226,6 +228,98 @@ describe("ingest-queue — enqueue & basic processing", () => {
})
})
+describe("ingest-queue — concurrent workers", () => {
+ it("fills the configured worker pool without exceeding its limit", async () => {
+ setIngestWorkerLimit(2)
+ const runs = new Map<string, ReturnType<typeof createDeferred<string[]>>>()
+ mockAutoIngest.mockImplementation((_projectPath, sourcePath) => {
+ const deferred = createDeferred<string[]>()
+ runs.set(sourcePath, deferred)
+ return deferred.promise
+ })
+
+ await enqueueBatch(TEST_ID, [
+ { sourcePath: "a.md", folderContext: "" },
+ { sourcePath: "b.md", folderContext: "" },
+ { sourcePath: "c.md", folderContext: "" },
+ ])
+ await waitFor(() => mockAutoIngest.mock.calls.length === 2)
+ expect(getQueueSummary()).toMatchObject({ processing: 2, pending: 1 })
+
+ const firstRun = [...runs.values()][0]
+ expect(firstRun).toBeDefined()
+ firstRun.resolve(["wiki/sources/a.md"])
+ await waitFor(() => mockAutoIngest.mock.calls.length === 3)
+ expect(getQueueSummary().processing).toBe(2)
+
+ for (const deferred of runs.values()) {
+ if (!deferred.settled) deferred.resolve(["wiki/sources/completed.md"])
+ }
+ await waitFor(() => getQueue().length === 0)
+ expect(getQueueSummary()).toMatchObject({ completed: 3, processing: 0, pending: 0 })
+ })
+
+ it("cancels one active worker without aborting its sibling", async () => {
+ setIngestWorkerLimit(2)
+ const signals = new Map<string, AbortSignal>()
+ const runs = new Map<string, ReturnType<typeof createDeferred<string[]>>>()
+ mockAutoIngest.mockImplementation((_projectPath, sourcePath, _config, signal) => {
+ const deferred = createDeferred<string[]>()
+ signals.set(sourcePath, signal as AbortSignal)
+ runs.set(sourcePath, deferred)
+ signal?.addEventListener("abort", () => deferred.reject(new Error("aborted")))
+ return deferred.promise
+ })
+
+ await enqueueBatch(TEST_ID, [
+ { sourcePath: "a.md", folderContext: "" },
+ { sourcePath: "b.md", folderContext: "" },
+ ])
+ await waitFor(() => mockAutoIngest.mock.calls.length === 2)
+ const firstTask = getQueue().find((task) => task.sourcePath === "a.md")
+ expect(firstTask).toBeDefined()
+ await cancelTask(firstTask?.id ?? "")
+
+ expect(signals.get(`${TEST_PATH}/a.md`)?.aborted).toBe(true)
+ expect(signals.get(`${TEST_PATH}/b.md`)?.aborted).toBe(false)
+ const siblingRun = [...runs.entries()].find(([path]) => path.endsWith("/b.md"))?.[1]
+ expect(siblingRun).toBeDefined()
+ siblingRun?.resolve(["wiki/sources/b.md"])
+ await waitFor(() => getQueueSummary().processing === 0)
+ expect(getQueue().find((task) => task.sourcePath === "a.md")?.status).toBe("cancelled")
+ })
+
+ it("pauses and aborts every active worker after a provider usage limit", async () => {
+ setIngestWorkerLimit(2)
+ const signals: AbortSignal[] = []
+ const firstRun = createDeferred<string[]>()
+ const secondRun = createDeferred<string[]>()
+ mockAutoIngest
+ .mockImplementationOnce((_projectPath, _sourcePath, _config, signal) => {
+ signals.push(signal as AbortSignal)
+ return firstRun.promise
+ })
+ .mockImplementationOnce((_projectPath, _sourcePath, _config, signal) => {
+ signals.push(signal as AbortSignal)
+ signal?.addEventListener("abort", () => secondRun.reject(new Error("aborted")))
+ return secondRun.promise
+ })
+
+ await enqueueBatch(TEST_ID, [
+ { sourcePath: "limited.md", folderContext: "" },
+ { sourcePath: "sibling.md", folderContext: "" },
+ ])
+ await waitFor(() => signals.length === 2)
+ firstRun.reject(new Error("429 usage limit exceeded"))
+ await waitFor(() => getQueueSummary().processing === 0)
+
+ expect(signals[1].aborted).toBe(true)
+ expect(isQueuePaused()).toBe(true)
+ expect(getQueue().map((task) => task.status)).toEqual(["pending", "pending"])
+ expect(getQueue().every((task) => task.retryCount === 0)).toBe(true)
+ })
+})
+
describe("ingest-queue — retry & failure", () => {
it("retries a failing task up to MAX_RETRIES=3 then marks failed", async () => {
mockAutoIngest.mockRejectedValue(new Error("LLM error"))
@@ -549,7 +643,7 @@ describe("ingest-queue — queue-drain triggers review sweep", () => {
// (We simulate an idle condition by enqueueing, processing, draining once)
mockAutoIngest.mockResolvedValue(["wiki/sources/foo.md"])
await enqueueIngest(TEST_ID, "a.md")
- await flushMicrotasks(20)
+ await waitFor(() => mockSweep.mock.calls.length === 1)
expect(mockSweep).toHaveBeenCalledTimes(1)
// Now the queue is empty. Calling cancelTask on a nonexistent id is a
@@ -928,7 +1022,7 @@ describe("ingest-queue — pause/resume processing", () => {
// Resume → aborted a.md restarts first, then b.md drains.
resumeProcessing()
- await flushMicrotasks(10)
+ await waitFor(() => mockAutoIngest.mock.calls.length === 3)
expect(mockAutoIngest).toHaveBeenCalledTimes(3)
expect(getQueue()).toHaveLength(0)
expect(isQueuePaused()).toBe(false)
diff --git a/src/lib/ingest-queue.ts b/src/lib/ingest-queue.ts
index 4c7377a3..ceb3cf14 100644
--- a/src/lib/ingest-queue.ts
+++ b/src/lib/ingest-queue.ts
@@ -4,6 +4,7 @@ import { normalizePath, isAbsolutePath } from "@/lib/path-utils"
import { getProjectPathById } from "@/lib/project-identity"
import { hasUsableLlm } from "@/lib/has-usable-llm"
import { getTaskLlmConfig } from "@/lib/llm-task-routing"
+import { IngestCommitCoordinator } from "@/lib/ingest-commit-coordinator"
// ── Types ─────────────────────────────────────────────────────────────────
@@ -26,7 +27,19 @@ export interface IngestTask {
// ── State ─────────────────────────────────────────────────────────────────
let queue: IngestTask[] = []
-let processing = false
+interface ActiveIngestRun {
+ taskId: string
+ epoch: number
+ controller: AbortController
+ writtenFiles: string[]
+ releaseCommitTurn: () => void
+}
+
+const activeRuns = new Map<string, ActiveIngestRun>()
+let commitCoordinator = new IngestCommitCoordinator()
+let workerLimit = 1
+let scheduling = false
+let queueEpoch = 0
/** User-controlled pause. When true, processNext stops handing pending
* tasks to the LLM. If a task is in flight, pauseProcessing aborts it
* and returns it to pending so token spend stops promptly. The drain-
@@ -47,8 +60,6 @@ let currentProjectId = ""
* step with `currentProjectId` by pauseQueue / restoreQueue so sync
* callers (saveQueue, cancelTask, etc.) don't need a registry lookup. */
let currentProjectPath = ""
-let currentAbortController: AbortController | null = null
-let lastWrittenFiles: string[] = [] // track files written by current ingest for cleanup
/** Task runs cancelled while still inside autoIngest. This is separate from
* the persisted task status because the user may restart a retained task
* before the old run observes its AbortSignal. In that case the old output
@@ -62,11 +73,28 @@ let processedSinceDrain = false
// cancels a long-running judgment instead of burning tokens.
let sweepAbortController: AbortController | null = null
let inactiveQueueWrite: Promise<void> = Promise.resolve()
+interface QueueWriteRequest {
+ path: string
+ snapshot: string
+ resolve: () => void
+}
+const pendingActiveQueueWrites: QueueWriteRequest[] = []
+let activeQueueWriteRunning = false
+let activeQueueWriteEpoch = 0
function resetQueueAccounting(): void {
completedSinceIdle = 0
}
+export function setIngestWorkerLimit(limit: number): void {
+ workerLimit = Math.max(1, Math.min(5, Math.floor(limit) || 1))
+ if (currentProjectId && !paused) processNext(currentProjectId)
+}
+
+export function getIngestWorkerLimit(): number {
+ return workerLimit
+}
+
// ── Persistence ───────────────────────────────────────────────────────────
function queueFilePath(projectPath: string): string {
@@ -74,12 +102,36 @@ function queueFilePath(projectPath: string): string {
}
async function saveQueue(projectPath: string): Promise<void> {
+ // Capture the snapshot now, then serialize writes so an older async write
+ // can never land after a newer queue state.
+ const toSave = queue.filter((t) => t.status !== "done")
+ const snapshot = JSON.stringify(toSave, null, 2)
+ await new Promise<void>((resolve) => {
+ pendingActiveQueueWrites.push({
+ path: queueFilePath(projectPath),
+ snapshot,
+ resolve,
+ })
+ void drainActiveQueueWrites()
+ })
+}
+
+async function drainActiveQueueWrites(): Promise<void> {
+ if (activeQueueWriteRunning) return
+ const request = pendingActiveQueueWrites.shift()
+ if (!request) return
+ const epoch = activeQueueWriteEpoch
+ activeQueueWriteRunning = true
try {
- // Only save pending and failed tasks (done tasks are removed)
- const toSave = queue.filter((t) => t.status !== "done")
- await writeFile(queueFilePath(projectPath), JSON.stringify(toSave, null, 2))
+ await writeFile(request.path, request.snapshot)
} catch {
- // non-critical
+ // Queue persistence is best-effort; the in-memory queue remains authoritative.
+ } finally {
+ request.resolve()
+ if (epoch === activeQueueWriteEpoch) {
+ activeQueueWriteRunning = false
+ void drainActiveQueueWrites()
+ }
}
}
@@ -134,7 +186,7 @@ function upsertQueuedIngestTask(
sourcePath: string,
folderContext: string,
): string {
- if (queue.length === 0 && !processing) {
+ if (queue.length === 0 && activeRuns.size === 0) {
resetQueueAccounting()
}
const normalizedSourcePath = normalizeSourcePathForQueue(sourcePath)
@@ -438,17 +490,12 @@ export async function cancelTask(taskId: string): Promise<void> {
const wasProcessing = task.status === "processing"
if (wasProcessing) {
cancelledInFlightTaskIds.add(task.id)
- // Abort the in-progress LLM call
- if (currentAbortController) {
- currentAbortController.abort()
- currentAbortController = null
- }
-
- // Clean up any files written by the interrupted ingest
- if (lastWrittenFiles.length > 0) {
- await cleanupWrittenFiles(currentProjectPath, lastWrittenFiles)
- console.log(`[Ingest Queue] Cleaned up ${lastWrittenFiles.length} files from cancelled task`)
- lastWrittenFiles = []
+ const run = activeRuns.get(task.id)
+ run?.controller.abort()
+ if (run && run.writtenFiles.length > 0) {
+ await cleanupWrittenFiles(currentProjectPath, run.writtenFiles)
+ console.log(`[Ingest Queue] Cleaned up ${run.writtenFiles.length} files from cancelled task`)
+ run.writtenFiles = []
}
}
@@ -463,8 +510,7 @@ export async function cancelTask(taskId: string): Promise<void> {
await saveQueue(currentProjectPath)
console.log(`[Ingest Queue] Cancelled and retained for restart: ${task.sourcePath}`)
- // The in-flight run owns `processing` until it observes cancellation.
- // Starting another task here would let the old abort handler race it.
+ // The in-flight run owns its worker slot until it observes cancellation.
if (!wasProcessing) processNext(currentProjectId)
}
@@ -482,13 +528,13 @@ export async function cancelTasks(taskIds: readonly string[]): Promise<number> {
for (const task of targets) {
if (task.status === "processing") cancelledInFlightTaskIds.add(task.id)
}
- if (hadProcessingTask && currentAbortController) {
- currentAbortController.abort()
- currentAbortController = null
- }
- if (hadProcessingTask && lastWrittenFiles.length > 0) {
- await cleanupWrittenFiles(currentProjectPath, lastWrittenFiles)
- lastWrittenFiles = []
+ for (const task of targets) {
+ const run = activeRuns.get(task.id)
+ run?.controller.abort()
+ if (run && run.writtenFiles.length > 0) {
+ await cleanupWrittenFiles(currentProjectPath, run.writtenFiles)
+ run.writtenFiles = []
+ }
}
for (const task of targets) {
restoredPausedTaskIds.delete(task.id)
@@ -534,13 +580,13 @@ export async function discardTasksForSources(
restoredPausedTaskIds.delete(task.id)
if (task.status === "processing") cancelledInFlightTaskIds.add(task.id)
}
- if (hadProcessingTask && currentAbortController) {
- currentAbortController.abort()
- currentAbortController = null
- }
- if (hadProcessingTask && lastWrittenFiles.length > 0) {
- await cleanupWrittenFiles(currentProjectPath, lastWrittenFiles)
- lastWrittenFiles = []
+ for (const task of targets) {
+ const run = activeRuns.get(task.id)
+ run?.controller.abort()
+ if (run && run.writtenFiles.length > 0) {
+ await cleanupWrittenFiles(currentProjectPath, run.writtenFiles)
+ run.writtenFiles = []
+ }
}
queue = queue.filter((task) => !targetIds.has(task.id))
@@ -578,14 +624,12 @@ export async function cancelAllTasks(): Promise<number> {
cancelledInFlightTaskIds.add(task.id)
}
}
- if (currentAbortController) {
- currentAbortController.abort()
- currentAbortController = null
- }
-
- if (lastWrittenFiles.length > 0) {
- await cleanupWrittenFiles(currentProjectPath, lastWrittenFiles)
- lastWrittenFiles = []
+ for (const run of activeRuns.values()) {
+ run.controller.abort()
+ if (run.writtenFiles.length > 0) {
+ await cleanupWrittenFiles(currentProjectPath, run.writtenFiles)
+ run.writtenFiles = []
+ }
}
let cancelled = 0
@@ -598,8 +642,6 @@ export async function cancelAllTasks(): Promise<number> {
}
paused = false
clearUsageLimitAutoResume()
- if (!hadProcessingTask) processing = false
-
await saveQueue(currentProjectPath)
console.log(`[Ingest Queue] Cancelled all and retained: ${cancelled} tasks`)
if (!hadProcessingTask) processNext(currentProjectId)
@@ -618,14 +660,13 @@ export async function cancelAllTasks(): Promise<number> {
export function pauseProcessing(): void {
clearUsageLimitAutoResume()
paused = true
- const processingTask = queue.find(
- (t) => t.projectId === currentProjectId && t.status === "processing",
+ const processingTasks = queue.filter(
+ (task) => task.projectId === currentProjectId && task.status === "processing",
)
- if (processingTask) {
- processingTask.status = "pending"
- if (currentAbortController) {
- currentAbortController.abort()
- currentAbortController = null
+ if (processingTasks.length > 0) {
+ for (const task of processingTasks) {
+ task.status = "pending"
+ activeRuns.get(task.id)?.controller.abort()
}
saveQueue(currentProjectPath).catch((err) => {
console.warn("[Ingest Queue] Failed to persist paused queue:", err)
@@ -637,7 +678,7 @@ export function pauseProcessing(): void {
/**
* Resume queue processing after pauseProcessing(). Kicks processNext so
* pending tasks start immediately. A no-op if a task is already running
- * (the existing `if (processing) return` guard handles that).
+ * Existing active workers keep their slots until they observe cancellation.
*/
export function resumeProcessing(): void {
clearUsageLimitAutoResume()
@@ -701,9 +742,11 @@ export function getQueueSummary(): {
* disk before clearing memory.
*/
export function clearQueueState(): void {
+ queueEpoch += 1
clearUsageLimitAutoResume()
- if (currentAbortController) {
- currentAbortController.abort()
+ for (const run of activeRuns.values()) {
+ run.controller.abort()
+ run.releaseCommitTurn()
}
if (sweepAbortController) {
sweepAbortController.abort()
@@ -711,15 +754,19 @@ export function clearQueueState(): void {
queue = []
restoredPausedTaskIds.clear()
cancelledInFlightTaskIds.clear()
- processing = false
+ activeRuns.clear()
+ scheduling = false
paused = false
currentProjectId = ""
currentProjectPath = ""
- currentAbortController = null
sweepAbortController = null
- lastWrittenFiles = []
processedSinceDrain = false
resetQueueAccounting()
+ activeQueueWriteEpoch += 1
+ activeQueueWriteRunning = false
+ for (const request of pendingActiveQueueWrites.splice(0)) request.resolve()
+ commitCoordinator = new IngestCommitCoordinator()
+ workerLimit = 1
}
/**
@@ -739,16 +786,18 @@ export async function pauseQueue(): Promise<void> {
}
const pausedProjectPath = currentProjectPath
+ queueEpoch += 1
- if (currentAbortController) {
- currentAbortController.abort()
- currentAbortController = null
+ for (const run of activeRuns.values()) {
+ run.controller.abort()
+ run.releaseCommitTurn()
}
if (sweepAbortController) {
sweepAbortController.abort()
sweepAbortController = null
}
- processing = false
+ activeRuns.clear()
+ scheduling = false
// Revert any in-flight processing task back to pending so when the
// user returns to this project, the task is re-tried from scratch.
@@ -760,13 +809,19 @@ export async function pauseQueue(): Promise<void> {
// Flush the paused state to THIS project's disk before wiping memory.
await saveQueue(pausedProjectPath)
+ // A scheduler that passed its registry guard just before the epoch changed
+ // may have persisted one final processing snapshot. Normalize once more
+ // after the write queue drains so the durable state is always restartable.
+ for (const task of queue) {
+ if (task.status === "processing") task.status = "pending"
+ }
+ await saveQueue(pausedProjectPath)
queue = []
restoredPausedTaskIds.clear()
cancelledInFlightTaskIds.clear()
currentProjectId = ""
currentProjectPath = ""
- lastWrittenFiles = []
processedSinceDrain = false
resetQueueAccounting()
}
@@ -785,19 +840,19 @@ export async function restoreQueue(
projectId: string,
projectPath: string,
): Promise<void> {
+ queueEpoch += 1
const pp = normalizePath(projectPath)
// Defensive: reset in-memory state (should already be empty via
// pauseQueue, but clearing again costs nothing).
queue = []
restoredPausedTaskIds.clear()
cancelledInFlightTaskIds.clear()
- processing = false
+ activeRuns.clear()
+ scheduling = false
clearUsageLimitAutoResume()
// Every project loads un-paused. Pause is a current-session control;
// it does not carry across project switches or app restarts.
paused = false
- currentAbortController = null
- lastWrittenFiles = []
resetQueueAccounting()
currentProjectId = projectId
currentProjectPath = pp
@@ -871,11 +926,11 @@ function scheduleUsageLimitAutoResume(projectId: string): void {
}, USAGE_LIMIT_AUTO_RESUME_MS)
}
-async function onQueueDrained(projectId: string, projectPath: string): Promise<void> {
+async function onQueueDrained(projectId: string, projectPath: string, epoch: number): Promise<void> {
if (!processedSinceDrain) return
// Stale-context guard — if we switched projects mid-drain, the sweep
// would burn tokens analyzing the wrong project.
- if (currentProjectId !== projectId) return
+ if (currentProjectId !== projectId || queueEpoch !== epoch) return
processedSinceDrain = false
sweepAbortController = new AbortController()
@@ -883,6 +938,7 @@ async function onQueueDrained(projectId: string, projectPath: string): Promise<v
try {
const { sweepResolvedReviews } = await import("@/lib/sweep-reviews")
+ if (currentProjectId !== projectId || queueEpoch !== epoch) return
await sweepResolvedReviews(projectPath, signal)
} catch (err) {
console.error("[Ingest Queue] Failed to load sweep-reviews:", err)
@@ -893,170 +949,84 @@ async function onQueueDrained(projectId: string, projectPath: string): Promise<v
}
}
-async function processNext(projectId: string): Promise<void> {
- if (processing) return
- // Stale-context guard: processNext may be invoked by an orphaned
- // recursion from a previous project. If we're no longer active, bail.
- if (currentProjectId !== projectId) return
- // User pause: don't hand the next pending task to the LLM. Also
- // skips the drain-sweep below — intended, since that's an LLM call.
- if (paused) {
- const hasPending = queue.some((t) => t.projectId === projectId && t.status === "pending")
- if (hasPending) return
- // Pause applies to the current queue, not to future imports forever.
- // If the in-flight task completed despite the abort and nothing is left,
- // clear it so a later explicit import can run normally.
- paused = false
- return
- }
-
- const next = queue.find((t) =>
- t.projectId === projectId &&
- t.status === "pending" &&
- !restoredPausedTaskIds.has(t.id)
- )
- if (!next) {
- const hasRestoredPending = queue.some((t) =>
- t.projectId === projectId &&
- t.status === "pending" &&
- restoredPausedTaskIds.has(t.id)
- )
- if (hasRestoredPending) return
- // Queue drained — trigger review cleanup (auto-resolve stale items)
- const pathAtDrain = currentProjectPath
- onQueueDrained(projectId, pathAtDrain).catch((err) =>
- console.error("[Ingest Queue] sweep failed:", err)
- )
- return
- }
-
- // Look up the project's current filesystem path from the registry —
- // it may have moved since the task was enqueued. If the project isn't
- // in the registry (was deleted or never registered), mark as failed.
- const registryPath = await getProjectPathById(projectId)
- const pp = registryPath ? normalizePath(registryPath) : ""
-
- // Check we're still active after the registry await.
- if (currentProjectId !== projectId) return
-
- if (!pp) {
- next.status = "failed"
- next.error = "Project not found in registry (was it deleted?)"
- await saveQueue(currentProjectPath)
- processNext(projectId)
- return
- }
-
- processing = true
- restoredPausedTaskIds.delete(next.id)
- next.status = "processing"
- await saveQueue(pp)
- if (currentProjectId !== projectId) return
-
- const llmConfig = getTaskLlmConfig("ingest")
-
- // Check if LLM is configured
- if (!hasUsableLlm(llmConfig)) {
- next.status = "failed"
- next.error = "LLM not configured — set API key in Settings"
- processing = false
- await saveQueue(pp)
- processNext(projectId)
- return
- }
-
- const fullSourcePath = isAbsolutePath(next.sourcePath)
- ? normalizePath(next.sourcePath)
- : `${pp}/${next.sourcePath}`
-
- console.log(`[Ingest Queue] Processing: ${next.sourcePath} (${queue.filter((t) => t.projectId === projectId && t.status === "pending").length} remaining)`)
-
- currentAbortController = new AbortController()
- lastWrittenFiles = []
+async function runTask(
+ projectId: string,
+ projectPath: string,
+ task: IngestTask,
+ run: ActiveIngestRun,
+ runCommit: ReturnType<IngestCommitCoordinator["reserve"]>["runCommit"],
+): Promise<void> {
+ const fullSourcePath = isAbsolutePath(task.sourcePath)
+ ? normalizePath(task.sourcePath)
+ : `${projectPath}/${task.sourcePath}`
const trackWrittenFile = (relativePath: string): void => {
- if (!lastWrittenFiles.includes(relativePath)) {
- lastWrittenFiles.push(relativePath)
- }
+ if (!run.writtenFiles.includes(relativePath)) run.writtenFiles.push(relativePath)
}
try {
const writtenFiles = await autoIngest(
- pp,
+ projectPath,
fullSourcePath,
- llmConfig,
- currentAbortController.signal,
- next.folderContext,
+ getTaskLlmConfig("ingest"),
+ run.controller.signal,
+ task.folderContext,
trackWrittenFile,
+ { runCommit },
)
- // Stale-context guard: project switched during the long LLM call.
- // Bail without mutating queue or writing to disk — pauseQueue has
- // already persisted the correct state to the old project's file,
- // and the new project's queue must not be touched by this orphan.
- if (currentProjectId !== projectId) return
- lastWrittenFiles = writtenFiles
-
- const currentTask = queue.find((task) => task.id === next.id)
- if (cancelledInFlightTaskIds.delete(next.id) || currentTask?.status === "cancelled") {
- if (lastWrittenFiles.length > 0) {
- await cleanupWrittenFiles(pp, lastWrittenFiles)
- lastWrittenFiles = []
+ if (currentProjectId !== projectId || run.epoch !== queueEpoch) return
+ run.writtenFiles = writtenFiles
+
+ const currentTask = queue.find((candidate) => candidate.id === task.id)
+ if (cancelledInFlightTaskIds.delete(task.id) || currentTask?.status === "cancelled") {
+ if (run.writtenFiles.length > 0) {
+ await cleanupWrittenFiles(projectPath, run.writtenFiles)
+ run.writtenFiles = []
}
- currentAbortController = null
- processing = false
- await saveQueue(pp)
- processNext(projectId)
+ await saveQueue(projectPath)
return
}
+ if (writtenFiles.length === 0) throw new Error("Ingest produced no output files")
- // Safety net: autoIngest resolving with zero files means nothing
- // was really ingested (e.g. abort during webview refresh where the
- // historical `return []` error path masqueraded as success). Treat
- // as failure so the task stays in the queue and retries.
- if (writtenFiles.length === 0) {
- throw new Error("Ingest produced no output files")
- }
-
- // Success: remove from queue
- currentAbortController = null
- lastWrittenFiles = []
- cancelledInFlightTaskIds.delete(next.id)
- queue = queue.filter((t) => t.id !== next.id)
+ cancelledInFlightTaskIds.delete(task.id)
+ queue = queue.filter((candidate) => candidate.id !== task.id)
completedSinceIdle++
processedSinceDrain = true
- await saveQueue(pp)
-
- console.log(`[Ingest Queue] Done: ${next.sourcePath}`)
+ await saveQueue(projectPath)
+ console.log(`[Ingest Queue] Done: ${task.sourcePath}`)
} catch (err) {
- if (currentProjectId !== projectId) return
- currentAbortController = null
- const currentTask = queue.find((t) => t.id === next.id)
- if (cancelledInFlightTaskIds.delete(next.id) || currentTask?.status === "cancelled") {
- if (lastWrittenFiles.length > 0) {
- await cleanupWrittenFiles(pp, lastWrittenFiles)
- lastWrittenFiles = []
+ if (currentProjectId !== projectId || run.epoch !== queueEpoch) return
+ const currentTask = queue.find((candidate) => candidate.id === task.id)
+ if (cancelledInFlightTaskIds.delete(task.id) || currentTask?.status === "cancelled") {
+ if (run.writtenFiles.length > 0) {
+ await cleanupWrittenFiles(projectPath, run.writtenFiles)
+ run.writtenFiles = []
}
- processing = false
- await saveQueue(pp)
- processNext(projectId)
+ await saveQueue(projectPath)
return
}
if (currentTask?.status === "pending") {
- if (lastWrittenFiles.length > 0) {
- await cleanupWrittenFiles(pp, lastWrittenFiles)
- lastWrittenFiles = []
+ if (run.writtenFiles.length > 0) {
+ await cleanupWrittenFiles(projectPath, run.writtenFiles)
+ run.writtenFiles = []
}
- processing = false
- await saveQueue(pp)
- if (!paused) processNext(projectId)
+ await saveQueue(projectPath)
return
}
+
const message = err instanceof Error ? err.message : String(err)
if (isUsageLimitError(message)) {
- next.status = "pending"
- next.error = `Paused after provider usage limit: ${message}`
+ task.status = "pending"
+ task.error = `Paused after provider usage limit: ${message}`
paused = true
- processing = false
- await saveQueue(pp)
+ for (const [taskId, activeRun] of activeRuns) {
+ if (taskId === task.id) continue
+ const sibling = queue.find((candidate) => candidate.id === taskId)
+ if (sibling?.projectId !== projectId || sibling.status !== "processing") continue
+ sibling.status = "pending"
+ sibling.error = `Paused after provider usage limit: ${message}`
+ activeRun.controller.abort()
+ }
+ await saveQueue(projectPath)
scheduleUsageLimitAutoResume(projectId)
console.log(
`[Ingest Queue] Paused after provider usage limit; will retry automatically in ${Math.round(USAGE_LIMIT_AUTO_RESUME_MS / 60000)} minutes: ${message}`,
@@ -1064,20 +1034,113 @@ async function processNext(projectId: string): Promise<void> {
return
}
- next.retryCount++
- next.error = message
-
- if (next.retryCount >= MAX_RETRIES) {
- next.status = "failed"
- console.log(`[Ingest Queue] Failed (${next.retryCount}x): ${next.sourcePath} — ${message}`)
+ task.retryCount++
+ task.error = message
+ if (task.retryCount >= MAX_RETRIES) {
+ task.status = "failed"
+ console.log(`[Ingest Queue] Failed (${task.retryCount}x): ${task.sourcePath} — ${message}`)
} else {
- next.status = "pending" // will retry
- console.log(`[Ingest Queue] Error (retry ${next.retryCount}/${MAX_RETRIES}): ${next.sourcePath} — ${message}`)
+ task.status = "pending"
+ console.log(`[Ingest Queue] Error (retry ${task.retryCount}/${MAX_RETRIES}): ${task.sourcePath} — ${message}`)
}
+ await saveQueue(projectPath)
+ } finally {
+ run.releaseCommitTurn()
+ if (activeRuns.get(task.id) === run) activeRuns.delete(task.id)
+ if (currentProjectId === projectId && run.epoch === queueEpoch && !paused) processNext(projectId)
+ }
+}
- await saveQueue(pp)
+async function startTask(projectId: string, task: IngestTask): Promise<boolean> {
+ const startEpoch = queueEpoch
+ const registryPath = await getProjectPathById(projectId)
+ const projectPath = registryPath ? normalizePath(registryPath) : ""
+ if (currentProjectId !== projectId || queueEpoch !== startEpoch) return false
+ if (!projectPath) {
+ task.status = "failed"
+ task.error = "Project not found in registry (was it deleted?)"
+ await saveQueue(currentProjectPath)
+ return true
}
- processing = false
- processNext(projectId)
+ const llmConfig = getTaskLlmConfig("ingest")
+ if (!hasUsableLlm(llmConfig)) {
+ task.status = "failed"
+ task.error = "LLM not configured — set API key in Settings"
+ await saveQueue(projectPath)
+ return true
+ }
+
+ const reservation = commitCoordinator.reserve()
+ const run: ActiveIngestRun = {
+ taskId: task.id,
+ epoch: startEpoch,
+ controller: new AbortController(),
+ writtenFiles: [],
+ releaseCommitTurn: reservation.release,
+ }
+ restoredPausedTaskIds.delete(task.id)
+ task.status = "processing"
+ activeRuns.set(task.id, run)
+ await saveQueue(projectPath)
+ if (currentProjectId !== projectId || run.epoch !== queueEpoch) {
+ run.controller.abort()
+ reservation.release()
+ activeRuns.delete(task.id)
+ return false
+ }
+
+ console.log(`[Ingest Queue] Processing: ${task.sourcePath} (${queue.filter((candidate) => candidate.projectId === projectId && candidate.status === "pending").length} remaining)`)
+ void runTask(projectId, projectPath, task, run, reservation.runCommit)
+ return true
+}
+
+async function processNext(projectId: string): Promise<void> {
+ if (scheduling || currentProjectId !== projectId) return
+ if (paused) return
+ const epoch = queueEpoch
+ scheduling = true
+ try {
+ while (activeRuns.size < workerLimit && currentProjectId === projectId && !paused) {
+ const next = queue.find((task) =>
+ task.projectId === projectId &&
+ task.status === "pending" &&
+ !restoredPausedTaskIds.has(task.id)
+ )
+ if (!next) break
+ await startTask(projectId, next)
+ }
+
+ const hasRunnablePending = queue.some((task) =>
+ task.projectId === projectId &&
+ task.status === "pending" &&
+ !restoredPausedTaskIds.has(task.id)
+ )
+ const hasRestoredPending = queue.some((task) =>
+ task.projectId === projectId &&
+ task.status === "pending" &&
+ restoredPausedTaskIds.has(task.id)
+ )
+ if (activeRuns.size === 0 && !hasRunnablePending && !hasRestoredPending) {
+ const pathAtDrain = currentProjectPath
+ onQueueDrained(projectId, pathAtDrain, epoch).catch((err) =>
+ console.error("[Ingest Queue] sweep failed:", err)
+ )
+ }
+ } finally {
+ scheduling = false
+ const hasRunnable = queue.some((task) =>
+ task.projectId === projectId &&
+ task.status === "pending" &&
+ !restoredPausedTaskIds.has(task.id)
+ )
+ const hasAnyPending = queue.some((task) =>
+ task.projectId === projectId && task.status === "pending"
+ )
+ const needsDrain = processedSinceDrain && activeRuns.size === 0 && !hasAnyPending
+ const hasWorkerCapacity = activeRuns.size < workerLimit
+ if (currentProjectId === projectId && !paused && ((hasWorkerCapacity && hasRunnable) || needsDrain)) {
+ queueMicrotask(() => void processNext(projectId))
+ }
+ }
}