Skip to content

LLM Wiki is a cross-platform desktop application that turns your documents into an organized, interlinked knowledge base — automatically. Instead of traditional RAG (retrieve-and-answer from scratch every time), the LLM incrementally builds and maintains a persistent wiki from your sources。

TypeScript

feat: add bounded concurrent source parsing

nash_su05b56b9a60b5
diff --git a/src/components/settings/sections/source-watch-section.tsx b/src/components/settings/sections/source-watch-section.tsx
index 0a3af9d0..da40096e 100644
--- a/src/components/settings/sections/source-watch-section.tsx
+++ b/src/components/settings/sections/source-watch-section.tsx
@@ -106,6 +106,29 @@ export function SourceWatchSection({ draft, setDraft, projectReady }: Props) {
</div>
</label>
+ <label className="block space-y-1.5">
+ <span className="text-sm font-semibold">
+ {t("settings.sections.sourceWatch.parsingConcurrency", {
+ defaultValue: "Concurrent document parsers",
+ })}
+ </span>
+ <input
+ type="number"
+ min={1}
+ max={8}
+ value={config.parsingConcurrency}
+ onChange={(event) => updateConfig({ parsingConcurrency: Number(event.target.value) || 1 })}
+ disabled={!projectReady}
+ className="w-24 rounded-md border border-input bg-background px-2 py-1 text-sm"
+ />
+ <p className="text-xs leading-relaxed text-muted-foreground">
+ {t("settings.sections.sourceWatch.parsingConcurrencyDescription", {
+ defaultValue:
+ "Parse up to this many imported documents at once before sequential Wiki generation. PDF parsing remains internally serialized for safety.",
+ })}
+ </p>
+ </label>
+
<label className="flex items-start gap-3">
<input
type="checkbox"
diff --git a/src/i18n/en.json b/src/i18n/en.json
index d080064a..0763a67c 100644
--- a/src/i18n/en.json
+++ b/src/i18n/en.json
@@ -704,6 +704,8 @@
"autoIngestDescription": "When disabled, external changes still refresh the file tree but do not create ingest tasks.",
"persistExtractedMarkdown": "Keep parsed Markdown",
"persistExtractedMarkdownDescription": "Save extracted document text under raw/parsed using the same folder structure as raw/sources.",
+ "parsingConcurrency": "Concurrent document parsers",
+ "parsingConcurrencyDescription": "Parse up to this many imported documents at once before sequential Wiki generation. PDF parsing remains internally serialized for safety.",
"noProject": "Open a project to change this project-level setting.",
"fileTypes": "Allowed file types",
"fileTypesDescription": "Document types are enabled by default. Media, binaries, source code, and config-like formats stay off unless selected.",
diff --git a/src/i18n/zh.json b/src/i18n/zh.json
index 0158a84b..81d36bb3 100644
--- a/src/i18n/zh.json
+++ b/src/i18n/zh.json
@@ -704,6 +704,8 @@
"autoIngestDescription": "关闭后,外部变化仍会刷新文件树,但不会创建提取任务。",
"persistExtractedMarkdown": "保留解析后的 Markdown",
"persistExtractedMarkdownDescription": "将文档解析出的文本按 raw/sources 的目录结构保存到 raw/parsed。",
+ "parsingConcurrency": "文档解析并发数",
+ "parsingConcurrencyDescription": "进入串行 Wiki 生成前,最多同时解析这些文档。为保证安全,PDF 解析器内部仍会串行执行。",
"noProject": "打开项目后才能修改这个项目级设置。",
"fileTypes": "允许的文件类型",
"fileTypesDescription": "常见文档类型默认开启。媒体、二进制、源码和配置类格式默认关闭,需要时再手动选择。",
diff --git a/src/lib/project-file-sync.test.ts b/src/lib/project-file-sync.test.ts
index 36c62eab..10a3a48e 100644
--- a/src/lib/project-file-sync.test.ts
+++ b/src/lib/project-file-sync.test.ts
@@ -468,6 +468,7 @@ describe("project file sync", () => {
enabled: true,
autoIngest: true,
persistExtractedMarkdown: false,
+ parsingConcurrency: 2,
includeExtensions: ["md", "xml"],
excludeExtensions: [],
excludeDirs: [],
diff --git a/src/lib/scheduled-import.ts b/src/lib/scheduled-import.ts
index 27873aae..8313c4bc 100644
--- a/src/lib/scheduled-import.ts
+++ b/src/lib/scheduled-import.ts
@@ -4,7 +4,6 @@ import {
getFileMd5,
getFileSize,
listDirectory,
- preprocessFile,
readFile,
writeFileAtomic,
} from "@/commands/fs"
@@ -428,7 +427,6 @@ export async function scanAndImport(
if (changedFiles.length > 0) {
const destPaths = changedFiles.map((file) => file.destPath)
- await Promise.all(destPaths.map((path) => preprocessFile(path).catch(() => {})))
if (isCurrentRun(project.id, options.runId)) {
const ids = await enqueueSourceIngest(project, destPaths, llmConfig)
if (ids.length > 0) {
diff --git a/src/lib/source-lifecycle.test.ts b/src/lib/source-lifecycle.test.ts
index 385b9391..2b2773da 100644
--- a/src/lib/source-lifecycle.test.ts
+++ b/src/lib/source-lifecycle.test.ts
@@ -95,6 +95,7 @@ describe("source-lifecycle path helpers", () => {
enabled: true,
autoIngest: true,
persistExtractedMarkdown: false,
+ parsingConcurrency: 2,
includeExtensions: ["md"],
excludeExtensions: ["json"],
excludeDirs: ["drafts"],
@@ -155,6 +156,7 @@ describe("source-lifecycle path helpers", () => {
enabled: true,
autoIngest: true,
persistExtractedMarkdown: false,
+ parsingConcurrency: 2,
includeExtensions: ["json", "yaml", "md"],
excludeExtensions: [],
excludeDirs: [],
@@ -224,6 +226,7 @@ describe("source-lifecycle path helpers", () => {
enabled: true,
autoIngest: true,
persistExtractedMarkdown: false,
+ parsingConcurrency: 2,
includeExtensions: ["md"],
excludeExtensions: [],
excludeDirs: ["drafts"],
@@ -260,6 +263,7 @@ describe("source-lifecycle path helpers", () => {
enabled: true,
autoIngest: true,
persistExtractedMarkdown: false,
+ parsingConcurrency: 2,
includeExtensions: ["md", "pdf"],
excludeExtensions: [],
excludeDirs: [],
@@ -324,6 +328,7 @@ describe("source-lifecycle path helpers", () => {
enabled: true,
autoIngest: true,
persistExtractedMarkdown: false,
+ parsingConcurrency: 2,
includeExtensions: ["md"],
excludeExtensions: [],
excludeDirs: [],
diff --git a/src/lib/source-lifecycle.ts b/src/lib/source-lifecycle.ts
index 946e1b68..d189874b 100644
--- a/src/lib/source-lifecycle.ts
+++ b/src/lib/source-lifecycle.ts
@@ -5,7 +5,6 @@ import {
fileExists,
getFileSize,
listDirectory,
- preprocessFile,
readFile,
writeFile,
} from "@/commands/fs"
@@ -39,6 +38,8 @@ import { isPathAllowedBySourceWatch, normalizeSourceWatchConfig } from "@/lib/so
import { isSensitiveConfigSourceFile } from "@/lib/source-filter"
import { naturalCompare } from "@/lib/natural-sort"
import type { SourceWatchConfig } from "@/stores/wiki-store"
+import { useWikiStore } from "@/stores/wiki-store"
+import { preprocessSourceFiles } from "@/lib/source-preprocess"
export const INGESTABLE_SOURCE_EXTENSIONS = new Set([
"md",
@@ -240,9 +241,8 @@ export async function enqueueSourceIngest(
project: WikiProject,
sourcePaths: string[],
llmConfig: LlmConfig,
- options: { sourceRoot?: string; rootContext?: string } = {},
+ options: { sourceRoot?: string; rootContext?: string; parsingConcurrency?: number } = {},
): Promise<string[]> {
- if (!hasUsableLlm(getTaskLlmConfig("ingest", llmConfig))) return []
const files = sourcePaths
.filter((sourcePath) =>
isIngestableSourcePath(sourcePath) &&
@@ -256,6 +256,10 @@ export async function enqueueSourceIngest(
),
}))
if (files.length === 0) return []
+ const parsingConcurrency = options.parsingConcurrency
+ ?? normalizeSourceWatchConfig(useWikiStore.getState().sourceWatchConfig).parsingConcurrency
+ await preprocessSourceFiles(files.map((file) => file.sourcePath), parsingConcurrency)
+ if (!hasUsableLlm(getTaskLlmConfig("ingest", llmConfig))) return []
return enqueueBatch(project.id, files)
}
@@ -295,13 +299,14 @@ export async function importSourceFiles(
try {
await copyFile(sourcePath, destPath)
importedPaths.push(destPath)
- preprocessFile(destPath).catch(() => {})
} catch (err) {
console.error(`Failed to import ${originalName}:`, err)
}
}
- await enqueueSourceIngest(project, importedPaths, llmConfig)
+ await enqueueSourceIngest(project, importedPaths, llmConfig, {
+ parsingConcurrency: cfg.parsingConcurrency,
+ })
return importedPaths
}
@@ -348,7 +353,6 @@ export async function importSourceFolder(
if (parent) await createDirectory(parent)
await copyFile(file.path, destPath)
allowedFiles.push(destPath)
- preprocessFile(destPath).catch(() => {})
}
const naturallyOrderedFiles = [...allowedFiles].sort((a, b) =>
@@ -359,6 +363,7 @@ export async function importSourceFolder(
await enqueueSourceIngest(project, naturallyOrderedFiles, llmConfig, {
sourceRoot: destDir,
rootContext: folderName,
+ parsingConcurrency: cfg.parsingConcurrency,
})
}
diff --git a/src/lib/source-preprocess.test.ts b/src/lib/source-preprocess.test.ts
new file mode 100644
index 00000000..f10d65da
--- /dev/null
+++ b/src/lib/source-preprocess.test.ts
@@ -0,0 +1,57 @@
+import { beforeEach, describe, expect, it, vi } from "vitest"
+
+const preprocessFile = vi.hoisted(() => vi.fn())
+vi.mock("@/commands/fs", () => ({ preprocessFile }))
+
+import { normalizeParsingConcurrency, preprocessSourceFiles } from "./source-preprocess"
+
+describe("source preprocessing", () => {
+ beforeEach(() => vi.clearAllMocks())
+
+ it("normalizes concurrency into the supported integer range", () => {
+ expect(normalizeParsingConcurrency(Number.NaN)).toBe(1)
+ expect(normalizeParsingConcurrency(0)).toBe(1)
+ expect(normalizeParsingConcurrency(3.9)).toBe(3)
+ expect(normalizeParsingConcurrency(99)).toBe(8)
+ })
+
+ it("never exceeds the configured concurrency and de-duplicates paths", async () => {
+ let active = 0
+ let peak = 0
+ preprocessFile.mockImplementation(async () => {
+ active += 1
+ peak = Math.max(peak, active)
+ await new Promise((resolve) => setTimeout(resolve, 5))
+ active -= 1
+ })
+
+ await preprocessSourceFiles(["a.pdf", "b.docx", "a.pdf", "c.epub", "d.org"], 2)
+
+ expect(peak).toBe(2)
+ expect(preprocessFile).toHaveBeenCalledTimes(4)
+ })
+
+ it("shares the concurrency ceiling across simultaneous batches", async () => {
+ let active = 0
+ let peak = 0
+ preprocessFile.mockImplementation(async () => {
+ active += 1
+ peak = Math.max(peak, active)
+ await new Promise((resolve) => setTimeout(resolve, 5))
+ active -= 1
+ })
+
+ await Promise.all([
+ preprocessSourceFiles(["a.pdf", "b.pdf", "c.pdf"], 2),
+ preprocessSourceFiles(["d.docx", "e.docx", "f.docx"], 2),
+ ])
+
+ expect(peak).toBe(2)
+ })
+
+ it("continues after one extractor fails", async () => {
+ preprocessFile.mockRejectedValueOnce(new Error("bad file")).mockResolvedValue("")
+ await expect(preprocessSourceFiles(["bad.pdf", "good.docx"], 1)).resolves.toBeUndefined()
+ expect(preprocessFile).toHaveBeenCalledTimes(2)
+ })
+})
diff --git a/src/lib/source-preprocess.ts b/src/lib/source-preprocess.ts
new file mode 100644
index 00000000..0a743019
--- /dev/null
+++ b/src/lib/source-preprocess.ts
@@ -0,0 +1,73 @@
+import { preprocessFile } from "@/commands/fs"
+
+export const MIN_PARSING_CONCURRENCY = 1
+export const MAX_PARSING_CONCURRENCY = 8
+
+export function normalizeParsingConcurrency(value: number): number {
+ if (!Number.isFinite(value)) return MIN_PARSING_CONCURRENCY
+ return Math.max(
+ MIN_PARSING_CONCURRENCY,
+ Math.min(MAX_PARSING_CONCURRENCY, Math.floor(value)),
+ )
+}
+
+interface WaitingParser {
+ limit: number
+ resolve: () => void
+}
+
+let activeParsers = 0
+const waitingParsers: WaitingParser[] = []
+
+function drainWaitingParsers(): void {
+ while (waitingParsers.length > 0 && activeParsers < waitingParsers[0].limit) {
+ const waiter = waitingParsers.shift()
+ if (!waiter) return
+ activeParsers += 1
+ waiter.resolve()
+ }
+}
+
+async function acquireParser(limit: number): Promise<() => void> {
+ await new Promise<void>((resolve) => {
+ waitingParsers.push({ limit, resolve })
+ drainWaitingParsers()
+ })
+ let released = false
+ return () => {
+ if (released) return
+ released = true
+ activeParsers -= 1
+ drainWaitingParsers()
+ }
+}
+
+/**
+ * Pre-extract source text with bounded concurrency before LLM work begins.
+ * Individual extractor failures are intentionally non-fatal: autoIngest will
+ * retry through read_file and surface a real source-read error if necessary.
+ */
+export async function preprocessSourceFiles(
+ paths: readonly string[],
+ concurrency: number,
+): Promise<void> {
+ const uniquePaths = [...new Set(paths)]
+ if (uniquePaths.length === 0) return
+ const limit = normalizeParsingConcurrency(concurrency)
+
+ await Promise.all(uniquePaths.map(async (path) => {
+ const release = await acquireParser(limit)
+ try {
+ try {
+ await preprocessFile(path)
+ } catch (err) {
+ console.warn(
+ `[source-preprocess] failed to pre-process "${path}"; ingest will retry when reading it:`,
+ err instanceof Error ? err.message : err,
+ )
+ }
+ } finally {
+ release()
+ }
+ }))
+}
diff --git a/src/lib/source-watch-config.test.ts b/src/lib/source-watch-config.test.ts
index 39a9438c..9fa4c113 100644
--- a/src/lib/source-watch-config.test.ts
+++ b/src/lib/source-watch-config.test.ts
@@ -10,9 +10,14 @@ describe("source watch config", () => {
it("uses the shared default fixture", () => {
expect(DEFAULT_SOURCE_WATCH_CONFIG).toEqual(sourceWatchDefaults)
expect(normalizeSourceWatchConfig({}).persistExtractedMarkdown).toBe(false)
+ expect(normalizeSourceWatchConfig({}).parsingConcurrency).toBe(2)
expect(
normalizeSourceWatchConfig({ persistExtractedMarkdown: true }).persistExtractedMarkdown,
).toBe(true)
+ expect(normalizeSourceWatchConfig({ parsingConcurrency: 20 }).parsingConcurrency).toBe(8)
+ expect(
+ normalizeSourceWatchConfig({ parsingConcurrency: Number.NaN }).parsingConcurrency,
+ ).toBe(2)
})
it("allows document types by default and rejects config/media/binaries", () => {
diff --git a/src/lib/source-watch-config.ts b/src/lib/source-watch-config.ts
index e819d54c..0aaded22 100644
--- a/src/lib/source-watch-config.ts
+++ b/src/lib/source-watch-config.ts
@@ -42,11 +42,17 @@ function normalizeList(values: readonly string[] | undefined): string[] {
}
export function normalizeSourceWatchConfig(config?: Partial<SourceWatchConfig> | null): SourceWatchConfig {
+ const rawParsingConcurrency = config?.parsingConcurrency
+ ?? DEFAULT_SOURCE_WATCH_CONFIG.parsingConcurrency
+ const parsingConcurrency = Number.isFinite(rawParsingConcurrency)
+ ? Math.max(1, Math.min(8, Math.floor(rawParsingConcurrency)))
+ : DEFAULT_SOURCE_WATCH_CONFIG.parsingConcurrency
return {
enabled: config?.enabled ?? DEFAULT_SOURCE_WATCH_CONFIG.enabled,
autoIngest: config?.autoIngest ?? DEFAULT_SOURCE_WATCH_CONFIG.autoIngest,
persistExtractedMarkdown:
config?.persistExtractedMarkdown ?? DEFAULT_SOURCE_WATCH_CONFIG.persistExtractedMarkdown,
+ parsingConcurrency,
includeExtensions: normalizeExtensions(config?.includeExtensions ?? DEFAULT_SOURCE_WATCH_CONFIG.includeExtensions),
excludeExtensions: normalizeExtensions(config?.excludeExtensions ?? DEFAULT_SOURCE_WATCH_CONFIG.excludeExtensions),
excludeDirs: normalizeList(config?.excludeDirs ?? DEFAULT_SOURCE_WATCH_CONFIG.excludeDirs),
diff --git a/src/lib/source-watch-defaults.json b/src/lib/source-watch-defaults.json
index 39054be2..52c3fbb1 100644
--- a/src/lib/source-watch-defaults.json
+++ b/src/lib/source-watch-defaults.json
@@ -2,6 +2,7 @@
"enabled": true,
"autoIngest": true,
"persistExtractedMarkdown": false,
+ "parsingConcurrency": 2,
"includeExtensions": [
"md",
"mdx",
diff --git a/src/stores/wiki-store.ts b/src/stores/wiki-store.ts
index 37c33fda..8b728600 100644
--- a/src/stores/wiki-store.ts
+++ b/src/stores/wiki-store.ts
@@ -261,6 +261,8 @@ interface SourceWatchConfig {
autoIngest: boolean
/** Keep extractor output in raw/parsed in addition to the internal cache. */
persistExtractedMarkdown: boolean
+ /** Maximum number of source text extraction jobs allowed at once. */
+ parsingConcurrency: number
includeExtensions: string[]
excludeExtensions: string[]
excludeDirs: string[]