LLM Wiki is a cross-platform desktop application that turns your documents into an organized, interlinked knowledge base — automatically. Instead of traditional RAG (retrieve-and-answer from scratch every time), the LLM incrementally builds and maintains a persistent wiki from your sources。
TypeScript
feat: add bounded concurrent source parsing
nash_su05b56b9a60b5
diff --git a/src/components/settings/sections/source-watch-section.tsx b/src/components/settings/sections/source-watch-section.tsxindex 0a3af9d0..da40096e 100644--- a/src/components/settings/sections/source-watch-section.tsx+++ b/src/components/settings/sections/source-watch-section.tsx@@ -106,6 +106,29 @@ export function SourceWatchSection({ draft, setDraft, projectReady }: Props) {</div></label>+ <label className="block space-y-1.5">+ <span className="text-sm font-semibold">+ {t("settings.sections.sourceWatch.parsingConcurrency", {+ defaultValue: "Concurrent document parsers",+ })}+ </span>+ <input+ type="number"+ min={1}+ max={8}+ value={config.parsingConcurrency}+ onChange={(event) => updateConfig({ parsingConcurrency: Number(event.target.value) || 1 })}+ disabled={!projectReady}+ className="w-24 rounded-md border border-input bg-background px-2 py-1 text-sm"+ />+ <p className="text-xs leading-relaxed text-muted-foreground">+ {t("settings.sections.sourceWatch.parsingConcurrencyDescription", {+ defaultValue:+ "Parse up to this many imported documents at once before sequential Wiki generation. PDF parsing remains internally serialized for safety.",+ })}+ </p>+ </label>+<label className="flex items-start gap-3"><inputtype="checkbox"diff --git a/src/i18n/en.json b/src/i18n/en.jsonindex d080064a..0763a67c 100644--- a/src/i18n/en.json+++ b/src/i18n/en.json@@ -704,6 +704,8 @@"autoIngestDescription": "When disabled, external changes still refresh the file tree but do not create ingest tasks.","persistExtractedMarkdown": "Keep parsed Markdown","persistExtractedMarkdownDescription": "Save extracted document text under raw/parsed using the same folder structure as raw/sources.",+ "parsingConcurrency": "Concurrent document parsers",+ "parsingConcurrencyDescription": "Parse up to this many imported documents at once before sequential Wiki generation. PDF parsing remains internally serialized for safety.","noProject": "Open a project to change this project-level setting.","fileTypes": "Allowed file types","fileTypesDescription": "Document types are enabled by default. Media, binaries, source code, and config-like formats stay off unless selected.",diff --git a/src/i18n/zh.json b/src/i18n/zh.jsonindex 0158a84b..81d36bb3 100644--- a/src/i18n/zh.json+++ b/src/i18n/zh.json@@ -704,6 +704,8 @@"autoIngestDescription": "关闭后,外部变化仍会刷新文件树,但不会创建提取任务。","persistExtractedMarkdown": "保留解析后的 Markdown","persistExtractedMarkdownDescription": "将文档解析出的文本按 raw/sources 的目录结构保存到 raw/parsed。",+ "parsingConcurrency": "文档解析并发数",+ "parsingConcurrencyDescription": "进入串行 Wiki 生成前,最多同时解析这些文档。为保证安全,PDF 解析器内部仍会串行执行。","noProject": "打开项目后才能修改这个项目级设置。","fileTypes": "允许的文件类型","fileTypesDescription": "常见文档类型默认开启。媒体、二进制、源码和配置类格式默认关闭,需要时再手动选择。",diff --git a/src/lib/project-file-sync.test.ts b/src/lib/project-file-sync.test.tsindex 36c62eab..10a3a48e 100644--- a/src/lib/project-file-sync.test.ts+++ b/src/lib/project-file-sync.test.ts@@ -468,6 +468,7 @@ describe("project file sync", () => {enabled: true,autoIngest: true,persistExtractedMarkdown: false,+ parsingConcurrency: 2,includeExtensions: ["md", "xml"],excludeExtensions: [],excludeDirs: [],diff --git a/src/lib/scheduled-import.ts b/src/lib/scheduled-import.tsindex 27873aae..8313c4bc 100644--- a/src/lib/scheduled-import.ts+++ b/src/lib/scheduled-import.ts@@ -4,7 +4,6 @@ import {getFileMd5,getFileSize,listDirectory,- preprocessFile,readFile,writeFileAtomic,} from "@/commands/fs"@@ -428,7 +427,6 @@ export async function scanAndImport(if (changedFiles.length > 0) {const destPaths = changedFiles.map((file) => file.destPath)- await Promise.all(destPaths.map((path) => preprocessFile(path).catch(() => {})))if (isCurrentRun(project.id, options.runId)) {const ids = await enqueueSourceIngest(project, destPaths, llmConfig)if (ids.length > 0) {diff --git a/src/lib/source-lifecycle.test.ts b/src/lib/source-lifecycle.test.tsindex 385b9391..2b2773da 100644--- a/src/lib/source-lifecycle.test.ts+++ b/src/lib/source-lifecycle.test.ts@@ -95,6 +95,7 @@ describe("source-lifecycle path helpers", () => {enabled: true,autoIngest: true,persistExtractedMarkdown: false,+ parsingConcurrency: 2,includeExtensions: ["md"],excludeExtensions: ["json"],excludeDirs: ["drafts"],@@ -155,6 +156,7 @@ describe("source-lifecycle path helpers", () => {enabled: true,autoIngest: true,persistExtractedMarkdown: false,+ parsingConcurrency: 2,includeExtensions: ["json", "yaml", "md"],excludeExtensions: [],excludeDirs: [],@@ -224,6 +226,7 @@ describe("source-lifecycle path helpers", () => {enabled: true,autoIngest: true,persistExtractedMarkdown: false,+ parsingConcurrency: 2,includeExtensions: ["md"],excludeExtensions: [],excludeDirs: ["drafts"],@@ -260,6 +263,7 @@ describe("source-lifecycle path helpers", () => {enabled: true,autoIngest: true,persistExtractedMarkdown: false,+ parsingConcurrency: 2,includeExtensions: ["md", "pdf"],excludeExtensions: [],excludeDirs: [],@@ -324,6 +328,7 @@ describe("source-lifecycle path helpers", () => {enabled: true,autoIngest: true,persistExtractedMarkdown: false,+ parsingConcurrency: 2,includeExtensions: ["md"],excludeExtensions: [],excludeDirs: [],diff --git a/src/lib/source-lifecycle.ts b/src/lib/source-lifecycle.tsindex 946e1b68..d189874b 100644--- a/src/lib/source-lifecycle.ts+++ b/src/lib/source-lifecycle.ts@@ -5,7 +5,6 @@ import {fileExists,getFileSize,listDirectory,- preprocessFile,readFile,writeFile,} from "@/commands/fs"@@ -39,6 +38,8 @@ import { isPathAllowedBySourceWatch, normalizeSourceWatchConfig } from "@/lib/soimport { isSensitiveConfigSourceFile } from "@/lib/source-filter"import { naturalCompare } from "@/lib/natural-sort"import type { SourceWatchConfig } from "@/stores/wiki-store"+import { useWikiStore } from "@/stores/wiki-store"+import { preprocessSourceFiles } from "@/lib/source-preprocess"export const INGESTABLE_SOURCE_EXTENSIONS = new Set(["md",@@ -240,9 +241,8 @@ export async function enqueueSourceIngest(project: WikiProject,sourcePaths: string[],llmConfig: LlmConfig,- options: { sourceRoot?: string; rootContext?: string } = {},+ options: { sourceRoot?: string; rootContext?: string; parsingConcurrency?: number } = {},): Promise<string[]> {- if (!hasUsableLlm(getTaskLlmConfig("ingest", llmConfig))) return []const files = sourcePaths.filter((sourcePath) =>isIngestableSourcePath(sourcePath) &&@@ -256,6 +256,10 @@ export async function enqueueSourceIngest(),}))if (files.length === 0) return []+ const parsingConcurrency = options.parsingConcurrency+ ?? normalizeSourceWatchConfig(useWikiStore.getState().sourceWatchConfig).parsingConcurrency+ await preprocessSourceFiles(files.map((file) => file.sourcePath), parsingConcurrency)+ if (!hasUsableLlm(getTaskLlmConfig("ingest", llmConfig))) return []return enqueueBatch(project.id, files)}@@ -295,13 +299,14 @@ export async function importSourceFiles(try {await copyFile(sourcePath, destPath)importedPaths.push(destPath)- preprocessFile(destPath).catch(() => {})} catch (err) {console.error(`Failed to import ${originalName}:`, err)}}- await enqueueSourceIngest(project, importedPaths, llmConfig)+ await enqueueSourceIngest(project, importedPaths, llmConfig, {+ parsingConcurrency: cfg.parsingConcurrency,+ })return importedPaths}@@ -348,7 +353,6 @@ export async function importSourceFolder(if (parent) await createDirectory(parent)await copyFile(file.path, destPath)allowedFiles.push(destPath)- preprocessFile(destPath).catch(() => {})}const naturallyOrderedFiles = [...allowedFiles].sort((a, b) =>@@ -359,6 +363,7 @@ export async function importSourceFolder(await enqueueSourceIngest(project, naturallyOrderedFiles, llmConfig, {sourceRoot: destDir,rootContext: folderName,+ parsingConcurrency: cfg.parsingConcurrency,})}diff --git a/src/lib/source-preprocess.test.ts b/src/lib/source-preprocess.test.tsnew file mode 100644index 00000000..f10d65da--- /dev/null+++ b/src/lib/source-preprocess.test.ts@@ -0,0 +1,57 @@+import { beforeEach, describe, expect, it, vi } from "vitest"++const preprocessFile = vi.hoisted(() => vi.fn())+vi.mock("@/commands/fs", () => ({ preprocessFile }))++import { normalizeParsingConcurrency, preprocessSourceFiles } from "./source-preprocess"++describe("source preprocessing", () => {+ beforeEach(() => vi.clearAllMocks())++ it("normalizes concurrency into the supported integer range", () => {+ expect(normalizeParsingConcurrency(Number.NaN)).toBe(1)+ expect(normalizeParsingConcurrency(0)).toBe(1)+ expect(normalizeParsingConcurrency(3.9)).toBe(3)+ expect(normalizeParsingConcurrency(99)).toBe(8)+ })++ it("never exceeds the configured concurrency and de-duplicates paths", async () => {+ let active = 0+ let peak = 0+ preprocessFile.mockImplementation(async () => {+ active += 1+ peak = Math.max(peak, active)+ await new Promise((resolve) => setTimeout(resolve, 5))+ active -= 1+ })++ await preprocessSourceFiles(["a.pdf", "b.docx", "a.pdf", "c.epub", "d.org"], 2)++ expect(peak).toBe(2)+ expect(preprocessFile).toHaveBeenCalledTimes(4)+ })++ it("shares the concurrency ceiling across simultaneous batches", async () => {+ let active = 0+ let peak = 0+ preprocessFile.mockImplementation(async () => {+ active += 1+ peak = Math.max(peak, active)+ await new Promise((resolve) => setTimeout(resolve, 5))+ active -= 1+ })++ await Promise.all([+ preprocessSourceFiles(["a.pdf", "b.pdf", "c.pdf"], 2),+ preprocessSourceFiles(["d.docx", "e.docx", "f.docx"], 2),+ ])++ expect(peak).toBe(2)+ })++ it("continues after one extractor fails", async () => {+ preprocessFile.mockRejectedValueOnce(new Error("bad file")).mockResolvedValue("")+ await expect(preprocessSourceFiles(["bad.pdf", "good.docx"], 1)).resolves.toBeUndefined()+ expect(preprocessFile).toHaveBeenCalledTimes(2)+ })+})diff --git a/src/lib/source-preprocess.ts b/src/lib/source-preprocess.tsnew file mode 100644index 00000000..0a743019--- /dev/null+++ b/src/lib/source-preprocess.ts@@ -0,0 +1,73 @@+import { preprocessFile } from "@/commands/fs"++export const MIN_PARSING_CONCURRENCY = 1+export const MAX_PARSING_CONCURRENCY = 8++export function normalizeParsingConcurrency(value: number): number {+ if (!Number.isFinite(value)) return MIN_PARSING_CONCURRENCY+ return Math.max(+ MIN_PARSING_CONCURRENCY,+ Math.min(MAX_PARSING_CONCURRENCY, Math.floor(value)),+ )+}++interface WaitingParser {+ limit: number+ resolve: () => void+}++let activeParsers = 0+const waitingParsers: WaitingParser[] = []++function drainWaitingParsers(): void {+ while (waitingParsers.length > 0 && activeParsers < waitingParsers[0].limit) {+ const waiter = waitingParsers.shift()+ if (!waiter) return+ activeParsers += 1+ waiter.resolve()+ }+}++async function acquireParser(limit: number): Promise<() => void> {+ await new Promise<void>((resolve) => {+ waitingParsers.push({ limit, resolve })+ drainWaitingParsers()+ })+ let released = false+ return () => {+ if (released) return+ released = true+ activeParsers -= 1+ drainWaitingParsers()+ }+}++/**+ * Pre-extract source text with bounded concurrency before LLM work begins.+ * Individual extractor failures are intentionally non-fatal: autoIngest will+ * retry through read_file and surface a real source-read error if necessary.+ */+export async function preprocessSourceFiles(+ paths: readonly string[],+ concurrency: number,+): Promise<void> {+ const uniquePaths = [...new Set(paths)]+ if (uniquePaths.length === 0) return+ const limit = normalizeParsingConcurrency(concurrency)++ await Promise.all(uniquePaths.map(async (path) => {+ const release = await acquireParser(limit)+ try {+ try {+ await preprocessFile(path)+ } catch (err) {+ console.warn(+ `[source-preprocess] failed to pre-process "${path}"; ingest will retry when reading it:`,+ err instanceof Error ? err.message : err,+ )+ }+ } finally {+ release()+ }+ }))+}diff --git a/src/lib/source-watch-config.test.ts b/src/lib/source-watch-config.test.tsindex 39a9438c..9fa4c113 100644--- a/src/lib/source-watch-config.test.ts+++ b/src/lib/source-watch-config.test.ts@@ -10,9 +10,14 @@ describe("source watch config", () => {it("uses the shared default fixture", () => {expect(DEFAULT_SOURCE_WATCH_CONFIG).toEqual(sourceWatchDefaults)expect(normalizeSourceWatchConfig({}).persistExtractedMarkdown).toBe(false)+ expect(normalizeSourceWatchConfig({}).parsingConcurrency).toBe(2)expect(normalizeSourceWatchConfig({ persistExtractedMarkdown: true }).persistExtractedMarkdown,).toBe(true)+ expect(normalizeSourceWatchConfig({ parsingConcurrency: 20 }).parsingConcurrency).toBe(8)+ expect(+ normalizeSourceWatchConfig({ parsingConcurrency: Number.NaN }).parsingConcurrency,+ ).toBe(2)})it("allows document types by default and rejects config/media/binaries", () => {diff --git a/src/lib/source-watch-config.ts b/src/lib/source-watch-config.tsindex e819d54c..0aaded22 100644--- a/src/lib/source-watch-config.ts+++ b/src/lib/source-watch-config.ts@@ -42,11 +42,17 @@ function normalizeList(values: readonly string[] | undefined): string[] {}export function normalizeSourceWatchConfig(config?: Partial<SourceWatchConfig> | null): SourceWatchConfig {+ const rawParsingConcurrency = config?.parsingConcurrency+ ?? DEFAULT_SOURCE_WATCH_CONFIG.parsingConcurrency+ const parsingConcurrency = Number.isFinite(rawParsingConcurrency)+ ? Math.max(1, Math.min(8, Math.floor(rawParsingConcurrency)))+ : DEFAULT_SOURCE_WATCH_CONFIG.parsingConcurrencyreturn {enabled: config?.enabled ?? DEFAULT_SOURCE_WATCH_CONFIG.enabled,autoIngest: config?.autoIngest ?? DEFAULT_SOURCE_WATCH_CONFIG.autoIngest,persistExtractedMarkdown:config?.persistExtractedMarkdown ?? DEFAULT_SOURCE_WATCH_CONFIG.persistExtractedMarkdown,+ parsingConcurrency,includeExtensions: normalizeExtensions(config?.includeExtensions ?? DEFAULT_SOURCE_WATCH_CONFIG.includeExtensions),excludeExtensions: normalizeExtensions(config?.excludeExtensions ?? DEFAULT_SOURCE_WATCH_CONFIG.excludeExtensions),excludeDirs: normalizeList(config?.excludeDirs ?? DEFAULT_SOURCE_WATCH_CONFIG.excludeDirs),diff --git a/src/lib/source-watch-defaults.json b/src/lib/source-watch-defaults.jsonindex 39054be2..52c3fbb1 100644--- a/src/lib/source-watch-defaults.json+++ b/src/lib/source-watch-defaults.json@@ -2,6 +2,7 @@"enabled": true,"autoIngest": true,"persistExtractedMarkdown": false,+ "parsingConcurrency": 2,"includeExtensions": ["md","mdx",diff --git a/src/stores/wiki-store.ts b/src/stores/wiki-store.tsindex 37c33fda..8b728600 100644--- a/src/stores/wiki-store.ts+++ b/src/stores/wiki-store.ts@@ -261,6 +261,8 @@ interface SourceWatchConfig {autoIngest: boolean/** Keep extractor output in raw/parsed in addition to the internal cache. */persistExtractedMarkdown: boolean+ /** Maximum number of source text extraction jobs allowed at once. */+ parsingConcurrency: numberincludeExtensions: string[]excludeExtensions: string[]excludeDirs: string[]