LLM Wiki is a cross-platform desktop application that turns your documents into an organized, interlinked knowledge base — automatically. Instead of traditional RAG (retrieve-and-answer from scratch every time), the LLM incrementally builds and maintains a persistent wiki from your sources。
TypeScript
refactor: split ingest preparation from commit
nash_subb601cfdf0d5
diff --git a/src/lib/ingest.scenarios.test.ts b/src/lib/ingest.scenarios.test.tsindex b9d1e903..f16af95e 100644--- a/src/lib/ingest.scenarios.test.ts+++ b/src/lib/ingest.scenarios.test.ts@@ -203,6 +203,90 @@ describe("ingest scenarios (fixture-driven)", () => {},)+ it("routes project mutations through the injected commit runner", async () => {+ const scenario = ingestScenarios[0]+ ctx = await setup(scenario)+ const sourceFullPath = path.join(ctx.tmp.path, scenario.source.path)+ let commitCalls = 0+ let commitActive = false+ let writeCallbacks = 0++ const written = await autoIngest(+ ctx.tmp.path,+ sourceFullPath,+ useWikiStore.getState().llmConfig,+ undefined,+ undefined,+ (relativePath) => {+ writeCallbacks += 1+ expect(commitActive, `write callback escaped commit boundary: ${relativePath}`).toBe(true)+ },+ {+ runCommit: async (operation) => {+ commitCalls += 1+ commitActive = true+ try {+ return await operation()+ } finally {+ commitActive = false+ }+ },+ },+ )++ expect(commitCalls).toBe(1)+ expect(written.length).toBeGreaterThan(0)+ expect(writeCallbacks).toBeGreaterThan(0)+ })++ it("routes cache-hit mutations through the injected commit runner", async () => {+ const scenario = ingestScenarios[0]+ ctx = await setup(scenario)+ const sourceFullPath = path.join(ctx.tmp.path, scenario.source.path)+ const firstWritten = await autoIngest(+ ctx.tmp.path,+ sourceFullPath,+ useWikiStore.getState().llmConfig,+ )+ let commitCalls = 0++ const cachedWritten = await autoIngest(+ ctx.tmp.path,+ sourceFullPath,+ useWikiStore.getState().llmConfig,+ undefined,+ undefined,+ undefined,+ {+ runCommit: async (operation) => {+ commitCalls += 1+ return operation()+ },+ },+ )++ expect(commitCalls).toBe(1)+ expect(cachedWritten).toEqual(firstWritten)+ const activities = useActivityStore.getState().items+ expect(activities.some((item) => item.detail.includes("Skipped (unchanged)"))).toBe(true)+ })++ it("serializes concurrent ingestion of the same source and reuses its cache", async () => {+ const scenario = ingestScenarios[0]+ ctx = await setup(scenario)+ const sourceFullPath = path.join(ctx.tmp.path, scenario.source.path)++ const [firstWritten, secondWritten] = await Promise.all([+ autoIngest(ctx.tmp.path, sourceFullPath, useWikiStore.getState().llmConfig),+ autoIngest(ctx.tmp.path, sourceFullPath, useWikiStore.getState().llmConfig),+ ])++ expect(streamCallCount).toBe(2)+ expect(secondWritten).toEqual(firstWritten)+ const activities = useActivityStore.getState().items+ expect(activities.some((item) => item.detail.includes("Skipped (unchanged)"))).toBe(true)+ })+it("drops generated pages whose frontmatter type disagrees with schema routing", async () => {ctx = { tmp: await createTempProject("ingest-schema-routing") }const projectPath = ctx.tmp.pathdiff --git a/src/lib/ingest.ts b/src/lib/ingest.tsindex e6761d5d..c428252d 100644--- a/src/lib/ingest.ts+++ b/src/lib/ingest.ts@@ -562,14 +562,20 @@ export function languageRule(sourceContent: string = ""): string {* Auto-ingest: reads source → LLM analyzes → LLM writes wiki pages, all in one go.* Used when importing new files.*- * Concurrency: this function holds a per-project lock for its full- * duration. Two simultaneous calls for the same project (e.g. queue- * + Save-to-Wiki) take turns. The lock is necessary because the- * analysis stage reads `wiki/index.md` and the generation stage- * overwrites it; without serialization, each call would emit an- * "updated" index based on the same pre-state and overwrite each- * other's additions.+ * Concurrency: source preparation and LLM generation for different sources may+ * overlap. Wiki-page and aggregate-file mutations run through a per-project+ * commit boundary. Queue callers can add stricter ordering around that boundary.*/+export type IngestCommitRunner = <T>(operation: () => Promise<T>) => Promise<T>++export interface AutoIngestOptions {+ /**+ * Adds queue ordering around a commit. The supplied runner must not acquire+ * the project mutex; autoIngest always acquires it inside this callback.+ */+ runCommit?: IngestCommitRunner+}+export async function autoIngest(projectPath: string,sourcePath: string,@@ -577,9 +583,21 @@ export async function autoIngest(signal?: AbortSignal,folderContext?: string,onFileWritten?: (relativePath: string) => void,+ options?: AutoIngestOptions,): Promise<string[]> {- return withProjectLock(normalizePath(projectPath), () =>- autoIngestImpl(projectPath, sourcePath, llmConfig, signal, folderContext, onFileWritten),+ const pp = normalizePath(projectPath)+ const sp = normalizePath(sourcePath)+ return withProjectLock(+ `ingest-source\0${pp}\0${sp}`,+ () => autoIngestImpl(+ projectPath,+ sourcePath,+ llmConfig,+ signal,+ folderContext,+ onFileWritten,+ options,+ ),)}@@ -645,6 +663,7 @@ async function autoIngestImpl(signal?: AbortSignal,folderContext?: string,onFileWritten?: (relativePath: string) => void,+ options?: AutoIngestOptions,): Promise<string[]> {const pp = normalizePath(projectPath)const sp = normalizePath(sourcePath)@@ -653,6 +672,10 @@ async function autoIngestImpl(const sourceIdentity = sourceIdentityForPath(pp, sp)const sourceSummarySlug = sourceSummarySlugFromIdentity(sourceIdentity)const sourceSummaryPath = `wiki/sources/${sourceSummarySlug}.md`+ const queueCommit = options?.runCommit+ const runCommit: IngestCommitRunner = queueCommit+ ? ((operation) => queueCommit(() => withProjectLock(pp, operation)))+ : ((operation) => withProjectLock(pp, operation))console.log(`[ingest:diag] autoIngestImpl ENTRY for "${fileName}" (project="${pp}", source="${sp}")`)const activityId = activity.addItem({type: "ingest",@@ -745,7 +768,9 @@ async function autoIngestImpl(const cachedFiles = await checkIngestCache(pp, sourceIdentity, sourceContent)console.log(`[ingest:diag] cache check for "${sourceIdentity}":`, cachedFiles === null ? "MISS (full pipeline)" : `HIT (${cachedFiles.length} cached files)`)if (cachedFiles !== null) {- try {+ return runCommit(async () => {+ throwIfIngestAborted(signal, activityId)+ try {console.log(`[ingest:diag] cache-hit branch: starting image extraction for ${sp}`)const skipNativePdfImageExtraction = isPdf && hasMineruImageRefs(sourceContent, sourceSummarySlug)let savedImages = skipNativePdfImageExtraction@@ -780,17 +805,19 @@ async function autoIngestImpl(const captionLlm = resolveCaptionConfig(mmCfg, llmConfig)if (captionLlm) {try {- await captionMarkdownImages(pp, appendSavedImageRefsForCaption(sourceContent, savedImages), captionLlm, {- signal,- shouldCaption: (url) =>- isSavedImagePromptUrl(pp, sourceSummarySlug, url),- urlToAbsPath: (url) => promptImageUrlToAbs(pp, url),- concurrency: mmCfg.concurrency,- onProgress: (done, total) =>- activity.updateItem(activityId, {- detail: `Captioning images... ${done}/${total}`,- }),- })+ await withProjectLock(`${pp}\0image-caption-cache`, () =>+ captionMarkdownImages(pp, appendSavedImageRefsForCaption(sourceContent, savedImages), captionLlm, {+ signal,+ shouldCaption: (url) =>+ isSavedImagePromptUrl(pp, sourceSummarySlug, url),+ urlToAbsPath: (url) => promptImageUrlToAbs(pp, url),+ concurrency: mmCfg.concurrency,+ onProgress: (done, total) =>+ activity.updateItem(activityId, {+ detail: `Captioning images... ${done}/${total}`,+ }),+ }),+ )} catch (err) {console.warn(`[ingest:caption] cache-hit caption pass failed:`,@@ -810,18 +837,19 @@ async function autoIngestImpl(} else {console.log(`[ingest:diag] cache-hit branch: skipping injection (no images returned from extraction)`)}- } catch (err) {- console.warn(- `[ingest:images] cache-hit injection failed for "${fileName}":`,- err instanceof Error ? err.message : err,- )- }- activity.updateItem(activityId, {- status: "done",- detail: `Skipped (unchanged) — ${cachedFiles.length} files from previous ingest`,- filesWritten: cachedFiles,+ } catch (err) {+ console.warn(+ `[ingest:images] cache-hit injection failed for "${fileName}":`,+ err instanceof Error ? err.message : err,+ )+ }+ activity.updateItem(activityId, {+ status: "done",+ detail: `Skipped (unchanged) — ${cachedFiles.length} files from previous ingest`,+ filesWritten: cachedFiles,+ })+ return cachedFiles})- return cachedFiles}// ── Step 0.5: Extract embedded images ─────────────────────────@@ -922,21 +950,23 @@ async function autoIngestImpl(activity.updateItem(activityId, { detail: "Captioning images..." })const ourMediaPrefix = `${pp}/wiki/media/${sourceSummarySlug}/`try {- const result = await captionMarkdownImages(pp, enrichedSourceContent, captionLlm, {- signal,- // Strict filter: only caption images we know we just- // extracted into this source's media directory. Skips any- // pre-existing markdown image refs the user may have typed- // into the source content (e.g. for hand-authored .md- // sources).- shouldCaption: (url) => url.startsWith(ourMediaPrefix) || isSavedImagePromptUrl(pp, sourceSummarySlug, url),- urlToAbsPath: (url) => promptImageUrlToAbs(pp, url),- concurrency: mmCfg.concurrency,- onProgress: (done, total) =>- activity.updateItem(activityId, {- detail: `Captioning images... ${done}/${total}`,- }),- })+ const result = await withProjectLock(`${pp}\0image-caption-cache`, () =>+ captionMarkdownImages(pp, enrichedSourceContent, captionLlm, {+ signal,+ // Strict filter: only caption images we know we just+ // extracted into this source's media directory. Skips any+ // pre-existing markdown image refs the user may have typed+ // into the source content (e.g. for hand-authored .md+ // sources).+ shouldCaption: (url) => url.startsWith(ourMediaPrefix) || isSavedImagePromptUrl(pp, sourceSummarySlug, url),+ urlToAbsPath: (url) => promptImageUrlToAbs(pp, url),+ concurrency: mmCfg.concurrency,+ onProgress: (done, total) =>+ activity.updateItem(activityId, {+ detail: `Captioning images... ${done}/${total}`,+ }),+ }),+ )enrichedSourceContent = stripWikiMediaAbsPaths(pp, result.enrichedMarkdown)console.log(`[ingest:caption] images=${savedImages.length} fresh=${result.freshCaptions} cached=${result.cachedCaptions} failed=${result.failed}`,@@ -1121,6 +1151,7 @@ async function autoIngestImpl(}// ── Step 3: Write files ───────────────────────────────────────+ return runCommit(async () => {throwIfIngestAborted(signal, activityId)activity.updateItem(activityId, { detail: "Writing files..." })await migrateLegacySourceSummaryIfSafe(pp, sourceIdentity, sourceSummaryPath)@@ -1416,6 +1447,7 @@ async function autoIngestImpl(})return writtenPaths+ })}/**diff --git a/src/lib/project-mutex.ts b/src/lib/project-mutex.tsindex 38327026..0e95356b 100644--- a/src/lib/project-mutex.ts+++ b/src/lib/project-mutex.ts@@ -1,20 +1,12 @@/*** Per-project async mutex.*- * Why this exists: `autoIngest` reads `wiki/index.md` at analysis- * time, asks the LLM to emit an updated `index.md`, then overwrites- * the file at write time. If two ingests run concurrently for the- * SAME project (queue-driven ingest happening while a Save-to-Wiki- * or deep-research auto-ingest fires), each LLM sees the same- * pre-state of `index.md`, each emits its own "updated" version,- * and whichever finishes second silently overwrites the first.- * Net effect: pages from the first ingest disappear from the index- * with no error surfaced anywhere.+ * Why this exists: multiple ingest and Save-to-Wiki entry points can mutate+ * shared project files such as `wiki/index.md`, `wiki/log.md`, page bodies,+ * and ingest caches. Those read-modify-write operations must not overlap.*- * The queue itself is already serial — but Save-to-Wiki and- * deep-research bypass the queue (`autoIngest(...).catch(...)`).- * Wrapping `autoIngest`'s body in `withProjectLock(projectPath, …)`- * forces all entry points to take turns.+ * Expensive source preparation may run concurrently, but commit sections use+ * this mutex so queue, Save-to-Wiki, and deep-research writes take turns.** The lock is a simple promise chain. No timeouts, no fairness, no* re-entrancy detection — those would all be overkill. If `fn`@@ -41,10 +33,8 @@ export async function withProjectLock<T>(const next = new Promise<void>((resolve) => {release = resolve})- locks.set(- projectPath,- prev.then(() => next),- )+ const tail = prev.then(() => next)+ locks.set(projectPath, tail)try {// Wait for the previous holder. Swallow rejections from `prev`// (a previous caller's failure shouldn't prevent us from running).@@ -54,20 +44,14 @@ export async function withProjectLock<T>(release()// Best-effort cleanup: if our promise is still the tail, drop the// map entry. Otherwise a later caller has chained on; leave it.- if (locks.get(projectPath) === next || locks.size > 1024) {- // Tail check is approximate (the map stores prev.then(() => next),- // not next directly). The size guard prevents pathological- // unbounded growth if many distinct projectPaths cycle through.- const tail = locks.get(projectPath)- if (tail) {- // Defer the delete one tick so a caller that just chained on- // doesn't see us yank the entry mid-chain.- Promise.resolve().then(() => {- if (locks.get(projectPath) === tail) {- locks.delete(projectPath)- }- })- }+ if (locks.get(projectPath) === tail) {+ // Defer the delete one tick so a caller that just chained on+ // doesn't see us yank the entry mid-chain.+ Promise.resolve().then(() => {+ if (locks.get(projectPath) === tail) {+ locks.delete(projectPath)+ }+ })}}}