Skip to content

LLM Wiki is a cross-platform desktop application that turns your documents into an organized, interlinked knowledge base — automatically. Instead of traditional RAG (retrieve-and-answer from scratch every time), the LLM incrementally builds and maintains a persistent wiki from your sources。

TypeScript

refactor: split ingest preparation from commit

nash_subb601cfdf0d5
diff --git a/src/lib/ingest.scenarios.test.ts b/src/lib/ingest.scenarios.test.ts
index b9d1e903..f16af95e 100644
--- a/src/lib/ingest.scenarios.test.ts
+++ b/src/lib/ingest.scenarios.test.ts
@@ -203,6 +203,90 @@ describe("ingest scenarios (fixture-driven)", () => {
},
)
+ it("routes project mutations through the injected commit runner", async () => {
+ const scenario = ingestScenarios[0]
+ ctx = await setup(scenario)
+ const sourceFullPath = path.join(ctx.tmp.path, scenario.source.path)
+ let commitCalls = 0
+ let commitActive = false
+ let writeCallbacks = 0
+
+ const written = await autoIngest(
+ ctx.tmp.path,
+ sourceFullPath,
+ useWikiStore.getState().llmConfig,
+ undefined,
+ undefined,
+ (relativePath) => {
+ writeCallbacks += 1
+ expect(commitActive, `write callback escaped commit boundary: ${relativePath}`).toBe(true)
+ },
+ {
+ runCommit: async (operation) => {
+ commitCalls += 1
+ commitActive = true
+ try {
+ return await operation()
+ } finally {
+ commitActive = false
+ }
+ },
+ },
+ )
+
+ expect(commitCalls).toBe(1)
+ expect(written.length).toBeGreaterThan(0)
+ expect(writeCallbacks).toBeGreaterThan(0)
+ })
+
+ it("routes cache-hit mutations through the injected commit runner", async () => {
+ const scenario = ingestScenarios[0]
+ ctx = await setup(scenario)
+ const sourceFullPath = path.join(ctx.tmp.path, scenario.source.path)
+ const firstWritten = await autoIngest(
+ ctx.tmp.path,
+ sourceFullPath,
+ useWikiStore.getState().llmConfig,
+ )
+ let commitCalls = 0
+
+ const cachedWritten = await autoIngest(
+ ctx.tmp.path,
+ sourceFullPath,
+ useWikiStore.getState().llmConfig,
+ undefined,
+ undefined,
+ undefined,
+ {
+ runCommit: async (operation) => {
+ commitCalls += 1
+ return operation()
+ },
+ },
+ )
+
+ expect(commitCalls).toBe(1)
+ expect(cachedWritten).toEqual(firstWritten)
+ const activities = useActivityStore.getState().items
+ expect(activities.some((item) => item.detail.includes("Skipped (unchanged)"))).toBe(true)
+ })
+
+ it("serializes concurrent ingestion of the same source and reuses its cache", async () => {
+ const scenario = ingestScenarios[0]
+ ctx = await setup(scenario)
+ const sourceFullPath = path.join(ctx.tmp.path, scenario.source.path)
+
+ const [firstWritten, secondWritten] = await Promise.all([
+ autoIngest(ctx.tmp.path, sourceFullPath, useWikiStore.getState().llmConfig),
+ autoIngest(ctx.tmp.path, sourceFullPath, useWikiStore.getState().llmConfig),
+ ])
+
+ expect(streamCallCount).toBe(2)
+ expect(secondWritten).toEqual(firstWritten)
+ const activities = useActivityStore.getState().items
+ expect(activities.some((item) => item.detail.includes("Skipped (unchanged)"))).toBe(true)
+ })
+
it("drops generated pages whose frontmatter type disagrees with schema routing", async () => {
ctx = { tmp: await createTempProject("ingest-schema-routing") }
const projectPath = ctx.tmp.path
diff --git a/src/lib/ingest.ts b/src/lib/ingest.ts
index e6761d5d..c428252d 100644
--- a/src/lib/ingest.ts
+++ b/src/lib/ingest.ts
@@ -562,14 +562,20 @@ export function languageRule(sourceContent: string = ""): string {
* Auto-ingest: reads source → LLM analyzes → LLM writes wiki pages, all in one go.
* Used when importing new files.
*
- * Concurrency: this function holds a per-project lock for its full
- * duration. Two simultaneous calls for the same project (e.g. queue
- * + Save-to-Wiki) take turns. The lock is necessary because the
- * analysis stage reads `wiki/index.md` and the generation stage
- * overwrites it; without serialization, each call would emit an
- * "updated" index based on the same pre-state and overwrite each
- * other's additions.
+ * Concurrency: source preparation and LLM generation for different sources may
+ * overlap. Wiki-page and aggregate-file mutations run through a per-project
+ * commit boundary. Queue callers can add stricter ordering around that boundary.
*/
+export type IngestCommitRunner = <T>(operation: () => Promise<T>) => Promise<T>
+
+export interface AutoIngestOptions {
+ /**
+ * Adds queue ordering around a commit. The supplied runner must not acquire
+ * the project mutex; autoIngest always acquires it inside this callback.
+ */
+ runCommit?: IngestCommitRunner
+}
+
export async function autoIngest(
projectPath: string,
sourcePath: string,
@@ -577,9 +583,21 @@ export async function autoIngest(
signal?: AbortSignal,
folderContext?: string,
onFileWritten?: (relativePath: string) => void,
+ options?: AutoIngestOptions,
): Promise<string[]> {
- return withProjectLock(normalizePath(projectPath), () =>
- autoIngestImpl(projectPath, sourcePath, llmConfig, signal, folderContext, onFileWritten),
+ const pp = normalizePath(projectPath)
+ const sp = normalizePath(sourcePath)
+ return withProjectLock(
+ `ingest-source\0${pp}\0${sp}`,
+ () => autoIngestImpl(
+ projectPath,
+ sourcePath,
+ llmConfig,
+ signal,
+ folderContext,
+ onFileWritten,
+ options,
+ ),
)
}
@@ -645,6 +663,7 @@ async function autoIngestImpl(
signal?: AbortSignal,
folderContext?: string,
onFileWritten?: (relativePath: string) => void,
+ options?: AutoIngestOptions,
): Promise<string[]> {
const pp = normalizePath(projectPath)
const sp = normalizePath(sourcePath)
@@ -653,6 +672,10 @@ async function autoIngestImpl(
const sourceIdentity = sourceIdentityForPath(pp, sp)
const sourceSummarySlug = sourceSummarySlugFromIdentity(sourceIdentity)
const sourceSummaryPath = `wiki/sources/${sourceSummarySlug}.md`
+ const queueCommit = options?.runCommit
+ const runCommit: IngestCommitRunner = queueCommit
+ ? ((operation) => queueCommit(() => withProjectLock(pp, operation)))
+ : ((operation) => withProjectLock(pp, operation))
console.log(`[ingest:diag] autoIngestImpl ENTRY for "${fileName}" (project="${pp}", source="${sp}")`)
const activityId = activity.addItem({
type: "ingest",
@@ -745,7 +768,9 @@ async function autoIngestImpl(
const cachedFiles = await checkIngestCache(pp, sourceIdentity, sourceContent)
console.log(`[ingest:diag] cache check for "${sourceIdentity}":`, cachedFiles === null ? "MISS (full pipeline)" : `HIT (${cachedFiles.length} cached files)`)
if (cachedFiles !== null) {
- try {
+ return runCommit(async () => {
+ throwIfIngestAborted(signal, activityId)
+ try {
console.log(`[ingest:diag] cache-hit branch: starting image extraction for ${sp}`)
const skipNativePdfImageExtraction = isPdf && hasMineruImageRefs(sourceContent, sourceSummarySlug)
let savedImages = skipNativePdfImageExtraction
@@ -780,17 +805,19 @@ async function autoIngestImpl(
const captionLlm = resolveCaptionConfig(mmCfg, llmConfig)
if (captionLlm) {
try {
- await captionMarkdownImages(pp, appendSavedImageRefsForCaption(sourceContent, savedImages), captionLlm, {
- signal,
- shouldCaption: (url) =>
- isSavedImagePromptUrl(pp, sourceSummarySlug, url),
- urlToAbsPath: (url) => promptImageUrlToAbs(pp, url),
- concurrency: mmCfg.concurrency,
- onProgress: (done, total) =>
- activity.updateItem(activityId, {
- detail: `Captioning images... ${done}/${total}`,
- }),
- })
+ await withProjectLock(`${pp}\0image-caption-cache`, () =>
+ captionMarkdownImages(pp, appendSavedImageRefsForCaption(sourceContent, savedImages), captionLlm, {
+ signal,
+ shouldCaption: (url) =>
+ isSavedImagePromptUrl(pp, sourceSummarySlug, url),
+ urlToAbsPath: (url) => promptImageUrlToAbs(pp, url),
+ concurrency: mmCfg.concurrency,
+ onProgress: (done, total) =>
+ activity.updateItem(activityId, {
+ detail: `Captioning images... ${done}/${total}`,
+ }),
+ }),
+ )
} catch (err) {
console.warn(
`[ingest:caption] cache-hit caption pass failed:`,
@@ -810,18 +837,19 @@ async function autoIngestImpl(
} else {
console.log(`[ingest:diag] cache-hit branch: skipping injection (no images returned from extraction)`)
}
- } catch (err) {
- console.warn(
- `[ingest:images] cache-hit injection failed for "${fileName}":`,
- err instanceof Error ? err.message : err,
- )
- }
- activity.updateItem(activityId, {
- status: "done",
- detail: `Skipped (unchanged) — ${cachedFiles.length} files from previous ingest`,
- filesWritten: cachedFiles,
+ } catch (err) {
+ console.warn(
+ `[ingest:images] cache-hit injection failed for "${fileName}":`,
+ err instanceof Error ? err.message : err,
+ )
+ }
+ activity.updateItem(activityId, {
+ status: "done",
+ detail: `Skipped (unchanged) — ${cachedFiles.length} files from previous ingest`,
+ filesWritten: cachedFiles,
+ })
+ return cachedFiles
})
- return cachedFiles
}
// ── Step 0.5: Extract embedded images ─────────────────────────
@@ -922,21 +950,23 @@ async function autoIngestImpl(
activity.updateItem(activityId, { detail: "Captioning images..." })
const ourMediaPrefix = `${pp}/wiki/media/${sourceSummarySlug}/`
try {
- const result = await captionMarkdownImages(pp, enrichedSourceContent, captionLlm, {
- signal,
- // Strict filter: only caption images we know we just
- // extracted into this source's media directory. Skips any
- // pre-existing markdown image refs the user may have typed
- // into the source content (e.g. for hand-authored .md
- // sources).
- shouldCaption: (url) => url.startsWith(ourMediaPrefix) || isSavedImagePromptUrl(pp, sourceSummarySlug, url),
- urlToAbsPath: (url) => promptImageUrlToAbs(pp, url),
- concurrency: mmCfg.concurrency,
- onProgress: (done, total) =>
- activity.updateItem(activityId, {
- detail: `Captioning images... ${done}/${total}`,
- }),
- })
+ const result = await withProjectLock(`${pp}\0image-caption-cache`, () =>
+ captionMarkdownImages(pp, enrichedSourceContent, captionLlm, {
+ signal,
+ // Strict filter: only caption images we know we just
+ // extracted into this source's media directory. Skips any
+ // pre-existing markdown image refs the user may have typed
+ // into the source content (e.g. for hand-authored .md
+ // sources).
+ shouldCaption: (url) => url.startsWith(ourMediaPrefix) || isSavedImagePromptUrl(pp, sourceSummarySlug, url),
+ urlToAbsPath: (url) => promptImageUrlToAbs(pp, url),
+ concurrency: mmCfg.concurrency,
+ onProgress: (done, total) =>
+ activity.updateItem(activityId, {
+ detail: `Captioning images... ${done}/${total}`,
+ }),
+ }),
+ )
enrichedSourceContent = stripWikiMediaAbsPaths(pp, result.enrichedMarkdown)
console.log(
`[ingest:caption] images=${savedImages.length} fresh=${result.freshCaptions} cached=${result.cachedCaptions} failed=${result.failed}`,
@@ -1121,6 +1151,7 @@ async function autoIngestImpl(
}
// ── Step 3: Write files ───────────────────────────────────────
+ return runCommit(async () => {
throwIfIngestAborted(signal, activityId)
activity.updateItem(activityId, { detail: "Writing files..." })
await migrateLegacySourceSummaryIfSafe(pp, sourceIdentity, sourceSummaryPath)
@@ -1416,6 +1447,7 @@ async function autoIngestImpl(
})
return writtenPaths
+ })
}
/**
diff --git a/src/lib/project-mutex.ts b/src/lib/project-mutex.ts
index 38327026..0e95356b 100644
--- a/src/lib/project-mutex.ts
+++ b/src/lib/project-mutex.ts
@@ -1,20 +1,12 @@
/**
* Per-project async mutex.
*
- * Why this exists: `autoIngest` reads `wiki/index.md` at analysis
- * time, asks the LLM to emit an updated `index.md`, then overwrites
- * the file at write time. If two ingests run concurrently for the
- * SAME project (queue-driven ingest happening while a Save-to-Wiki
- * or deep-research auto-ingest fires), each LLM sees the same
- * pre-state of `index.md`, each emits its own "updated" version,
- * and whichever finishes second silently overwrites the first.
- * Net effect: pages from the first ingest disappear from the index
- * with no error surfaced anywhere.
+ * Why this exists: multiple ingest and Save-to-Wiki entry points can mutate
+ * shared project files such as `wiki/index.md`, `wiki/log.md`, page bodies,
+ * and ingest caches. Those read-modify-write operations must not overlap.
*
- * The queue itself is already serial — but Save-to-Wiki and
- * deep-research bypass the queue (`autoIngest(...).catch(...)`).
- * Wrapping `autoIngest`'s body in `withProjectLock(projectPath, …)`
- * forces all entry points to take turns.
+ * Expensive source preparation may run concurrently, but commit sections use
+ * this mutex so queue, Save-to-Wiki, and deep-research writes take turns.
*
* The lock is a simple promise chain. No timeouts, no fairness, no
* re-entrancy detection — those would all be overkill. If `fn`
@@ -41,10 +33,8 @@ export async function withProjectLock<T>(
const next = new Promise<void>((resolve) => {
release = resolve
})
- locks.set(
- projectPath,
- prev.then(() => next),
- )
+ const tail = prev.then(() => next)
+ locks.set(projectPath, tail)
try {
// Wait for the previous holder. Swallow rejections from `prev`
// (a previous caller's failure shouldn't prevent us from running).
@@ -54,20 +44,14 @@ export async function withProjectLock<T>(
release()
// Best-effort cleanup: if our promise is still the tail, drop the
// map entry. Otherwise a later caller has chained on; leave it.
- if (locks.get(projectPath) === next || locks.size > 1024) {
- // Tail check is approximate (the map stores prev.then(() => next),
- // not next directly). The size guard prevents pathological
- // unbounded growth if many distinct projectPaths cycle through.
- const tail = locks.get(projectPath)
- if (tail) {
- // Defer the delete one tick so a caller that just chained on
- // doesn't see us yank the entry mid-chain.
- Promise.resolve().then(() => {
- if (locks.get(projectPath) === tail) {
- locks.delete(projectPath)
- }
- })
- }
+ if (locks.get(projectPath) === tail) {
+ // Defer the delete one tick so a caller that just chained on
+ // doesn't see us yank the entry mid-chain.
+ Promise.resolve().then(() => {
+ if (locks.get(projectPath) === tail) {
+ locks.delete(projectPath)
+ }
+ })
}
}
}