Skip to content

LLM Wiki is a cross-platform desktop application that turns your documents into an organized, interlinked knowledge base — automatically. Instead of traditional RAG (retrieve-and-answer from scratch every time), the LLM incrementally builds and maintains a persistent wiki from your sources。

TypeScript

feat: merge sources[] on ingest write to prevent shared-page data loss

nash_su21eed06b0ace
diff --git a/src/lib/ingest.ts b/src/lib/ingest.ts
index 1bf4fb7a..87fe196b 100644
--- a/src/lib/ingest.ts
+++ b/src/lib/ingest.ts
@@ -475,8 +475,33 @@ async function writeFileBlocks(
const existing = await tryReadFile(fullPath)
const appended = existing ? `${existing}\n\n${content.trim()}` : content.trim()
await writeFile(fullPath, appended)
- } else {
+ } else if (
+ relativePath === "wiki/index.md" ||
+ relativePath.endsWith("/index.md") ||
+ relativePath === "wiki/overview.md" ||
+ relativePath.endsWith("/overview.md")
+ ) {
+ // Listing pages (index / overview) are always overwritten
+ // wholesale — their sources field is incidental and merging
+ // wouldn't make semantic sense (they aren't source-derived
+ // content pages).
await writeFile(fullPath, content)
+ } else {
+ // Content pages (entities / concepts / queries / synthesis /
+ // comparisons / sources summaries): MERGE the sources field
+ // with what's already on disk before overwriting, so pages
+ // that multiple source documents contribute to retain the
+ // full `sources: [...]` history. Without this, every
+ // re-ingest clobbers sources to a single entry and the
+ // source-delete flow would later treat the page as single-
+ // sourced and delete it outright — silent data loss.
+ //
+ // See src/lib/sources-merge.ts for the merge semantics
+ // (case-insensitive dedup, preserves existing order).
+ const { mergeSourcesIntoContent } = await import("./sources-merge")
+ const existing = await tryReadFile(fullPath)
+ const toWrite = mergeSourcesIntoContent(content, existing)
+ await writeFile(fullPath, toWrite)
}
writtenPaths.push(relativePath)
} catch (err) {
diff --git a/src/lib/sources-merge.test.ts b/src/lib/sources-merge.test.ts
new file mode 100644
index 00000000..c9a37cbf
--- /dev/null
+++ b/src/lib/sources-merge.test.ts
@@ -0,0 +1,312 @@
+/**
+ * Regression suite for the sources-merge helpers used during ingest.
+ *
+ * Why this exists: the stage-2 prompt tells the LLM to emit
+ * `sources: ["<current>"]` on every FILE block, but doesn't feed the
+ * existing page's body in. Without this merge step, each re-ingest
+ * would clobber the sources array to a single entry, and the
+ * source-delete flow would later treat the page as single-sourced →
+ * delete it → lose content contributed by the earlier source.
+ *
+ * Every failure mode below, if it regressed, would surface as
+ * silent data loss on the user's next source-delete.
+ */
+import { describe, it, expect } from "vitest"
+import {
+ parseSources,
+ writeSources,
+ mergeSourcesLists,
+ mergeSourcesIntoContent,
+} from "./sources-merge"
+
+const WRAP = (fm: string, body = "body\n") => `---\n${fm}\n---\n${body}`
+
+// ── parseSources ────────────────────────────────────────────────────
+
+describe("parseSources — inline `sources: [...]`", () => {
+ it("extracts a single-entry array", () => {
+ expect(parseSources(WRAP('sources: ["a.md"]'))).toEqual(["a.md"])
+ })
+
+ it("extracts a multi-entry array", () => {
+ expect(parseSources(WRAP('sources: ["a.md", "b.md"]'))).toEqual(["a.md", "b.md"])
+ })
+
+ it("handles single quotes", () => {
+ expect(parseSources(WRAP("sources: ['a.md', 'b.md']"))).toEqual(["a.md", "b.md"])
+ })
+
+ it("handles bare (unquoted) entries", () => {
+ expect(parseSources(WRAP("sources: [a.md, b.md]"))).toEqual(["a.md", "b.md"])
+ })
+
+ it("handles no-space-after-comma", () => {
+ expect(parseSources(WRAP('sources: ["a.md","b.md"]'))).toEqual(["a.md", "b.md"])
+ })
+
+ it("returns empty array for empty brackets", () => {
+ expect(parseSources(WRAP("sources: []"))).toEqual([])
+ })
+
+ it("returns empty array when frontmatter has no sources field", () => {
+ expect(parseSources(WRAP("title: X"))).toEqual([])
+ })
+
+ it("returns empty array when no frontmatter at all", () => {
+ expect(parseSources("# heading\n\nbody")).toEqual([])
+ })
+
+ it("handles CJK-named source files", () => {
+ expect(parseSources(WRAP('sources: ["测试.md", "test.md"]'))).toEqual([
+ "测试.md",
+ "test.md",
+ ])
+ })
+})
+
+describe("parseSources — multi-line YAML list form", () => {
+ it("extracts a multi-line list", () => {
+ const content = WRAP(
+ ["sources:", " - a.md", " - b.md"].join("\n"),
+ )
+ expect(parseSources(content)).toEqual(["a.md", "b.md"])
+ })
+
+ it("handles quoted multi-line entries", () => {
+ const content = WRAP(
+ ["sources:", ' - "a.md"', " - 'b.md'"].join("\n"),
+ )
+ expect(parseSources(content)).toEqual(["a.md", "b.md"])
+ })
+})
+
+// ── writeSources ────────────────────────────────────────────────────
+
+describe("writeSources", () => {
+ it("replaces an existing inline sources array", () => {
+ const before = WRAP('title: X\nsources: ["a.md"]')
+ const after = writeSources(before, ["a.md", "b.md"])
+ expect(parseSources(after)).toEqual(["a.md", "b.md"])
+ // Other frontmatter fields preserved.
+ expect(after).toContain("title: X")
+ })
+
+ it("preserves field order (sources stays in its original position)", () => {
+ const before = WRAP('title: X\nsources: ["a.md"]\ntags: []')
+ const after = writeSources(before, ["a.md", "b.md"])
+ const fmLines = after
+ .match(/^---\n([\s\S]*?)\n---/)![1]
+ .split("\n")
+ .map((l) => l.split(":")[0].trim())
+ expect(fmLines).toEqual(["title", "sources", "tags"])
+ })
+
+ it("inserts a sources field when none exists", () => {
+ const before = WRAP("title: X\ntags: []")
+ const after = writeSources(before, ["a.md"])
+ expect(parseSources(after)).toEqual(["a.md"])
+ expect(after).toContain("title: X")
+ expect(after).toContain("tags: []")
+ })
+
+ it("collapses multi-line YAML list form into inline form", () => {
+ const before = WRAP(
+ ["title: X", "sources:", " - a.md", " - b.md"].join("\n"),
+ )
+ const after = writeSources(before, ["a.md", "b.md", "c.md"])
+ expect(parseSources(after)).toEqual(["a.md", "b.md", "c.md"])
+ expect(after).toContain('sources: ["a.md", "b.md", "c.md"]')
+ // Multi-line artefacts removed.
+ expect(after).not.toMatch(/^\s+-\s+a\.md/m)
+ })
+
+ it("returns content unchanged when there is no frontmatter", () => {
+ const before = "# heading\n\nbody"
+ expect(writeSources(before, ["a.md"])).toBe(before)
+ })
+
+ it("preserves the body of the document", () => {
+ const before = WRAP(
+ 'title: X\nsources: ["a.md"]',
+ "# Title\n\nLots of prose here.\nAnd more prose.",
+ )
+ const after = writeSources(before, ["a.md", "b.md"])
+ expect(after).toContain("# Title")
+ expect(after).toContain("Lots of prose here.")
+ expect(after).toContain("And more prose.")
+ })
+})
+
+// ── mergeSourcesLists ───────────────────────────────────────────────
+
+describe("mergeSourcesLists", () => {
+ it("unions disjoint lists in order", () => {
+ expect(mergeSourcesLists(["a.md"], ["b.md"])).toEqual(["a.md", "b.md"])
+ })
+
+ it("dedupes exact duplicates", () => {
+ expect(mergeSourcesLists(["a.md"], ["a.md"])).toEqual(["a.md"])
+ })
+
+ it("dedupes case-insensitively, keeping the existing casing", () => {
+ expect(mergeSourcesLists(["Test.md"], ["test.md"])).toEqual(["Test.md"])
+ })
+
+ it("preserves order of existing entries", () => {
+ expect(mergeSourcesLists(["b.md", "a.md"], ["c.md"])).toEqual([
+ "b.md",
+ "a.md",
+ "c.md",
+ ])
+ })
+
+ it("is a no-op for empty+empty", () => {
+ expect(mergeSourcesLists([], [])).toEqual([])
+ })
+
+ it("preserves existing when incoming is empty", () => {
+ expect(mergeSourcesLists(["a.md", "b.md"], [])).toEqual(["a.md", "b.md"])
+ })
+
+ it("preserves incoming when existing is empty", () => {
+ expect(mergeSourcesLists([], ["a.md", "b.md"])).toEqual(["a.md", "b.md"])
+ })
+})
+
+// ── mergeSourcesIntoContent — the real ingest entry point ──────────
+
+describe("mergeSourcesIntoContent — happy paths", () => {
+ it("returns newContent unchanged when page is new (no existingContent)", () => {
+ const newContent = WRAP('sources: ["test2.md"]')
+ expect(mergeSourcesIntoContent(newContent, null)).toBe(newContent)
+ })
+
+ it("merges sources when the page already exists with a different source", () => {
+ const existing = WRAP('sources: ["test1.md"]', "old body")
+ const incoming = WRAP('sources: ["test2.md"]', "fresh body from re-ingest")
+ const merged = mergeSourcesIntoContent(incoming, existing)
+ expect(parseSources(merged)).toEqual(["test1.md", "test2.md"])
+ // Body is the NEW body — this function only rewrites the sources
+ // field, it doesn't preserve old content.
+ expect(merged).toContain("fresh body from re-ingest")
+ expect(merged).not.toContain("old body")
+ })
+
+ it("idempotent: merging the same current source is a no-op on sources", () => {
+ const existing = WRAP('sources: ["test1.md", "test2.md"]')
+ const incoming = WRAP('sources: ["test1.md"]')
+ const merged = mergeSourcesIntoContent(incoming, existing)
+ expect(parseSources(merged)).toEqual(["test1.md", "test2.md"])
+ })
+
+ it("short-circuits (returns newContent verbatim) when old is a subset of new", () => {
+ // Covers the no-op optimization branch: when the merged list
+ // equals newSources element-for-element, the function should hand
+ // back the unchanged newContent string (same reference, no
+ // rewrite). Exercises the `merged === newSources` fast path.
+ const existing = WRAP('sources: ["a.md"]')
+ const incoming = WRAP('sources: ["a.md"]', "body content")
+ const merged = mergeSourcesIntoContent(incoming, existing)
+ // Same reference — no writeSources call happened.
+ expect(merged).toBe(incoming)
+ })
+
+ it("handles case-insensitive source filenames (keeps original casing)", () => {
+ const existing = WRAP('sources: ["Test1.md"]')
+ const incoming = WRAP('sources: ["test1.md"]')
+ const merged = mergeSourcesIntoContent(incoming, existing)
+ // Should be a single entry with the existing (original) casing,
+ // not a duplicate.
+ expect(parseSources(merged)).toEqual(["Test1.md"])
+ })
+
+ it("appends the new source to an already multi-source page", () => {
+ const existing = WRAP('sources: ["a.md", "b.md"]')
+ const incoming = WRAP('sources: ["c.md"]')
+ const merged = mergeSourcesIntoContent(incoming, existing)
+ expect(parseSources(merged)).toEqual(["a.md", "b.md", "c.md"])
+ })
+})
+
+describe("mergeSourcesIntoContent — degenerate inputs", () => {
+ it("is a no-op when existingContent has no sources field at all", () => {
+ // Pre-0.3.x pages without sources field should not spontaneously
+ // acquire phantom sources from the existing file.
+ const existing = WRAP("title: X")
+ const incoming = WRAP('sources: ["test2.md"]')
+ expect(mergeSourcesIntoContent(incoming, existing)).toBe(incoming)
+ })
+
+ it("is a no-op when existingContent is empty string", () => {
+ const incoming = WRAP('sources: ["test2.md"]')
+ expect(mergeSourcesIntoContent(incoming, "")).toBe(incoming)
+ })
+
+ it("tolerates newContent missing a sources field (LLM slipup)", () => {
+ // If the LLM forgot to emit sources in the new content but the
+ // existing page has one, we should NOT drop the existing sources.
+ const existing = WRAP('sources: ["test1.md"]')
+ const incomingNoSources = WRAP("title: X")
+ const merged = mergeSourcesIntoContent(incomingNoSources, existing)
+ expect(parseSources(merged)).toEqual(["test1.md"])
+ })
+
+ it("tolerates existingContent with no frontmatter at all", () => {
+ const existing = "# legacy non-frontmatter page"
+ const incoming = WRAP('sources: ["test2.md"]')
+ expect(mergeSourcesIntoContent(incoming, existing)).toBe(incoming)
+ })
+})
+
+// ── The end-to-end scenario that motivated this module ────────────
+
+describe("Regression: the data-loss path in the user's diagnosis", () => {
+ it("two ingests against the same page result in union, not last-wins", () => {
+ // Step 1 of user's failure scenario: ingest test1.md creates
+ // attention.md with sources=[test1.md].
+ const afterFirstIngest = mergeSourcesIntoContent(
+ WRAP('title: Attention\nsources: ["test1.md"]', "content from test1"),
+ null,
+ )
+ expect(parseSources(afterFirstIngest)).toEqual(["test1.md"])
+
+ // Step 2: ingest test2.md, which also covers attention. LLM emits
+ // `sources: ["test2.md"]` per the prompt template, but we merge
+ // before writing — so the on-disk sources field becomes BOTH.
+ const afterSecondIngest = mergeSourcesIntoContent(
+ WRAP('title: Attention\nsources: ["test2.md"]', "content from test2"),
+ afterFirstIngest,
+ )
+ expect(parseSources(afterSecondIngest)).toEqual(["test1.md", "test2.md"])
+
+ // Step 3: user later deletes test2.md. sources-view's deletion
+ // flow reads sources = ["test1.md", "test2.md"] (length 2),
+ // recognises the page is shared, and keeps it on disk — no data
+ // loss. Before this fix, step 2 would have clobbered sources to
+ // just ["test2.md"], and step 3 would have deleted the page.
+ const remainingSourcesAfterDeletingTest2 = parseSources(afterSecondIngest)
+ .filter((s) => s.toLowerCase() !== "test2.md")
+ expect(remainingSourcesAfterDeletingTest2).toEqual(["test1.md"])
+ })
+
+ it("three-way re-ingest: each ingest contributes its own source to the union", () => {
+ let page = mergeSourcesIntoContent(
+ WRAP('sources: ["a.md"]', "body v1"),
+ null,
+ )
+ page = mergeSourcesIntoContent(
+ WRAP('sources: ["b.md"]', "body v2"),
+ page,
+ )
+ page = mergeSourcesIntoContent(
+ WRAP('sources: ["c.md"]', "body v3"),
+ page,
+ )
+ expect(parseSources(page)).toEqual(["a.md", "b.md", "c.md"])
+ // Body is always the most recent emission — we don't diff-merge
+ // the body, only the sources list.
+ expect(page).toContain("body v3")
+ expect(page).not.toContain("body v1")
+ expect(page).not.toContain("body v2")
+ })
+})
diff --git a/src/lib/sources-merge.ts b/src/lib/sources-merge.ts
new file mode 100644
index 00000000..e8b6b86c
--- /dev/null
+++ b/src/lib/sources-merge.ts
@@ -0,0 +1,148 @@
+/**
+ * Merge a page's YAML frontmatter `sources:` array with what the LLM
+ * just emitted, so re-ingesting a page that already has a history from
+ * another source doesn't silently clobber that history.
+ *
+ * Why this exists: the stage-2 prompt instructs the LLM to emit
+ * `sources: ["${sourceFileName}"]` — with JUST the current source — on
+ * every FILE block. The stage-2 prompt also doesn't feed existing page
+ * bodies into the context, so the LLM can't see the old sources. If
+ * the ingest write were naive, each re-ingest would overwrite the
+ * sources array with a single-element list, and the downstream
+ * source-delete logic would later treat the page as single-sourced and
+ * delete it — losing content contributed by the earlier source.
+ *
+ * The fix: before writing, read the existing file (if any), parse its
+ * sources, union with the freshly emitted sources, rewrite the frontmatter.
+ */
+
+/**
+ * Extract `sources: [...]` from the YAML frontmatter of a wiki page.
+ * Returns `[]` when no sources line is found or parsing fails.
+ *
+ * Handles both single-line form (`sources: ["a.md", "b.md"]`) and the
+ * multi-line YAML list form (`sources:\n - a.md\n - b.md`). Single
+ * and double quotes on items are stripped; bare items are accepted.
+ */
+export function parseSources(content: string): string[] {
+ const fmMatch = content.match(/^---\n([\s\S]*?)\n---/)
+ const fm = fmMatch ? fmMatch[1] : content
+
+ // Multi-line YAML list form: `sources:\n - "a.md"\n - "b.md"`
+ const multi = fm.match(/^sources:\s*\n((?:[ \t]+-\s+.+\n?)+)/m)
+ if (multi) {
+ const out: string[] = []
+ for (const line of multi[1].split("\n")) {
+ const m = line.match(/^\s+-\s+["']?(.+?)["']?\s*$/)
+ if (m && m[1]) out.push(m[1].trim())
+ }
+ return out
+ }
+
+ // Inline form: `sources: ["a.md", "b.md"]` or `sources: []`.
+ const inline = fm.match(/^sources:\s*\[([^\]]*)\]/m)
+ if (!inline) return []
+ const body = inline[1].trim()
+ if (body === "") return []
+ return body
+ .split(",")
+ .map((s) => s.trim().replace(/^["']|["']$/g, ""))
+ .filter((s) => s.length > 0)
+}
+
+/**
+ * Rewrite the `sources:` field of a markdown page's frontmatter to the
+ * provided array. Preserves every other frontmatter line. If no
+ * `sources:` line exists (LLM forgot it), one is inserted just before
+ * the closing `---`. If no frontmatter exists at all, returns the
+ * content unchanged — we don't manufacture frontmatter for pages the
+ * LLM didn't frontmatter-prefix, since that almost certainly means
+ * the emission was already malformed and the caller should surface it.
+ */
+export function writeSources(content: string, sources: string[]): string {
+ const fmMatch = content.match(/^(---\n)([\s\S]*?)(\n---)/)
+ if (!fmMatch) return content
+
+ const [, openDelim, fmBody, closeDelim] = fmMatch
+ const serialized = sources.map((s) => `"${s}"`).join(", ")
+ const newLine = `sources: [${serialized}]`
+
+ // Prefer replacing an existing inline `sources:` line in-place so
+ // field ordering within the frontmatter stays the same as the LLM
+ // emitted it — users don't see fields shuffle around on every
+ // re-ingest.
+ if (/^sources:\s*\[[^\]]*\]/m.test(fmBody)) {
+ const rewritten = fmBody.replace(/^sources:\s*\[[^\]]*\]/m, newLine)
+ return `${openDelim}${rewritten}${closeDelim}${content.slice(fmMatch[0].length)}`
+ }
+
+ // Replace a multi-line YAML list form with an inline form too —
+ // consistent shape across pages makes downstream parsing simpler.
+ if (/^sources:\s*\n((?:[ \t]+-\s+.+\n?)+)/m.test(fmBody)) {
+ const rewritten = fmBody.replace(
+ /^sources:\s*\n((?:[ \t]+-\s+.+\n?)+)/m,
+ newLine,
+ )
+ return `${openDelim}${rewritten}${closeDelim}${content.slice(fmMatch[0].length)}`
+ }
+
+ // No sources field at all — append one at the end of the frontmatter.
+ const rewritten = `${fmBody}\n${newLine}`
+ return `${openDelim}${rewritten}${closeDelim}${content.slice(fmMatch[0].length)}`
+}
+
+/**
+ * Merge two source lists, case-insensitively deduped. Order of existing
+ * entries is preserved; new entries not already present are appended
+ * in the order they appear in `incoming`.
+ *
+ * Case handling: if both lists contain the same name but with different
+ * casing (e.g. "Test.md" and "test.md"), the first-seen form wins.
+ * This keeps the user's original filename casing stable on disk.
+ */
+export function mergeSourcesLists(
+ existing: readonly string[],
+ incoming: readonly string[],
+): string[] {
+ const seen = new Set<string>()
+ const out: string[] = []
+ for (const s of [...existing, ...incoming]) {
+ const key = s.toLowerCase()
+ if (seen.has(key)) continue
+ seen.add(key)
+ out.push(s)
+ }
+ return out
+}
+
+/**
+ * The main entry point used from ingest: given the content the LLM
+ * just emitted for a page (`newContent`), and whatever is currently on
+ * disk at that path (`existingContent`, or null if the page is new),
+ * return content whose `sources:` field is the union of both.
+ *
+ * For new pages: returns newContent unchanged.
+ * For existing pages with no frontmatter: returns newContent unchanged
+ * (don't corrupt unconventional files).
+ * For existing pages with frontmatter: merges sources, rewrites.
+ */
+export function mergeSourcesIntoContent(
+ newContent: string,
+ existingContent: string | null,
+): string {
+ if (!existingContent) return newContent
+ const oldSources = parseSources(existingContent)
+ if (oldSources.length === 0) return newContent
+ const newSources = parseSources(newContent)
+ const merged = mergeSourcesLists(oldSources, newSources)
+ // Avoid writing a no-op change: if nothing actually needs merging,
+ // hand back the original newContent verbatim so hashes / caches stay
+ // stable.
+ if (
+ merged.length === newSources.length &&
+ merged.every((s, i) => s === newSources[i])
+ ) {
+ return newContent
+ }
+ return writeSources(newContent, merged)
+}