LLM Wiki is a cross-platform desktop application that turns your documents into an organized, interlinked knowledge base — automatically. Instead of traditional RAG (retrieve-and-answer from scratch every time), the LLM incrementally builds and maintains a persistent wiki from your sources。
TypeScript
feat: merge sources[] on ingest write to prevent shared-page data loss
nash_su21eed06b0ace
diff --git a/src/lib/ingest.ts b/src/lib/ingest.tsindex 1bf4fb7a..87fe196b 100644--- a/src/lib/ingest.ts+++ b/src/lib/ingest.ts@@ -475,8 +475,33 @@ async function writeFileBlocks(const existing = await tryReadFile(fullPath)const appended = existing ? `${existing}\n\n${content.trim()}` : content.trim()await writeFile(fullPath, appended)- } else {+ } else if (+ relativePath === "wiki/index.md" ||+ relativePath.endsWith("/index.md") ||+ relativePath === "wiki/overview.md" ||+ relativePath.endsWith("/overview.md")+ ) {+ // Listing pages (index / overview) are always overwritten+ // wholesale — their sources field is incidental and merging+ // wouldn't make semantic sense (they aren't source-derived+ // content pages).await writeFile(fullPath, content)+ } else {+ // Content pages (entities / concepts / queries / synthesis /+ // comparisons / sources summaries): MERGE the sources field+ // with what's already on disk before overwriting, so pages+ // that multiple source documents contribute to retain the+ // full `sources: [...]` history. Without this, every+ // re-ingest clobbers sources to a single entry and the+ // source-delete flow would later treat the page as single-+ // sourced and delete it outright — silent data loss.+ //+ // See src/lib/sources-merge.ts for the merge semantics+ // (case-insensitive dedup, preserves existing order).+ const { mergeSourcesIntoContent } = await import("./sources-merge")+ const existing = await tryReadFile(fullPath)+ const toWrite = mergeSourcesIntoContent(content, existing)+ await writeFile(fullPath, toWrite)}writtenPaths.push(relativePath)} catch (err) {diff --git a/src/lib/sources-merge.test.ts b/src/lib/sources-merge.test.tsnew file mode 100644index 00000000..c9a37cbf--- /dev/null+++ b/src/lib/sources-merge.test.ts@@ -0,0 +1,312 @@+/**+ * Regression suite for the sources-merge helpers used during ingest.+ *+ * Why this exists: the stage-2 prompt tells the LLM to emit+ * `sources: ["<current>"]` on every FILE block, but doesn't feed the+ * existing page's body in. Without this merge step, each re-ingest+ * would clobber the sources array to a single entry, and the+ * source-delete flow would later treat the page as single-sourced →+ * delete it → lose content contributed by the earlier source.+ *+ * Every failure mode below, if it regressed, would surface as+ * silent data loss on the user's next source-delete.+ */+import { describe, it, expect } from "vitest"+import {+ parseSources,+ writeSources,+ mergeSourcesLists,+ mergeSourcesIntoContent,+} from "./sources-merge"++const WRAP = (fm: string, body = "body\n") => `---\n${fm}\n---\n${body}`++// ── parseSources ────────────────────────────────────────────────────++describe("parseSources — inline `sources: [...]`", () => {+ it("extracts a single-entry array", () => {+ expect(parseSources(WRAP('sources: ["a.md"]'))).toEqual(["a.md"])+ })++ it("extracts a multi-entry array", () => {+ expect(parseSources(WRAP('sources: ["a.md", "b.md"]'))).toEqual(["a.md", "b.md"])+ })++ it("handles single quotes", () => {+ expect(parseSources(WRAP("sources: ['a.md', 'b.md']"))).toEqual(["a.md", "b.md"])+ })++ it("handles bare (unquoted) entries", () => {+ expect(parseSources(WRAP("sources: [a.md, b.md]"))).toEqual(["a.md", "b.md"])+ })++ it("handles no-space-after-comma", () => {+ expect(parseSources(WRAP('sources: ["a.md","b.md"]'))).toEqual(["a.md", "b.md"])+ })++ it("returns empty array for empty brackets", () => {+ expect(parseSources(WRAP("sources: []"))).toEqual([])+ })++ it("returns empty array when frontmatter has no sources field", () => {+ expect(parseSources(WRAP("title: X"))).toEqual([])+ })++ it("returns empty array when no frontmatter at all", () => {+ expect(parseSources("# heading\n\nbody")).toEqual([])+ })++ it("handles CJK-named source files", () => {+ expect(parseSources(WRAP('sources: ["测试.md", "test.md"]'))).toEqual([+ "测试.md",+ "test.md",+ ])+ })+})++describe("parseSources — multi-line YAML list form", () => {+ it("extracts a multi-line list", () => {+ const content = WRAP(+ ["sources:", " - a.md", " - b.md"].join("\n"),+ )+ expect(parseSources(content)).toEqual(["a.md", "b.md"])+ })++ it("handles quoted multi-line entries", () => {+ const content = WRAP(+ ["sources:", ' - "a.md"', " - 'b.md'"].join("\n"),+ )+ expect(parseSources(content)).toEqual(["a.md", "b.md"])+ })+})++// ── writeSources ────────────────────────────────────────────────────++describe("writeSources", () => {+ it("replaces an existing inline sources array", () => {+ const before = WRAP('title: X\nsources: ["a.md"]')+ const after = writeSources(before, ["a.md", "b.md"])+ expect(parseSources(after)).toEqual(["a.md", "b.md"])+ // Other frontmatter fields preserved.+ expect(after).toContain("title: X")+ })++ it("preserves field order (sources stays in its original position)", () => {+ const before = WRAP('title: X\nsources: ["a.md"]\ntags: []')+ const after = writeSources(before, ["a.md", "b.md"])+ const fmLines = after+ .match(/^---\n([\s\S]*?)\n---/)![1]+ .split("\n")+ .map((l) => l.split(":")[0].trim())+ expect(fmLines).toEqual(["title", "sources", "tags"])+ })++ it("inserts a sources field when none exists", () => {+ const before = WRAP("title: X\ntags: []")+ const after = writeSources(before, ["a.md"])+ expect(parseSources(after)).toEqual(["a.md"])+ expect(after).toContain("title: X")+ expect(after).toContain("tags: []")+ })++ it("collapses multi-line YAML list form into inline form", () => {+ const before = WRAP(+ ["title: X", "sources:", " - a.md", " - b.md"].join("\n"),+ )+ const after = writeSources(before, ["a.md", "b.md", "c.md"])+ expect(parseSources(after)).toEqual(["a.md", "b.md", "c.md"])+ expect(after).toContain('sources: ["a.md", "b.md", "c.md"]')+ // Multi-line artefacts removed.+ expect(after).not.toMatch(/^\s+-\s+a\.md/m)+ })++ it("returns content unchanged when there is no frontmatter", () => {+ const before = "# heading\n\nbody"+ expect(writeSources(before, ["a.md"])).toBe(before)+ })++ it("preserves the body of the document", () => {+ const before = WRAP(+ 'title: X\nsources: ["a.md"]',+ "# Title\n\nLots of prose here.\nAnd more prose.",+ )+ const after = writeSources(before, ["a.md", "b.md"])+ expect(after).toContain("# Title")+ expect(after).toContain("Lots of prose here.")+ expect(after).toContain("And more prose.")+ })+})++// ── mergeSourcesLists ───────────────────────────────────────────────++describe("mergeSourcesLists", () => {+ it("unions disjoint lists in order", () => {+ expect(mergeSourcesLists(["a.md"], ["b.md"])).toEqual(["a.md", "b.md"])+ })++ it("dedupes exact duplicates", () => {+ expect(mergeSourcesLists(["a.md"], ["a.md"])).toEqual(["a.md"])+ })++ it("dedupes case-insensitively, keeping the existing casing", () => {+ expect(mergeSourcesLists(["Test.md"], ["test.md"])).toEqual(["Test.md"])+ })++ it("preserves order of existing entries", () => {+ expect(mergeSourcesLists(["b.md", "a.md"], ["c.md"])).toEqual([+ "b.md",+ "a.md",+ "c.md",+ ])+ })++ it("is a no-op for empty+empty", () => {+ expect(mergeSourcesLists([], [])).toEqual([])+ })++ it("preserves existing when incoming is empty", () => {+ expect(mergeSourcesLists(["a.md", "b.md"], [])).toEqual(["a.md", "b.md"])+ })++ it("preserves incoming when existing is empty", () => {+ expect(mergeSourcesLists([], ["a.md", "b.md"])).toEqual(["a.md", "b.md"])+ })+})++// ── mergeSourcesIntoContent — the real ingest entry point ──────────++describe("mergeSourcesIntoContent — happy paths", () => {+ it("returns newContent unchanged when page is new (no existingContent)", () => {+ const newContent = WRAP('sources: ["test2.md"]')+ expect(mergeSourcesIntoContent(newContent, null)).toBe(newContent)+ })++ it("merges sources when the page already exists with a different source", () => {+ const existing = WRAP('sources: ["test1.md"]', "old body")+ const incoming = WRAP('sources: ["test2.md"]', "fresh body from re-ingest")+ const merged = mergeSourcesIntoContent(incoming, existing)+ expect(parseSources(merged)).toEqual(["test1.md", "test2.md"])+ // Body is the NEW body — this function only rewrites the sources+ // field, it doesn't preserve old content.+ expect(merged).toContain("fresh body from re-ingest")+ expect(merged).not.toContain("old body")+ })++ it("idempotent: merging the same current source is a no-op on sources", () => {+ const existing = WRAP('sources: ["test1.md", "test2.md"]')+ const incoming = WRAP('sources: ["test1.md"]')+ const merged = mergeSourcesIntoContent(incoming, existing)+ expect(parseSources(merged)).toEqual(["test1.md", "test2.md"])+ })++ it("short-circuits (returns newContent verbatim) when old is a subset of new", () => {+ // Covers the no-op optimization branch: when the merged list+ // equals newSources element-for-element, the function should hand+ // back the unchanged newContent string (same reference, no+ // rewrite). Exercises the `merged === newSources` fast path.+ const existing = WRAP('sources: ["a.md"]')+ const incoming = WRAP('sources: ["a.md"]', "body content")+ const merged = mergeSourcesIntoContent(incoming, existing)+ // Same reference — no writeSources call happened.+ expect(merged).toBe(incoming)+ })++ it("handles case-insensitive source filenames (keeps original casing)", () => {+ const existing = WRAP('sources: ["Test1.md"]')+ const incoming = WRAP('sources: ["test1.md"]')+ const merged = mergeSourcesIntoContent(incoming, existing)+ // Should be a single entry with the existing (original) casing,+ // not a duplicate.+ expect(parseSources(merged)).toEqual(["Test1.md"])+ })++ it("appends the new source to an already multi-source page", () => {+ const existing = WRAP('sources: ["a.md", "b.md"]')+ const incoming = WRAP('sources: ["c.md"]')+ const merged = mergeSourcesIntoContent(incoming, existing)+ expect(parseSources(merged)).toEqual(["a.md", "b.md", "c.md"])+ })+})++describe("mergeSourcesIntoContent — degenerate inputs", () => {+ it("is a no-op when existingContent has no sources field at all", () => {+ // Pre-0.3.x pages without sources field should not spontaneously+ // acquire phantom sources from the existing file.+ const existing = WRAP("title: X")+ const incoming = WRAP('sources: ["test2.md"]')+ expect(mergeSourcesIntoContent(incoming, existing)).toBe(incoming)+ })++ it("is a no-op when existingContent is empty string", () => {+ const incoming = WRAP('sources: ["test2.md"]')+ expect(mergeSourcesIntoContent(incoming, "")).toBe(incoming)+ })++ it("tolerates newContent missing a sources field (LLM slipup)", () => {+ // If the LLM forgot to emit sources in the new content but the+ // existing page has one, we should NOT drop the existing sources.+ const existing = WRAP('sources: ["test1.md"]')+ const incomingNoSources = WRAP("title: X")+ const merged = mergeSourcesIntoContent(incomingNoSources, existing)+ expect(parseSources(merged)).toEqual(["test1.md"])+ })++ it("tolerates existingContent with no frontmatter at all", () => {+ const existing = "# legacy non-frontmatter page"+ const incoming = WRAP('sources: ["test2.md"]')+ expect(mergeSourcesIntoContent(incoming, existing)).toBe(incoming)+ })+})++// ── The end-to-end scenario that motivated this module ────────────++describe("Regression: the data-loss path in the user's diagnosis", () => {+ it("two ingests against the same page result in union, not last-wins", () => {+ // Step 1 of user's failure scenario: ingest test1.md creates+ // attention.md with sources=[test1.md].+ const afterFirstIngest = mergeSourcesIntoContent(+ WRAP('title: Attention\nsources: ["test1.md"]', "content from test1"),+ null,+ )+ expect(parseSources(afterFirstIngest)).toEqual(["test1.md"])++ // Step 2: ingest test2.md, which also covers attention. LLM emits+ // `sources: ["test2.md"]` per the prompt template, but we merge+ // before writing — so the on-disk sources field becomes BOTH.+ const afterSecondIngest = mergeSourcesIntoContent(+ WRAP('title: Attention\nsources: ["test2.md"]', "content from test2"),+ afterFirstIngest,+ )+ expect(parseSources(afterSecondIngest)).toEqual(["test1.md", "test2.md"])++ // Step 3: user later deletes test2.md. sources-view's deletion+ // flow reads sources = ["test1.md", "test2.md"] (length 2),+ // recognises the page is shared, and keeps it on disk — no data+ // loss. Before this fix, step 2 would have clobbered sources to+ // just ["test2.md"], and step 3 would have deleted the page.+ const remainingSourcesAfterDeletingTest2 = parseSources(afterSecondIngest)+ .filter((s) => s.toLowerCase() !== "test2.md")+ expect(remainingSourcesAfterDeletingTest2).toEqual(["test1.md"])+ })++ it("three-way re-ingest: each ingest contributes its own source to the union", () => {+ let page = mergeSourcesIntoContent(+ WRAP('sources: ["a.md"]', "body v1"),+ null,+ )+ page = mergeSourcesIntoContent(+ WRAP('sources: ["b.md"]', "body v2"),+ page,+ )+ page = mergeSourcesIntoContent(+ WRAP('sources: ["c.md"]', "body v3"),+ page,+ )+ expect(parseSources(page)).toEqual(["a.md", "b.md", "c.md"])+ // Body is always the most recent emission — we don't diff-merge+ // the body, only the sources list.+ expect(page).toContain("body v3")+ expect(page).not.toContain("body v1")+ expect(page).not.toContain("body v2")+ })+})diff --git a/src/lib/sources-merge.ts b/src/lib/sources-merge.tsnew file mode 100644index 00000000..e8b6b86c--- /dev/null+++ b/src/lib/sources-merge.ts@@ -0,0 +1,148 @@+/**+ * Merge a page's YAML frontmatter `sources:` array with what the LLM+ * just emitted, so re-ingesting a page that already has a history from+ * another source doesn't silently clobber that history.+ *+ * Why this exists: the stage-2 prompt instructs the LLM to emit+ * `sources: ["${sourceFileName}"]` — with JUST the current source — on+ * every FILE block. The stage-2 prompt also doesn't feed existing page+ * bodies into the context, so the LLM can't see the old sources. If+ * the ingest write were naive, each re-ingest would overwrite the+ * sources array with a single-element list, and the downstream+ * source-delete logic would later treat the page as single-sourced and+ * delete it — losing content contributed by the earlier source.+ *+ * The fix: before writing, read the existing file (if any), parse its+ * sources, union with the freshly emitted sources, rewrite the frontmatter.+ */++/**+ * Extract `sources: [...]` from the YAML frontmatter of a wiki page.+ * Returns `[]` when no sources line is found or parsing fails.+ *+ * Handles both single-line form (`sources: ["a.md", "b.md"]`) and the+ * multi-line YAML list form (`sources:\n - a.md\n - b.md`). Single+ * and double quotes on items are stripped; bare items are accepted.+ */+export function parseSources(content: string): string[] {+ const fmMatch = content.match(/^---\n([\s\S]*?)\n---/)+ const fm = fmMatch ? fmMatch[1] : content++ // Multi-line YAML list form: `sources:\n - "a.md"\n - "b.md"`+ const multi = fm.match(/^sources:\s*\n((?:[ \t]+-\s+.+\n?)+)/m)+ if (multi) {+ const out: string[] = []+ for (const line of multi[1].split("\n")) {+ const m = line.match(/^\s+-\s+["']?(.+?)["']?\s*$/)+ if (m && m[1]) out.push(m[1].trim())+ }+ return out+ }++ // Inline form: `sources: ["a.md", "b.md"]` or `sources: []`.+ const inline = fm.match(/^sources:\s*\[([^\]]*)\]/m)+ if (!inline) return []+ const body = inline[1].trim()+ if (body === "") return []+ return body+ .split(",")+ .map((s) => s.trim().replace(/^["']|["']$/g, ""))+ .filter((s) => s.length > 0)+}++/**+ * Rewrite the `sources:` field of a markdown page's frontmatter to the+ * provided array. Preserves every other frontmatter line. If no+ * `sources:` line exists (LLM forgot it), one is inserted just before+ * the closing `---`. If no frontmatter exists at all, returns the+ * content unchanged — we don't manufacture frontmatter for pages the+ * LLM didn't frontmatter-prefix, since that almost certainly means+ * the emission was already malformed and the caller should surface it.+ */+export function writeSources(content: string, sources: string[]): string {+ const fmMatch = content.match(/^(---\n)([\s\S]*?)(\n---)/)+ if (!fmMatch) return content++ const [, openDelim, fmBody, closeDelim] = fmMatch+ const serialized = sources.map((s) => `"${s}"`).join(", ")+ const newLine = `sources: [${serialized}]`++ // Prefer replacing an existing inline `sources:` line in-place so+ // field ordering within the frontmatter stays the same as the LLM+ // emitted it — users don't see fields shuffle around on every+ // re-ingest.+ if (/^sources:\s*\[[^\]]*\]/m.test(fmBody)) {+ const rewritten = fmBody.replace(/^sources:\s*\[[^\]]*\]/m, newLine)+ return `${openDelim}${rewritten}${closeDelim}${content.slice(fmMatch[0].length)}`+ }++ // Replace a multi-line YAML list form with an inline form too —+ // consistent shape across pages makes downstream parsing simpler.+ if (/^sources:\s*\n((?:[ \t]+-\s+.+\n?)+)/m.test(fmBody)) {+ const rewritten = fmBody.replace(+ /^sources:\s*\n((?:[ \t]+-\s+.+\n?)+)/m,+ newLine,+ )+ return `${openDelim}${rewritten}${closeDelim}${content.slice(fmMatch[0].length)}`+ }++ // No sources field at all — append one at the end of the frontmatter.+ const rewritten = `${fmBody}\n${newLine}`+ return `${openDelim}${rewritten}${closeDelim}${content.slice(fmMatch[0].length)}`+}++/**+ * Merge two source lists, case-insensitively deduped. Order of existing+ * entries is preserved; new entries not already present are appended+ * in the order they appear in `incoming`.+ *+ * Case handling: if both lists contain the same name but with different+ * casing (e.g. "Test.md" and "test.md"), the first-seen form wins.+ * This keeps the user's original filename casing stable on disk.+ */+export function mergeSourcesLists(+ existing: readonly string[],+ incoming: readonly string[],+): string[] {+ const seen = new Set<string>()+ const out: string[] = []+ for (const s of [...existing, ...incoming]) {+ const key = s.toLowerCase()+ if (seen.has(key)) continue+ seen.add(key)+ out.push(s)+ }+ return out+}++/**+ * The main entry point used from ingest: given the content the LLM+ * just emitted for a page (`newContent`), and whatever is currently on+ * disk at that path (`existingContent`, or null if the page is new),+ * return content whose `sources:` field is the union of both.+ *+ * For new pages: returns newContent unchanged.+ * For existing pages with no frontmatter: returns newContent unchanged+ * (don't corrupt unconventional files).+ * For existing pages with frontmatter: merges sources, rewrites.+ */+export function mergeSourcesIntoContent(+ newContent: string,+ existingContent: string | null,+): string {+ if (!existingContent) return newContent+ const oldSources = parseSources(existingContent)+ if (oldSources.length === 0) return newContent+ const newSources = parseSources(newContent)+ const merged = mergeSourcesLists(oldSources, newSources)+ // Avoid writing a no-op change: if nothing actually needs merging,+ // hand back the original newContent verbatim so hashes / caches stay+ // stable.+ if (+ merged.length === newSources.length &&+ merged.every((s, i) => s === newSources[i])+ ) {+ return newContent+ }+ return writeSources(newContent, merged)+}