Skip to content

LLM Wiki is a cross-platform desktop application that turns your documents into an organized, interlinked knowledge base — automatically. Instead of traditional RAG (retrieve-and-answer from scratch every time), the LLM incrementally builds and maintains a persistent wiki from your sources。

TypeScript

fix: normalize malformed generated wikilinks

nash_succ7573d1f9ad
diff --git a/src/lib/deep-research.test.ts b/src/lib/deep-research.test.ts
index ce261dbf..8463e29c 100644
--- a/src/lib/deep-research.test.ts
+++ b/src/lib/deep-research.test.ts
@@ -80,6 +80,16 @@ describe("buildResearchPageContent", () => {
expect(content).toContain("![[media/chart.png]]")
})
+ it("normalizes malformed citation wikilinks before saving", () => {
+ const content = buildResearchPageContent(
+ "Topic",
+ "2026-09-27",
+ "Evidence [[1]] and combined [[2], [3], [4]].",
+ "1. [Source](https://example.test) — web",
+ )
+ expect(content).toContain("Evidence [1] and combined [2], [3], [4].")
+ })
+
it("keeps multiline topics inside one safe YAML scalar and heading", () => {
const content = buildResearchPageContent(
"first line\nsecond: \"quoted\"",
diff --git a/src/lib/deep-research.ts b/src/lib/deep-research.ts
index 5abf0230..9bc0722d 100644
--- a/src/lib/deep-research.ts
+++ b/src/lib/deep-research.ts
@@ -11,6 +11,7 @@ import { makeQueryFileName } from "@/lib/wiki-filename"
import { refreshProjectFileTree } from "@/lib/project-file-tree-refresh"
import { useReviewStore } from "@/stores/review-store"
import { stripBodyWikilinkPathPrefixes } from "./page-merge"
+import { normalizeMalformedWikilinks } from "./ingest-sanitize"
const MAX_RESEARCH_SOURCES = 20
const MIN_RESEARCH_CONTENT_CHARS = 120
@@ -30,7 +31,7 @@ export function buildResearchPageContent(
references: string,
): string {
const displayTopic = topic.replace(/\s+/g, " ").trim()
- return stripBodyWikilinkPathPrefixes([
+ return normalizeMalformedWikilinks(stripBodyWikilinkPathPrefixes([
"---",
"type: query",
`title: ${JSON.stringify(`Research: ${displayTopic}`)}`,
@@ -47,7 +48,7 @@ export function buildResearchPageContent(
"",
references,
"",
- ].join("\n"))
+ ].join("\n")))
}
export async function makeAvailableResearchFilePath(
diff --git a/src/lib/ingest-sanitize.test.ts b/src/lib/ingest-sanitize.test.ts
index 57b80b4d..4fac910b 100644
--- a/src/lib/ingest-sanitize.test.ts
+++ b/src/lib/ingest-sanitize.test.ts
@@ -1,5 +1,8 @@
import { describe, it, expect } from "vitest"
-import { sanitizeIngestedFileContent } from "./ingest-sanitize"
+import {
+ normalizeMalformedWikilinks,
+ sanitizeIngestedFileContent,
+} from "./ingest-sanitize"
describe("sanitizeIngestedFileContent", () => {
it("returns clean content unchanged", () => {
@@ -123,3 +126,114 @@ describe("sanitizeIngestedFileContent", () => {
)
})
})
+
+describe("normalizeMalformedWikilinks", () => {
+ it("converts numeric wikilinks and wrapped citation sequences to citations", () => {
+ const input = "One [[1]], several [[12], [14], [15]], and page [[2026]]."
+ expect(normalizeMalformedWikilinks(input)).toBe(
+ "One [1], several [12], [14], [15], and page [[2026]].",
+ )
+ })
+
+ it("repairs an aliased wikilink with one missing closing bracket", () => {
+ expect(normalizeMalformedWikilinks("See **[[target|label]** now."))
+ .toBe("See **[[target|label]]** now.")
+ })
+
+ it("preserves frontmatter, code, embeds, and Markdown links", () => {
+ const input = [
+ "---",
+ "related: ['[[4]]']",
+ "---",
+ "`[[5]]` and ![[asset|preview]",
+ "[label](https://example.test) with [[target|alias]",
+ "```md",
+ "[[6]] [[code|sample]",
+ "```",
+ ].join("\n")
+ expect(normalizeMalformedWikilinks(input)).toBe(input)
+ })
+
+ it.each([
+ "[[1]](https://example.test)",
+ "![[1]]",
+ "[[2024]]",
+ "[[a|b]]",
+ "[[a]",
+ "[[a|see [1]]",
+ "[[a|b]]]",
+ ])("does not rewrite valid or ambiguous syntax: %s", (input) => {
+ expect(normalizeMalformedWikilinks(input)).toBe(input)
+ })
+
+ it("preserves numeric wikilinks in inline and blockquoted fenced code", () => {
+ const input = [
+ "a `` b `[[1]]` c",
+ "> ```md",
+ "> [[2]]",
+ "> ```",
+ "> > ~~~~md",
+ "> > [[3]]",
+ "> > ~~~~",
+ ].join("\n")
+ expect(normalizeMalformedWikilinks(input)).toBe(input)
+ })
+
+ it("recognizes empty frontmatter and preserves CRLF", () => {
+ expect(normalizeMalformedWikilinks("---\n---\nbody [[1]]\n---\nx"))
+ .toBe("---\n---\nbody [1]\n---\nx")
+ expect(normalizeMalformedWikilinks("x\r\n[[1]]\r\n"))
+ .toBe("x\r\n[1]\r\n")
+ })
+
+ it("does not mistake body horizontal rules for frontmatter", () => {
+ const input = "Intro\n\n---\n\nSee [[1]]\n\n---\n\nTail [[2]]\n"
+ expect(normalizeMalformedWikilinks(input)).toBe(
+ "Intro\n\n---\n\nSee [1]\n\n---\n\nTail [2]\n",
+ )
+ })
+
+ it("normalizes nested list citations while preserving code and math blocks", () => {
+ const input = [
+ "- item",
+ " - nested [[1]]",
+ " const matrix = [[2]]",
+ "$$",
+ "[[3], [4]]",
+ "$$",
+ "<pre>",
+ "[[5]]",
+ "</pre>",
+ ].join("\n")
+ expect(normalizeMalformedWikilinks(input)).toBe([
+ "- item",
+ " - nested [1]",
+ " const matrix = [[2]]",
+ "$$",
+ "[[3], [4]]",
+ "$$",
+ "<pre>",
+ "[[5]]",
+ "</pre>",
+ ].join("\n"))
+ })
+
+ it("preserves BOM frontmatter and is idempotent for triple brackets", () => {
+ const input = "\uFEFF---\nrelated: ['[[1]]']\n---\n[[[1]]] and [[2]]"
+ const once = normalizeMalformedWikilinks(input)
+ expect(once).toBe("\uFEFF---\nrelated: ['[[1]]']\n---\n[[[1]]] and [2]")
+ expect(normalizeMalformedWikilinks(once)).toBe(once)
+ })
+
+ it("applies body normalization through the public ingest sanitizer", () => {
+ const input = "---\ntype: entity\nrelated: ['[[4]]']\n---\nBody [[1]] and [[target|alias]"
+ expect(sanitizeIngestedFileContent(input)).toBe(
+ "---\ntype: entity\nrelated: ['[[4]]']\n---\nBody [1] and [[target|alias]]",
+ )
+ })
+
+ it("is idempotent", () => {
+ const once = normalizeMalformedWikilinks("[[1]] [[target|alias] [[2], [3]]")
+ expect(normalizeMalformedWikilinks(once)).toBe(once)
+ })
+})
diff --git a/src/lib/ingest-sanitize.ts b/src/lib/ingest-sanitize.ts
index 03cd24e1..bf510f37 100644
--- a/src/lib/ingest-sanitize.ts
+++ b/src/lib/ingest-sanitize.ts
@@ -85,9 +85,106 @@ export function sanitizeIngestedFileContent(content: string): string {
// link transform applied at read time.
cleaned = repairWikilinkListsInFrontmatter(cleaned)
+ // (4) Normalize recurring malformed body links emitted by generation and
+ // synthesis models. Frontmatter and code examples remain byte-identical.
+ cleaned = normalizeMalformedWikilinks(cleaned)
+
return cleaned
}
+export function normalizeMalformedWikilinks(content: string): string {
+ const bom = content.startsWith("\uFEFF") ? "\uFEFF" : ""
+ const withoutBom = content.slice(bom.length)
+ const frontmatter = withoutBom.match(
+ /^---[ \t]*\r?\n(?:---[ \t]*(?:\r?\n|$)|[\s\S]*?\r?\n---[ \t]*(?:\r?\n|$))/,
+ )
+ const prefix = `${bom}${frontmatter?.[0] ?? ""}`
+ const body = withoutBom.slice(frontmatter?.[0].length ?? 0)
+ if (!body.includes("[[")) return content
+
+ let fence: { marker: "`" | "~"; length: number } | null = null
+ let mathFence = false
+ let htmlCodeFence = false
+ const normalized = body.replace(/.*(?:\r?\n|$)/g, (line) => {
+ const lineContent = line.replace(/\r?\n$/, "")
+ const markerMatch = lineContent.match(/^ {0,3}(?:>\s*)*(`{3,}|~{3,})/)
+ if (markerMatch) {
+ const marker = markerMatch[1][0] as "`" | "~"
+ const length = markerMatch[1].length
+ if (!fence) fence = { marker, length }
+ else if (
+ marker === fence.marker &&
+ length >= fence.length &&
+ lineContent.slice(markerMatch[0].length).trim() === ""
+ ) fence = null
+ return line
+ }
+ if (fence) return line
+ const trimmed = lineContent.trim()
+ if (trimmed === "$$") {
+ mathFence = !mathFence
+ return line
+ }
+ if (mathFence) return line
+ if (/<(?:pre|code)\b/i.test(lineContent)) htmlCodeFence = true
+ if (htmlCodeFence) {
+ if (/<\/(?:pre|code)>/i.test(lineContent)) htmlCodeFence = false
+ return line
+ }
+ const indented = /^(?: {4}|\t)/.test(lineContent)
+ const nestedList = /^(?: {4}|\t)(?:[-+*]|\d+[.)])\s+/.test(lineContent)
+ if (indented && !nestedList) return line
+ return normalizeMalformedLinksOutsideInlineCode(line)
+ })
+ return `${prefix}${normalized}`
+}
+
+function normalizeMalformedLinksOutsideInlineCode(text: string): string {
+ let output = ""
+ let cursor = 0
+ while (cursor < text.length) {
+ const opening = text.indexOf("`", cursor)
+ if (opening < 0) return output + normalizeMalformedLinkSegment(text.slice(cursor))
+ output += normalizeMalformedLinkSegment(text.slice(cursor, opening))
+ let runEnd = opening + 1
+ while (text[runEnd] === "`") runEnd++
+ const delimiter = text.slice(opening, runEnd)
+ const closing = text.indexOf(delimiter, runEnd)
+ if (closing < 0) {
+ output += normalizeMalformedLinkSegment(text.slice(opening, runEnd))
+ cursor = runEnd
+ continue
+ }
+ output += text.slice(opening, closing + delimiter.length)
+ cursor = closing + delimiter.length
+ }
+ return output
+}
+
+function normalizeMalformedLinkSegment(segment: string): string {
+ const citationPattern = /\[\[(\d+\](?:\s*,\s*\[\d+\])*)\]/g
+ let normalized = segment.replace(
+ citationPattern,
+ (match, citations: string, offset: number) => {
+ if (offset > 0 && (segment[offset - 1] === "!" || segment[offset - 1] === "[")) {
+ return match
+ }
+ if (segment[offset + match.length] === "(") return match
+ const numbers = citations.match(/\d+/g) ?? []
+ if (numbers.some((value) => value.length > 3)) return match
+ return `[${citations}`
+ },
+ )
+ if (normalized.includes("](")) return normalized
+ return normalized.replace(
+ /\[\[([^\]|\n]+)\|([^\]\n]+)\](?!\])/g,
+ (match, target: string, alias: string, offset: number) => {
+ if (offset > 0 && normalized[offset - 1] === "!") return match
+ return `[[${target}|${alias}]]`
+ },
+ )
+}
+
/** Top-level fence wrapper. Removes the open + matching close fence lines. */
function stripOuterCodeFence(content: string): string {
const open = content.match(
diff --git a/src/lib/ingest-source-path-collision.test.ts b/src/lib/ingest-source-path-collision.test.ts
index d1987d59..6cb5a124 100644
--- a/src/lib/ingest-source-path-collision.test.ts
+++ b/src/lib/ingest-source-path-collision.test.ts
@@ -892,4 +892,24 @@ describe("autoIngest source summary paths", () => {
expect(written).toEqual([])
await expect(fs.access(path.join(tmp.path, "escape.md"))).rejects.toThrow()
})
+
+ it("sanitizes malformed citation links in interactive writes", async () => {
+ if (!tmp) throw new Error("missing temp project")
+ interactiveGenerationOverride = [
+ "---FILE: wiki/entities/citations.md---",
+ "---",
+ "type: entity",
+ "title: Citations",
+ "---",
+ "Evidence [[1]] and [[target|label].",
+ "---END FILE---",
+ ].join("\n")
+ useChatStore.setState({ ingestSource: null })
+
+ const written = await executeIngestWrites(tmp.path, useWikiStore.getState().llmConfig)
+
+ expect(written).toHaveLength(1)
+ const content = await fs.readFile(written[0], "utf8")
+ expect(content).toContain("Evidence [1] and [[target|label]].")
+ })
})
diff --git a/src/lib/ingest.ts b/src/lib/ingest.ts
index 8495044b..3e09f4b5 100644
--- a/src/lib/ingest.ts
+++ b/src/lib/ingest.ts
@@ -3524,6 +3524,8 @@ async function executeIngestWritesImpl(
continue
}
+ content = sanitizeIngestedFileContent(content)
+
if (
activeSourceIdentity &&
!isLogPath(relativePath) &&
diff --git a/src/lib/page-merge.test.ts b/src/lib/page-merge.test.ts
index f5a2154e..f1a8acbd 100644
--- a/src/lib/page-merge.test.ts
+++ b/src/lib/page-merge.test.ts
@@ -44,6 +44,14 @@ describe("mergePageContent — fast paths", () => {
expect(merger).not.toHaveBeenCalled()
})
+ it("repairs malformed citations on an otherwise byte-identical page", async () => {
+ const merger = vi.fn()
+ const content = PAGE("type: entity\ntitle: Foo", "Evidence [[1]].")
+ const out = await mergePageContent(content, content, merger, baseOpts)
+ expect(out).toContain("Evidence [1].")
+ expect(merger).not.toHaveBeenCalled()
+ })
+
it("skips LLM when bodies are identical (only sources differ)", async () => {
// Re-ingest of the same file from a different source just adds
// its source filename — body is byte-identical. Don't waste an
@@ -183,6 +191,18 @@ describe("mergePageContent — LLM merge", () => {
expect(out).not.toContain("type: concept")
})
+ it("normalizes malformed citations introduced by the merge model", async () => {
+ const existing = PAGE("type: entity\ntitle: Foo", "Existing detailed evidence for the topic.")
+ const incoming = PAGE("type: entity\ntitle: Foo", "Incoming detailed evidence for the topic.")
+ const merger = vi.fn().mockResolvedValue(PAGE(
+ "type: entity\ntitle: Foo",
+ "Existing and incoming evidence are retained [[1]] with sources [[2], [3]].",
+ ))
+
+ const out = await mergePageContent(incoming, existing, merger, baseOpts)
+ expect(out).toContain("retained [1] with sources [2], [3].")
+ })
+
it("strips directory prefixes from merged body wikilinks only", async () => {
const existing = PAGE(
"type: entity\ntitle: Foo",
diff --git a/src/lib/page-merge.ts b/src/lib/page-merge.ts
index 8132c91b..4883bc56 100644
--- a/src/lib/page-merge.ts
+++ b/src/lib/page-merge.ts
@@ -25,6 +25,7 @@
*/
import { parseFrontmatter } from "./frontmatter"
import { mergeArrayFieldsIntoContent } from "./sources-merge"
+import { normalizeMalformedWikilinks } from "./ingest-sanitize"
/** Frontmatter array fields unioned across re-ingests. */
const UNION_FIELDS = ["sources", "tags", "related"] as const
@@ -98,11 +99,11 @@ export async function mergePageContent(
opts: MergePageOptions,
): Promise<string> {
// Fast path 1: brand-new page.
- if (!existingContent) return newContent
+ if (!existingContent) return normalizeMalformedWikilinks(newContent)
// Fast path 2: byte-identical — re-ingest of the same source file
// with no actual change. Stable reference helps callers.
- if (newContent === existingContent) return existingContent
+ if (newContent === existingContent) return normalizeMalformedWikilinks(existingContent)
// Step 1 — always-on: union the array frontmatter fields.
const arrayMerged = mergeArrayFieldsIntoContent(
@@ -125,14 +126,14 @@ export async function mergePageContent(
replacement = setFrontmatterScalar(replacement, field, existingValue)
}
}
- return setFrontmatterScalar(
+ return normalizeMalformedWikilinks(setFrontmatterScalar(
replacement,
"updated",
(opts.today ?? defaultToday)(),
- )
+ ))
}
if (oldParsed.body.trim() === arrayMergedParsed.body.trim()) {
- return arrayMerged
+ return normalizeMalformedWikilinks(arrayMerged)
}
// Step 2 — ask the merger to produce a unified body. Failures fall
@@ -150,7 +151,7 @@ export async function mergePageContent(
`[page-merge] LLM merge failed for ${opts.pagePath}, falling back to incoming + array-field union: ${err instanceof Error ? err.message : err}`,
)
await tryBackup(opts, existingContent)
- return arrayMerged
+ return normalizeMalformedWikilinks(arrayMerged)
}
// Sanity 1: LLM output must parse as a frontmatter-bearing wiki
@@ -161,7 +162,7 @@ export async function mergePageContent(
`[page-merge] LLM output for ${opts.pagePath} has no frontmatter — rejecting, falling back`,
)
await tryBackup(opts, existingContent)
- return arrayMerged
+ return normalizeMalformedWikilinks(arrayMerged)
}
// Sanity 2: body length. Reject obvious truncation / lazy summary.
@@ -174,7 +175,7 @@ export async function mergePageContent(
`[page-merge] LLM merge for ${opts.pagePath} produced body ${llmBodyLen} chars, below threshold ${minThreshold.toFixed(0)} (max input was ${Math.max(oldBodyLen, newBodyLen)}) — rejecting, falling back`,
)
await tryBackup(opts, existingContent)
- return arrayMerged
+ return normalizeMalformedWikilinks(arrayMerged)
}
// Step 3 — apply deterministic post-processing: lock fields back
@@ -197,7 +198,7 @@ export async function mergePageContent(
const todayFn = opts.today ?? defaultToday
final = setFrontmatterScalar(final, "updated", todayFn())
- return stripBodyWikilinkPathPrefixes(final)
+ return normalizeMalformedWikilinks(stripBodyWikilinkPathPrefixes(final))
}
/**