Skip to content

LLM Wiki is a cross-platform desktop application that turns your documents into an organized, interlinked knowledge base — automatically. Instead of traditional RAG (retrieve-and-answer from scratch every time), the LLM incrementally builds and maintains a persistent wiki from your sources。

TypeScript
19.3 KB
Raw Blame
/**
 * Merge a wiki page that the LLM just generated with whatever's
 * already on disk. Solves silent data loss across re-ingests where
 * a second source contributes content to the same entity / concept
 * page.
 *
 * Architecture: pure logic, LLM call injected as a parameter.
 * Production wires this up against `streamChat`; tests use mocks.
 *
 * Three layers of protection:
 *   1. Frontmatter array fields (sources / tags / related) — always
 *      union-merged at the application layer regardless of whether
 *      the LLM is involved. Zero-cost, deterministic.
 *   2. Body — if old and new bodies differ, ask the LLM to produce
 *      a coherent merge. Sanity-checked on length and structure
 *      before accepting.
 *   3. Locked frontmatter fields (type / title / created) — even if
 *      the LLM rewrote them, the existing values are forced back.
 *      type/title shifting breaks wikilinks; created is a one-time
 *      stamp.
 *
 * Fallback: any LLM failure or sanity-check rejection falls back to
 * the previous (Array-merged-frontmatter + new body) behavior, with
 * an optional backup of the existing content for user recovery.
 */
import { parseFrontmatter } from "./frontmatter"
import { mergeArrayFieldsIntoContent } from "./sources-merge"
import { normalizeMalformedWikilinks } from "./ingest-sanitize"

/** Frontmatter array fields unioned across re-ingests. */
const UNION_FIELDS = ["sources", "tags", "related"] as const

/**
 * Frontmatter scalar fields whose existing value MUST survive an
 * ingest, even if the LLM emits a different value:
 *   - type: changing it would re-categorize the page across folders;
 *     the wiki schema isn't designed for that.
 *   - title: every wikilink that resolves to this page goes through
 *     `[[slug]]` keyed off the slug, but the displayed title is the
 *     `title` field — silently rewriting it would break user mental
 *     maps and any prose that references the title verbatim.
 *   - created: a one-time stamp; an "updated" stamp is computed
 *     separately.
 */
const LOCKED_FIELDS = ["type", "title", "created"] as const

/**
 * Body length safety threshold. If the LLM's merged body is shorter
 * than 70% of the longer of (existing body, incoming body), reject
 * the merge — the LLM almost certainly stripped content rather than
 * legitimately deduplicating. 0.7 allows for ~30% legitimate dedup
 * compression while catching obvious truncation / lazy summaries.
 */
const BODY_SHRINK_THRESHOLD = 0.7
const HTTP_URL_RE = /https?:\/\/[^\s<>"'\])]+/gi

export interface MergeFn {
  /**
   * Implements the LLM-call portion of the merge. Receives the
   * existing on-disk content and the incoming new content (already
   * array-field-merged); returns the LLM's full proposed merged
   * file (frontmatter + body) as a string.
   */
  (
    existingContent: string,
    incomingContent: string,
    sourceFileName: string,
    signal?: AbortSignal,
  ): Promise<string>
}

export interface MergePageOptions {
  /** Source filename being ingested — passed through to the merger
   *  and used in fallback log messages. */
  sourceFileName: string
  /** Wiki-relative page path (e.g. wiki/entities/foo.md). Used only
   *  for log messages today; the file write itself is the caller's
   *  responsibility. */
  pagePath: string
  /** Abort signal forwarded to the merger. */
  signal?: AbortSignal
  /** Best-effort hook to back up the existing content before any
   *  fallback write. Errors are swallowed — backup must never
   *  block the merge. */
  backup?: (existingContent: string) => Promise<void>
  /** Date provider for the `updated` field. Injectable for
   *  deterministic tests. Defaults to today's UTC date. */
  today?: () => string
  /**
   * Replace the old body after a corrected source is regenerated, while
   * retaining union fields, locked metadata, and a recovery backup.
   */
  replaceExistingBody?: boolean
}

export async function mergePageContent(
  newContent: string,
  existingContent: string | null,
  merger: MergeFn,
  opts: MergePageOptions,
): Promise<string> {
  // Fast path 1: brand-new page.
  if (!existingContent) return normalizeMalformedWikilinks(newContent)

  // Fast path 2: byte-identical — re-ingest of the same source file
  // with no actual change. Stable reference helps callers.
  if (newContent === existingContent) return normalizeMalformedWikilinks(existingContent)

  // Step 1 — always-on: union the array frontmatter fields.
  const arrayMerged = mergeArrayFieldsIntoContent(
    newContent,
    existingContent,
    [...UNION_FIELDS],
  )

  // Fast path 3: bodies are identical (only frontmatter array-fields
  // differed). The array merge has already produced the right output;
  // skip the LLM.
  const oldParsed = parseFrontmatter(existingContent)
  const arrayMergedParsed = parseFrontmatter(arrayMerged)
  if (opts.replaceExistingBody) {
    await tryBackup(opts, existingContent)
    let replacement = arrayMerged
    for (const field of LOCKED_FIELDS) {
      const existingValue = oldParsed.frontmatter?.[field]
      if (typeof existingValue === "string" && existingValue !== "") {
        replacement = setFrontmatterScalar(replacement, field, existingValue)
      }
    }
    return normalizeMalformedWikilinks(setFrontmatterScalar(
      replacement,
      "updated",
      (opts.today ?? defaultToday)(),
    ))
  }
  if (oldParsed.body.trim() === arrayMergedParsed.body.trim()) {
    return normalizeMalformedWikilinks(arrayMerged)
  }

  // Step 2 — ask the merger to produce a unified body. Failures fall
  // through to the array-merged-only path with a best-effort backup.
  let llmOutput: string
  try {
    llmOutput = await merger(
      existingContent,
      arrayMerged,
      opts.sourceFileName,
      opts.signal,
    )
  } catch (err) {
    console.warn(
      `[page-merge] LLM merge failed for ${opts.pagePath}, falling back to incoming + array-field union: ${err instanceof Error ? err.message : err}`,
    )
    await tryBackup(opts, existingContent)
    return normalizeMalformedWikilinks(arrayMerged)
  }

  // Sanity 1: LLM output must parse as a frontmatter-bearing wiki
  // page. No-frontmatter output would silently corrupt the page.
  const llmParsed = parseFrontmatter(llmOutput)
  if (llmParsed.frontmatter === null) {
    console.warn(
      `[page-merge] LLM output for ${opts.pagePath} has no frontmatter — rejecting, falling back`,
    )
    await tryBackup(opts, existingContent)
    return normalizeMalformedWikilinks(arrayMerged)
  }

  // Sanity 2: body length. Reject obvious truncation / lazy summary.
  const oldBodyLen = oldParsed.body.length
  const newBodyLen = arrayMergedParsed.body.length
  const llmBodyLen = llmParsed.body.length
  const minThreshold = Math.max(oldBodyLen, newBodyLen) * BODY_SHRINK_THRESHOLD
  if (llmBodyLen < minThreshold) {
    console.warn(
      `[page-merge] LLM merge for ${opts.pagePath} produced body ${llmBodyLen} chars, below threshold ${minThreshold.toFixed(0)} (max input was ${Math.max(oldBodyLen, newBodyLen)}) — rejecting, falling back`,
    )
    await tryBackup(opts, existingContent)
    return normalizeMalformedWikilinks(arrayMerged)
  }

  llmOutput = removeIntroducedReservedExampleUrls(
    llmOutput,
    existingContent,
    arrayMerged,
  )

  // Step 3 — apply deterministic post-processing: lock fields back
  // to existing values, force array fields to the union, set
  // updated to today.
  let final = llmOutput
  for (const field of LOCKED_FIELDS) {
    const existingValue = oldParsed.frontmatter?.[field]
    if (typeof existingValue === "string" && existingValue !== "") {
      final = setFrontmatterScalar(final, field, existingValue)
    }
  }
  // Re-apply union merges on top of the LLM's frontmatter using
  // arrayMerged (which is already existing+incoming union) as the
  // reference. The LLM may have output a subset of array values —
  // defensively re-union against BOTH sides so neither contributor
  // is dropped, regardless of which side the LLM chose to echo.
  final = mergeArrayFieldsIntoContent(final, arrayMerged, [...UNION_FIELDS])
  // Updated is always today on a successful merge.
  const todayFn = opts.today ?? defaultToday
  final = setFrontmatterScalar(final, "updated", todayFn())

  return normalizeMalformedWikilinks(
    stripBodyWikilinkPathPrefixes(
      stripMergeScaffolding(final, [existingContent, arrayMerged]),
    ),
  )
}

interface ReservedUrlMatch {
  canonical: string
  raw: string
  start: number
  end: number
}

function reservedExampleUrls(content: string): ReservedUrlMatch[] {
  const urls: ReservedUrlMatch[] = []
  for (const match of content.matchAll(HTTP_URL_RE)) {
    if (match.index === undefined) continue
    const raw = match[0].replace(/[.,;:!?]+$/, "")
    try {
      const parsed = new URL(raw)
      const host = parsed.hostname.replace(/\.$/, "").toLowerCase()
      if (["example.com", "example.org", "example.net"].some(
        (domain) => host === domain || host.endsWith(`.${domain}`),
      )) urls.push({
        canonical: parsed.toString(),
        raw,
        start: match.index,
        end: match.index + raw.length,
      })
    } catch {
      // Invalid URLs are left to the ordinary Markdown/content validators.
    }
  }
  return urls
}

function removeIntroducedReservedExampleUrls(
  output: string,
  existing: string,
  incoming: string,
): string {
  const inputUrls = new Set([
    ...reservedExampleUrls(existing).map((match) => match.canonical),
    ...reservedExampleUrls(incoming).map((match) => match.canonical),
  ])
  const introduced = reservedExampleUrls(output)
    .filter((match) => !inputUrls.has(match.canonical))
  if (introduced.length === 0) return output
  console.warn("[page-merge] removed invented reserved example-domain URL(s) from merge output")
  const introducedUrls = new Set(introduced.map((match) => match.canonical))
  const frontmatter = output.match(/^---\r?\n[\s\S]*?\r?\n---(?:\r?\n|$)/)
  const prefix = frontmatter?.[0] ?? ""
  const body = output.slice(prefix.length)
  let fence: { marker: string; length: number } | null = null
  const cleanedBody = body.split(/(\r?\n)/).map((part) => {
    if (/^\r?\n$/.test(part)) return part
    const marker = part.match(/^ {0,3}(`{3,}|~{3,})/)
    if (marker) {
      if (!fence) fence = { marker: marker[1][0], length: marker[1].length }
      else if (
        marker[1][0] === fence.marker &&
        marker[1].length >= fence.length &&
        part.slice(marker[0].length).trim() === ""
      ) fence = null
      return part
    }
    if (fence || /^(?: {4}|\t)/.test(part)) return part

    const shouldRemove = (raw: string): boolean => {
      try {
        return introducedUrls.has(new URL(raw.replace(/[.,;:!?]+$/, "")).toString())
      } catch {
        return false
      }
    }
    let line = part.replace(
      /!?\[([^\]\r\n]+)\]\((https?:\/\/[^\s<>")]+)(?:\s+["'][^"']*["'])?\)/gi,
      (whole, label: string, url: string) => shouldRemove(url) ? label : whole,
    )
    line = line.replace(/<(https?:\/\/[^\s<>]+)>/gi, (whole, url: string) => (
      shouldRemove(url) ? "" : whole
    ))
    line = line.replace(HTTP_URL_RE, (url) => shouldRemove(url) ? "" : url)
    if (/^\s*\[[^\]]+\]:\s*$/.test(line) || /^\s*[-*+]\s*$/.test(line)) return ""
    return line.replace(/[ \t]+$/, "")
  }).join("")
  return `${prefix}${cleanedBody}`
}

export function stripMergeScaffolding(
  content: string,
  protectedInputs: string[] = [],
): string {
  const frontmatter = content.match(/^---\r?\n[\s\S]*?\r?\n---(?:\r?\n|$)/)
  const prefix = frontmatter?.[0] ?? ""
  const body = content.slice(prefix.length)
  let fence: { marker: string; length: number } | null = null
  const label = "(?:现有|新生成|新|原始|旧|已有)(?:内容|版本|材料)(?:的?(?:定义|说明|描述))?|(?:补充材料|此前收集的材料)|(?:existing|newly generated|new|original|old|incoming|additional|previously collected) (?:content|version|material)(?: (?:definition|description))?"
  const sourceNote = new RegExp(
    `^[ \\t]*(?:[-*][ \\t]*)?\\*?[((]?[ \\t]*(?:来源|source)[ \\t]*[::][ \\t]*(?:${label})[ \\t]*[))]?\\*?[ \\t]*$`,
    "i",
  )
  const bracketedSuffix = new RegExp(`[ \\t]*[((【][ \\t]*(?:${label})[ \\t]*[))】][ \\t]*$`, "i")
  const bareSuffix = new RegExp(`^(#{1,6}\\s+.+?)[ \\t]+[-—:][ \\t]*(?:${label})[ \\t]*$`, "i")
  const protectedLines = new Set(
    protectedInputs.flatMap((input) => input.split(/\r?\n/).map((line) => line.trim())),
  )
  const protectedHeadings = new Set(
    protectedInputs.flatMap((input) => input.split(/\r?\n/))
      .map((line) => line.match(/^#{1,6}\s+(.+?)\s*$/)?.[1])
      .filter((heading): heading is string => Boolean(heading)),
  )
  const cleanedBody = body.replace(/.*(?:\r?\n|$)/g, (line) => {
    const lineContent = line.replace(/\r?\n$/, "")
    const newline = line.slice(lineContent.length)
    const marker = lineContent.match(/^ {0,3}(`{3,}|~{3,})/)
    if (marker) {
      if (!fence) fence = { marker: marker[1][0], length: marker[1].length }
      else if (
        marker[1][0] === fence.marker &&
        marker[1].length >= fence.length &&
        lineContent.slice(marker[0].length).trim() === ""
      ) fence = null
      return line
    }
    if (fence || /^(?: {4}|\t)/.test(lineContent)) return line
    if (protectedLines.has(lineContent.trim())) return line
    if (sourceNote.test(lineContent)) return ""
    if (!/^#{1,6}\s+/.test(lineContent)) return line
    const headingText = lineContent.replace(/^#{1,6}\s+/, "").trim()
    if (protectedHeadings.has(headingText)) return line
    const withoutBracket = lineContent.replace(bracketedSuffix, "")
    const withoutBare = withoutBracket.replace(bareSuffix, "$1")
    return `${withoutBare}${newline}`
  })
  return `${prefix}${cleanedBody}`
}

/**
 * Normalize page links after an LLM merge without touching frontmatter,
 * prose paths, code examples, or Obsidian image/file embeds.
 *
 * Project schemas route pages into directories on disk, but cross-page links
 * use bare slugs. Older page bodies may still contain targets such as
 * `[[clients/foo-overview]]`; merge prompts intentionally preserve existing
 * links, so the application must repair those targets deterministically.
 */
export function stripBodyWikilinkPathPrefixes(content: string): string {
  const frontmatter = content.match(/^---\r?\n[\s\S]*?\r?\n---(?:\r?\n|$)/)
  if (!frontmatter) return content

  const body = content.slice(frontmatter[0].length)
  if (!body.includes("[[")) return content

  const normalizedBody = normalizeWikilinksOutsideCode(body)

  return `${frontmatter[0]}${normalizedBody}`
}

function normalizeWikilinksOutsideCode(body: string): string {
  let fence: { marker: "`" | "~"; length: number } | null = null

  return body.replace(/.*(?:\r?\n|$)/g, (line) => {
    const content = line.replace(/\r?\n$/, "")
    const markerMatch = content.match(/^ {0,3}(`{3,}|~{3,})/)
    if (markerMatch) {
      const marker = markerMatch[1][0] as "`" | "~"
      const length = markerMatch[1].length
      if (!fence) fence = { marker, length }
      else if (
        marker === fence.marker &&
        length >= fence.length &&
        content.slice(markerMatch[0].length).trim() === ""
      ) {
        fence = null
      }
      return line
    }
    if (fence || /^(?: {4}|\t)/.test(content)) return line

    return replaceOutsideInlineCode(line)
  })
}

function replaceOutsideInlineCode(text: string): string {
  let output = ""
  let cursor = 0

  while (cursor < text.length) {
    const opening = text.indexOf("`", cursor)
    if (opening < 0) return output + replaceWikilinkPrefixes(text.slice(cursor))

    output += replaceWikilinkPrefixes(text.slice(cursor, opening))
    let runEnd = opening + 1
    while (text[runEnd] === "`") runEnd += 1
    const delimiter = text.slice(opening, runEnd)
    const closing = text.indexOf(delimiter, runEnd)
    if (closing < 0) {
      // An unmatched run is ordinary Markdown text, not an inline code span.
      output += replaceWikilinkPrefixes(text.slice(opening, runEnd))
      cursor = runEnd
      continue
    }

    output += text.slice(opening, closing + delimiter.length)
    cursor = closing + delimiter.length
  }

  return output
}

const PAGE_WIKILINK_RE = /\[\[([^\]|\n]+)(?:\|([^\]\n]*))?\]\]/g

function replaceWikilinkPrefixes(text: string): string {
  return text.replace(
    PAGE_WIKILINK_RE,
    (match, rawTarget: string, rawAlias: string | undefined, offset: number) => {
      if (offset > 0 && text[offset - 1] === "!") return match
      const preceding = text.slice(0, offset).match(/\\+$/)?.[0].length ?? 0
      if (preceding % 2 === 1) return match

      const target = rawTarget.trim()
      const normalizedTarget = bareWikilinkTarget(target)
      if (normalizedTarget === target) return match

      const alias = rawAlias === undefined ? "" : `|${rawAlias}`
      return `[[${normalizedTarget}${alias}]]`
    },
  )
}

function bareWikilinkTarget(target: string): string {
  if (!target || target.startsWith("#")) return target
  // URI-like targets are not wiki page paths.
  if (/^[a-z][a-z0-9+.-]*:/i.test(target)) return target

  const fragmentIndex = target.indexOf("#")
  const pageTarget = fragmentIndex >= 0 ? target.slice(0, fragmentIndex) : target
  const fragment = fragmentIndex >= 0 ? target.slice(fragmentIndex) : ""
  const normalizedPath = pageTarget.replace(/\\/g, "/")
  if (!normalizedPath.includes("/")) return target

  const leaf = normalizedPath.split("/").pop()
  if (!leaf) return target
  const extensionIndex = leaf.lastIndexOf(".")
  if (extensionIndex > 0 && leaf.slice(extensionIndex).toLowerCase() !== ".md") {
    return target
  }

  return `${leaf}${fragment}`
}

async function tryBackup(
  opts: MergePageOptions,
  existingContent: string,
): Promise<void> {
  if (!opts.backup) return
  try {
    await opts.backup(existingContent)
  } catch (err) {
    console.warn(
      `[page-merge] backup failed for ${opts.pagePath}: ${err instanceof Error ? err.message : err}`,
    )
  }
}

function defaultToday(): string {
  return new Date().toISOString().slice(0, 10)
}

/**
 * Set a scalar frontmatter field to `value` in the inline form
 * `field: value`. If the field already exists in the frontmatter,
 * the line is replaced in place. If it doesn't, the field is
 * appended at the end of the frontmatter block.
 *
 * Keeps the rest of the document unchanged. Returns content
 * unchanged if it has no frontmatter at all.
 *
 * Quoting: the value is written verbatim; the caller is
 * responsible for quoting if the value contains characters that
 * would otherwise break YAML parsing (`:` etc). Today's callers
 * pass plain identifiers and ISO dates so this hasn't been an
 * issue.
 */
function setFrontmatterScalar(
  content: string,
  fieldName: string,
  value: string,
): string {
  const fmMatch = content.match(/^(---\n)([\s\S]*?)(\n---)/)
  if (!fmMatch) return content
  const [, openDelim, fmBody, closeDelim] = fmMatch
  const escapedName = fieldName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")
  const newLine = `${fieldName}: ${value}`

  // Only match scalar form (no `[`, no `\n  -`). Array-form fields
  // are handled by sources-merge.
  const lineRe = new RegExp(`^${escapedName}:\\s*(?!\\[)([^\\n]*)`, "m")
  if (lineRe.test(fmBody)) {
    const rewritten = fmBody.replace(lineRe, newLine)
    return `${openDelim}${rewritten}${closeDelim}${content.slice(fmMatch[0].length)}`
  }
  // Field absent — append.
  const rewritten = `${fmBody}\n${newLine}`
  return `${openDelim}${rewritten}${closeDelim}${content.slice(fmMatch[0].length)}`
}