Skip to content

LLM Wiki is a cross-platform desktop application that turns your documents into an organized, interlinked knowledge base — automatically. Instead of traditional RAG (retrieve-and-answer from scratch every time), the LLM incrementally builds and maintains a persistent wiki from your sources。

TypeScript
8.7 KB
Raw Blame
/**
 * Frontmatter array-field merging during ingest.
 *
 * Originally written for the `sources:` field alone — re-ingesting a
 * page from a second source would clobber `sources: [...]` to a
 * single entry, and the source-delete flow would later treat the page
 * as single-sourced and delete it (silent data loss). The fix unions
 * old and new sources before writing.
 *
 * Generalized to handle any frontmatter array field (`sources`,
 * `tags`, `related`, …): the same loss-on-clobber pattern applied to
 * `tags` and `related` too — old contributing tags / wikilinks
 * disappeared every time a different source brought new ones. The
 * generic API merges any subset of fields the caller names.
 *
 * Two callers today:
 *   - sources-view.tsx — uses `parseSources` / `writeSources` for
 *     the source-delete flow (operates on the single sources field)
 *   - ingest.ts — uses `mergeArrayFieldsIntoContent` to union
 *     sources + tags + related when writing a content page that
 *     already exists on disk
 */

// ─── Generic helpers (the implementation core) ────────────────────

/**
 * Extract a frontmatter array field by name. Handles both:
 *   inline form:    `name: ["a", "b"]` or `name: [a, b]`
 *   block form:     `name:\n  - a\n  - b`
 * Strips quotes (single or double) from items. Returns `[]` for
 * missing field, malformed parse, or content with no frontmatter.
 *
 * The field name is matched as a whole word at line start, so
 * `parseFrontmatterArray(c, "rel")` won't match `related: [...]`.
 */
export function parseFrontmatterArray(content: string, fieldName: string): string[] {
  const fmMatch = content.match(/^---\r?\n([\s\S]*?)\r?\n---/)
  if (!fmMatch) return []
  const fm = fmMatch[1]
  // Anchor to start of line + exact field name + colon. The negative
  // lookahead-style check is done by requiring `:` immediately after.
  const escapedName = fieldName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")
  const blockRe = new RegExp(
    `^${escapedName}:\\s*\\r?\\n((?:[ \\t]+-\\s+.+(?:\\r?\\n|$))+)`,
    "m",
  )
  const block = fm.match(blockRe)
  if (block) {
    const out: string[] = []
    for (const line of block[1].split(/\r?\n/)) {
      const m = line.match(/^\s+-\s+["']?(.+?)["']?\s*$/)
      if (m && m[1]) out.push(m[1].trim())
    }
    return out
  }

  const inlineRe = new RegExp(`^${escapedName}:\\s*\\[([^\\]]*)\\]`, "m")
  const inline = fm.match(inlineRe)
  if (!inline) return []
  const body = inline[1].trim()
  if (body === "") return []
  return splitInlineArray(body)
}

function splitInlineArray(body: string): string[] {
  const out: string[] = []
  let current = ""
  let quote: "\"" | "'" | null = null
  let escaped = false

  for (const ch of body) {
    if (escaped) {
      current += ch
      escaped = false
      continue
    }
    if (quote === "\"" && ch === "\\") {
      escaped = true
      continue
    }
    if ((ch === "\"" || ch === "'") && quote === null) {
      quote = ch
      continue
    }
    if (quote === ch) {
      quote = null
      continue
    }
    if (ch === "," && quote === null) {
      const value = current.trim()
      if (value) out.push(value)
      current = ""
      continue
    }
    current += ch
  }

  const value = current.trim()
  if (value) out.push(value)
  return out
}

/**
 * Rewrite (or insert) a frontmatter array field. Preserves all other
 * frontmatter lines and order. Returns content unchanged if the
 * input has no frontmatter at all (don't manufacture frontmatter for
 * unconventional pages — almost certainly malformed emission worth
 * surfacing rather than silently fixing).
 *
 * Always emits the inline form `name: ["a", "b"]` so downstream
 * parsers see a consistent shape regardless of the original input
 * shape.
 */
export function writeFrontmatterArray(
  content: string,
  fieldName: string,
  values: string[],
): string {
  const fmMatch = content.match(/^(---\r?\n)([\s\S]*?)(\r?\n---)/)
  if (!fmMatch) return content

  const [, openDelim, fmBody, closeDelim] = fmMatch
  const newline = openDelim.endsWith("\r\n") ? "\r\n" : "\n"
  const escapedName = fieldName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")
  const serialized = values.map(quoteInlineArrayValue).join(", ")
  const newLine = `${fieldName}: [${serialized}]`

  // Replace inline form in place — preserves field ordering.
  const inlineRe = new RegExp(`^${escapedName}:\\s*\\[[^\\]]*\\]`, "m")
  if (inlineRe.test(fmBody)) {
    const rewritten = fmBody.replace(inlineRe, newLine)
    return `${openDelim}${rewritten}${closeDelim}${content.slice(fmMatch[0].length)}`
  }

  // Replace block form in place, normalized to inline form.
  const blockRe = new RegExp(
    `^${escapedName}:\\s*\\r?\\n((?:[ \\t]+-\\s+.+(?:\\r?\\n|$))+)`,
    "m",
  )
  if (blockRe.test(fmBody)) {
    const rewritten = fmBody.replace(blockRe, (matched) =>
      `${newLine}${/\r?\n$/.test(matched) ? newline : ""}`,
    )
    return `${openDelim}${rewritten}${closeDelim}${content.slice(fmMatch[0].length)}`
  }

  // Field absent — append at end of frontmatter.
  const rewritten = `${fmBody}${newline}${newLine}`
  return `${openDelim}${rewritten}${closeDelim}${content.slice(fmMatch[0].length)}`
}

function quoteInlineArrayValue(value: string): string {
  return `"${value.replace(/\\/g, "\\\\").replace(/"/g, '\\"')}"`
}

/**
 * Union-merge two array values. Case-insensitive dedup. First-seen
 * casing wins (matches sources-merge's historical contract — keeps
 * users' original filename casing stable across re-ingests).
 */
function mergeLists(
  existing: readonly string[],
  incoming: readonly string[],
): string[] {
  const seen = new Set<string>()
  const out: string[] = []
  for (const s of [...existing, ...incoming]) {
    const key = s.toLowerCase()
    if (seen.has(key)) continue
    seen.add(key)
    out.push(s)
  }
  return out
}

/**
 * Multi-field merge entry point. For each requested field, union the
 * existing-on-disk value with the LLM-emitted new value, and rewrite
 * the new content's frontmatter with the merged values.
 *
 * Fast-paths:
 *   - existingContent null/empty → return newContent verbatim
 *   - existing has no frontmatter at all → return newContent verbatim
 *   - no field actually changes → return newContent verbatim (stable
 *     reference, callers can rely on this for cache-key invariance)
 */
export function mergeArrayFieldsIntoContent(
  newContent: string,
  existingContent: string | null,
  fields: readonly string[],
): string {
  if (!existingContent) return newContent
  if (!/^---\r?\n/.test(existingContent)) return newContent

  let result = newContent
  let changed = false
  for (const field of fields) {
    const oldValues = parseFrontmatterArray(existingContent, field)
    if (oldValues.length === 0) continue // field absent in existing → nothing to preserve
    const newValues = parseFrontmatterArray(result, field)
    const merged = mergeLists(oldValues, newValues)
    if (
      merged.length === newValues.length &&
      merged.every((s, i) => s === newValues[i])
    ) {
      continue // no-op for this field
    }
    result = writeFrontmatterArray(result, field, merged)
    changed = true
  }
  return changed ? result : newContent
}

// ─── Backward-compatible single-field exports ─────────────────────

/**
 * Extract `sources: [...]` from a wiki page's frontmatter.
 * Handles inline and block forms; strips quotes; returns [] when
 * absent. Thin wrapper around the generic parser, kept stable for
 * sources-view.tsx (source-delete flow).
 */
export function parseSources(content: string): string[] {
  return parseFrontmatterArray(content, "sources")
}

/**
 * Rewrite the `sources:` field. Thin wrapper around the generic
 * writer, kept stable for sources-view.tsx (writes the post-delete
 * sources list back).
 */
export function writeSources(content: string, sources: string[]): string {
  return writeFrontmatterArray(content, "sources", sources)
}

/**
 * Merge two source lists into one (case-insensitive dedup, first-
 * seen casing wins). Exported for tests and sources-view consumers
 * that prefer to call the merge directly without going through the
 * content-rewrite path.
 */
export function mergeSourcesLists(
  existing: readonly string[],
  incoming: readonly string[],
): string[] {
  return mergeLists(existing, incoming)
}

/**
 * Sources-only convenience wrapper — equivalent to
 * `mergeArrayFieldsIntoContent(newContent, existingContent, ["sources"])`.
 * Kept for ingest-flow callers that pre-date the multi-field
 * generalization; new code should prefer the generic version which
 * also unions tags / related.
 */
export function mergeSourcesIntoContent(
  newContent: string,
  existingContent: string | null,
): string {
  return mergeArrayFieldsIntoContent(newContent, existingContent, ["sources"])
}