Skip to content

LLM Wiki is a cross-platform desktop application that turns your documents into an organized, interlinked knowledge base — automatically. Instead of traditional RAG (retrieve-and-answer from scratch every time), the LLM incrementally builds and maintains a persistent wiki from your sources。

TypeScript

feat: graph relevance model + tokenized search + budget-controlled retrieval

nash_sucf58eacc5eba
diff --git a/src/components/chat/chat-panel.tsx b/src/components/chat/chat-panel.tsx
index a62c447c..63ae0217 100644
--- a/src/components/chat/chat-panel.tsx
+++ b/src/components/chat/chat-panel.tsx
@@ -9,6 +9,7 @@ import { streamChat, type ChatMessage as LLMMessage } from "@/lib/llm-client"
import { executeIngestWrites } from "@/lib/ingest"
import { listDirectory, readFile, writeFile } from "@/commands/fs"
import { searchWiki } from "@/lib/search"
+import { buildRetrievalGraph, getRelatedNodes } from "@/lib/graph-relevance"
import { useReviewStore } from "@/stores/review-store"
import type { FileNode } from "@/types/wiki"
@@ -47,41 +48,90 @@ export function ChatPanel() {
addMessage("user", text)
setStreaming(true)
- // Build system prompt with wiki context: search for relevant pages and include their content
+ // Build system prompt with wiki context using graph-enhanced retrieval
const systemMessages: LLMMessage[] = []
if (project) {
+ const dataVersion = useWikiStore.getState().dataVersion
const [index, purpose] = await Promise.all([
readFile(`${project.path}/wiki/index.md`).catch(() => ""),
readFile(`${project.path}/purpose.md`).catch(() => ""),
])
- // Search wiki + raw sources for pages relevant to the current question only
+ // --- Phase 1: Tokenized search -> top 10 ---
const searchResults = await searchWiki(project.path, text)
-
- // Read the full content of top relevant pages (max 5 to keep context focused)
- const relevantPages: { title: string; path: string; content: string }[] = []
- for (const result of searchResults.slice(0, 5)) {
+ const topSearchResults = searchResults.slice(0, 10)
+
+ // --- Phase 2: Graph 1-level expansion ---
+ const graph = await buildRetrievalGraph(project.path, dataVersion)
+ const expandedIds = new Set<string>()
+ const searchHitPaths = new Set(topSearchResults.map((r) => r.path))
+ const graphExpansions: { title: string; path: string; relevance: number }[] = []
+
+ for (const result of topSearchResults) {
+ // Derive node ID from file path (kebab-case filename without .md)
+ const fileName = result.path.split("/").pop() ?? ""
+ const nodeId = fileName.replace(/\.md$/, "")
+ const related = getRelatedNodes(nodeId, graph, 3)
+ for (const { node, relevance } of related) {
+ if (relevance < 2.0) continue
+ if (searchHitPaths.has(node.path)) continue
+ if (expandedIds.has(node.id)) continue
+ expandedIds.add(node.id)
+ graphExpansions.push({
+ title: node.title,
+ path: node.path,
+ relevance,
+ })
+ }
+ }
+ // Sort graph expansions by relevance descending
+ graphExpansions.sort((a, b) => b.relevance - a.relevance)
+
+ // --- Phase 3 & 4: Budget control with priority queue ---
+ const TOTAL_BUDGET = 30_000
+ const MAX_PAGE_SIZE = 8_000
+ let usedChars = 0
+
+ type PageEntry = { title: string; path: string; content: string; priority: number }
+ const relevantPages: PageEntry[] = []
+
+ // Helper to add a page within budget
+ const tryAddPage = async (
+ title: string,
+ filePath: string,
+ priority: number,
+ ): Promise<boolean> => {
+ if (usedChars >= TOTAL_BUDGET) return false
try {
- const content = await readFile(result.path)
- const relativePath = result.path.replace(project.path + "/", "")
- // Truncate large files
- const truncated = content.length > 15000
- ? content.slice(0, 15000) + "\n\n[...truncated...]"
- : content
- relevantPages.push({ title: result.title, path: relativePath, content: truncated })
+ const raw = await readFile(filePath)
+ const relativePath = filePath.replace(project.path + "/", "")
+ const truncated = raw.length > MAX_PAGE_SIZE
+ ? raw.slice(0, MAX_PAGE_SIZE) + "\n\n[...truncated...]"
+ : raw
+ if (usedChars + truncated.length > TOTAL_BUDGET) return false
+ usedChars += truncated.length
+ relevantPages.push({ title, path: relativePath, content: truncated, priority })
+ return true
} catch {
- // skip unreadable pages
+ return false
}
}
- // If search found nothing (new wiki or very specific question), read index + overview only
+ // P0: Title matches from search
+ for (const r of topSearchResults.filter((r) => r.titleMatch)) {
+ await tryAddPage(r.title, r.path, 0)
+ }
+ // P1: Content matches from search
+ for (const r of topSearchResults.filter((r) => !r.titleMatch)) {
+ await tryAddPage(r.title, r.path, 1)
+ }
+ // P2: Graph expansions (already sorted by relevance)
+ for (const exp of graphExpansions) {
+ await tryAddPage(exp.title, exp.path, 2)
+ }
+ // P3: Overview fallback if nothing found
if (relevantPages.length === 0) {
- try {
- const overview = await readFile(`${project.path}/wiki/overview.md`)
- relevantPages.push({ title: "Overview", path: "wiki/overview.md", content: overview })
- } catch {
- // no overview
- }
+ await tryAddPage("Overview", `${project.path}/wiki/overview.md`, 3)
}
// Build numbered wiki context so AI can cite by number
diff --git a/src/lib/graph-relevance.ts b/src/lib/graph-relevance.ts
new file mode 100644
index 00000000..d2821573
--- /dev/null
+++ b/src/lib/graph-relevance.ts
@@ -0,0 +1,311 @@
+import { readFile, listDirectory } from "@/commands/fs"
+import type { FileNode } from "@/types/wiki"
+
+// ---------------------------------------------------------------------------
+// Types
+// ---------------------------------------------------------------------------
+
+export interface RetrievalNode {
+ readonly id: string
+ readonly title: string
+ readonly type: string
+ readonly path: string
+ readonly sources: readonly string[]
+ readonly outLinks: ReadonlySet<string>
+ readonly inLinks: ReadonlySet<string>
+}
+
+export interface RetrievalGraph {
+ readonly nodes: ReadonlyMap<string, RetrievalNode>
+ readonly dataVersion: number
+}
+
+// ---------------------------------------------------------------------------
+// Constants
+// ---------------------------------------------------------------------------
+
+const WIKILINK_REGEX = /\[\[([^\]|]+?)(?:\|[^\]]+?)?\]\]/g
+
+const WEIGHTS = {
+ directLink: 3.0,
+ sourceOverlap: 4.0,
+ commonNeighbor: 1.5,
+ typeAffinity: 1.0,
+} as const
+
+const TYPE_AFFINITY: Record<string, Record<string, number>> = {
+ entity: { concept: 1.2, entity: 0.8, source: 1.0, synthesis: 1.0, query: 0.8 },
+ concept: { entity: 1.2, concept: 0.8, source: 1.0, synthesis: 1.2, query: 1.0 },
+ source: { entity: 1.0, concept: 1.0, source: 0.5, query: 0.8, synthesis: 1.0 },
+ query: { concept: 1.0, entity: 0.8, synthesis: 1.0, source: 0.8, query: 0.5 },
+ synthesis: { concept: 1.2, entity: 1.0, source: 1.0, query: 1.0, synthesis: 0.8 },
+}
+
+// ---------------------------------------------------------------------------
+// Module-level cache
+// ---------------------------------------------------------------------------
+
+let cachedGraph: RetrievalGraph | null = null
+
+// ---------------------------------------------------------------------------
+// Helpers (pure)
+// ---------------------------------------------------------------------------
+
+function flattenMdFiles(nodes: readonly FileNode[]): FileNode[] {
+ const files: FileNode[] = []
+ for (const node of nodes) {
+ if (node.is_dir && node.children) {
+ files.push(...flattenMdFiles(node.children))
+ } else if (!node.is_dir && node.name.endsWith(".md")) {
+ files.push(node)
+ }
+ }
+ return files
+}
+
+function fileNameToId(fileName: string): string {
+ return fileName.replace(/\.md$/, "")
+}
+
+function extractFrontmatter(content: string): { title: string; type: string; sources: string[] } {
+ const fmMatch = content.match(/^---\n([\s\S]*?)\n---/)
+ const fm = fmMatch ? fmMatch[1] : ""
+
+ const titleMatch = fm.match(/^title:\s*["']?(.+?)["']?\s*$/m)
+ const typeMatch = fm.match(/^type:\s*["']?(.+?)["']?\s*$/m)
+
+ // Parse sources array from YAML frontmatter
+ const sources: string[] = []
+ const sourcesBlockMatch = fm.match(/^sources:\s*\n((?:\s+-\s+.+\n?)*)/m)
+ if (sourcesBlockMatch) {
+ const lines = sourcesBlockMatch[1].split("\n")
+ for (const line of lines) {
+ const itemMatch = line.match(/^\s+-\s+["']?(.+?)["']?\s*$/)
+ if (itemMatch) {
+ sources.push(itemMatch[1])
+ }
+ }
+ } else {
+ // Single-line: sources: ["a.pdf", "b.pdf"] or sources: [a.pdf]
+ const inlineMatch = fm.match(/^sources:\s*\[([^\]]*)\]/m)
+ if (inlineMatch) {
+ const items = inlineMatch[1].split(",")
+ for (const item of items) {
+ const trimmed = item.trim().replace(/^["']|["']$/g, "")
+ if (trimmed) sources.push(trimmed)
+ }
+ }
+ }
+
+ let title = titleMatch ? titleMatch[1].trim() : ""
+ if (!title) {
+ const headingMatch = content.match(/^#\s+(.+)$/m)
+ title = headingMatch ? headingMatch[1].trim() : ""
+ }
+
+ return {
+ title,
+ type: typeMatch ? typeMatch[1].trim().toLowerCase() : "other",
+ sources,
+ }
+}
+
+function extractWikilinks(content: string): string[] {
+ const links: string[] = []
+ const regex = new RegExp(WIKILINK_REGEX.source, "g")
+ let match: RegExpExecArray | null
+ while ((match = regex.exec(content)) !== null) {
+ links.push(match[1].trim())
+ }
+ return links
+}
+
+function resolveTarget(
+ raw: string,
+ nodeIds: ReadonlySet<string>,
+): string | null {
+ if (nodeIds.has(raw)) return raw
+
+ const normalized = raw.toLowerCase().replace(/\s+/g, "-")
+ for (const id of nodeIds) {
+ const idLower = id.toLowerCase()
+ if (idLower === normalized) return id
+ if (idLower === raw.toLowerCase()) return id
+ if (idLower.replace(/\s+/g, "-") === normalized) return id
+ }
+ return null
+}
+
+function getNeighbors(node: RetrievalNode): ReadonlySet<string> {
+ const neighbors = new Set<string>()
+ for (const id of node.outLinks) neighbors.add(id)
+ for (const id of node.inLinks) neighbors.add(id)
+ return neighbors
+}
+
+function getNodeDegree(node: RetrievalNode): number {
+ return node.outLinks.size + node.inLinks.size
+}
+
+// ---------------------------------------------------------------------------
+// Core API
+// ---------------------------------------------------------------------------
+
+export async function buildRetrievalGraph(
+ projectPath: string,
+ dataVersion: number = 0,
+): Promise<RetrievalGraph> {
+ // Return cached if version matches
+ if (cachedGraph !== null && cachedGraph.dataVersion === dataVersion) {
+ return cachedGraph
+ }
+
+ const wikiRoot = `${projectPath}/wiki`
+ let tree: FileNode[]
+ try {
+ tree = await listDirectory(wikiRoot)
+ } catch {
+ const emptyGraph: RetrievalGraph = { nodes: new Map(), dataVersion }
+ cachedGraph = emptyGraph
+ return emptyGraph
+ }
+
+ const mdFiles = flattenMdFiles(tree)
+
+ // First pass: read all files and build raw node data
+ const rawNodes: Array<{
+ id: string
+ title: string
+ type: string
+ path: string
+ sources: string[]
+ rawLinks: string[]
+ fileName: string
+ }> = []
+
+ for (const file of mdFiles) {
+ const id = fileNameToId(file.name)
+ let content = ""
+ try {
+ content = await readFile(file.path)
+ } catch {
+ continue
+ }
+
+ const fm = extractFrontmatter(content)
+ rawNodes.push({
+ id,
+ title: fm.title || file.name.replace(/\.md$/, "").replace(/-/g, " "),
+ type: fm.type,
+ path: file.path,
+ sources: fm.sources,
+ rawLinks: extractWikilinks(content),
+ fileName: file.name,
+ })
+ }
+
+ const nodeIds = new Set(rawNodes.map((n) => n.id))
+
+ // Second pass: resolve links and build graph nodes
+ const outLinksMap = new Map<string, Set<string>>()
+ const inLinksMap = new Map<string, Set<string>>()
+
+ for (const id of nodeIds) {
+ outLinksMap.set(id, new Set())
+ inLinksMap.set(id, new Set())
+ }
+
+ for (const raw of rawNodes) {
+ for (const linkTarget of raw.rawLinks) {
+ const resolvedId = resolveTarget(linkTarget, nodeIds)
+ if (resolvedId === null || resolvedId === raw.id) continue
+ outLinksMap.get(raw.id)!.add(resolvedId)
+ inLinksMap.get(resolvedId)!.add(raw.id)
+ }
+ }
+
+ // Build immutable nodes map
+ const nodes = new Map<string, RetrievalNode>()
+ for (const raw of rawNodes) {
+ nodes.set(raw.id, {
+ id: raw.id,
+ title: raw.title,
+ type: raw.type,
+ path: raw.path,
+ sources: Object.freeze([...raw.sources]),
+ outLinks: Object.freeze(outLinksMap.get(raw.id) ?? new Set()),
+ inLinks: Object.freeze(inLinksMap.get(raw.id) ?? new Set()),
+ })
+ }
+
+ const graph: RetrievalGraph = { nodes, dataVersion }
+ cachedGraph = graph
+ return graph
+}
+
+export function calculateRelevance(
+ nodeA: RetrievalNode,
+ nodeB: RetrievalNode,
+ graph: RetrievalGraph,
+): number {
+ if (nodeA.id === nodeB.id) return 0
+
+ // Signal 1: Direct links (weight 3.0)
+ const forwardLinks = nodeA.outLinks.has(nodeB.id) ? 1 : 0
+ const backwardLinks = nodeB.outLinks.has(nodeA.id) ? 1 : 0
+ const directLinkScore = (forwardLinks + backwardLinks) * WEIGHTS.directLink
+
+ // Signal 2: Source overlap (weight 4.0)
+ const sourcesA = new Set(nodeA.sources)
+ let sharedSourceCount = 0
+ for (const src of nodeB.sources) {
+ if (sourcesA.has(src)) sharedSourceCount += 1
+ }
+ const sourceOverlapScore = sharedSourceCount * WEIGHTS.sourceOverlap
+
+ // Signal 3: Common neighbors - Adamic-Adar (weight 1.5)
+ const neighborsA = getNeighbors(nodeA)
+ const neighborsB = getNeighbors(nodeB)
+ let adamicAdar = 0
+ for (const neighborId of neighborsA) {
+ if (neighborsB.has(neighborId)) {
+ const neighbor = graph.nodes.get(neighborId)
+ if (neighbor) {
+ const degree = getNodeDegree(neighbor)
+ adamicAdar += 1 / Math.log(Math.max(degree, 2))
+ }
+ }
+ }
+ const commonNeighborScore = adamicAdar * WEIGHTS.commonNeighbor
+
+ // Signal 4: Type affinity (weight 1.0)
+ const affinityMap = TYPE_AFFINITY[nodeA.type]
+ const typeAffinityScore = (affinityMap?.[nodeB.type] ?? 0.5) * WEIGHTS.typeAffinity
+
+ return directLinkScore + sourceOverlapScore + commonNeighborScore + typeAffinityScore
+}
+
+export function getRelatedNodes(
+ nodeId: string,
+ graph: RetrievalGraph,
+ limit: number = 5,
+): ReadonlyArray<{ node: RetrievalNode; relevance: number }> {
+ const sourceNode = graph.nodes.get(nodeId)
+ if (!sourceNode) return []
+
+ const scored: Array<{ node: RetrievalNode; relevance: number }> = []
+ for (const [id, node] of graph.nodes) {
+ if (id === nodeId) continue
+ const relevance = calculateRelevance(sourceNode, node, graph)
+ if (relevance > 0) {
+ scored.push({ node, relevance })
+ }
+ }
+
+ scored.sort((a, b) => b.relevance - a.relevance)
+ return scored.slice(0, limit)
+}
+
+export function clearGraphCache(): void {
+ cachedGraph = null
+}
diff --git a/src/lib/search.ts b/src/lib/search.ts
index 1e14f8c7..68b7fd55 100644
--- a/src/lib/search.ts
+++ b/src/lib/search.ts
@@ -6,10 +6,37 @@ export interface SearchResult {
title: string
snippet: string
titleMatch: boolean
+ score: number
}
const MAX_RESULTS = 20
const SNIPPET_CONTEXT = 80
+const TITLE_MATCH_BONUS = 10
+
+const STOP_WORDS = new Set([
+ "的", "是", "了", "什么", "在", "有", "和", "与", "对", "从",
+ "the", "is", "a", "an", "what", "how", "are", "was", "were",
+ "do", "does", "did", "be", "been", "being", "have", "has", "had",
+ "it", "its", "in", "on", "at", "to", "for", "of", "with", "by",
+ "this", "that", "these", "those",
+])
+
+export function tokenizeQuery(query: string): string[] {
+ return query
+ .toLowerCase()
+ .split(/[\s,,。!?、;:""''()()\-_/\\]+/)
+ .filter((t) => t.length > 1)
+ .filter((t) => !STOP_WORDS.has(t))
+}
+
+function tokenMatchScore(text: string, tokens: readonly string[]): number {
+ const lower = text.toLowerCase()
+ let score = 0
+ for (const token of tokens) {
+ if (lower.includes(token)) score += 1
+ }
+ return score
+}
function flattenMdFiles(nodes: FileNode[]): FileNode[] {
const files: FileNode[] = []
@@ -56,15 +83,16 @@ export async function searchWiki(
): Promise<SearchResult[]> {
if (!query.trim()) return []
- const lowerQuery = query.toLowerCase()
- const titleMatches: SearchResult[] = []
- const contentMatches: SearchResult[] = []
+ const tokens = tokenizeQuery(query)
+ // Fallback: if all tokens were filtered out, use the trimmed query as a single token
+ const effectiveTokens = tokens.length > 0 ? tokens : [query.trim().toLowerCase()]
+ const results: SearchResult[] = []
// Search wiki pages
try {
const wikiTree = await listDirectory(`${projectPath}/wiki`)
const wikiFiles = flattenMdFiles(wikiTree)
- await searchFiles(wikiFiles, lowerQuery, query, titleMatches, contentMatches)
+ await searchFiles(wikiFiles, effectiveTokens, query, results)
} catch {
// no wiki directory
}
@@ -73,20 +101,21 @@ export async function searchWiki(
try {
const rawTree = await listDirectory(`${projectPath}/raw/sources`)
const rawFiles = flattenAllFiles(rawTree)
- await searchFiles(rawFiles, lowerQuery, query, titleMatches, contentMatches)
+ await searchFiles(rawFiles, effectiveTokens, query, results)
} catch {
// no raw sources
}
- return [...titleMatches, ...contentMatches].slice(0, MAX_RESULTS)
+ // Sort by score descending
+ results.sort((a, b) => b.score - a.score)
+ return results.slice(0, MAX_RESULTS)
}
async function searchFiles(
files: FileNode[],
- lowerQuery: string,
+ tokens: readonly string[],
query: string,
- titleMatches: SearchResult[],
- contentMatches: SearchResult[],
+ results: SearchResult[],
): Promise<void> {
for (const file of files) {
let content = ""
@@ -97,29 +126,28 @@ async function searchFiles(
}
const title = extractTitle(content, file.name)
- const titleLower = title.toLowerCase()
- const fileNameLower = file.name.toLowerCase()
- const contentLower = content.toLowerCase()
+ const titleText = `${title} ${file.name}`
- const matchesTitle =
- titleLower.includes(lowerQuery) || fileNameLower.includes(lowerQuery)
- const matchesContent = contentLower.includes(lowerQuery)
+ const titleScore = tokenMatchScore(titleText, tokens)
+ const contentScore = tokenMatchScore(content, tokens)
- if (!matchesTitle && !matchesContent) continue
+ if (titleScore === 0 && contentScore === 0) continue
- const snippet = buildSnippet(content, query)
- const result: SearchResult = {
+ const isTitleMatch = titleScore > 0
+ const score = contentScore + (isTitleMatch ? TITLE_MATCH_BONUS : 0)
+
+ const firstMatchingToken = tokens.find((t) =>
+ content.toLowerCase().includes(t),
+ ) ?? query
+ const snippet = buildSnippet(content, firstMatchingToken)
+
+ results.push({
path: file.path,
title,
snippet,
- titleMatch: matchesTitle,
- }
-
- if (matchesTitle) {
- titleMatches.push(result)
- } else {
- contentMatches.push(result)
- }
+ titleMatch: isTitleMatch,
+ score,
+ })
}
}