LLM Wiki is a cross-platform desktop application that turns your documents into an organized, interlinked knowledge base — automatically. Instead of traditional RAG (retrieve-and-answer from scratch every time), the LLM incrementally builds and maintains a persistent wiki from your sources。
TypeScript
feat: graph relevance model + tokenized search + budget-controlled retrieval
nash_sucf58eacc5eba
diff --git a/src/components/chat/chat-panel.tsx b/src/components/chat/chat-panel.tsxindex a62c447c..63ae0217 100644--- a/src/components/chat/chat-panel.tsx+++ b/src/components/chat/chat-panel.tsx@@ -9,6 +9,7 @@ import { streamChat, type ChatMessage as LLMMessage } from "@/lib/llm-client"import { executeIngestWrites } from "@/lib/ingest"import { listDirectory, readFile, writeFile } from "@/commands/fs"import { searchWiki } from "@/lib/search"+import { buildRetrievalGraph, getRelatedNodes } from "@/lib/graph-relevance"import { useReviewStore } from "@/stores/review-store"import type { FileNode } from "@/types/wiki"@@ -47,41 +48,90 @@ export function ChatPanel() {addMessage("user", text)setStreaming(true)- // Build system prompt with wiki context: search for relevant pages and include their content+ // Build system prompt with wiki context using graph-enhanced retrievalconst systemMessages: LLMMessage[] = []if (project) {+ const dataVersion = useWikiStore.getState().dataVersionconst [index, purpose] = await Promise.all([readFile(`${project.path}/wiki/index.md`).catch(() => ""),readFile(`${project.path}/purpose.md`).catch(() => ""),])- // Search wiki + raw sources for pages relevant to the current question only+ // --- Phase 1: Tokenized search -> top 10 ---const searchResults = await searchWiki(project.path, text)-- // Read the full content of top relevant pages (max 5 to keep context focused)- const relevantPages: { title: string; path: string; content: string }[] = []- for (const result of searchResults.slice(0, 5)) {+ const topSearchResults = searchResults.slice(0, 10)++ // --- Phase 2: Graph 1-level expansion ---+ const graph = await buildRetrievalGraph(project.path, dataVersion)+ const expandedIds = new Set<string>()+ const searchHitPaths = new Set(topSearchResults.map((r) => r.path))+ const graphExpansions: { title: string; path: string; relevance: number }[] = []++ for (const result of topSearchResults) {+ // Derive node ID from file path (kebab-case filename without .md)+ const fileName = result.path.split("/").pop() ?? ""+ const nodeId = fileName.replace(/\.md$/, "")+ const related = getRelatedNodes(nodeId, graph, 3)+ for (const { node, relevance } of related) {+ if (relevance < 2.0) continue+ if (searchHitPaths.has(node.path)) continue+ if (expandedIds.has(node.id)) continue+ expandedIds.add(node.id)+ graphExpansions.push({+ title: node.title,+ path: node.path,+ relevance,+ })+ }+ }+ // Sort graph expansions by relevance descending+ graphExpansions.sort((a, b) => b.relevance - a.relevance)++ // --- Phase 3 & 4: Budget control with priority queue ---+ const TOTAL_BUDGET = 30_000+ const MAX_PAGE_SIZE = 8_000+ let usedChars = 0++ type PageEntry = { title: string; path: string; content: string; priority: number }+ const relevantPages: PageEntry[] = []++ // Helper to add a page within budget+ const tryAddPage = async (+ title: string,+ filePath: string,+ priority: number,+ ): Promise<boolean> => {+ if (usedChars >= TOTAL_BUDGET) return falsetry {- const content = await readFile(result.path)- const relativePath = result.path.replace(project.path + "/", "")- // Truncate large files- const truncated = content.length > 15000- ? content.slice(0, 15000) + "\n\n[...truncated...]"- : content- relevantPages.push({ title: result.title, path: relativePath, content: truncated })+ const raw = await readFile(filePath)+ const relativePath = filePath.replace(project.path + "/", "")+ const truncated = raw.length > MAX_PAGE_SIZE+ ? raw.slice(0, MAX_PAGE_SIZE) + "\n\n[...truncated...]"+ : raw+ if (usedChars + truncated.length > TOTAL_BUDGET) return false+ usedChars += truncated.length+ relevantPages.push({ title, path: relativePath, content: truncated, priority })+ return true} catch {- // skip unreadable pages+ return false}}- // If search found nothing (new wiki or very specific question), read index + overview only+ // P0: Title matches from search+ for (const r of topSearchResults.filter((r) => r.titleMatch)) {+ await tryAddPage(r.title, r.path, 0)+ }+ // P1: Content matches from search+ for (const r of topSearchResults.filter((r) => !r.titleMatch)) {+ await tryAddPage(r.title, r.path, 1)+ }+ // P2: Graph expansions (already sorted by relevance)+ for (const exp of graphExpansions) {+ await tryAddPage(exp.title, exp.path, 2)+ }+ // P3: Overview fallback if nothing foundif (relevantPages.length === 0) {- try {- const overview = await readFile(`${project.path}/wiki/overview.md`)- relevantPages.push({ title: "Overview", path: "wiki/overview.md", content: overview })- } catch {- // no overview- }+ await tryAddPage("Overview", `${project.path}/wiki/overview.md`, 3)}// Build numbered wiki context so AI can cite by numberdiff --git a/src/lib/graph-relevance.ts b/src/lib/graph-relevance.tsnew file mode 100644index 00000000..d2821573--- /dev/null+++ b/src/lib/graph-relevance.ts@@ -0,0 +1,311 @@+import { readFile, listDirectory } from "@/commands/fs"+import type { FileNode } from "@/types/wiki"++// ---------------------------------------------------------------------------+// Types+// ---------------------------------------------------------------------------++export interface RetrievalNode {+ readonly id: string+ readonly title: string+ readonly type: string+ readonly path: string+ readonly sources: readonly string[]+ readonly outLinks: ReadonlySet<string>+ readonly inLinks: ReadonlySet<string>+}++export interface RetrievalGraph {+ readonly nodes: ReadonlyMap<string, RetrievalNode>+ readonly dataVersion: number+}++// ---------------------------------------------------------------------------+// Constants+// ---------------------------------------------------------------------------++const WIKILINK_REGEX = /\[\[([^\]|]+?)(?:\|[^\]]+?)?\]\]/g++const WEIGHTS = {+ directLink: 3.0,+ sourceOverlap: 4.0,+ commonNeighbor: 1.5,+ typeAffinity: 1.0,+} as const++const TYPE_AFFINITY: Record<string, Record<string, number>> = {+ entity: { concept: 1.2, entity: 0.8, source: 1.0, synthesis: 1.0, query: 0.8 },+ concept: { entity: 1.2, concept: 0.8, source: 1.0, synthesis: 1.2, query: 1.0 },+ source: { entity: 1.0, concept: 1.0, source: 0.5, query: 0.8, synthesis: 1.0 },+ query: { concept: 1.0, entity: 0.8, synthesis: 1.0, source: 0.8, query: 0.5 },+ synthesis: { concept: 1.2, entity: 1.0, source: 1.0, query: 1.0, synthesis: 0.8 },+}++// ---------------------------------------------------------------------------+// Module-level cache+// ---------------------------------------------------------------------------++let cachedGraph: RetrievalGraph | null = null++// ---------------------------------------------------------------------------+// Helpers (pure)+// ---------------------------------------------------------------------------++function flattenMdFiles(nodes: readonly FileNode[]): FileNode[] {+ const files: FileNode[] = []+ for (const node of nodes) {+ if (node.is_dir && node.children) {+ files.push(...flattenMdFiles(node.children))+ } else if (!node.is_dir && node.name.endsWith(".md")) {+ files.push(node)+ }+ }+ return files+}++function fileNameToId(fileName: string): string {+ return fileName.replace(/\.md$/, "")+}++function extractFrontmatter(content: string): { title: string; type: string; sources: string[] } {+ const fmMatch = content.match(/^---\n([\s\S]*?)\n---/)+ const fm = fmMatch ? fmMatch[1] : ""++ const titleMatch = fm.match(/^title:\s*["']?(.+?)["']?\s*$/m)+ const typeMatch = fm.match(/^type:\s*["']?(.+?)["']?\s*$/m)++ // Parse sources array from YAML frontmatter+ const sources: string[] = []+ const sourcesBlockMatch = fm.match(/^sources:\s*\n((?:\s+-\s+.+\n?)*)/m)+ if (sourcesBlockMatch) {+ const lines = sourcesBlockMatch[1].split("\n")+ for (const line of lines) {+ const itemMatch = line.match(/^\s+-\s+["']?(.+?)["']?\s*$/)+ if (itemMatch) {+ sources.push(itemMatch[1])+ }+ }+ } else {+ // Single-line: sources: ["a.pdf", "b.pdf"] or sources: [a.pdf]+ const inlineMatch = fm.match(/^sources:\s*\[([^\]]*)\]/m)+ if (inlineMatch) {+ const items = inlineMatch[1].split(",")+ for (const item of items) {+ const trimmed = item.trim().replace(/^["']|["']$/g, "")+ if (trimmed) sources.push(trimmed)+ }+ }+ }++ let title = titleMatch ? titleMatch[1].trim() : ""+ if (!title) {+ const headingMatch = content.match(/^#\s+(.+)$/m)+ title = headingMatch ? headingMatch[1].trim() : ""+ }++ return {+ title,+ type: typeMatch ? typeMatch[1].trim().toLowerCase() : "other",+ sources,+ }+}++function extractWikilinks(content: string): string[] {+ const links: string[] = []+ const regex = new RegExp(WIKILINK_REGEX.source, "g")+ let match: RegExpExecArray | null+ while ((match = regex.exec(content)) !== null) {+ links.push(match[1].trim())+ }+ return links+}++function resolveTarget(+ raw: string,+ nodeIds: ReadonlySet<string>,+): string | null {+ if (nodeIds.has(raw)) return raw++ const normalized = raw.toLowerCase().replace(/\s+/g, "-")+ for (const id of nodeIds) {+ const idLower = id.toLowerCase()+ if (idLower === normalized) return id+ if (idLower === raw.toLowerCase()) return id+ if (idLower.replace(/\s+/g, "-") === normalized) return id+ }+ return null+}++function getNeighbors(node: RetrievalNode): ReadonlySet<string> {+ const neighbors = new Set<string>()+ for (const id of node.outLinks) neighbors.add(id)+ for (const id of node.inLinks) neighbors.add(id)+ return neighbors+}++function getNodeDegree(node: RetrievalNode): number {+ return node.outLinks.size + node.inLinks.size+}++// ---------------------------------------------------------------------------+// Core API+// ---------------------------------------------------------------------------++export async function buildRetrievalGraph(+ projectPath: string,+ dataVersion: number = 0,+): Promise<RetrievalGraph> {+ // Return cached if version matches+ if (cachedGraph !== null && cachedGraph.dataVersion === dataVersion) {+ return cachedGraph+ }++ const wikiRoot = `${projectPath}/wiki`+ let tree: FileNode[]+ try {+ tree = await listDirectory(wikiRoot)+ } catch {+ const emptyGraph: RetrievalGraph = { nodes: new Map(), dataVersion }+ cachedGraph = emptyGraph+ return emptyGraph+ }++ const mdFiles = flattenMdFiles(tree)++ // First pass: read all files and build raw node data+ const rawNodes: Array<{+ id: string+ title: string+ type: string+ path: string+ sources: string[]+ rawLinks: string[]+ fileName: string+ }> = []++ for (const file of mdFiles) {+ const id = fileNameToId(file.name)+ let content = ""+ try {+ content = await readFile(file.path)+ } catch {+ continue+ }++ const fm = extractFrontmatter(content)+ rawNodes.push({+ id,+ title: fm.title || file.name.replace(/\.md$/, "").replace(/-/g, " "),+ type: fm.type,+ path: file.path,+ sources: fm.sources,+ rawLinks: extractWikilinks(content),+ fileName: file.name,+ })+ }++ const nodeIds = new Set(rawNodes.map((n) => n.id))++ // Second pass: resolve links and build graph nodes+ const outLinksMap = new Map<string, Set<string>>()+ const inLinksMap = new Map<string, Set<string>>()++ for (const id of nodeIds) {+ outLinksMap.set(id, new Set())+ inLinksMap.set(id, new Set())+ }++ for (const raw of rawNodes) {+ for (const linkTarget of raw.rawLinks) {+ const resolvedId = resolveTarget(linkTarget, nodeIds)+ if (resolvedId === null || resolvedId === raw.id) continue+ outLinksMap.get(raw.id)!.add(resolvedId)+ inLinksMap.get(resolvedId)!.add(raw.id)+ }+ }++ // Build immutable nodes map+ const nodes = new Map<string, RetrievalNode>()+ for (const raw of rawNodes) {+ nodes.set(raw.id, {+ id: raw.id,+ title: raw.title,+ type: raw.type,+ path: raw.path,+ sources: Object.freeze([...raw.sources]),+ outLinks: Object.freeze(outLinksMap.get(raw.id) ?? new Set()),+ inLinks: Object.freeze(inLinksMap.get(raw.id) ?? new Set()),+ })+ }++ const graph: RetrievalGraph = { nodes, dataVersion }+ cachedGraph = graph+ return graph+}++export function calculateRelevance(+ nodeA: RetrievalNode,+ nodeB: RetrievalNode,+ graph: RetrievalGraph,+): number {+ if (nodeA.id === nodeB.id) return 0++ // Signal 1: Direct links (weight 3.0)+ const forwardLinks = nodeA.outLinks.has(nodeB.id) ? 1 : 0+ const backwardLinks = nodeB.outLinks.has(nodeA.id) ? 1 : 0+ const directLinkScore = (forwardLinks + backwardLinks) * WEIGHTS.directLink++ // Signal 2: Source overlap (weight 4.0)+ const sourcesA = new Set(nodeA.sources)+ let sharedSourceCount = 0+ for (const src of nodeB.sources) {+ if (sourcesA.has(src)) sharedSourceCount += 1+ }+ const sourceOverlapScore = sharedSourceCount * WEIGHTS.sourceOverlap++ // Signal 3: Common neighbors - Adamic-Adar (weight 1.5)+ const neighborsA = getNeighbors(nodeA)+ const neighborsB = getNeighbors(nodeB)+ let adamicAdar = 0+ for (const neighborId of neighborsA) {+ if (neighborsB.has(neighborId)) {+ const neighbor = graph.nodes.get(neighborId)+ if (neighbor) {+ const degree = getNodeDegree(neighbor)+ adamicAdar += 1 / Math.log(Math.max(degree, 2))+ }+ }+ }+ const commonNeighborScore = adamicAdar * WEIGHTS.commonNeighbor++ // Signal 4: Type affinity (weight 1.0)+ const affinityMap = TYPE_AFFINITY[nodeA.type]+ const typeAffinityScore = (affinityMap?.[nodeB.type] ?? 0.5) * WEIGHTS.typeAffinity++ return directLinkScore + sourceOverlapScore + commonNeighborScore + typeAffinityScore+}++export function getRelatedNodes(+ nodeId: string,+ graph: RetrievalGraph,+ limit: number = 5,+): ReadonlyArray<{ node: RetrievalNode; relevance: number }> {+ const sourceNode = graph.nodes.get(nodeId)+ if (!sourceNode) return []++ const scored: Array<{ node: RetrievalNode; relevance: number }> = []+ for (const [id, node] of graph.nodes) {+ if (id === nodeId) continue+ const relevance = calculateRelevance(sourceNode, node, graph)+ if (relevance > 0) {+ scored.push({ node, relevance })+ }+ }++ scored.sort((a, b) => b.relevance - a.relevance)+ return scored.slice(0, limit)+}++export function clearGraphCache(): void {+ cachedGraph = null+}diff --git a/src/lib/search.ts b/src/lib/search.tsindex 1e14f8c7..68b7fd55 100644--- a/src/lib/search.ts+++ b/src/lib/search.ts@@ -6,10 +6,37 @@ export interface SearchResult {title: stringsnippet: stringtitleMatch: boolean+ score: number}const MAX_RESULTS = 20const SNIPPET_CONTEXT = 80+const TITLE_MATCH_BONUS = 10++const STOP_WORDS = new Set([+ "的", "是", "了", "什么", "在", "有", "和", "与", "对", "从",+ "the", "is", "a", "an", "what", "how", "are", "was", "were",+ "do", "does", "did", "be", "been", "being", "have", "has", "had",+ "it", "its", "in", "on", "at", "to", "for", "of", "with", "by",+ "this", "that", "these", "those",+])++export function tokenizeQuery(query: string): string[] {+ return query+ .toLowerCase()+ .split(/[\s,,。!?、;:""''()()\-_/\\]+/)+ .filter((t) => t.length > 1)+ .filter((t) => !STOP_WORDS.has(t))+}++function tokenMatchScore(text: string, tokens: readonly string[]): number {+ const lower = text.toLowerCase()+ let score = 0+ for (const token of tokens) {+ if (lower.includes(token)) score += 1+ }+ return score+}function flattenMdFiles(nodes: FileNode[]): FileNode[] {const files: FileNode[] = []@@ -56,15 +83,16 @@ export async function searchWiki(): Promise<SearchResult[]> {if (!query.trim()) return []- const lowerQuery = query.toLowerCase()- const titleMatches: SearchResult[] = []- const contentMatches: SearchResult[] = []+ const tokens = tokenizeQuery(query)+ // Fallback: if all tokens were filtered out, use the trimmed query as a single token+ const effectiveTokens = tokens.length > 0 ? tokens : [query.trim().toLowerCase()]+ const results: SearchResult[] = []// Search wiki pagestry {const wikiTree = await listDirectory(`${projectPath}/wiki`)const wikiFiles = flattenMdFiles(wikiTree)- await searchFiles(wikiFiles, lowerQuery, query, titleMatches, contentMatches)+ await searchFiles(wikiFiles, effectiveTokens, query, results)} catch {// no wiki directory}@@ -73,20 +101,21 @@ export async function searchWiki(try {const rawTree = await listDirectory(`${projectPath}/raw/sources`)const rawFiles = flattenAllFiles(rawTree)- await searchFiles(rawFiles, lowerQuery, query, titleMatches, contentMatches)+ await searchFiles(rawFiles, effectiveTokens, query, results)} catch {// no raw sources}- return [...titleMatches, ...contentMatches].slice(0, MAX_RESULTS)+ // Sort by score descending+ results.sort((a, b) => b.score - a.score)+ return results.slice(0, MAX_RESULTS)}async function searchFiles(files: FileNode[],- lowerQuery: string,+ tokens: readonly string[],query: string,- titleMatches: SearchResult[],- contentMatches: SearchResult[],+ results: SearchResult[],): Promise<void> {for (const file of files) {let content = ""@@ -97,29 +126,28 @@ async function searchFiles(}const title = extractTitle(content, file.name)- const titleLower = title.toLowerCase()- const fileNameLower = file.name.toLowerCase()- const contentLower = content.toLowerCase()+ const titleText = `${title} ${file.name}`- const matchesTitle =- titleLower.includes(lowerQuery) || fileNameLower.includes(lowerQuery)- const matchesContent = contentLower.includes(lowerQuery)+ const titleScore = tokenMatchScore(titleText, tokens)+ const contentScore = tokenMatchScore(content, tokens)- if (!matchesTitle && !matchesContent) continue+ if (titleScore === 0 && contentScore === 0) continue- const snippet = buildSnippet(content, query)- const result: SearchResult = {+ const isTitleMatch = titleScore > 0+ const score = contentScore + (isTitleMatch ? TITLE_MATCH_BONUS : 0)++ const firstMatchingToken = tokens.find((t) =>+ content.toLowerCase().includes(t),+ ) ?? query+ const snippet = buildSnippet(content, firstMatchingToken)++ results.push({path: file.path,title,snippet,- titleMatch: matchesTitle,- }-- if (matchesTitle) {- titleMatches.push(result)- } else {- contentMatches.push(result)- }+ titleMatch: isTitleMatch,+ score,+ })}}