LLM Wiki is a cross-platform desktop application that turns your documents into an organized, interlinked knowledge base — automatically. Instead of traditional RAG (retrieve-and-answer from scratch every time), the LLM incrementally builds and maintains a persistent wiki from your sources。
TypeScript
feat: add ingest orchestrator with LLM-driven wiki page generation
nash_su09fd6eb98ae1
diff --git a/src/lib/ingest.ts b/src/lib/ingest.tsnew file mode 100644index 00000000..8268779d--- /dev/null+++ b/src/lib/ingest.ts@@ -0,0 +1,196 @@+import { readFile, writeFile } from "@/commands/fs"+import { streamChat } from "@/lib/llm-client"+import type { LlmConfig } from "@/stores/wiki-store"+import { useChatStore } from "@/stores/chat-store"++const FILE_BLOCK_REGEX = /---FILE:\s*([^\n-]+?)\s*---\n([\s\S]*?)---END FILE---/g++function getStore() {+ return useChatStore.getState()+}++async function tryReadFile(path: string): Promise<string> {+ try {+ return await readFile(path)+ } catch {+ return ""+ }+}++export async function startIngest(+ projectPath: string,+ sourcePath: string,+ llmConfig: LlmConfig,+ signal?: AbortSignal,+): Promise<void> {+ const store = getStore()+ store.setMode("ingest")+ store.setIngestSource(sourcePath)+ store.clearMessages()+ store.setStreaming(false)++ const [sourceContent, schema, purpose, index] = await Promise.all([+ tryReadFile(sourcePath),+ tryReadFile(`${projectPath}/wiki/schema.md`),+ tryReadFile(`${projectPath}/wiki/purpose.md`),+ tryReadFile(`${projectPath}/wiki/index.md`),+ ])++ const fileName = sourcePath.split("/").pop() ?? sourcePath++ const systemPrompt = [+ "You are a knowledgeable assistant helping to build a wiki from source documents.",+ "",+ purpose ? `## Wiki Purpose\n${purpose}` : "",+ schema ? `## Wiki Schema\n${schema}` : "",+ index ? `## Current Wiki Index\n${index}` : "",+ ]+ .filter(Boolean)+ .join("\n\n")++ const userMessage = [+ `I'm ingesting the following source file into my wiki: **${fileName}**`,+ "",+ "Please read it carefully and present the key takeaways, important concepts, and information that would be valuable to capture in the wiki. Highlight anything that relates to the wiki's purpose and schema.",+ "",+ "---",+ `**File: ${fileName}**`,+ "```",+ sourceContent || "(empty file)",+ "```",+ ].join("\n")++ store.addMessage("user", userMessage)+ store.setStreaming(true)++ let accumulated = ""++ await streamChat(+ llmConfig,+ [+ { role: "system", content: systemPrompt },+ { role: "user", content: userMessage },+ ],+ {+ onToken: (token) => {+ accumulated += token+ getStore().appendStreamToken(token)+ },+ onDone: () => {+ getStore().finalizeStream(accumulated)+ },+ onError: (err) => {+ getStore().finalizeStream(`Error during ingest: ${err.message}`)+ },+ },+ signal,+ )+}++export async function executeIngestWrites(+ projectPath: string,+ llmConfig: LlmConfig,+ userGuidance?: string,+ signal?: AbortSignal,+): Promise<string[]> {+ const store = getStore()++ const [schema, index] = await Promise.all([+ tryReadFile(`${projectPath}/wiki/schema.md`),+ tryReadFile(`${projectPath}/wiki/index.md`),+ ])++ const conversationHistory = store.messages+ .filter((m) => m.role !== "system")+ .map((m) => ({ role: m.role as "user" | "assistant", content: m.content }))++ const writePrompt = [+ "Based on our discussion, please generate the wiki files that should be created or updated.",+ "",+ userGuidance ? `Additional guidance: ${userGuidance}` : "",+ "",+ schema ? `## Wiki Schema\n${schema}` : "",+ index ? `## Current Wiki Index\n${index}` : "",+ "",+ "Output ONLY the file contents in this exact format for each file:",+ "```",+ "---FILE: wiki/path/to/file.md---",+ "(file content here)",+ "---END FILE---",+ "```",+ "",+ "For wiki/log.md, include a log entry to append. For all other files, output the complete file content.",+ "Use relative paths from the project root (e.g., wiki/sources/topic.md).",+ "Do not include any other text outside the FILE blocks.",+ ]+ .filter((line) => line !== undefined)+ .join("\n")++ conversationHistory.push({ role: "user", content: writePrompt })++ store.addMessage("user", writePrompt)+ store.setStreaming(true)++ let accumulated = ""++ const systemPrompt = [+ "You are a wiki generation assistant. Your task is to produce structured wiki file contents.",+ schema ? `## Wiki Schema\n${schema}` : "",+ ]+ .filter(Boolean)+ .join("\n\n")++ await streamChat(+ llmConfig,+ [{ role: "system", content: systemPrompt }, ...conversationHistory],+ {+ onToken: (token) => {+ accumulated += token+ getStore().appendStreamToken(token)+ },+ onDone: () => {+ getStore().finalizeStream(accumulated)+ },+ onError: (err) => {+ getStore().finalizeStream(`Error generating wiki files: ${err.message}`)+ },+ },+ signal,+ )++ const writtenPaths: string[] = []+ const matches = accumulated.matchAll(FILE_BLOCK_REGEX)++ for (const match of matches) {+ const relativePath = match[1].trim()+ const content = match[2]++ if (!relativePath) continue++ const fullPath = `${projectPath}/${relativePath}`++ try {+ if (relativePath === "wiki/log.md" || relativePath.endsWith("/log.md")) {+ const existing = await tryReadFile(fullPath)+ const appended = existing+ ? `${existing}\n\n${content.trim()}`+ : content.trim()+ await writeFile(fullPath, appended)+ } else {+ await writeFile(fullPath, content)+ }+ writtenPaths.push(fullPath)+ } catch (err) {+ console.error(`Failed to write ${fullPath}:`, err)+ }+ }++ if (writtenPaths.length > 0) {+ const fileList = writtenPaths.map((p) => `- ${p}`).join("\n")+ getStore().addMessage("system", `Files written to wiki:\n${fileList}`)+ } else {+ getStore().addMessage("system", "No files were written. The LLM response did not contain valid FILE blocks.")+ }++ return writtenPaths+}