LLM Wiki is a cross-platform desktop application that turns your documents into an organized, interlinked knowledge base — automatically. Instead of traditional RAG (retrieve-and-answer from scratch every time), the LLM incrementally builds and maintains a persistent wiki from your sources。
TypeScript
import {
copyFile,
fileExists,
getFileMd5,
getFileModifiedTime,
getFileSize,
listDirectory,
readFile,
writeFileAtomic,
} from "@/commands/fs"
import type { FileNode, WikiProject } from "@/types/wiki"
import { isAbsolutePath, normalizePath } from "@/lib/path-utils"
import { useWikiStore } from "@/stores/wiki-store"
import type { ScheduledImportConfig, SourceWatchConfig } from "@/stores/wiki-store"
import {
loadScheduledImportConfig,
loadSourceWatchConfig,
saveScheduledImportConfig,
getRecentProjects,
} from "@/lib/project-store"
import {
isPathAllowedBySourceWatch,
normalizeSourceWatchConfig,
} from "@/lib/source-watch-config"
import {
deleteSourceFile,
enqueueSourceIngest,
folderContextForSourcePath,
isIngestableSourcePath,
} from "@/lib/source-lifecycle"
import { discardTasksForSources, enqueueInactiveProjectBatch } from "@/lib/ingest-queue"
import { useActivityStore } from "@/stores/activity-store"
import { refreshProjectFileTree } from "@/lib/project-file-tree-refresh"
interface ImportStat {
size: number
mtime: number
}
interface ImportDb {
files: Record<string, string>
/**
* Size + mtime (ms) per source path. Lets a scan reuse the recorded content
* hash for files that did not change instead of re-reading and re-hashing
* every file on every run. Optional and additive: databases written by
* older builds simply lack it and fall back to hashing once.
*/
statCache?: Record<string, ImportStat>
lastScan: number | null
}
interface ImportDbStore {
version: 1
/**
* Kept as a map for backward compatibility with early scheduled-import
* builds. The current UI supports one watched directory per project, so
* saveImportDb intentionally writes only the active directory key.
*/
directories: Record<string, ImportDb>
}
type ScanOptions = {
runId?: number
allowInactive?: boolean
sourceWatchConfig?: SourceWatchConfig
}
const EMPTY_DB: ImportDb = {
files: {},
statCache: {},
lastScan: null,
}
let scanTimer: ReturnType<typeof setInterval> | null = null
let scanning = false
let activeRunId = 0
const managedPathWarningKeys = new Set<string>()
const DB_PATH = ".llm-wiki/scheduled-import-db.json"
const LEGACY_DB_DIR = ".llm-wiki-imported"
const SCHEDULED_IMPORT_DIR = "scheduled-import"
const MAX_SCHEDULED_IMPORT_BYTES = 100 * 1024 * 1024
const SCHEDULED_IMPORT_CONFIG_EXTENSIONS = new Set(["json", "yaml", "yml", "xml"])
const RESERVED_WINDOWS_NAMES = new Set([
"con",
"prn",
"aux",
"nul",
"com1",
"com2",
"com3",
"com4",
"com5",
"com6",
"com7",
"com8",
"com9",
"lpt1",
"lpt2",
"lpt3",
"lpt4",
"lpt5",
"lpt6",
"lpt7",
"lpt8",
"lpt9",
])
function emptyStore(): ImportDbStore {
return { version: 1, directories: {} }
}
function dbFilePath(projectPath: string): string {
return `${normalizePath(projectPath)}/${DB_PATH}`
}
function dbDirectoryKey(importPath: string): string {
return caseFoldPath(normalizePath(importPath))
}
// Windows drive-letter and UNC paths are case-insensitive; fold them for
// comparison purposes only (never for the paths actually written to disk).
function caseFoldPath(normalized: string): string {
return /^[A-Za-z]:\//.test(normalized) || normalized.startsWith("//")
? normalized.toLowerCase()
: normalized
}
function cloneDb(db: ImportDb): ImportDb {
return {
files: { ...db.files },
statCache: { ...(db.statCache ?? {}) },
lastScan: db.lastScan,
}
}
function isPathInside(path: string, parent: string): boolean {
const normalizedPath = dbDirectoryKey(path)
const normalizedParent = dbDirectoryKey(parent).replace(/\/+$/, "")
return (
normalizedPath === normalizedParent ||
normalizedPath.startsWith(`${normalizedParent}/`)
)
}
function projectSubpath(projectPath: string, relPath: string): string {
return `${normalizePath(projectPath)}/${relPath}`
}
export function isProjectManagedScheduledImportPath(
projectPath: string,
importPath: string,
): boolean {
const project = normalizePath(projectPath).replace(/\/+$/, "")
const root = normalizePath(importPath).replace(/\/+$/, "")
return (
root === project ||
isPathInside(project, root) ||
isPathInside(root, projectSubpath(project, "raw")) ||
isPathInside(root, projectSubpath(project, "raw/sources")) ||
isPathInside(root, projectSubpath(project, "wiki")) ||
isPathInside(root, projectSubpath(project, ".llm-wiki"))
)
}
function notifyManagedScheduledImportPath(project: WikiProject, importRoot: string): void {
const key = `${project.id}:${normalizePath(importRoot)}`
if (managedPathWarningKeys.has(key)) return
managedPathWarningKeys.add(key)
useActivityStore.getState().addItem({
type: "ingest",
title: "Scheduled import skipped",
status: "error",
detail: `Scheduled import path is inside or contains the current LLM Wiki project: ${importRoot}. Choose an external folder; project sources are handled by source folder monitoring.`,
filesWritten: [],
})
}
function stableSuffix(input: string): string {
let hash = 2166136261
for (let i = 0; i < input.length; i += 1) {
hash ^= input.charCodeAt(i)
hash = Math.imul(hash, 16777619)
}
return (hash >>> 0).toString(36).slice(0, 6)
}
function sanitizePathSegment(segment: string): string {
let value = segment
.replace(/[<>:"|?*\x00-\x1F]/g, "_")
.replace(/[. ]+$/g, "")
.trim()
if (!value) {
value = "_"
}
const stem = value.split(".")[0]?.toLowerCase() ?? value.toLowerCase()
if (RESERVED_WINDOWS_NAMES.has(stem)) {
value = `_${value}`
}
return value
}
function appendSuffixToFileName(fileName: string, suffix: string): string {
const dot = fileName.lastIndexOf(".")
if (dot > 0) {
return `${fileName.slice(0, dot)}-${suffix}${fileName.slice(dot)}`
}
return `${fileName}-${suffix}`
}
function safeRelativePath(path: string): string {
const normalized = normalizePath(path)
const parts = normalized
.split("/")
.filter((part) => part && part !== "." && part !== "..")
const safeParts = parts.map(sanitizePathSegment)
if (safeParts.length === 0) {
return "_"
}
const safePath = safeParts.join("/")
if (safePath !== parts.join("/")) {
const last = safeParts[safeParts.length - 1]
safeParts[safeParts.length - 1] = appendSuffixToFileName(last, stableSuffix(normalized))
}
return safeParts.join("/")
}
export function isScheduledImportInternalPath(path: string): boolean {
const parts = normalizePath(path).split("/")
return parts.includes(LEGACY_DB_DIR) || parts.includes(".llm-wiki")
}
export function shouldSkipScheduledImportFile(
projectPath: string,
filePath: string,
): boolean {
const path = normalizePath(filePath)
const project = normalizePath(projectPath)
if (isScheduledImportInternalPath(path)) {
return true
}
if (isPathInside(path, projectSubpath(project, "wiki"))) {
return true
}
if (isPathInside(path, projectSubpath(project, "raw/sources/.cache"))) {
return true
}
const name = path.split("/").pop() ?? ""
return name.startsWith(".")
}
export function shouldSkipScheduledImportConfigFile(path: string): boolean {
const name = normalizePath(path).split("/").pop() ?? ""
const ext = name.includes(".") ? name.split(".").pop()?.toLowerCase() : ""
return Boolean(ext && SCHEDULED_IMPORT_CONFIG_EXTENSIONS.has(ext))
}
export function resolveImportPath(projectPath: string, configPath: string): string {
const path = normalizePath(configPath || "raw/sources")
if (isAbsolutePath(path)) {
return path
}
return `${normalizePath(projectPath)}/${path}`
}
export function scheduledImportDestinationForFile(
projectPath: string,
importPath: string,
file: Pick<FileNode, "path" | "name">,
): string {
const project = normalizePath(projectPath)
const source = normalizePath(file.path)
const sourcesRoot = projectSubpath(project, "raw/sources")
if (isPathInside(source, sourcesRoot)) {
return source
}
const importRoot = normalizePath(importPath).replace(/\/+$/, "")
const sourceKey = caseFoldPath(source)
const importRootKey = caseFoldPath(importRoot)
const relative =
sourceKey === importRootKey || !sourceKey.startsWith(`${importRootKey}/`)
? file.name
: source.slice(importRoot.length + 1)
return `${sourcesRoot}/${SCHEDULED_IMPORT_DIR}/${safeRelativePath(relative)}`
}
function collectFiles(nodes: FileNode[]): FileNode[] {
const files: FileNode[] = []
for (const node of nodes) {
if (!node.is_dir) {
files.push(node)
} else if (node.children) {
files.push(...collectFiles(node.children))
}
}
return files
}
function scheduledImportDestinationForSourceKey(
projectPath: string,
importPath: string,
sourceKey: string,
): string {
const name = normalizePath(sourceKey).split("/").pop() ?? "_"
return scheduledImportDestinationForFile(projectPath, importPath, {
name,
path: sourceKey,
})
}
async function cleanupRemovedScheduledImports(
project: WikiProject,
importRoot: string,
db: ImportDb,
observedEligibleKeys: ReadonlySet<string>,
nextDb: ImportDb,
): Promise<boolean> {
const removedKeys = Object.keys(db.files).filter(
(key) => !observedEligibleKeys.has(key) && !(key in nextDb.files),
)
if (removedKeys.length === 0) return false
const destinations = removedKeys.map((key) => ({
key,
path: scheduledImportDestinationForSourceKey(project.path, importRoot, key),
}))
await discardTasksForSources(destinations.map((entry) => entry.path))
let changed = false
for (const destination of destinations) {
try {
const exists = await fileExists(destination.path)
await deleteSourceFile(project.path, destination.path, {
fileAlreadyDeleted: !exists,
logReason: "scheduled import source removed or excluded",
})
changed = true
} catch (err) {
// Preserve the old record so a later scan retries cleanup rather than
// silently forgetting a stale mirror or its generated wiki pages.
nextDb.files[destination.key] = db.files[destination.key]
console.warn(
`[scheduled-import] failed to clean removed source ${destination.path}:`,
err,
)
}
}
return changed
}
async function loadDbStore(projectPath: string): Promise<ImportDbStore> {
const path = dbFilePath(projectPath)
try {
if (!(await fileExists(path))) {
return emptyStore()
}
const content = await readFile(path)
const parsed = JSON.parse(content) as Partial<ImportDbStore>
if (!parsed.directories || typeof parsed.directories !== "object") {
return emptyStore()
}
return {
version: 1,
directories: parsed.directories as Record<string, ImportDb>,
}
} catch (err) {
console.warn("Failed to load scheduled import database:", err)
return emptyStore()
}
}
async function loadImportDb(
projectPath: string,
importPath: string,
): Promise<ImportDb> {
const store = await loadDbStore(projectPath)
const directoryKey = dbDirectoryKey(importPath)
const db = store.directories[directoryKey] ??
Object.entries(store.directories).find(
([storedKey]) => dbDirectoryKey(storedKey) === directoryKey,
)?.[1]
if (!db) return cloneDb(EMPTY_DB)
// Older builds persisted Windows directory and file keys with their
// observed casing. Normalize both levels while loading so upgrading does
// not make the first scan treat every unchanged file as new.
return {
files: Object.fromEntries(
Object.entries(db.files).map(([path, md5]) => [dbDirectoryKey(path), md5]),
),
statCache: Object.fromEntries(
Object.entries(db.statCache ?? {}).map(([path, stat]) => [dbDirectoryKey(path), stat]),
),
lastScan: db.lastScan,
}
}
async function saveImportDb(
projectPath: string,
importPath: string,
db: ImportDb,
): Promise<void> {
const store: ImportDbStore = {
version: 1,
directories: {
[dbDirectoryKey(importPath)]: cloneDb(db),
},
}
await writeFileAtomic(dbFilePath(projectPath), JSON.stringify(store, null, 2))
}
function isCurrentProject(projectId: string): boolean {
return useWikiStore.getState().project?.id === projectId
}
function isCurrentRun(
projectId: string,
runId?: number,
allowInactive = false,
): boolean {
return (allowInactive || isCurrentProject(projectId)) &&
(runId === undefined || runId === activeRunId)
}
export function isScheduledImportDue(
config: ScheduledImportConfig,
now = Date.now(),
): boolean {
if (!config.enabled || !config.path || config.interval <= 0) return false
if (!config.lastScan) return true
const intervalMs = Math.max(1, Math.min(1440, config.interval)) * 60 * 1000
return now - config.lastScan >= intervalMs
}
export async function scanAndImport(
project: WikiProject,
importPath: string,
options: ScanOptions = {},
): Promise<void> {
if (!importPath) return
const projectPath = normalizePath(project.path)
const importRoot = resolveImportPath(projectPath, importPath)
if (isProjectManagedScheduledImportPath(projectPath, importRoot)) {
console.warn(
`[scheduled-import] skipped self-referential path "${importRoot}". Use source folder monitoring for project sources instead.`,
)
if (isCurrentProject(project.id)) {
notifyManagedScheduledImportPath(project, importRoot)
}
return
}
if (scanning) return
scanning = true
try {
if (!isCurrentRun(project.id, options.runId, options.allowInactive)) {
return
}
const tree = await listDirectory(importRoot)
const sourceWatchConfig = normalizeSourceWatchConfig(
options.sourceWatchConfig ??
(isCurrentProject(project.id)
? useWikiStore.getState().sourceWatchConfig
: await loadSourceWatchConfig(project.id)),
)
const maxFileSizeBytes = sourceWatchConfig.maxFileSizeMb * 1024 * 1024
const db = await loadImportDb(projectPath, importRoot)
const nextDb: ImportDb = { files: {}, lastScan: Date.now() }
const nextStatCache: Record<string, ImportStat> = {}
const changedFiles: Array<{ key: string; md5: string; destPath: string }> = []
const observedEligibleKeys = new Set<string>()
for (const file of collectFiles(tree)) {
try {
const sourcePath = normalizePath(file.path)
if (
shouldSkipScheduledImportFile(projectPath, sourcePath) ||
shouldSkipScheduledImportConfigFile(sourcePath) ||
!isPathAllowedBySourceWatch(sourcePath, sourceWatchConfig) ||
!isIngestableSourcePath(sourcePath)
) {
continue
}
const key = dbDirectoryKey(sourcePath)
if (!isCurrentRun(project.id, options.runId, options.allowInactive)) {
return
}
const size = await getFileSize(sourcePath)
if (size > Math.min(MAX_SCHEDULED_IMPORT_BYTES, maxFileSizeBytes)) {
const limitMb = Math.min(100, sourceWatchConfig.maxFileSizeMb)
console.warn(
`[scheduled-import] skipping ${sourcePath}: ${(size / 1024 / 1024).toFixed(1)} MB exceeds ${limitMb} MB limit`,
)
continue
}
observedEligibleKeys.add(key)
// Fast path: if size and mtime are unchanged since the last scan,
// reuse the recorded hash instead of re-reading and re-hashing the
// file. Files whose stat differs, including imports from older
// builds that have no cache entry yet, are still verified by content
// hash below.
const cachedStat = db.statCache?.[key]
const mtime = await getFileModifiedTime(sourcePath)
if (cachedStat && db.files[key] && cachedStat.size === size && cachedStat.mtime === mtime) {
nextDb.files[key] = db.files[key]
nextStatCache[key] = cachedStat
continue
}
const md5 = await getFileMd5(sourcePath)
nextStatCache[key] = { size, mtime }
if (db.files[key] === md5) {
nextDb.files[key] = md5
continue
}
const destPath = scheduledImportDestinationForFile(projectPath, importRoot, file)
if (normalizePath(destPath) !== sourcePath) {
await copyFile(sourcePath, destPath)
}
changedFiles.push({ key, md5, destPath })
} catch (err) {
const key = dbDirectoryKey(file.path)
if (db.files[key]) {
nextDb.files[key] = db.files[key]
}
console.warn(`[scheduled-import] skipped ${file.path}:`, err)
}
}
if (!isCurrentRun(project.id, options.runId, options.allowInactive)) {
return
}
let removedSourcesChanged = false
if (isCurrentProject(project.id)) {
removedSourcesChanged = await cleanupRemovedScheduledImports(
project,
importRoot,
db,
observedEligibleKeys,
nextDb,
)
} else {
// Defer destructive cleanup until the project is opened. Preserve old
// records so the first active scan can still identify removals.
for (const [key, md5] of Object.entries(db.files)) {
if (!observedEligibleKeys.has(key)) nextDb.files[key] = md5
}
}
if (!isCurrentRun(project.id, options.runId, options.allowInactive)) {
return
}
if (changedFiles.length > 0) {
const destPaths = changedFiles.map((file) => file.destPath)
if (isCurrentRun(project.id, options.runId, options.allowInactive)) {
const ids = isCurrentProject(project.id)
? await enqueueSourceIngest(
project,
destPaths,
useWikiStore.getState().llmConfig,
)
: await enqueueInactiveProjectBatch(
project.id,
project.path,
destPaths.map((sourcePath) => ({
sourcePath,
folderContext: folderContextForSourcePath(sourcePath),
})),
)
if (ids.length > 0) {
for (const file of changedFiles) {
nextDb.files[file.key] = file.md5
}
} else {
console.warn("[scheduled-import] LLM is not configured; changed files were not marked imported")
}
}
}
if (
isCurrentProject(project.id) &&
(removedSourcesChanged || changedFiles.some((file) => nextDb.files[file.key]))
) {
await refreshProjectFileTree(projectPath, {
projectId: project.id,
bumpDataVersion: true,
})
}
nextDb.statCache = nextStatCache
await saveImportDb(projectPath, importRoot, nextDb)
const currentConfig = await loadScheduledImportConfig(projectPath)
if (currentConfig) {
await saveScheduledImportConfig(projectPath, {
...currentConfig,
lastScan: nextDb.lastScan,
})
}
if (isCurrentProject(project.id) && currentConfig) {
useWikiStore.getState().setScheduledImportConfig({
...currentConfig,
lastScan: nextDb.lastScan,
})
}
} catch (err) {
console.error("Scheduled import scan failed:", err)
} finally {
scanning = false
}
}
export function startScheduledImport(
project: WikiProject,
config: ScheduledImportConfig,
): void {
stopScheduledImport()
const runId = ++activeRunId
void runScheduledImportSweep(project, config, runId, true).catch((err) => {
console.error("Scheduled import sweep failed:", err)
})
scanTimer = setInterval(() => {
void runScheduledImportSweep(project, config, runId, false).catch((err) => {
console.error("Scheduled import sweep failed:", err)
})
}, 60 * 1000)
}
async function runScheduledImportSweep(
activeProject: WikiProject,
activeConfig: ScheduledImportConfig,
runId: number,
forceActive: boolean,
): Promise<void> {
const recents = await getRecentProjects()
const projects = [activeProject, ...recents.filter((item) => item.id !== activeProject.id)]
for (const project of projects) {
if (runId !== activeRunId) return
let config: ScheduledImportConfig | null
try {
config = project.id === activeProject.id
? forceActive
? activeConfig
: (await loadScheduledImportConfig(project.path)) ?? activeConfig
: await loadScheduledImportConfig(project.path)
} catch (err) {
console.warn(`[scheduled-import] failed to load config for ${project.path}:`, err)
continue
}
if (!config) continue
if (!isScheduledImportDue(config) && !(project.id === activeProject.id && forceActive)) {
continue
}
if (!config.enabled || !config.path || config.interval <= 0) {
continue
}
await scanAndImport(project, config.path, {
runId,
allowInactive: project.id !== activeProject.id,
})
}
}
export function stopScheduledImport(): void {
activeRunId += 1
if (scanTimer) {
clearInterval(scanTimer)
scanTimer = null
}
}