Skip to content

LLM Wiki is a cross-platform desktop application that turns your documents into an organized, interlinked knowledge base — automatically. Instead of traditional RAG (retrieve-and-answer from scratch every time), the LLM incrementally builds and maintains a persistent wiki from your sources。

TypeScript

Harden scheduled directory import

nash_su41902df00896
diff --git a/src-tauri/src/commands/fs.rs b/src-tauri/src/commands/fs.rs
index 56303ab2..6959f7e6 100644
--- a/src-tauri/src/commands/fs.rs
+++ b/src-tauri/src/commands/fs.rs
@@ -1375,10 +1375,19 @@ pub async fn get_file_md5(path: String) -> Result<String, String> {
use md5::{Digest, Md5};
tauri::async_runtime::spawn_blocking(move || {
run_guarded("get_file_md5", || {
- let bytes = fs::read(&path)
- .map_err(|e| format!("Failed to read file '{}': {}", path, e))?;
+ let mut file = fs::File::open(&path)
+ .map_err(|e| format!("Failed to open file '{}': {}", path, e))?;
let mut hasher = Md5::new();
- hasher.update(&bytes);
+ let mut buffer = [0u8; 64 * 1024];
+ loop {
+ let read = file
+ .read(&mut buffer)
+ .map_err(|e| format!("Failed to read file '{}': {}", path, e))?;
+ if read == 0 {
+ break;
+ }
+ hasher.update(&buffer[..read]);
+ }
let result = hasher.finalize();
Ok(format!("{:x}", result))
})
diff --git a/src/App.tsx b/src/App.tsx
index 182d9403..d0b02b81 100644
--- a/src/App.tsx
+++ b/src/App.tsx
@@ -307,7 +307,7 @@ function App() {
const scheduledImportConfig = useWikiStore.getState().scheduledImportConfig
if (scheduledImportConfig.enabled && scheduledImportConfig.path && scheduledImportConfig.interval > 0) {
import("@/lib/scheduled-import").then(({ startScheduledImport }) => {
- startScheduledImport(proj.path, scheduledImportConfig)
+ startScheduledImport(proj, scheduledImportConfig)
}).catch((err) =>
console.error("Failed to start scheduled import:", err)
)
diff --git a/src/components/project/template-picker.tsx b/src/components/project/template-picker.tsx
index 82250aba..ff193534 100644
--- a/src/components/project/template-picker.tsx
+++ b/src/components/project/template-picker.tsx
@@ -1,5 +1,6 @@
import { templates } from "@/lib/templates"
import { cn } from "@/lib/utils"
+import { useTranslation } from "react-i18next"
interface TemplatePickerProps {
selected: string
@@ -7,25 +8,34 @@ interface TemplatePickerProps {
}
export function TemplatePicker({ selected, onSelect }: TemplatePickerProps) {
+ const { t } = useTranslation()
+
return (
<div className="grid grid-cols-2 gap-2 sm:grid-cols-3">
- {templates.map((template) => (
- <button
- key={template.id}
- type="button"
- onClick={() => onSelect(template.id)}
- className={cn(
- "flex flex-col gap-1 rounded-md border p-3 text-left transition-colors hover:bg-accent",
- selected === template.id
- ? "border-primary bg-accent ring-1 ring-primary"
- : "border-border bg-background"
- )}
- >
- <span className="text-xl leading-none">{template.icon}</span>
- <span className="text-sm font-medium leading-tight">{template.name}</span>
- <span className="text-xs text-muted-foreground leading-tight">{template.description}</span>
- </button>
- ))}
+ {templates.map((template) => {
+ const name = t(`templates.${template.id}.name`, { defaultValue: template.name })
+ const description = t(`templates.${template.id}.description`, {
+ defaultValue: template.description,
+ })
+
+ return (
+ <button
+ key={template.id}
+ type="button"
+ onClick={() => onSelect(template.id)}
+ className={cn(
+ "flex flex-col gap-1 rounded-md border p-3 text-left transition-colors hover:bg-accent",
+ selected === template.id
+ ? "border-primary bg-accent ring-1 ring-primary"
+ : "border-border bg-background",
+ )}
+ >
+ <span className="text-xl leading-none">{template.icon}</span>
+ <span className="text-sm font-medium leading-tight">{name}</span>
+ <span className="text-xs text-muted-foreground leading-tight">{description}</span>
+ </button>
+ )
+ })}
</div>
)
}
diff --git a/src/components/settings/sections/scheduled-import-section.tsx b/src/components/settings/sections/scheduled-import-section.tsx
index 5faf1f9f..506d9d8d 100644
--- a/src/components/settings/sections/scheduled-import-section.tsx
+++ b/src/components/settings/sections/scheduled-import-section.tsx
@@ -1,4 +1,4 @@
-import { useState, useEffect, useCallback } from "react"
+import { useState, useCallback } from "react"
import { useTranslation } from "react-i18next"
import { open } from "@tauri-apps/plugin-dialog"
import { Input } from "@/components/ui/input"
@@ -7,7 +7,7 @@ import { Button } from "@/components/ui/button"
import { Folder, Play, RefreshCw } from "lucide-react"
import type { SettingsDraft, DraftSetter } from "../settings-types"
import { useWikiStore } from "@/stores/wiki-store"
-import { startScheduledImport, stopScheduledImport, scanAndImport } from "@/lib/scheduled-import"
+import { scanAndImport } from "@/lib/scheduled-import"
interface Props {
draft: SettingsDraft
@@ -38,7 +38,7 @@ export function ScheduledImportSection({ draft, setDraft }: Props) {
setIsScanning(true)
try {
- await scanAndImport(project.path, draft.scheduledImportPath)
+ await scanAndImport(project, draft.scheduledImportPath)
} catch (err) {
console.error("[Scheduled Import] Manual scan failed:", err)
} finally {
@@ -46,26 +46,6 @@ export function ScheduledImportSection({ draft, setDraft }: Props) {
}
}, [project, draft.scheduledImportPath, isScanning])
- // Start/stop scheduled import when config changes
- useEffect(() => {
- if (!project) return
-
- if (draft.scheduledImportEnabled && draft.scheduledImportPath && draft.scheduledImportInterval > 0) {
- startScheduledImport(project.path, {
- enabled: true,
- path: draft.scheduledImportPath,
- interval: draft.scheduledImportInterval,
- lastScan: scheduledImportConfig.lastScan,
- })
- } else {
- stopScheduledImport()
- }
-
- return () => {
- stopScheduledImport()
- }
- }, [project, draft.scheduledImportEnabled, draft.scheduledImportPath, draft.scheduledImportInterval])
-
const lastScanDate = scheduledImportConfig.lastScan
? new Date(scheduledImportConfig.lastScan).toLocaleString()
: t("settings.sections.scheduledImport.never", { defaultValue: "Never" })
@@ -100,6 +80,15 @@ export function ScheduledImportSection({ draft, setDraft }: Props) {
</span>
</label>
+ {draft.scheduledImportEnabled && (
+ <div className="rounded-md border border-amber-200 bg-amber-50 px-3 py-2 text-xs text-amber-900 dark:border-amber-900/50 dark:bg-amber-950/40 dark:text-amber-200">
+ {t("settings.sections.scheduledImport.privacyNotice", {
+ defaultValue:
+ "Files from the selected directory may be copied into this project and sent to your configured LLM during ingest. Removed files are not automatically deleted from the project.",
+ })}
+ </div>
+ )}
+
<div className="space-y-2">
<Label>
{t("settings.sections.scheduledImport.directory", {
diff --git a/src/components/settings/settings-view.tsx b/src/components/settings/settings-view.tsx
index b8985ae4..9659d715 100644
--- a/src/components/settings/settings-view.tsx
+++ b/src/components/settings/settings-view.tsx
@@ -317,12 +317,22 @@ export function SettingsView() {
const newScheduledImport = {
enabled: draft.scheduledImportEnabled,
path: draft.scheduledImportPath,
- interval: draft.scheduledImportInterval,
+ interval: Math.max(1, Math.min(1440, draft.scheduledImportInterval || 60)),
lastScan: scheduledImportConfig.lastScan,
}
setScheduledImportConfig(newScheduledImport)
if (project) {
await saveScheduledImportConfig(project.path, newScheduledImport)
+ const { startScheduledImport, stopScheduledImport } = await import("@/lib/scheduled-import")
+ if (
+ newScheduledImport.enabled &&
+ newScheduledImport.path &&
+ newScheduledImport.interval > 0
+ ) {
+ startScheduledImport(project, newScheduledImport)
+ } else {
+ stopScheduledImport()
+ }
}
setMaxHistoryMessages(draft.maxHistoryMessages)
@@ -345,7 +355,6 @@ export function SettingsView() {
scheduledImportConfig,
setMaxHistoryMessages,
outputLanguage,
- project,
])
const body = useMemo(() => {
diff --git a/src/i18n/en.json b/src/i18n/en.json
index 5ac18767..e125eb27 100644
--- a/src/i18n/en.json
+++ b/src/i18n/en.json
@@ -115,20 +115,6 @@
"noResults": "No results found",
"startSearching": "Type to search wiki pages"
},
- "lint": {
- "title": "Lint",
- "runLint": "Run Lint",
- "running": "Running...",
- "semantic": "Semantic (LLM)",
- "allClear": "All clear — no issues found",
- "orphan": "Orphan Page",
- "brokenLink": "Broken Link",
- "noOutlinks": "No Outbound Links",
- "contradiction": "Contradiction",
- "stale": "Stale Content",
- "missingPage": "Missing Page",
- "suggestion": "Suggestion"
- },
"lint": {
"title": "Wiki Lint",
"semantic": "Semantic (LLM)",
@@ -286,6 +272,7 @@
"enable": "Enable scheduled import",
"directory": "Monitor Directory",
"directoryHelp": "Relative path from project root (e.g., 'raw', 'docs/input'). Files in this directory will be automatically imported. Defaults to 'raw'.",
+ "privacyNotice": "Files from the selected directory may be copied into this project and sent to your configured LLM during ingest. Removed files are not automatically deleted from the project.",
"browse": "Browse",
"selectDirectory": "Select Directory to Monitor",
"interval": "Scan Interval (minutes)",
diff --git a/src/i18n/i18n-parity.test.ts b/src/i18n/i18n-parity.test.ts
index 48a189d3..c8ed88c3 100644
--- a/src/i18n/i18n-parity.test.ts
+++ b/src/i18n/i18n-parity.test.ts
@@ -11,6 +11,9 @@
* anyone notices in the UI.
*/
import { describe, it, expect } from "vitest"
+import { dirname, join } from "node:path"
+import { fileURLToPath } from "node:url"
+import { readFileSync } from "node:fs"
import en from "./en.json"
import zh from "./zh.json"
@@ -30,9 +33,27 @@ function flattenKeys(obj: unknown, prefix = ""): string[] {
}
describe("i18n bundle parity (en.json ↔ zh.json)", () => {
+ const i18nDir = dirname(fileURLToPath(import.meta.url))
const enKeys = new Set(flattenKeys(en))
const zhKeys = new Set(flattenKeys(zh))
+ it("does not contain duplicate top-level JSON keys", () => {
+ const findDuplicates = (fileName: string) => {
+ const text = readFileSync(join(i18nDir, fileName), "utf8")
+ const seen = new Set<string>()
+ const duplicates = new Set<string>()
+ for (const match of text.matchAll(/^ "([^"]+)":/gm)) {
+ const key = match[1]
+ if (seen.has(key)) duplicates.add(key)
+ seen.add(key)
+ }
+ return [...duplicates].sort()
+ }
+
+ expect(findDuplicates("en.json"), "duplicate top-level keys in en.json").toEqual([])
+ expect(findDuplicates("zh.json"), "duplicate top-level keys in zh.json").toEqual([])
+ })
+
it("every en.json key is also in zh.json", () => {
const missing = [...enKeys].filter((k) => !zhKeys.has(k)).sort()
expect(
diff --git a/src/i18n/zh.json b/src/i18n/zh.json
index c261623a..4eb0fa56 100644
--- a/src/i18n/zh.json
+++ b/src/i18n/zh.json
@@ -47,6 +47,28 @@
"aiOutputLanguageHint": "所有 AI 生成的内容(Wiki 页面、聊天回复、Research 结果)都将使用此语言。您可以在设置 → 输出 中稍后更改。",
"pickLanguage": "选择语言..."
},
+ "templates": {
+ "research": {
+ "name": "研究",
+ "description": "用于深度研究,包含假设跟踪和方法论记录"
+ },
+ "reading": {
+ "name": "阅读",
+ "description": "跟踪书籍的人物、主题、情节线和章节笔记"
+ },
+ "personal": {
+ "name": "个人成长",
+ "description": "记录目标、习惯、反思和成长日志"
+ },
+ "business": {
+ "name": "商务",
+ "description": "管理团队会议、决策、项目和干系人背景"
+ },
+ "general": {
+ "name": "通用",
+ "description": "最小化初始结构,适合从空白开始"
+ }
+ },
"sources": {
"title": "原始资料",
"import": "导入",
@@ -93,20 +115,6 @@
"noResults": "未找到相关结果",
"startSearching": "输入关键词搜索 Wiki 页面"
},
- "lint": {
- "title": "检查",
- "runLint": "开始检查",
- "running": "检查中...",
- "semantic": "语义检查(LLM)",
- "allClear": "一切正常,未发现问题",
- "orphan": "孤立页面",
- "brokenLink": "失效链接",
- "noOutlinks": "无出链",
- "contradiction": "内容矛盾",
- "stale": "内容过时",
- "missingPage": "缺失页面",
- "suggestion": "建议"
- },
"lint": {
"title": "Wiki 检查",
"semantic": "语义检查 (LLM)",
@@ -264,6 +272,7 @@
"enable": "启用定时导入",
"directory": "监控目录",
"directoryHelp": "相对于项目根目录的路径(如 'raw'、'docs/input')。该目录中的文件将被自动导入。默认为 'raw'。",
+ "privacyNotice": "所选目录中的文件可能会被复制到当前项目,并在提取时发送给你配置的 LLM。源目录中删除文件不会自动删除项目内已导入内容。",
"browse": "浏览",
"selectDirectory": "选择监控目录",
"interval": "扫描间隔(分钟)",
diff --git a/src/lib/reset-project-state.ts b/src/lib/reset-project-state.ts
index 32353f69..6d660060 100644
--- a/src/lib/reset-project-state.ts
+++ b/src/lib/reset-project-state.ts
@@ -40,11 +40,12 @@ export async function resetProjectState(): Promise<void> {
// Module-level caches — load in parallel and clear each, surfacing any
// failure instead of swallowing it.
- const [queueMod, dedupQueueMod, graphMod, fileSyncMod] = await Promise.allSettled([
+ const [queueMod, dedupQueueMod, graphMod, fileSyncMod, scheduledImportMod] = await Promise.allSettled([
import("@/lib/ingest-queue"),
import("@/lib/dedup-queue"),
import("@/lib/graph-relevance"),
import("@/lib/project-file-sync"),
+ import("@/lib/scheduled-import"),
])
if (queueMod.status === "fulfilled") {
@@ -90,4 +91,14 @@ export async function resetProjectState(): Promise<void> {
} else {
console.warn("[Reset Project State] Failed to load project-file-sync:", fileSyncMod.reason)
}
+
+ if (scheduledImportMod.status === "fulfilled") {
+ try {
+ scheduledImportMod.value.stopScheduledImport()
+ } catch (err) {
+ console.warn("[Reset Project State] stopScheduledImport failed:", err)
+ }
+ } else {
+ console.warn("[Reset Project State] Failed to load scheduled-import:", scheduledImportMod.reason)
+ }
}
diff --git a/src/lib/scheduled-import.test.ts b/src/lib/scheduled-import.test.ts
new file mode 100644
index 00000000..87d3bcd8
--- /dev/null
+++ b/src/lib/scheduled-import.test.ts
@@ -0,0 +1,67 @@
+import { describe, expect, it } from "vitest"
+import {
+ scheduledImportDestinationForFile,
+ shouldSkipScheduledImportFile,
+} from "./scheduled-import"
+
+describe("scheduled import path handling", () => {
+ const projectPath = "/Users/me/wiki-project"
+
+ it("preserves nested relative paths for external directories", () => {
+ const dest = scheduledImportDestinationForFile(
+ projectPath,
+ "/Users/me/inbox",
+ {
+ name: "report.pdf",
+ path: "/Users/me/inbox/a/report.pdf",
+ },
+ )
+
+ expect(dest).toBe(
+ "/Users/me/wiki-project/raw/sources/scheduled-import/a/report.pdf",
+ )
+ })
+
+ it("does not copy files that are already under raw/sources", () => {
+ const dest = scheduledImportDestinationForFile(
+ projectPath,
+ `${projectPath}/raw/sources`,
+ {
+ name: "source.md",
+ path: `${projectPath}/raw/sources/source.md`,
+ },
+ )
+
+ expect(dest).toBe(`${projectPath}/raw/sources/source.md`)
+ })
+
+ it("sanitizes Windows-unsafe destination path segments", () => {
+ const dest = scheduledImportDestinationForFile(
+ projectPath,
+ "/Users/me/inbox",
+ {
+ name: "ignored.md",
+ path: "/Users/me/inbox/CON/Article: Why?.md",
+ },
+ )
+
+ expect(dest).toBe(
+ "/Users/me/wiki-project/raw/sources/scheduled-import/_CON/Article_ Why_.md",
+ )
+ })
+
+ it("skips project internals and generated wiki/cache files", () => {
+ expect(
+ shouldSkipScheduledImportFile(projectPath, `${projectPath}/.llm-wiki/db.json`),
+ ).toBe(true)
+ expect(
+ shouldSkipScheduledImportFile(projectPath, `${projectPath}/wiki/index.md`),
+ ).toBe(true)
+ expect(
+ shouldSkipScheduledImportFile(
+ projectPath,
+ `${projectPath}/raw/sources/.cache/source.pdf.txt`,
+ ),
+ ).toBe(true)
+ })
+})
diff --git a/src/lib/scheduled-import.ts b/src/lib/scheduled-import.ts
index 28560f5d..11400d2a 100644
--- a/src/lib/scheduled-import.ts
+++ b/src/lib/scheduled-import.ts
@@ -1,304 +1,362 @@
-import { listDirectory, copyFile, preprocessFile, getFileMd5, createDirectory, fileExists, readFile, writeFile } from "@/commands/fs"
-import { enqueueIngest } from "@/lib/ingest-queue"
-import { normalizePath, getFileName } from "@/lib/path-utils"
-import { hasUsableLlm } from "@/lib/has-usable-llm"
-import { useWikiStore, type ScheduledImportConfig } from "@/stores/wiki-store"
-import type { FileNode } from "@/types/wiki"
-
-// Supported file extensions for import
-const IMPORTABLE_EXTENSIONS = new Set([
- "md", "mdx", "txt", "rtf", "pdf",
- "html", "htm", "xml",
- "doc", "docx", "xls", "xlsx", "ppt", "pptx",
- "odt", "ods", "odp", "epub", "pages", "numbers", "key",
- "json", "jsonl", "csv", "tsv", "yaml", "yml", "ndjson",
- "py", "js", "ts", "jsx", "tsx", "rs", "go", "java",
- "c", "cpp", "h", "rb", "php", "swift", "sql", "sh",
-])
+import {
+ copyFile,
+ fileExists,
+ getFileMd5,
+ listDirectory,
+ preprocessFile,
+ readFile,
+ writeFile,
+} from "@/commands/fs"
+import type { FileNode, WikiProject } from "@/types/wiki"
+import { normalizePath } from "@/lib/path-utils"
+import { useWikiStore } from "@/stores/wiki-store"
+import type { ScheduledImportConfig } from "@/stores/wiki-store"
+import {
+ loadScheduledImportConfig,
+ saveScheduledImportConfig,
+} from "@/lib/project-store"
+import {
+ enqueueSourceIngest,
+ isIngestableSourcePath,
+} from "@/lib/source-lifecycle"
+
+interface ImportDb {
+ files: Record<string, string>
+ lastScan: number | null
+}
+
+interface ImportDbStore {
+ version: 1
+ directories: Record<string, ImportDb>
+}
-// Hidden config folder name in the monitored directory
-const IMPORT_DB_FOLDER = ".llm-wiki-imported"
-const IMPORT_DB_FILE = "db.json"
+type ScanOptions = {
+ runId?: number
+}
+
+const EMPTY_DB: ImportDb = {
+ files: {},
+ lastScan: null,
+}
let scanTimer: ReturnType<typeof setInterval> | null = null
let scanning = false
+let activeRunId = 0
+
+const DB_PATH = ".llm-wiki/scheduled-import-db.json"
+const LEGACY_DB_DIR = ".llm-wiki-imported"
+const SCHEDULED_IMPORT_DIR = "scheduled-import"
+const RESERVED_WINDOWS_NAMES = new Set([
+ "con",
+ "prn",
+ "aux",
+ "nul",
+ "com1",
+ "com2",
+ "com3",
+ "com4",
+ "com5",
+ "com6",
+ "com7",
+ "com8",
+ "com9",
+ "lpt1",
+ "lpt2",
+ "lpt3",
+ "lpt4",
+ "lpt5",
+ "lpt6",
+ "lpt7",
+ "lpt8",
+ "lpt9",
+])
-interface ImportDbEntry {
- md5: string
- importedAt: number
+function emptyStore(): ImportDbStore {
+ return { version: 1, directories: {} }
}
-interface ImportDb {
- files: Record<string, ImportDbEntry>
- lastScan: number
+function dbFilePath(projectPath: string): string {
+ return `${normalizePath(projectPath)}/${DB_PATH}`
}
-/**
- * Get all files recursively from a directory tree (FileNode structure).
- */
-function collectFiles(nodes: FileNode[], prefix: string = ""): Array<{ name: string; path: string }> {
- const files: Array<{ name: string; path: string }> = []
- for (const node of nodes) {
- if (node.is_dir && node.children) {
- files.push(...collectFiles(node.children, `${prefix}${node.name}/`))
- } else if (!node.is_dir) {
- files.push({ name: `${prefix}${node.name}`, path: node.path })
- }
+function dbDirectoryKey(importPath: string): string {
+ return normalizePath(importPath)
+}
+
+function cloneDb(db: ImportDb): ImportDb {
+ return {
+ files: { ...db.files },
+ lastScan: db.lastScan,
}
- return files
}
-/**
- * Check if a file has an importable extension.
- */
-function isImportableFile(fileName: string): boolean {
- const ext = fileName.split(".").pop()?.toLowerCase() ?? ""
- return IMPORTABLE_EXTENSIONS.has(ext)
+function isPathInside(path: string, parent: string): boolean {
+ const normalizedPath = normalizePath(path)
+ const normalizedParent = normalizePath(parent).replace(/\/+$/, "")
+ return (
+ normalizedPath === normalizedParent ||
+ normalizedPath.startsWith(`${normalizedParent}/`)
+ )
+}
+
+function projectSubpath(projectPath: string, relPath: string): string {
+ return `${normalizePath(projectPath)}/${relPath}`
+}
+
+function sanitizePathSegment(segment: string): string {
+ let value = segment
+ .replace(/[<>:"|?*\x00-\x1F]/g, "_")
+ .replace(/[. ]+$/g, "")
+ .trim()
+
+ if (!value) {
+ value = "_"
+ }
+
+ const stem = value.split(".")[0]?.toLowerCase() ?? value.toLowerCase()
+ if (RESERVED_WINDOWS_NAMES.has(stem)) {
+ value = `_${value}`
+ }
+
+ return value
}
-/**
- * Get the path to the import db folder within the monitored directory.
- */
-function getDbFolderPath(importPath: string): string {
- return `${importPath}/${IMPORT_DB_FOLDER}`
+function safeRelativePath(path: string): string {
+ const parts = normalizePath(path)
+ .split("/")
+ .filter((part) => part && part !== "." && part !== "..")
+ .map(sanitizePathSegment)
+
+ return parts.length > 0 ? parts.join("/") : "_"
}
-/**
- * Get the path to the import db.json file.
- */
-function getDbFilePath(importPath: string): string {
- return `${importPath}/${IMPORT_DB_FOLDER}/${IMPORT_DB_FILE}`
+export function isScheduledImportInternalPath(path: string): boolean {
+ const parts = normalizePath(path).split("/")
+ return parts.includes(LEGACY_DB_DIR) || parts.includes(".llm-wiki")
}
-/**
- * Ensure the .llm-wiki-imported/ directory exists in the monitored directory.
- */
-async function ensureImportDb(importPath: string): Promise<void> {
- const dbFolder = getDbFolderPath(importPath)
- try {
- await createDirectory(dbFolder)
- } catch {
- // Directory may already exist
+export function shouldSkipScheduledImportFile(
+ projectPath: string,
+ filePath: string,
+): boolean {
+ const path = normalizePath(filePath)
+ const project = normalizePath(projectPath)
+
+ if (isScheduledImportInternalPath(path)) {
+ return true
+ }
+
+ if (isPathInside(path, projectSubpath(project, "wiki"))) {
+ return true
}
+
+ if (isPathInside(path, projectSubpath(project, "raw/sources/.cache"))) {
+ return true
+ }
+
+ const name = path.split("/").pop() ?? ""
+ return name.startsWith(".")
}
-/**
- * Load the import database from .llm-wiki-imported/db.json.
- * Returns an empty database if the file doesn't exist.
- */
-async function loadImportDb(importPath: string): Promise<ImportDb> {
- const dbFilePath = getDbFilePath(importPath)
- try {
- const exists = await fileExists(dbFilePath)
- if (!exists) return { files: {}, lastScan: 0 }
- const content = await readFile(dbFilePath)
- const parsed = JSON.parse(content)
- return {
- files: parsed.files || {},
- lastScan: parsed.lastScan || 0,
+export function scheduledImportDestinationForFile(
+ projectPath: string,
+ importPath: string,
+ file: Pick<FileNode, "path" | "name">,
+): string {
+ const project = normalizePath(projectPath)
+ const source = normalizePath(file.path)
+ const sourcesRoot = projectSubpath(project, "raw/sources")
+
+ if (isPathInside(source, sourcesRoot)) {
+ return source
+ }
+
+ const importRoot = normalizePath(importPath).replace(/\/+$/, "")
+ const relative =
+ source === importRoot || !source.startsWith(`${importRoot}/`)
+ ? file.name
+ : source.slice(importRoot.length + 1)
+
+ return `${sourcesRoot}/${SCHEDULED_IMPORT_DIR}/${safeRelativePath(relative)}`
+}
+
+function collectFiles(nodes: FileNode[]): FileNode[] {
+ const files: FileNode[] = []
+ for (const node of nodes) {
+ if (!node.is_dir) {
+ files.push(node)
+ } else if (node.children) {
+ files.push(...collectFiles(node.children))
}
- } catch {
- return { files: {}, lastScan: 0 }
}
+ return files
}
-/**
- * Save the import database to .llm-wiki-imported/db.json.
- */
-async function saveImportDb(importPath: string, db: ImportDb): Promise<void> {
- const dbFilePath = getDbFilePath(importPath)
+async function loadDbStore(projectPath: string): Promise<ImportDbStore> {
+ const path = dbFilePath(projectPath)
try {
- await writeFile(dbFilePath, JSON.stringify(db, null, 2))
+ if (!(await fileExists(path))) {
+ return emptyStore()
+ }
+ const content = await readFile(path)
+ const parsed = JSON.parse(content) as Partial<ImportDbStore>
+ if (!parsed.directories || typeof parsed.directories !== "object") {
+ return emptyStore()
+ }
+ return {
+ version: 1,
+ directories: parsed.directories as Record<string, ImportDb>,
+ }
} catch (err) {
- console.error("[Scheduled Import] Failed to save import db:", err)
+ console.warn("Failed to load scheduled import database:", err)
+ return emptyStore()
}
}
-/**
- * Scan the monitored directory and import new or modified files.
- * Uses MD5 hashing stored in .llm-wiki-imported/db.json to detect changes.
- */
-export async function scanAndImport(projectPath: string, importPath: string): Promise<void> {
- if (scanning) {
- console.log("[Scheduled Import] Scan already in progress, skipping")
- return
- }
+async function loadImportDb(
+ projectPath: string,
+ importPath: string,
+): Promise<ImportDb> {
+ const store = await loadDbStore(projectPath)
+ const db = store.directories[dbDirectoryKey(importPath)]
+ return db ? cloneDb(db) : cloneDb(EMPTY_DB)
+}
+
+async function saveImportDb(
+ projectPath: string,
+ importPath: string,
+ db: ImportDb,
+): Promise<void> {
+ const store = await loadDbStore(projectPath)
+ store.directories[dbDirectoryKey(importPath)] = cloneDb(db)
+ await writeFile(dbFilePath(projectPath), JSON.stringify(store, null, 2))
+}
+
+function isCurrentProject(projectId: string): boolean {
+ return useWikiStore.getState().project?.id === projectId
+}
+
+function isCurrentRun(projectId: string, runId?: number): boolean {
+ return isCurrentProject(projectId) && (runId === undefined || runId === activeRunId)
+}
- const pp = normalizePath(projectPath)
- const ip = normalizePath(importPath)
+export async function scanAndImport(
+ project: WikiProject,
+ importPath: string,
+ options: ScanOptions = {},
+): Promise<void> {
+ if (!importPath || scanning) return
scanning = true
- console.log(`[Scheduled Import] Starting scan of ${ip}`)
+ const projectPath = normalizePath(project.path)
+ const importRoot = normalizePath(importPath)
try {
- // Ensure the hidden config folder exists
- await ensureImportDb(ip)
-
- // Load existing import database
- const db = await loadImportDb(ip)
-
- // List all files in the monitored directory
- let monitoredFiles: Array<{ name: string; path: string }> = []
- try {
- const tree = await listDirectory(ip)
- monitoredFiles = collectFiles(tree)
- } catch (err) {
- console.error("[Scheduled Import] Failed to list monitored directory:", err)
+ if (!isCurrentRun(project.id, options.runId)) {
return
}
- // Filter to importable files only, excluding the hidden config folder
- const dbFolderPrefix = `${ip}/${IMPORT_DB_FOLDER}`
- const importableFiles = monitoredFiles.filter(f => {
- // Skip files inside .llm-wiki-imported/ (compare normalized paths)
- if (normalizePath(f.path).startsWith(dbFolderPrefix)) {
- return false
+ const tree = await listDirectory(importRoot)
+ const db = await loadImportDb(projectPath, importRoot)
+ const nextDb: ImportDb = {
+ files: {},
+ lastScan: Date.now(),
+ }
+ const llmConfig = useWikiStore.getState().llmConfig
+ const destPaths: string[] = []
+
+ for (const file of collectFiles(tree)) {
+ const sourcePath = normalizePath(file.path)
+ if (
+ shouldSkipScheduledImportFile(projectPath, sourcePath) ||
+ !isIngestableSourcePath(sourcePath)
+ ) {
+ continue
}
- return isImportableFile(f.name)
- })
- if (importableFiles.length === 0) {
- console.log("[Scheduled Import] No importable files found")
- return
- }
+ if (!isCurrentRun(project.id, options.runId)) {
+ return
+ }
- console.log(`[Scheduled Import] Found ${importableFiles.length} importable files`)
+ const key = sourcePath
+ const md5 = await getFileMd5(sourcePath)
+ nextDb.files[key] = md5
- const llmConfig = useWikiStore.getState().llmConfig
- const hasLlm = hasUsableLlm(llmConfig)
- let importedCount = 0
- let skippedCount = 0
-
- for (const file of importableFiles) {
- try {
- // Use relative path (from monitored dir) as the key
- const relativePath = file.name
- const currentMd5 = await getFileMd5(file.path)
- const existing = db.files[relativePath]
-
- // Check if file has changed
- if (existing && existing.md5 === currentMd5) {
- skippedCount++
- continue
- }
-
- // File is new or modified - import it
- const baseName = getFileName(file.path) || file.name
- const destPath = `${pp}/raw/sources/${baseName}`
-
- // Copy file to sources directory (skip if source IS the destination)
- const normalizedSource = normalizePath(file.path)
- const normalizedDest = normalizePath(destPath)
- if (normalizedSource !== normalizedDest) {
- await copyFile(file.path, destPath)
- }
-
- // Preprocess the file
- preprocessFile(destPath).catch(() => {})
-
- // Enqueue for ingest if LLM is configured
- if (hasLlm) {
- const project = useWikiStore.getState().project
- if (project) {
- await enqueueIngest(project.id, destPath)
- }
- }
-
- // Update the database with the new MD5
- db.files[relativePath] = {
- md5: currentMd5,
- importedAt: Date.now(),
- }
-
- importedCount++
- console.log(`[Scheduled Import] ${existing ? "Updated" : "Imported"}: ${baseName}`)
- } catch (err) {
- console.error(`[Scheduled Import] Failed to process ${file.name}:`, err)
+ if (db.files[key] === md5) {
+ continue
}
+
+ const destPath = scheduledImportDestinationForFile(projectPath, importRoot, file)
+ if (normalizePath(destPath) !== sourcePath) {
+ await copyFile(sourcePath, destPath)
+ }
+ destPaths.push(destPath)
}
- // Save the updated database
- db.lastScan = Date.now()
- await saveImportDb(ip, db)
+ if (!isCurrentRun(project.id, options.runId)) {
+ return
+ }
- // Update last scan time in the store
- const config = useWikiStore.getState().scheduledImportConfig
- const updatedConfig = { ...config, lastScan: Date.now() }
- useWikiStore.getState().setScheduledImportConfig(updatedConfig)
- const { saveScheduledImportConfig } = await import("@/lib/project-store")
- await saveScheduledImportConfig(projectPath, updatedConfig)
+ await saveImportDb(projectPath, importRoot, nextDb)
- // Refresh file tree
- const { listDirectory: listDir } = await import("@/commands/fs")
- const tree = await listDir(pp)
- useWikiStore.getState().setFileTree(tree)
- useWikiStore.getState().bumpDataVersion()
+ if (destPaths.length > 0) {
+ await Promise.all(destPaths.map((path) => preprocessFile(path).catch(() => {})))
+ if (isCurrentRun(project.id, options.runId)) {
+ await enqueueSourceIngest(project, destPaths, llmConfig)
+ const projectTree = await listDirectory(projectPath)
+ useWikiStore.getState().setFileTree(projectTree)
+ useWikiStore.getState().bumpDataVersion()
+ }
+ }
- console.log(`[Scheduled Import] Scan complete: ${importedCount} imported, ${skippedCount} skipped`)
+ const currentConfig = (await loadScheduledImportConfig(projectPath)) ?? {
+ enabled: false,
+ path: importRoot,
+ interval: 60,
+ lastScan: null,
+ }
+ await saveScheduledImportConfig(projectPath, {
+ ...currentConfig,
+ lastScan: nextDb.lastScan,
+ })
+
+ if (isCurrentProject(project.id)) {
+ useWikiStore.getState().setScheduledImportConfig({
+ ...currentConfig,
+ lastScan: nextDb.lastScan,
+ })
+ }
} catch (err) {
- console.error("[Scheduled Import] Scan failed:", err)
+ console.error("Scheduled import scan failed:", err)
} finally {
scanning = false
}
}
-/**
- * Resolve the absolute path for scheduled import.
- * If the config path is relative (not absolute), prepend the project path.
- * If the config path is empty, use the default "raw/sources" directory.
- */
-export function resolveImportPath(projectPath: string, configPath: string): string {
- const pp = normalizePath(projectPath)
- const path = configPath || "raw/sources"
- if (path.startsWith("/") || path.match(/^[a-zA-Z]:[/\\]/)) {
- return normalizePath(path)
- }
- return `${pp}/${path}`
-}
-
-/**
- * Start the scheduled import timer.
- */
-export function startScheduledImport(projectPath: string, config: ScheduledImportConfig): void {
+export function startScheduledImport(
+ project: WikiProject,
+ config: ScheduledImportConfig,
+): void {
stopScheduledImport()
- if (!config.enabled || config.interval <= 0) {
+ if (!config.enabled || !config.path || config.interval <= 0) {
return
}
- const pp = normalizePath(projectPath)
- const ip = resolveImportPath(pp, config.path)
+ const runId = ++activeRunId
+ const intervalMs = Math.max(1, Math.min(1440, config.interval)) * 60 * 1000
- console.log(`[Scheduled Import] Starting with interval ${config.interval} minutes, path: ${ip}`)
+ void scanAndImport(project, config.path, { runId })
- // Run first scan immediately
- scanAndImport(pp, ip).catch((err) => {
- console.error("[Scheduled Import] Initial scan failed:", err)
- })
-
- // Set up interval
- const intervalMs = config.interval * 60 * 1000
scanTimer = setInterval(() => {
- scanAndImport(pp, ip).catch((err) => {
- console.error("[Scheduled Import] Scheduled scan failed:", err)
- })
+ void scanAndImport(project, config.path, { runId })
}, intervalMs)
}
-/**
- * Stop the scheduled import timer.
- */
export function stopScheduledImport(): void {
+ activeRunId += 1
if (scanTimer) {
clearInterval(scanTimer)
scanTimer = null
- console.log("[Scheduled Import] Stopped")
}
}
-
-/**
- * Check if scheduled import is currently running.
- */
-export function isScanning(): boolean {
- return scanning
-}