LLM Wiki is a cross-platform desktop application that turns your documents into an organized, interlinked knowledge base — automatically. Instead of traditional RAG (retrieve-and-answer from scratch every time), the LLM incrementally builds and maintains a persistent wiki from your sources。
TypeScript
Harden scheduled directory import
nash_su41902df00896
diff --git a/src-tauri/src/commands/fs.rs b/src-tauri/src/commands/fs.rsindex 56303ab2..6959f7e6 100644--- a/src-tauri/src/commands/fs.rs+++ b/src-tauri/src/commands/fs.rs@@ -1375,10 +1375,19 @@ pub async fn get_file_md5(path: String) -> Result<String, String> {use md5::{Digest, Md5};tauri::async_runtime::spawn_blocking(move || {run_guarded("get_file_md5", || {- let bytes = fs::read(&path)- .map_err(|e| format!("Failed to read file '{}': {}", path, e))?;+ let mut file = fs::File::open(&path)+ .map_err(|e| format!("Failed to open file '{}': {}", path, e))?;let mut hasher = Md5::new();- hasher.update(&bytes);+ let mut buffer = [0u8; 64 * 1024];+ loop {+ let read = file+ .read(&mut buffer)+ .map_err(|e| format!("Failed to read file '{}': {}", path, e))?;+ if read == 0 {+ break;+ }+ hasher.update(&buffer[..read]);+ }let result = hasher.finalize();Ok(format!("{:x}", result))})diff --git a/src/App.tsx b/src/App.tsxindex 182d9403..d0b02b81 100644--- a/src/App.tsx+++ b/src/App.tsx@@ -307,7 +307,7 @@ function App() {const scheduledImportConfig = useWikiStore.getState().scheduledImportConfigif (scheduledImportConfig.enabled && scheduledImportConfig.path && scheduledImportConfig.interval > 0) {import("@/lib/scheduled-import").then(({ startScheduledImport }) => {- startScheduledImport(proj.path, scheduledImportConfig)+ startScheduledImport(proj, scheduledImportConfig)}).catch((err) =>console.error("Failed to start scheduled import:", err))diff --git a/src/components/project/template-picker.tsx b/src/components/project/template-picker.tsxindex 82250aba..ff193534 100644--- a/src/components/project/template-picker.tsx+++ b/src/components/project/template-picker.tsx@@ -1,5 +1,6 @@import { templates } from "@/lib/templates"import { cn } from "@/lib/utils"+import { useTranslation } from "react-i18next"interface TemplatePickerProps {selected: string@@ -7,25 +8,34 @@ interface TemplatePickerProps {}export function TemplatePicker({ selected, onSelect }: TemplatePickerProps) {+ const { t } = useTranslation()+return (<div className="grid grid-cols-2 gap-2 sm:grid-cols-3">- {templates.map((template) => (- <button- key={template.id}- type="button"- onClick={() => onSelect(template.id)}- className={cn(- "flex flex-col gap-1 rounded-md border p-3 text-left transition-colors hover:bg-accent",- selected === template.id- ? "border-primary bg-accent ring-1 ring-primary"- : "border-border bg-background"- )}- >- <span className="text-xl leading-none">{template.icon}</span>- <span className="text-sm font-medium leading-tight">{template.name}</span>- <span className="text-xs text-muted-foreground leading-tight">{template.description}</span>- </button>- ))}+ {templates.map((template) => {+ const name = t(`templates.${template.id}.name`, { defaultValue: template.name })+ const description = t(`templates.${template.id}.description`, {+ defaultValue: template.description,+ })++ return (+ <button+ key={template.id}+ type="button"+ onClick={() => onSelect(template.id)}+ className={cn(+ "flex flex-col gap-1 rounded-md border p-3 text-left transition-colors hover:bg-accent",+ selected === template.id+ ? "border-primary bg-accent ring-1 ring-primary"+ : "border-border bg-background",+ )}+ >+ <span className="text-xl leading-none">{template.icon}</span>+ <span className="text-sm font-medium leading-tight">{name}</span>+ <span className="text-xs text-muted-foreground leading-tight">{description}</span>+ </button>+ )+ })}</div>)}diff --git a/src/components/settings/sections/scheduled-import-section.tsx b/src/components/settings/sections/scheduled-import-section.tsxindex 5faf1f9f..506d9d8d 100644--- a/src/components/settings/sections/scheduled-import-section.tsx+++ b/src/components/settings/sections/scheduled-import-section.tsx@@ -1,4 +1,4 @@-import { useState, useEffect, useCallback } from "react"+import { useState, useCallback } from "react"import { useTranslation } from "react-i18next"import { open } from "@tauri-apps/plugin-dialog"import { Input } from "@/components/ui/input"@@ -7,7 +7,7 @@ import { Button } from "@/components/ui/button"import { Folder, Play, RefreshCw } from "lucide-react"import type { SettingsDraft, DraftSetter } from "../settings-types"import { useWikiStore } from "@/stores/wiki-store"-import { startScheduledImport, stopScheduledImport, scanAndImport } from "@/lib/scheduled-import"+import { scanAndImport } from "@/lib/scheduled-import"interface Props {draft: SettingsDraft@@ -38,7 +38,7 @@ export function ScheduledImportSection({ draft, setDraft }: Props) {setIsScanning(true)try {- await scanAndImport(project.path, draft.scheduledImportPath)+ await scanAndImport(project, draft.scheduledImportPath)} catch (err) {console.error("[Scheduled Import] Manual scan failed:", err)} finally {@@ -46,26 +46,6 @@ export function ScheduledImportSection({ draft, setDraft }: Props) {}}, [project, draft.scheduledImportPath, isScanning])- // Start/stop scheduled import when config changes- useEffect(() => {- if (!project) return-- if (draft.scheduledImportEnabled && draft.scheduledImportPath && draft.scheduledImportInterval > 0) {- startScheduledImport(project.path, {- enabled: true,- path: draft.scheduledImportPath,- interval: draft.scheduledImportInterval,- lastScan: scheduledImportConfig.lastScan,- })- } else {- stopScheduledImport()- }-- return () => {- stopScheduledImport()- }- }, [project, draft.scheduledImportEnabled, draft.scheduledImportPath, draft.scheduledImportInterval])-const lastScanDate = scheduledImportConfig.lastScan? new Date(scheduledImportConfig.lastScan).toLocaleString(): t("settings.sections.scheduledImport.never", { defaultValue: "Never" })@@ -100,6 +80,15 @@ export function ScheduledImportSection({ draft, setDraft }: Props) {</span></label>+ {draft.scheduledImportEnabled && (+ <div className="rounded-md border border-amber-200 bg-amber-50 px-3 py-2 text-xs text-amber-900 dark:border-amber-900/50 dark:bg-amber-950/40 dark:text-amber-200">+ {t("settings.sections.scheduledImport.privacyNotice", {+ defaultValue:+ "Files from the selected directory may be copied into this project and sent to your configured LLM during ingest. Removed files are not automatically deleted from the project.",+ })}+ </div>+ )}+<div className="space-y-2"><Label>{t("settings.sections.scheduledImport.directory", {diff --git a/src/components/settings/settings-view.tsx b/src/components/settings/settings-view.tsxindex b8985ae4..9659d715 100644--- a/src/components/settings/settings-view.tsx+++ b/src/components/settings/settings-view.tsx@@ -317,12 +317,22 @@ export function SettingsView() {const newScheduledImport = {enabled: draft.scheduledImportEnabled,path: draft.scheduledImportPath,- interval: draft.scheduledImportInterval,+ interval: Math.max(1, Math.min(1440, draft.scheduledImportInterval || 60)),lastScan: scheduledImportConfig.lastScan,}setScheduledImportConfig(newScheduledImport)if (project) {await saveScheduledImportConfig(project.path, newScheduledImport)+ const { startScheduledImport, stopScheduledImport } = await import("@/lib/scheduled-import")+ if (+ newScheduledImport.enabled &&+ newScheduledImport.path &&+ newScheduledImport.interval > 0+ ) {+ startScheduledImport(project, newScheduledImport)+ } else {+ stopScheduledImport()+ }}setMaxHistoryMessages(draft.maxHistoryMessages)@@ -345,7 +355,6 @@ export function SettingsView() {scheduledImportConfig,setMaxHistoryMessages,outputLanguage,- project,])const body = useMemo(() => {diff --git a/src/i18n/en.json b/src/i18n/en.jsonindex 5ac18767..e125eb27 100644--- a/src/i18n/en.json+++ b/src/i18n/en.json@@ -115,20 +115,6 @@"noResults": "No results found","startSearching": "Type to search wiki pages"},- "lint": {- "title": "Lint",- "runLint": "Run Lint",- "running": "Running...",- "semantic": "Semantic (LLM)",- "allClear": "All clear — no issues found",- "orphan": "Orphan Page",- "brokenLink": "Broken Link",- "noOutlinks": "No Outbound Links",- "contradiction": "Contradiction",- "stale": "Stale Content",- "missingPage": "Missing Page",- "suggestion": "Suggestion"- },"lint": {"title": "Wiki Lint","semantic": "Semantic (LLM)",@@ -286,6 +272,7 @@"enable": "Enable scheduled import","directory": "Monitor Directory","directoryHelp": "Relative path from project root (e.g., 'raw', 'docs/input'). Files in this directory will be automatically imported. Defaults to 'raw'.",+ "privacyNotice": "Files from the selected directory may be copied into this project and sent to your configured LLM during ingest. Removed files are not automatically deleted from the project.","browse": "Browse","selectDirectory": "Select Directory to Monitor","interval": "Scan Interval (minutes)",diff --git a/src/i18n/i18n-parity.test.ts b/src/i18n/i18n-parity.test.tsindex 48a189d3..c8ed88c3 100644--- a/src/i18n/i18n-parity.test.ts+++ b/src/i18n/i18n-parity.test.ts@@ -11,6 +11,9 @@* anyone notices in the UI.*/import { describe, it, expect } from "vitest"+import { dirname, join } from "node:path"+import { fileURLToPath } from "node:url"+import { readFileSync } from "node:fs"import en from "./en.json"import zh from "./zh.json"@@ -30,9 +33,27 @@ function flattenKeys(obj: unknown, prefix = ""): string[] {}describe("i18n bundle parity (en.json ↔ zh.json)", () => {+ const i18nDir = dirname(fileURLToPath(import.meta.url))const enKeys = new Set(flattenKeys(en))const zhKeys = new Set(flattenKeys(zh))+ it("does not contain duplicate top-level JSON keys", () => {+ const findDuplicates = (fileName: string) => {+ const text = readFileSync(join(i18nDir, fileName), "utf8")+ const seen = new Set<string>()+ const duplicates = new Set<string>()+ for (const match of text.matchAll(/^ "([^"]+)":/gm)) {+ const key = match[1]+ if (seen.has(key)) duplicates.add(key)+ seen.add(key)+ }+ return [...duplicates].sort()+ }++ expect(findDuplicates("en.json"), "duplicate top-level keys in en.json").toEqual([])+ expect(findDuplicates("zh.json"), "duplicate top-level keys in zh.json").toEqual([])+ })+it("every en.json key is also in zh.json", () => {const missing = [...enKeys].filter((k) => !zhKeys.has(k)).sort()expect(diff --git a/src/i18n/zh.json b/src/i18n/zh.jsonindex c261623a..4eb0fa56 100644--- a/src/i18n/zh.json+++ b/src/i18n/zh.json@@ -47,6 +47,28 @@"aiOutputLanguageHint": "所有 AI 生成的内容(Wiki 页面、聊天回复、Research 结果)都将使用此语言。您可以在设置 → 输出 中稍后更改。","pickLanguage": "选择语言..."},+ "templates": {+ "research": {+ "name": "研究",+ "description": "用于深度研究,包含假设跟踪和方法论记录"+ },+ "reading": {+ "name": "阅读",+ "description": "跟踪书籍的人物、主题、情节线和章节笔记"+ },+ "personal": {+ "name": "个人成长",+ "description": "记录目标、习惯、反思和成长日志"+ },+ "business": {+ "name": "商务",+ "description": "管理团队会议、决策、项目和干系人背景"+ },+ "general": {+ "name": "通用",+ "description": "最小化初始结构,适合从空白开始"+ }+ },"sources": {"title": "原始资料","import": "导入",@@ -93,20 +115,6 @@"noResults": "未找到相关结果","startSearching": "输入关键词搜索 Wiki 页面"},- "lint": {- "title": "检查",- "runLint": "开始检查",- "running": "检查中...",- "semantic": "语义检查(LLM)",- "allClear": "一切正常,未发现问题",- "orphan": "孤立页面",- "brokenLink": "失效链接",- "noOutlinks": "无出链",- "contradiction": "内容矛盾",- "stale": "内容过时",- "missingPage": "缺失页面",- "suggestion": "建议"- },"lint": {"title": "Wiki 检查","semantic": "语义检查 (LLM)",@@ -264,6 +272,7 @@"enable": "启用定时导入","directory": "监控目录","directoryHelp": "相对于项目根目录的路径(如 'raw'、'docs/input')。该目录中的文件将被自动导入。默认为 'raw'。",+ "privacyNotice": "所选目录中的文件可能会被复制到当前项目,并在提取时发送给你配置的 LLM。源目录中删除文件不会自动删除项目内已导入内容。","browse": "浏览","selectDirectory": "选择监控目录","interval": "扫描间隔(分钟)",diff --git a/src/lib/reset-project-state.ts b/src/lib/reset-project-state.tsindex 32353f69..6d660060 100644--- a/src/lib/reset-project-state.ts+++ b/src/lib/reset-project-state.ts@@ -40,11 +40,12 @@ export async function resetProjectState(): Promise<void> {// Module-level caches — load in parallel and clear each, surfacing any// failure instead of swallowing it.- const [queueMod, dedupQueueMod, graphMod, fileSyncMod] = await Promise.allSettled([+ const [queueMod, dedupQueueMod, graphMod, fileSyncMod, scheduledImportMod] = await Promise.allSettled([import("@/lib/ingest-queue"),import("@/lib/dedup-queue"),import("@/lib/graph-relevance"),import("@/lib/project-file-sync"),+ import("@/lib/scheduled-import"),])if (queueMod.status === "fulfilled") {@@ -90,4 +91,14 @@ export async function resetProjectState(): Promise<void> {} else {console.warn("[Reset Project State] Failed to load project-file-sync:", fileSyncMod.reason)}++ if (scheduledImportMod.status === "fulfilled") {+ try {+ scheduledImportMod.value.stopScheduledImport()+ } catch (err) {+ console.warn("[Reset Project State] stopScheduledImport failed:", err)+ }+ } else {+ console.warn("[Reset Project State] Failed to load scheduled-import:", scheduledImportMod.reason)+ }}diff --git a/src/lib/scheduled-import.test.ts b/src/lib/scheduled-import.test.tsnew file mode 100644index 00000000..87d3bcd8--- /dev/null+++ b/src/lib/scheduled-import.test.ts@@ -0,0 +1,67 @@+import { describe, expect, it } from "vitest"+import {+ scheduledImportDestinationForFile,+ shouldSkipScheduledImportFile,+} from "./scheduled-import"++describe("scheduled import path handling", () => {+ const projectPath = "/Users/me/wiki-project"++ it("preserves nested relative paths for external directories", () => {+ const dest = scheduledImportDestinationForFile(+ projectPath,+ "/Users/me/inbox",+ {+ name: "report.pdf",+ path: "/Users/me/inbox/a/report.pdf",+ },+ )++ expect(dest).toBe(+ "/Users/me/wiki-project/raw/sources/scheduled-import/a/report.pdf",+ )+ })++ it("does not copy files that are already under raw/sources", () => {+ const dest = scheduledImportDestinationForFile(+ projectPath,+ `${projectPath}/raw/sources`,+ {+ name: "source.md",+ path: `${projectPath}/raw/sources/source.md`,+ },+ )++ expect(dest).toBe(`${projectPath}/raw/sources/source.md`)+ })++ it("sanitizes Windows-unsafe destination path segments", () => {+ const dest = scheduledImportDestinationForFile(+ projectPath,+ "/Users/me/inbox",+ {+ name: "ignored.md",+ path: "/Users/me/inbox/CON/Article: Why?.md",+ },+ )++ expect(dest).toBe(+ "/Users/me/wiki-project/raw/sources/scheduled-import/_CON/Article_ Why_.md",+ )+ })++ it("skips project internals and generated wiki/cache files", () => {+ expect(+ shouldSkipScheduledImportFile(projectPath, `${projectPath}/.llm-wiki/db.json`),+ ).toBe(true)+ expect(+ shouldSkipScheduledImportFile(projectPath, `${projectPath}/wiki/index.md`),+ ).toBe(true)+ expect(+ shouldSkipScheduledImportFile(+ projectPath,+ `${projectPath}/raw/sources/.cache/source.pdf.txt`,+ ),+ ).toBe(true)+ })+})diff --git a/src/lib/scheduled-import.ts b/src/lib/scheduled-import.tsindex 28560f5d..11400d2a 100644--- a/src/lib/scheduled-import.ts+++ b/src/lib/scheduled-import.ts@@ -1,304 +1,362 @@-import { listDirectory, copyFile, preprocessFile, getFileMd5, createDirectory, fileExists, readFile, writeFile } from "@/commands/fs"-import { enqueueIngest } from "@/lib/ingest-queue"-import { normalizePath, getFileName } from "@/lib/path-utils"-import { hasUsableLlm } from "@/lib/has-usable-llm"-import { useWikiStore, type ScheduledImportConfig } from "@/stores/wiki-store"-import type { FileNode } from "@/types/wiki"--// Supported file extensions for import-const IMPORTABLE_EXTENSIONS = new Set([- "md", "mdx", "txt", "rtf", "pdf",- "html", "htm", "xml",- "doc", "docx", "xls", "xlsx", "ppt", "pptx",- "odt", "ods", "odp", "epub", "pages", "numbers", "key",- "json", "jsonl", "csv", "tsv", "yaml", "yml", "ndjson",- "py", "js", "ts", "jsx", "tsx", "rs", "go", "java",- "c", "cpp", "h", "rb", "php", "swift", "sql", "sh",-])+import {+ copyFile,+ fileExists,+ getFileMd5,+ listDirectory,+ preprocessFile,+ readFile,+ writeFile,+} from "@/commands/fs"+import type { FileNode, WikiProject } from "@/types/wiki"+import { normalizePath } from "@/lib/path-utils"+import { useWikiStore } from "@/stores/wiki-store"+import type { ScheduledImportConfig } from "@/stores/wiki-store"+import {+ loadScheduledImportConfig,+ saveScheduledImportConfig,+} from "@/lib/project-store"+import {+ enqueueSourceIngest,+ isIngestableSourcePath,+} from "@/lib/source-lifecycle"++interface ImportDb {+ files: Record<string, string>+ lastScan: number | null+}++interface ImportDbStore {+ version: 1+ directories: Record<string, ImportDb>+}-// Hidden config folder name in the monitored directory-const IMPORT_DB_FOLDER = ".llm-wiki-imported"-const IMPORT_DB_FILE = "db.json"+type ScanOptions = {+ runId?: number+}++const EMPTY_DB: ImportDb = {+ files: {},+ lastScan: null,+}let scanTimer: ReturnType<typeof setInterval> | null = nulllet scanning = false+let activeRunId = 0++const DB_PATH = ".llm-wiki/scheduled-import-db.json"+const LEGACY_DB_DIR = ".llm-wiki-imported"+const SCHEDULED_IMPORT_DIR = "scheduled-import"+const RESERVED_WINDOWS_NAMES = new Set([+ "con",+ "prn",+ "aux",+ "nul",+ "com1",+ "com2",+ "com3",+ "com4",+ "com5",+ "com6",+ "com7",+ "com8",+ "com9",+ "lpt1",+ "lpt2",+ "lpt3",+ "lpt4",+ "lpt5",+ "lpt6",+ "lpt7",+ "lpt8",+ "lpt9",+])-interface ImportDbEntry {- md5: string- importedAt: number+function emptyStore(): ImportDbStore {+ return { version: 1, directories: {} }}-interface ImportDb {- files: Record<string, ImportDbEntry>- lastScan: number+function dbFilePath(projectPath: string): string {+ return `${normalizePath(projectPath)}/${DB_PATH}`}-/**- * Get all files recursively from a directory tree (FileNode structure).- */-function collectFiles(nodes: FileNode[], prefix: string = ""): Array<{ name: string; path: string }> {- const files: Array<{ name: string; path: string }> = []- for (const node of nodes) {- if (node.is_dir && node.children) {- files.push(...collectFiles(node.children, `${prefix}${node.name}/`))- } else if (!node.is_dir) {- files.push({ name: `${prefix}${node.name}`, path: node.path })- }+function dbDirectoryKey(importPath: string): string {+ return normalizePath(importPath)+}++function cloneDb(db: ImportDb): ImportDb {+ return {+ files: { ...db.files },+ lastScan: db.lastScan,}- return files}-/**- * Check if a file has an importable extension.- */-function isImportableFile(fileName: string): boolean {- const ext = fileName.split(".").pop()?.toLowerCase() ?? ""- return IMPORTABLE_EXTENSIONS.has(ext)+function isPathInside(path: string, parent: string): boolean {+ const normalizedPath = normalizePath(path)+ const normalizedParent = normalizePath(parent).replace(/\/+$/, "")+ return (+ normalizedPath === normalizedParent ||+ normalizedPath.startsWith(`${normalizedParent}/`)+ )+}++function projectSubpath(projectPath: string, relPath: string): string {+ return `${normalizePath(projectPath)}/${relPath}`+}++function sanitizePathSegment(segment: string): string {+ let value = segment+ .replace(/[<>:"|?*\x00-\x1F]/g, "_")+ .replace(/[. ]+$/g, "")+ .trim()++ if (!value) {+ value = "_"+ }++ const stem = value.split(".")[0]?.toLowerCase() ?? value.toLowerCase()+ if (RESERVED_WINDOWS_NAMES.has(stem)) {+ value = `_${value}`+ }++ return value}-/**- * Get the path to the import db folder within the monitored directory.- */-function getDbFolderPath(importPath: string): string {- return `${importPath}/${IMPORT_DB_FOLDER}`+function safeRelativePath(path: string): string {+ const parts = normalizePath(path)+ .split("/")+ .filter((part) => part && part !== "." && part !== "..")+ .map(sanitizePathSegment)++ return parts.length > 0 ? parts.join("/") : "_"}-/**- * Get the path to the import db.json file.- */-function getDbFilePath(importPath: string): string {- return `${importPath}/${IMPORT_DB_FOLDER}/${IMPORT_DB_FILE}`+export function isScheduledImportInternalPath(path: string): boolean {+ const parts = normalizePath(path).split("/")+ return parts.includes(LEGACY_DB_DIR) || parts.includes(".llm-wiki")}-/**- * Ensure the .llm-wiki-imported/ directory exists in the monitored directory.- */-async function ensureImportDb(importPath: string): Promise<void> {- const dbFolder = getDbFolderPath(importPath)- try {- await createDirectory(dbFolder)- } catch {- // Directory may already exist+export function shouldSkipScheduledImportFile(+ projectPath: string,+ filePath: string,+): boolean {+ const path = normalizePath(filePath)+ const project = normalizePath(projectPath)++ if (isScheduledImportInternalPath(path)) {+ return true+ }++ if (isPathInside(path, projectSubpath(project, "wiki"))) {+ return true}++ if (isPathInside(path, projectSubpath(project, "raw/sources/.cache"))) {+ return true+ }++ const name = path.split("/").pop() ?? ""+ return name.startsWith(".")}-/**- * Load the import database from .llm-wiki-imported/db.json.- * Returns an empty database if the file doesn't exist.- */-async function loadImportDb(importPath: string): Promise<ImportDb> {- const dbFilePath = getDbFilePath(importPath)- try {- const exists = await fileExists(dbFilePath)- if (!exists) return { files: {}, lastScan: 0 }- const content = await readFile(dbFilePath)- const parsed = JSON.parse(content)- return {- files: parsed.files || {},- lastScan: parsed.lastScan || 0,+export function scheduledImportDestinationForFile(+ projectPath: string,+ importPath: string,+ file: Pick<FileNode, "path" | "name">,+): string {+ const project = normalizePath(projectPath)+ const source = normalizePath(file.path)+ const sourcesRoot = projectSubpath(project, "raw/sources")++ if (isPathInside(source, sourcesRoot)) {+ return source+ }++ const importRoot = normalizePath(importPath).replace(/\/+$/, "")+ const relative =+ source === importRoot || !source.startsWith(`${importRoot}/`)+ ? file.name+ : source.slice(importRoot.length + 1)++ return `${sourcesRoot}/${SCHEDULED_IMPORT_DIR}/${safeRelativePath(relative)}`+}++function collectFiles(nodes: FileNode[]): FileNode[] {+ const files: FileNode[] = []+ for (const node of nodes) {+ if (!node.is_dir) {+ files.push(node)+ } else if (node.children) {+ files.push(...collectFiles(node.children))}- } catch {- return { files: {}, lastScan: 0 }}+ return files}-/**- * Save the import database to .llm-wiki-imported/db.json.- */-async function saveImportDb(importPath: string, db: ImportDb): Promise<void> {- const dbFilePath = getDbFilePath(importPath)+async function loadDbStore(projectPath: string): Promise<ImportDbStore> {+ const path = dbFilePath(projectPath)try {- await writeFile(dbFilePath, JSON.stringify(db, null, 2))+ if (!(await fileExists(path))) {+ return emptyStore()+ }+ const content = await readFile(path)+ const parsed = JSON.parse(content) as Partial<ImportDbStore>+ if (!parsed.directories || typeof parsed.directories !== "object") {+ return emptyStore()+ }+ return {+ version: 1,+ directories: parsed.directories as Record<string, ImportDb>,+ }} catch (err) {- console.error("[Scheduled Import] Failed to save import db:", err)+ console.warn("Failed to load scheduled import database:", err)+ return emptyStore()}}-/**- * Scan the monitored directory and import new or modified files.- * Uses MD5 hashing stored in .llm-wiki-imported/db.json to detect changes.- */-export async function scanAndImport(projectPath: string, importPath: string): Promise<void> {- if (scanning) {- console.log("[Scheduled Import] Scan already in progress, skipping")- return- }+async function loadImportDb(+ projectPath: string,+ importPath: string,+): Promise<ImportDb> {+ const store = await loadDbStore(projectPath)+ const db = store.directories[dbDirectoryKey(importPath)]+ return db ? cloneDb(db) : cloneDb(EMPTY_DB)+}++async function saveImportDb(+ projectPath: string,+ importPath: string,+ db: ImportDb,+): Promise<void> {+ const store = await loadDbStore(projectPath)+ store.directories[dbDirectoryKey(importPath)] = cloneDb(db)+ await writeFile(dbFilePath(projectPath), JSON.stringify(store, null, 2))+}++function isCurrentProject(projectId: string): boolean {+ return useWikiStore.getState().project?.id === projectId+}++function isCurrentRun(projectId: string, runId?: number): boolean {+ return isCurrentProject(projectId) && (runId === undefined || runId === activeRunId)+}- const pp = normalizePath(projectPath)- const ip = normalizePath(importPath)+export async function scanAndImport(+ project: WikiProject,+ importPath: string,+ options: ScanOptions = {},+): Promise<void> {+ if (!importPath || scanning) returnscanning = true- console.log(`[Scheduled Import] Starting scan of ${ip}`)+ const projectPath = normalizePath(project.path)+ const importRoot = normalizePath(importPath)try {- // Ensure the hidden config folder exists- await ensureImportDb(ip)-- // Load existing import database- const db = await loadImportDb(ip)-- // List all files in the monitored directory- let monitoredFiles: Array<{ name: string; path: string }> = []- try {- const tree = await listDirectory(ip)- monitoredFiles = collectFiles(tree)- } catch (err) {- console.error("[Scheduled Import] Failed to list monitored directory:", err)+ if (!isCurrentRun(project.id, options.runId)) {return}- // Filter to importable files only, excluding the hidden config folder- const dbFolderPrefix = `${ip}/${IMPORT_DB_FOLDER}`- const importableFiles = monitoredFiles.filter(f => {- // Skip files inside .llm-wiki-imported/ (compare normalized paths)- if (normalizePath(f.path).startsWith(dbFolderPrefix)) {- return false+ const tree = await listDirectory(importRoot)+ const db = await loadImportDb(projectPath, importRoot)+ const nextDb: ImportDb = {+ files: {},+ lastScan: Date.now(),+ }+ const llmConfig = useWikiStore.getState().llmConfig+ const destPaths: string[] = []++ for (const file of collectFiles(tree)) {+ const sourcePath = normalizePath(file.path)+ if (+ shouldSkipScheduledImportFile(projectPath, sourcePath) ||+ !isIngestableSourcePath(sourcePath)+ ) {+ continue}- return isImportableFile(f.name)- })- if (importableFiles.length === 0) {- console.log("[Scheduled Import] No importable files found")- return- }+ if (!isCurrentRun(project.id, options.runId)) {+ return+ }- console.log(`[Scheduled Import] Found ${importableFiles.length} importable files`)+ const key = sourcePath+ const md5 = await getFileMd5(sourcePath)+ nextDb.files[key] = md5- const llmConfig = useWikiStore.getState().llmConfig- const hasLlm = hasUsableLlm(llmConfig)- let importedCount = 0- let skippedCount = 0-- for (const file of importableFiles) {- try {- // Use relative path (from monitored dir) as the key- const relativePath = file.name- const currentMd5 = await getFileMd5(file.path)- const existing = db.files[relativePath]-- // Check if file has changed- if (existing && existing.md5 === currentMd5) {- skippedCount++- continue- }-- // File is new or modified - import it- const baseName = getFileName(file.path) || file.name- const destPath = `${pp}/raw/sources/${baseName}`-- // Copy file to sources directory (skip if source IS the destination)- const normalizedSource = normalizePath(file.path)- const normalizedDest = normalizePath(destPath)- if (normalizedSource !== normalizedDest) {- await copyFile(file.path, destPath)- }-- // Preprocess the file- preprocessFile(destPath).catch(() => {})-- // Enqueue for ingest if LLM is configured- if (hasLlm) {- const project = useWikiStore.getState().project- if (project) {- await enqueueIngest(project.id, destPath)- }- }-- // Update the database with the new MD5- db.files[relativePath] = {- md5: currentMd5,- importedAt: Date.now(),- }-- importedCount++- console.log(`[Scheduled Import] ${existing ? "Updated" : "Imported"}: ${baseName}`)- } catch (err) {- console.error(`[Scheduled Import] Failed to process ${file.name}:`, err)+ if (db.files[key] === md5) {+ continue}++ const destPath = scheduledImportDestinationForFile(projectPath, importRoot, file)+ if (normalizePath(destPath) !== sourcePath) {+ await copyFile(sourcePath, destPath)+ }+ destPaths.push(destPath)}- // Save the updated database- db.lastScan = Date.now()- await saveImportDb(ip, db)+ if (!isCurrentRun(project.id, options.runId)) {+ return+ }- // Update last scan time in the store- const config = useWikiStore.getState().scheduledImportConfig- const updatedConfig = { ...config, lastScan: Date.now() }- useWikiStore.getState().setScheduledImportConfig(updatedConfig)- const { saveScheduledImportConfig } = await import("@/lib/project-store")- await saveScheduledImportConfig(projectPath, updatedConfig)+ await saveImportDb(projectPath, importRoot, nextDb)- // Refresh file tree- const { listDirectory: listDir } = await import("@/commands/fs")- const tree = await listDir(pp)- useWikiStore.getState().setFileTree(tree)- useWikiStore.getState().bumpDataVersion()+ if (destPaths.length > 0) {+ await Promise.all(destPaths.map((path) => preprocessFile(path).catch(() => {})))+ if (isCurrentRun(project.id, options.runId)) {+ await enqueueSourceIngest(project, destPaths, llmConfig)+ const projectTree = await listDirectory(projectPath)+ useWikiStore.getState().setFileTree(projectTree)+ useWikiStore.getState().bumpDataVersion()+ }+ }- console.log(`[Scheduled Import] Scan complete: ${importedCount} imported, ${skippedCount} skipped`)+ const currentConfig = (await loadScheduledImportConfig(projectPath)) ?? {+ enabled: false,+ path: importRoot,+ interval: 60,+ lastScan: null,+ }+ await saveScheduledImportConfig(projectPath, {+ ...currentConfig,+ lastScan: nextDb.lastScan,+ })++ if (isCurrentProject(project.id)) {+ useWikiStore.getState().setScheduledImportConfig({+ ...currentConfig,+ lastScan: nextDb.lastScan,+ })+ }} catch (err) {- console.error("[Scheduled Import] Scan failed:", err)+ console.error("Scheduled import scan failed:", err)} finally {scanning = false}}-/**- * Resolve the absolute path for scheduled import.- * If the config path is relative (not absolute), prepend the project path.- * If the config path is empty, use the default "raw/sources" directory.- */-export function resolveImportPath(projectPath: string, configPath: string): string {- const pp = normalizePath(projectPath)- const path = configPath || "raw/sources"- if (path.startsWith("/") || path.match(/^[a-zA-Z]:[/\\]/)) {- return normalizePath(path)- }- return `${pp}/${path}`-}--/**- * Start the scheduled import timer.- */-export function startScheduledImport(projectPath: string, config: ScheduledImportConfig): void {+export function startScheduledImport(+ project: WikiProject,+ config: ScheduledImportConfig,+): void {stopScheduledImport()- if (!config.enabled || config.interval <= 0) {+ if (!config.enabled || !config.path || config.interval <= 0) {return}- const pp = normalizePath(projectPath)- const ip = resolveImportPath(pp, config.path)+ const runId = ++activeRunId+ const intervalMs = Math.max(1, Math.min(1440, config.interval)) * 60 * 1000- console.log(`[Scheduled Import] Starting with interval ${config.interval} minutes, path: ${ip}`)+ void scanAndImport(project, config.path, { runId })- // Run first scan immediately- scanAndImport(pp, ip).catch((err) => {- console.error("[Scheduled Import] Initial scan failed:", err)- })-- // Set up interval- const intervalMs = config.interval * 60 * 1000scanTimer = setInterval(() => {- scanAndImport(pp, ip).catch((err) => {- console.error("[Scheduled Import] Scheduled scan failed:", err)- })+ void scanAndImport(project, config.path, { runId })}, intervalMs)}-/**- * Stop the scheduled import timer.- */export function stopScheduledImport(): void {+ activeRunId += 1if (scanTimer) {clearInterval(scanTimer)scanTimer = null- console.log("[Scheduled Import] Stopped")}}--/**- * Check if scheduled import is currently running.- */-export function isScanning(): boolean {- return scanning-}