Skip to content

LLM Wiki is a cross-platform desktop application that turns your documents into an organized, interlinked knowledge base — automatically. Instead of traditional RAG (retrieve-and-answer from scratch every time), the LLM incrementally builds and maintains a persistent wiki from your sources。

TypeScript

Fix source import and graph UX issues

nash_su8f0a488ff7c3
diff --git a/src/components/editor/wiki-editor.tsx b/src/components/editor/wiki-editor.tsx
index 390f72ba..a083bae3 100644
--- a/src/components/editor/wiki-editor.tsx
+++ b/src/components/editor/wiki-editor.tsx
@@ -1,4 +1,4 @@
-import { useMemo, useRef, useState } from "react"
+import { useCallback, useEffect, useMemo, useRef, useState } from "react"
import { Editor, rootCtx, defaultValueCtx } from "@milkdown/kit/core"
import { commonmark } from "@milkdown/kit/preset/commonmark"
import { gfm } from "@milkdown/kit/preset/gfm"
@@ -16,10 +16,11 @@ import { WikiReader } from "@/components/editor/wiki-reader"
interface WikiEditorInnerProps {
content: string
- onSave: (markdown: string) => void
+ onSave: (markdown: string, options?: { immediate?: boolean }) => void
+ onMarkdownChange: (markdown: string) => void
}
-function WikiEditorInner({ content, onSave }: WikiEditorInnerProps) {
+function WikiEditorInner({ content, onSave, onMarkdownChange }: WikiEditorInnerProps) {
// Milkdown fires `markdownUpdated` once on initial parse before any
// user interaction. That one emit must not be forwarded as a save,
// otherwise just opening a file can overwrite its content with
@@ -40,6 +41,7 @@ function WikiEditorInner({ content, onSave }: WikiEditorInnerProps) {
initialEmitConsumedRef.current = true
return
}
+ onMarkdownChange(markdown)
onSave(markdown)
})
})
@@ -56,7 +58,7 @@ function WikiEditorInner({ content, onSave }: WikiEditorInnerProps) {
interface WikiEditorProps {
content: string
- onSave: (markdown: string) => void
+ onSave: (markdown: string, options?: { immediate?: boolean }) => void
}
function wrapBareMathBlocks(text: string): string {
@@ -92,17 +94,39 @@ export function WikiEditor({ content, onSave }: WikiEditorProps) {
)
const processedBody = useMemo(() => wrapBareMathBlocks(body), [body])
+ const latestBodyRef = useRef(processedBody)
+
+ useEffect(() => {
+ latestBodyRef.current = processedBody
+ }, [processedBody])
const handleSave = useMemo(
- () => (markdown: string) => onSave(rawBlock + markdown),
+ () => (markdown: string, options?: { immediate?: boolean }) => onSave(rawBlock + markdown, options),
[onSave, rawBlock],
)
+ const saveLatestNow = useCallback(() => {
+ onSave(rawBlock + latestBodyRef.current, { immediate: true })
+ }, [onSave, rawBlock])
+
return (
- <div className="relative h-full overflow-auto">
+ <div
+ className="relative h-full overflow-auto"
+ tabIndex={-1}
+ onKeyDownCapture={(event) => {
+ if (mode !== "edit") return
+ if ((event.metaKey || event.ctrlKey) && event.key.toLowerCase() === "s") {
+ event.preventDefault()
+ saveLatestNow()
+ }
+ }}
+ >
<button
type="button"
- onClick={() => setMode((m) => (m === "read" ? "edit" : "read"))}
+ onClick={() => {
+ if (mode === "edit") saveLatestNow()
+ setMode((m) => (m === "read" ? "edit" : "read"))
+ }}
title={mode === "read" ? "Edit (raw markdown)" : "Done editing"}
className="absolute right-3 top-3 z-10 inline-flex items-center gap-1 rounded-md border border-border/60 bg-background/90 px-2 py-1 text-xs text-muted-foreground shadow-sm hover:bg-accent hover:text-foreground"
>
@@ -119,7 +143,13 @@ export function WikiEditor({ content, onSave }: WikiEditorProps) {
<MilkdownProvider>
<div className="prose prose-invert min-w-0 max-w-none overflow-hidden p-6">
{frontmatter && <FrontmatterPanel data={frontmatter} />}
- <WikiEditorInner content={processedBody} onSave={handleSave} />
+ <WikiEditorInner
+ content={processedBody}
+ onSave={handleSave}
+ onMarkdownChange={(markdown) => {
+ latestBodyRef.current = markdown
+ }}
+ />
</div>
</MilkdownProvider>
)}
diff --git a/src/components/graph/graph-view.tsx b/src/components/graph/graph-view.tsx
index 34482123..74ae9407 100644
--- a/src/components/graph/graph-view.tsx
+++ b/src/components/graph/graph-view.tsx
@@ -17,6 +17,7 @@ import { optimizeResearchTopic } from "@/lib/optimize-research-topic"
import { normalizePath } from "@/lib/path-utils"
import { applyGraphFilters, DEFAULT_GRAPH_FILTERS, hasActiveGraphFilters, type GraphFilterState } from "@/lib/graph-filters"
import { applyGraphSearch } from "@/lib/graph-search"
+import { wikiTypeLabel } from "@/lib/wiki-page-types"
import { useTranslation } from "react-i18next"
const NODE_TYPE_COLORS: Record<string, string> = {
@@ -33,6 +34,17 @@ const NODE_TYPE_COLORS: Record<string, string> = {
other: "#94a3b8", // slate-400
}
+const CUSTOM_NODE_COLORS = [
+ "#38bdf8",
+ "#34d399",
+ "#fbbf24",
+ "#fb7185",
+ "#a78bfa",
+ "#22d3ee",
+ "#f97316",
+ "#84cc16",
+]
+
const COMMUNITY_COLORS = [
"#60a5fa", // blue-400
"#4ade80", // green-400
@@ -52,9 +64,15 @@ type ColorMode = "type" | "community"
const BASE_NODE_SIZE = 8
const MAX_NODE_SIZE = 28
+const DEFAULT_NODE_SCALE = 1
+const DEFAULT_GRAPH_SPACING = 1
+const GRAPH_SPACING_DEBOUNCE_MS = 180
function nodeColor(type: string): string {
- return NODE_TYPE_COLORS[type] ?? NODE_TYPE_COLORS.other
+ if (NODE_TYPE_COLORS[type]) return NODE_TYPE_COLORS[type]
+ let hash = 0
+ for (const char of type) hash = (hash * 31 + char.charCodeAt(0)) >>> 0
+ return CUSTOM_NODE_COLORS[hash % CUSTOM_NODE_COLORS.length] ?? NODE_TYPE_COLORS.other
}
function hexToRgba(hex: string, alpha: number): string {
@@ -74,10 +92,16 @@ function mixColor(color1: string, color2: string, ratio: number): string {
return `#${r.toString(16).padStart(2, "0")}${g.toString(16).padStart(2, "0")}${b.toString(16).padStart(2, "0")}`
}
-function nodeSize(linkCount: number, maxLinks: number): number {
+function graphDensityScale(nodeCount: number): number {
+ if (nodeCount <= 150) return 1
+ return Math.max(0.35, Math.sqrt(150 / nodeCount))
+}
+
+function nodeSize(linkCount: number, maxLinks: number, nodeCount: number, userScale: number): number {
if (maxLinks === 0) return BASE_NODE_SIZE
const ratio = linkCount / maxLinks
- return BASE_NODE_SIZE + Math.sqrt(ratio) * (MAX_NODE_SIZE - BASE_NODE_SIZE)
+ const size = BASE_NODE_SIZE + Math.sqrt(ratio) * (MAX_NODE_SIZE - BASE_NODE_SIZE)
+ return size * graphDensityScale(nodeCount) * userScale
}
// --- Inner components ---
@@ -86,11 +110,23 @@ function nodeSize(linkCount: number, maxLinks: number): number {
const positionCache = new Map<string, { x: number; y: number }>()
let lastLayoutDataKey = ""
-function GraphLoader({ nodes, edges, colorMode }: { nodes: GraphNode[]; edges: GraphEdge[]; colorMode: ColorMode }) {
+function GraphLoader({
+ nodes,
+ edges,
+ colorMode,
+ nodeScale,
+ graphSpacing,
+}: {
+ nodes: GraphNode[]
+ edges: GraphEdge[]
+ colorMode: ColorMode
+ nodeScale: number
+ graphSpacing: number
+}) {
const loadGraph = useLoadGraph()
useEffect(() => {
- const dataKey = nodes.map((n) => n.id).sort().join(",") + "|" + edges.length
+ const dataKey = nodes.map((n) => n.id).sort().join(",") + "|" + edges.length + `|spacing:${graphSpacing.toFixed(2)}`
const needsLayout = dataKey !== lastLayoutDataKey
const graph = new Graph()
@@ -104,7 +140,7 @@ function GraphLoader({ nodes, edges, colorMode }: { nodes: GraphNode[]; edges: G
graph.addNode(node.id, {
x: cached?.x ?? Math.random() * 100,
y: cached?.y ?? Math.random() * 100,
- size: nodeSize(node.linkCount, maxLinks),
+ size: nodeSize(node.linkCount, maxLinks, nodes.length, nodeScale),
color,
label: node.label,
nodeType: node.type,
@@ -142,7 +178,7 @@ function GraphLoader({ nodes, edges, colorMode }: { nodes: GraphNode[]; edges: G
settings: {
...settings,
gravity: 1,
- scalingRatio: 2,
+ scalingRatio: graphSpacing * (nodes.length > 400 ? 3 : 2),
strongGravityMode: true,
barnesHutOptimize: nodes.length > 50,
},
@@ -156,7 +192,7 @@ function GraphLoader({ nodes, edges, colorMode }: { nodes: GraphNode[]; edges: G
}
loadGraph(graph)
- }, [loadGraph, nodes, edges, colorMode])
+ }, [loadGraph, nodes, edges, colorMode, nodeScale, graphSpacing])
return null
}
@@ -339,6 +375,9 @@ export function GraphView() {
const [showFilters, setShowFilters] = useState(false)
const [graphSearchOpen, setGraphSearchOpen] = useState(false)
const [graphSearch, setGraphSearch] = useState("")
+ const [nodeScale, setNodeScale] = useState(DEFAULT_NODE_SCALE)
+ const [graphSpacingDraft, setGraphSpacingDraft] = useState(DEFAULT_GRAPH_SPACING)
+ const [graphSpacing, setGraphSpacing] = useState(DEFAULT_GRAPH_SPACING)
const [filters, setFilters] = useState<GraphFilterState>(() => ({
...DEFAULT_GRAPH_FILTERS,
hiddenTypes: new Set(),
@@ -346,6 +385,7 @@ export function GraphView() {
}))
const [nodeMenu, setNodeMenu] = useState<{ nodeId: string; x: number; y: number } | null>(null)
const graphContainerRef = useRef<HTMLDivElement>(null)
+ const researchDialogTokenRef = useRef(0)
// i18n node type labels (populated after mount to support language switching)
const [nodeTypeLabels, setNodeTypeLabels] = useState<Record<string, string>>({})
const graphSearchInputRef = useRef<HTMLInputElement>(null)
@@ -355,6 +395,7 @@ export function GraphView() {
loading: boolean
topic: string
queries: string[]
+ dismissKey?: string
} | null>(null)
const lastLoadedVersion = useRef(-1)
@@ -395,6 +436,13 @@ export function GraphView() {
})
}, [t])
+ // Spacing changes trigger ForceAtlas2 layout through GraphLoader's dataKey.
+ // Keep the slider responsive while debouncing the expensive relayout.
+ useEffect(() => {
+ const timer = window.setTimeout(() => setGraphSpacing(graphSpacingDraft), GRAPH_SPACING_DEBOUNCE_MS)
+ return () => window.clearTimeout(timer)
+ }, [graphSpacingDraft])
+
useEffect(() => {
if (dataVersion !== lastLoadedVersion.current) {
loadGraph()
@@ -441,16 +489,37 @@ export function GraphView() {
hiddenTypes: new Set(),
hiddenNodeIds: new Set(),
})
+ setNodeScale(DEFAULT_NODE_SCALE)
+ setGraphSpacingDraft(DEFAULT_GRAPH_SPACING)
+ setGraphSpacing(DEFAULT_GRAPH_SPACING)
setNodeMenu(null)
}, [])
- const handleResearchClick = useCallback(async (gapTitle: string, gapDescription: string, gapType: string) => {
+ const knowledgeGapKey = useCallback((gap: KnowledgeGap) => (
+ `gap:${gap.type}:${gap.title}:${gap.nodeIds.join(",")}`
+ ), [])
+
+ const visibleKnowledgeGaps = useMemo(
+ () => knowledgeGaps.filter((gap) => !dismissedInsights.has(knowledgeGapKey(gap))),
+ [dismissedInsights, knowledgeGaps, knowledgeGapKey],
+ )
+
+ const dismissInsight = useCallback((key: string, ids?: Set<string>) => {
+ setDismissedInsights((prev) => new Set([...prev, key]))
+ if (ids && highlightedNodes.size === ids.size && [...ids].every((id) => highlightedNodes.has(id))) {
+ setHighlightedNodes(new Set())
+ }
+ }, [highlightedNodes])
+
+ const handleResearchClick = useCallback(async (gapTitle: string, gapDescription: string, gapType: string, dismissKey?: string) => {
const store = useWikiStore.getState()
if (!store.project) return
const pp = normalizePath(store.project.path)
+ const token = researchDialogTokenRef.current + 1
+ researchDialogTokenRef.current = token
// Show loading state
- setResearchDialog({ loading: true, topic: "", queries: [] })
+ setResearchDialog({ loading: true, topic: "", queries: [], dismissKey })
try {
// Read overview and purpose for context
@@ -467,10 +536,12 @@ export function GraphView() {
overview,
purpose,
)
- setResearchDialog({ loading: false, topic: result.topic, queries: result.searchQueries })
+ if (researchDialogTokenRef.current !== token) return
+ setResearchDialog({ loading: false, topic: result.topic, queries: result.searchQueries, dismissKey })
} catch {
+ if (researchDialogTokenRef.current !== token) return
// Fallback: use raw title
- setResearchDialog({ loading: false, topic: gapTitle, queries: [gapTitle] })
+ setResearchDialog({ loading: false, topic: gapTitle, queries: [gapTitle], dismissKey })
}
}, [])
@@ -485,6 +556,10 @@ export function GraphView() {
store.searchApiConfig,
researchDialog.queries,
)
+ if (researchDialog.dismissKey) {
+ setDismissedInsights((prev) => new Set([...prev, researchDialog.dismissKey!]))
+ setHighlightedNodes(new Set())
+ }
setResearchDialog(null)
}, [researchDialog])
@@ -534,6 +609,13 @@ export function GraphView() {
acc[n.type] = (acc[n.type] ?? 0) + 1
return acc
}, {})
+ const nodeTypeLabelMap = useMemo(() => {
+ const labels = { ...nodeTypeLabels }
+ for (const type of Object.keys(typeCounts)) {
+ labels[type] ??= wikiTypeLabel(type)
+ }
+ return labels
+ }, [nodeTypeLabels, typeCounts])
const filteredGraph = useMemo(
() => applyGraphFilters(nodes, edges, filters),
@@ -690,7 +772,7 @@ export function GraphView() {
<Layers className="h-3 w-3" />
{t("graph.community")}
</Button>
- {(surprisingConns.filter((c) => !dismissedInsights.has(c.key)).length > 0 || knowledgeGaps.length > 0) && (
+ {(surprisingConns.filter((c) => !dismissedInsights.has(c.key)).length > 0 || visibleKnowledgeGaps.length > 0) && (
<Button
variant={showInsights ? "secondary" : "ghost"}
size="sm"
@@ -705,7 +787,7 @@ export function GraphView() {
<Lightbulb className="h-3 w-3" />
{t("graph.insights")}
<span className="rounded bg-muted px-1 text-[10px]">
- {surprisingConns.filter((c) => !dismissedInsights.has(c.key)).length + knowledgeGaps.length}
+ {surprisingConns.filter((c) => !dismissedInsights.has(c.key)).length + visibleKnowledgeGaps.length}
</span>
</Button>
)}
@@ -789,7 +871,13 @@ export function GraphView() {
},
}}
>
- <GraphLoader nodes={searchedGraph.nodes} edges={searchedGraph.edges} colorMode={colorMode} />
+ <GraphLoader
+ nodes={searchedGraph.nodes}
+ edges={searchedGraph.edges}
+ colorMode={colorMode}
+ nodeScale={nodeScale}
+ graphSpacing={graphSpacing}
+ />
<EventHandler onNodeClick={handleNodeClick} onNodeContextMenu={handleNodeContextMenu} />
<HighlightManager highlightedNodes={searchActive ? searchedGraph.matchedNodeIds : highlightedNodes} />
<ZoomControls />
@@ -857,10 +945,47 @@ export function GraphView() {
</div>
</div>
+ <div className="space-y-2">
+ <div className="font-medium text-muted-foreground">{t("graph.displayTuning")}</div>
+ <label className="block space-y-1">
+ <div className="flex items-center justify-between">
+ <span>{t("graph.nodeSize")}</span>
+ <span className="text-muted-foreground">{Math.round(nodeScale * 100)}%</span>
+ </div>
+ <input
+ type="range"
+ min={0.5}
+ max={1.5}
+ step={0.05}
+ value={nodeScale}
+ onChange={(e) => setNodeScale(Number(e.target.value))}
+ className="w-full"
+ />
+ </label>
+ <label className="block space-y-1">
+ <div className="flex items-center justify-between">
+ <span>{t("graph.spacing")}</span>
+ <span className="text-muted-foreground">{Math.round(graphSpacingDraft * 100)}%</span>
+ </div>
+ <input
+ type="range"
+ min={0.6}
+ max={2.2}
+ step={0.05}
+ value={graphSpacingDraft}
+ onChange={(e) => setGraphSpacingDraft(Number(e.target.value))}
+ className="w-full"
+ />
+ </label>
+ <p className="text-[11px] leading-relaxed text-muted-foreground">
+ {t("graph.displayTuningHint")}
+ </p>
+ </div>
+
<div className="space-y-1.5">
<div className="font-medium text-muted-foreground">{t("graph.nodeTypes")}</div>
<div className="grid grid-cols-2 gap-1">
- {Object.entries(nodeTypeLabels)
+ {Object.entries(nodeTypeLabelMap)
.filter(([type]) => (typeCounts[type] ?? 0) > 0)
.map(([type, label]) => (
<label key={type} className="flex min-w-0 items-center gap-1.5">
@@ -977,7 +1102,7 @@ export function GraphView() {
colorMode === "type" ? (
<div className="flex flex-col gap-0.5 max-h-48 overflow-y-auto legend-scroll" style={{ direction: "rtl" }}>
<div className="flex flex-col gap-0.5" style={{ direction: "ltr" }}>
- {Object.entries(nodeTypeLabels)
+ {Object.entries(nodeTypeLabelMap)
.filter(([type]) => (typeCounts[type] ?? 0) > 0)
.map(([type, label]) => {
const isHidden = filters.hiddenTypes.has(type)
@@ -1003,8 +1128,8 @@ export function GraphView() {
<span
className="inline-block h-3 w-3 rounded-full shrink-0 shadow-sm"
style={{
- backgroundColor: isHidden ? "#94a3b8" : NODE_TYPE_COLORS[type],
- boxShadow: `0 0 4px ${hexToRgba(isHidden ? "#94a3b8" : NODE_TYPE_COLORS[type] ?? "#94a3b8", 0.4)}`,
+ backgroundColor: isHidden ? "#94a3b8" : nodeColor(type),
+ boxShadow: `0 0 4px ${hexToRgba(isHidden ? "#94a3b8" : nodeColor(type), 0.4)}`,
}}
/>
<span className={hoveredType === type ? "text-foreground font-medium" : "text-muted-foreground"}>
@@ -1098,8 +1223,7 @@ export function GraphView() {
className="shrink-0 rounded p-0.5 text-muted-foreground hover:bg-destructive/20 hover:text-destructive"
onClick={(e) => {
e.stopPropagation()
- setDismissedInsights((prev) => new Set([...prev, conn.key]))
- if (isActive) setHighlightedNodes(new Set())
+ dismissInsight(conn.key, ids)
}}
>
<X className="h-3.5 w-3.5" />
@@ -1116,14 +1240,15 @@ export function GraphView() {
)}
{/* Knowledge Gaps */}
- {knowledgeGaps.length > 0 && (
+ {visibleKnowledgeGaps.length > 0 && (
<div>
<div className="flex items-center gap-1.5 mb-2 text-xs font-semibold text-foreground">
<AlertTriangle className="h-3.5 w-3.5 text-amber-500" />
{t("graph.knowledgeGaps")}
</div>
<div className="flex flex-col gap-2">
- {knowledgeGaps.map((gap, i) => {
+ {visibleKnowledgeGaps.map((gap, i) => {
+ const gapKey = knowledgeGapKey(gap)
const ids = new Set(gap.nodeIds)
const isActive = highlightedNodes.size > 0 &&
[...ids].every((id) => highlightedNodes.has(id)) &&
@@ -1134,7 +1259,19 @@ export function GraphView() {
className={`rounded-lg border p-3 text-sm cursor-pointer transition-colors ${isActive ? "bg-amber-500/10 border-amber-500/40" : "hover:bg-muted/50"}`}
onClick={() => setHighlightedNodes(isActive ? new Set() : ids)}
>
- <div className="font-medium text-xs text-foreground mb-1">{gap.title}</div>
+ <div className="flex items-start justify-between gap-2 mb-1">
+ <div className="font-medium text-xs text-foreground">{gap.title}</div>
+ <button
+ className="shrink-0 rounded p-0.5 text-muted-foreground hover:bg-destructive/20 hover:text-destructive"
+ title={t("common.dismiss")}
+ onClick={(e) => {
+ e.stopPropagation()
+ dismissInsight(gapKey, ids)
+ }}
+ >
+ <X className="h-3.5 w-3.5" />
+ </button>
+ </div>
<p className="text-xs text-muted-foreground mb-2">{gap.description}</p>
<p className="text-xs text-muted-foreground/80 italic mb-2">{gap.suggestion}</p>
<Button
@@ -1143,7 +1280,7 @@ export function GraphView() {
className="h-7 text-xs gap-1"
onClick={(e) => {
e.stopPropagation()
- handleResearchClick(gap.title, gap.description, gap.type)
+ handleResearchClick(gap.title, gap.description, gap.type, gapKey)
}}
>
<Search className="h-3.5 w-3.5" />
@@ -1169,14 +1306,15 @@ export function GraphView() {
<Search className="h-4 w-4 text-primary" />
<span className="font-medium text-sm">{t("graph.deepResearch")}</span>
</div>
- {!researchDialog.loading && (
- <button
- className="p-1 rounded hover:bg-muted text-muted-foreground"
- onClick={() => setResearchDialog(null)}
- >
- <X className="h-4 w-4" />
- </button>
- )}
+ <button
+ className="p-1 rounded hover:bg-muted text-muted-foreground"
+ onClick={() => {
+ researchDialogTokenRef.current += 1
+ setResearchDialog(null)
+ }}
+ >
+ <X className="h-4 w-4" />
+ </button>
</div>
{researchDialog.loading ? (
@@ -1221,7 +1359,14 @@ export function GraphView() {
</div>
</div>
<div className="flex justify-end gap-2">
- <Button variant="outline" size="sm" onClick={() => setResearchDialog(null)}>
+ <Button
+ variant="outline"
+ size="sm"
+ onClick={() => {
+ researchDialogTokenRef.current += 1
+ setResearchDialog(null)
+ }}
+ >
{t("graph.cancel")}
</Button>
<Button
diff --git a/src/components/layout/activity-panel.tsx b/src/components/layout/activity-panel.tsx
index ecd4ef6a..842501db 100644
--- a/src/components/layout/activity-panel.tsx
+++ b/src/components/layout/activity-panel.tsx
@@ -181,7 +181,7 @@ export function ActivityPanel() {
// Build status text
let statusText = ""
if (queueSummary.processing > 0 || queueSummary.pending > 0) {
- const done = queueSummary.total - queueSummary.pending - queueSummary.processing
+ const done = queueSummary.completed + queueSummary.failed
statusText = `Queue: ${done}/${queueSummary.total}`
if (queueSummary.failed > 0) statusText += ` (${queueSummary.failed} failed)`
} else if (runningCount > 0) {
@@ -255,7 +255,7 @@ export function ActivityPanel() {
<div className="flex items-center justify-between text-[10px] text-muted-foreground mb-1 gap-2">
<span>Ingest Queue</span>
<span className="flex-1 text-right">
- {queueSummary.total - queueSummary.pending - queueSummary.processing}/{queueSummary.total} complete
+ {queueSummary.completed + queueSummary.failed}/{queueSummary.total} complete
</span>
{queueSummary.pending + queueSummary.processing >= 2 && (
<button
@@ -279,7 +279,7 @@ export function ActivityPanel() {
<div className="h-1.5 rounded-full bg-muted overflow-hidden">
<div
className="h-full rounded-full bg-primary transition-all"
- style={{ width: `${((queueSummary.total - queueSummary.pending - queueSummary.processing) / Math.max(queueSummary.total, 1)) * 100}%` }}
+ style={{ width: `${((queueSummary.completed + queueSummary.failed) / Math.max(queueSummary.total, 1)) * 100}%` }}
/>
</div>
</div>
diff --git a/src/components/layout/knowledge-tree.tsx b/src/components/layout/knowledge-tree.tsx
index 1368f792..dd172b15 100644
--- a/src/components/layout/knowledge-tree.tsx
+++ b/src/components/layout/knowledge-tree.tsx
@@ -9,7 +9,7 @@ import { readFile, listDirectory } from "@/commands/fs"
import type { FileNode } from "@/types/wiki"
import { normalizePath } from "@/lib/path-utils"
import { cascadeDeleteWikiPagesWithRefs } from "@/lib/wiki-page-delete"
-import { inferWikiTypeFromPath } from "@/lib/wiki-page-types"
+import { inferWikiTypeFromPath, wikiTypeLabel } from "@/lib/wiki-page-types"
interface WikiPageInfo {
path: string
@@ -32,7 +32,9 @@ const TYPE_CONFIG: Record<string, { icon: typeof FileText; label: string; color:
query: { icon: HelpCircle, label: "Queries", color: "text-green-500", order: 9 },
}
-const DEFAULT_CONFIG = { icon: FileText, label: "Other", color: "text-muted-foreground", order: 99 }
+function typeConfig(type: string): { icon: typeof FileText; label: string; color: string; order: number } {
+ return TYPE_CONFIG[type] ?? { icon: FileText, label: wikiTypeLabel(type), color: "text-muted-foreground", order: 99 }
+}
export function KnowledgeTree() {
const project = useWikiStore((s) => s.project)
@@ -135,8 +137,9 @@ export function KnowledgeTree() {
// Sort groups by configured order
const sortedGroups = [...grouped.entries()].sort((a, b) => {
- const orderA = TYPE_CONFIG[a[0]]?.order ?? DEFAULT_CONFIG.order
- const orderB = TYPE_CONFIG[b[0]]?.order ?? DEFAULT_CONFIG.order
+ const orderA = typeConfig(a[0]).order
+ const orderB = typeConfig(b[0]).order
+ if (orderA === orderB) return wikiTypeLabel(a[0]).localeCompare(wikiTypeLabel(b[0]))
return orderA - orderB
})
@@ -163,7 +166,7 @@ export function KnowledgeTree() {
)}
{sortedGroups.map(([type, items]) => {
- const config = TYPE_CONFIG[type] ?? DEFAULT_CONFIG
+ const config = typeConfig(type)
const Icon = config.icon
const isExpanded = expandedTypes.has(type)
diff --git a/src/components/layout/preview-panel.tsx b/src/components/layout/preview-panel.tsx
index d14ea95d..dfc79b7d 100644
--- a/src/components/layout/preview-panel.tsx
+++ b/src/components/layout/preview-panel.tsx
@@ -46,26 +46,33 @@ export function PreviewPanel() {
})
}, [selectedFile, setFileContent])
+ const writeNow = useCallback((path: string, markdown: string, syncStore = false) => {
+ writeFile(path, markdown)
+ .then(() => {
+ lastLoadedRef.current = markdown
+ if (syncStore) setFileContent(markdown)
+ })
+ .catch((err) => console.error("Failed to save:", err))
+ }, [setFileContent])
+
const handleSave = useCallback(
- (markdown: string) => {
+ (markdown: string, options?: { immediate?: boolean }) => {
if (!selectedFile) return
// Ignore no-op saves from the editor's initial re-emit. Only write
// when the user has actually changed the content relative to the
// last disk read.
if (markdown === lastLoadedRef.current) return
if (saveTimerRef.current) clearTimeout(saveTimerRef.current)
+ if (options?.immediate) {
+ setFileContent(markdown)
+ writeNow(selectedFile, markdown, true)
+ return
+ }
saveTimerRef.current = setTimeout(() => {
- writeFile(selectedFile, markdown)
- .then(() => {
- // Our own write becomes the new "last loaded" — subsequent
- // re-emits from Milkdown that match this content must not
- // trigger another save.
- lastLoadedRef.current = markdown
- })
- .catch((err) => console.error("Failed to save:", err))
+ writeNow(selectedFile, markdown, true)
}, 1000)
},
- [selectedFile]
+ [selectedFile, setFileContent, writeNow]
)
useEffect(() => {
diff --git a/src/components/settings/sections/source-watch-section.tsx b/src/components/settings/sections/source-watch-section.tsx
index 913983b3..79e2800b 100644
--- a/src/components/settings/sections/source-watch-section.tsx
+++ b/src/components/settings/sections/source-watch-section.tsx
@@ -1,4 +1,4 @@
-import { useMemo } from "react"
+import { useEffect, useMemo, useState } from "react"
import { FolderSync, ShieldAlert } from "lucide-react"
import { useTranslation } from "react-i18next"
import type { SettingsDraft, DraftSetter } from "../settings-types"
@@ -28,11 +28,37 @@ export function SourceWatchSection({ draft, setDraft, projectReady }: Props) {
const { t } = useTranslation()
const config = normalizeSourceWatchConfig(draft.sourceWatchConfig)
const selected = useMemo(() => new Set(config.includeExtensions), [config.includeExtensions])
+ const [editingList, setEditingList] = useState<"dirs" | "extensions" | "globs" | null>(null)
+ const [excludeDirsText, setExcludeDirsText] = useState(() => joinList(config.excludeDirs))
+ const [excludeExtensionsText, setExcludeExtensionsText] = useState(() => joinList(config.excludeExtensions))
+ const [excludeGlobsText, setExcludeGlobsText] = useState(() => joinList(config.excludeGlobs))
+ const excludeDirsKey = config.excludeDirs.join("\0")
+ const excludeExtensionsKey = config.excludeExtensions.join("\0")
+ const excludeGlobsKey = config.excludeGlobs.join("\0")
+
+ useEffect(() => {
+ if (editingList !== "dirs") setExcludeDirsText(joinList(config.excludeDirs))
+ if (editingList !== "extensions") setExcludeExtensionsText(joinList(config.excludeExtensions))
+ if (editingList !== "globs") setExcludeGlobsText(joinList(config.excludeGlobs))
+ }, [editingList, excludeDirsKey, excludeExtensionsKey, excludeGlobsKey])
const updateConfig = (patch: Partial<typeof config>) => {
setDraft("sourceWatchConfig", normalizeSourceWatchConfig({ ...config, ...patch }))
}
+ const finishListEdit = (field: "dirs" | "extensions" | "globs", value: string) => {
+ const normalized = normalizeSourceWatchConfig({
+ ...config,
+ ...(field === "dirs" ? { excludeDirs: updateListValue(value) } : {}),
+ ...(field === "extensions" ? { excludeExtensions: updateListValue(value) } : {}),
+ ...(field === "globs" ? { excludeGlobs: updateListValue(value) } : {}),
+ })
+ setEditingList(null)
+ if (field === "dirs") setExcludeDirsText(joinList(normalized.excludeDirs))
+ if (field === "extensions") setExcludeExtensionsText(joinList(normalized.excludeExtensions))
+ if (field === "globs") setExcludeGlobsText(joinList(normalized.excludeGlobs))
+ }
+
const toggleExtension = (ext: string, checked: boolean) => {
const next = new Set(config.includeExtensions)
if (checked) {
@@ -179,8 +205,13 @@ export function SourceWatchSection({ draft, setDraft, projectReady }: Props) {
{t("settings.sections.sourceWatch.excludeDirs", { defaultValue: "Excluded folders" })}
</span>
<textarea
- value={joinList(config.excludeDirs)}
- onChange={(event) => updateConfig({ excludeDirs: updateListValue(event.target.value) })}
+ value={excludeDirsText}
+ onFocus={() => setEditingList("dirs")}
+ onChange={(event) => {
+ setExcludeDirsText(event.target.value)
+ updateConfig({ excludeDirs: updateListValue(event.target.value) })
+ }}
+ onBlur={() => finishListEdit("dirs", excludeDirsText)}
placeholder={t("settings.sections.sourceWatch.excludeDirsPlaceholder", {
defaultValue: ".git, node_modules, drafts, subdir/drafts",
})}
@@ -200,8 +231,13 @@ export function SourceWatchSection({ draft, setDraft, projectReady }: Props) {
{t("settings.sections.sourceWatch.excludeExtensions", { defaultValue: "Excluded file extensions" })}
</span>
<textarea
- value={joinList(config.excludeExtensions)}
- onChange={(event) => updateConfig({ excludeExtensions: updateListValue(event.target.value) })}
+ value={excludeExtensionsText}
+ onFocus={() => setEditingList("extensions")}
+ onChange={(event) => {
+ setExcludeExtensionsText(event.target.value)
+ updateConfig({ excludeExtensions: updateListValue(event.target.value) })
+ }}
+ onBlur={() => finishListEdit("extensions", excludeExtensionsText)}
placeholder={t("settings.sections.sourceWatch.excludeExtensionsPlaceholder", {
defaultValue: "tmp, bak, exe, dll, iso, dmg",
})}
@@ -221,8 +257,13 @@ export function SourceWatchSection({ draft, setDraft, projectReady }: Props) {
{t("settings.sections.sourceWatch.excludeGlobs", { defaultValue: "Excluded filename patterns" })}
</span>
<textarea
- value={joinList(config.excludeGlobs)}
- onChange={(event) => updateConfig({ excludeGlobs: updateListValue(event.target.value) })}
+ value={excludeGlobsText}
+ onFocus={() => setEditingList("globs")}
+ onChange={(event) => {
+ setExcludeGlobsText(event.target.value)
+ updateConfig({ excludeGlobs: updateListValue(event.target.value) })
+ }}
+ onBlur={() => finishListEdit("globs", excludeGlobsText)}
disabled={!projectReady || !config.enabled}
rows={2}
className="w-full rounded-md border border-input bg-background px-2 py-1 text-sm"
diff --git a/src/components/sources/sources-view.tsx b/src/components/sources/sources-view.tsx
index 7dee9c52..a191ee0d 100644
--- a/src/components/sources/sources-view.tsx
+++ b/src/components/sources/sources-view.tsx
@@ -30,6 +30,7 @@ export function SourcesView() {
const setFileContent = useWikiStore((s) => s.setFileContent)
const setFileTree = useWikiStore((s) => s.setFileTree)
const llmConfig = useWikiStore((s) => s.llmConfig)
+ const sourceWatchConfig = useWikiStore((s) => s.sourceWatchConfig)
const dataVersion = useWikiStore((s) => s.dataVersion)
const [sources, setSources] = useState<FileNode[]>([])
const [importing, setImporting] = useState(false)
@@ -137,7 +138,7 @@ export function SourcesView() {
setImporting(true)
const paths = Array.isArray(selected) ? selected : [selected]
try {
- await importSourceFiles(project, paths, llmConfig)
+ await importSourceFiles(project, paths, llmConfig, sourceWatchConfig)
await loadSources()
} finally {
setImporting(false)
@@ -156,7 +157,7 @@ export function SourcesView() {
setImporting(true)
try {
- await importSourceFolder(project, selected, llmConfig)
+ await importSourceFolder(project, selected, llmConfig, sourceWatchConfig)
await loadSources()
} catch (err) {
console.error(`Failed to import folder:`, err)
diff --git a/src/components/ui/scroll-area.tsx b/src/components/ui/scroll-area.tsx
index 72000e5c..56e1f60f 100644
--- a/src/components/ui/scroll-area.tsx
+++ b/src/components/ui/scroll-area.tsx
@@ -17,6 +17,7 @@ function ScrollArea({
>
<ScrollAreaPrimitive.Viewport
data-slot="scroll-area-viewport"
+ tabIndex={0}
className="size-full rounded-[inherit] transition-[color,box-shadow] outline-none focus-visible:ring-[3px] focus-visible:ring-ring/50 focus-visible:outline-1"
>
{children}
diff --git a/src/i18n/en.json b/src/i18n/en.json
index 02ac8689..05080c05 100644
--- a/src/i18n/en.json
+++ b/src/i18n/en.json
@@ -585,6 +585,10 @@
"hideIsolated": "Hide isolated nodes",
"maxLinks": "Max links",
"maxLinksHint": "Hide nodes above this link count",
+ "displayTuning": "Display tuning",
+ "nodeSize": "Node size",
+ "spacing": "Spacing",
+ "displayTuningHint": "Use these controls to keep dense graphs readable on smaller screens.",
"nodeTypes": "Node types",
"hiddenNodes": "Hidden nodes",
"show": "Show",
diff --git a/src/i18n/zh.json b/src/i18n/zh.json
index 7c9f8dd7..54c47e49 100644
--- a/src/i18n/zh.json
+++ b/src/i18n/zh.json
@@ -585,6 +585,10 @@
"hideIsolated": "隐藏孤立节点",
"maxLinks": "最大链接数",
"maxLinksHint": "隐藏链接数超过此值的节点",
+ "displayTuning": "显示调节",
+ "nodeSize": "节点大小",
+ "spacing": "间距",
+ "displayTuningHint": "用这些控制项让大型关系图在小屏幕上更容易阅读。",
"nodeTypes": "节点类型",
"hiddenNodes": "已隐藏节点",
"show": "显示",
diff --git a/src/lib/extract-source-images.test.ts b/src/lib/extract-source-images.test.ts
new file mode 100644
index 00000000..6bff1152
--- /dev/null
+++ b/src/lib/extract-source-images.test.ts
@@ -0,0 +1,23 @@
+import { describe, expect, it } from "vitest"
+import { findLocalMarkdownImageRefs } from "./extract-source-images"
+
+describe("findLocalMarkdownImageRefs", () => {
+ it("extracts Obsidian and markdown local image references", () => {
+ const refs = findLocalMarkdownImageRefs(`
+![[attachments/chart.png]]
+![Figure](images/plot%201.jpg "title")
+![Remote](https://example.com/a.png)
+![[attachments/chart.png|400]]
+`)
+ expect(refs).toEqual(["attachments/chart.png", "images/plot 1.jpg"])
+ })
+
+ it("ignores non-image links and remote/data references", () => {
+ const refs = findLocalMarkdownImageRefs(`
+![Doc](notes/page.md)
+![Data](data:image/png;base64,abc)
+![[draft.txt]]
+`)
+ expect(refs).toEqual([])
+ })
+})
diff --git a/src/lib/extract-source-images.ts b/src/lib/extract-source-images.ts
index 4ba92344..67af8d2b 100644
--- a/src/lib/extract-source-images.ts
+++ b/src/lib/extract-source-images.ts
@@ -14,6 +14,7 @@
* image and the markdown line uses that instead.
*/
import { invoke } from "@tauri-apps/api/core"
+import { copyFile, createDirectory, fileExists, readFileAsBase64 } from "@/commands/fs"
import { getFileName, normalizePath } from "@/lib/path-utils"
/** Mirrors `commands::extract_images::SavedImage` on the Rust side. */
@@ -40,6 +41,99 @@ const SUPPORTED_OFFICE_EXTS = ["pptx", "docx", "ppt", "doc"] as const
// Note: ppt / doc (legacy binary formats) won't actually work — they
// aren't ZIP. Listed for completeness; Rust side will return Err which
// the caller treats as "no images" gracefully.
+const MARKDOWN_IMAGE_EXTS = new Set([
+ "png",
+ "jpg",
+ "jpeg",
+ "gif",
+ "webp",
+ "bmp",
+ "tif",
+ "tiff",
+ "svg",
+])
+
+function dirname(path: string): string {
+ const idx = normalizePath(path).lastIndexOf("/")
+ return idx >= 0 ? normalizePath(path).slice(0, idx) : ""
+}
+
+function isRemoteOrDataImageRef(raw: string): boolean {
+ return /^(https?:|data:|blob:|file:|tauri:)/i.test(raw)
+}
+
+function cleanMarkdownImageRef(raw: string): string {
+ const stripped = raw.trim().replace(/^<(.+)>$/, "$1")
+ try {
+ return decodeURIComponent(stripped)
+ } catch {
+ return stripped
+ }
+}
+
+function imageMimeType(path: string): string {
+ const ext = getFileName(path).split(".").pop()?.toLowerCase() ?? ""
+ switch (ext) {
+ case "jpg":
+ case "jpeg":
+ return "image/jpeg"
+ case "png":
+ return "image/png"
+ case "gif":
+ return "image/gif"
+ case "webp":
+ return "image/webp"
+ case "bmp":
+ return "image/bmp"
+ case "svg":
+ return "image/svg+xml"
+ case "tif":
+ case "tiff":
+ return "image/tiff"
+ default:
+ return "application/octet-stream"
+ }
+}
+
+function uniqueDestName(index: number, sourcePath: string): string {
+ const name = getFileName(sourcePath).replace(/[<>:"|?*\x00-\x1f]/g, "_")
+ return `${String(index).padStart(3, "0")}-${name}`
+}
+
+async function sha256OfFile(path: string): Promise<string> {
+ const bytes = await readFileAsBase64(path)
+ const raw = Uint8Array.from(atob(bytes.base64), (c) => c.charCodeAt(0))
+ const digest = await crypto.subtle.digest("SHA-256", raw)
+ return Array.from(new Uint8Array(digest))
+ .map((b) => b.toString(16).padStart(2, "0"))
+ .join("")
+}
+
+export function findLocalMarkdownImageRefs(markdown: string): string[] {
+ const refs: string[] = []
+ const seen = new Set<string>()
+
+ const add = (raw: string) => {
+ const ref = cleanMarkdownImageRef(raw.split("#")[0].split("|")[0])
+ if (!ref || isRemoteOrDataImageRef(ref)) return
+ const ext = getFileName(ref).split(".").pop()?.toLowerCase() ?? ""
+ if (!MARKDOWN_IMAGE_EXTS.has(ext)) return
+ const key = normalizePath(ref)
+ if (seen.has(key)) return
+ seen.add(key)
+ refs.push(ref)
+ }
+
+ for (const match of markdown.matchAll(/!\[\[([^\]]+)\]\]/g)) {
+ add(match[1] ?? "")
+ }
+
+ for (const match of markdown.matchAll(/!\[[^\]]*]\(([^)\s]+)(?:\s+["'][^"']*["'])?\)/g)) {
+ add(match[1] ?? "")
+ }
+
+ return refs
+}
/**
* Extract every embedded image from `sourcePath` and save them to
@@ -104,6 +198,62 @@ export async function extractAndSaveSourceImages(
}
}
+export async function extractAndSaveMarkdownImages(
+ projectPath: string,
+ sourcePath: string,
+ markdown: string,
+ slugOverride?: string,
+): Promise<SavedImage[]> {
+ const refs = findLocalMarkdownImageRefs(markdown)
+ if (refs.length === 0) return []
+
+ const pp = normalizePath(projectPath)
+ const sp = normalizePath(sourcePath)
+ const sourceDir = dirname(sp)
+ const slug = slugOverride ?? getFileName(sp).replace(/\.[^.]+$/, "")
+ const destDir = `${pp}/wiki/media/${slug}`
+ const images: SavedImage[] = []
+
+ try {
+ await createDirectory(destDir)
+ } catch (err) {
+ console.warn("[ingest:images] failed to create markdown image directory:", err)
+ return []
+ }
+
+ for (const ref of refs) {
+ const abs = normalizePath(
+ ref.startsWith("/") || /^[a-zA-Z]:/.test(ref) || ref.startsWith("\\\\")
+ ? ref
+ : `${sourceDir}/${ref}`,
+ )
+ try {
+ if (!(await fileExists(abs))) continue
+ const destName = uniqueDestName(images.length + 1, abs)
+ const dest = `${destDir}/${destName}`
+ await copyFile(abs, dest)
+ const sha256 = await sha256OfFile(dest)
+ images.push({
+ index: images.length + 1,
+ mimeType: imageMimeType(dest),
+ page: null,
+ width: 0,
+ height: 0,
+ relPath: `media/${slug}/${destName}`,
+ absPath: dest,
+ sha256,
+ })
+ } catch (err) {
+ console.warn(
+ `[ingest:images] markdown image import failed for "${ref}":`,
+ err instanceof Error ? err.message : err,
+ )
+ }
+ }
+
+ return images
+}
+
/**
* Build the markdown section to splice into `sourceContent` so the
* generation LLM sees the available images. Each image is referenced
diff --git a/src/lib/ingest-queue.test.ts b/src/lib/ingest-queue.test.ts
index 4b2e7461..43c32b1c 100644
--- a/src/lib/ingest-queue.test.ts
+++ b/src/lib/ingest-queue.test.ts
@@ -121,6 +121,7 @@ describe("ingest-queue — enqueue & basic processing", () => {
// Task should have been processed and removed
expect(mockAutoIngest).toHaveBeenCalledOnce()
expect(getQueue()).toHaveLength(0)
+ expect(getQueueSummary()).toMatchObject({ completed: 1, total: 1 })
})
it("persists queue to disk on enqueue", async () => {
@@ -332,10 +333,27 @@ describe("ingest-queue — clearCompletedTasks & summary", () => {
const summary = getQueueSummary()
expect(summary.failed).toBe(1)
+ expect(summary.completed).toBe(0)
expect(summary.pending).toBe(0)
expect(summary.total).toBe(1)
})
+ it("tracks successful completions until a new idle queue starts", async () => {
+ mockAutoIngest.mockResolvedValue(["wiki/sources/foo.md"])
+ await enqueueBatch(TEST_ID, [
+ { sourcePath: "a.md", folderContext: "" },
+ { sourcePath: "b.md", folderContext: "" },
+ ])
+ await flushMicrotasks(40)
+
+ expect(getQueueSummary()).toMatchObject({ completed: 2, failed: 0, total: 2 })
+
+ await enqueueIngest(TEST_ID, "c.md")
+ await flushMicrotasks(20)
+
+ expect(getQueueSummary()).toMatchObject({ completed: 1, failed: 0, total: 1 })
+ })
+
it("clearCompletedTasks drops failed tasks", async () => {
mockAutoIngest.mockRejectedValue(new Error("fail"))
await enqueueIngest(TEST_ID, "f.md")
diff --git a/src/lib/ingest-queue.ts b/src/lib/ingest-queue.ts
index 7060908a..2580a0ef 100644
--- a/src/lib/ingest-queue.ts
+++ b/src/lib/ingest-queue.ts
@@ -35,6 +35,7 @@ let currentProjectId = ""
let currentProjectPath = ""
let currentAbortController: AbortController | null = null
let lastWrittenFiles: string[] = [] // track files written by current ingest for cleanup
+let completedSinceIdle = 0
// Track whether any task has been processed since the last drain.
// Prevents the sweep from running on every idle/no-op call.
let processedSinceDrain = false
@@ -42,6 +43,10 @@ let processedSinceDrain = false
// cancels a long-running judgment instead of burning tokens.
let sweepAbortController: AbortController | null = null
+function resetQueueAccounting(): void {
+ completedSinceIdle = 0
+}
+
// ── Persistence ───────────────────────────────────────────────────────────
function queueFilePath(projectPath: string): string {
@@ -97,6 +102,9 @@ function upsertQueuedIngestTask(
sourcePath: string,
folderContext: string,
): string {
+ if (queue.length === 0 && !processing) {
+ resetQueueAccounting()
+ }
const normalizedSourcePath = normalizeSourcePathForQueue(sourcePath)
const pendingOrFailed = queue.find((t) =>
t.projectId === projectId &&
@@ -113,12 +121,12 @@ function upsertQueuedIngestTask(
return pendingOrFailed.id
}
- const processing = queue.find((t) =>
+ const processingTask = queue.find((t) =>
t.projectId === projectId &&
t.status === "processing" &&
sameQueuedSourcePath(t.sourcePath, normalizedSourcePath)
)
- const pendingRerun = processing
+ const pendingRerun = processingTask
? queue.find((t) =>
t.projectId === projectId &&
t.status === "pending" &&
@@ -340,12 +348,14 @@ export function getQueue(): readonly IngestTask[] {
/**
* Get queue summary.
*/
-export function getQueueSummary(): { pending: number; processing: number; failed: number; total: number } {
+export function getQueueSummary(): { pending: number; processing: number; failed: number; completed: number; total: number } {
+ const activeTotal = queue.length + completedSinceIdle
return {
pending: queue.filter((t) => t.status === "pending").length,
processing: queue.filter((t) => t.status === "processing").length,
failed: queue.filter((t) => t.status === "failed").length,
- total: queue.length,
+ completed: completedSinceIdle,
+ total: activeTotal,
}
}
@@ -370,6 +380,7 @@ export function clearQueueState(): void {
sweepAbortController = null
lastWrittenFiles = []
processedSinceDrain = false
+ resetQueueAccounting()
}
/**
@@ -415,6 +426,7 @@ export async function pauseQueue(): Promise<void> {
currentProjectPath = ""
lastWrittenFiles = []
processedSinceDrain = false
+ resetQueueAccounting()
}
// ── Restore on startup ───────────────────────────────────────────────────
@@ -436,6 +448,7 @@ export async function restoreQueue(
processing = false
currentAbortController = null
lastWrittenFiles = []
+ resetQueueAccounting()
currentProjectId = projectId
currentProjectPath = pp
@@ -579,6 +592,7 @@ async function processNext(projectId: string): Promise<void> {
currentAbortController = null
lastWrittenFiles = []
queue = queue.filter((t) => t.id !== next.id)
+ completedSinceIdle++
processedSinceDrain = true
await saveQueue(pp)
diff --git a/src/lib/ingest-sanitize.test.ts b/src/lib/ingest-sanitize.test.ts
index 6e1074e3..01dba74c 100644
--- a/src/lib/ingest-sanitize.test.ts
+++ b/src/lib/ingest-sanitize.test.ts
@@ -51,6 +51,19 @@ describe("sanitizeIngestedFileContent", () => {
)
})
+ it("repairs a missing opening frontmatter fence when the closing fence is present", () => {
+ const input =
+ "\n\ntype: entity\ntitle: \"Foo: Bar\"\nsources: [foo.pdf]\n---\n\n# Foo\n\nBody"
+ expect(sanitizeIngestedFileContent(input)).toBe(
+ "---\ntype: entity\ntitle: \"Foo: Bar\"\nsources: [foo.pdf]\n---\n\n# Foo\n\nBody",
+ )
+ })
+
+ it("does NOT invent frontmatter when a body line only looks like metadata", () => {
+ const input = "title: A research question\n\n# Notes\n\nBody"
+ expect(sanitizeIngestedFileContent(input)).toBe(input)
+ })
+
it("does NOT strip the word `frontmatter:` when it appears mid-document (in prose)", () => {
const input = "---\ntype: x\n---\n\nThe frontmatter: of this doc is above."
expect(sanitizeIngestedFileContent(input)).toBe(input)
diff --git a/src/lib/ingest-sanitize.ts b/src/lib/ingest-sanitize.ts
index d6a6bcf6..ccbdcaaa 100644
--- a/src/lib/ingest-sanitize.ts
+++ b/src/lib/ingest-sanitize.ts
@@ -33,7 +33,16 @@
* — semantically what the model wanted (a list of wikilinks),
* but not valid YAML flow syntax.
*
- * This sanitizer rewrites all three shapes into the standard
+ * 4. A frontmatter payload whose opening `---` is missing but whose
+ * closing `---` is present, e.g.
+ *
+ * type: entity
+ * title: Foo
+ * ---
+ *
+ * — common when the model starts "inside" the YAML block.
+ *
+ * This sanitizer rewrites these shapes into the standard
* `---\n…\n---\n` frontmatter form before write. It's deliberately
* conservative: each pattern is anchored at the very start of the
* document (or at top-level frontmatter scope), so a legitimate
@@ -64,6 +73,10 @@ export function sanitizeIngestedFileContent(content: string): string {
// frontmatter block".
cleaned = stripFrontmatterKeyPrefix(cleaned)
+ // (2.5) Repair a missing opening frontmatter fence when the model
+ // clearly emitted frontmatter lines followed by a closing fence.
+ cleaned = addMissingOpeningFrontmatterFence(cleaned)
+
// (3) Repair `key: [[a]], [[b]], [[c]]` lines inside the
// frontmatter block so they're valid YAML. Body wikilinks are
// left alone — those render fine via the wikilink → markdown
@@ -98,6 +111,30 @@ function stripFrontmatterKeyPrefix(content: string): string {
return content.slice(m[0].length)
}
+function addMissingOpeningFrontmatterFence(content: string): string {
+ if (/^[ \t]*---\s*(\r?\n|$)/.test(content)) return content
+
+ const lines = content.split(/\r?\n/)
+ const firstContentIdx = lines.findIndex((line) => line.trim().length > 0)
+ if (firstContentIdx < 0) return content
+
+ const first = lines[firstContentIdx].trim()
+ if (!/^(type|title|created|updated|tags|related|sources)\s*:/i.test(first)) {
+ return content
+ }
+
+ const searchEnd = Math.min(lines.length, firstContentIdx + 30)
+ for (let i = firstContentIdx + 1; i < searchEnd; i += 1) {
+ const trimmed = lines[i].trim()
+ if (trimmed === "---") {
+ return `---\n${lines.slice(firstContentIdx).join("\n")}`
+ }
+ if (/^#{1,6}\s+/.test(trimmed)) break
+ }
+
+ return content
+}
+
/**
* Inside the frontmatter block (between the opening `---` and the
* closing `---`), rewrite invalid wikilink-list lines. Lines
diff --git a/src/lib/ingest.prompt.test.ts b/src/lib/ingest.prompt.test.ts
index 0de0172f..83ee323b 100644
--- a/src/lib/ingest.prompt.test.ts
+++ b/src/lib/ingest.prompt.test.ts
@@ -72,6 +72,19 @@ describe("buildGenerationPrompt language directive", () => {
expect(prompt).toContain("my-paper.pdf")
})
+ it("makes project schema routing authoritative over default entity and concept folders", () => {
+ const prompt = buildGenerationPrompt(
+ "Use wiki/people/ for people. Use wiki/technologies/ for technical methods.",
+ "",
+ "",
+ "source.pdf",
+ )
+ expect(prompt).toContain("## Project Schema and Routing (AUTHORITATIVE)")
+ expect(prompt).toContain("write pages into those schema-defined folders")
+ expect(prompt).toContain("otherwise use wiki/entities/")
+ expect(prompt).not.toContain("Entity pages in wiki/entities/ for key entities")
+ })
+
it("respects user setting regardless of source content language", () => {
useWikiStore.getState().setOutputLanguage("English")
const prompt = buildGenerationPrompt("", "", "", "x.pdf", undefined, "私は日本語の文章を書きます")
diff --git a/src/lib/ingest.ts b/src/lib/ingest.ts
index a45ecb17..f1a00cf0 100644
--- a/src/lib/ingest.ts
+++ b/src/lib/ingest.ts
@@ -18,7 +18,9 @@ import { withProjectLock } from "@/lib/project-mutex"
import type { FileNode } from "@/types/wiki"
import {
extractAndSaveSourceImages,
+ extractAndSaveMarkdownImages,
buildImageMarkdownSection,
+ type SavedImage,
} from "@/lib/extract-source-images"
import { captionMarkdownImages, loadCaptionCache } from "@/lib/image-caption-pipeline"
import type { MultimodalConfig } from "@/stores/wiki-store"
@@ -38,6 +40,31 @@ const INGEST_GENERATION_TOKENS_512K = 32_768
const REVIEW_STAGE_MIN_SIGNAL_CHARS = 10_000
const REVIEW_STAGE_MIN_FILE_BLOCKS = 4
+function appendSavedImageRefsForCaption(content: string, images: SavedImage[]): string {
+ if (images.length === 0) return content
+ const refs = images
+ .map((img) => img.relPath)
+ .filter(Boolean)
+ .map((relPath) => `![](${relPath})`)
+ if (refs.length === 0) return content
+ return `${content}\n\n## Referenced Local Images\n\n${refs.join("\n")}\n`
+}
+
+function isSavedImagePromptUrl(projectPath: string, sourceSummarySlug: string, url: string): boolean {
+ return (
+ url.startsWith(`${projectPath}/wiki/media/${sourceSummarySlug}/`) ||
+ url.startsWith(`media/${sourceSummarySlug}/`)
+ )
+}
+
+function promptImageUrlToAbs(projectPath: string, url: string): string {
+ return url.startsWith("media/") ? `${projectPath}/wiki/${url}` : url
+}
+
+function stripWikiMediaAbsPaths(projectPath: string, content: string): string {
+ return content.split(`${projectPath}/wiki/media/`).join("media/")
+}
+
interface SourceChunk {
id: string
index: number
@@ -396,7 +423,9 @@ async function autoIngestImpl(
if (cachedFiles !== null) {
try {
console.log(`[ingest:diag] cache-hit branch: starting image extraction for ${sp}`)
- const savedImages = await extractAndSaveSourceImages(pp, sp, sourceSummarySlug)
+ let savedImages = await extractAndSaveSourceImages(pp, sp, sourceSummarySlug)
+ const markdownImages = await extractAndSaveMarkdownImages(pp, sp, sourceContent, sourceSummarySlug)
+ savedImages = [...savedImages, ...markdownImages]
console.log(`[ingest:diag] cache-hit branch: got ${savedImages.length} image(s)`)
if (savedImages.length > 0) {
// Caption first (populates the cache), THEN inject — the
@@ -424,11 +453,11 @@ async function autoIngestImpl(
const captionLlm = resolveCaptionConfig(mmCfg, llmConfig)
if (captionLlm) {
try {
- await captionMarkdownImages(pp, sourceContent, captionLlm, {
+ await captionMarkdownImages(pp, appendSavedImageRefsForCaption(sourceContent, savedImages), captionLlm, {
signal,
shouldCaption: (url) =>
- url.startsWith(`${pp}/wiki/media/${sourceSummarySlug}/`),
- urlToAbsPath: (url) => url,
+ isSavedImagePromptUrl(pp, sourceSummarySlug, url),
+ urlToAbsPath: (url) => promptImageUrlToAbs(pp, url),
concurrency: mmCfg.concurrency,
onProgress: (done, total) =>
activity.updateItem(activityId, {
@@ -487,7 +516,9 @@ async function autoIngestImpl(
// and returns [] on any error.
activity.updateItem(activityId, { detail: "Extracting embedded images..." })
console.log(`[ingest:diag] full-pipeline branch: starting image extraction for ${sp}`)
- const savedImages = await extractAndSaveSourceImages(pp, sp, sourceSummarySlug)
+ let savedImages = await extractAndSaveSourceImages(pp, sp, sourceSummarySlug)
+ const markdownImages = await extractAndSaveMarkdownImages(pp, sp, sourceContent, sourceSummarySlug)
+ savedImages = [...savedImages, ...markdownImages]
console.log(`[ingest:diag] full-pipeline branch: got ${savedImages.length} image(s)`)
if (savedImages.length > 0) {
console.log(
@@ -534,7 +565,10 @@ async function autoIngestImpl(
// (which renders read_file output directly) still shows them —
// that surface is "the source document as-is", separate from
// "the curated wiki knowledge".
- let enrichedSourceContent = sourceContent
+ let enrichedSourceContent = stripWikiMediaAbsPaths(
+ pp,
+ appendSavedImageRefsForCaption(sourceContent, markdownImages),
+ )
const mmCfg = useWikiStore.getState().multimodalConfig
const captionLlm = resolveCaptionConfig(mmCfg, llmConfig)
if (!mmCfg.enabled && savedImages.length > 0) {
@@ -551,27 +585,27 @@ async function autoIngestImpl(
} else if (
captionLlm &&
savedImages.length > 0 &&
- /!\[\]\(/.test(sourceContent)
+ /!\[\]\(/.test(enrichedSourceContent)
) {
activity.updateItem(activityId, { detail: "Captioning images..." })
const ourMediaPrefix = `${pp}/wiki/media/${sourceSummarySlug}/`
try {
- const result = await captionMarkdownImages(pp, sourceContent, captionLlm, {
+ const result = await captionMarkdownImages(pp, enrichedSourceContent, captionLlm, {
signal,
// Strict filter: only caption images we know we just
// extracted into this source's media directory. Skips any
// pre-existing markdown image refs the user may have typed
// into the source content (e.g. for hand-authored .md
// sources).
- shouldCaption: (url) => url.startsWith(ourMediaPrefix),
- urlToAbsPath: (url) => url, // already absolute in our extraction output
+ shouldCaption: (url) => url.startsWith(ourMediaPrefix) || isSavedImagePromptUrl(pp, sourceSummarySlug, url),
+ urlToAbsPath: (url) => promptImageUrlToAbs(pp, url),
concurrency: mmCfg.concurrency,
onProgress: (done, total) =>
activity.updateItem(activityId, {
detail: `Captioning images... ${done}/${total}`,
}),
})
- enrichedSourceContent = result.enrichedMarkdown
+ enrichedSourceContent = stripWikiMediaAbsPaths(pp, result.enrichedMarkdown)
console.log(
`[ingest:caption] images=${savedImages.length} fresh=${result.freshCaptions} cached=${result.cachedCaptions} failed=${result.failed}`,
)
@@ -1343,11 +1377,22 @@ export function buildGenerationPrompt(
`The original source file is: **${sourceFileName}**`,
`All wiki pages generated from this source MUST include this filename in their frontmatter \`sources\` field.`,
"",
+ schema
+ ? [
+ "## Project Schema and Routing (AUTHORITATIVE)",
+ schema,
+ "",
+ "Use this schema as the primary routing rule for page types and directories.",
+ "If it defines custom folders or distinctions (for example people, technologies, organizations, methods, or cases), write pages into those schema-defined folders instead of forcing them into wiki/entities/ or wiki/concepts/.",
+ "Use wiki/entities/ and wiki/concepts/ only when the schema does not provide a more specific destination.",
+ ].join("\n")
+ : "",
+ "",
"## What to generate",
"",
`1. A source summary page at **${summaryPath}** (MUST use this exact path)`,
- "2. Entity pages in wiki/entities/ for key entities identified in the analysis",
- "3. Concept pages in wiki/concepts/ for key concepts identified in the analysis",
+ "2. Entity or schema-defined typed pages for key named things identified in the analysis. Prefer schema-defined directories when present; otherwise use wiki/entities/.",
+ "3. Concept or schema-defined typed pages for key ideas, methods, techniques, and abstractions. Prefer schema-defined directories when present; otherwise use wiki/concepts/.",
"4. An updated wiki/index.md — add new entries to existing categories, preserve all existing entries",
"5. A log entry for wiki/log.md (just the new entry to append, format: ## [YYYY-MM-DD] ingest | Title)",
"6. An updated wiki/overview.md — a high-level summary of what the entire wiki covers, updated to reflect the newly ingested source. This should be a comprehensive 2-5 paragraph overview of ALL topics in the wiki, not just the new source.",
@@ -1367,7 +1412,7 @@ export function buildGenerationPrompt(
" write `related: [a, b]` with bare slugs.",
"",
"Required fields and types:",
- ` • type — one of: ${GENERATION_WIKI_TYPES.join(" | ")}`,
+ ` • type — one of the known types (${GENERATION_WIKI_TYPES.join(" | ")}), or a custom type explicitly defined by the project schema`,
" • title — string (quote it if it contains a colon, e.g. `title: \"Foo: Bar\"`)",
" • created — date in YYYY-MM-DD form (no quotes)",
" • updated — same as created",
@@ -1395,6 +1440,7 @@ export function buildGenerationPrompt(
"",
"Other rules:",
"- Use [[wikilink]] syntax in the BODY for cross-references between pages",
+ "- If you include images, use wiki-root-relative paths such as `media/source-slug/image.png`; never output absolute filesystem paths.",
"- Use kebab-case filenames",
"- Follow the analysis recommendations on what to emphasize",
"- If the analysis found connections to existing pages, add cross-references",
@@ -1425,7 +1471,6 @@ export function buildGenerationPrompt(
" SEARCH: automated technical debt detection AI generated code | software quality metrics LLM code generation | static analysis tools agentic software development",
"",
purpose ? `## Wiki Purpose\n${purpose}` : "",
- schema ? `## Wiki Schema\n${schema}` : "",
index ? `## Current Wiki Index (preserve all existing entries, add new ones)\n${index}` : "",
overview ? `## Current Overview (update this to reflect the new source)\n${overview}` : "",
"",
diff --git a/src/lib/source-lifecycle.test.ts b/src/lib/source-lifecycle.test.ts
index 6c3c25aa..8fd7e20a 100644
--- a/src/lib/source-lifecycle.test.ts
+++ b/src/lib/source-lifecycle.test.ts
@@ -1,9 +1,53 @@
-import { describe, expect, it } from "vitest"
+import { describe, expect, it, vi, beforeEach } from "vitest"
+
+const mocks = vi.hoisted(() => ({
+ copyFile: vi.fn(),
+ createDirectory: vi.fn(),
+ deleteFile: vi.fn(),
+ fileExists: vi.fn(),
+ getFileSize: vi.fn(),
+ listDirectory: vi.fn(),
+ preprocessFile: vi.fn(),
+ enqueueBatch: vi.fn(),
+}))
+
+vi.mock("@/commands/fs", async () => {
+ const actual = await vi.importActual<typeof import("@/commands/fs")>("@/commands/fs")
+ return {
+ ...actual,
+ copyFile: mocks.copyFile,
+ createDirectory: mocks.createDirectory,
+ deleteFile: mocks.deleteFile,
+ fileExists: mocks.fileExists,
+ getFileSize: mocks.getFileSize,
+ listDirectory: mocks.listDirectory,
+ preprocessFile: mocks.preprocessFile,
+ }
+})
+
+vi.mock("@/lib/ingest-queue", () => ({
+ enqueueBatch: mocks.enqueueBatch,
+}))
+
import {
folderContextForSourcePath,
+ importSourceFiles,
+ importSourceFolder,
isIngestableSourcePath,
} from "./source-lifecycle"
+beforeEach(() => {
+ vi.clearAllMocks()
+ mocks.copyFile.mockResolvedValue(undefined)
+ mocks.createDirectory.mockResolvedValue(undefined)
+ mocks.deleteFile.mockResolvedValue(undefined)
+ mocks.fileExists.mockResolvedValue(false)
+ mocks.getFileSize.mockResolvedValue(1024)
+ mocks.listDirectory.mockResolvedValue([])
+ mocks.preprocessFile.mockResolvedValue("")
+ mocks.enqueueBatch.mockResolvedValue(["task"])
+})
+
describe("source-lifecycle path helpers", () => {
it("does not treat preprocessed cache files as ingestable sources", () => {
expect(isIngestableSourcePath("raw/sources/.cache/report.pdf.txt")).toBe(false)
@@ -15,4 +59,91 @@ describe("source-lifecycle path helpers", () => {
folderContextForSourcePath("/tmp/project/raw/sources/reports/2026/report.pdf"),
).toBe("reports > 2026")
})
+
+ it("applies source watch exclusions during folder import before preprocess and ingest", async () => {
+ mocks.listDirectory.mockResolvedValue([
+ { name: "keep.md", path: "/external/imported/keep.md", is_dir: false },
+ { name: "config.json", path: "/external/imported/config.json", is_dir: false },
+ {
+ name: "drafts",
+ path: "/external/imported/drafts",
+ is_dir: true,
+ children: [
+ { name: "skip.md", path: "/external/imported/drafts/skip.md", is_dir: false },
+ ],
+ },
+ ])
+
+ const copied = await importSourceFolder(
+ { id: "p1", name: "Project", path: "/project" },
+ "/external/imported",
+ {
+ provider: "openai",
+ endpoint: "https://api.example.com/v1",
+ apiKey: "key",
+ model: "model",
+ customModel: "",
+ reasoning: { enabled: false, effort: "low" },
+ } as never,
+ {
+ enabled: true,
+ autoIngest: true,
+ includeExtensions: ["md"],
+ excludeExtensions: ["json"],
+ excludeDirs: ["drafts"],
+ excludeGlobs: [],
+ maxFileSizeMb: 100,
+ },
+ )
+
+ expect(copied).toEqual(["/project/raw/sources/imported/keep.md"])
+ expect(mocks.copyFile).toHaveBeenCalledTimes(1)
+ expect(mocks.copyFile).toHaveBeenCalledWith("/external/imported/keep.md", "/project/raw/sources/imported/keep.md")
+ expect(mocks.copyFile).not.toHaveBeenCalledWith("/external/imported/config.json", expect.anything())
+ expect(mocks.copyFile).not.toHaveBeenCalledWith("/external/imported/drafts/skip.md", expect.anything())
+ expect(mocks.deleteFile).not.toHaveBeenCalled()
+ expect(mocks.preprocessFile).toHaveBeenCalledOnce()
+ expect(mocks.preprocessFile).toHaveBeenCalledWith("/project/raw/sources/imported/keep.md")
+ expect(mocks.enqueueBatch).toHaveBeenCalledWith("p1", [
+ {
+ sourcePath: "/project/raw/sources/imported/keep.md",
+ folderContext: "imported",
+ },
+ ])
+ })
+
+ it("filters single-file imports using the original source path before copying", async () => {
+ const copied = await importSourceFiles(
+ { id: "p1", name: "Project", path: "/project" },
+ ["/external/drafts/spec.md", "/external/ready.md"],
+ {
+ provider: "openai",
+ endpoint: "https://api.example.com/v1",
+ apiKey: "key",
+ model: "model",
+ customModel: "",
+ reasoning: { enabled: false, effort: "low" },
+ } as never,
+ {
+ enabled: true,
+ autoIngest: true,
+ includeExtensions: ["md"],
+ excludeExtensions: [],
+ excludeDirs: ["drafts"],
+ excludeGlobs: [],
+ maxFileSizeMb: 100,
+ },
+ )
+
+ expect(copied).toEqual(["/project/raw/sources/ready.md"])
+ expect(mocks.copyFile).toHaveBeenCalledTimes(1)
+ expect(mocks.copyFile).toHaveBeenCalledWith("/external/ready.md", "/project/raw/sources/ready.md")
+ expect(mocks.copyFile).not.toHaveBeenCalledWith("/external/drafts/spec.md", expect.anything())
+ expect(mocks.enqueueBatch).toHaveBeenCalledWith("p1", [
+ {
+ sourcePath: "/project/raw/sources/ready.md",
+ folderContext: "",
+ },
+ ])
+ })
})
diff --git a/src/lib/source-lifecycle.ts b/src/lib/source-lifecycle.ts
index 81e40a35..e13c5337 100644
--- a/src/lib/source-lifecycle.ts
+++ b/src/lib/source-lifecycle.ts
@@ -1,8 +1,9 @@
import {
- copyDirectory,
copyFile,
+ createDirectory,
deleteFile,
fileExists,
+ getFileSize,
listDirectory,
preprocessFile,
readFile,
@@ -12,7 +13,7 @@ import type { WikiProject, FileNode } from "@/types/wiki"
import type { LlmConfig } from "@/stores/wiki-store"
import { enqueueBatch } from "@/lib/ingest-queue"
import { hasUsableLlm } from "@/lib/has-usable-llm"
-import { getFileName, getFileStem, normalizePath } from "@/lib/path-utils"
+import { getFileName, getFileStem, getRelativePath, normalizePath } from "@/lib/path-utils"
import {
sourceIdentityForPath,
sourceReferenceIdentity,
@@ -32,6 +33,8 @@ import {
stripDeletedWikilinks,
} from "@/lib/wiki-cleanup"
import { collectAllFilesIncludingDot } from "@/lib/sources-tree-delete"
+import { isPathAllowedBySourceWatch, normalizeSourceWatchConfig } from "@/lib/source-watch-config"
+import type { SourceWatchConfig } from "@/stores/wiki-store"
export const INGESTABLE_SOURCE_EXTENSIONS = new Set([
"md",
@@ -55,6 +58,24 @@ export const INGESTABLE_SOURCE_EXTENSIONS = new Set([
"yml",
])
+function flattenFiles(nodes: FileNode[]): FileNode[] {
+ const files: FileNode[] = []
+ for (const node of nodes) {
+ if (node.is_dir) {
+ files.push(...flattenFiles(node.children ?? []))
+ } else {
+ files.push(node)
+ }
+ }
+ return files
+}
+
+function parentPath(path: string): string {
+ const normalized = normalizePath(path)
+ const index = normalized.lastIndexOf("/")
+ return index > 0 ? normalized.slice(0, index) : ""
+}
+
export interface DeleteSourceResult {
deletedWikiPaths: string[]
rewrittenSourcePages: number
@@ -117,12 +138,25 @@ export async function importSourceFiles(
project: WikiProject,
sourcePaths: string[],
llmConfig: LlmConfig,
+ sourceWatchConfig?: SourceWatchConfig,
): Promise<string[]> {
const pp = normalizePath(project.path)
const importedPaths: string[] = []
+ const cfg = normalizeSourceWatchConfig(sourceWatchConfig)
+ const maxBytes = cfg.maxFileSizeMb * 1024 * 1024
for (const sourcePath of sourcePaths) {
const originalName = getFileName(sourcePath) || "unknown"
+ let allowed = isPathAllowedBySourceWatch(sourcePath, cfg)
+ if (allowed) {
+ try {
+ allowed = await getFileSize(sourcePath) <= maxBytes
+ } catch {
+ allowed = false
+ }
+ }
+ if (!allowed) continue
+
const destPath = await getUniqueDestPath(`${pp}/raw/sources`, originalName)
try {
await copyFile(sourcePath, destPath)
@@ -142,24 +176,45 @@ export async function importSourceFolder(
project: WikiProject,
selectedFolder: string,
llmConfig: LlmConfig,
+ sourceWatchConfig?: SourceWatchConfig,
): Promise<string[]> {
const pp = normalizePath(project.path)
+ const sourceRoot = normalizePath(selectedFolder)
const folderName = getFileName(selectedFolder) || "imported"
const destDir = `${pp}/raw/sources/${folderName}`
- const copiedFiles = await copyDirectory(selectedFolder, destDir)
-
- for (const filePath of copiedFiles) {
- preprocessFile(filePath).catch(() => {})
+ const cfg = normalizeSourceWatchConfig(sourceWatchConfig)
+ const maxBytes = cfg.maxFileSizeMb * 1024 * 1024
+ const allowedFiles: string[] = []
+ const sourceFiles = flattenFiles(await listDirectory(selectedFolder))
+
+ for (const file of sourceFiles) {
+ const relativeSourcePath = getRelativePath(file.path, sourceRoot)
+ const destPath = `${destDir}/${relativeSourcePath}`
+ const relPath = `raw/sources/${folderName}/${relativeSourcePath}`
+ let allowed = isPathAllowedBySourceWatch(relPath, cfg)
+ if (allowed) {
+ try {
+ allowed = await getFileSize(file.path) <= maxBytes
+ } catch {
+ allowed = false
+ }
+ }
+ if (!allowed) continue
+ const parent = parentPath(destPath)
+ if (parent) await createDirectory(parent)
+ await copyFile(file.path, destPath)
+ allowedFiles.push(destPath)
+ preprocessFile(destPath).catch(() => {})
}
if (hasUsableLlm(llmConfig)) {
- await enqueueSourceIngest(project, copiedFiles, llmConfig, {
+ await enqueueSourceIngest(project, allowedFiles, llmConfig, {
sourceRoot: destDir,
rootContext: folderName,
})
}
- return copiedFiles
+ return allowedFiles
}
export async function deleteSourceFile(
diff --git a/src/lib/source-watch-config.test.ts b/src/lib/source-watch-config.test.ts
index c9bed7f3..7987194d 100644
--- a/src/lib/source-watch-config.test.ts
+++ b/src/lib/source-watch-config.test.ts
@@ -34,4 +34,18 @@ describe("source watch config", () => {
expect(isPathAllowedBySourceWatch("raw/sources/plan.private.md", config)).toBe(false)
expect(isPathAllowedBySourceWatch("raw/sources/~$Document.docx", config)).toBe(false)
})
+
+ it("normalizes comma-separated exclusion values from text fields", () => {
+ const config = normalizeSourceWatchConfig({
+ includeExtensions: ["md, pdf", "docx"],
+ excludeExtensions: ["json, yaml\nxml,dll"],
+ excludeDirs: [".git, node_modules", "drafts,wip"],
+ excludeGlobs: ["*.private.*, ~$*"],
+ })
+
+ expect(config.includeExtensions).toEqual(["md", "pdf", "docx"])
+ expect(config.excludeExtensions).toEqual(["json", "yaml", "xml", "dll"])
+ expect(config.excludeDirs).toEqual([".git", "node_modules", "drafts", "wip"])
+ expect(config.excludeGlobs).toEqual(["*.private.*", "~$*"])
+ })
})
diff --git a/src/lib/source-watch-config.ts b/src/lib/source-watch-config.ts
index 7a798923..95489f13 100644
--- a/src/lib/source-watch-config.ts
+++ b/src/lib/source-watch-config.ts
@@ -29,12 +29,16 @@ export const SOURCE_WATCH_FILE_TYPE_GROUPS = [
function normalizeExtensions(values: readonly string[] | undefined): string[] {
return [...new Set((values ?? [])
+ .flatMap((value) => value.split(/[,,\n]/))
.map((value) => value.trim().replace(/^\./, "").toLowerCase())
.filter(Boolean))]
}
function normalizeList(values: readonly string[] | undefined): string[] {
- return [...new Set((values ?? []).map((value) => value.trim()).filter(Boolean))]
+ return [...new Set((values ?? [])
+ .flatMap((value) => value.split(/[,,\n]/))
+ .map((value) => value.trim())
+ .filter(Boolean))]
}
export function normalizeSourceWatchConfig(config?: Partial<SourceWatchConfig> | null): SourceWatchConfig {
diff --git a/src/lib/wiki-page-types.test.ts b/src/lib/wiki-page-types.test.ts
index f51a9832..fa35809e 100644
--- a/src/lib/wiki-page-types.test.ts
+++ b/src/lib/wiki-page-types.test.ts
@@ -20,7 +20,11 @@ describe("inferWikiTypeFromPath", () => {
it("handles Windows separators and overview pages", () => {
expect(inferWikiTypeFromPath("C:\\wiki\\findings\\result.md")).toBe("finding")
expect(inferWikiTypeFromPath("/project/wiki/overview.md")).toBe("overview")
- expect(inferWikiTypeFromPath("/project/wiki/misc/foo.md")).toBeNull()
+ })
+
+ it("uses custom wiki subdirectories as dynamic types", () => {
+ expect(inferWikiTypeFromPath("/project/wiki/people/ada-lovelace.md")).toBe("people")
+ expect(inferWikiTypeFromPath("/project/wiki/technologies/vector-db.md")).toBe("technologies")
})
})
@@ -29,6 +33,7 @@ describe("wikiTypeLabel", () => {
expect(wikiTypeLabel("finding")).toBe("Finding")
expect(wikiTypeLabel("thesis")).toBe("Thesis")
expect(wikiTypeLabel("methodology")).toBe("Methodology")
+ expect(wikiTypeLabel("custom-topic")).toBe("Custom Topic")
})
it("keeps generation prompt type list aligned with research-template types", () => {
diff --git a/src/lib/wiki-page-types.ts b/src/lib/wiki-page-types.ts
index cbf61718..5c445bb6 100644
--- a/src/lib/wiki-page-types.ts
+++ b/src/lib/wiki-page-types.ts
@@ -31,6 +31,8 @@ export function inferWikiTypeFromPath(path: string, fileName?: string): string |
}
const name = (fileName ?? normalized.split("/").pop() ?? "").toLowerCase()
if (name === "overview.md" || normalized.includes("/overview.md")) return "overview"
+ const customDir = normalized.match(/(?:^|\/)wiki\/([^/.][^/]*)\/[^/]+\.md$/)?.[1]
+ if (customDir) return customDir
return null
}
@@ -38,5 +40,9 @@ export function wikiTypeLabel(type: string): string {
if (type === "thesis") return "Thesis"
if (type === "methodology") return "Methodology"
if (type === "finding") return "Finding"
- return type.charAt(0).toUpperCase() + type.slice(1)
+ return type
+ .split(/[-_\s]+/)
+ .filter(Boolean)
+ .map((part) => part.charAt(0).toUpperCase() + part.slice(1))
+ .join(" ")
}