Skip to content

LLM Wiki is a cross-platform desktop application that turns your documents into an organized, interlinked knowledge base — automatically. Instead of traditional RAG (retrieve-and-answer from scratch every time), the LLM incrementally builds and maintains a persistent wiki from your sources。

TypeScript

Harden source watch filtering

nash_su0efcd68eb252
diff --git a/src-tauri/src/commands/file_sync.rs b/src-tauri/src/commands/file_sync.rs
index e428b8e2..0bfd75d8 100644
--- a/src-tauri/src/commands/file_sync.rs
+++ b/src-tauri/src/commands/file_sync.rs
@@ -3,6 +3,7 @@ use std::fs;
use std::io::Read;
use std::panic::AssertUnwindSafe;
use std::path::{Component, Path, PathBuf};
+use std::sync::atomic::{AtomicU64, Ordering};
use std::sync::mpsc;
use std::sync::{Arc, Mutex, OnceLock};
use std::time::Duration;
@@ -24,10 +25,12 @@ const MAX_RETRY_COUNT: u32 = 3;
const APP_WRITE_IGNORE_MS: i64 = 4_000;
const QUEUE_EMIT_EVERY: usize = 25;
const LINUX_RESCAN_INTERVAL_MS: i64 = 10_000;
-const DEFAULT_SOURCE_WATCH_MAX_MB: u64 = 100;
+const DEFAULT_SOURCE_WATCH_CONFIG_JSON: &str =
+ include_str!("../../../src/lib/source-watch-defaults.json");
static QUEUE_LOCKS: OnceLock<Mutex<BTreeMap<String, Arc<Mutex<()>>>>> = OnceLock::new();
static APP_WRITE_IGNORES: OnceLock<Mutex<BTreeMap<String, i64>>> = OnceLock::new();
+static WATCHER_GENERATION: AtomicU64 = AtomicU64::new(0);
#[derive(Default)]
pub struct FileSyncState {
@@ -104,67 +107,61 @@ pub struct FileChangeQueue {
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct SourceWatchConfig {
+ #[serde(default = "default_source_watch_enabled")]
enabled: bool,
+ #[serde(default = "default_source_watch_auto_ingest")]
auto_ingest: bool,
+ #[serde(default = "default_source_watch_include_extensions")]
include_extensions: Vec<String>,
+ #[serde(default = "default_source_watch_exclude_extensions")]
exclude_extensions: Vec<String>,
+ #[serde(default = "default_source_watch_exclude_dirs")]
exclude_dirs: Vec<String>,
+ #[serde(default = "default_source_watch_exclude_globs")]
exclude_globs: Vec<String>,
+ #[serde(default = "default_source_watch_max_file_size_mb")]
max_file_size_mb: u64,
}
impl Default for SourceWatchConfig {
fn default() -> Self {
- Self {
- enabled: true,
- auto_ingest: true,
- include_extensions: [
- "md", "mdx", "txt", "pdf", "doc", "docx", "ppt", "pptx", "xls", "xlsx",
- "rtf", "html", "htm", "csv",
- ]
- .into_iter()
- .map(str::to_string)
- .collect(),
- exclude_extensions: [
- "tmp",
- "temp",
- "bak",
- "swp",
- "part",
- "partial",
- "crdownload",
- "exe",
- "dll",
- "so",
- "dylib",
- "bin",
- ]
- .into_iter()
- .map(str::to_string)
- .collect(),
- exclude_dirs: [
- ".git",
- ".svn",
- ".hg",
- ".obsidian",
- ".idea",
- ".vscode",
- "node_modules",
- ".cache",
- "__pycache__",
- ]
- .into_iter()
- .map(str::to_string)
- .collect(),
- exclude_globs: ["~$*", ".~lock.*#", "*.draft.*", "draft-*", "*.private.*"]
- .into_iter()
- .map(str::to_string)
- .collect(),
- max_file_size_mb: DEFAULT_SOURCE_WATCH_MAX_MB,
- }
+ serde_json::from_str(DEFAULT_SOURCE_WATCH_CONFIG_JSON)
+ .expect("source-watch-defaults.json must match SourceWatchConfig")
}
}
+fn default_source_watch_config() -> SourceWatchConfig {
+ SourceWatchConfig::default()
+}
+
+fn default_source_watch_enabled() -> bool {
+ default_source_watch_config().enabled
+}
+
+fn default_source_watch_auto_ingest() -> bool {
+ default_source_watch_config().auto_ingest
+}
+
+fn default_source_watch_include_extensions() -> Vec<String> {
+ default_source_watch_config().include_extensions
+}
+
+fn default_source_watch_exclude_extensions() -> Vec<String> {
+ default_source_watch_config().exclude_extensions
+}
+
+fn default_source_watch_exclude_dirs() -> Vec<String> {
+ default_source_watch_config().exclude_dirs
+}
+
+fn default_source_watch_exclude_globs() -> Vec<String> {
+ default_source_watch_config().exclude_globs
+}
+
+fn default_source_watch_max_file_size_mb() -> u64 {
+ default_source_watch_config().max_file_size_mb
+}
+
fn normalize_source_watch_config(config: Option<SourceWatchConfig>) -> SourceWatchConfig {
let mut config = config.unwrap_or_default();
config.include_extensions = normalize_ext_list(config.include_extensions);
@@ -193,6 +190,7 @@ pub fn start_project_file_watcher(
run_guarded("start_project_file_watcher", || {
let root = PathBuf::from(project_path);
let source_watch_config = normalize_source_watch_config(source_watch_config);
+ let watcher_generation = WATCHER_GENERATION.fetch_add(1, Ordering::SeqCst) + 1;
ensure_sync_dir(&root)?;
with_queue_lock(&root, || reset_processing_tasks(&root, &project_id))?;
enqueue_rescan_changes(&root, &project_id, &source_watch_config)?;
@@ -221,6 +219,7 @@ pub fn start_project_file_watcher(
&root_for_thread,
&project_for_thread,
&config_for_thread,
+ watcher_generation,
&mut last_periodic_rescan,
);
continue;
@@ -233,6 +232,7 @@ pub fn start_project_file_watcher(
&root_for_thread,
&project_for_thread,
&config_for_thread,
+ watcher_generation,
paths,
)
}));
@@ -246,6 +246,7 @@ pub fn start_project_file_watcher(
&root_for_thread,
&project_for_thread,
&config_for_thread,
+ watcher_generation,
&mut last_periodic_rescan,
);
}
@@ -308,6 +309,7 @@ pub fn start_project_file_watcher(
#[tauri::command]
pub fn stop_project_file_watcher(state: State<FileSyncState>) -> Result<(), String> {
run_guarded("stop_project_file_watcher", || {
+ WATCHER_GENERATION.fetch_add(1, Ordering::SeqCst);
let mut inner = state.inner.lock().map_err(|_| "file sync state poisoned")?;
inner.watcher = None;
inner.project_id = None;
@@ -411,28 +413,33 @@ fn handle_changed_paths(
root: &Path,
project_id: &str,
source_watch_config: &SourceWatchConfig,
+ watcher_generation: u64,
paths: Vec<PathBuf>,
) -> Result<(), String> {
+ if !is_active_watcher_generation(watcher_generation) {
+ return Ok(());
+ }
+ let rules = SourceWatchRules::new(source_watch_config);
let mut rels = BTreeSet::<String>::new();
let mut app_written_rels = BTreeSet::<String>::new();
let snapshot = with_queue_lock(root, || read_snapshot(root))?;
for path in paths {
if is_app_write_ignored(&path) {
- collect_known_paths(root, &path, &snapshot, &mut app_written_rels, source_watch_config);
+ collect_known_paths(root, &path, &snapshot, &mut app_written_rels, &rules);
continue;
}
if path.is_dir() {
for entry in WalkDir::new(&path).into_iter().filter_map(Result::ok) {
if entry.file_type().is_file() && !is_app_write_ignored(entry.path()) {
- if let Some(rel) = relative_watch_path(root, entry.path(), source_watch_config) {
+ if let Some(rel) = relative_watch_path(root, entry.path(), &rules, entry.metadata().ok().map(|m| m.len())) {
rels.insert(rel);
}
}
}
- } else if let Some(rel) = relative_watch_path(root, &path, source_watch_config) {
+ } else if let Some(rel) = relative_watch_path(root, &path, &rules, None) {
rels.insert(rel);
} else if !path.exists() {
- collect_known_paths(root, &path, &snapshot, &mut rels, source_watch_config);
+ collect_known_paths(root, &path, &snapshot, &mut rels, &rules);
}
}
if !app_written_rels.is_empty() {
@@ -441,9 +448,18 @@ fn handle_changed_paths(
if rels.is_empty() {
return Ok(());
}
+ if !is_active_watcher_generation(watcher_generation) {
+ return Ok(());
+ }
enqueue_paths(root, project_id, rels)?;
+ if !is_active_watcher_generation(watcher_generation) {
+ return Ok(());
+ }
process_queue(app, root, project_id)?;
let queue = with_queue_lock(root, || read_queue(root))?;
+ if !is_active_watcher_generation(watcher_generation) {
+ return Ok(());
+ }
emit_queue(app, project_id, &queue);
Ok(())
}
@@ -453,6 +469,7 @@ fn maybe_periodic_rescan(
root: &Path,
project_id: &str,
source_watch_config: &SourceWatchConfig,
+ watcher_generation: u64,
last_periodic_rescan: &mut i64,
) {
if !cfg!(target_os = "linux") || now_ms() - *last_periodic_rescan < LINUX_RESCAN_INTERVAL_MS {
@@ -460,7 +477,7 @@ fn maybe_periodic_rescan(
}
*last_periodic_rescan = now_ms();
let result = std::panic::catch_unwind(AssertUnwindSafe(|| {
- rescan_watch_roots(app, root, project_id, source_watch_config)
+ rescan_watch_roots(app, root, project_id, source_watch_config, watcher_generation)
}));
match result {
Ok(Ok(())) => {}
@@ -474,30 +491,44 @@ fn rescan_watch_roots(
root: &Path,
project_id: &str,
source_watch_config: &SourceWatchConfig,
+ watcher_generation: u64,
) -> Result<(), String> {
+ if !is_active_watcher_generation(watcher_generation) {
+ return Ok(());
+ }
enqueue_rescan_changes_for_prefixes(
root,
project_id,
&["raw/sources", "wiki", "purpose.md", "schema.md"],
source_watch_config,
)?;
+ if !is_active_watcher_generation(watcher_generation) {
+ return Ok(());
+ }
process_queue(app, root, project_id)?;
let queue = with_queue_lock(root, || read_queue(root))?;
+ if !is_active_watcher_generation(watcher_generation) {
+ return Ok(());
+ }
emit_queue(app, project_id, &queue);
Ok(())
}
+fn is_active_watcher_generation(generation: u64) -> bool {
+ WATCHER_GENERATION.load(Ordering::SeqCst) == generation
+}
+
fn collect_known_paths(
root: &Path,
path: &Path,
snapshot: &FileSnapshot,
rels: &mut BTreeSet<String>,
- source_watch_config: &SourceWatchConfig,
+ rules: &SourceWatchRules,
) {
if path.is_dir() {
for entry in WalkDir::new(path).into_iter().filter_map(Result::ok) {
if entry.file_type().is_file() {
- if let Some(rel) = relative_watch_path(root, entry.path(), source_watch_config) {
+ if let Some(rel) = relative_watch_path(root, entry.path(), rules, entry.metadata().ok().map(|m| m.len())) {
rels.insert(rel);
}
}
@@ -520,7 +551,7 @@ fn collect_known_paths(
return;
}
- if should_watch_rel(&rel, source_watch_config) {
+ if should_watch_rel(&rel, rules) {
rels.insert(rel);
}
}
@@ -554,10 +585,11 @@ fn enqueue_rescan_changes(
project_id: &str,
source_watch_config: &SourceWatchConfig,
) -> Result<(), String> {
+ let rules = SourceWatchRules::new(source_watch_config);
let mut rels = BTreeSet::<String>::new();
for entry in WalkDir::new(root).into_iter().filter_map(Result::ok) {
if entry.file_type().is_file() {
- if let Some(rel) = relative_watch_path(root, entry.path(), source_watch_config) {
+ if let Some(rel) = relative_watch_path(root, entry.path(), &rules, entry.metadata().ok().map(|m| m.len())) {
rels.insert(rel);
}
}
@@ -578,12 +610,13 @@ fn enqueue_rescan_changes_for_prefixes(
prefixes: &[&str],
source_watch_config: &SourceWatchConfig,
) -> Result<(), String> {
+ let rules = SourceWatchRules::new(source_watch_config);
let mut rels = BTreeSet::<String>::new();
let snapshot = with_queue_lock(root, || read_snapshot(root))?;
for prefix in prefixes {
let path = root.join(prefix);
if path.is_file() {
- if let Some(rel) = relative_watch_path(root, &path, source_watch_config) {
+ if let Some(rel) = relative_watch_path(root, &path, &rules, fs::metadata(&path).ok().map(|m| m.len())) {
let old = snapshot.files.get(&rel);
let fast = read_meta_fast(root, &rel)?;
if old.map(|m| (m.size, m.mtime_ms)) != fast.as_ref().map(|m| (m.size, m.mtime_ms)) {
@@ -593,7 +626,7 @@ fn enqueue_rescan_changes_for_prefixes(
} else if path.exists() {
for entry in WalkDir::new(&path).into_iter().filter_map(Result::ok) {
if entry.file_type().is_file() {
- if let Some(rel) = relative_watch_path(root, entry.path(), source_watch_config) {
+ if let Some(rel) = relative_watch_path(root, entry.path(), &rules, entry.metadata().ok().map(|m| m.len())) {
let old = snapshot.files.get(&rel);
let fast = read_meta_fast(root, &rel)?;
if old.map(|m| (m.size, m.mtime_ms)) != fast.as_ref().map(|m| (m.size, m.mtime_ms)) {
@@ -941,15 +974,21 @@ fn md5_file(path: &Path) -> Result<String, String> {
Ok(format!("{:x}", hasher.finalize()))
}
-fn relative_watch_path(root: &Path, path: &Path, source_watch_config: &SourceWatchConfig) -> Option<String> {
+fn relative_watch_path(
+ root: &Path,
+ path: &Path,
+ rules: &SourceWatchRules,
+ size: Option<u64>,
+) -> Option<String> {
let rel = path.strip_prefix(root).ok()?;
let rel = normalize_rel_path(rel)?;
- if !should_watch_rel(&rel, source_watch_config) {
+ if !should_watch_rel(&rel, rules) {
return None;
}
if rel.starts_with("raw/sources/") && path.exists() {
- let max_bytes = source_watch_config.max_file_size_mb.saturating_mul(1024 * 1024);
- if fs::metadata(path).ok()?.len() > max_bytes {
+ let max_bytes = rules.config.max_file_size_mb.saturating_mul(1024 * 1024);
+ let size = size.or_else(|| fs::metadata(path).ok().map(|m| m.len()))?;
+ if size > max_bytes {
return None;
}
}
@@ -968,51 +1007,29 @@ fn normalize_rel_path(path: &Path) -> Option<String> {
Some(parts.join("/"))
}
-fn should_watch_rel(rel: &str, source_watch_config: &SourceWatchConfig) -> bool {
+fn should_watch_rel(rel: &str, rules: &SourceWatchRules) -> bool {
if rel.is_empty() {
return false;
}
let lower = rel.to_lowercase();
if lower.contains("/.llm-wiki/")
|| lower.starts_with(".llm-wiki/")
- || lower.starts_with(".cache/")
- || lower.starts_with(".obsidian/")
- || lower.starts_with(".idea/")
- || lower.starts_with(".vscode/")
// App-managed generated media is intentionally ignored here. The
// source markdown references drive graph/index refresh; media bytes
// themselves are not analyzed by the wiki pipeline.
|| lower.starts_with("wiki/media/")
- || lower.ends_with(".tmp")
- || lower.ends_with(".swp")
- || lower.ends_with('~')
|| lower.ends_with(".ds_store")
- || lower.ends_with(".crdownload")
- || lower.ends_with(".part")
- || lower.ends_with(".partial")
{
return false;
}
let name = lower.rsplit('/').next().unwrap_or(&lower);
- if name == "thumbs.db"
- || name == "desktop.ini"
- || name.starts_with("~$")
- || (name.starts_with(".~lock.") && name.ends_with('#'))
- {
+ if name == "thumbs.db" || name == "desktop.ini" {
return false;
}
- let excluded_dirs = source_watch_config
- .exclude_dirs
- .iter()
- .map(|dir| dir.to_lowercase())
- .collect::<BTreeSet<_>>();
- if lower
- .split('/')
- .any(|part| !part.is_empty() && excluded_dirs.contains(part))
- {
+ if rules.matches_excluded_dir(&lower) {
return false;
}
- if source_watch_config
+ if rules
.exclude_globs
.iter()
.any(|pattern| wildcard_match(pattern, rel) || wildcard_match(pattern, name))
@@ -1022,19 +1039,12 @@ fn should_watch_rel(rel: &str, source_watch_config: &SourceWatchConfig) -> bool
if rel.starts_with("raw/sources/") {
let ext = extension_of(name);
if !ext.is_empty()
- && source_watch_config
- .exclude_extensions
- .iter()
- .any(|candidate| candidate == ext)
+ && rules.exclude_extensions.contains(ext)
{
return false;
}
- if !source_watch_config.include_extensions.is_empty()
- && (ext.is_empty()
- || !source_watch_config
- .include_extensions
- .iter()
- .any(|candidate| candidate == ext))
+ if !rules.include_extensions.is_empty()
+ && (ext.is_empty() || !rules.include_extensions.contains(ext))
{
return false;
}
@@ -1045,6 +1055,47 @@ fn should_watch_rel(rel: &str, source_watch_config: &SourceWatchConfig) -> bool
|| (rel.starts_with("wiki/") && rel.ends_with(".md"))
}
+struct SourceWatchRules<'a> {
+ config: &'a SourceWatchConfig,
+ include_extensions: BTreeSet<String>,
+ exclude_extensions: BTreeSet<String>,
+ exclude_dirs: BTreeSet<String>,
+ exclude_globs: Vec<String>,
+}
+
+impl<'a> SourceWatchRules<'a> {
+ fn new(config: &'a SourceWatchConfig) -> Self {
+ Self {
+ config,
+ include_extensions: config.include_extensions.iter().cloned().collect(),
+ exclude_extensions: config.exclude_extensions.iter().cloned().collect(),
+ exclude_dirs: config
+ .exclude_dirs
+ .iter()
+ .map(|dir| normalize_rel_string(dir).to_lowercase())
+ .filter(|dir| !dir.is_empty())
+ .collect(),
+ exclude_globs: config.exclude_globs.iter().cloned().collect(),
+ }
+ }
+
+ fn matches_excluded_dir(&self, rel_lower: &str) -> bool {
+ self.exclude_dirs.iter().any(|dir| {
+ if dir.contains('/') {
+ rel_lower == dir
+ || rel_lower.starts_with(&format!("{dir}/"))
+ || rel_lower.contains(&format!("/{dir}/"))
+ } else {
+ rel_lower.split('/').any(|part| part == dir)
+ }
+ })
+ }
+}
+
+fn normalize_rel_string(value: &str) -> String {
+ value.replace('\\', "/").trim_matches('/').to_string()
+}
+
fn extension_of(name: &str) -> &str {
name.rsplit_once('.').map(|(_, ext)| ext).unwrap_or("")
}
@@ -1070,21 +1121,20 @@ fn normalize_string_list(values: Vec<String>) -> Vec<String> {
}
fn wildcard_match(pattern: &str, value: &str) -> bool {
- wildcard_match_inner(
- pattern.to_lowercase().as_bytes(),
- value.to_lowercase().as_bytes(),
- )
+ let pattern = pattern.to_lowercase().chars().collect::<Vec<_>>();
+ let value = value.to_lowercase().chars().collect::<Vec<_>>();
+ wildcard_match_inner(&pattern, &value)
}
-fn wildcard_match_inner(pattern: &[u8], value: &[u8]) -> bool {
+fn wildcard_match_inner(pattern: &[char], value: &[char]) -> bool {
let (mut p, mut v) = (0usize, 0usize);
let mut star: Option<usize> = None;
let mut match_after_star = 0usize;
while v < value.len() {
- if p < pattern.len() && (pattern[p] == b'?' || pattern[p] == value[v]) {
+ if p < pattern.len() && (pattern[p] == '?' || pattern[p] == value[v]) {
p += 1;
v += 1;
- } else if p < pattern.len() && pattern[p] == b'*' {
+ } else if p < pattern.len() && pattern[p] == '*' {
star = Some(p);
match_after_star = v;
p += 1;
@@ -1096,7 +1146,7 @@ fn wildcard_match_inner(pattern: &[u8], value: &[u8]) -> bool {
return false;
}
}
- while p < pattern.len() && pattern[p] == b'*' {
+ while p < pattern.len() && pattern[p] == '*' {
p += 1;
}
p == pattern.len()
@@ -1369,12 +1419,14 @@ mod tests {
let mut rels = BTreeSet::new();
let snapshot = read_snapshot(&root).unwrap();
+ let config = default_watch_config();
+ let rules = SourceWatchRules::new(&config);
collect_known_paths(
&root,
&root.join("raw/sources/folder"),
&snapshot,
&mut rels,
- &default_watch_config(),
+ &rules,
);
assert_eq!(rels, BTreeSet::from([a.to_string(), b.to_string()]));
@@ -1423,7 +1475,9 @@ mod tests {
let mut app_written_rels = BTreeSet::new();
let snapshot = read_snapshot(&root).unwrap();
if is_app_write_ignored(&path) {
- collect_known_paths(&root, &path, &snapshot, &mut app_written_rels, &default_watch_config());
+ let config = default_watch_config();
+ let rules = SourceWatchRules::new(&config);
+ collect_known_paths(&root, &path, &snapshot, &mut app_written_rels, &rules);
}
sync_snapshot_paths(&root, app_written_rels).unwrap();
@@ -1484,33 +1538,36 @@ mod tests {
#[test]
fn watch_rules_exclude_temporary_and_app_dirs() {
let config = default_watch_config();
- assert!(should_watch_rel("raw/sources/doc.md", &config));
- assert!(should_watch_rel("wiki/concepts/topic.md", &config));
- assert!(!should_watch_rel(".llm-wiki/file-change-queue.json", &config));
- assert!(!should_watch_rel("raw/sources/~$Document.docx", &config));
- assert!(!should_watch_rel("raw/sources/.~lock.Document.odt#", &config));
- assert!(!should_watch_rel("raw/sources/Thumbs.db", &config));
- assert!(!should_watch_rel("raw/sources/desktop.ini", &config));
- assert!(!should_watch_rel("raw/sources/download.crdownload", &config));
- assert!(!should_watch_rel(".vscode/settings.json", &config));
- assert!(!should_watch_rel("wiki/media/image.png", &config));
+ let rules = SourceWatchRules::new(&config);
+ assert!(should_watch_rel("raw/sources/document.docx", &rules));
+ assert!(should_watch_rel("wiki/concepts/topic.md", &rules));
+ assert!(!should_watch_rel(".llm-wiki/file-change-queue.json", &rules));
+ assert!(!should_watch_rel("raw/sources/~$Document.docx", &rules));
+ assert!(!should_watch_rel("raw/sources/.~lock.Document.odt#", &rules));
+ assert!(!should_watch_rel("raw/sources/Thumbs.db", &rules));
+ assert!(!should_watch_rel("raw/sources/desktop.ini", &rules));
+ assert!(!should_watch_rel("raw/sources/download.crdownload", &rules));
+ assert!(!should_watch_rel(".vscode/settings.json", &rules));
+ assert!(!should_watch_rel("wiki/media/image.png", &rules));
}
#[test]
fn source_watch_config_filters_raw_source_extensions_and_dirs() {
let config = SourceWatchConfig {
include_extensions: vec!["md".into(), "pdf".into()],
- exclude_dirs: vec!["drafts".into()],
+ exclude_dirs: vec!["drafts".into(), "subdir/drafts".into()],
exclude_globs: vec!["*.private.*".into()],
..SourceWatchConfig::default()
};
let config = normalize_source_watch_config(Some(config));
-
- assert!(should_watch_rel("raw/sources/final.md", &config));
- assert!(!should_watch_rel("raw/sources/data.json", &config));
- assert!(!should_watch_rel("raw/sources/drafts/final.md", &config));
- assert!(!should_watch_rel("raw/sources/report.private.md", &config));
- assert!(should_watch_rel("wiki/index.md", &config));
+ let rules = SourceWatchRules::new(&config);
+
+ assert!(should_watch_rel("raw/sources/final.md", &rules));
+ assert!(!should_watch_rel("raw/sources/data.json", &rules));
+ assert!(!should_watch_rel("raw/sources/drafts/final.md", &rules));
+ assert!(!should_watch_rel("raw/sources/subdir/drafts/final.md", &rules));
+ assert!(!should_watch_rel("raw/sources/report.private.md", &rules));
+ assert!(should_watch_rel("wiki/index.md", &rules));
}
#[test]
@@ -1523,12 +1580,33 @@ mod tests {
..SourceWatchConfig::default()
};
let config = normalize_source_watch_config(Some(config));
+ let rules = SourceWatchRules::new(&config);
- assert_eq!(relative_watch_path(&root, &root.join(rel), &config), None);
+ assert_eq!(relative_watch_path(&root, &root.join(rel), &rules, None), None);
let _ = fs::remove_dir_all(root);
}
+ #[test]
+ fn source_watch_defaults_are_loaded_from_shared_json_and_tolerate_missing_fields() {
+ let default = SourceWatchConfig::default();
+ assert!(default.include_extensions.contains(&"md".to_string()));
+ assert!(default.exclude_dirs.contains(&".git".to_string()));
+
+ let partial: SourceWatchConfig =
+ serde_json::from_str(r#"{"enabled":false,"includeExtensions":["md"]}"#).unwrap();
+ assert!(!partial.enabled);
+ assert!(partial.auto_ingest);
+ assert!(partial.exclude_dirs.contains(&".git".to_string()));
+ }
+
+ #[test]
+ fn wildcard_match_is_unicode_character_based() {
+ assert!(wildcard_match("?稿.md", "草稿.md"));
+ assert!(wildcard_match("草*.md", "草稿文件.md"));
+ assert!(!wildcard_match("??.md", "草稿文件.md"));
+ }
+
#[test]
fn normalize_rel_path_tolerates_current_dir_segments() {
let path = Path::new("raw").join(".").join("sources").join("doc.md");
diff --git a/src/components/settings/sections/source-watch-section.tsx b/src/components/settings/sections/source-watch-section.tsx
index 4958d95f..913983b3 100644
--- a/src/components/settings/sections/source-watch-section.tsx
+++ b/src/components/settings/sections/source-watch-section.tsx
@@ -181,10 +181,39 @@ export function SourceWatchSection({ draft, setDraft, projectReady }: Props) {
<textarea
value={joinList(config.excludeDirs)}
onChange={(event) => updateConfig({ excludeDirs: updateListValue(event.target.value) })}
+ placeholder={t("settings.sections.sourceWatch.excludeDirsPlaceholder", {
+ defaultValue: ".git, node_modules, drafts, subdir/drafts",
+ })}
+ disabled={!projectReady || !config.enabled}
+ rows={2}
+ className="w-full rounded-md border border-input bg-background px-2 py-1 text-sm"
+ />
+ <p className="text-[11px] leading-relaxed text-muted-foreground">
+ {t("settings.sections.sourceWatch.excludeDirsHint", {
+ defaultValue: "Folder names match any path segment. Paths such as subdir/drafts match that nested folder.",
+ })}
+ </p>
+ </label>
+
+ <label className="block space-y-1.5">
+ <span className="text-xs font-medium">
+ {t("settings.sections.sourceWatch.excludeExtensions", { defaultValue: "Excluded file extensions" })}
+ </span>
+ <textarea
+ value={joinList(config.excludeExtensions)}
+ onChange={(event) => updateConfig({ excludeExtensions: updateListValue(event.target.value) })}
+ placeholder={t("settings.sections.sourceWatch.excludeExtensionsPlaceholder", {
+ defaultValue: "tmp, bak, exe, dll, iso, dmg",
+ })}
disabled={!projectReady || !config.enabled}
rows={2}
className="w-full rounded-md border border-input bg-background px-2 py-1 text-sm"
/>
+ <p className="text-[11px] leading-relaxed text-muted-foreground">
+ {t("settings.sections.sourceWatch.excludeExtensionsHint", {
+ defaultValue: "Use extensions without dots. These override the allowed file type checkboxes.",
+ })}
+ </p>
</label>
<label className="block space-y-1.5">
diff --git a/src/i18n/en.json b/src/i18n/en.json
index 9ac94b72..5795ca4b 100644
--- a/src/i18n/en.json
+++ b/src/i18n/en.json
@@ -281,6 +281,11 @@
"exclusions": "Exclusions and limits",
"maxSize": "Maximum auto-ingest file size (MB)",
"excludeDirs": "Excluded folders",
+ "excludeDirsPlaceholder": ".git, node_modules, drafts, subdir/drafts",
+ "excludeDirsHint": "Folder names match any path segment. Paths such as subdir/drafts match that nested folder.",
+ "excludeExtensions": "Excluded file extensions",
+ "excludeExtensionsPlaceholder": "tmp, bak, exe, dll, iso, dmg",
+ "excludeExtensionsHint": "Use extensions without dots. These override the allowed file type checkboxes.",
"excludeGlobs": "Excluded filename patterns",
"excludeGlobsHint": "Use comma-separated patterns such as *.draft.*, ~$*, .~lock.*#.",
"groups": {
diff --git a/src/i18n/zh.json b/src/i18n/zh.json
index 431b9233..6ea4f97f 100644
--- a/src/i18n/zh.json
+++ b/src/i18n/zh.json
@@ -178,7 +178,7 @@
"multimodal": "图片描述",
"webSearch": "网页搜索",
"network": "网络",
- "sourceWatch": "原始资料监控",
+ "sourceWatch": "资料文件夹监控",
"scheduledImport": "定时导入",
"output": "输出偏好",
"interface": "界面",
@@ -269,9 +269,9 @@
"bypassLocalHelp": "对 localhost、127.0.0.0/8、10.0.0.0/8、172.16.0.0/12、192.168.0.0/16 以及 *.local 的请求会绕过代理。如果你用 Ollama / LM Studio 或局域网内的 LLM 服务,请保持开启。"
},
"sourceWatch": {
- "title": "原始资料文件夹自动监控",
+ "title": "资料文件夹自动监控",
"description": "监控 raw/sources 的外部变化,并选择哪些文件允许自动进入提取队列。",
- "enable": "监控项目原始资料文件夹",
+ "enable": "监控项目资料文件夹",
"enableDescription": "刷新原始资料列表,并为 raw/sources 下允许的新文件或修改文件建立提取任务。",
"autoIngest": "自动提取允许的原始资料文件",
"autoIngestDescription": "关闭后,外部变化仍会刷新文件树,但不会创建提取任务。",
@@ -281,6 +281,11 @@
"exclusions": "排除规则和限制",
"maxSize": "自动提取的最大文件大小(MB)",
"excludeDirs": "排除的文件夹",
+ "excludeDirsPlaceholder": ".git, node_modules, drafts, subdir/drafts",
+ "excludeDirsHint": "文件夹名称会匹配任意路径段。subdir/drafts 这类路径会匹配对应的嵌套文件夹。",
+ "excludeExtensions": "排除的文件扩展名",
+ "excludeExtensionsPlaceholder": "tmp, bak, exe, dll, iso, dmg",
+ "excludeExtensionsHint": "不需要写点号。这些扩展名会覆盖上面的允许文件类型勾选。",
"excludeGlobs": "排除的文件名模式",
"excludeGlobsHint": "用逗号分隔模式,例如 *.draft.*、~$*、.~lock.*#。",
"groups": {
diff --git a/src/lib/changelog.ts b/src/lib/changelog.ts
index a1be1762..90908cbd 100644
--- a/src/lib/changelog.ts
+++ b/src/lib/changelog.ts
@@ -25,6 +25,18 @@ export interface ChangelogEntry {
}
export const CHANGELOG: ChangelogEntry[] = [
+ {
+ version: "0.4.10",
+ date: "2026-05-14",
+ highlights: {
+ en: [
+ "Source folder monitoring now has configurable file type and exclusion rules. Existing projects should review Settings → Source Watch after upgrading because automatic ingest now defaults to common document formats only.",
+ ],
+ zh: [
+ "资料文件夹监控新增文件类型和排除规则配置。升级后建议到「设置 → 资料文件夹监控」检查配置;自动提取现在默认只纳入常见文档格式。",
+ ],
+ },
+ },
{
version: "0.4.9",
date: "2026-05-11",
diff --git a/src/lib/project-file-sync.ts b/src/lib/project-file-sync.ts
index f9fb50f5..dad7445d 100644
--- a/src/lib/project-file-sync.ts
+++ b/src/lib/project-file-sync.ts
@@ -1,5 +1,5 @@
import { listen, type UnlistenFn } from "@tauri-apps/api/event"
-import { readFile, listDirectory, getFileSize } from "@/commands/fs"
+import { readFile, listDirectory } from "@/commands/fs"
import {
startProjectFileWatcher,
stopProjectFileWatcher,
@@ -156,19 +156,7 @@ async function enqueueRawSourceChanges(project: WikiProject, tasks: FileChangeTa
.map((task) => task.path)
.filter(isIngestableRawSource)
- const paths: string[] = []
- const maxBytes = config.maxFileSizeMb * 1024 * 1024
- for (const rel of candidates) {
- if (!isPathAllowedBySourceWatch(rel, config)) continue
- try {
- const size = await getFileSize(`${normalizePath(project.path)}/${rel}`)
- if (size > maxBytes) continue
- } catch (err) {
- console.warn("[file-sync] failed to inspect changed source file:", err)
- continue
- }
- paths.push(rel)
- }
+ const paths = candidates.filter((rel) => isPathAllowedBySourceWatch(rel, config))
if (paths.length === 0) return
diff --git a/src/lib/source-lifecycle.ts b/src/lib/source-lifecycle.ts
index 37d71cb0..7e5de899 100644
--- a/src/lib/source-lifecycle.ts
+++ b/src/lib/source-lifecycle.ts
@@ -37,6 +37,9 @@ export const INGESTABLE_SOURCE_EXTENSIONS = new Set([
"docx",
"pptx",
"xlsx",
+ "odt",
+ "odp",
+ "ods",
"xls",
"csv",
"json",
diff --git a/src/lib/source-watch-config.test.ts b/src/lib/source-watch-config.test.ts
index 09a50c53..c9bed7f3 100644
--- a/src/lib/source-watch-config.test.ts
+++ b/src/lib/source-watch-config.test.ts
@@ -4,11 +4,17 @@ import {
isPathAllowedBySourceWatch,
normalizeSourceWatchConfig,
} from "@/lib/source-watch-config"
+import sourceWatchDefaults from "@/lib/source-watch-defaults.json"
describe("source watch config", () => {
+ it("uses the shared default fixture", () => {
+ expect(DEFAULT_SOURCE_WATCH_CONFIG).toEqual(sourceWatchDefaults)
+ })
+
it("allows document types by default and rejects config/media/binaries", () => {
expect(isPathAllowedBySourceWatch("raw/sources/report.pdf", DEFAULT_SOURCE_WATCH_CONFIG)).toBe(true)
expect(isPathAllowedBySourceWatch("raw/sources/notes.md", DEFAULT_SOURCE_WATCH_CONFIG)).toBe(true)
+ expect(isPathAllowedBySourceWatch("raw/sources/report.doc", DEFAULT_SOURCE_WATCH_CONFIG)).toBe(false)
expect(isPathAllowedBySourceWatch("raw/sources/secrets.json", DEFAULT_SOURCE_WATCH_CONFIG)).toBe(false)
expect(isPathAllowedBySourceWatch("raw/sources/video.mp4", DEFAULT_SOURCE_WATCH_CONFIG)).toBe(false)
expect(isPathAllowedBySourceWatch("raw/sources/tool.exe", DEFAULT_SOURCE_WATCH_CONFIG)).toBe(false)
@@ -23,6 +29,7 @@ describe("source watch config", () => {
expect(isPathAllowedBySourceWatch("raw/sources/ready.md", config)).toBe(true)
expect(isPathAllowedBySourceWatch("raw/sources/drafts/ready.md", config)).toBe(false)
+ expect(isPathAllowedBySourceWatch("raw/sources/subdir/drafts/ready.md", config)).toBe(false)
expect(isPathAllowedBySourceWatch("raw/sources/.obsidian/index.md", config)).toBe(false)
expect(isPathAllowedBySourceWatch("raw/sources/plan.private.md", config)).toBe(false)
expect(isPathAllowedBySourceWatch("raw/sources/~$Document.docx", config)).toBe(false)
diff --git a/src/lib/source-watch-config.ts b/src/lib/source-watch-config.ts
index 9d650877..7a798923 100644
--- a/src/lib/source-watch-config.ts
+++ b/src/lib/source-watch-config.ts
@@ -1,68 +1,21 @@
import type { SourceWatchConfig } from "@/stores/wiki-store"
import { normalizePath } from "@/lib/path-utils"
+import sourceWatchDefaults from "@/lib/source-watch-defaults.json"
-export const SOURCE_WATCH_DEFAULT_DOCUMENT_EXTENSIONS = [
- "md",
- "mdx",
- "txt",
- "pdf",
- "doc",
- "docx",
- "ppt",
- "pptx",
- "xls",
- "xlsx",
- "rtf",
- "html",
- "htm",
- "csv",
-]
-
-export const DEFAULT_SOURCE_WATCH_CONFIG: SourceWatchConfig = {
- enabled: true,
- autoIngest: true,
- includeExtensions: SOURCE_WATCH_DEFAULT_DOCUMENT_EXTENSIONS,
- excludeExtensions: [
- "tmp",
- "temp",
- "bak",
- "swp",
- "part",
- "partial",
- "crdownload",
- "exe",
- "dll",
- "so",
- "dylib",
- "bin",
- ],
- excludeDirs: [
- ".git",
- ".svn",
- ".hg",
- ".obsidian",
- ".idea",
- ".vscode",
- "node_modules",
- ".cache",
- "__pycache__",
- ],
- excludeGlobs: ["~$*", ".~lock.*#", "*.draft.*", "draft-*", "*.private.*"],
- maxFileSizeMb: 100,
-}
+export const DEFAULT_SOURCE_WATCH_CONFIG: SourceWatchConfig = sourceWatchDefaults
export const SOURCE_WATCH_FILE_TYPE_GROUPS = [
{
id: "documents",
- extensions: ["md", "mdx", "txt", "pdf", "doc", "docx", "rtf"],
+ extensions: ["md", "mdx", "txt", "pdf", "docx", "odt", "rtf"],
},
{
id: "presentations",
- extensions: ["ppt", "pptx"],
+ extensions: ["pptx", "odp"],
},
{
id: "spreadsheets",
- extensions: ["xls", "xlsx", "csv"],
+ extensions: ["xls", "xlsx", "ods", "csv"],
},
{
id: "web",
@@ -72,18 +25,6 @@ export const SOURCE_WATCH_FILE_TYPE_GROUPS = [
id: "data",
extensions: ["json", "yaml", "yml", "xml"],
},
- {
- id: "images",
- extensions: ["png", "jpg", "jpeg", "webp", "gif", "tiff", "bmp", "svg"],
- },
- {
- id: "media",
- extensions: ["mp3", "wav", "m4a", "mp4", "mov", "avi", "mkv", "webm"],
- },
- {
- id: "code",
- extensions: ["js", "ts", "tsx", "py", "rs", "go", "java", "cpp", "c", "h", "sh"],
- },
]
function normalizeExtensions(values: readonly string[] | undefined): string[] {
@@ -129,13 +70,21 @@ function matchesGlob(path: string, pattern: string): boolean {
return re.test(name) || re.test(normalized)
}
+function pathMatchesExcludedDir(path: string, excludedDir: string): boolean {
+ const normalized = normalizePath(path).toLowerCase()
+ const dir = normalizePath(excludedDir).toLowerCase()
+ if (!dir) return false
+ if (dir.includes("/")) {
+ return normalized === dir || normalized.includes(`/${dir}/`) || normalized.startsWith(`${dir}/`)
+ }
+ return normalized.split("/").some((part) => part === dir)
+}
+
export function isPathAllowedBySourceWatch(path: string, config: SourceWatchConfig): boolean {
const cfg = normalizeSourceWatchConfig(config)
const normalized = normalizePath(path)
const parts = normalized.split("/").filter(Boolean)
- const lowerParts = parts.map((part) => part.toLowerCase())
- const excludedDirs = new Set(cfg.excludeDirs.map((dir) => dir.toLowerCase()))
- if (lowerParts.some((part) => excludedDirs.has(part))) return false
+ if (cfg.excludeDirs.some((dir) => pathMatchesExcludedDir(normalized, dir))) return false
if (cfg.excludeGlobs.some((pattern) => matchesGlob(normalized, pattern))) return false
const name = parts[parts.length - 1] ?? ""
if (!name || name.startsWith(".")) return false
diff --git a/src/lib/source-watch-defaults.json b/src/lib/source-watch-defaults.json
new file mode 100644
index 00000000..bcf72c73
--- /dev/null
+++ b/src/lib/source-watch-defaults.json
@@ -0,0 +1,56 @@
+{
+ "enabled": true,
+ "autoIngest": true,
+ "includeExtensions": [
+ "md",
+ "mdx",
+ "txt",
+ "pdf",
+ "docx",
+ "pptx",
+ "xls",
+ "xlsx",
+ "odt",
+ "odp",
+ "ods",
+ "rtf",
+ "html",
+ "htm",
+ "csv"
+ ],
+ "excludeExtensions": [
+ "tmp",
+ "temp",
+ "bak",
+ "swp",
+ "part",
+ "partial",
+ "crdownload",
+ "exe",
+ "dll",
+ "so",
+ "dylib",
+ "bin",
+ "iso",
+ "dmg"
+ ],
+ "excludeDirs": [
+ ".git",
+ ".svn",
+ ".hg",
+ ".obsidian",
+ ".idea",
+ ".vscode",
+ "node_modules",
+ ".cache",
+ "__pycache__"
+ ],
+ "excludeGlobs": [
+ "~$*",
+ ".~lock.*#",
+ "*.draft.*",
+ "draft-*",
+ "*.private.*"
+ ],
+ "maxFileSizeMb": 100
+}
diff --git a/src/stores/wiki-store.ts b/src/stores/wiki-store.ts
index 13697ed8..b1be2ed0 100644
--- a/src/stores/wiki-store.ts
+++ b/src/stores/wiki-store.ts
@@ -1,5 +1,6 @@
import { create } from "zustand"
import type { WikiProject, FileNode } from "@/types/wiki"
+import { DEFAULT_SOURCE_WATCH_CONFIG } from "@/lib/source-watch-config"
/**
* Wire protocol used when `provider === "custom"`. Other providers have a
@@ -343,15 +344,7 @@ export const useWikiStore = create<WikiState>((set) => ({
lastScan: null,
},
- sourceWatchConfig: {
- enabled: true,
- autoIngest: true,
- includeExtensions: ["md", "mdx", "txt", "pdf", "doc", "docx", "ppt", "pptx", "xls", "xlsx", "rtf", "html", "htm", "csv"],
- excludeExtensions: ["tmp", "temp", "bak", "swp", "part", "partial", "crdownload", "exe", "dll", "so", "dylib", "bin"],
- excludeDirs: [".git", ".svn", ".hg", ".obsidian", ".idea", ".vscode", "node_modules", ".cache", "__pycache__"],
- excludeGlobs: ["~$*", ".~lock.*#", "*.draft.*", "draft-*", "*.private.*"],
- maxFileSizeMb: 100,
- },
+ sourceWatchConfig: DEFAULT_SOURCE_WATCH_CONFIG,
setLlmConfig: (llmConfig) => set({ llmConfig }),
setProviderConfigs: (providerConfigs) => set({ providerConfigs }),
diff --git a/tsconfig.app.json b/tsconfig.app.json
index 583c0681..ce73dbb4 100644
--- a/tsconfig.app.json
+++ b/tsconfig.app.json
@@ -11,6 +11,7 @@
"allowImportingTsExtensions": true,
"isolatedModules": true,
"moduleDetection": "force",
+ "resolveJsonModule": true,
"noEmit": true,
"jsx": "react-jsx",