AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
fix(autoindex): resolve only through links, keep the resolution, and stop overclaiming
xerj-org1f61fc4bc13c
diff --git a/engine/crates/xerj-autoindex/src/ignore_rules.rs b/engine/crates/xerj-autoindex/src/ignore_rules.rsindex 1aaef6e7b..4e4092c02 100644--- a/engine/crates/xerj-autoindex/src/ignore_rules.rs+++ b/engine/crates/xerj-autoindex/src/ignore_rules.rs@@ -97,9 +97,14 @@ pub const DEFAULT_IGNORE_PATTERNS: &[&str] = &[];/// Rule label used for the pre-existing dotfile skip. It is not one of the-/// ignore rules and `--no-ignore` does not disable it (it is what keeps `.env`-/// and `.ssh` out of a queryable brain), but a user whose file vanished still-/// needs it named, so it shares the reporting.+/// ignore rules and `--no-ignore` does not disable it (it is the main thing+/// keeping `.env` and `.ssh` out of a queryable brain), but a user whose file+/// vanished still needs it named, so it shares the reporting.+///+/// It is a rule about names, and under `--follow-symlinks` about where a link+/// resolves. A HARD link has neither, so a visible name hard-linked to a file+/// inside a hidden directory is still indexed — see the note in+/// [`crate::walk`].pub const HIDDEN_RULE: &str = "hidden:dotfile";/// A symlink whose resolved target lies outside the folder the operatordiff --git a/engine/crates/xerj-autoindex/src/walk.rs b/engine/crates/xerj-autoindex/src/walk.rsindex 32cc5ddbe..cbcafeefd 100644--- a/engine/crates/xerj-autoindex/src/walk.rs+++ b/engine/crates/xerj-autoindex/src/walk.rs@@ -90,8 +90,10 @@ enum SymlinkVerdict {/// in it at once, which is what makes a chain of links (`a -> b -> .secret/c`)/// and a link partway up a directory path behave the same as a direct one.///-/// `None` means the entry is where it appears to be and the ordinary rules-/// govern it.+/// `Ok(Some(real))` is the resolved path, which the caller stores on the+/// `FileEntry` so every later `open()` reads the file that was judged.+/// `Ok(None)` means there is nothing to resolve and the ordinary rules govern+/// the entry.////// A resolution FAILURE is not `None`. An earlier version of this function/// wrote `path.canonicalize().ok()?`, which collapsed every error into "the@@ -109,19 +111,22 @@ enum SymlinkVerdict {/// reports a dangling link through its error arm. Everything else is refused/// and named in the report, because an answer that could not be computed is not/// an answer of "yes".-fn escaped_or_hidden_target(root_canon: &Path, path: &Path) -> Option<SymlinkVerdict> {+fn escaped_or_hidden_target(+ root_canon: &Path,+ path: &Path,+) -> Result<Option<PathBuf>, SymlinkVerdict> {let real = match path.canonicalize() {Ok(real) => real,- Err(e) if e.kind() == std::io::ErrorKind::NotFound => return None,- Err(_) => return Some(SymlinkVerdict::Unresolvable),+ Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(None),+ Err(_) => return Err(SymlinkVerdict::Unresolvable),};- let rel = real.strip_prefix(root_canon).ok();- let Some(rel) = rel else {- return Some(SymlinkVerdict::OutsideRoot);+ let Some(rel) = real.strip_prefix(root_canon).ok() else {+ return Err(SymlinkVerdict::OutsideRoot);};- rel.components()- .any(|c| is_hidden_name(c.as_os_str()))- .then_some(SymlinkVerdict::HiddenTarget)+ if rel.components().any(|c| is_hidden_name(c.as_os_str())) {+ return Err(SymlinkVerdict::HiddenTarget);+ }+ Ok(Some(real))}/// Walk with the default ignore rules on and the report discarded.@@ -147,6 +152,8 @@ pub fn walk_reporting(anyhow::bail!("{} is not a directory", root_canon.display());}let mut out = Vec::new();+ // Shallowest depth from which the walk is inside a symlink; see the guard.+ let mut link_depth: Option<usize> = None;let marker_defaults = ignore.enabled && ignore.defaults;let mut stack = IgnoreStack::new(&root_canon, ignore);// Manual iteration rather than `filter_entry`, because the ignore stack has@@ -183,7 +190,7 @@ pub fn walk_reporting(// `IgnoreStack::truncate_to` for why every entry needs it, not just// directories.stack.truncate_to(entry.depth());- // SECURITY / hygiene: never index hidden files or descend into hidden+ // SECURITY / hygiene: do not index hidden files or descend into hidden// directories. Without this the walker happily indexed `.env` (secrets,// API tokens), `.git`, `.ssh`, `.aws`, and other dotfiles into a// queryable brain with no per-brain authorization — a real exposure for@@ -191,7 +198,17 @@ pub fn walk_reporting(// pruned before descending, so `.git/` is skipped whole. The root// itself (depth 0) is exempt so a brain over a dot-named folder still// works. `--no-ignore` does NOT turn this off: it is not one of the- // ignore rules, it is the reason secrets stay out.+ // ignore rules, it is the main reason secrets stay out.+ //+ // The limit of that claim, stated because the rest of this file used to+ // assert it without one: this is a rule about NAMES and, under+ // `--follow-symlinks`, about where a link RESOLVES. A hard link has+ // neither — it is a second directory entry for the same inode, so+ // `ln .secretdir/k.txt notes.txt` reads the secret under a visible name+ // and nothing here can tell. Defending that needs `(dev, ino)` of every+ // hidden-rule exclusion, which a walk that prunes hidden directories+ // without descending does not have. `cp -al`, `rsync --link-dest` and+ // hard-linked package stores all produce that shape without an attacker.if entry.depth() > 0 && is_hidden_name(entry.file_name()) {stack.record_hidden(is_dir);if is_dir {@@ -213,30 +230,54 @@ pub fn walk_reporting(// So under the flag the same two questions are asked of the RESOLVED// path instead of the name. Loop-safety is walkdir's and unrelated;// this is the secret rule and the root boundary.- if follow_symlinks && entry.depth() > 0 {+ // Resolving is only necessary once the walk has actually gone through a+ // link. `canonicalize` is `realpath(3)` and issues one `readlinkat` per+ // path component, so doing it for every entry costs 8-14 failing+ // syscalls per file on an ordinary tree and made the walk 4.2x slower+ // with 7.6x the syscalls — all of it wasted on paths where no link+ // exists. `link_depth` is the shallowest depth at or below which some+ // ancestor (or the entry itself) is a symlink; below that the walk path+ // and the real path cannot differ.+ if entry.path_is_symlink() {+ link_depth = Some(link_depth.map_or(entry.depth(), |d: usize| d.min(entry.depth())));+ } else if is_dir && link_depth.is_some_and(|d| entry.depth() <= d) {+ link_depth = None;+ }+ let through_link = entry.path_is_symlink() || link_depth.is_some_and(|d| entry.depth() > d);+ // The path every later stage should open. The guard resolves the entry+ // to decide on it; keeping that resolution is what stops the decision+ // being made about one file and the bytes being read from another.+ // `content.rs` hashes and `lib.rs` extracts from `FileEntry::path`+ // minutes after the walk on a large corpus, each a fresh `open()` that+ // re-follows the link — so a link re-pointed in between handed the+ // refused content straight into a document body. `rel` still comes from+ // the walk path: that is the name the operator pointed at and the+ // identity the index is keyed on.+ let mut resolved: Option<PathBuf> = None;+ if follow_symlinks && entry.depth() > 0 && through_link {match escaped_or_hidden_target(&root_canon, entry.path()) {- Some(SymlinkVerdict::OutsideRoot) => {+ Ok(real) => resolved = real,+ Err(SymlinkVerdict::OutsideRoot) => {stack.record_symlink_escape(is_dir);if is_dir {it.skip_current_dir();}continue;}- Some(SymlinkVerdict::HiddenTarget) => {+ Err(SymlinkVerdict::HiddenTarget) => {stack.record_hidden(is_dir);if is_dir {it.skip_current_dir();}continue;}- Some(SymlinkVerdict::Unresolvable) => {+ Err(SymlinkVerdict::Unresolvable) => {stack.record_symlink_unresolved(is_dir);if is_dir {it.skip_current_dir();}continue;}- None => {}}}if is_dir {@@ -280,7 +321,7 @@ pub fn walk_reporting(.replace('\\', "/");let rel_id = stable_path_id(p.strip_prefix(&root_canon).unwrap_or(&p));out.push(FileEntry {- path: p,+ path: resolved.unwrap_or(p),rel,rel_id,is_symlink: entry@@ -347,13 +388,19 @@ mod hidden_skip_tests {// Exists and resolves inside the root: admitted.std::fs::write(root.join("plain.txt"), "x").unwrap();assert!(- escaped_or_hidden_target(&root, &root.join("plain.txt")).is_none(),- "an ordinary in-root file must be governed by the ordinary rules"+ matches!(+ escaped_or_hidden_target(&root, &root.join("plain.txt")),+ Ok(Some(_))+ ),+ "an ordinary in-root file must resolve and be governed by the ordinary rules");// Does not exist: nothing to leak, so also admitted — walkdir reports it.assert!(- escaped_or_hidden_target(&root, &root.join("gone.txt")).is_none(),+ matches!(+ escaped_or_hidden_target(&root, &root.join("gone.txt")),+ Ok(None)+ ),"a dangling link has no target and must not be reported as a refusal");@@ -362,7 +409,7 @@ mod hidden_skip_tests {assert!(matches!(escaped_or_hidden_target(&root, &bad),- Some(SymlinkVerdict::Unresolvable)+ Err(SymlinkVerdict::Unresolvable)),"an unresolvable path must be refused — `could not compute` is not `yes`");