AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
fix(autoindex): say that a .xerjignore !pattern cannot re-include a hidden dir (#1844) (#1916)
xerj-orgc238592e5098
diff --git a/engine/crates/xerj-autoindex/src/cli.rs b/engine/crates/xerj-autoindex/src/cli.rsindex 25b275e34..bf64af1ad 100644--- a/engine/crates/xerj-autoindex/src/cli.rs+++ b/engine/crates/xerj-autoindex/src/cli.rs@@ -407,6 +407,8 @@ pub fn help_text_with(feedback: bool) -> String {\x20\x20\x20\x20\x20\x20.xerjignore same syntax as .gitignore, XERJ-only. Use it to exclude\n\\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20something git tracks, or to re-include (!pattern) something\n\\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20git ignores. Outranks .gitignore at any depth.\n\+ \x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20!pattern cannot re-include a hidden (dot-named)\n\+ \x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20entry: see HIDDEN below.\n\\x20\x20\x20\x20\x20\x20.gitignore including nested ones and negation; the closest file wins.\n\\x20\x20\x20\x20\x20\x20.git/info/exclude per-checkout excludes, including nested checkouts.\n\\x20\x20\x20\x20\x20\x20built-in {defaults}\n\@@ -424,6 +426,11 @@ pub fn help_text_with(feedback: bool) -> String {\x20\x20\x20\x20ignored_files_in_pruned_dirs_exact=false to say so).\n\\x20\x20\x20\x20The folder you name is never rejected: if it is itself ignored, it is\n\\x20\x20\x20\x20indexed anyway and the run says which rule it would have matched.\n\+ \x20\x20\x20\x20HIDDEN: dot-named files and directories below the folder (.env, .git,\n\+ \x20\x20\x20\x20.ssh, .<tool>/) are skipped by a separate rule that runs before any\n\+ \x20\x20\x20\x20ignore file. No .xerjignore !pattern and no --no-ignore lifts it (the\n\+ \x20\x20\x20\x20run warns when a .xerjignore tries). To index one, name it as the\n\+ \x20\x20\x20\x20root, which is never rejected: xerj autoindex ./.openfox --prefix openfox\n\\n\OBJECT STORAGE:\n\\x20\x20\x20\x20`xerj autoindex s3://bucket/prefix` indexes objects instead of files.\n\diff --git a/engine/crates/xerj-autoindex/src/ignore_rules.rs b/engine/crates/xerj-autoindex/src/ignore_rules.rsindex 4cfe9c63d..2a111a354 100644--- a/engine/crates/xerj-autoindex/src/ignore_rules.rs+++ b/engine/crates/xerj-autoindex/src/ignore_rules.rs@@ -745,6 +745,39 @@ impl IgnoreStack {}}+ /// #1844: a `.xerjignore` `!pattern` that names a hidden entry looks like it+ /// re-includes it, and `--help` documents `!pattern` as the re-include. It+ /// cannot: the dotfile rule is not an ignore rule (see [`HIDDEN_RULE`]) and+ /// runs before any ignore file is consulted. Saying so — once per entry,+ /// with the named-root workaround — replaces a silent no-op.+ ///+ /// Only `.xerjignore` is asked: a `.gitignore` `!.env.example` is about+ /// what git tracks, not a request to XERJ, and warning on it would fire on+ /// half the repositories in existence. Call it for the hidden entry the+ /// walker is about to skip, after [`IgnoreStack::truncate_to`].+ pub fn note_hidden_reinclude(&mut self, path: &Path, is_dir: bool) {+ if !self.opts.enabled {+ return;+ }+ let decision = self+ .layers+ .iter()+ .rev()+ .find_map(|layer| self.decide_with(layer.xerj.as_ref(), path, is_dir));+ let Some((false, label)) = decision else {+ return;+ };+ let kind = if is_dir { "directory" } else { "file" };+ self.warn(format!(+ "`{label}` re-includes the hidden {kind} {rel}, but hidden (dot-named) entries are \+ never indexed: that rule keeps .env and .ssh out, and no ignore file, !pattern or \+ --no-ignore lifts it. To index it, name it as its own root: `xerj autoindex {abs} \+ --prefix <name>`",+ rel = self.rel(path),+ abs = path.display(),+ ));+ }+/// A symlink target outside the indexed root, also decided by the walker/// (it needs the canonical root, which this stack does not carry) and/// reported through the same accounting. See [`ESCAPED_ROOT_RULE`].diff --git a/engine/crates/xerj-autoindex/src/walk.rs b/engine/crates/xerj-autoindex/src/walk.rsindex 67325cd96..493233a29 100644--- a/engine/crates/xerj-autoindex/src/walk.rs+++ b/engine/crates/xerj-autoindex/src/walk.rs@@ -367,6 +367,7 @@ fn walk_impl(// without descending does not have. `cp -al`, `rsync --link-dest` and// hard-linked package stores all produce that shape without an attacker.if entry.depth() > 0 && is_hidden_name(entry.file_name()) {+ stack.note_hidden_reinclude(entry.path(), is_dir);stack.record_hidden(is_dir);if is_dir {it.skip_current_dir();@@ -755,6 +756,60 @@ mod hidden_skip_tests {assert_eq!(off.dirs_pruned, 0, "{off:?}");}+ /// #1844: `!.openfox/` in a `.xerjignore` had no effect on the dotfile+ /// prune and nothing said so. The hidden directory stays out (the rule is+ /// what keeps `.env` and `.ssh` out), but the run now names the pattern+ /// that tried and the named-root command that works; a `.gitignore`+ /// negation of a dotfile (`!.env.example`) is git's business and stays+ /// silent.+ #[test]+ fn a_xerjignore_re_include_of_a_hidden_dir_is_refused_out_loud() {+ use super::walk_reporting;+ use crate::ignore_rules::{IgnoreOptions, HIDDEN_RULE};++ let dir = tempfile::TempDir::new().unwrap();+ let root = dir.path();+ fs::write(root.join(".xerjignore"), "!.openfox/\n").unwrap();+ fs::write(root.join(".gitignore"), ".env*\n!.env.example\n").unwrap();+ fs::write(root.join(".env.example"), "KEY=").unwrap();+ fs::write(root.join("top.md"), "# top").unwrap();+ fs::create_dir_all(root.join(".openfox/workflows")).unwrap();+ fs::write(root.join(".openfox/workflows/w.md"), "# wf").unwrap();++ let (files, report) = walk_reporting(root, false, IgnoreOptions::default()).unwrap();+ let rels: Vec<String> = files.into_iter().map(|e| e.rel).collect();+ assert_eq!(rels, vec!["top.md".to_string()], "{rels:?}");+ assert_eq!(report.by_rule[HIDDEN_RULE].dirs, 1, "{report:?}");++ let warned: Vec<&String> = report+ .warnings+ .iter()+ .filter(|w| w.contains("re-includes the hidden"))+ .collect();+ assert_eq!(warned.len(), 1, "{:?}", report.warnings);+ let warning = warned[0];+ assert!(warning.contains(".xerjignore:!.openfox/"), "{warning}");+ assert!(warning.contains("directory .openfox"), "{warning}");+ assert!(warning.contains("never indexed"), "{warning}");+ assert!(warning.contains("xerj autoindex "), "{warning}");+ assert!(warning.contains(".openfox --prefix <name>"), "{warning}");+ assert!(+ report+ .summary_lines()+ .iter()+ .any(|l| l.starts_with("ignore rules: ") && l.contains("re-includes the hidden")),+ "the warning must reach the printed report"+ );++ // --no-ignore reads no ignore file, so there is no pattern to answer.+ let (_, off) = walk_reporting(root, false, IgnoreOptions::off()).unwrap();+ assert!(+ off.warnings.iter().all(|w| !w.contains("re-includes")),+ "{:?}",+ off.warnings+ );+ }+/// Create a real checkout at `path`. Returns false when the `git` binary/// is unavailable, in which case the caller still gets the on-disk shape/// that matters (`.git/HEAD`) but skips the parity cross-check.