Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

fix(autoindex): say that a .xerjignore !pattern cannot re-include a hidden dir (#1844) (#1916)

xerj-orgc238592e5098
diff --git a/engine/crates/xerj-autoindex/src/cli.rs b/engine/crates/xerj-autoindex/src/cli.rs
index 25b275e34..bf64af1ad 100644
--- a/engine/crates/xerj-autoindex/src/cli.rs
+++ b/engine/crates/xerj-autoindex/src/cli.rs
@@ -407,6 +407,8 @@ pub fn help_text_with(feedback: bool) -> String {
\x20\x20\x20\x20\x20\x20.xerjignore same syntax as .gitignore, XERJ-only. Use it to exclude\n\
\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20something git tracks, or to re-include (!pattern) something\n\
\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20git ignores. Outranks .gitignore at any depth.\n\
+ \x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20!pattern cannot re-include a hidden (dot-named)\n\
+ \x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20entry: see HIDDEN below.\n\
\x20\x20\x20\x20\x20\x20.gitignore including nested ones and negation; the closest file wins.\n\
\x20\x20\x20\x20\x20\x20.git/info/exclude per-checkout excludes, including nested checkouts.\n\
\x20\x20\x20\x20\x20\x20built-in {defaults}\n\
@@ -424,6 +426,11 @@ pub fn help_text_with(feedback: bool) -> String {
\x20\x20\x20\x20ignored_files_in_pruned_dirs_exact=false to say so).\n\
\x20\x20\x20\x20The folder you name is never rejected: if it is itself ignored, it is\n\
\x20\x20\x20\x20indexed anyway and the run says which rule it would have matched.\n\
+ \x20\x20\x20\x20HIDDEN: dot-named files and directories below the folder (.env, .git,\n\
+ \x20\x20\x20\x20.ssh, .<tool>/) are skipped by a separate rule that runs before any\n\
+ \x20\x20\x20\x20ignore file. No .xerjignore !pattern and no --no-ignore lifts it (the\n\
+ \x20\x20\x20\x20run warns when a .xerjignore tries). To index one, name it as the\n\
+ \x20\x20\x20\x20root, which is never rejected: xerj autoindex ./.openfox --prefix openfox\n\
\n\
OBJECT STORAGE:\n\
\x20\x20\x20\x20`xerj autoindex s3://bucket/prefix` indexes objects instead of files.\n\
diff --git a/engine/crates/xerj-autoindex/src/ignore_rules.rs b/engine/crates/xerj-autoindex/src/ignore_rules.rs
index 4cfe9c63d..2a111a354 100644
--- a/engine/crates/xerj-autoindex/src/ignore_rules.rs
+++ b/engine/crates/xerj-autoindex/src/ignore_rules.rs
@@ -745,6 +745,39 @@ impl IgnoreStack {
}
}
+ /// #1844: a `.xerjignore` `!pattern` that names a hidden entry looks like it
+ /// re-includes it, and `--help` documents `!pattern` as the re-include. It
+ /// cannot: the dotfile rule is not an ignore rule (see [`HIDDEN_RULE`]) and
+ /// runs before any ignore file is consulted. Saying so — once per entry,
+ /// with the named-root workaround — replaces a silent no-op.
+ ///
+ /// Only `.xerjignore` is asked: a `.gitignore` `!.env.example` is about
+ /// what git tracks, not a request to XERJ, and warning on it would fire on
+ /// half the repositories in existence. Call it for the hidden entry the
+ /// walker is about to skip, after [`IgnoreStack::truncate_to`].
+ pub fn note_hidden_reinclude(&mut self, path: &Path, is_dir: bool) {
+ if !self.opts.enabled {
+ return;
+ }
+ let decision = self
+ .layers
+ .iter()
+ .rev()
+ .find_map(|layer| self.decide_with(layer.xerj.as_ref(), path, is_dir));
+ let Some((false, label)) = decision else {
+ return;
+ };
+ let kind = if is_dir { "directory" } else { "file" };
+ self.warn(format!(
+ "`{label}` re-includes the hidden {kind} {rel}, but hidden (dot-named) entries are \
+ never indexed: that rule keeps .env and .ssh out, and no ignore file, !pattern or \
+ --no-ignore lifts it. To index it, name it as its own root: `xerj autoindex {abs} \
+ --prefix <name>`",
+ rel = self.rel(path),
+ abs = path.display(),
+ ));
+ }
+
/// A symlink target outside the indexed root, also decided by the walker
/// (it needs the canonical root, which this stack does not carry) and
/// reported through the same accounting. See [`ESCAPED_ROOT_RULE`].
diff --git a/engine/crates/xerj-autoindex/src/walk.rs b/engine/crates/xerj-autoindex/src/walk.rs
index 67325cd96..493233a29 100644
--- a/engine/crates/xerj-autoindex/src/walk.rs
+++ b/engine/crates/xerj-autoindex/src/walk.rs
@@ -367,6 +367,7 @@ fn walk_impl(
// without descending does not have. `cp -al`, `rsync --link-dest` and
// hard-linked package stores all produce that shape without an attacker.
if entry.depth() > 0 && is_hidden_name(entry.file_name()) {
+ stack.note_hidden_reinclude(entry.path(), is_dir);
stack.record_hidden(is_dir);
if is_dir {
it.skip_current_dir();
@@ -755,6 +756,60 @@ mod hidden_skip_tests {
assert_eq!(off.dirs_pruned, 0, "{off:?}");
}
+ /// #1844: `!.openfox/` in a `.xerjignore` had no effect on the dotfile
+ /// prune and nothing said so. The hidden directory stays out (the rule is
+ /// what keeps `.env` and `.ssh` out), but the run now names the pattern
+ /// that tried and the named-root command that works; a `.gitignore`
+ /// negation of a dotfile (`!.env.example`) is git's business and stays
+ /// silent.
+ #[test]
+ fn a_xerjignore_re_include_of_a_hidden_dir_is_refused_out_loud() {
+ use super::walk_reporting;
+ use crate::ignore_rules::{IgnoreOptions, HIDDEN_RULE};
+
+ let dir = tempfile::TempDir::new().unwrap();
+ let root = dir.path();
+ fs::write(root.join(".xerjignore"), "!.openfox/\n").unwrap();
+ fs::write(root.join(".gitignore"), ".env*\n!.env.example\n").unwrap();
+ fs::write(root.join(".env.example"), "KEY=").unwrap();
+ fs::write(root.join("top.md"), "# top").unwrap();
+ fs::create_dir_all(root.join(".openfox/workflows")).unwrap();
+ fs::write(root.join(".openfox/workflows/w.md"), "# wf").unwrap();
+
+ let (files, report) = walk_reporting(root, false, IgnoreOptions::default()).unwrap();
+ let rels: Vec<String> = files.into_iter().map(|e| e.rel).collect();
+ assert_eq!(rels, vec!["top.md".to_string()], "{rels:?}");
+ assert_eq!(report.by_rule[HIDDEN_RULE].dirs, 1, "{report:?}");
+
+ let warned: Vec<&String> = report
+ .warnings
+ .iter()
+ .filter(|w| w.contains("re-includes the hidden"))
+ .collect();
+ assert_eq!(warned.len(), 1, "{:?}", report.warnings);
+ let warning = warned[0];
+ assert!(warning.contains(".xerjignore:!.openfox/"), "{warning}");
+ assert!(warning.contains("directory .openfox"), "{warning}");
+ assert!(warning.contains("never indexed"), "{warning}");
+ assert!(warning.contains("xerj autoindex "), "{warning}");
+ assert!(warning.contains(".openfox --prefix <name>"), "{warning}");
+ assert!(
+ report
+ .summary_lines()
+ .iter()
+ .any(|l| l.starts_with("ignore rules: ") && l.contains("re-includes the hidden")),
+ "the warning must reach the printed report"
+ );
+
+ // --no-ignore reads no ignore file, so there is no pattern to answer.
+ let (_, off) = walk_reporting(root, false, IgnoreOptions::off()).unwrap();
+ assert!(
+ off.warnings.iter().all(|w| !w.contains("re-includes")),
+ "{:?}",
+ off.warnings
+ );
+ }
+
/// Create a real checkout at `path`. Returns false when the `git` binary
/// is unavailable, in which case the caller still gets the on-disk shape
/// that matters (`.git/HEAD`) but skips the parity cross-check.