Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

feat(autoindex): honour .gitignore/.xerjignore so junk is never indexed (#276)

xerj-org5c7b369635f7
diff --git a/docs/recipes/zero-config-indexing.md b/docs/recipes/zero-config-indexing.md
index f3c92f1c3..c2a33f3e6 100644
--- a/docs/recipes/zero-config-indexing.md
+++ b/docs/recipes/zero-config-indexing.md
@@ -270,6 +270,20 @@ phase), `--prefix` to namespace the indices (default `ax`),
print the inferred plan without indexing anything, `--follow-symlinks`
(loop-safe), `--sample N` records per file for inference (default 500).
+Build output never reaches the index in the first place. The walk honours
+`.xerjignore`, `.gitignore` (nested files and `!negation` included) and
+`.git/info/exclude`, plus a built-in list — `node_modules/ vendor/ target/
+dist/ build/ .venv/ __pycache__/` — in that order of precedence; an excluded
+directory is never descended, so nothing inside it is stat-ed, hashed or sent.
+Hidden files (`.env`, `.git/`) are skipped by a separate, non-optional rule.
+Every run prints what was dropped and by which rule (`ignore rules: …` on
+stderr), and `--dry-run` also counts the files inside each pruned directory, so
+"where did my file go?" always has an answer. `--no-ignore` turns the rules off;
+`--no-default-ignores` keeps your ignore files and drops only the built-in list.
+The folder you point at is never excluded: if it is itself ignored — say you
+run this on `~/proj/target` — it is indexed anyway and the run says which rule
+it would have matched.
+
`--workers` is a ceiling, not a promise: when the server answers `429 Too Many
Requests` the run halves its own bulk concurrency on the spot and probes back
up one worker at a time only after a clean streak, so a busy server is not
diff --git a/engine/Cargo.lock b/engine/Cargo.lock
index 3b18ec0b4..3ef85a9eb 100644
--- a/engine/Cargo.lock
+++ b/engine/Cargo.lock
@@ -529,6 +529,16 @@ dependencies = [
"alloc-stdlib",
]
+[[package]]
+name = "bstr"
+version = "1.13.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6bb31b46c14244e20ee9984b11bf5c992b91fb6939fea616e3512c8baecdbe5f"
+dependencies = [
+ "memchr",
+ "serde_core",
+]
+
[[package]]
name = "bumpalo"
version = "3.20.2"
@@ -1930,6 +1940,19 @@ version = "0.3.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0cc23270f6e1808e30a928bdc84dea0b9b4136a8bc82338574f23baf47bbd280"
+[[package]]
+name = "globset"
+version = "0.4.19"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e47d37d2ae4464254884b60ab7071be2b876a9c35b696bd018ddcc76847309cd"
+dependencies = [
+ "aho-corasick",
+ "bstr",
+ "log",
+ "regex-automata",
+ "regex-syntax",
+]
+
[[package]]
name = "grid"
version = "1.0.1"
@@ -2370,6 +2393,22 @@ dependencies = [
"icu_properties",
]
+[[package]]
+name = "ignore"
+version = "0.4.32"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0b17771570a2b94107741a7b033f19132c2eee21d59d21b24d2ced26500bd66e"
+dependencies = [
+ "crossbeam-deque",
+ "globset",
+ "log",
+ "memchr",
+ "regex-automata",
+ "same-file",
+ "walkdir",
+ "winapi-util",
+]
+
[[package]]
name = "image"
version = "0.25.10"
@@ -6437,6 +6476,7 @@ dependencies = [
"encoding_rs",
"flate2",
"fs2",
+ "ignore",
"libc",
"memchr",
"pdf_oxide",
diff --git a/engine/crates/xerj-autoindex/Cargo.toml b/engine/crates/xerj-autoindex/Cargo.toml
index f2784edf6..a5c0d9849 100644
--- a/engine/crates/xerj-autoindex/Cargo.toml
+++ b/engine/crates/xerj-autoindex/Cargo.toml
@@ -16,6 +16,11 @@ serde_json.workspace = true
chrono.workspace = true
regex = "1"
walkdir = "2"
+# gitignore/.xerjignore matching (#276). ripgrep's matcher, used for its
+# semantics only — traversal stays on walkdir so the existing symlink,
+# hidden-file and unreadable-entry behaviour is unchanged.
+# Licence: Unlicense OR MIT — compatible with XERJ's Apache-2.0.
+ignore = "0.4"
encoding_rs = "0.8"
xxhash-rust.workspace = true
memchr.workspace = true
diff --git a/engine/crates/xerj-autoindex/src/cli.rs b/engine/crates/xerj-autoindex/src/cli.rs
index 85196c78c..ce648eff1 100644
--- a/engine/crates/xerj-autoindex/src/cli.rs
+++ b/engine/crates/xerj-autoindex/src/cli.rs
@@ -1,5 +1,6 @@
//! Hand-rolled arg parser (house style of xerj-server — no clap).
+use crate::ignore_rules::IgnoreOptions;
use crate::progress::ProgressMode;
use std::path::PathBuf;
use std::time::Duration;
@@ -30,6 +31,9 @@ pub struct IndexCfg {
pub state_dir: Option<PathBuf>,
pub fresh: bool,
pub follow_symlinks: bool,
+ /// `.gitignore` / `.xerjignore` / built-in build-output rules (#276).
+ /// `deep_count` is set from `--dry-run`.
+ pub ignore: IgnoreOptions,
pub max_file_gb: u64,
pub sample: usize,
pub no_semantic: bool,
@@ -114,6 +118,12 @@ pub fn print_help() {
bytes (default 64); excludes filesystem/manifest overhead\n\
--fresh {fresh_help}\n\
--follow-symlinks follow symlinks (loop-safe); off by default\n\
+ --no-ignore index everything: no .gitignore, no .xerjignore, no\n\
+ .git/info/exclude, no built-in defaults. Hidden files\n\
+ (.env, .git/, .ssh) stay skipped either way — that is\n\
+ not an ignore rule, it is what keeps secrets out.\n\
+ --no-default-ignores keep the ignore files, drop only the built-in list\n\
+ ({defaults})\n\
--max-file-gb <N> skip+record oversized non-streamable files (default 2)\n\
--sample <N> records sampled per file for inference (default 500)\n\
--no-semantic skip semantic_text on body fields (pure BM25+keyword)\n\
@@ -136,6 +146,24 @@ pub fn print_help() {
--dataset <SLUG> (map) show a single dataset\n\
--help, -h this help\n\
\n\
+ IGNORE RULES:\n\
+ The fastest file is the one that is never read, so junk is dropped during\n\
+ the walk, not after it. Honoured, highest precedence first:\n\
+ .xerjignore same syntax as .gitignore, XERJ-only. Use it to exclude\n\
+ something git tracks, or to re-include (!pattern) something\n\
+ git ignores. Outranks .gitignore at any depth.\n\
+ .gitignore including nested ones and negation; the closest file wins.\n\
+ .git/info/exclude per-checkout excludes, including nested checkouts.\n\
+ built-in {defaults}\n\
+ Your global gitignore (core.excludesFile) is NOT read: a machine-wide\n\
+ preference should not silently decide what is in your index.\n\
+ A directory the rules reject is never descended, so nothing inside it is\n\
+ stat-ed, hashed or sent. Every run prints what was dropped and by which\n\
+ rule; --dry-run additionally counts the files inside each pruned directory\n\
+ (bounded — past the budget the count is reported as `at least N`).\n\
+ The folder you name is never rejected: if it is itself ignored, it is\n\
+ indexed anyway and the run says which rule it would have matched.\n\
+ \n\
PDF EXTRACTION:\n\
Each PDF uses a fresh process. Limits: 512 MiB input, 32 MiB worker output,\n\
100,000 pages, and 1.5 GiB address space on Unix. This is crash/resource\n\
@@ -221,7 +249,8 @@ pub fn print_help() {
2 usage; 1 endpoint/journal failure, a refused corpus removal, or a\n\
refused unsafe state transition\n",
fresh_help = FRESH_HELP,
- resume_policy_help = RESUME_POLICY_HELP
+ resume_policy_help = RESUME_POLICY_HELP,
+ defaults = crate::ignore_rules::DEFAULT_IGNORE_PATTERNS.join(" ")
);
}
@@ -246,6 +275,8 @@ pub fn parse(args: Vec<String>) -> Result<Cmd, String> {
let mut state_dir: Option<PathBuf> = None;
let mut fresh = false;
let mut follow_symlinks = false;
+ let mut no_ignore = false;
+ let mut no_default_ignores = false;
let mut max_file_gb = 2u64;
let mut sample = 500usize;
let mut no_semantic = false;
@@ -333,6 +364,8 @@ pub fn parse(args: Vec<String>) -> Result<Cmd, String> {
"--state-dir" => state_dir = it.next().map(PathBuf::from),
"--fresh" => fresh = true,
"--follow-symlinks" => follow_symlinks = true,
+ "--no-ignore" => no_ignore = true,
+ "--no-default-ignores" => no_default_ignores = true,
"--max-file-gb" => {
max_file_gb = it
.next()
@@ -416,6 +449,17 @@ pub fn parse(args: Vec<String>) -> Result<Cmd, String> {
} else {
progress.unwrap_or(ProgressMode::Auto)
};
+ // Redundant-but-honoured is fine; accepted-and-ignored is the #204 class.
+ // `--no-ignore` already removes the built-in defaults, so there is nothing
+ // left for `--no-default-ignores` to do — say so instead of taking a flag
+ // that changes nothing.
+ if no_ignore && no_default_ignores {
+ return Err(
+ "--no-ignore already turns off the built-in default rules that --no-default-ignores \
+ targets. Drop one of the two"
+ .into(),
+ );
+ }
if progress_interval.is_some() && progress == ProgressMode::None {
return Err(
"--progress-interval sets the cadence of a progress stream that --progress none / \
@@ -466,6 +510,13 @@ pub fn parse(args: Vec<String>) -> Result<Cmd, String> {
state_dir,
fresh,
follow_symlinks,
+ ignore: IgnoreOptions {
+ enabled: !no_ignore,
+ defaults: !no_ignore && !no_default_ignores,
+ // Only --dry-run pays for counting what is inside a pruned
+ // directory; a real run's whole point is not touching it.
+ deep_count: dry_run,
+ },
max_file_gb,
sample: sample.max(50),
no_semantic,
@@ -702,4 +753,32 @@ mod tests {
assert!(err.contains("applies only to indexing"), "{err}");
}
}
+
+ /// #276. Ignore rules are on by default — that is the whole point of the
+ /// issue — and each flag turns off exactly what it names.
+ #[test]
+ fn ignore_rules_default_on_and_each_flag_turns_off_what_it_names() {
+ let cfg = index(&["data"]);
+ assert!(cfg.ignore.enabled);
+ assert!(cfg.ignore.defaults);
+ assert!(!cfg.ignore.deep_count, "only --dry-run pays for the count");
+
+ let none = index(&["data", "--no-ignore"]);
+ assert!(!none.ignore.enabled);
+ assert!(!none.ignore.defaults);
+
+ let keep_files = index(&["data", "--no-default-ignores"]);
+ assert!(keep_files.ignore.enabled, "ignore files still apply");
+ assert!(!keep_files.ignore.defaults);
+
+ assert!(index(&["data", "--dry-run"]).ignore.deep_count);
+ }
+
+ /// A flag that cannot change anything is refused rather than accepted and
+ /// quietly dropped (#204's class).
+ #[test]
+ fn no_default_ignores_under_no_ignore_is_refused() {
+ let err = err(&["data", "--no-ignore", "--no-default-ignores"]);
+ assert!(err.contains("--no-ignore already turns off"), "{err}");
+ }
}
diff --git a/engine/crates/xerj-autoindex/src/detect/e2e.rs b/engine/crates/xerj-autoindex/src/detect/e2e.rs
index 59174c85a..af2848899 100644
--- a/engine/crates/xerj-autoindex/src/detect/e2e.rs
+++ b/engine/crates/xerj-autoindex/src/detect/e2e.rs
@@ -268,6 +268,7 @@ fn cfg(root: &Path, state_dir: &Path, url: &str) -> IndexCfg {
state_dir: Some(state_dir.to_owned()),
fresh: false,
follow_symlinks: false,
+ ignore: crate::ignore_rules::IgnoreOptions::default(),
max_file_gb: 1,
sample: 50,
no_semantic: true,
diff --git a/engine/crates/xerj-autoindex/src/failure_resume_http_tests.rs b/engine/crates/xerj-autoindex/src/failure_resume_http_tests.rs
index a18f5ad3e..4a3df54dc 100644
--- a/engine/crates/xerj-autoindex/src/failure_resume_http_tests.rs
+++ b/engine/crates/xerj-autoindex/src/failure_resume_http_tests.rs
@@ -515,6 +515,7 @@ fn cfg(root: &Path, state_dir: &Path, url: &str) -> IndexCfg {
state_dir: Some(state_dir.to_owned()),
fresh: false,
follow_symlinks: false,
+ ignore: crate::ignore_rules::IgnoreOptions::default(),
max_file_gb: 1,
sample: 50,
no_semantic: true,
diff --git a/engine/crates/xerj-autoindex/src/ignore_rules.rs b/engine/crates/xerj-autoindex/src/ignore_rules.rs
new file mode 100644
index 000000000..6731af882
--- /dev/null
+++ b/engine/crates/xerj-autoindex/src/ignore_rules.rs
@@ -0,0 +1,950 @@
+//! Ignore rules for the inventory walk (#276).
+//!
+//! The cheapest file to index is the one that is never read. A repository's
+//! `.gitignore` already says which files are build output; honouring it means
+//! `target/`, `node_modules/` and friends are pruned before the walk descends
+//! into them, so they cost nothing at all — not a stat, not a hash, not a bulk
+//! body.
+//!
+//! # Where the semantics come from
+//!
+//! gitignore matching is easy to get 80% right and very hard to get right:
+//! nested ignore files, negation (`!keep.log`), directory-only patterns
+//! (`build/`), anchoring (`/only-here`) and precedence between an ignore file
+//! in a subdirectory and one at the root. We do not hand-roll any of that. The
+//! matcher is ripgrep's `ignore` crate (`Unlicense OR MIT`, compatible with
+//! XERJ's Apache-2.0), used for its **matching** only — traversal stays on
+//! `walkdir` so the existing symlink, hidden-file and unreadable-entry
+//! behaviour of [`crate::walk`] is unchanged, and so we can report *which*
+//! rule discarded *what*, which `ignore`'s own walker does not expose.
+//!
+//! Two behaviours are copied deliberately from that reference implementation
+//! (`ignore-0.4.32/src/dir.rs`, read at `~/.cargo/registry/src/*/ignore-0.4.32`):
+//!
+//! * **Deepest ignore file wins, per kind.** `dir.rs:563-594` walks matchers
+//! from the current directory outward and keeps the first decision each kind
+//! produces (`if m_gi.is_none() { m_gi = … }`).
+//! * **Kinds are ranked, and the rank beats depth.** `dir.rs:678-684` combines
+//! them `m_custom_ignore.or(m_ignore).or(m_gi).or(m_gi_exclude)`. So a
+//! `.xerjignore` at the root outranks a `.gitignore` in a deep subdirectory.
+//! That is what makes `.xerjignore` useful as the stated "exclude things git
+//! tracks" (and re-include things git ignores) override rather than just a
+//! second `.gitignore`.
+//!
+//! Within one file the last matching pattern wins; that is `Gitignore::matched`
+//! and we do not reimplement it.
+//!
+//! # What is deliberately not honoured
+//!
+//! The user's *global* gitignore (`core.excludesFile`) is not read: a machine-
+//! wide preference silently deciding what is in someone's search index is a
+//! surprise, and it is not visible in the folder being indexed. `.gitignore`,
+//! `.xerjignore` and `.git/info/exclude` all live in the tree the user pointed
+//! at, so they are.
+
+use ignore::gitignore::{Gitignore, GitignoreBuilder, Glob};
+use ignore::Match;
+use std::collections::BTreeMap;
+use std::path::{Path, PathBuf};
+
+/// Ignore file honoured in every directory, ranked below `.xerjignore`.
+pub const GITIGNORE: &str = ".gitignore";
+/// XERJ's own ignore file. Same syntax as `.gitignore`, higher precedence, so
+/// a user can exclude something git tracks (or re-include something git
+/// ignores) without editing `.gitignore`.
+pub const XERJIGNORE: &str = ".xerjignore";
+
+/// Build output that is junk in practice but is *not* reliably gitignored —
+/// `vendor/` is committed by Go projects, `dist/` and `build/` are gitignored
+/// in one repo and checked in the next, and a folder full of downloaded
+/// packages is very often not a git repository at all.
+///
+/// These are defaults, not law: `--no-default-ignores` turns them off, any
+/// ignore file in the tree outranks them, and `--dry-run` names them as the
+/// reason a directory disappeared.
+pub const DEFAULT_IGNORE_PATTERNS: &[&str] = &[
+ "node_modules/",
+ "vendor/",
+ "target/",
+ "dist/",
+ "build/",
+ ".venv/",
+ "__pycache__/",
+];
+
+/// Rule label used for the pre-existing dotfile skip. It is not one of the
+/// ignore rules and `--no-ignore` does not disable it (it is what keeps `.env`
+/// and `.ssh` out of a queryable brain), but a user whose file vanished still
+/// needs it named, so it shares the reporting.
+pub const HIDDEN_RULE: &str = "hidden:dotfile";
+
+const MAX_WARNINGS: usize = 20;
+/// Directory entries the `--dry-run` deep count may examine in total. A
+/// pathological `node_modules` is not allowed to turn "tell me the plan" into
+/// minutes of stat storm on a laptop; past the budget the count is reported as
+/// a floor instead of a total.
+const DEEP_COUNT_BUDGET: u64 = 1_000_000;
+/// How far above the root we look for an ignore file that covers the root
+/// itself. Bounded so a root near `/` cannot walk the whole filesystem.
+const MAX_ANCESTOR_LEVELS: usize = 64;
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub struct IgnoreOptions {
+ /// `--no-ignore` clears this: no `.gitignore`, no `.xerjignore`, no
+ /// defaults. The dotfile skip is separate and stays on.
+ pub enabled: bool,
+ /// `--no-default-ignores` clears this: ignore files still apply,
+ /// [`DEFAULT_IGNORE_PATTERNS`] does not.
+ pub defaults: bool,
+ /// Count the files inside a pruned directory instead of only reporting
+ /// that the directory was pruned. Worth its syscalls in `--dry-run`, whose
+ /// whole job is explaining the plan, and not otherwise — the point of
+ /// pruning is to never touch those paths.
+ pub deep_count: bool,
+}
+
+impl Default for IgnoreOptions {
+ fn default() -> Self {
+ Self {
+ enabled: true,
+ defaults: true,
+ deep_count: false,
+ }
+ }
+}
+
+impl IgnoreOptions {
+ /// `--no-ignore`.
+ pub const fn off() -> Self {
+ Self {
+ enabled: false,
+ defaults: false,
+ deep_count: false,
+ }
+ }
+
+ /// `--dry-run` explains itself; every other run stays cheap.
+ pub const fn with_deep_count(mut self, deep_count: bool) -> Self {
+ self.deep_count = deep_count;
+ self
+ }
+}
+
+#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
+pub struct RuleCount {
+ /// Files matched directly by this rule.
+ pub files: u64,
+ /// Directories pruned by this rule (their contents were never walked).
+ pub dirs: u64,
+ /// Files found inside those pruned directories. Only ever non-zero when
+ /// the deep count ran (`--dry-run`).
+ pub files_inside_pruned_dirs: u64,
+}
+
+#[derive(Debug, Clone)]
+pub struct IgnoreReport {
+ /// rule label → what it discarded.
+ pub by_rule: BTreeMap<String, RuleCount>,
+ pub files_skipped: u64,
+ pub dirs_pruned: u64,
+ pub files_inside_pruned_dirs: u64,
+ /// True unless the deep count hit [`DEEP_COUNT_BUDGET`]. When false,
+ /// `files_inside_pruned_dirs` is a floor and every line that prints it
+ /// says "at least".
+ pub deep_count_complete: bool,
+ /// Whether the deep count ran at all. When it did not, no file count
+ /// inside a pruned directory is printed — an unmeasured number is not
+ /// printed as zero.
+ pub deep_count_ran: bool,
+ pub ignore_files_read: u64,
+ /// Set when the folder the user pointed at is itself covered by an ignore
+ /// rule. It is indexed anyway; this is the "and here is why you are seeing
+ /// files you thought were ignored" line.
+ pub root_exemption: Option<String>,
+ /// Unreadable or unparseable ignore files. Never fatal: an ignore file we
+ /// cannot read means we index more, not less.
+ pub warnings: Vec<String>,
+}
+
+impl Default for IgnoreReport {
+ fn default() -> Self {
+ Self {
+ by_rule: BTreeMap::new(),
+ files_skipped: 0,
+ dirs_pruned: 0,
+ files_inside_pruned_dirs: 0,
+ deep_count_complete: true,
+ deep_count_ran: false,
+ ignore_files_read: 0,
+ root_exemption: None,
+ warnings: Vec::new(),
+ }
+ }
+}
+
+impl IgnoreReport {
+ pub fn nothing_to_report(&self) -> bool {
+ self.by_rule.is_empty() && self.root_exemption.is_none() && self.warnings.is_empty()
+ }
+
+ /// Human-readable lines for the progress surface. Every number here was
+ /// counted by this run; the ones that were deliberately not counted are
+ /// named as such instead of being printed as zero.
+ pub fn summary_lines(&self) -> Vec<String> {
+ let mut out = Vec::new();
+ for warning in &self.warnings {
+ out.push(format!("ignore rules: {warning}"));
+ }
+ if let Some(reason) = &self.root_exemption {
+ out.push(format!("ignore rules: {reason}"));
+ }
+ if self.by_rule.is_empty() {
+ return out;
+ }
+ let mut headline = format!(
+ "ignore rules: skipped {} file{} and pruned {} director{}",
+ self.files_skipped,
+ plural(self.files_skipped),
+ self.dirs_pruned,
+ if self.dirs_pruned == 1 { "y" } else { "ies" },
+ );
+ if self.deep_count_ran && self.dirs_pruned > 0 {
+ headline.push_str(&format!(
+ " ({}{} file{} inside them)",
+ if self.deep_count_complete {
+ ""
+ } else {
+ "at least "
+ },
+ self.files_inside_pruned_dirs,
+ plural(self.files_inside_pruned_dirs),
+ ));
+ } else if self.dirs_pruned > 0 {
+ headline.push_str(" (contents never walked, so not counted)");
+ }
+ if self.ignore_files_read > 0 {
+ headline.push_str(&format!(
+ "; {} ignore file{} read",
+ self.ignore_files_read,
+ plural(self.ignore_files_read)
+ ));
+ }
+ out.push(headline);
+
+ let mut ranked: Vec<(&String, &RuleCount)> = self.by_rule.iter().collect();
+ ranked.sort_by(|a, b| {
+ let weight = |c: &RuleCount| c.files + c.files_inside_pruned_dirs + c.dirs;
+ weight(b.1).cmp(&weight(a.1)).then_with(|| a.0.cmp(b.0))
+ });
+ const SHOWN: usize = 10;
+ for (label, count) in ranked.iter().take(SHOWN) {
+ let mut parts = Vec::new();
+ if count.files > 0 {
+ parts.push(format!("{} file{}", count.files, plural(count.files)));
+ }
+ if count.dirs > 0 {
+ let inside = if self.deep_count_ran {
+ format!(
+ " ({}{} file{} inside)",
+ if self.deep_count_complete {
+ ""
+ } else {
+ "at least "
+ },
+ count.files_inside_pruned_dirs,
+ plural(count.files_inside_pruned_dirs)
+ )
+ } else {
+ String::new()
+ };
+ parts.push(format!(
+ "{} director{} pruned{inside}",
+ count.dirs,
+ if count.dirs == 1 { "y" } else { "ies" }
+ ));
+ }
+ out.push(format!("ignore rules: {label} — {}", parts.join(", ")));
+ }
+ if ranked.len() > SHOWN {
+ out.push(format!(
+ "ignore rules: … and {} more rule{}",
+ ranked.len() - SHOWN,
+ plural((ranked.len() - SHOWN) as u64)
+ ));
+ }
+ out
+ }
+}
+
+fn plural(n: u64) -> &'static str {
+ if n == 1 {
+ ""
+ } else {
+ "s"
+ }
+}
+
+#[derive(Default)]
+struct Layer {
+ xerj: Option<Gitignore>,
+ git: Option<Gitignore>,
+ exclude: Option<Gitignore>,
+}
+
+/// A stack of ignore matchers, one entry per directory on the current
+/// root-to-here path, plus the built-in defaults.
+///
+/// The caller drives it: [`IgnoreStack::enter_dir`] when a directory is
+/// yielded, [`IgnoreStack::skip_dir`] / [`IgnoreStack::skip_file`] to ask
+/// whether an entry survives.
+pub struct IgnoreStack {
+ opts: IgnoreOptions,
+ root: PathBuf,
+ layers: Vec<Layer>,
+ defaults: Option<Gitignore>,
+ budget: u64,
+ pub report: IgnoreReport,
+}
+
+impl IgnoreStack {
+ /// `root` must be the canonicalised root folder.
+ pub fn new(root: &Path, opts: IgnoreOptions) -> Self {
+ let mut stack = Self {
+ opts,
+ root: root.to_path_buf(),
+ layers: Vec::new(),
+ defaults: None,
+ budget: DEEP_COUNT_BUDGET,
+ report: IgnoreReport {
+ deep_count_ran: opts.deep_count,
+ ..IgnoreReport::default()
+ },
+ };
+ if opts.enabled && opts.defaults {
+ let defaults = stack.build_defaults(root);
+ stack.defaults = defaults;
+ }
+ let exemption = stack.root_exemption(root);
+ stack.report.root_exemption = exemption;
+ stack
+ }
+
+ fn build_defaults(&mut self, root: &Path) -> Option<Gitignore> {
+ let mut builder = GitignoreBuilder::new(root);
+ for pattern in DEFAULT_IGNORE_PATTERNS {
+ if let Err(err) = builder.add_line(None, pattern) {
+ self.warn(format!("built-in rule `{pattern}` is not usable: {err}"));
+ }
+ }
+ match builder.build() {
+ Ok(gi) if gi.is_empty() => None,
+ Ok(gi) => Some(gi),
+ Err(err) => {
+ self.warn(format!("built-in default rules are unusable: {err}"));
+ None
+ }
+ }
+ }
+
+ /// Detect that the folder the user pointed at is itself ignored.
+ ///
+ /// We never act on it — pointing autoindex at `~/proj/target` is an
+ /// explicit instruction and silently indexing nothing would be the worst
+ /// possible answer. We only explain it, because the alternative is a user
+ /// staring at a brain full of build output wondering why their ignore file
+ /// did nothing.
+ fn root_exemption(&self, root: &Path) -> Option<String> {
+ if !self.opts.enabled {
+ return None;
+ }
+ let name = root.file_name()?;
+ let parent = root.parent()?;
+ // The defaults are anchored at the root itself, so they cannot see the
+ // root's own name. Re-anchor a copy one level up to ask the question.
+ if self.opts.defaults {
+ let mut builder = GitignoreBuilder::new(parent);
+ for pattern in DEFAULT_IGNORE_PATTERNS {
+ let _ = builder.add_line(None, pattern);
+ }
+ if let Ok(gi) = builder.build() {
+ if let Match::Ignore(glob) = gi.matched(Path::new(name), true) {
+ return Some(format!(
+ "{} matches the built-in default rule `{}`, but you pointed autoindex at \
+ it, so it is being indexed — pass --no-default-ignores to silence this \
+ note",
+ root.display(),
+ glob.original()
+ ));
+ }
+ }
+ }
+ // Ignore files *above* the root are consulted for this note only, and
+ // never to prune: git itself stops at the repository root, so we do
+ // too (plus a hard level cap).
+ let mut dir = parent.to_path_buf();
+ for _ in 0..MAX_ANCESTOR_LEVELS {
+ for file in [XERJIGNORE, GITIGNORE] {
+ let path = dir.join(file);
+ if !path.is_file() {
+ continue;
+ }
+ let mut builder = GitignoreBuilder::new(&dir);
+ let _ = builder.add(&path);
+ let Ok(gi) = builder.build() else { continue };
+ if gi.is_empty() {
+ continue;
+ }
+ if let Match::Ignore(glob) = gi.matched_path_or_any_parents(root, true) {
+ return Some(format!(
+ "{} is ignored by `{}` in {}, but you pointed autoindex at it, so it is \
+ being indexed",
+ root.display(),
+ glob.original(),
+ path.display()
+ ));
+ }
+ }
+ if dir.join(".git").exists() {
+ break;
+ }
+ match dir.parent() {
+ Some(up) => dir = up.to_path_buf(),
+ None => break,
+ }
+ }
+ None
+ }
+
+ /// Drop the layers of directories the walk has finished with.
+ ///
+ /// Must be called for **every** entry, not only directories. A depth-first
+ /// walk interleaves files and subdirectories in readdir order, so after it
+ /// descends into `a/` and comes back it can hand us `b.txt` at the shallower
+ /// depth while `a/`'s layer is still on the stack. Without this, `a/`'s
+ /// `.gitignore` would silently judge its parent's files.
+ pub fn truncate_to(&mut self, depth: usize) {
+ self.layers.truncate(depth);
+ }
+
+ /// Called for every directory the walk yields, root first, before any of
+ /// its children. `depth` is walkdir's depth, so the stack is indexed by
+ /// depth and a sibling directory replaces its predecessor's layer.
+ pub fn enter_dir(&mut self, dir: &Path, depth: usize) {
+ self.layers.truncate(depth);
+ let layer = if self.opts.enabled {
+ Layer {
+ xerj: self.load(dir, dir.join(XERJIGNORE)),
+ git: self.load(dir, dir.join(GITIGNORE)),
+ // Per-repository excludes, which is where people put "junk
+ // only I have". Loaded per directory so a nested checkout gets
+ // its own.
+ exclude: self.load(dir, dir.join(".git").join("info").join("exclude")),
+ }
+ } else {
+ Layer::default()
+ };
+ self.layers.push(layer);
+ }
+
+ fn load(&mut self, dir: &Path, file: PathBuf) -> Option<Gitignore> {
+ if !file.is_file() {
+ return None;
+ }
+ let mut builder = GitignoreBuilder::new(dir);
+ if let Some(err) = builder.add(&file) {
+ self.warn(format!("{}: {err}", file.display()));
+ }
+ match builder.build() {
+ Ok(gi) => {
+ self.report.ignore_files_read += 1;
+ if gi.is_empty() {
+ None
+ } else {
+ Some(gi)
+ }
+ }
+ Err(err) => {
+ self.warn(format!("{} is not usable: {err}", file.display()));
+ None
+ }
+ }
+ }
+
+ fn warn(&mut self, message: String) {
+ if self.report.warnings.len() < MAX_WARNINGS {
+ self.report.warnings.push(message);
+ }
+ }
+
+ /// ripgrep's precedence, reproduced: the deepest match *per kind*, then
+ /// the kinds ranked `.xerjignore` > `.gitignore` > `.git/info/exclude` >
+ /// built-in defaults (`ignore-0.4.32/src/dir.rs:563-594` and `:678-684`).
+ fn lookup(&self, path: &Path, is_dir: bool) -> Option<(bool, String)> {
+ if !self.opts.enabled {
+ return None;
+ }
+ let mut xerj = None;
+ let mut git = None;
+ let mut exclude = None;
+ for layer in self.layers.iter().rev() {
+ if xerj.is_none() {
+ xerj = self.decide_with(layer.xerj.as_ref(), path, is_dir);
+ }
+ if git.is_none() {
+ git = self.decide_with(layer.git.as_ref(), path, is_dir);
+ }
+ if exclude.is_none() {
+ exclude = self.decide_with(layer.exclude.as_ref(), path, is_dir);
+ }
+ if xerj.is_some() {
+ // Nothing ranked lower can change the answer.
+ break;
+ }
+ }
+ xerj.or(git)
+ .or(exclude)
+ .or_else(|| self.decide_with(self.defaults.as_ref(), path, is_dir))
+ }
+
+ fn decide_with(
+ &self,
+ matcher: Option<&Gitignore>,
+ path: &Path,
+ is_dir: bool,
+ ) -> Option<(bool, String)> {
+ match matcher?.matched(path, is_dir) {
+ Match::None => None,
+ Match::Ignore(glob) => Some((true, self.label(glob))),
+ Match::Whitelist(glob) => Some((false, self.label(glob))),
+ }
+ }
+
+ fn label(&self, glob: &Glob) -> String {
+ let from = glob
+ .from()
+ .map(|p| self.rel(p))
+ .unwrap_or_else(|| "<built-in>".to_string());
+ format!("{from}:{}", glob.original())
+ }
+
+ fn rel(&self, path: &Path) -> String {
+ path.strip_prefix(&self.root)
+ .unwrap_or(path)
+ .to_string_lossy()
+ .replace('\\', "/")
+ }
+
+ /// Ask whether a directory must be pruned. `Some(label)` means do not
+ /// descend; the report has already been updated.
+ pub fn skip_dir(&mut self, path: &Path) -> Option<String> {
+ let (ignored, label) = self.lookup(path, true)?;
+ if !ignored {
+ return None;
+ }
+ self.record_dir(path, &label);
+ Some(label)
+ }
+
+ /// Ask whether a file must be skipped.
+ pub fn skip_file(&mut self, path: &Path) -> Option<String> {
+ let (ignored, label) = self.lookup(path, false)?;
+ if !ignored {
+ return None;
+ }
+ self.record_file(&label);
+ Some(label)
+ }
+
+ /// The dotfile skip is decided by the walker, not here, but it is reported
+ /// through the same accounting so `--dry-run` explains every missing file
+ /// with one mechanism.
+ pub fn record_hidden(&mut self, path: &Path, is_dir: bool) {
+ if is_dir {
+ self.record_dir(path, HIDDEN_RULE);
+ } else {
+ self.record_file(HIDDEN_RULE);
+ }
+ }
+
+ fn record_file(&mut self, label: &str) {
+ self.report.files_skipped += 1;
+ self.report
+ .by_rule
+ .entry(label.to_string())
+ .or_default()
+ .files += 1;
+ }
+
+ fn record_dir(&mut self, path: &Path, label: &str) {
+ self.report.dirs_pruned += 1;
+ self.report
+ .by_rule
+ .entry(label.to_string())
+ .or_default()
+ .dirs += 1;
+ let inside = self.count_inside(path);
+ self.report.files_inside_pruned_dirs += inside;
+ self.report
+ .by_rule
+ .entry(label.to_string())
+ .or_default()
+ .files_inside_pruned_dirs += inside;
+ }
+
+ /// Only under `--dry-run`. Symlinks are never followed here whatever the
+ /// run's `--follow-symlinks` setting: this is an accounting pass over a
+ /// directory we have already decided not to index, and it must not be able
+ /// to wander outside it or loop.
+ ///
+ /// The dotfile rule is applied here too, so the number means "files that
+ /// would otherwise have been indexed" and not "inodes under this path".
+ /// Without it a pruned directory containing a nested checkout reports its
+ /// whole `.git`, which was never indexable in the first place — a number
+ /// that does not reconcile with the before/after file count is worse than
+ /// no number.
+ fn count_inside(&mut self, dir: &Path) -> u64 {
+ if !self.opts.deep_count {
+ return 0;
+ }
+ let mut files = 0;
+ let walker = walkdir::WalkDir::new(dir)
+ .follow_links(false)
+ .min_depth(1)
+ .into_iter()
+ .filter_entry(|e| {
+ !e.file_name()
+ .to_str()
+ .is_some_and(|name| name.starts_with('.'))
+ });
+ for entry in walker {
+ if self.budget == 0 {
+ self.report.deep_count_complete = false;
+ break;
+ }
+ self.budget -= 1;
+ if entry.is_ok_and(|e| e.file_type().is_file()) {
+ files += 1;
+ }
+ }
+ files
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use std::fs;
+
+ /// Drive the stack the way `crate::walk` does, over a sorted listing, and
+ /// return the surviving root-relative file paths.
+ fn survivors(root: &Path, opts: IgnoreOptions) -> (Vec<String>, IgnoreReport) {
+ let root = root.canonicalize().unwrap();
+ let mut stack = IgnoreStack::new(&root, opts);
+ let mut kept = Vec::new();
+ let mut it = walkdir::WalkDir::new(&root).into_iter();
+ while let Some(next) = it.next() {
+ let Ok(entry) = next else { continue };
+ let is_dir = entry.file_type().is_dir();
+ if entry.depth() > 0
+ && entry
+ .file_name()
+ .to_str()
+ .is_some_and(|n| n.starts_with('.'))
+ {
+ stack.record_hidden(entry.path(), is_dir);
+ if is_dir {
+ it.skip_current_dir();
+ }
+ continue;
+ }
+ stack.truncate_to(entry.depth());
+ if is_dir {
+ if entry.depth() > 0 && stack.skip_dir(entry.path()).is_some() {
+ it.skip_current_dir();
+ continue;
+ }
+ stack.enter_dir(entry.path(), entry.depth());
+ continue;
+ }
+ if !entry.file_type().is_file() {
+ continue;
+ }
+ if stack.skip_file(entry.path()).is_some() {
+ continue;
+ }
+ kept.push(
+ entry
+ .path()
+ .strip_prefix(&root)
+ .unwrap()
+ .to_string_lossy()
+ .replace('\\', "/"),
+ );
+ }
+ kept.sort();
+ (kept, stack.report)
+ }
+
+ fn write(root: &Path, rel: &str, body: &str) {
+ let path = root.join(rel);
+ fs::create_dir_all(path.parent().unwrap()).unwrap();
+ fs::write(path, body).unwrap();
+ }
+
+ #[test]
+ fn root_gitignore_prunes_and_negation_re_includes() {
+ let dir = tempfile::TempDir::new().unwrap();
+ let root = dir.path();
+ write(root, ".gitignore", "*.log\n!keep.log\nbuild/\n");
+ write(root, "a.log", "x");
+ write(root, "keep.log", "x");
+ write(root, "src/main.rs", "x");
+ write(root, "src/nested.log", "x");
+ write(root, "build/out.bin", "x");
+
+ let (kept, report) = survivors(root, IgnoreOptions::default());
+ assert_eq!(kept, vec!["keep.log", "src/main.rs"], "{kept:?}");
+ // `*.log` has no slash, so it matches at any depth — git's rule.
+ assert!(
+ report.by_rule.contains_key(".gitignore:*.log"),
+ "{report:?}"
+ );
+ assert!(
+ report.by_rule.contains_key(".gitignore:build/"),
+ "{report:?}"
+ );
+ assert_eq!(report.dirs_pruned, 1);
+ assert_eq!(
+ report.by_rule[".gitignore:*.log"].files, 2,
+ "a.log and src/nested.log"
+ );
+ // 3, not 2: `.gitignore` is itself a dotfile, so the pre-existing
+ // hidden-file rule accounts for it in the same report.
+ assert_eq!(report.files_skipped, 3, "{report:?}");
+ assert_eq!(report.by_rule[HIDDEN_RULE].files, 1, "{report:?}");
+ }
+
+ #[test]
+ fn a_nested_gitignore_outranks_the_root_one() {
+ let dir = tempfile::TempDir::new().unwrap();
+ let root = dir.path();
+ write(root, ".gitignore", "*.log\n");
+ write(root, "logs/.gitignore", "!*.log\n");
+ write(root, "top.log", "x");
+ write(root, "logs/kept.log", "x");
+
+ let (kept, _) = survivors(root, IgnoreOptions::default());
+ assert_eq!(kept, vec!["logs/kept.log"], "{kept:?}");
+ }
+
+ #[test]
+ fn xerjignore_outranks_gitignore_at_any_depth() {
+ let dir = tempfile::TempDir::new().unwrap();
+ let root = dir.path();
+ // First line: git tracks it, the user does not want it in the index.
+ // Second: git ignores it, the user *does* want it — and a root
+ // .xerjignore outranks a deep .gitignore, which is ripgrep's rule.
+ write(
+ root,
+ ".xerjignore",
+ "tracked-but-huge.csv\n!deep/wanted.bin\n",
+ );
+ write(root, "deep/.gitignore", "*.bin\n");
+ write(root, "tracked-but-huge.csv", "x");
+ write(root, "deep/wanted.bin", "x");
+ write(root, "deep/other.bin", "x");
+ write(root, "ok.txt", "x");
+
+ let (kept, report) = survivors(root, IgnoreOptions::default());
+ assert_eq!(kept, vec!["deep/wanted.bin", "ok.txt"], "{kept:?}");
+ assert!(
+ report
+ .by_rule
+ .contains_key(".xerjignore:tracked-but-huge.csv"),
+ "{report:?}"
+ );
+ }
+
+ #[test]
+ fn defaults_prune_build_output_that_no_ignore_file_mentions() {
+ let dir = tempfile::TempDir::new().unwrap();
+ let root = dir.path();
+ write(root, "index.js", "x");
+ write(root, "node_modules/left-pad/index.js", "x");
+ write(root, "target/debug/huge.rlib", "x");
+ write(root, "__pycache__/mod.cpython-311.pyc", "x");
+
+ let (kept, report) = survivors(root, IgnoreOptions::default());
+ assert_eq!(kept, vec!["index.js"], "{kept:?}");
+ assert_eq!(report.dirs_pruned, 3);
+ assert!(
+ report.by_rule.contains_key("<built-in>:node_modules/"),
+ "{report:?}"
+ );
+
+ let (all, _) = survivors(
+ root,
+ IgnoreOptions {
+ defaults: false,
+ ..IgnoreOptions::default()
+ },
+ );
+ assert_eq!(all.len(), 4, "--no-default-ignores must keep them: {all:?}");
+ }
+
+ #[test]
+ fn an_ignore_file_outranks_a_default() {
+ let dir = tempfile::TempDir::new().unwrap();
+ let root = dir.path();
+ // A Go project that commits its vendor tree.
+ write(root, ".gitignore", "!vendor/\n");
+ write(root, "vendor/lib/lib.go", "x");
+ let (kept, _) = survivors(root, IgnoreOptions::default());
+ assert_eq!(kept, vec!["vendor/lib/lib.go"], "{kept:?}");
+ }
+
+ #[test]
+ fn no_ignore_disables_every_rule_but_not_the_dotfile_skip() {
+ let dir = tempfile::TempDir::new().unwrap();
+ let root = dir.path();
+ write(root, ".gitignore", "*.log\n");
+ write(root, "a.log", "x");
+ write(root, "node_modules/x.js", "x");
+ write(root, ".env", "SECRET=1");
+
+ let (kept, report) = survivors(root, IgnoreOptions::off());
+ assert_eq!(kept, vec!["a.log", "node_modules/x.js"], "{kept:?}");
+ assert!(!kept.iter().any(|k| k == ".env"), "{kept:?}");
+ assert_eq!(report.by_rule.get(HIDDEN_RULE).unwrap().files, 2);
+ }
+
+ #[test]
+ fn git_info_exclude_is_honoured() {
+ let dir = tempfile::TempDir::new().unwrap();
+ let root = dir.path();
+ write(root, ".git/info/exclude", "scratch.txt\n");
+ write(root, "scratch.txt", "x");
+ write(root, "real.txt", "x");
+ let (kept, _) = survivors(root, IgnoreOptions::default());
+ assert_eq!(kept, vec!["real.txt"], "{kept:?}");
+ }
+
+ #[test]
+ fn an_ignored_root_is_indexed_anyway_and_says_why() {
+ let dir = tempfile::TempDir::new().unwrap();
+ let repo = dir.path().join("repo");
+ fs::create_dir_all(repo.join(".git")).unwrap();
+ write(&repo, ".gitignore", "generated/\n");
+ write(&repo, "generated/out.o", "x");
+
+ let root = repo.join("generated");
+ let (kept, report) = survivors(&root, IgnoreOptions::default());
+ assert_eq!(kept, vec!["out.o"], "an explicit root must be indexed");
+ let reason = report.root_exemption.expect("must explain itself");
+ assert!(reason.contains("generated/"), "{reason}");
+ assert!(reason.contains(".gitignore"), "{reason}");
+ }
+
+ #[test]
+ fn a_root_named_like_a_default_is_indexed_anyway_and_says_why() {
+ let dir = tempfile::TempDir::new().unwrap();
+ let root = dir.path().join("node_modules");
+ fs::create_dir_all(&root).unwrap();
+ write(&root, "left-pad/index.js", "x");
+ let (kept, report) = survivors(&root, IgnoreOptions::default());
+ assert_eq!(kept, vec!["left-pad/index.js"], "{kept:?}");
+ let reason = report.root_exemption.expect("must explain itself");
+ assert!(reason.contains("node_modules/"), "{reason}");
+ }
+
+ #[test]
+ fn deep_count_is_off_by_default_and_counts_under_dry_run() {
+ let dir = tempfile::TempDir::new().unwrap();
+ let root = dir.path();
+ write(root, "keep.txt", "x");
+ for i in 0..7 {
+ write(root, &format!("node_modules/pkg/f{i}.js"), "x");
+ }
+
+ let (_, cheap) = survivors(root, IgnoreOptions::default());
+ assert_eq!(cheap.files_inside_pruned_dirs, 0);
+ assert!(!cheap.deep_count_ran);
+ assert!(
+ cheap
+ .summary_lines()
+ .iter()
+ .any(|l| l.contains("contents never walked")),
+ "{:?}",
+ cheap.summary_lines()
+ );
+
+ let (_, deep) = survivors(root, IgnoreOptions::default().with_deep_count(true));
+ assert_eq!(deep.files_inside_pruned_dirs, 7);
+ assert!(deep.deep_count_complete);
+ assert!(
+ deep.summary_lines().iter().any(|l| l.contains("7 files")),
+ "{:?}",
+ deep.summary_lines()
+ );
+ }
+
+ /// Regression, order-independent: a depth-first walk interleaves files and
+ /// subdirectories, so a nested `.gitignore` must be off the stack again by
+ /// the time the walk hands back a file from the parent directory. Driven
+ /// through the exact call sequence `crate::walk` makes, because readdir
+ /// order is not something a test can pin.
+ #[test]
+ fn a_nested_ignore_file_never_judges_its_parents_files() {
+ let dir = tempfile::TempDir::new().unwrap();
+ let root = dir.path().canonicalize().unwrap();
+ write(&root, "a/.gitignore", "*.txt\n");
+ write(&root, "a/inside.txt", "x");
+ write(&root, "b.txt", "x");
+
+ let mut stack = IgnoreStack::new(&root, IgnoreOptions::default());
+ stack.truncate_to(0);
+ stack.enter_dir(&root, 0);
+ // …descend into a/ …
+ stack.truncate_to(1);
+ stack.enter_dir(&root.join("a"), 1);
+ stack.truncate_to(2);
+ assert!(
+ stack
+ .skip_file(&root.join("a").join("inside.txt"))
+ .is_some(),
+ "a/.gitignore must apply inside a/"
+ );
+ // …and come back out to a sibling file of a/.
+ stack.truncate_to(1);
+ assert!(
+ stack.skip_file(&root.join("b.txt")).is_none(),
+ "a/.gitignore leaked onto the parent's files"
+ );
+ }
+
+ /// A broken ignore file must never be able to make files disappear
+ /// silently: it is reported, and its only possible effect is that we index
+ /// MORE than the user asked for, never less.
+ #[test]
+ fn an_unreadable_ignore_file_warns_and_indexes_more_not_less() {
+ let dir = tempfile::TempDir::new().unwrap();
+ let root = dir.path();
+ fs::write(root.join(".gitignore"), b"\xff\xfe not text\n*.txt\n").unwrap();
+ fs::write(root.join("a.txt"), "x").unwrap();
+ let (kept, report) = survivors(root, IgnoreOptions::default());
+ assert_eq!(
+ kept,
+ vec!["a.txt"],
+ "an ignore file we cannot read must not hide anything"
+ );
+ assert!(!report.warnings.is_empty(), "must warn: {report:?}");
+ assert!(
+ report
+ .summary_lines()
+ .iter()
+ .any(|l| l.contains(".gitignore")),
+ "the warning must reach the user: {:?}",
+ report.summary_lines()
+ );
+ }
+}
diff --git a/engine/crates/xerj-autoindex/src/incremental_reconcile_http_tests.rs b/engine/crates/xerj-autoindex/src/incremental_reconcile_http_tests.rs
index 6db75e62a..ae70c788d 100644
--- a/engine/crates/xerj-autoindex/src/incremental_reconcile_http_tests.rs
+++ b/engine/crates/xerj-autoindex/src/incremental_reconcile_http_tests.rs
@@ -400,6 +400,7 @@ fn cfg(root: &Path, state_dir: &Path, url: &str, semantic: bool) -> IndexCfg {
state_dir: Some(state_dir.to_owned()),
fresh: false,
follow_symlinks: false,
+ ignore: crate::ignore_rules::IgnoreOptions::default(),
max_file_gb: 1,
sample: 50,
no_semantic: !semantic,
diff --git a/engine/crates/xerj-autoindex/src/lib.rs b/engine/crates/xerj-autoindex/src/lib.rs
index 205487df3..5fe5ce843 100644
--- a/engine/crates/xerj-autoindex/src/lib.rs
+++ b/engine/crates/xerj-autoindex/src/lib.rs
@@ -16,6 +16,7 @@ mod generation_catalog;
#[cfg(test)]
mod generation_catalog_http_tests;
pub mod ids;
+pub mod ignore_rules;
pub mod infer;
pub mod pool;
pub mod progress;
@@ -943,6 +944,7 @@ mod phase_a_grouping_tests {
state_dir: None,
fresh: true,
follow_symlinks: false,
+ ignore: crate::ignore_rules::IgnoreOptions::default(),
max_file_gb: 2,
sample: 500,
no_semantic: false,
@@ -2266,7 +2268,8 @@ pub fn run_index_report(cfg: IndexCfg) -> Result<(i32, Option<Value>)> {
// Totals are unknown until the walk returns, so this phase honestly
// reports `pct=unknown` and proves liveness with the clock alone.
pr.phase("walk", 0, 0);
- let discovered_files = walk::walk(&cfg.root, cfg.follow_symlinks)?;
+ let (discovered_files, ignore_report) =
+ walk::walk_reporting(&cfg.root, cfg.follow_symlinks, cfg.ignore)?;
let discovered_bytes: u64 = discovered_files.iter().map(|f| f.size).sum();
pr.note(&format!(
"autoindex: {} files ({} MB) under {}",
@@ -2274,6 +2277,11 @@ pub fn run_index_report(cfg: IndexCfg) -> Result<(i32, Option<Value>)> {
discovered_bytes / (1 << 20),
root_str
));
+ // "Where did my files go?" is answered here, on every run — the rules are
+ // named, not just the totals (#276).
+ for line in ignore_report.summary_lines() {
+ pr.note(&format!("autoindex: {line}"));
+ }
// An empty folder is only "nothing to do" when there is also no durable
// state: with a journal present, zero files is a deletion of the whole
// corpus and has to be reconciled, not shrugged off.
@@ -2320,7 +2328,15 @@ pub fn run_index_report(cfg: IndexCfg) -> Result<(i32, Option<Value>)> {
true,
0,
"dry-run",
- &[("files", inventory.files.len() as u64)],
+ &[
+ ("files", inventory.files.len() as u64),
+ ("ignored_files", ignore_report.files_skipped),
+ ("ignored_dirs", ignore_report.dirs_pruned),
+ (
+ "ignored_files_in_pruned_dirs",
+ ignore_report.files_inside_pruned_dirs,
+ ),
+ ],
);
return Ok((0, None));
}
@@ -2760,7 +2776,20 @@ pub fn run_index_report(cfg: IndexCfg) -> Result<(i32, Option<Value>)> {
if cfg.dry_run {
println!("{}", serde_json::to_string_pretty(&plan)?);
pr.note("(dry run — nothing indexed)");
- pr.finish(true, 0, "dry-run", &[("files", files.len() as u64)]);
+ pr.finish(
+ true,
+ 0,
+ "dry-run",
+ &[
+ ("files", files.len() as u64),
+ ("ignored_files", ignore_report.files_skipped),
+ ("ignored_dirs", ignore_report.dirs_pruned),
+ (
+ "ignored_files_in_pruned_dirs",
+ ignore_report.files_inside_pruned_dirs,
+ ),
+ ],
+ );
return Ok((0, None));
}
diff --git a/engine/crates/xerj-autoindex/src/walk.rs b/engine/crates/xerj-autoindex/src/walk.rs
index 27278f2c5..c230cac04 100644
--- a/engine/crates/xerj-autoindex/src/walk.rs
+++ b/engine/crates/xerj-autoindex/src/walk.rs
@@ -1,7 +1,13 @@
//! Inventory: recursive walk of the target folder.
//! Symlinks are NOT followed by default; with --follow-symlinks walkdir's
//! ancestor-loop detection keeps the traversal loop-safe.
+//!
+//! Ignore rules (`.gitignore`, `.xerjignore`, built-in build-output defaults)
+//! are applied *during* the walk, not after it: an ignored directory is never
+//! descended, so its contents cost nothing — no stat, no hash, no bulk body.
+//! The matching itself lives in [`crate::ignore_rules`].
+use crate::ignore_rules::{IgnoreOptions, IgnoreReport, IgnoreStack};
use anyhow::{Context, Result};
use std::path::{Path, PathBuf};
@@ -17,7 +23,22 @@ pub struct FileEntry {
pub size: u64,
}
+/// Walk with the default ignore rules on and the report discarded.
pub fn walk(root: &Path, follow_symlinks: bool) -> Result<Vec<FileEntry>> {
+ walk_reporting(root, follow_symlinks, IgnoreOptions::default()).map(|(files, _)| files)
+}
+
+/// Walk, and say what was left out and why.
+///
+/// The returned [`IgnoreReport`] is the answer to "my files vanished" — it
+/// names every rule that discarded something and how much. It is reported on
+/// every run and, under `--dry-run`, also counts what was inside each pruned
+/// directory.
+pub fn walk_reporting(
+ root: &Path,
+ follow_symlinks: bool,
+ ignore: IgnoreOptions,
+) -> Result<(Vec<FileEntry>, IgnoreReport)> {
let root_canon = root
.canonicalize()
.with_context(|| format!("resolve root folder {}", root.display()))?;
@@ -25,30 +46,66 @@ pub fn walk(root: &Path, follow_symlinks: bool) -> Result<Vec<FileEntry>> {
anyhow::bail!("{} is not a directory", root_canon.display());
}
let mut out = Vec::new();
- // SECURITY / hygiene: never index hidden files or descend into hidden
- // directories. Without this the walker happily indexed `.env` (secrets,
- // API tokens), `.git`, `.ssh`, `.aws`, and other dotfiles into a queryable
- // brain with no per-brain authorization — a real exposure for the
- // "point it at my project folder" use case. `filter_entry` prunes a hidden
- // directory before descending, so `.git/` is skipped whole. The root itself
- // (depth 0) is exempt so a brain over a dot-named folder still works.
- let walker = walkdir::WalkDir::new(&root_canon)
+ let mut stack = IgnoreStack::new(&root_canon, ignore);
+ // Manual iteration rather than `filter_entry`, because the ignore stack has
+ // to load a directory's ignore files *after* that directory is admitted and
+ // *before* its children are judged. `skip_current_dir` on a just-yielded
+ // directory prunes its contents — walkdir's own documented pattern, and the
+ // same effect `filter_entry` had.
+ let mut it = walkdir::WalkDir::new(&root_canon)
.follow_links(follow_symlinks)
- .into_iter()
- .filter_entry(|e| {
- e.depth() == 0 || !e.file_name().to_str().is_some_and(|n| n.starts_with('.'))
- });
- for entry in walker {
- let entry = match entry {
+ .into_iter();
+ while let Some(next) = it.next() {
+ let entry = match next {
Ok(e) => e,
Err(e) => {
eprintln!("walk: skipping unreadable entry: {e}");
continue;
}
};
+ let is_dir = entry.file_type().is_dir();
+ // Retire the layers of directories this entry is no longer inside; see
+ // `IgnoreStack::truncate_to` for why every entry needs it, not just
+ // directories.
+ stack.truncate_to(entry.depth());
+ // SECURITY / hygiene: never index hidden files or descend into hidden
+ // directories. Without this the walker happily indexed `.env` (secrets,
+ // API tokens), `.git`, `.ssh`, `.aws`, and other dotfiles into a
+ // queryable brain with no per-brain authorization — a real exposure for
+ // the "point it at my project folder" use case. A hidden directory is
+ // pruned before descending, so `.git/` is skipped whole. The root
+ // itself (depth 0) is exempt so a brain over a dot-named folder still
+ // works. `--no-ignore` does NOT turn this off: it is not one of the
+ // ignore rules, it is the reason secrets stay out.
+ if entry.depth() > 0
+ && entry
+ .file_name()
+ .to_str()
+ .is_some_and(|n| n.starts_with('.'))
+ {
+ stack.record_hidden(entry.path(), is_dir);
+ if is_dir {
+ it.skip_current_dir();
+ }
+ continue;
+ }
+ if is_dir {
+ // The root is never judged by the rules below it: pointing
+ // autoindex at a folder is an explicit instruction, and the note
+ // explaining that it *would* have been ignored is on the report.
+ if entry.depth() > 0 && stack.skip_dir(entry.path()).is_some() {
+ it.skip_current_dir();
+ continue;
+ }
+ stack.enter_dir(entry.path(), entry.depth());
+ continue;
+ }
if !entry.file_type().is_file() {
continue;
}
+ if stack.skip_file(entry.path()).is_some() {
+ continue;
+ }
let md = match entry.metadata() {
Ok(m) => m,
Err(_) => continue,
@@ -73,7 +130,7 @@ pub fn walk(root: &Path, follow_symlinks: bool) -> Result<Vec<FileEntry>> {
}
// Deterministic order for clustering / naming.
out.sort_by(|a, b| a.rel.cmp(&b.rel));
- Ok(out)
+ Ok((out, stack.report))
}
#[cfg(unix)]
@@ -139,6 +196,49 @@ mod hidden_skip_tests {
);
}
+ /// #276: the walk itself must drop gitignored junk, not a later stage —
+ /// an ignored directory is never descended, so it never reaches hashing.
+ #[test]
+ fn the_walk_honours_gitignore_and_the_built_in_defaults() {
+ use super::walk_reporting;
+ use crate::ignore_rules::IgnoreOptions;
+
+ let dir = tempfile::TempDir::new().unwrap();
+ let root = dir.path();
+ fs::write(root.join(".gitignore"), "*.tmp\ndocs/private/\n").unwrap();
+ fs::write(root.join("README.md"), "hello").unwrap();
+ fs::write(root.join("scratch.tmp"), "junk").unwrap();
+ fs::create_dir_all(root.join("docs/private")).unwrap();
+ fs::write(root.join("docs/private/salary.md"), "secret").unwrap();
+ fs::create_dir_all(root.join("target/debug")).unwrap();
+ fs::write(root.join("target/debug/huge.rlib"), "bytes").unwrap();
+
+ let (files, report) = walk_reporting(root, false, IgnoreOptions::default()).unwrap();
+ let rels: Vec<String> = files.into_iter().map(|e| e.rel).collect();
+ assert_eq!(rels, vec!["README.md".to_string()], "{rels:?}");
+ // 2 = scratch.tmp, plus `.gitignore` itself under the (separate,
+ // non-optional) hidden-file rule.
+ assert_eq!(report.files_skipped, 2, "{report:?}");
+ assert_eq!(report.dirs_pruned, 2, "{report:?}");
+ assert_eq!(report.ignore_files_read, 1, "{report:?}");
+
+ // --no-ignore brings all of it back, except the dotfiles.
+ let (all, off) = walk_reporting(root, false, IgnoreOptions::off()).unwrap();
+ let mut rels: Vec<String> = all.into_iter().map(|e| e.rel).collect();
+ rels.sort();
+ assert_eq!(
+ rels,
+ vec![
+ "README.md".to_string(),
+ "docs/private/salary.md".to_string(),
+ "scratch.tmp".to_string(),
+ "target/debug/huge.rlib".to_string(),
+ ],
+ "{rels:?}"
+ );
+ assert_eq!(off.dirs_pruned, 0, "{off:?}");
+ }
+
#[test]
fn a_brain_over_a_dot_named_root_still_works() {
let dir = tempfile::TempDir::new().unwrap();
diff --git a/engine/crates/xerj-server/src/brain.rs b/engine/crates/xerj-server/src/brain.rs
index 1d4d5a70b..cddd4c61e 100644
--- a/engine/crates/xerj-server/src/brain.rs
+++ b/engine/crates/xerj-server/src/brain.rs
@@ -515,6 +515,10 @@ fn index_cfg(cfg: &BrainCfg, brain: &str, api_key: Option<String>) -> IndexCfg {
state_dir: None,
fresh: cfg.fresh,
follow_symlinks: false,
+ // `xerj brain` composes autoindex, so it inherits its ignore rules
+ // rather than inventing its own: build output does not belong in a
+ // second brain either (#276).
+ ignore: xerj_autoindex::ignore_rules::IgnoreOptions::default(),
max_file_gb: 2,
sample: 500,
no_semantic: false,
diff --git a/landing/docs/recipes/zero-config-autoindex.html b/landing/docs/recipes/zero-config-autoindex.html
index 44de3d05d..21c8b259e 100644
--- a/landing/docs/recipes/zero-config-autoindex.html
+++ b/landing/docs/recipes/zero-config-autoindex.html
@@ -267,6 +267,7 @@ xerj autoindex map
curl localhost:9200/ax-*/_search -H 'Content-Type: application/json' \
-d '{"query":{"match_all":{}},"size":3}'</pre>
<p>Useful knobs (all optional): <code class="inline">--url</code> for a non-default endpoint, <code class="inline">--workers N</code> (default: every core, reduced when the memory safe zone cannot pay for that many in-flight bulk buffers; bounds both the scan and the indexing phase), <code class="inline">--prefix</code> to namespace the indices (default <code class="inline">ax</code>), <code class="inline">--no-semantic</code> for pure BM25+keyword without vector fields, <code class="inline">--dry-run</code> to print the inferred plan without indexing anything, <code class="inline">--follow-symlinks</code> (loop-safe), <code class="inline">--sample N</code> records per file for inference (default 500).</p>
+ <p>Build output never reaches the index in the first place. The walk honours <code class="inline">.xerjignore</code>, <code class="inline">.gitignore</code> (nested files and <code class="inline">!negation</code> included) and <code class="inline">.git/info/exclude</code>, plus a built-in list — <code class="inline">node_modules/ vendor/ target/ dist/ build/ .venv/ __pycache__/</code> — in that order of precedence; an excluded directory is never descended, so nothing inside it is stat-ed, hashed or sent. Hidden files (<code class="inline">.env</code>, <code class="inline">.git/</code>) are skipped by a separate, non-optional rule. Every run prints what was dropped and by which rule (<code class="inline">ignore rules: &hellip;</code> on stderr), and <code class="inline">--dry-run</code> also counts the files inside each pruned directory, so &ldquo;where did my file go?&rdquo; always has an answer. <code class="inline">--no-ignore</code> turns the rules off; <code class="inline">--no-default-ignores</code> keeps your ignore files and drops only the built-in list. The folder you point at is never excluded: if it is itself ignored &mdash; say you run this on <code class="inline">~/proj/target</code> &mdash; it is indexed anyway and the run says which rule it would have matched.</p>
<p><code class="inline">--workers</code> is a ceiling, not a promise: when the server answers <code class="inline">429 Too Many Requests</code> the run halves its own bulk concurrency on the spot and probes back up one worker at a time only after a clean streak, so a busy server is not handed the same load six times over. The run's summary records where it ended up (<code class="inline">bulk_concurrency_final</code>, <code class="inline">bulk_congestion_events</code>).</p>
<p>Your dataset names, field types, and counts will reflect <em>your</em> folder — that is the point. The behaviors shown here (decimal-comma → <code class="inline">double</code>, date strings → typed <code class="inline">date</code>, keyword inference, junk recording, resume-to-identical-counts) are the invariants.</p>
diff --git a/landing/llms-full.txt b/landing/llms-full.txt
index 7fae28b23..c90f80402 100644
--- a/landing/llms-full.txt
+++ b/landing/llms-full.txt
@@ -137,6 +137,7 @@ destabilize the server.
[--bulk-mb N] [--prefix ax] [--state-dir P] [--fresh]
[--follow-symlinks] [--max-file-gb N] [--sample N]
[--no-semantic] [--dry-run] [--json] [--quiet]
+ [--no-ignore] [--no-default-ignores]
xerj autoindex map [--url U] [--json] [--dataset SLUG]
xerj autoindex status [--url U] [--state-dir P]
@@ -159,6 +160,19 @@ shared autoindex-catalog and the old target require explicit cleanup. A
--no-graph state directory written before the generation format cannot be
adopted in place and must be rebuilt the same way.
+Ignore rules: the walk honours .xerjignore, .gitignore (nested files and
+!negation included) and .git/info/exclude, plus a built-in build-output list
+(node_modules/ vendor/ target/ dist/ build/ .venv/ __pycache__/), in that order
+of precedence. An excluded directory is never descended, so nothing inside it
+is stat-ed, hashed or sent. Hidden files (.env, .git/, .ssh) are skipped by a
+separate rule that no flag turns off. Every run prints `ignore rules: ...` on
+stderr naming each rule and what it dropped; --dry-run also counts the files
+inside each pruned directory. --no-ignore turns the rules off entirely;
+--no-default-ignores keeps your ignore files and drops only the built-in list.
+Your global gitignore (core.excludesFile) is deliberately not read. The folder
+you point at is never excluded: if it is itself ignored, it is indexed anyway
+and the run says which rule it would have matched.
+
Exit codes: 0 complete · 3 completed-with-junk (junk recorded, never fatal) ·
2 usage · 1 endpoint/journal error, a refused corpus removal, or a refused
unsafe corpus-state transition.
diff --git a/landing/llms.txt b/landing/llms.txt
index 2c9f2adc4..4272cdcac 100644
--- a/landing/llms.txt
+++ b/landing/llms.txt
@@ -103,8 +103,9 @@ Two honesty notes the payload states about itself and you should pass on: the es
- **If it is going to be long, ask instead of deciding for them.** Present the real trade-off:
* **Full** — everything: binary/hostile format extraction (PDF, DOCX, SQLite, gzip…), inferred types, embedded body fields, relationship edges.
* **Fast** — `--no-semantic --no-graph`: skips embedding-backed body fields and relationship detection. You still get typed, queryable BM25 + keyword indices over the same files — a better-than-grep experience for a fraction of the time. (Note: XERJ's default embedder is lexical feature-hashing, not neural, so `--no-semantic` costs you less than the word "semantic" suggests unless the node runs `--embed-mode neural`.)
- * **Narrower** — index the subdirectory that matters, or skip `node_modules`/`vendor`/`target`/build outputs. Usually the biggest win available.
+ * **Narrower** — index the subdirectory that matters. Build output is already handled for you (see the ignore-rules bullet below); narrowing beyond that is still usually the biggest win available.
* Also useful when their machine must stay usable: `--workers N` and `--pdf-workers N` (which accepts 1–4 only). They bound the **client-side extractor**, not the server — indexing itself runs in the node process, so do not promise a user these cap the machine's total CPU.
+- **Junk is already excluded — do not re-derive it, and do read the report.** The walk honours `.xerjignore`, `.gitignore` (nested files and `!negation` included), `.git/info/exclude`, and a built-in list — `node_modules/ vendor/ target/ dist/ build/ .venv/ __pycache__/` — highest precedence first. An excluded directory is never descended, so nothing inside it is stat-ed, hashed or sent. Hidden files (`.env`, `.git/`, `.ssh`) are skipped separately and *stay* skipped: that rule is what keeps secrets out of a queryable brain, and no flag turns it off. Two things follow for you: (a) the job-size line already reflects the exclusions, so estimate from it and do not subtract build output a second time; (b) when your user says "my file is missing", the answer is on stderr — every run prints `ignore rules: …` lines naming each rule and what it dropped, and `--dry-run` additionally counts the files inside each pruned directory. Relay those lines rather than guessing. `--no-ignore` indexes everything the rules would have dropped; `--no-default-ignores` keeps the ignore files and drops only the built-in list. Your *global* gitignore is deliberately not read.
- **Read the progress stream — it is designed for you.** `autoindex` writes its RESULT to stdout and its LIVENESS to stderr, so `--json` stdout stays one clean parseable document while stderr narrates. When stderr is not a terminal — which is you — a tick writes **two lines, in one write**: a display line to hand to your user, and the machine line to parse.
```