Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

feat(autoindex): add --walk-only so a large tree can be mapped without hashing (#2006)

xerj-orge1348b4d751b
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 0078c9c56..e6bee5674 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -24,6 +24,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
token lineage is stored: the only insert of a new API token row refuses
bearer auth before it writes, so a revoked token has no descendants.
+### Added
+
+- `xerj autoindex --walk-only` stops after the walk. It prints each top-level
+ folder's file count and size, what the ignore rules would drop and which
+ rule, and the totals. It does not hash or sniff file contents. `--dry-run`
+ still hashes, sniffs and prints the plan (#1912).
+
### Fixed
- A bare hub question that names a language or a technology
diff --git a/engine/crates/xerj-autoindex/src/cli.rs b/engine/crates/xerj-autoindex/src/cli.rs
index e95c4934e..7ed13c1f8 100644
--- a/engine/crates/xerj-autoindex/src/cli.rs
+++ b/engine/crates/xerj-autoindex/src/cli.rs
@@ -86,7 +86,8 @@ pub struct IndexCfg {
/// existence-only name card, contents never opened.
pub stub_globs: Vec<String>,
/// `.gitignore` / `.xerjignore` / built-in build-output rules (#276).
- /// `deep_count` is set from `--dry-run`.
+ /// `deep_count` is set from `--dry-run` (and from `--walk-only`, which
+ /// implies it).
pub ignore: IgnoreOptions,
/// Whole-file size cap in BYTES for non-streaming families (JSON, HTML,
/// prose, PDF, …). Streaming families are never capped. Set on the CLI by
@@ -105,6 +106,10 @@ pub struct IndexCfg {
/// Disable edge detection entirely (no `.xerj-memory-*-edges` writes).
pub no_graph: bool,
pub dry_run: bool,
+ /// `--walk-only`: print the per-folder walk summary and return before
+ /// hashing. The parser also sets [`Self::dry_run`], so nothing is indexed.
+ /// `--dry-run` without this flag still hashes, sniffs and prints the plan.
+ pub walk_only: bool,
/// Stop and ask before indexing when phase A's measured estimate exceeds
/// this many minutes. `0` disables the gate outright.
pub max_minutes: u64,
@@ -401,6 +406,13 @@ pub fn help_text_with(feedback: bool) -> String {
\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20when the estimate says it does not fit (warned).\n\
\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20See DISK BUDGET.\n\
\x20\x20\x20\x20--dry-run walk+sniff+infer, print the plan, index nothing\n\
+ \x20\x20\x20\x20--walk-only stop after the walk. Prints each top-level folder's\n\
+ \x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20file count and size, what the ignore rules would drop\n\
+ \x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20and which rule, and the totals. Stops before hashing:\n\
+ \x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20no digest, no sniff, nothing indexed. Ignore files are\n\
+ \x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20still read, as on every walk. The node is still\n\
+ \x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20contacted, as with --dry-run. --dry-run itself still\n\
+ \x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20hashes, sniffs and prints the plan\n\
\x20\x20\x20\x20--label <FILE> label each document at ingest through POST /_decide\n\
\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20(#1062): the question-set file's noul/choice questions\n\
\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20are rendered against each record's fields and the vote\n\
@@ -454,7 +466,7 @@ pub fn help_text_with(feedback: bool) -> String {
\x20\x20\x20\x20preference should not silently decide what is in your index.\n\
\x20\x20\x20\x20A directory the rules reject is never descended, so nothing inside it is\n\
\x20\x20\x20\x20stat-ed, hashed or sent. Every run prints what was dropped and by which\n\
- \x20\x20\x20\x20rule; --dry-run additionally counts the non-hidden files inside each\n\
+ \x20\x20\x20\x20rule; --dry-run and --walk-only additionally count the non-hidden files inside each\n\
\x20\x20\x20\x20pruned directory (bounded — past the budget the count is reported as\n\
\x20\x20\x20\x20`at least N`, and xerj-done carries\n\
\x20\x20\x20\x20ignored_files_in_pruned_dirs_exact=false to say so).\n\
@@ -859,6 +871,7 @@ pub fn parse(args: Vec<String>) -> Result<Cmd, String> {
let mut watch = false;
let mut debounce_ms: Option<u64> = None;
let mut dry_run = false;
+ let mut walk_only = false;
let mut max_minutes = DEFAULT_MAX_MINUTES;
let mut max_minutes_explicit = false;
let mut approve: Option<Approval> = None;
@@ -1122,6 +1135,7 @@ pub fn parse(args: Vec<String>) -> Result<Cmd, String> {
approve_explicit = true;
}
"--dry-run" => dry_run = true,
+ "--walk-only" => walk_only = true,
"--json" => json = true,
"--md" => json = false,
"--code-analyzer" => {
@@ -1362,6 +1376,28 @@ pub fn parse(args: Vec<String>) -> Result<Cmd, String> {
no_semantic = true;
no_graph = true;
}
+ // `--walk-only` is a dry run that returns after the walk. Implying
+ // `dry_run` here, before the contradiction checks and the object-watch
+ // router, keeps every "index nothing" guard armed. `--dry-run` alone
+ // leaves `walk_only` false, so that path still hashes and prints the plan.
+ if walk_only && watch {
+ return Err(
+ "--walk-only and --watch contradict each other: a walk-only run stops after the \
+ folder walk and indexes nothing, and a watcher exists to index changes as they \
+ land. Drop one of the two"
+ .into(),
+ );
+ }
+ if walk_only && (max_minutes_explicit || approve_explicit) {
+ return Err(
+ "--max-minutes/--approve/--yes are the indexing decision gate, which --walk-only \
+ never reaches: it stops after the walk and does not sniff or index. Drop them"
+ .into(),
+ );
+ }
+ if walk_only {
+ dry_run = true;
+ }
if approve == Some(Approval::Cancel) && dry_run {
return Err(
"--approve cancel and --dry-run contradict each other: a dry run already indexes \
@@ -1670,6 +1706,11 @@ pub fn parse(args: Vec<String>) -> Result<Cmd, String> {
}
match (sub.as_deref(), folder) {
+ (Some("map"), _) | (Some("status"), _) if walk_only => Err(format!(
+ "--walk-only summarises a folder walk and does not apply to `autoindex {}`: that \
+ subcommand never walks a tree. Drop it",
+ sub.as_deref().unwrap_or_default()
+ )),
(Some("map"), _) | (Some("status"), _)
if max_minutes_explicit
|| approve_explicit
@@ -1761,6 +1802,13 @@ pub fn parse(args: Vec<String>) -> Result<Cmd, String> {
.into(),
);
}
+ if walk_only && label.is_some() {
+ return Err(
+ "--walk-only and --label contradict each other: a walk-only run indexes \
+ nothing, so there is nowhere for the decide labels to go. Drop one of the two"
+ .into(),
+ );
+ }
if dry_run && label.is_some() {
return Err(
"--dry-run and --label contradict each other: a dry run indexes nothing, so \
@@ -1801,8 +1849,9 @@ pub fn parse(args: Vec<String>) -> Result<Cmd, String> {
ignore: IgnoreOptions {
enabled: !no_ignore,
defaults: !no_ignore && !no_default_ignores,
- // Only --dry-run pays for counting what is inside a pruned
- // directory; a real run's whole point is not touching it.
+ // Only --dry-run and --walk-only (which implies --dry-run)
+ // pay for counting what is inside a pruned directory; a
+ // real run's whole point is not touching it.
deep_count: dry_run,
},
max_file_bytes: max_file_bytes.unwrap_or(2u64 << 30),
@@ -1812,6 +1861,7 @@ pub fn parse(args: Vec<String>) -> Result<Cmd, String> {
brain,
no_graph,
dry_run,
+ walk_only,
max_minutes,
approve,
max_index_bytes,
@@ -2669,6 +2719,58 @@ mod tests {
assert!(err(&["data", "--approve"]).contains("--approve"));
}
+ /// `--walk-only` is the folder map that stops before hashing. `--dry-run`
+ /// stays the plan preview: it still hashes. The two together are the
+ /// walk-only run, because that is the stricter "index nothing".
+ #[test]
+ fn walk_only_is_accepted_and_dry_run_still_hashes() {
+ let help = super::help_text();
+ assert!(
+ help.contains("--walk-only"),
+ "the flag must be in `xerj autoindex --help`"
+ );
+ assert!(
+ help.contains("Stops before hashing") && help.contains("no digest, no sniff"),
+ "help must say the walk does not hash or sniff file contents: {help}"
+ );
+ assert!(
+ help.contains("Ignore files are"),
+ "help must not claim ignore files are unread; the walk has always read them"
+ );
+ assert!(
+ help.contains("--dry-run itself still"),
+ "help must keep --dry-run's hash-and-plan behaviour distinct"
+ );
+
+ let plain = index(&["data"]);
+ assert!(!plain.dry_run && !plain.walk_only);
+ assert!(!plain.ignore.deep_count);
+
+ let dry = index(&["data", "--dry-run"]);
+ assert!(dry.dry_run);
+ assert!(!dry.walk_only, "--dry-run must not become walk-only");
+ assert!(dry.ignore.deep_count);
+
+ let walk = index(&["data", "--walk-only"]);
+ assert!(walk.walk_only);
+ assert!(
+ walk.dry_run,
+ "walk-only indexes nothing, so the dry-run guards have to be armed"
+ );
+ assert!(walk.ignore.deep_count);
+
+ let both = index(&["data", "--dry-run", "--walk-only"]);
+ assert!(both.dry_run && both.walk_only);
+
+ assert!(err(&["data", "--watch", "--no-graph", "--walk-only"]).contains("--watch"));
+ assert!(err(&["s3://b/", "--watch", "--walk-only"]).contains("--watch"));
+ assert!(err(&["map", "--walk-only"]).contains("autoindex map"));
+ assert!(err(&["status", "--walk-only"]).contains("autoindex status"));
+ assert!(err(&["data", "--walk-only", "--max-minutes", "1"]).contains("--max-minutes"));
+ assert!(err(&["data", "--walk-only", "--yes"]).contains("--approve"));
+ assert!(err(&["data", "--walk-only", "--label", "q.json"]).contains("--label"));
+ }
+
#[test]
fn cancel_and_dry_run_are_refused_rather_than_silently_merged() {
let err = err(&["data", "--approve", "cancel", "--dry-run"]);
diff --git a/engine/crates/xerj-autoindex/src/detect/e2e.rs b/engine/crates/xerj-autoindex/src/detect/e2e.rs
index 529b8d48c..c9aaf7d9a 100644
--- a/engine/crates/xerj-autoindex/src/detect/e2e.rs
+++ b/engine/crates/xerj-autoindex/src/detect/e2e.rs
@@ -379,6 +379,7 @@ pub(super) fn cfg(root: &Path, state_dir: &Path, url: &str) -> IndexCfg {
max_index_bytes: None,
budget_plan: None,
dry_run: false,
+ walk_only: false,
json: false,
quiet: true,
progress: crate::progress::ProgressMode::None,
diff --git a/engine/crates/xerj-autoindex/src/failure_resume_http_tests.rs b/engine/crates/xerj-autoindex/src/failure_resume_http_tests.rs
index cf9cf7431..e8182aa8b 100644
--- a/engine/crates/xerj-autoindex/src/failure_resume_http_tests.rs
+++ b/engine/crates/xerj-autoindex/src/failure_resume_http_tests.rs
@@ -817,6 +817,7 @@ fn cfg(root: &Path, state_dir: &Path, url: &str) -> IndexCfg {
max_index_bytes: None,
budget_plan: None,
dry_run: false,
+ walk_only: false,
json: false,
quiet: true,
progress: crate::progress::ProgressMode::None,
diff --git a/engine/crates/xerj-autoindex/src/ignore_rules.rs b/engine/crates/xerj-autoindex/src/ignore_rules.rs
index 2a111a354..d009f4a0e 100644
--- a/engine/crates/xerj-autoindex/src/ignore_rules.rs
+++ b/engine/crates/xerj-autoindex/src/ignore_rules.rs
@@ -243,6 +243,13 @@ pub struct RuleCount {
pub struct IgnoreReport {
/// rule label → what it discarded.
pub by_rule: BTreeMap<String, RuleCount>,
+ /// The same discards as [`Self::by_rule`], split by top-level folder.
+ ///
+ /// `"."` is a file that sits directly in the indexed root. A pruned
+ /// directory is charged to itself when it is top-level, and to the
+ /// top-level directory that contains it otherwise. [`crate::walk`] turns
+ /// this into the `--walk-only` per-folder summary; it is not a second walk.
+ pub by_top: BTreeMap<String, BTreeMap<String, RuleCount>>,
pub files_skipped: u64,
pub dirs_pruned: u64,
/// Non-hidden files inside the pruned directories — see
@@ -274,6 +281,7 @@ impl Default for IgnoreReport {
fn default() -> Self {
Self {
by_rule: BTreeMap::new(),
+ by_top: BTreeMap::new(),
files_skipped: 0,
dirs_pruned: 0,
files_inside_pruned_dirs: 0,
@@ -409,6 +417,24 @@ fn plural(n: u64) -> &'static str {
}
}
+/// Top-level folder of `path` relative to the canonical walk root.
+///
+/// A file with no parent under the root — `README.md` — is `"."`. A directory
+/// the walk prunes at the root — `target/` — is that directory's own name, so
+/// the folder summary can say the directory itself was dropped.
+pub(crate) fn folder_key(root: &Path, path: &Path, is_dir: bool) -> String {
+ let rel = path.strip_prefix(root).unwrap_or(path);
+ let mut comps = rel.components();
+ let Some(first) = comps.next() else {
+ return ".".to_string();
+ };
+ if comps.next().is_none() && !is_dir {
+ ".".to_string()
+ } else {
+ first.as_os_str().to_string_lossy().into_owned()
+ }
+}
+
#[derive(Default)]
struct Layer {
xerj: Option<Gitignore>,
@@ -717,7 +743,7 @@ impl IgnoreStack {
if !ignored {
return None;
}
- self.record_file(&label);
+ self.record_file(path, &label);
Some(label)
}
@@ -732,7 +758,7 @@ impl IgnoreStack {
/// [`IgnoreReport::files_inside_pruned_dirs`] with work that no run was
/// ever going to do. On this repository that inflation was 97,731 files
/// against a real exclusion of 273,441 (#279).
- pub fn record_hidden(&mut self, is_dir: bool) {
+ pub fn record_hidden(&mut self, path: &Path, is_dir: bool) {
if is_dir {
self.report.dirs_pruned += 1;
self.report
@@ -740,8 +766,9 @@ impl IgnoreStack {
.entry(HIDDEN_RULE.to_string())
.or_default()
.dirs += 1;
+ self.note_top(path, true, HIDDEN_RULE, |count| count.dirs += 1);
} else {
- self.record_file(HIDDEN_RULE);
+ self.record_file(path, HIDDEN_RULE);
}
}
@@ -781,7 +808,7 @@ impl IgnoreStack {
/// A symlink target outside the indexed root, also decided by the walker
/// (it needs the canonical root, which this stack does not carry) and
/// reported through the same accounting. See [`ESCAPED_ROOT_RULE`].
- pub fn record_symlink_escape(&mut self, is_dir: bool) {
+ pub fn record_symlink_escape(&mut self, path: &Path, is_dir: bool) {
if is_dir {
self.report.dirs_pruned += 1;
self.report
@@ -789,14 +816,15 @@ impl IgnoreStack {
.entry(ESCAPED_ROOT_RULE.to_string())
.or_default()
.dirs += 1;
+ self.note_top(path, true, ESCAPED_ROOT_RULE, |count| count.dirs += 1);
} else {
- self.record_file(ESCAPED_ROOT_RULE);
+ self.record_file(path, ESCAPED_ROOT_RULE);
}
}
/// A followed link that left the root through a dotted component.
/// See [`HIDDEN_OUTSIDE_ROOT_RULE`].
- pub fn record_symlink_hidden_escape(&mut self, is_dir: bool) {
+ pub fn record_symlink_hidden_escape(&mut self, path: &Path, is_dir: bool) {
if is_dir {
self.report.dirs_pruned += 1;
self.report
@@ -804,14 +832,17 @@ impl IgnoreStack {
.entry(HIDDEN_OUTSIDE_ROOT_RULE.to_string())
.or_default()
.dirs += 1;
+ self.note_top(path, true, HIDDEN_OUTSIDE_ROOT_RULE, |count| {
+ count.dirs += 1;
+ });
} else {
- self.record_file(HIDDEN_OUTSIDE_ROOT_RULE);
+ self.record_file(path, HIDDEN_OUTSIDE_ROOT_RULE);
}
}
/// A symlink left unfollowed. See [`NOT_FOLLOWED_RULE`]. The warning is
/// once per walk: the rule count already says how many links.
- pub fn record_symlink_not_followed(&mut self) {
+ pub fn record_symlink_not_followed(&mut self, path: &Path) {
if !self.report.by_rule.contains_key(NOT_FOLLOWED_RULE) {
self.warn(
"symlink: not followed (pass --follow-symlinks to index a target inside \
@@ -819,11 +850,11 @@ impl IgnoreStack {
.into(),
);
}
- self.record_file(NOT_FOLLOWED_RULE);
+ self.record_file(path, NOT_FOLLOWED_RULE);
}
/// A followed entry that could not be resolved. See [`UNRESOLVED_LINK_RULE`].
- pub fn record_symlink_unresolved(&mut self, is_dir: bool) {
+ pub fn record_symlink_unresolved(&mut self, path: &Path, is_dir: bool) {
if is_dir {
self.report.dirs_pruned += 1;
self.report
@@ -831,8 +862,9 @@ impl IgnoreStack {
.entry(UNRESOLVED_LINK_RULE.to_string())
.or_default()
.dirs += 1;
+ self.note_top(path, true, UNRESOLVED_LINK_RULE, |count| count.dirs += 1);
} else {
- self.record_file(UNRESOLVED_LINK_RULE);
+ self.record_file(path, UNRESOLVED_LINK_RULE);
}
}
@@ -847,8 +879,8 @@ impl IgnoreStack {
/// The file-level twin of [`Self::record_marker_dir`]: a built-in rule the
/// walker decided (it needs a SIBLING or PARENT marker, which the matcher
/// stack cannot see), reported through the same accounting.
- pub fn record_marker_file(&mut self, label: &str) {
- self.record_file(label);
+ pub fn record_marker_file(&mut self, path: &Path, label: &str) {
+ self.record_file(path, label);
}
/// Did an ignore file explicitly RE-INCLUDE this path (`!name`)? A
@@ -859,25 +891,53 @@ impl IgnoreStack {
matches!(self.lookup(path, is_dir), Some((false, _)))
}
- fn record_file(&mut self, label: &str) {
+ fn record_file(&mut self, path: &Path, label: &str) {
self.report.files_skipped += 1;
self.report
.by_rule
.entry(label.to_string())
.or_default()
.files += 1;
+ self.note_top(path, false, label, |count| count.files += 1);
}
fn record_dir(&mut self, path: &Path, label: &str) {
let inside = self.count_inside(path);
+ let deep = self.opts.deep_count;
let entry = self.report.by_rule.entry(label.to_string()).or_default();
entry.dirs += 1;
entry.files_inside_pruned_dirs += inside;
- if self.opts.deep_count {
+ if deep {
entry.dirs_deep_counted += 1;
}
self.report.dirs_pruned += 1;
self.report.files_inside_pruned_dirs += inside;
+ self.note_top(path, true, label, |count| {
+ count.dirs += 1;
+ count.files_inside_pruned_dirs += inside;
+ if deep {
+ count.dirs_deep_counted += 1;
+ }
+ });
+ }
+
+ /// Charge one discard to the top-level folder [`folder_key`] names.
+ fn note_top(
+ &mut self,
+ path: &Path,
+ is_dir: bool,
+ label: &str,
+ update: impl FnOnce(&mut RuleCount),
+ ) {
+ let key = folder_key(&self.root, path, is_dir);
+ let count = self
+ .report
+ .by_top
+ .entry(key)
+ .or_default()
+ .entry(label.to_string())
+ .or_default();
+ update(count);
}
/// Only under `--dry-run`. Symlinks are never followed here whatever the
@@ -944,7 +1004,7 @@ mod tests {
let Ok(entry) = next else { continue };
let is_dir = entry.file_type().is_dir();
if entry.depth() > 0 && is_hidden_name(entry.file_name()) {
- stack.record_hidden(is_dir);
+ stack.record_hidden(entry.path(), is_dir);
if is_dir {
it.skip_current_dir();
}
diff --git a/engine/crates/xerj-autoindex/src/incremental_reconcile_http_tests.rs b/engine/crates/xerj-autoindex/src/incremental_reconcile_http_tests.rs
index 3027e6e8e..106eeedd4 100644
--- a/engine/crates/xerj-autoindex/src/incremental_reconcile_http_tests.rs
+++ b/engine/crates/xerj-autoindex/src/incremental_reconcile_http_tests.rs
@@ -973,6 +973,7 @@ fn cfg(root: &Path, state_dir: &Path, url: &str, semantic: bool) -> IndexCfg {
max_index_bytes: None,
budget_plan: None,
dry_run: false,
+ walk_only: false,
json: false,
quiet: true,
progress: crate::progress::ProgressMode::None,
diff --git a/engine/crates/xerj-autoindex/src/label_http_tests.rs b/engine/crates/xerj-autoindex/src/label_http_tests.rs
index 941150495..148ed845e 100644
--- a/engine/crates/xerj-autoindex/src/label_http_tests.rs
+++ b/engine/crates/xerj-autoindex/src/label_http_tests.rs
@@ -387,6 +387,7 @@ fn cfg(root: &Path, state_dir: &Path, url: &str, label: &Path) -> IndexCfg {
max_index_bytes: None,
budget_plan: None,
dry_run: false,
+ walk_only: false,
json: false,
quiet: true,
progress: crate::progress::ProgressMode::None,
diff --git a/engine/crates/xerj-autoindex/src/lib.rs b/engine/crates/xerj-autoindex/src/lib.rs
index 92e96e5b0..eb17e34ad 100644
--- a/engine/crates/xerj-autoindex/src/lib.rs
+++ b/engine/crates/xerj-autoindex/src/lib.rs
@@ -3524,6 +3524,7 @@ mod phase_a_grouping_tests {
max_index_bytes: None,
budget_plan: None,
dry_run: true,
+ walk_only: false,
json: false,
quiet: true,
progress: crate::progress::ProgressMode::None,
@@ -6979,6 +6980,43 @@ fn run_index_report_inner(
for line in ignore_report.summary_lines() {
pr.note(&format!("autoindex: {line}"));
}
+ // `--walk-only` (#1912) is the map of a tree that has not been indexed yet.
+ // The walk above is the same one `--dry-run` runs, and `summary_lines` is
+ // the same ignore report. Hashing is the next step, and it reads every
+ // admitted byte — on a large drive that is the day-long part. Return
+ // before it. `walk_only` implies `dry_run`, so a pending generation or an
+ // object download already returned above without writing.
+ if cfg.walk_only {
+ let summary = walk::summarise_walk_only(&root_str, &discovered_files, &ignore_report);
+ if cfg.json {
+ println!("{}", serde_json::to_string_pretty(&summary)?);
+ } else {
+ print!("{}", summary.render());
+ }
+ pr.note(
+ "(walk-only — stopped before hashing; file contents were not hashed and nothing was indexed)",
+ );
+ pr.finish_with_flags(
+ true,
+ 0,
+ "walk-only",
+ &[
+ ("files", summary.files),
+ ("bytes", summary.bytes),
+ ("ignored_files", ignore_report.files_skipped),
+ ("ignored_dirs", ignore_report.dirs_pruned),
+ (
+ "ignored_files_in_pruned_dirs",
+ ignore_report.files_inside_pruned_dirs,
+ ),
+ ],
+ &[(
+ "ignored_files_in_pruned_dirs_exact",
+ ignore_report.files_inside_pruned_dirs_is_exact(),
+ )],
+ );
+ return Ok((0, None));
+ }
// An empty folder is only "nothing to do" when there is also no durable
// state: with a journal present, zero files is a deletion of the whole
// corpus and has to be reconciled, not shrugged off.
diff --git a/engine/crates/xerj-autoindex/src/walk.rs b/engine/crates/xerj-autoindex/src/walk.rs
index 493233a29..3a3fd9a5f 100644
--- a/engine/crates/xerj-autoindex/src/walk.rs
+++ b/engine/crates/xerj-autoindex/src/walk.rs
@@ -9,6 +9,8 @@
use crate::ignore_rules::{is_hidden_name, IgnoreOptions, IgnoreReport, IgnoreStack};
use anyhow::{Context, Result};
+use serde::Serialize;
+use std::collections::{BTreeMap, BTreeSet};
use std::path::{Path, PathBuf};
#[derive(Debug, Clone)]
@@ -368,7 +370,7 @@ fn walk_impl(
// hard-linked package stores all produce that shape without an attacker.
if entry.depth() > 0 && is_hidden_name(entry.file_name()) {
stack.note_hidden_reinclude(entry.path(), is_dir);
- stack.record_hidden(is_dir);
+ stack.record_hidden(entry.path(), is_dir);
if is_dir {
it.skip_current_dir();
}
@@ -423,28 +425,28 @@ fn walk_impl(
resolved = Some(real);
}
Err(SymlinkVerdict::HiddenOutsideRoot) => {
- stack.record_symlink_hidden_escape(is_dir);
+ stack.record_symlink_hidden_escape(entry.path(), is_dir);
if is_dir {
it.skip_current_dir();
}
continue;
}
Err(SymlinkVerdict::OutsideRoot(_)) => {
- stack.record_symlink_escape(is_dir);
+ stack.record_symlink_escape(entry.path(), is_dir);
if is_dir {
it.skip_current_dir();
}
continue;
}
Err(SymlinkVerdict::HiddenTarget) => {
- stack.record_hidden(is_dir);
+ stack.record_hidden(entry.path(), is_dir);
if is_dir {
it.skip_current_dir();
}
continue;
}
Err(SymlinkVerdict::Unresolvable) => {
- stack.record_symlink_unresolved(is_dir);
+ stack.record_symlink_unresolved(entry.path(), is_dir);
if is_dir {
it.skip_current_dir();
}
@@ -491,7 +493,7 @@ fn walk_impl(
// with no report made a tree of links look empty: neither the
// file list nor the ignore summary mentioned them.
if !follow_symlinks && entry.path_is_symlink() {
- stack.record_symlink_not_followed();
+ stack.record_symlink_not_followed(entry.path());
}
continue;
}
@@ -505,7 +507,7 @@ fn walk_impl(
if marker_defaults {
if let Some(label) = takeout_noise(entry.path()) {
if !stack.is_reincluded(entry.path(), false) {
- stack.record_marker_file(label);
+ stack.record_marker_file(entry.path(), label);
continue;
}
}
@@ -575,6 +577,218 @@ fn stable_path_id(path: &Path) -> String {
format!("other:{}", path.to_string_lossy())
}
+/// One top-level folder in a [`WalkOnlySummary`].
+///
+/// `folder` is `(root)` for files that sit directly in the indexed directory
+/// and `name/` for everything else. `files` and `bytes` count only what the
+/// walk admitted (metadata sizes — the file was not opened). `rules` is the
+/// ignore report's per-folder slice of the same discards [`IgnoreReport::summary_lines`]
+/// describes globally.
+#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
+pub struct WalkOnlyFolder {
+ pub folder: String,
+ pub files: u64,
+ pub bytes: u64,
+ pub rules: Vec<WalkOnlyRule>,
+}
+
+/// One ignore rule's discards inside a top-level folder.
+#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
+pub struct WalkOnlyRule {
+ pub rule: String,
+ pub files: u64,
+ pub dirs: u64,
+ pub files_inside_pruned_dirs: u64,
+}
+
+/// The `--walk-only` result: admitted files grouped by top-level folder, plus
+/// the ignore summary every run already prints.
+///
+/// Built from the walk's [`FileEntry`] list and [`IgnoreReport`]. It does not
+/// open a file. `--dry-run` keeps hashing after this point; `--walk-only` prints
+/// this and returns.
+#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
+pub struct WalkOnlySummary {
+ pub root: String,
+ pub files: u64,
+ pub bytes: u64,
+ pub folders: Vec<WalkOnlyFolder>,
+ pub ignored_files: u64,
+ pub ignored_dirs: u64,
+ pub files_inside_pruned_dirs: u64,
+ pub files_inside_pruned_dirs_exact: bool,
+ /// [`IgnoreReport::summary_lines`], unchanged.
+ pub ignore_summary: Vec<String>,
+}
+
+/// Folder key for an admitted file's portable relative path. Matches
+/// [`crate::ignore_rules::folder_key`]: a root-level file is `"."`, and
+/// `docs/a.txt` is `docs`.
+fn admitted_folder_key(rel: &str) -> &str {
+ match rel.split_once('/') {
+ Some((dir, rest)) if !dir.is_empty() && !rest.is_empty() => dir,
+ _ => ".",
+ }
+}
+
+fn folder_label(key: &str) -> String {
+ if key == "." {
+ "(root)".to_string()
+ } else {
+ format!("{key}/")
+ }
+}
+
+fn plural_s(n: u64) -> &'static str {
+ if n == 1 {
+ ""
+ } else {
+ "s"
+ }
+}
+
+/// Per-folder file counts and sizes, and the ignore-rule summary, from a walk
+/// that has already finished.
+///
+/// `root_display` is the path the run prints (the canonical folder, or an
+/// object-source identity). `files` are the admitted entries — callers that
+/// drop `exclude_rels` pass the filtered list, the same list hashing would
+/// have seen. `report` is that walk's ignore report, including its per-folder
+/// `by_top` breakdown.
+pub fn summarise_walk_only(
+ root_display: &str,
+ files: &[FileEntry],
+ report: &IgnoreReport,
+) -> WalkOnlySummary {
+ let mut by_folder: BTreeMap<String, (u64, u64)> = BTreeMap::new();
+ let mut bytes = 0u64;
+ for file in files {
+ let key = admitted_folder_key(&file.rel).to_string();
+ let row = by_folder.entry(key).or_insert((0, 0));
+ row.0 += 1;
+ row.1 += file.size;
+ bytes += file.size;
+ }
+
+ let mut keys: BTreeSet<String> = BTreeSet::new();
+ keys.extend(by_folder.keys().cloned());
+ keys.extend(report.by_top.keys().cloned());
+
+ let mut folders = Vec::with_capacity(keys.len());
+ for key in keys {
+ let (folder_files, folder_bytes) = by_folder.get(&key).copied().unwrap_or((0, 0));
+ let mut rules = Vec::new();
+ if let Some(by_rule) = report.by_top.get(&key) {
+ for (rule, count) in by_rule {
+ rules.push(WalkOnlyRule {
+ rule: rule.clone(),
+ files: count.files,
+ dirs: count.dirs,
+ files_inside_pruned_dirs: count.files_inside_pruned_dirs,
+ });
+ }
+ rules.sort_by(|a, b| a.rule.cmp(&b.rule));
+ }
+ folders.push(WalkOnlyFolder {
+ folder: folder_label(&key),
+ files: folder_files,
+ bytes: folder_bytes,
+ rules,
+ });
+ }
+
+ WalkOnlySummary {
+ root: root_display.to_string(),
+ files: files.len() as u64,
+ bytes,
+ folders,
+ ignored_files: report.files_skipped,
+ ignored_dirs: report.dirs_pruned,
+ files_inside_pruned_dirs: report.files_inside_pruned_dirs,
+ files_inside_pruned_dirs_exact: report.files_inside_pruned_dirs_is_exact(),
+ ignore_summary: report.summary_lines(),
+ }
+}
+
+impl WalkOnlySummary {
+ /// Human-readable result. Stdout of `xerj autoindex <folder> --walk-only`.
+ pub fn render(&self) -> String {
+ let mut out = format!(
+ "walk-only: {} file{}, {} byte{} under {}\n",
+ self.files,
+ plural_s(self.files),
+ self.bytes,
+ plural_s(self.bytes),
+ self.root
+ );
+ if self.folders.is_empty() {
+ out.push_str(" (no files)\n");
+ }
+ for folder in &self.folders {
+ out.push_str(&format!(
+ " {} {} file{}, {} byte{}\n",
+ folder.folder,
+ folder.files,
+ plural_s(folder.files),
+ folder.bytes,
+ plural_s(folder.bytes)
+ ));
+ for rule in &folder.rules {
+ out.push_str(&format!(" {}\n", self.rule_line(rule)));
+ }
+ }
+ for line in &self.ignore_summary {
+ out.push_str(line);
+ out.push('\n');
+ }
+ let dirs = if self.ignored_dirs == 1 { "y" } else { "ies" };
+ out.push_str(&format!(
+ "total: {} file{}, {} byte{}; ignored {} file{}, {} director{}\n",
+ self.files,
+ plural_s(self.files),
+ self.bytes,
+ plural_s(self.bytes),
+ self.ignored_files,
+ plural_s(self.ignored_files),
+ self.ignored_dirs,
+ dirs
+ ));
+ out
+ }
+
+ fn rule_line(&self, rule: &WalkOnlyRule) -> String {
+ let mut parts = Vec::new();
+ if rule.files > 0 {
+ parts.push(format!("{} file{}", rule.files, plural_s(rule.files)));
+ }
+ if rule.dirs > 0 {
+ let inside = if rule.files_inside_pruned_dirs > 0 {
+ format!(
+ " ({}{} non-hidden file{} inside)",
+ if self.files_inside_pruned_dirs_exact {
+ ""
+ } else {
+ "at least "
+ },
+ rule.files_inside_pruned_dirs,
+ plural_s(rule.files_inside_pruned_dirs)
+ )
+ } else {
+ String::new()
+ };
+ parts.push(format!(
+ "{} director{} pruned{inside}",
+ rule.dirs,
+ if rule.dirs == 1 { "y" } else { "ies" }
+ ));
+ }
+ if parts.is_empty() {
+ parts.push("nothing counted".to_string());
+ }
+ format!("{} — {}", rule.rule, parts.join(", "))
+ }
+}
+
#[cfg(test)]
mod hidden_skip_tests {
use super::{escaped_or_hidden_target, SymlinkVerdict};
@@ -1350,3 +1564,273 @@ mod portable_rel_tests {
assert!(rels.contains(&"src/utils/paths.rs"), "{rels:?}");
}
}
+
+#[cfg(test)]
+mod walk_only_tests {
+ use super::{summarise_walk_only, walk_reporting};
+ use crate::ignore_rules::IgnoreOptions;
+ use std::fs;
+ use std::path::PathBuf;
+
+ fn folder<'a>(summary: &'a super::WalkOnlySummary, name: &str) -> &'a super::WalkOnlyFolder {
+ summary
+ .folders
+ .iter()
+ .find(|f| f.folder == name)
+ .unwrap_or_else(|| panic!("missing folder {name} in {summary:?}"))
+ }
+
+ /// `--walk-only` is this function: the same walk `--dry-run` already runs,
+ /// then a summary, and no content hash. The payloads are watched for
+ /// `open`/`read`. Ignore files are not watched — the walk has always read
+ /// `.gitignore` to apply the rules.
+ #[test]
+ fn walk_only_summarises_without_opening_file_contents() {
+ let dir = tempfile::tempdir().unwrap();
+ let root = dir.path();
+ let readme = b"PAYLOAD-README";
+ let docs_a = b"PAYLOAD-DOCS-A";
+ let docs_b = b"PAYLOAD-DOCS-B";
+ let main_rs = b"PAYLOAD-MAIN-RS";
+ let skipped = b"PAYLOAD-IGNORED-LOG";
+ let pruned = b"PAYLOAD-INSIDE-PRUNED";
+ fs::write(root.join(".gitignore"), "*.log\n").unwrap();
+ fs::write(root.join("README.md"), readme).unwrap();
+ fs::create_dir(root.join("docs")).unwrap();
+ fs::write(root.join("docs/a.txt"), docs_a).unwrap();
+ fs::write(root.join("docs/b.txt"), docs_b).unwrap();
+ fs::write(root.join("docs/skip.log"), skipped).unwrap();
+ fs::create_dir(root.join("src")).unwrap();
+ fs::write(root.join("src/main.rs"), main_rs).unwrap();
+ fs::create_dir(root.join("target")).unwrap();
+ fs::write(root.join("target/huge.bin"), pruned).unwrap();
+
+ let watched = [
+ root.join("README.md"),
+ root.join("docs/a.txt"),
+ root.join("docs/b.txt"),
+ root.join("docs/skip.log"),
+ root.join("src/main.rs"),
+ root.join("target/huge.bin"),
+ ];
+ let summary = {
+ let _guard = OpenWatch::arm(&watched);
+ let (files, report) =
+ walk_reporting(root, false, IgnoreOptions::default().with_deep_count(true))
+ .unwrap();
+ let summary = summarise_walk_only(&root.display().to_string(), &files, &report);
+ let top_files: u64 = report
+ .by_top
+ .values()
+ .flat_map(|rules| rules.values())
+ .map(|c| c.files)
+ .sum();
+ let top_dirs: u64 = report
+ .by_top
+ .values()
+ .flat_map(|rules| rules.values())
+ .map(|c| c.dirs)
+ .sum();
+ assert_eq!(top_files, report.files_skipped, "{report:?}");
+ assert_eq!(top_dirs, report.dirs_pruned, "{report:?}");
+ summary
+ };
+
+ let rendered = summary.render();
+ for marker in [
+ "PAYLOAD-README",
+ "PAYLOAD-DOCS-A",
+ "PAYLOAD-DOCS-B",
+ "PAYLOAD-MAIN-RS",
+ "PAYLOAD-IGNORED-LOG",
+ "PAYLOAD-INSIDE-PRUNED",
+ ] {
+ assert!(
+ !rendered.contains(marker),
+ "walk-only echoed file contents ({marker}): {rendered}"
+ );
+ }
+
+ assert_eq!(summary.files, 4, "{rendered}");
+ assert_eq!(
+ summary.bytes,
+ (readme.len() + docs_a.len() + docs_b.len() + main_rs.len()) as u64,
+ "{rendered}"
+ );
+ let root_row = folder(&summary, "(root)");
+ assert_eq!(root_row.files, 1);
+ assert_eq!(root_row.bytes, readme.len() as u64);
+ assert!(
+ root_row
+ .rules
+ .iter()
+ .any(|r| r.rule == "hidden:dotfile" && r.files == 1),
+ "{root_row:?}"
+ );
+
+ let docs = folder(&summary, "docs/");
+ assert_eq!(docs.files, 2);
+ assert_eq!(docs.bytes, (docs_a.len() + docs_b.len()) as u64);
+ assert!(
+ docs.rules
+ .iter()
+ .any(|r| r.rule == ".gitignore:*.log" && r.files == 1),
+ "{docs:?}"
+ );
+
+ let src = folder(&summary, "src/");
+ assert_eq!(src.files, 1);
+ assert_eq!(src.bytes, main_rs.len() as u64);
+ assert!(src.rules.is_empty(), "{src:?}");
+
+ let target = folder(&summary, "target/");
+ assert_eq!(target.files, 0);
+ assert_eq!(target.bytes, 0);
+ let pruned_rule = target
+ .rules
+ .iter()
+ .find(|r| r.rule == "<built-in>:target/")
+ .unwrap_or_else(|| panic!("target rule missing: {target:?}"));
+ assert_eq!(pruned_rule.dirs, 1);
+ assert_eq!(pruned_rule.files_inside_pruned_dirs, 1);
+ assert!(summary.files_inside_pruned_dirs_exact);
+
+ assert!(rendered.contains("docs/ 2 files,"), "{rendered}");
+ assert!(rendered.contains(".gitignore:*.log — 1 file"), "{rendered}");
+ assert!(
+ rendered.contains("<built-in>:target/ — 1 directory pruned (1 non-hidden file inside)"),
+ "{rendered}"
+ );
+ assert!(
+ summary
+ .ignore_summary
+ .iter()
+ .any(|l| l.contains("ignore rules:")),
+ "the existing ignore summary must be reused: {:?}",
+ summary.ignore_summary
+ );
+ assert!(rendered.contains("total: 4 files,"), "{rendered}");
+ assert_eq!(summary.ignored_files, 2, "{rendered}");
+ assert_eq!(summary.ignored_dirs, 1, "{rendered}");
+ }
+
+ /// Watches payload paths for `open` and `read` across a closure.
+ /// Events queued while the watches were installed are discarded first.
+ struct OpenWatch {
+ fd: i32,
+ wds: Vec<(i32, PathBuf)>,
+ }
+
+ impl OpenWatch {
+ fn arm(paths: &[PathBuf]) -> Self {
+ #[cfg(target_os = "linux")]
+ {
+ Self::arm_linux(paths)
+ }
+ #[cfg(not(target_os = "linux"))]
+ {
+ let _ = paths;
+ Self {
+ fd: -1,
+ wds: Vec::new(),
+ }
+ }
+ }
+
+ #[cfg(target_os = "linux")]
+ fn arm_linux(paths: &[PathBuf]) -> Self {
+ let fd = unsafe { libc::inotify_init1(libc::IN_NONBLOCK | libc::IN_CLOEXEC) };
+ assert!(
+ fd >= 0,
+ "inotify_init1: {}",
+ std::io::Error::last_os_error()
+ );
+ let mut wds = Vec::new();
+ for path in paths {
+ let c = std::ffi::CString::new(path.to_str().expect("utf-8 fixture path")).unwrap();
+ let wd = unsafe {
+ libc::inotify_add_watch(fd, c.as_ptr(), libc::IN_OPEN | libc::IN_ACCESS)
+ };
+ assert!(
+ wd >= 0,
+ "inotify_add_watch {}: {}",
+ path.display(),
+ std::io::Error::last_os_error()
+ );
+ wds.push((wd, path.clone()));
+ }
+ // Installing a watch can itself queue IN_OPEN. Drop that, then
+ // anything still queued after the walk was the walk.
+ let _ = read_events(fd);
+ Self { fd, wds }
+ }
+ }
+
+ impl Drop for OpenWatch {
+ fn drop(&mut self) {
+ #[cfg(target_os = "linux")]
+ {
+ if self.fd < 0 {
+ return;
+ }
+ let events = read_events(self.fd);
+ let names: Vec<String> = events
+ .iter()
+ .map(|(wd, mask)| {
+ let path = self
+ .wds
+ .iter()
+ .find(|(id, _)| id == wd)
+ .map(|(_, p)| p.display().to_string())
+ .unwrap_or_else(|| format!("wd {wd}"));
+ format!("{path} mask={mask:#x}")
+ })
+ .collect();
+ for (wd, _) in &self.wds {
+ unsafe {
+ libc::inotify_rm_watch(self.fd, *wd);
+ }
+ }
+ unsafe {
+ libc::close(self.fd);
+ }
+ assert!(
+ names.is_empty(),
+ "walk-only opened or read file contents: {names:?}"
+ );
+ }
+ #[cfg(not(target_os = "linux"))]
+ {
+ let _ = (self.fd, &self.wds);
+ }
+ }
+ }
+
+ #[cfg(target_os = "linux")]
+ fn read_events(fd: i32) -> Vec<(i32, u32)> {
+ #[repr(align(8))]
+ struct Buf([u8; 4096]);
+ let mut buf = Buf([0u8; 4096]);
+ let mut out = Vec::new();
+ loop {
+ let n = unsafe { libc::read(fd, buf.0.as_mut_ptr().cast(), buf.0.len()) };
+ if n < 0 {
+ let err = std::io::Error::last_os_error();
+ if err.kind() == std::io::ErrorKind::WouldBlock {
+ break;
+ }
+ panic!("inotify read: {err}");
+ }
+ if n == 0 {
+ break;
+ }
+ let mut off = 0usize;
+ while off < n as usize {
+ let ev = unsafe { &*buf.0.as_ptr().add(off).cast::<libc::inotify_event>() };
+ out.push((ev.wd, ev.mask));
+ off += std::mem::size_of::<libc::inotify_event>() + ev.len as usize;
+ }
+ }
+ out
+ }
+}
diff --git a/engine/crates/xerj-server/src/brain.rs b/engine/crates/xerj-server/src/brain.rs
index 27fea7b1f..3a2378957 100644
--- a/engine/crates/xerj-server/src/brain.rs
+++ b/engine/crates/xerj-server/src/brain.rs
@@ -635,6 +635,7 @@ fn index_cfg(cfg: &BrainCfg, brain: &str, api_key: Option<String>) -> IndexCfg {
brain: Some(brain.to_string()),
no_graph: false,
dry_run: false,
+ walk_only: false,
// The estimate/decision gate is an `xerj autoindex` CLI feature, and
// `xerj brain` has no `--approve` of its own to answer it with. Arming
// it here would stop the run with exit 4 and instructions the caller