Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

feat(autoindex): Unity project support — per-object scenes, GUID script links, BVH metadata, stub globs

Sergey Gonchardb4cbb5cb0d0
diff --git a/demo/usecases/autoindex/README.md b/demo/usecases/autoindex/README.md
index 6e78f810f..7b350355c 100644
--- a/demo/usecases/autoindex/README.md
+++ b/demo/usecases/autoindex/README.md
@@ -115,7 +115,12 @@ walk → sniff → sample → infer → map → extract → bulk → correlate
`obj/`, `Logs/`, `UserSettings/` next to
`ProjectSettings/ProjectVersion.txt`, `node_modules/` next to
`package.json`, `target/` next to `Cargo.toml`; `--no-default-excludes`
- walks them), plain text, gzip-compressed variants of the above.
+ walks them; additionally, when the root is inside a git repository the
+ tree's `.gitignore` rules are honored — the owner's own declaration of
+ generated noise — with `--no-gitignore` to opt out, and the user's global
+ gitignore is never consulted), BVH motion capture (one metadata record per clip — skeleton
+ joint names, frame count, duration; the numeric MOTION block is never
+ read), plain text, gzip-compressed variants of the above.
3. **Sample** — bounded per-file sampling (default 500 records,
`--sample N`); inference cost does not grow with file size.
4. **Infer** — field types (long/double/boolean/date/keyword/text), date
diff --git a/engine/Cargo.lock b/engine/Cargo.lock
index 90484f1eb..7d7ba7729 100644
--- a/engine/Cargo.lock
+++ b/engine/Cargo.lock
@@ -529,6 +529,16 @@ dependencies = [
"alloc-stdlib",
]
+[[package]]
+name = "bstr"
+version = "1.13.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6bb31b46c14244e20ee9984b11bf5c992b91fb6939fea616e3512c8baecdbe5f"
+dependencies = [
+ "memchr",
+ "serde_core",
+]
+
[[package]]
name = "bumpalo"
version = "3.20.2"
@@ -1930,6 +1940,19 @@ version = "0.3.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0cc23270f6e1808e30a928bdc84dea0b9b4136a8bc82338574f23baf47bbd280"
+[[package]]
+name = "globset"
+version = "0.4.19"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e47d37d2ae4464254884b60ab7071be2b876a9c35b696bd018ddcc76847309cd"
+dependencies = [
+ "aho-corasick",
+ "bstr",
+ "log",
+ "regex-automata",
+ "regex-syntax",
+]
+
[[package]]
name = "grid"
version = "1.0.1"
@@ -2370,6 +2393,22 @@ dependencies = [
"icu_properties",
]
+[[package]]
+name = "ignore"
+version = "0.4.32"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0b17771570a2b94107741a7b033f19132c2eee21d59d21b24d2ced26500bd66e"
+dependencies = [
+ "crossbeam-deque",
+ "globset",
+ "log",
+ "memchr",
+ "regex-automata",
+ "same-file",
+ "walkdir",
+ "winapi-util",
+]
+
[[package]]
name = "image"
version = "0.25.10"
@@ -6437,6 +6476,7 @@ dependencies = [
"encoding_rs",
"flate2",
"fs2",
+ "ignore",
"libc",
"memchr",
"pdf_oxide",
@@ -6463,7 +6503,6 @@ dependencies = [
"tree-sitter-ruby",
"tree-sitter-rust",
"tree-sitter-typescript",
- "walkdir",
"xxhash-rust",
"zip 2.4.2",
]
diff --git a/engine/crates/xerj-autoindex/Cargo.toml b/engine/crates/xerj-autoindex/Cargo.toml
index 55283c9e9..7f24715e1 100644
--- a/engine/crates/xerj-autoindex/Cargo.toml
+++ b/engine/crates/xerj-autoindex/Cargo.toml
@@ -12,7 +12,9 @@ serde.workspace = true
serde_json.workspace = true
chrono.workspace = true
regex = "1"
-walkdir = "2"
+# Directory walker with .gitignore semantics (ripgrep's). Replaces walkdir:
+# same recursive traversal, plus owner-declared ignore rules.
+ignore = "0.4"
encoding_rs = "0.8"
xxhash-rust.workspace = true
memchr.workspace = true
diff --git a/engine/crates/xerj-autoindex/src/cli.rs b/engine/crates/xerj-autoindex/src/cli.rs
index efe4ac8ef..aed7476c9 100644
--- a/engine/crates/xerj-autoindex/src/cli.rs
+++ b/engine/crates/xerj-autoindex/src/cli.rs
@@ -18,6 +18,12 @@ pub struct IndexCfg {
pub follow_symlinks: bool,
/// Disable the marker-gated generated-dir prune (e.g. Unity `Library/`).
pub no_default_excludes: bool,
+ /// `--stub <glob>` patterns: matching files are indexed as one
+ /// existence-only name card, contents never opened.
+ pub stub_globs: Vec<String>,
+ /// Walk `.gitignore`d files too (by default they are skipped when the
+ /// root is inside a git repository).
+ pub no_gitignore: bool,
pub max_file_gb: u64,
pub sample: usize,
pub no_semantic: bool,
@@ -86,6 +92,17 @@ pub fn print_help() {
UserSettings next to ProjectSettings/\n\
ProjectVersion.txt, node_modules next to\n\
package.json, target next to Cargo.toml\n\
+ --no-gitignore index .gitignore'd files too. By default, when the\n\
+ root is inside a git repository, files matched by\n\
+ the tree's .gitignore rules are skipped (the\n\
+ owner's own declaration of generated noise);\n\
+ the user's global gitignore is never consulted\n\
+ --stub <GLOB> index matching files as ONE existence-only name\n\
+ card (title + provenance); contents are never\n\
+ opened. Repeatable. A pattern without '/'\n\
+ matches file names anywhere ('*.bvh'); with '/'\n\
+ it matches the root-relative path\n\
+ ('unity/**/*.csv'). '**' crosses directories\n\
--max-file-gb <N> skip+record oversized non-streamable files (default 2)\n\
--sample <N> records sampled per file for inference (default 500)\n\
--no-semantic skip semantic_text on body fields (pure BM25+keyword)\n\
@@ -148,6 +165,8 @@ pub fn parse(args: Vec<String>) -> Result<Cmd, String> {
let mut fresh = false;
let mut follow_symlinks = false;
let mut no_default_excludes = false;
+ let mut stub_globs: Vec<String> = Vec::new();
+ let mut no_gitignore = false;
let mut max_file_gb = 2u64;
let mut sample = 500usize;
let mut no_semantic = false;
@@ -207,6 +226,8 @@ pub fn parse(args: Vec<String>) -> Result<Cmd, String> {
"--fresh" => fresh = true,
"--follow-symlinks" => follow_symlinks = true,
"--no-default-excludes" => no_default_excludes = true,
+ "--stub" => stub_globs.push(it.next().ok_or("--stub needs a glob pattern")?),
+ "--no-gitignore" => no_gitignore = true,
"--max-file-gb" => {
max_file_gb = it
.next()
@@ -283,6 +304,8 @@ pub fn parse(args: Vec<String>) -> Result<Cmd, String> {
fresh,
follow_symlinks,
no_default_excludes,
+ stub_globs,
+ no_gitignore,
max_file_gb,
sample: sample.max(50),
no_semantic,
diff --git a/engine/crates/xerj-autoindex/src/content.rs b/engine/crates/xerj-autoindex/src/content.rs
index 4111e7b22..3bf0d45fb 100644
--- a/engine/crates/xerj-autoindex/src/content.rs
+++ b/engine/crates/xerj-autoindex/src/content.rs
@@ -187,7 +187,7 @@ mod tests {
let dir = tempfile::tempdir().unwrap();
fs::write(dir.path().join("a.txt"), b"same records\n").unwrap();
fs::write(dir.path().join("b.txt"), b"same records\n").unwrap();
- let inv = resolve(crate::walk::walk(dir.path(), false, true).unwrap().0).unwrap();
+ let inv = resolve(crate::walk::walk(dir.path(), false, true, true).unwrap().0).unwrap();
assert_eq!(inv.files.len(), 1);
assert_eq!(inv.files[0].rel, "a.txt");
assert_eq!(inv.duplicates.len(), 1);
@@ -204,7 +204,7 @@ mod tests {
second[65_536] = b'b';
fs::write(dir.path().join("a.txt"), first).unwrap();
fs::write(dir.path().join("b.txt"), second).unwrap();
- let inv = resolve(crate::walk::walk(dir.path(), false, true).unwrap().0).unwrap();
+ let inv = resolve(crate::walk::walk(dir.path(), false, true, true).unwrap().0).unwrap();
assert_eq!(inv.files.len(), 2);
assert!(inv.duplicates.is_empty());
assert_ne!(inv.keys[0], inv.keys[1]);
@@ -215,7 +215,7 @@ mod tests {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("a.txt");
fs::write(&path, b"content-a").unwrap();
- let inv = resolve(crate::walk::walk(dir.path(), false, true).unwrap().0).unwrap();
+ let inv = resolve(crate::walk::walk(dir.path(), false, true, true).unwrap().0).unwrap();
fs::write(&path, b"content-b").unwrap();
assert!(verify(&path, 9, &inv.digests[0]).is_err());
}
@@ -225,7 +225,7 @@ mod tests {
let dir = tempfile::tempdir().unwrap();
fs::write(dir.path().join("a.txt"), b"unequal-a").unwrap();
fs::write(dir.path().join("b.txt"), b"unequal-b").unwrap();
- let files = crate::walk::walk(dir.path(), false, true).unwrap().0;
+ let files = crate::walk::walk(dir.path(), false, true, true).unwrap().0;
let forced = vec!["axf2-forced-9".to_string(); 2];
let inv = resolve_with_digests(files, forced).unwrap();
assert_eq!(inv.files.len(), 2);
@@ -242,7 +242,7 @@ mod tests {
let b = dir.path().join("b.txt");
fs::write(&a, b"hardlinked").unwrap();
fs::hard_link(&a, &b).unwrap();
- let inv = resolve(crate::walk::walk(dir.path(), false, true).unwrap().0).unwrap();
+ let inv = resolve(crate::walk::walk(dir.path(), false, true, true).unwrap().0).unwrap();
assert_eq!(inv.files.len(), 1);
assert_eq!(inv.duplicates.len(), 1);
}
@@ -264,7 +264,7 @@ mod tests {
b"same",
)
.unwrap();
- let inv = resolve(crate::walk::walk(dir.path(), false, true).unwrap().0).unwrap();
+ let inv = resolve(crate::walk::walk(dir.path(), false, true, true).unwrap().0).unwrap();
assert_eq!(inv.files.len(), 1);
assert_eq!(inv.duplicates.len(), 2);
assert_ne!(inv.duplicates[0].path_id, inv.duplicates[1].path_id);
@@ -278,7 +278,7 @@ mod tests {
let real = dir.path().join("z-real");
fs::write(&real, b"same").unwrap();
symlink(&real, dir.path().join("a-link")).unwrap();
- let inv = resolve(crate::walk::walk(dir.path(), true, true).unwrap().0).unwrap();
+ let inv = resolve(crate::walk::walk(dir.path(), true, true, true).unwrap().0).unwrap();
assert_eq!(inv.files[0].rel, "z-real");
assert_eq!(inv.duplicates[0].rel, "a-link");
}
diff --git a/engine/crates/xerj-autoindex/src/detect/e2e.rs b/engine/crates/xerj-autoindex/src/detect/e2e.rs
index 30b95dede..15684ef61 100644
--- a/engine/crates/xerj-autoindex/src/detect/e2e.rs
+++ b/engine/crates/xerj-autoindex/src/detect/e2e.rs
@@ -271,6 +271,8 @@ fn cfg(root: &Path, state_dir: &Path, url: &str) -> IndexCfg {
fresh: false,
follow_symlinks: false,
no_default_excludes: false,
+ stub_globs: Vec::new(),
+ no_gitignore: false,
max_file_gb: 1,
sample: 50,
no_semantic: true,
diff --git a/engine/crates/xerj-autoindex/src/extract/bvh.rs b/engine/crates/xerj-autoindex/src/extract/bvh.rs
new file mode 100644
index 000000000..f835f907f
--- /dev/null
+++ b/engine/crates/xerj-autoindex/src/extract/bvh.rs
@@ -0,0 +1,175 @@
+//! BVH motion capture — ONE metadata record per file, motion data never read.
+//!
+//! A BVH file is a small skeleton HIERARCHY header followed by a numeric
+//! MOTION block that can run to hundreds of MB (one line per frame). The
+//! frames carry no retrieval value — they are float soup — but the header
+//! does: joint names make the skeleton searchable ("which clips animate
+//! LeftHand?"), and frame count/time give the clip's duration. So this
+//! extractor streams the header, captures that metadata, and STOPS as soon
+//! as the two `Frames:`/`Frame Time:` lines after `MOTION` are read — the
+//! motion block is never pulled off disk.
+//!
+//! Emitted fields: `title`, `joints` (names, document order), `joint_count`,
+//! `frames`, `frame_time_s`, `duration_s`. All extractor vocabulary
+//! (`FieldOrigin::Extractor`), so every BVH file clusters into one dataset.
+
+use super::{ExtractStats, FieldOrigin, RawRecord, Sink};
+use anyhow::Result;
+use serde_json::{Map, Value};
+use std::path::Path;
+
+/// Skeleton headers are a few KB; a "header" that runs past this is not a
+/// BVH we understand, and the file junks rather than streaming megabytes.
+const HEADER_CAP: usize = 1 << 20;
+
+pub fn extract(path: &Path, gzip: bool, sink: Sink) -> Result<ExtractStats> {
+ let mut stats = ExtractStats::default();
+ let mut r = super::open_reader(path, gzip, None)?;
+ let mut joints: Vec<String> = Vec::new();
+ let mut frames: Option<i64> = None;
+ let mut frame_time: Option<f64> = None;
+ let mut in_motion = false;
+ let mut consumed = 0usize;
+ let mut line: Vec<u8> = Vec::new();
+ loop {
+ line.clear();
+ let n = super::jsonl::read_capped_line(&mut r, &mut line)?;
+ if n == 0 {
+ break;
+ }
+ consumed += n;
+ if consumed > HEADER_CAP {
+ stats.junk += 1;
+ return Ok(stats);
+ }
+ let text = String::from_utf8_lossy(&line);
+ let t = text.trim();
+ if !in_motion {
+ if let Some(name) = t
+ .strip_prefix("ROOT ")
+ .or_else(|| t.strip_prefix("JOINT "))
+ {
+ let name = name.trim();
+ if !name.is_empty() && name.len() <= 200 {
+ joints.push(name.to_string());
+ }
+ } else if t == "MOTION" {
+ in_motion = true;
+ }
+ continue;
+ }
+ if let Some(v) = t.strip_prefix("Frames:") {
+ frames = v.trim().parse().ok();
+ } else if let Some(v) = t.strip_prefix("Frame Time:") {
+ frame_time = v.trim().parse().ok();
+ }
+ if frames.is_some() && frame_time.is_some() {
+ break; // metadata complete — the motion block is never read
+ }
+ }
+ if joints.is_empty() {
+ stats.junk += 1;
+ return Ok(stats);
+ }
+
+ let title = path
+ .file_name()
+ .and_then(|n| n.to_str())
+ .unwrap_or("motion")
+ .to_string();
+ let mut fields = Map::new();
+ fields.insert("title".into(), Value::String(title));
+ fields.insert(
+ "joints".into(),
+ Value::Array(joints.iter().map(|j| Value::String(j.clone())).collect()),
+ );
+ fields.insert(
+ "joint_count".into(),
+ Value::Number((joints.len() as u64).into()),
+ );
+ if let Some(f) = frames {
+ fields.insert("frames".into(), Value::Number(f.into()));
+ }
+ if let Some(ft) = frame_time {
+ if let Some(num) = serde_json::Number::from_f64(ft) {
+ fields.insert("frame_time_s".into(), Value::Number(num));
+ }
+ if let (Some(f), true) = (frames, ft.is_finite()) {
+ if let Some(num) = serde_json::Number::from_f64(f as f64 * ft) {
+ fields.insert("duration_s".into(), Value::Number(num));
+ }
+ }
+ }
+ stats.records += 1;
+ sink(RawRecord {
+ fields,
+ locator: "bvh".into(),
+ group: None,
+ origin: FieldOrigin::Extractor,
+ });
+ Ok(stats)
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ const SAMPLE: &str = "HIERARCHY\nROOT Hips\n{\n OFFSET 0 90 0\n CHANNELS 6 Xposition Yposition Zposition Zrotation Xrotation Yrotation\n JOINT LeftUpLeg\n {\n OFFSET 8.5 0 -2.5\n CHANNELS 3 Zrotation Xrotation Yrotation\n End Site\n {\n OFFSET 6.9 0 0\n }\n }\n}\nMOTION\nFrames: 120\nFrame Time: 0.033333\n0.0 90.0 0.0 1.1 2.2 3.3 4.4 5.5 6.6\n0.1 90.1 0.1 1.2 2.3 3.4 4.5 5.6 6.7\n";
+
+ fn run(text: &str) -> (ExtractStats, Vec<RawRecord>) {
+ let dir = tempfile::tempdir().unwrap();
+ let path = dir.path().join("clip.bvh");
+ std::fs::write(&path, text).unwrap();
+ let mut recs = Vec::new();
+ let stats = extract(&path, false, &mut |r| {
+ recs.push(r);
+ true
+ })
+ .unwrap();
+ (stats, recs)
+ }
+
+ #[test]
+ fn one_metadata_record_per_file_and_no_motion_rows() {
+ let (stats, recs) = run(SAMPLE);
+ assert_eq!((stats.records, stats.junk), (1, 0));
+ let r = &recs[0];
+ assert_eq!(r.fields["title"], "clip.bvh");
+ assert_eq!(r.fields["joints"], serde_json::json!(["Hips", "LeftUpLeg"]));
+ assert_eq!(r.fields["joint_count"], 2);
+ assert_eq!(r.fields["frames"], 120);
+ assert_eq!(r.fields["duration_s"].as_f64().unwrap().round(), 4.0);
+ assert_eq!(r.locator, "bvh");
+ }
+
+ /// The point of the format: a clip with a multi-GB motion block must cost
+ /// only its header. The reader stops after `Frame Time:`, so the frames
+ /// after it are never even pulled through the BufReader into the record.
+ #[test]
+ fn extraction_stops_at_the_motion_header_not_the_motion_data() {
+ let mut text = SAMPLE.to_string();
+ text.push_str(&"9.9 ".repeat(500_000));
+ let (stats, recs) = run(&text);
+ assert_eq!(stats.records, 1);
+ assert_eq!(recs[0].fields["frames"], 120);
+ }
+
+ #[test]
+ fn a_hierarchy_without_joints_is_junk_not_an_empty_record() {
+ let (stats, recs) = run("HIERARCHY\nMOTION\nFrames: 1\nFrame Time: 0.03\n1 2 3\n");
+ assert_eq!(stats.records, 0);
+ assert_eq!(stats.junk, 1);
+ assert!(recs.is_empty());
+ }
+
+ #[test]
+ fn a_runaway_header_is_junk_rather_than_streamed_forever() {
+ let mut text = String::from("HIERARCHY\nROOT Hips\n{\n");
+ for i in 0..40_000 {
+ text.push_str(&format!(" JOINT J{i}\n {{\n OFFSET 0 0 0\n"));
+ }
+ let (stats, _) = run(&text);
+ assert_eq!(stats.records, 0);
+ assert_eq!(stats.junk, 1);
+ }
+}
diff --git a/engine/crates/xerj-autoindex/src/extract/mod.rs b/engine/crates/xerj-autoindex/src/extract/mod.rs
index a3c8c6199..ac0f45fa1 100644
--- a/engine/crates/xerj-autoindex/src/extract/mod.rs
+++ b/engine/crates/xerj-autoindex/src/extract/mod.rs
@@ -2,6 +2,7 @@
//! Every extractor is bounded-memory and never fatal: parse failures
//! downgrade (family → txt → junk-with-metadata) and are counted.
+pub mod bvh;
pub mod code;
pub mod csv_x;
pub mod docx;
@@ -142,6 +143,26 @@ pub fn extract(
Family::Code => code::extract(path, sink),
Family::UnityYaml => unity::extract_unity(path, sn.gzip, limit_bytes, sink),
Family::UnityMeta => unity::extract_meta(path, sn.gzip, sink),
+ Family::Bvh => bvh::extract(path, sn.gzip, sink),
+ // `--stub`-designated file: one name card, the file is never opened.
+ Family::Stub => {
+ let mut stats = ExtractStats::default();
+ let mut fields = Map::new();
+ let title = path
+ .file_name()
+ .and_then(|n| n.to_str())
+ .unwrap_or("file")
+ .to_string();
+ fields.insert("title".into(), Value::String(title));
+ stats.records += 1;
+ sink(RawRecord {
+ fields,
+ locator: "stub".into(),
+ group: None,
+ origin: FieldOrigin::Extractor,
+ });
+ Ok(stats)
+ }
Family::Binary => Ok(ExtractStats::default()),
}
}
diff --git a/engine/crates/xerj-autoindex/src/failure_resume_http_tests.rs b/engine/crates/xerj-autoindex/src/failure_resume_http_tests.rs
index 7be372a48..84481d716 100644
--- a/engine/crates/xerj-autoindex/src/failure_resume_http_tests.rs
+++ b/engine/crates/xerj-autoindex/src/failure_resume_http_tests.rs
@@ -205,6 +205,8 @@ fn cfg(root: &Path, state_dir: &Path, url: &str) -> IndexCfg {
fresh: false,
follow_symlinks: false,
no_default_excludes: false,
+ stub_globs: Vec::new(),
+ no_gitignore: false,
max_file_gb: 1,
sample: 50,
no_semantic: true,
diff --git a/engine/crates/xerj-autoindex/src/lib.rs b/engine/crates/xerj-autoindex/src/lib.rs
index 3aa83fe4f..1a3c29e54 100644
--- a/engine/crates/xerj-autoindex/src/lib.rs
+++ b/engine/crates/xerj-autoindex/src/lib.rs
@@ -173,6 +173,137 @@ fn section_label(locator: &str) -> Option<String> {
(digits(page) && digits(sec)).then(|| format!("page {page} section {sec}"))
}
+// ─── --stub glob matcher ──────────────────────────────────────────────────
+
+/// Compiled `--stub <glob>` patterns. A matching file is indexed as ONE
+/// name-card record (`Family::Stub`) and its contents are never opened —
+/// the owner's way of saying "this data blob should be referenceable but
+/// not parsed" without the engine hardcoding per-corpus rules.
+///
+/// Glob semantics (gitignore-flavored): `**` crosses `/`, `*` and `?` do
+/// not; a pattern without `/` matches against the file NAME anywhere in the
+/// tree, a pattern with `/` matches the full root-relative path.
+pub struct StubMatcher {
+ by_name: Vec<regex::Regex>,
+ by_path: Vec<regex::Regex>,
+}
+
+impl StubMatcher {
+ pub fn compile(globs: &[String]) -> Result<Self> {
+ let mut by_name = Vec::new();
+ let mut by_path = Vec::new();
+ for g in globs {
+ let re = regex::Regex::new(&glob_to_regex(g))
+ .with_context(|| format!("--stub {g}: invalid pattern"))?;
+ if g.contains('/') {
+ by_path.push(re);
+ } else {
+ by_name.push(re);
+ }
+ }
+ Ok(Self { by_name, by_path })
+ }
+
+ pub fn is_empty(&self) -> bool {
+ self.by_name.is_empty() && self.by_path.is_empty()
+ }
+
+ /// `rel` is the root-relative path with forward slashes.
+ pub fn matches(&self, rel: &str) -> bool {
+ if self.by_path.iter().any(|re| re.is_match(rel)) {
+ return true;
+ }
+ if self.by_name.is_empty() {
+ return false;
+ }
+ let name = rel.rsplit('/').next().unwrap_or(rel);
+ self.by_name.iter().any(|re| re.is_match(name))
+ }
+}
+
+fn glob_to_regex(glob: &str) -> String {
+ let mut out = String::from("^");
+ let mut chars = glob.chars().peekable();
+ while let Some(c) = chars.next() {
+ match c {
+ '*' => {
+ if chars.peek() == Some(&'*') {
+ chars.next();
+ // `**/` also swallows its slash so `**/x` matches a
+ // top-level `x`.
+ if chars.peek() == Some(&'/') {
+ chars.next();
+ out.push_str("(?:.*/)?");
+ } else {
+ out.push_str(".*");
+ }
+ } else {
+ out.push_str("[^/]*");
+ }
+ }
+ '?' => out.push_str("[^/]"),
+ other => out.push_str(&regex::escape(&other.to_string())),
+ }
+ }
+ out.push('$');
+ out
+}
+
+/// The synthetic sniff result for a `--stub`-designated file.
+fn stub_sniffed() -> Sniffed {
+ Sniffed {
+ family: Family::Stub,
+ gzip: false,
+ binary_kind: None,
+ csv: None,
+ encoding: "utf-8",
+ }
+}
+
+#[cfg(test)]
+mod stub_matcher_tests {
+ use super::StubMatcher;
+
+ fn m(globs: &[&str]) -> StubMatcher {
+ StubMatcher::compile(&globs.iter().map(|s| s.to_string()).collect::<Vec<_>>()).unwrap()
+ }
+
+ #[test]
+ fn a_bare_pattern_matches_file_names_anywhere() {
+ let s = m(&["*.csv"]);
+ assert!(s.matches("unity/Assets/Face/f_roommate_004.csv"));
+ assert!(s.matches("top.csv"));
+ assert!(!s.matches("unity/Assets/notes.csv.md"));
+ }
+
+ #[test]
+ fn a_path_pattern_matches_the_root_relative_path() {
+ let s = m(&["unity/**/*.csv"]);
+ assert!(s.matches("unity/Assets/Face/f_roommate_004.csv"));
+ assert!(s.matches("unity/top.csv"), "**/ also matches zero dirs");
+ assert!(!s.matches("backend/data/users.csv"), "scoped to unity/");
+ }
+
+ #[test]
+ fn single_star_does_not_cross_directories() {
+ let s = m(&["unity/*.csv"]);
+ assert!(s.matches("unity/top.csv"));
+ assert!(!s.matches("unity/Assets/deep.csv"));
+ }
+
+ #[test]
+ fn regex_metacharacters_in_patterns_are_literal() {
+ let s = m(&["data(v1).csv"]);
+ assert!(s.matches("x/data(v1).csv"));
+ assert!(!s.matches("x/dataXv1Y.csv"));
+ }
+
+ #[test]
+ fn an_invalid_pattern_fails_loudly_at_startup() {
+ assert!(StubMatcher::compile(&["ok.csv".into()]).is_ok());
+ }
+}
+
// ─── Phase A: per-file scan (sniff + bounded sampling) ───────────────────
struct FileScan {
@@ -191,17 +322,21 @@ struct GroupSketch {
records: u64,
}
-fn scan_file(path: &Path, size: u64, sample: usize, max_file_gb: u64) -> FileScan {
+fn scan_file(path: &Path, size: u64, sample: usize, max_file_gb: u64, stub: bool) -> FileScan {
let mut out = FileScan {
sniffed: None,
sketches: Vec::new(),
junk: None,
};
- let sn = match sniff::sniff(path) {
- Ok(s) => s,
- Err(e) => {
- out.junk = Some(("junk".into(), format!("unreadable: {e}")));
- return out;
+ let sn = if stub {
+ stub_sniffed()
+ } else {
+ match sniff::sniff(path) {
+ Ok(s) => s,
+ Err(e) => {
+ out.junk = Some(("junk".into(), format!("unreadable: {e}")));
+ return out;
+ }
}
};
if sn.family == Family::Binary {
@@ -315,7 +450,7 @@ mod clustering_key_tests {
let path = dir.join(name);
std::fs::write(&path, body).unwrap();
let size = std::fs::metadata(&path).unwrap().len();
- scan_file(&path, size, 500, 2)
+ scan_file(&path, size, 500, 2, false)
}
/// The #178 mechanism, from the extractor to the clustering key: a source
@@ -675,8 +810,14 @@ pub fn run_index_report(cfg: IndexCfg) -> Result<(i32, Option<Value>)> {
let es = Es::with_bulk_timeout(&cfg.url, cfg.api_key.clone(), cfg.bulk_timeout_secs)?;
es.ping()?;
+ let stub_matcher = StubMatcher::compile(&cfg.stub_globs)?;
let (discovered_files, skipped_dirs) =
- walk::walk(&cfg.root, cfg.follow_symlinks, !cfg.no_default_excludes)?;
+ walk::walk(
+ &cfg.root,
+ cfg.follow_symlinks,
+ !cfg.no_default_excludes,
+ !cfg.no_gitignore,
+ )?;
if !skipped_dirs.is_empty() && !cfg.quiet {
let names: Vec<&str> = skipped_dirs.iter().map(|s| s.rel.as_str()).collect();
eprintln!(
@@ -886,7 +1027,15 @@ pub fn run_index_report(cfg: IndexCfg) -> Result<(i32, Option<Value>)> {
}
let scans: Vec<FileScan> = files
.par_iter()
- .map(|f| scan_file(&f.path, f.size, cfg.sample, cfg.max_file_gb))
+ .map(|f| {
+ scan_file(
+ &f.path,
+ f.size,
+ cfg.sample,
+ cfg.max_file_gb,
+ stub_matcher.matches(&f.rel),
+ )
+ })
.collect();
let rels: Vec<String> = files.iter().map(|f| f.rel.clone()).collect();
@@ -1352,7 +1501,12 @@ pub fn run_index_report(cfg: IndexCfg) -> Result<(i32, Option<Value>)> {
let fa = plan.files.get(key).unwrap();
let asg: HashMap<Option<String>, String> =
fa.assignments.iter().cloned().collect();
- let sn = match sniff::sniff(&f.path) {
+ let sn = if stub_matcher.matches(&f.rel) {
+ Ok(stub_sniffed())
+ } else {
+ sniff::sniff(&f.path)
+ };
+ let sn = match sn {
Ok(s) => s,
Err(e) => {
extra_junk.lock().unwrap().push(JunkFile {
@@ -2497,7 +2651,7 @@ mod duplicate_integration_tests {
b[65_536] = b'b';
fs::write(corpus.path().join("a.txt"), a).unwrap();
fs::write(corpus.path().join("b.txt"), b).unwrap();
- let files = walk::walk(corpus.path(), false, true).unwrap().0;
+ let files = walk::walk(corpus.path(), false, true, true).unwrap().0;
let inventory = content::resolve(files.clone()).unwrap();
let legacy = ids::file_key(&files[0].path, files[0].size).unwrap();
assert_eq!(
@@ -2534,7 +2688,7 @@ mod duplicate_integration_tests {
// with — so b.txt's content key IS the planned key a.txt claims by rel.
fs::write(corpus.path().join("a.txt"), b"rewritten content\n").unwrap();
fs::write(corpus.path().join("b.txt"), b"original planned content\n").unwrap();
- let inventory = content::resolve(walk::walk(corpus.path(), false, true).unwrap().0).unwrap();
+ let inventory = content::resolve(walk::walk(corpus.path(), false, true, true).unwrap().0).unwrap();
let planned_key = inventory.keys[1].clone();
let mut plan = Plan::default();
plan.files
@@ -2604,7 +2758,7 @@ mod duplicate_integration_tests {
fs::write(corpus.path().join("report-original.txt"), body).unwrap();
fs::write(corpus.path().join("report-copy.txt"), body).unwrap();
- let discovered = walk::walk(corpus.path(), false, true).unwrap().0;
+ let discovered = walk::walk(corpus.path(), false, true, true).unwrap().0;
let inventory = content::resolve(discovered).unwrap();
assert_eq!(inventory.files.len(), 1);
assert_eq!(inventory.duplicates.len(), 1);
@@ -2677,7 +2831,7 @@ mod duplicate_integration_tests {
}
fs::write(&path, csv).unwrap();
- let inventory = content::resolve(walk::walk(corpus.path(), false, true).unwrap().0).unwrap();
+ let inventory = content::resolve(walk::walk(corpus.path(), false, true, true).unwrap().0).unwrap();
let expected_size = inventory.files[0].size;
let expected_digest = inventory.digests[0].clone();
let sniffed = sniff::sniff(&path).unwrap();
diff --git a/engine/crates/xerj-autoindex/src/sniff.rs b/engine/crates/xerj-autoindex/src/sniff.rs
index 0dbf3a489..49f31bdfb 100644
--- a/engine/crates/xerj-autoindex/src/sniff.rs
+++ b/engine/crates/xerj-autoindex/src/sniff.rs
@@ -30,6 +30,15 @@ pub enum Family {
/// Unity `.meta` sidecar: plain YAML opening with `fileFormatVersion:`
/// and carrying the asset `guid` — the join key for everything Unity.
UnityMeta,
+ /// Biovision motion capture: a skeleton HIERARCHY header followed by a
+ /// large numeric MOTION block. Indexed as ONE metadata record per file
+ /// (joints, frame count, duration) — the motion numbers are never read.
+ Bvh,
+ /// User-designated existence-only file (`--stub <glob>`): ONE name-card
+ /// record, contents never opened. For corpus-specific data blobs the
+ /// owner wants referenceable but not parsed — never assigned by
+ /// sniffing, only by the CLI flag.
+ Stub,
Binary,
}
@@ -52,6 +61,8 @@ impl Family {
Family::Code => "code",
Family::UnityYaml => "unity",
Family::UnityMeta => "unity-meta",
+ Family::Bvh => "bvh",
+ Family::Stub => "stub",
Family::Binary => "binary",
}
}
@@ -224,6 +235,19 @@ fn sniff_bytes(prefix: &[u8], path: &Path, gzip: bool) -> Result<Sniffed> {
s.encoding = encoding;
return Ok(s);
}
+ // BVH motion capture: `HIERARCHY` opener with a `ROOT <name>` next.
+ // Without this the numeric MOTION block classified as txt-lines and
+ // indexed millions of meaningless number rows.
+ if first_line.trim() == "HIERARCHY"
+ && body
+ .lines()
+ .nth(1)
+ .is_some_and(|l| l.trim_start().starts_with("ROOT "))
+ {
+ let mut s = mk(Family::Bvh);
+ s.encoding = encoding;
+ return Ok(s);
+ }
}
// 2c. Source code: a known code extension whose content is text. We only
@@ -624,6 +648,19 @@ mod unity_sniff_tests {
assert_ne!(sniff_str(no_guid, "x.meta"), Family::UnityMeta);
}
+ #[test]
+ fn bvh_is_detected_by_hierarchy_root_header() {
+ let bvh = "HIERARCHY\nROOT Hips\n{\n OFFSET 0 90 0\n CHANNELS 6 Xposition Yposition Zposition Zrotation Xrotation Yrotation\n}\nMOTION\nFrames: 2\nFrame Time: 0.033\n1 2 3\n4 5 6\n";
+ assert_eq!(sniff_str(bvh, "clip.bvh"), Family::Bvh);
+ assert_eq!(
+ sniff_str(bvh, "clip.txt"),
+ Family::Bvh,
+ "content decides, never the extension"
+ );
+ let not_bvh = "HIERARCHY\nof needs (Maslow):\n- physiological\n- safety\n";
+ assert_ne!(sniff_str(not_bvh, "notes.txt"), Family::Bvh);
+ }
+
#[test]
fn plain_yaml_without_the_unity_tag_stays_yaml() {
let plain = "%YAML 1.2\n---\nkey: value\nother: 1\nnested:\n a: 2\n";
diff --git a/engine/crates/xerj-autoindex/src/walk.rs b/engine/crates/xerj-autoindex/src/walk.rs
index e316aafc0..bdfd7fd55 100644
--- a/engine/crates/xerj-autoindex/src/walk.rs
+++ b/engine/crates/xerj-autoindex/src/walk.rs
@@ -1,5 +1,14 @@
//! Inventory: recursive walk of the target folder.
-//! Symlinks are NOT followed by default; with --follow-symlinks walkdir's
+//!
+//! Built on the `ignore` crate (ripgrep's walker), which gives three skip
+//! layers, all evidence-based:
+//! 1. hidden files/dirs — always (security: `.env`, `.git`, `.ssh`, …);
+//! 2. `.gitignore` rules — the corpus OWNER's own declaration of generated
+//! noise; honored only inside a real git repo (`.git` present), and
+//! disabled with `--no-gitignore`;
+//! 3. marker-gated generated dirs (`GENERATED_DIR_RULES`) — for corpora
+//! that are not git repos or whose ignore files are incomplete.
+//! Symlinks are NOT followed by default; with --follow-symlinks the walker's
//! ancestor-loop detection keeps the traversal loop-safe.
use anyhow::{Context, Result};
@@ -75,6 +84,7 @@ pub fn walk(
root: &Path,
follow_symlinks: bool,
default_excludes: bool,
+ gitignore: bool,
) -> Result<(Vec<FileEntry>, Vec<SkippedDir>)> {
let root_canon = root
.canonicalize()
@@ -82,28 +92,39 @@ pub fn walk(
if !root_canon.is_dir() {
anyhow::bail!("{} is not a directory", root_canon.display());
}
- let skipped: std::rc::Rc<std::cell::RefCell<Vec<SkippedDir>>> = Default::default();
+ // Arc<Mutex>: the `ignore` walker's filter must be Send+Sync even for the
+ // sequential build() we use.
+ let skipped: std::sync::Arc<std::sync::Mutex<Vec<SkippedDir>>> = Default::default();
let mut out = Vec::new();
// SECURITY / hygiene: never index hidden files or descend into hidden
- // directories. Without this the walker happily indexed `.env` (secrets,
- // API tokens), `.git`, `.ssh`, `.aws`, and other dotfiles into a queryable
- // brain with no per-brain authorization — a real exposure for the
- // "point it at my project folder" use case. `filter_entry` prunes a hidden
- // directory before descending, so `.git/` is skipped whole. The root itself
- // (depth 0) is exempt so a brain over a dot-named folder still works.
- let sk = std::rc::Rc::clone(&skipped);
+ // directories (`hidden(true)`). Without this the walker happily indexed
+ // `.env` (secrets, API tokens), `.git`, `.ssh`, `.aws`, and other dotfiles
+ // into a queryable brain with no per-brain authorization — a real exposure
+ // for the "point it at my project folder" use case. The root itself is
+ // exempt (the walk origin is always yielded) so a brain over a dot-named
+ // folder still works.
+ //
+ // `.gitignore` handling stays deterministic and corpus-local: tree
+ // `.gitignore`s + `.git/info/exclude` only, never the user's global
+ // gitignore config; `require_git` keeps the semantics evidence-based (a
+ // stray .gitignore outside a repo is not an owner declaration).
+ let sk = std::sync::Arc::clone(&skipped);
let filter_root = root_canon.clone();
- let walker = walkdir::WalkDir::new(&root_canon)
+ let mut builder = ignore::WalkBuilder::new(&root_canon);
+ builder
.follow_links(follow_symlinks)
- .into_iter()
+ .hidden(true)
+ .ignore(false)
+ .parents(gitignore)
+ .git_ignore(gitignore)
+ .git_exclude(gitignore)
+ .git_global(false)
+ .require_git(true)
.filter_entry(move |e| {
if e.depth() == 0 {
return true;
}
- if e.file_name().to_str().is_some_and(|n| n.starts_with('.')) {
- return false;
- }
- if default_excludes && e.file_type().is_dir() {
+ if default_excludes && e.file_type().is_some_and(|t| t.is_dir()) {
for (name, marker, reason) in GENERATED_DIR_RULES {
if e.file_name() == *name
&& e.path()
@@ -116,7 +137,7 @@ pub fn walk(
.unwrap_or(e.path())
.to_string_lossy()
.replace('\\', "/");
- sk.borrow_mut().push(SkippedDir {
+ sk.lock().expect("skip list lock").push(SkippedDir {
rel,
reason: (*reason).to_string(),
});
@@ -126,7 +147,7 @@ pub fn walk(
}
true
});
- for entry in walker {
+ for entry in builder.build() {
let entry = match entry {
Ok(e) => e,
Err(e) => {
@@ -134,7 +155,7 @@ pub fn walk(
continue;
}
};
- if !entry.file_type().is_file() {
+ if !entry.file_type().is_some_and(|t| t.is_file()) {
continue;
}
let md = match entry.metadata() {
@@ -152,18 +173,15 @@ pub fn walk(
path: p,
rel,
rel_id,
- is_symlink: entry
- .path()
- .symlink_metadata()
- .is_ok_and(|m| m.file_type().is_symlink()),
+ is_symlink: entry.path_is_symlink(),
size: md.len(),
});
}
// Deterministic order for clustering / naming.
out.sort_by(|a, b| a.rel.cmp(&b.rel));
- let skipped = std::rc::Rc::try_unwrap(skipped)
- .expect("walker iterator consumed; sole owner remains")
- .into_inner();
+ // The builder keeps its own clone of the filter closure (and thus of the
+ // Arc), so drain under the lock instead of unwrapping ownership.
+ let skipped = std::mem::take(&mut *skipped.lock().expect("skip list lock"));
Ok((out, skipped))
}
@@ -211,7 +229,7 @@ mod hidden_skip_tests {
fs::write(root.join("src").join("main.rs"), "fn main() {}").unwrap();
fs::write(root.join("src").join(".secret"), "nope").unwrap();
- let rels: Vec<String> = walk(root, false, true)
+ let rels: Vec<String> = walk(root, false, true, true)
.unwrap()
.0
.into_iter()
@@ -237,7 +255,7 @@ mod hidden_skip_tests {
let root = dir.path().join(".notes");
fs::create_dir(&root).unwrap();
fs::write(root.join("a.md"), "x").unwrap();
- let rels: Vec<String> = walk(&root, false, true)
+ let rels: Vec<String> = walk(&root, false, true, true)
.unwrap()
.0
.into_iter()
@@ -278,7 +296,7 @@ mod generated_dir_tests {
fn unity_marker_prunes_generated_dirs_and_records_them() {
let dir = tempfile::TempDir::new().unwrap();
make_unity_root(dir.path(), true);
- let (files, skipped) = walk(dir.path(), false, true).unwrap();
+ let (files, skipped) = walk(dir.path(), false, true, true).unwrap();
let rels: Vec<&str> = files.iter().map(|e| e.rel.as_str()).collect();
assert!(rels.contains(&"Assets/Player.cs"));
assert!(
@@ -295,7 +313,7 @@ mod generated_dir_tests {
fn without_the_marker_nothing_is_pruned() {
let dir = tempfile::TempDir::new().unwrap();
make_unity_root(dir.path(), false);
- let (files, skipped) = walk(dir.path(), false, true).unwrap();
+ let (files, skipped) = walk(dir.path(), false, true, true).unwrap();
let rels: Vec<&str> = files.iter().map(|e| e.rel.as_str()).collect();
assert!(
rels.iter().any(|r| r.starts_with("Library/")),
@@ -308,7 +326,7 @@ mod generated_dir_tests {
fn no_default_excludes_walks_everything() {
let dir = tempfile::TempDir::new().unwrap();
make_unity_root(dir.path(), true);
- let (files, skipped) = walk(dir.path(), false, false).unwrap();
+ let (files, skipped) = walk(dir.path(), false, false, true).unwrap();
let rels: Vec<&str> = files.iter().map(|e| e.rel.as_str()).collect();
assert!(rels.iter().any(|r| r.starts_with("Library/")));
assert!(skipped.is_empty());
@@ -319,7 +337,7 @@ mod generated_dir_tests {
let dir = tempfile::TempDir::new().unwrap();
make_unity_root(dir.path(), true);
fs::write(dir.path().join("obj"), "a FILE named obj").unwrap();
- let (files, _) = walk(dir.path(), false, true).unwrap();
+ let (files, _) = walk(dir.path(), false, true, true).unwrap();
assert!(
files.iter().any(|e| e.rel == "obj"),
"the prune applies to directories only"
@@ -336,7 +354,7 @@ mod generated_dir_tests {
fs::create_dir_all(&unity).unwrap();
make_unity_root(&unity, true);
fs::write(dir.path().join("README.md"), "monorepo").unwrap();
- let (files, skipped) = walk(dir.path(), false, true).unwrap();
+ let (files, skipped) = walk(dir.path(), false, true, true).unwrap();
let rels: Vec<&str> = files.iter().map(|e| e.rel.as_str()).collect();
assert!(rels.contains(&"apps/game/Assets/Player.cs"));
assert!(!rels.iter().any(|r| r.contains("game/Library/")));
@@ -354,7 +372,7 @@ mod generated_dir_tests {
let bare = dir.path().join("data/node_modules");
fs::create_dir_all(&bare).unwrap();
fs::write(bare.join("notes.txt"), "not an npm tree").unwrap();
- let (files, skipped) = walk(dir.path(), false, true).unwrap();
+ let (files, skipped) = walk(dir.path(), false, true, true).unwrap();
let rels: Vec<&str> = files.iter().map(|e| e.rel.as_str()).collect();
assert!(rels.contains(&"web/index.js"));
assert!(!rels.iter().any(|r| r.starts_with("web/node_modules")));
@@ -366,6 +384,73 @@ mod generated_dir_tests {
assert_eq!(skipped[0].rel, "web/node_modules");
}
+ #[test]
+ fn gitignored_files_are_skipped_only_inside_a_git_repo() {
+ let dir = tempfile::TempDir::new().unwrap();
+ let root = dir.path();
+ fs::create_dir(root.join(".git")).unwrap();
+ fs::write(root.join(".gitignore"), "build/\n*.log\n").unwrap();
+ fs::create_dir_all(root.join("build/intermediates")).unwrap();
+ fs::write(root.join("build/intermediates/R.txt"), "generated").unwrap();
+ fs::write(root.join("debug.log"), "noise").unwrap();
+ fs::write(root.join("main.rs"), "fn main() {}").unwrap();
+ fs::create_dir(root.join("src")).unwrap();
+ fs::write(root.join("src/app.log"), "noise").unwrap();
+ fs::write(root.join("src/lib.rs"), "pub fn f() {}").unwrap();
+
+ let (files, _) = walk(root, false, true, true).unwrap();
+ let rels: Vec<&str> = files.iter().map(|e| e.rel.as_str()).collect();
+ assert!(rels.contains(&"main.rs"));
+ assert!(rels.contains(&"src/lib.rs"));
+ assert!(
+ !rels.iter().any(|r| r.starts_with("build/")),
+ "gitignored dir must be pruned: {rels:?}"
+ );
+ assert!(
+ !rels.iter().any(|r| r.ends_with(".log")),
+ "gitignored glob applies at every depth: {rels:?}"
+ );
+
+ // --no-gitignore restores the full walk.
+ let (files, _) = walk(root, false, true, false).unwrap();
+ let rels: Vec<&str> = files.iter().map(|e| e.rel.as_str()).collect();
+ assert!(rels.contains(&"build/intermediates/R.txt"));
+ assert!(rels.contains(&"debug.log"));
+ }
+
+ /// A `.gitignore` OUTSIDE a git repository is not an owner declaration —
+ /// require_git keeps the rule evidence-based.
+ #[test]
+ fn a_gitignore_without_a_git_repo_is_inert() {
+ let dir = tempfile::TempDir::new().unwrap();
+ let root = dir.path();
+ fs::write(root.join(".gitignore"), "*.log\n").unwrap();
+ fs::write(root.join("debug.log"), "kept").unwrap();
+ let (files, _) = walk(root, false, true, true).unwrap();
+ assert!(
+ files.iter().any(|e| e.rel == "debug.log"),
+ "no .git dir, so gitignore semantics must not apply"
+ );
+ }
+
+ /// Nested .gitignore files scope to their own subtree, like git itself.
+ #[test]
+ fn nested_gitignore_files_scope_to_their_subtree() {
+ let dir = tempfile::TempDir::new().unwrap();
+ let root = dir.path();
+ fs::create_dir(root.join(".git")).unwrap();
+ fs::create_dir(root.join("web")).unwrap();
+ fs::write(root.join("web/.gitignore"), "dist/\n").unwrap();
+ fs::create_dir_all(root.join("web/dist")).unwrap();
+ fs::write(root.join("web/dist/bundle.js"), "x").unwrap();
+ fs::create_dir_all(root.join("docs/dist")).unwrap();
+ fs::write(root.join("docs/dist/manual.md"), "kept").unwrap();
+ let (files, _) = walk(root, false, true, true).unwrap();
+ let rels: Vec<&str> = files.iter().map(|e| e.rel.as_str()).collect();
+ assert!(!rels.iter().any(|r| r.starts_with("web/dist/")));
+ assert!(rels.contains(&"docs/dist/manual.md"));
+ }
+
#[test]
fn cargo_target_is_pruned_only_next_to_a_cargo_toml() {
let dir = tempfile::TempDir::new().unwrap();
@@ -377,7 +462,7 @@ mod generated_dir_tests {
let plain = dir.path().join("shooting/target");
fs::create_dir_all(&plain).unwrap();
fs::write(plain.join("scores.csv"), "a,b\n1,2\n").unwrap();
- let (files, skipped) = walk(dir.path(), false, true).unwrap();
+ let (files, skipped) = walk(dir.path(), false, true, true).unwrap();
let rels: Vec<&str> = files.iter().map(|e| e.rel.as_str()).collect();
assert!(rels.contains(&"mycrate/main.rs"));
assert!(!rels.iter().any(|r| r.starts_with("mycrate/target")));
diff --git a/engine/crates/xerj-server/src/brain.rs b/engine/crates/xerj-server/src/brain.rs
index 8fd418ab9..4951aa6ba 100644
--- a/engine/crates/xerj-server/src/brain.rs
+++ b/engine/crates/xerj-server/src/brain.rs
@@ -177,7 +177,7 @@ fn run(cfg: BrainCfg) -> Result<i32> {
if !cfg.root.exists() {
bail!("{} does not exist", cfg.root.display());
}
- let (files, _skipped_dirs) = walk::walk(&cfg.root, false, true)?;
+ let (files, _skipped_dirs) = walk::walk(&cfg.root, false, true, true)?;
if files.is_empty() {
eprintln!(
"xerj brain: no files under {} — nothing to build a brain from.\n\
@@ -433,6 +433,8 @@ fn index_cfg(cfg: &BrainCfg, brain: &str, api_key: Option<String>) -> IndexCfg {
fresh: cfg.fresh,
follow_symlinks: false,
no_default_excludes: false,
+ stub_globs: Vec::new(),
+ no_gitignore: false,
max_file_gb: 2,
sample: 500,
no_semantic: false,