AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
feat(autoindex): Unity project support — per-object scenes, GUID script links, BVH metadata, stub globs
Sergey Gonchardb4cbb5cb0d0
diff --git a/demo/usecases/autoindex/README.md b/demo/usecases/autoindex/README.mdindex 6e78f810f..7b350355c 100644--- a/demo/usecases/autoindex/README.md+++ b/demo/usecases/autoindex/README.md@@ -115,7 +115,12 @@ walk → sniff → sample → infer → map → extract → bulk → correlate`obj/`, `Logs/`, `UserSettings/` next to`ProjectSettings/ProjectVersion.txt`, `node_modules/` next to`package.json`, `target/` next to `Cargo.toml`; `--no-default-excludes`- walks them), plain text, gzip-compressed variants of the above.+ walks them; additionally, when the root is inside a git repository the+ tree's `.gitignore` rules are honored — the owner's own declaration of+ generated noise — with `--no-gitignore` to opt out, and the user's global+ gitignore is never consulted), BVH motion capture (one metadata record per clip — skeleton+ joint names, frame count, duration; the numeric MOTION block is never+ read), plain text, gzip-compressed variants of the above.3. **Sample** — bounded per-file sampling (default 500 records,`--sample N`); inference cost does not grow with file size.4. **Infer** — field types (long/double/boolean/date/keyword/text), datediff --git a/engine/Cargo.lock b/engine/Cargo.lockindex 90484f1eb..7d7ba7729 100644--- a/engine/Cargo.lock+++ b/engine/Cargo.lock@@ -529,6 +529,16 @@ dependencies = ["alloc-stdlib",]+[[package]]+name = "bstr"+version = "1.13.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "6bb31b46c14244e20ee9984b11bf5c992b91fb6939fea616e3512c8baecdbe5f"+dependencies = [+ "memchr",+ "serde_core",+]+[[package]]name = "bumpalo"version = "3.20.2"@@ -1930,6 +1940,19 @@ version = "0.3.3"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "0cc23270f6e1808e30a928bdc84dea0b9b4136a8bc82338574f23baf47bbd280"+[[package]]+name = "globset"+version = "0.4.19"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "e47d37d2ae4464254884b60ab7071be2b876a9c35b696bd018ddcc76847309cd"+dependencies = [+ "aho-corasick",+ "bstr",+ "log",+ "regex-automata",+ "regex-syntax",+]+[[package]]name = "grid"version = "1.0.1"@@ -2370,6 +2393,22 @@ dependencies = ["icu_properties",]+[[package]]+name = "ignore"+version = "0.4.32"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "0b17771570a2b94107741a7b033f19132c2eee21d59d21b24d2ced26500bd66e"+dependencies = [+ "crossbeam-deque",+ "globset",+ "log",+ "memchr",+ "regex-automata",+ "same-file",+ "walkdir",+ "winapi-util",+]+[[package]]name = "image"version = "0.25.10"@@ -6437,6 +6476,7 @@ dependencies = ["encoding_rs","flate2","fs2",+ "ignore","libc","memchr","pdf_oxide",@@ -6463,7 +6503,6 @@ dependencies = ["tree-sitter-ruby","tree-sitter-rust","tree-sitter-typescript",- "walkdir","xxhash-rust","zip 2.4.2",]diff --git a/engine/crates/xerj-autoindex/Cargo.toml b/engine/crates/xerj-autoindex/Cargo.tomlindex 55283c9e9..7f24715e1 100644--- a/engine/crates/xerj-autoindex/Cargo.toml+++ b/engine/crates/xerj-autoindex/Cargo.toml@@ -12,7 +12,9 @@ serde.workspace = trueserde_json.workspace = truechrono.workspace = trueregex = "1"-walkdir = "2"+# Directory walker with .gitignore semantics (ripgrep's). Replaces walkdir:+# same recursive traversal, plus owner-declared ignore rules.+ignore = "0.4"encoding_rs = "0.8"xxhash-rust.workspace = truememchr.workspace = truediff --git a/engine/crates/xerj-autoindex/src/cli.rs b/engine/crates/xerj-autoindex/src/cli.rsindex efe4ac8ef..aed7476c9 100644--- a/engine/crates/xerj-autoindex/src/cli.rs+++ b/engine/crates/xerj-autoindex/src/cli.rs@@ -18,6 +18,12 @@ pub struct IndexCfg {pub follow_symlinks: bool,/// Disable the marker-gated generated-dir prune (e.g. Unity `Library/`).pub no_default_excludes: bool,+ /// `--stub <glob>` patterns: matching files are indexed as one+ /// existence-only name card, contents never opened.+ pub stub_globs: Vec<String>,+ /// Walk `.gitignore`d files too (by default they are skipped when the+ /// root is inside a git repository).+ pub no_gitignore: bool,pub max_file_gb: u64,pub sample: usize,pub no_semantic: bool,@@ -86,6 +92,17 @@ pub fn print_help() {UserSettings next to ProjectSettings/\n\ProjectVersion.txt, node_modules next to\n\package.json, target next to Cargo.toml\n\+ --no-gitignore index .gitignore'd files too. By default, when the\n\+ root is inside a git repository, files matched by\n\+ the tree's .gitignore rules are skipped (the\n\+ owner's own declaration of generated noise);\n\+ the user's global gitignore is never consulted\n\+ --stub <GLOB> index matching files as ONE existence-only name\n\+ card (title + provenance); contents are never\n\+ opened. Repeatable. A pattern without '/'\n\+ matches file names anywhere ('*.bvh'); with '/'\n\+ it matches the root-relative path\n\+ ('unity/**/*.csv'). '**' crosses directories\n\--max-file-gb <N> skip+record oversized non-streamable files (default 2)\n\--sample <N> records sampled per file for inference (default 500)\n\--no-semantic skip semantic_text on body fields (pure BM25+keyword)\n\@@ -148,6 +165,8 @@ pub fn parse(args: Vec<String>) -> Result<Cmd, String> {let mut fresh = false;let mut follow_symlinks = false;let mut no_default_excludes = false;+ let mut stub_globs: Vec<String> = Vec::new();+ let mut no_gitignore = false;let mut max_file_gb = 2u64;let mut sample = 500usize;let mut no_semantic = false;@@ -207,6 +226,8 @@ pub fn parse(args: Vec<String>) -> Result<Cmd, String> {"--fresh" => fresh = true,"--follow-symlinks" => follow_symlinks = true,"--no-default-excludes" => no_default_excludes = true,+ "--stub" => stub_globs.push(it.next().ok_or("--stub needs a glob pattern")?),+ "--no-gitignore" => no_gitignore = true,"--max-file-gb" => {max_file_gb = it.next()@@ -283,6 +304,8 @@ pub fn parse(args: Vec<String>) -> Result<Cmd, String> {fresh,follow_symlinks,no_default_excludes,+ stub_globs,+ no_gitignore,max_file_gb,sample: sample.max(50),no_semantic,diff --git a/engine/crates/xerj-autoindex/src/content.rs b/engine/crates/xerj-autoindex/src/content.rsindex 4111e7b22..3bf0d45fb 100644--- a/engine/crates/xerj-autoindex/src/content.rs+++ b/engine/crates/xerj-autoindex/src/content.rs@@ -187,7 +187,7 @@ mod tests {let dir = tempfile::tempdir().unwrap();fs::write(dir.path().join("a.txt"), b"same records\n").unwrap();fs::write(dir.path().join("b.txt"), b"same records\n").unwrap();- let inv = resolve(crate::walk::walk(dir.path(), false, true).unwrap().0).unwrap();+ let inv = resolve(crate::walk::walk(dir.path(), false, true, true).unwrap().0).unwrap();assert_eq!(inv.files.len(), 1);assert_eq!(inv.files[0].rel, "a.txt");assert_eq!(inv.duplicates.len(), 1);@@ -204,7 +204,7 @@ mod tests {second[65_536] = b'b';fs::write(dir.path().join("a.txt"), first).unwrap();fs::write(dir.path().join("b.txt"), second).unwrap();- let inv = resolve(crate::walk::walk(dir.path(), false, true).unwrap().0).unwrap();+ let inv = resolve(crate::walk::walk(dir.path(), false, true, true).unwrap().0).unwrap();assert_eq!(inv.files.len(), 2);assert!(inv.duplicates.is_empty());assert_ne!(inv.keys[0], inv.keys[1]);@@ -215,7 +215,7 @@ mod tests {let dir = tempfile::tempdir().unwrap();let path = dir.path().join("a.txt");fs::write(&path, b"content-a").unwrap();- let inv = resolve(crate::walk::walk(dir.path(), false, true).unwrap().0).unwrap();+ let inv = resolve(crate::walk::walk(dir.path(), false, true, true).unwrap().0).unwrap();fs::write(&path, b"content-b").unwrap();assert!(verify(&path, 9, &inv.digests[0]).is_err());}@@ -225,7 +225,7 @@ mod tests {let dir = tempfile::tempdir().unwrap();fs::write(dir.path().join("a.txt"), b"unequal-a").unwrap();fs::write(dir.path().join("b.txt"), b"unequal-b").unwrap();- let files = crate::walk::walk(dir.path(), false, true).unwrap().0;+ let files = crate::walk::walk(dir.path(), false, true, true).unwrap().0;let forced = vec!["axf2-forced-9".to_string(); 2];let inv = resolve_with_digests(files, forced).unwrap();assert_eq!(inv.files.len(), 2);@@ -242,7 +242,7 @@ mod tests {let b = dir.path().join("b.txt");fs::write(&a, b"hardlinked").unwrap();fs::hard_link(&a, &b).unwrap();- let inv = resolve(crate::walk::walk(dir.path(), false, true).unwrap().0).unwrap();+ let inv = resolve(crate::walk::walk(dir.path(), false, true, true).unwrap().0).unwrap();assert_eq!(inv.files.len(), 1);assert_eq!(inv.duplicates.len(), 1);}@@ -264,7 +264,7 @@ mod tests {b"same",).unwrap();- let inv = resolve(crate::walk::walk(dir.path(), false, true).unwrap().0).unwrap();+ let inv = resolve(crate::walk::walk(dir.path(), false, true, true).unwrap().0).unwrap();assert_eq!(inv.files.len(), 1);assert_eq!(inv.duplicates.len(), 2);assert_ne!(inv.duplicates[0].path_id, inv.duplicates[1].path_id);@@ -278,7 +278,7 @@ mod tests {let real = dir.path().join("z-real");fs::write(&real, b"same").unwrap();symlink(&real, dir.path().join("a-link")).unwrap();- let inv = resolve(crate::walk::walk(dir.path(), true, true).unwrap().0).unwrap();+ let inv = resolve(crate::walk::walk(dir.path(), true, true, true).unwrap().0).unwrap();assert_eq!(inv.files[0].rel, "z-real");assert_eq!(inv.duplicates[0].rel, "a-link");}diff --git a/engine/crates/xerj-autoindex/src/detect/e2e.rs b/engine/crates/xerj-autoindex/src/detect/e2e.rsindex 30b95dede..15684ef61 100644--- a/engine/crates/xerj-autoindex/src/detect/e2e.rs+++ b/engine/crates/xerj-autoindex/src/detect/e2e.rs@@ -271,6 +271,8 @@ fn cfg(root: &Path, state_dir: &Path, url: &str) -> IndexCfg {fresh: false,follow_symlinks: false,no_default_excludes: false,+ stub_globs: Vec::new(),+ no_gitignore: false,max_file_gb: 1,sample: 50,no_semantic: true,diff --git a/engine/crates/xerj-autoindex/src/extract/bvh.rs b/engine/crates/xerj-autoindex/src/extract/bvh.rsnew file mode 100644index 000000000..f835f907f--- /dev/null+++ b/engine/crates/xerj-autoindex/src/extract/bvh.rs@@ -0,0 +1,175 @@+//! BVH motion capture — ONE metadata record per file, motion data never read.+//!+//! A BVH file is a small skeleton HIERARCHY header followed by a numeric+//! MOTION block that can run to hundreds of MB (one line per frame). The+//! frames carry no retrieval value — they are float soup — but the header+//! does: joint names make the skeleton searchable ("which clips animate+//! LeftHand?"), and frame count/time give the clip's duration. So this+//! extractor streams the header, captures that metadata, and STOPS as soon+//! as the two `Frames:`/`Frame Time:` lines after `MOTION` are read — the+//! motion block is never pulled off disk.+//!+//! Emitted fields: `title`, `joints` (names, document order), `joint_count`,+//! `frames`, `frame_time_s`, `duration_s`. All extractor vocabulary+//! (`FieldOrigin::Extractor`), so every BVH file clusters into one dataset.++use super::{ExtractStats, FieldOrigin, RawRecord, Sink};+use anyhow::Result;+use serde_json::{Map, Value};+use std::path::Path;++/// Skeleton headers are a few KB; a "header" that runs past this is not a+/// BVH we understand, and the file junks rather than streaming megabytes.+const HEADER_CAP: usize = 1 << 20;++pub fn extract(path: &Path, gzip: bool, sink: Sink) -> Result<ExtractStats> {+ let mut stats = ExtractStats::default();+ let mut r = super::open_reader(path, gzip, None)?;+ let mut joints: Vec<String> = Vec::new();+ let mut frames: Option<i64> = None;+ let mut frame_time: Option<f64> = None;+ let mut in_motion = false;+ let mut consumed = 0usize;+ let mut line: Vec<u8> = Vec::new();+ loop {+ line.clear();+ let n = super::jsonl::read_capped_line(&mut r, &mut line)?;+ if n == 0 {+ break;+ }+ consumed += n;+ if consumed > HEADER_CAP {+ stats.junk += 1;+ return Ok(stats);+ }+ let text = String::from_utf8_lossy(&line);+ let t = text.trim();+ if !in_motion {+ if let Some(name) = t+ .strip_prefix("ROOT ")+ .or_else(|| t.strip_prefix("JOINT "))+ {+ let name = name.trim();+ if !name.is_empty() && name.len() <= 200 {+ joints.push(name.to_string());+ }+ } else if t == "MOTION" {+ in_motion = true;+ }+ continue;+ }+ if let Some(v) = t.strip_prefix("Frames:") {+ frames = v.trim().parse().ok();+ } else if let Some(v) = t.strip_prefix("Frame Time:") {+ frame_time = v.trim().parse().ok();+ }+ if frames.is_some() && frame_time.is_some() {+ break; // metadata complete — the motion block is never read+ }+ }+ if joints.is_empty() {+ stats.junk += 1;+ return Ok(stats);+ }++ let title = path+ .file_name()+ .and_then(|n| n.to_str())+ .unwrap_or("motion")+ .to_string();+ let mut fields = Map::new();+ fields.insert("title".into(), Value::String(title));+ fields.insert(+ "joints".into(),+ Value::Array(joints.iter().map(|j| Value::String(j.clone())).collect()),+ );+ fields.insert(+ "joint_count".into(),+ Value::Number((joints.len() as u64).into()),+ );+ if let Some(f) = frames {+ fields.insert("frames".into(), Value::Number(f.into()));+ }+ if let Some(ft) = frame_time {+ if let Some(num) = serde_json::Number::from_f64(ft) {+ fields.insert("frame_time_s".into(), Value::Number(num));+ }+ if let (Some(f), true) = (frames, ft.is_finite()) {+ if let Some(num) = serde_json::Number::from_f64(f as f64 * ft) {+ fields.insert("duration_s".into(), Value::Number(num));+ }+ }+ }+ stats.records += 1;+ sink(RawRecord {+ fields,+ locator: "bvh".into(),+ group: None,+ origin: FieldOrigin::Extractor,+ });+ Ok(stats)+}++#[cfg(test)]+mod tests {+ use super::*;++ const SAMPLE: &str = "HIERARCHY\nROOT Hips\n{\n OFFSET 0 90 0\n CHANNELS 6 Xposition Yposition Zposition Zrotation Xrotation Yrotation\n JOINT LeftUpLeg\n {\n OFFSET 8.5 0 -2.5\n CHANNELS 3 Zrotation Xrotation Yrotation\n End Site\n {\n OFFSET 6.9 0 0\n }\n }\n}\nMOTION\nFrames: 120\nFrame Time: 0.033333\n0.0 90.0 0.0 1.1 2.2 3.3 4.4 5.5 6.6\n0.1 90.1 0.1 1.2 2.3 3.4 4.5 5.6 6.7\n";++ fn run(text: &str) -> (ExtractStats, Vec<RawRecord>) {+ let dir = tempfile::tempdir().unwrap();+ let path = dir.path().join("clip.bvh");+ std::fs::write(&path, text).unwrap();+ let mut recs = Vec::new();+ let stats = extract(&path, false, &mut |r| {+ recs.push(r);+ true+ })+ .unwrap();+ (stats, recs)+ }++ #[test]+ fn one_metadata_record_per_file_and_no_motion_rows() {+ let (stats, recs) = run(SAMPLE);+ assert_eq!((stats.records, stats.junk), (1, 0));+ let r = &recs[0];+ assert_eq!(r.fields["title"], "clip.bvh");+ assert_eq!(r.fields["joints"], serde_json::json!(["Hips", "LeftUpLeg"]));+ assert_eq!(r.fields["joint_count"], 2);+ assert_eq!(r.fields["frames"], 120);+ assert_eq!(r.fields["duration_s"].as_f64().unwrap().round(), 4.0);+ assert_eq!(r.locator, "bvh");+ }++ /// The point of the format: a clip with a multi-GB motion block must cost+ /// only its header. The reader stops after `Frame Time:`, so the frames+ /// after it are never even pulled through the BufReader into the record.+ #[test]+ fn extraction_stops_at_the_motion_header_not_the_motion_data() {+ let mut text = SAMPLE.to_string();+ text.push_str(&"9.9 ".repeat(500_000));+ let (stats, recs) = run(&text);+ assert_eq!(stats.records, 1);+ assert_eq!(recs[0].fields["frames"], 120);+ }++ #[test]+ fn a_hierarchy_without_joints_is_junk_not_an_empty_record() {+ let (stats, recs) = run("HIERARCHY\nMOTION\nFrames: 1\nFrame Time: 0.03\n1 2 3\n");+ assert_eq!(stats.records, 0);+ assert_eq!(stats.junk, 1);+ assert!(recs.is_empty());+ }++ #[test]+ fn a_runaway_header_is_junk_rather_than_streamed_forever() {+ let mut text = String::from("HIERARCHY\nROOT Hips\n{\n");+ for i in 0..40_000 {+ text.push_str(&format!(" JOINT J{i}\n {{\n OFFSET 0 0 0\n"));+ }+ let (stats, _) = run(&text);+ assert_eq!(stats.records, 0);+ assert_eq!(stats.junk, 1);+ }+}diff --git a/engine/crates/xerj-autoindex/src/extract/mod.rs b/engine/crates/xerj-autoindex/src/extract/mod.rsindex a3c8c6199..ac0f45fa1 100644--- a/engine/crates/xerj-autoindex/src/extract/mod.rs+++ b/engine/crates/xerj-autoindex/src/extract/mod.rs@@ -2,6 +2,7 @@//! Every extractor is bounded-memory and never fatal: parse failures//! downgrade (family → txt → junk-with-metadata) and are counted.+pub mod bvh;pub mod code;pub mod csv_x;pub mod docx;@@ -142,6 +143,26 @@ pub fn extract(Family::Code => code::extract(path, sink),Family::UnityYaml => unity::extract_unity(path, sn.gzip, limit_bytes, sink),Family::UnityMeta => unity::extract_meta(path, sn.gzip, sink),+ Family::Bvh => bvh::extract(path, sn.gzip, sink),+ // `--stub`-designated file: one name card, the file is never opened.+ Family::Stub => {+ let mut stats = ExtractStats::default();+ let mut fields = Map::new();+ let title = path+ .file_name()+ .and_then(|n| n.to_str())+ .unwrap_or("file")+ .to_string();+ fields.insert("title".into(), Value::String(title));+ stats.records += 1;+ sink(RawRecord {+ fields,+ locator: "stub".into(),+ group: None,+ origin: FieldOrigin::Extractor,+ });+ Ok(stats)+ }Family::Binary => Ok(ExtractStats::default()),}}diff --git a/engine/crates/xerj-autoindex/src/failure_resume_http_tests.rs b/engine/crates/xerj-autoindex/src/failure_resume_http_tests.rsindex 7be372a48..84481d716 100644--- a/engine/crates/xerj-autoindex/src/failure_resume_http_tests.rs+++ b/engine/crates/xerj-autoindex/src/failure_resume_http_tests.rs@@ -205,6 +205,8 @@ fn cfg(root: &Path, state_dir: &Path, url: &str) -> IndexCfg {fresh: false,follow_symlinks: false,no_default_excludes: false,+ stub_globs: Vec::new(),+ no_gitignore: false,max_file_gb: 1,sample: 50,no_semantic: true,diff --git a/engine/crates/xerj-autoindex/src/lib.rs b/engine/crates/xerj-autoindex/src/lib.rsindex 3aa83fe4f..1a3c29e54 100644--- a/engine/crates/xerj-autoindex/src/lib.rs+++ b/engine/crates/xerj-autoindex/src/lib.rs@@ -173,6 +173,137 @@ fn section_label(locator: &str) -> Option<String> {(digits(page) && digits(sec)).then(|| format!("page {page} section {sec}"))}+// ─── --stub glob matcher ──────────────────────────────────────────────────++/// Compiled `--stub <glob>` patterns. A matching file is indexed as ONE+/// name-card record (`Family::Stub`) and its contents are never opened —+/// the owner's way of saying "this data blob should be referenceable but+/// not parsed" without the engine hardcoding per-corpus rules.+///+/// Glob semantics (gitignore-flavored): `**` crosses `/`, `*` and `?` do+/// not; a pattern without `/` matches against the file NAME anywhere in the+/// tree, a pattern with `/` matches the full root-relative path.+pub struct StubMatcher {+ by_name: Vec<regex::Regex>,+ by_path: Vec<regex::Regex>,+}++impl StubMatcher {+ pub fn compile(globs: &[String]) -> Result<Self> {+ let mut by_name = Vec::new();+ let mut by_path = Vec::new();+ for g in globs {+ let re = regex::Regex::new(&glob_to_regex(g))+ .with_context(|| format!("--stub {g}: invalid pattern"))?;+ if g.contains('/') {+ by_path.push(re);+ } else {+ by_name.push(re);+ }+ }+ Ok(Self { by_name, by_path })+ }++ pub fn is_empty(&self) -> bool {+ self.by_name.is_empty() && self.by_path.is_empty()+ }++ /// `rel` is the root-relative path with forward slashes.+ pub fn matches(&self, rel: &str) -> bool {+ if self.by_path.iter().any(|re| re.is_match(rel)) {+ return true;+ }+ if self.by_name.is_empty() {+ return false;+ }+ let name = rel.rsplit('/').next().unwrap_or(rel);+ self.by_name.iter().any(|re| re.is_match(name))+ }+}++fn glob_to_regex(glob: &str) -> String {+ let mut out = String::from("^");+ let mut chars = glob.chars().peekable();+ while let Some(c) = chars.next() {+ match c {+ '*' => {+ if chars.peek() == Some(&'*') {+ chars.next();+ // `**/` also swallows its slash so `**/x` matches a+ // top-level `x`.+ if chars.peek() == Some(&'/') {+ chars.next();+ out.push_str("(?:.*/)?");+ } else {+ out.push_str(".*");+ }+ } else {+ out.push_str("[^/]*");+ }+ }+ '?' => out.push_str("[^/]"),+ other => out.push_str(®ex::escape(&other.to_string())),+ }+ }+ out.push('$');+ out+}++/// The synthetic sniff result for a `--stub`-designated file.+fn stub_sniffed() -> Sniffed {+ Sniffed {+ family: Family::Stub,+ gzip: false,+ binary_kind: None,+ csv: None,+ encoding: "utf-8",+ }+}++#[cfg(test)]+mod stub_matcher_tests {+ use super::StubMatcher;++ fn m(globs: &[&str]) -> StubMatcher {+ StubMatcher::compile(&globs.iter().map(|s| s.to_string()).collect::<Vec<_>>()).unwrap()+ }++ #[test]+ fn a_bare_pattern_matches_file_names_anywhere() {+ let s = m(&["*.csv"]);+ assert!(s.matches("unity/Assets/Face/f_roommate_004.csv"));+ assert!(s.matches("top.csv"));+ assert!(!s.matches("unity/Assets/notes.csv.md"));+ }++ #[test]+ fn a_path_pattern_matches_the_root_relative_path() {+ let s = m(&["unity/**/*.csv"]);+ assert!(s.matches("unity/Assets/Face/f_roommate_004.csv"));+ assert!(s.matches("unity/top.csv"), "**/ also matches zero dirs");+ assert!(!s.matches("backend/data/users.csv"), "scoped to unity/");+ }++ #[test]+ fn single_star_does_not_cross_directories() {+ let s = m(&["unity/*.csv"]);+ assert!(s.matches("unity/top.csv"));+ assert!(!s.matches("unity/Assets/deep.csv"));+ }++ #[test]+ fn regex_metacharacters_in_patterns_are_literal() {+ let s = m(&["data(v1).csv"]);+ assert!(s.matches("x/data(v1).csv"));+ assert!(!s.matches("x/dataXv1Y.csv"));+ }++ #[test]+ fn an_invalid_pattern_fails_loudly_at_startup() {+ assert!(StubMatcher::compile(&["ok.csv".into()]).is_ok());+ }+}+// ─── Phase A: per-file scan (sniff + bounded sampling) ───────────────────struct FileScan {@@ -191,17 +322,21 @@ struct GroupSketch {records: u64,}-fn scan_file(path: &Path, size: u64, sample: usize, max_file_gb: u64) -> FileScan {+fn scan_file(path: &Path, size: u64, sample: usize, max_file_gb: u64, stub: bool) -> FileScan {let mut out = FileScan {sniffed: None,sketches: Vec::new(),junk: None,};- let sn = match sniff::sniff(path) {- Ok(s) => s,- Err(e) => {- out.junk = Some(("junk".into(), format!("unreadable: {e}")));- return out;+ let sn = if stub {+ stub_sniffed()+ } else {+ match sniff::sniff(path) {+ Ok(s) => s,+ Err(e) => {+ out.junk = Some(("junk".into(), format!("unreadable: {e}")));+ return out;+ }}};if sn.family == Family::Binary {@@ -315,7 +450,7 @@ mod clustering_key_tests {let path = dir.join(name);std::fs::write(&path, body).unwrap();let size = std::fs::metadata(&path).unwrap().len();- scan_file(&path, size, 500, 2)+ scan_file(&path, size, 500, 2, false)}/// The #178 mechanism, from the extractor to the clustering key: a source@@ -675,8 +810,14 @@ pub fn run_index_report(cfg: IndexCfg) -> Result<(i32, Option<Value>)> {let es = Es::with_bulk_timeout(&cfg.url, cfg.api_key.clone(), cfg.bulk_timeout_secs)?;es.ping()?;+ let stub_matcher = StubMatcher::compile(&cfg.stub_globs)?;let (discovered_files, skipped_dirs) =- walk::walk(&cfg.root, cfg.follow_symlinks, !cfg.no_default_excludes)?;+ walk::walk(+ &cfg.root,+ cfg.follow_symlinks,+ !cfg.no_default_excludes,+ !cfg.no_gitignore,+ )?;if !skipped_dirs.is_empty() && !cfg.quiet {let names: Vec<&str> = skipped_dirs.iter().map(|s| s.rel.as_str()).collect();eprintln!(@@ -886,7 +1027,15 @@ pub fn run_index_report(cfg: IndexCfg) -> Result<(i32, Option<Value>)> {}let scans: Vec<FileScan> = files.par_iter()- .map(|f| scan_file(&f.path, f.size, cfg.sample, cfg.max_file_gb))+ .map(|f| {+ scan_file(+ &f.path,+ f.size,+ cfg.sample,+ cfg.max_file_gb,+ stub_matcher.matches(&f.rel),+ )+ }).collect();let rels: Vec<String> = files.iter().map(|f| f.rel.clone()).collect();@@ -1352,7 +1501,12 @@ pub fn run_index_report(cfg: IndexCfg) -> Result<(i32, Option<Value>)> {let fa = plan.files.get(key).unwrap();let asg: HashMap<Option<String>, String> =fa.assignments.iter().cloned().collect();- let sn = match sniff::sniff(&f.path) {+ let sn = if stub_matcher.matches(&f.rel) {+ Ok(stub_sniffed())+ } else {+ sniff::sniff(&f.path)+ };+ let sn = match sn {Ok(s) => s,Err(e) => {extra_junk.lock().unwrap().push(JunkFile {@@ -2497,7 +2651,7 @@ mod duplicate_integration_tests {b[65_536] = b'b';fs::write(corpus.path().join("a.txt"), a).unwrap();fs::write(corpus.path().join("b.txt"), b).unwrap();- let files = walk::walk(corpus.path(), false, true).unwrap().0;+ let files = walk::walk(corpus.path(), false, true, true).unwrap().0;let inventory = content::resolve(files.clone()).unwrap();let legacy = ids::file_key(&files[0].path, files[0].size).unwrap();assert_eq!(@@ -2534,7 +2688,7 @@ mod duplicate_integration_tests {// with — so b.txt's content key IS the planned key a.txt claims by rel.fs::write(corpus.path().join("a.txt"), b"rewritten content\n").unwrap();fs::write(corpus.path().join("b.txt"), b"original planned content\n").unwrap();- let inventory = content::resolve(walk::walk(corpus.path(), false, true).unwrap().0).unwrap();+ let inventory = content::resolve(walk::walk(corpus.path(), false, true, true).unwrap().0).unwrap();let planned_key = inventory.keys[1].clone();let mut plan = Plan::default();plan.files@@ -2604,7 +2758,7 @@ mod duplicate_integration_tests {fs::write(corpus.path().join("report-original.txt"), body).unwrap();fs::write(corpus.path().join("report-copy.txt"), body).unwrap();- let discovered = walk::walk(corpus.path(), false, true).unwrap().0;+ let discovered = walk::walk(corpus.path(), false, true, true).unwrap().0;let inventory = content::resolve(discovered).unwrap();assert_eq!(inventory.files.len(), 1);assert_eq!(inventory.duplicates.len(), 1);@@ -2677,7 +2831,7 @@ mod duplicate_integration_tests {}fs::write(&path, csv).unwrap();- let inventory = content::resolve(walk::walk(corpus.path(), false, true).unwrap().0).unwrap();+ let inventory = content::resolve(walk::walk(corpus.path(), false, true, true).unwrap().0).unwrap();let expected_size = inventory.files[0].size;let expected_digest = inventory.digests[0].clone();let sniffed = sniff::sniff(&path).unwrap();diff --git a/engine/crates/xerj-autoindex/src/sniff.rs b/engine/crates/xerj-autoindex/src/sniff.rsindex 0dbf3a489..49f31bdfb 100644--- a/engine/crates/xerj-autoindex/src/sniff.rs+++ b/engine/crates/xerj-autoindex/src/sniff.rs@@ -30,6 +30,15 @@ pub enum Family {/// Unity `.meta` sidecar: plain YAML opening with `fileFormatVersion:`/// and carrying the asset `guid` — the join key for everything Unity.UnityMeta,+ /// Biovision motion capture: a skeleton HIERARCHY header followed by a+ /// large numeric MOTION block. Indexed as ONE metadata record per file+ /// (joints, frame count, duration) — the motion numbers are never read.+ Bvh,+ /// User-designated existence-only file (`--stub <glob>`): ONE name-card+ /// record, contents never opened. For corpus-specific data blobs the+ /// owner wants referenceable but not parsed — never assigned by+ /// sniffing, only by the CLI flag.+ Stub,Binary,}@@ -52,6 +61,8 @@ impl Family {Family::Code => "code",Family::UnityYaml => "unity",Family::UnityMeta => "unity-meta",+ Family::Bvh => "bvh",+ Family::Stub => "stub",Family::Binary => "binary",}}@@ -224,6 +235,19 @@ fn sniff_bytes(prefix: &[u8], path: &Path, gzip: bool) -> Result<Sniffed> {s.encoding = encoding;return Ok(s);}+ // BVH motion capture: `HIERARCHY` opener with a `ROOT <name>` next.+ // Without this the numeric MOTION block classified as txt-lines and+ // indexed millions of meaningless number rows.+ if first_line.trim() == "HIERARCHY"+ && body+ .lines()+ .nth(1)+ .is_some_and(|l| l.trim_start().starts_with("ROOT "))+ {+ let mut s = mk(Family::Bvh);+ s.encoding = encoding;+ return Ok(s);+ }}// 2c. Source code: a known code extension whose content is text. We only@@ -624,6 +648,19 @@ mod unity_sniff_tests {assert_ne!(sniff_str(no_guid, "x.meta"), Family::UnityMeta);}+ #[test]+ fn bvh_is_detected_by_hierarchy_root_header() {+ let bvh = "HIERARCHY\nROOT Hips\n{\n OFFSET 0 90 0\n CHANNELS 6 Xposition Yposition Zposition Zrotation Xrotation Yrotation\n}\nMOTION\nFrames: 2\nFrame Time: 0.033\n1 2 3\n4 5 6\n";+ assert_eq!(sniff_str(bvh, "clip.bvh"), Family::Bvh);+ assert_eq!(+ sniff_str(bvh, "clip.txt"),+ Family::Bvh,+ "content decides, never the extension"+ );+ let not_bvh = "HIERARCHY\nof needs (Maslow):\n- physiological\n- safety\n";+ assert_ne!(sniff_str(not_bvh, "notes.txt"), Family::Bvh);+ }+#[test]fn plain_yaml_without_the_unity_tag_stays_yaml() {let plain = "%YAML 1.2\n---\nkey: value\nother: 1\nnested:\n a: 2\n";diff --git a/engine/crates/xerj-autoindex/src/walk.rs b/engine/crates/xerj-autoindex/src/walk.rsindex e316aafc0..bdfd7fd55 100644--- a/engine/crates/xerj-autoindex/src/walk.rs+++ b/engine/crates/xerj-autoindex/src/walk.rs@@ -1,5 +1,14 @@//! Inventory: recursive walk of the target folder.-//! Symlinks are NOT followed by default; with --follow-symlinks walkdir's+//!+//! Built on the `ignore` crate (ripgrep's walker), which gives three skip+//! layers, all evidence-based:+//! 1. hidden files/dirs — always (security: `.env`, `.git`, `.ssh`, …);+//! 2. `.gitignore` rules — the corpus OWNER's own declaration of generated+//! noise; honored only inside a real git repo (`.git` present), and+//! disabled with `--no-gitignore`;+//! 3. marker-gated generated dirs (`GENERATED_DIR_RULES`) — for corpora+//! that are not git repos or whose ignore files are incomplete.+//! Symlinks are NOT followed by default; with --follow-symlinks the walker's//! ancestor-loop detection keeps the traversal loop-safe.use anyhow::{Context, Result};@@ -75,6 +84,7 @@ pub fn walk(root: &Path,follow_symlinks: bool,default_excludes: bool,+ gitignore: bool,) -> Result<(Vec<FileEntry>, Vec<SkippedDir>)> {let root_canon = root.canonicalize()@@ -82,28 +92,39 @@ pub fn walk(if !root_canon.is_dir() {anyhow::bail!("{} is not a directory", root_canon.display());}- let skipped: std::rc::Rc<std::cell::RefCell<Vec<SkippedDir>>> = Default::default();+ // Arc<Mutex>: the `ignore` walker's filter must be Send+Sync even for the+ // sequential build() we use.+ let skipped: std::sync::Arc<std::sync::Mutex<Vec<SkippedDir>>> = Default::default();let mut out = Vec::new();// SECURITY / hygiene: never index hidden files or descend into hidden- // directories. Without this the walker happily indexed `.env` (secrets,- // API tokens), `.git`, `.ssh`, `.aws`, and other dotfiles into a queryable- // brain with no per-brain authorization — a real exposure for the- // "point it at my project folder" use case. `filter_entry` prunes a hidden- // directory before descending, so `.git/` is skipped whole. The root itself- // (depth 0) is exempt so a brain over a dot-named folder still works.- let sk = std::rc::Rc::clone(&skipped);+ // directories (`hidden(true)`). Without this the walker happily indexed+ // `.env` (secrets, API tokens), `.git`, `.ssh`, `.aws`, and other dotfiles+ // into a queryable brain with no per-brain authorization — a real exposure+ // for the "point it at my project folder" use case. The root itself is+ // exempt (the walk origin is always yielded) so a brain over a dot-named+ // folder still works.+ //+ // `.gitignore` handling stays deterministic and corpus-local: tree+ // `.gitignore`s + `.git/info/exclude` only, never the user's global+ // gitignore config; `require_git` keeps the semantics evidence-based (a+ // stray .gitignore outside a repo is not an owner declaration).+ let sk = std::sync::Arc::clone(&skipped);let filter_root = root_canon.clone();- let walker = walkdir::WalkDir::new(&root_canon)+ let mut builder = ignore::WalkBuilder::new(&root_canon);+ builder.follow_links(follow_symlinks)- .into_iter()+ .hidden(true)+ .ignore(false)+ .parents(gitignore)+ .git_ignore(gitignore)+ .git_exclude(gitignore)+ .git_global(false)+ .require_git(true).filter_entry(move |e| {if e.depth() == 0 {return true;}- if e.file_name().to_str().is_some_and(|n| n.starts_with('.')) {- return false;- }- if default_excludes && e.file_type().is_dir() {+ if default_excludes && e.file_type().is_some_and(|t| t.is_dir()) {for (name, marker, reason) in GENERATED_DIR_RULES {if e.file_name() == *name&& e.path()@@ -116,7 +137,7 @@ pub fn walk(.unwrap_or(e.path()).to_string_lossy().replace('\\', "/");- sk.borrow_mut().push(SkippedDir {+ sk.lock().expect("skip list lock").push(SkippedDir {rel,reason: (*reason).to_string(),});@@ -126,7 +147,7 @@ pub fn walk(}true});- for entry in walker {+ for entry in builder.build() {let entry = match entry {Ok(e) => e,Err(e) => {@@ -134,7 +155,7 @@ pub fn walk(continue;}};- if !entry.file_type().is_file() {+ if !entry.file_type().is_some_and(|t| t.is_file()) {continue;}let md = match entry.metadata() {@@ -152,18 +173,15 @@ pub fn walk(path: p,rel,rel_id,- is_symlink: entry- .path()- .symlink_metadata()- .is_ok_and(|m| m.file_type().is_symlink()),+ is_symlink: entry.path_is_symlink(),size: md.len(),});}// Deterministic order for clustering / naming.out.sort_by(|a, b| a.rel.cmp(&b.rel));- let skipped = std::rc::Rc::try_unwrap(skipped)- .expect("walker iterator consumed; sole owner remains")- .into_inner();+ // The builder keeps its own clone of the filter closure (and thus of the+ // Arc), so drain under the lock instead of unwrapping ownership.+ let skipped = std::mem::take(&mut *skipped.lock().expect("skip list lock"));Ok((out, skipped))}@@ -211,7 +229,7 @@ mod hidden_skip_tests {fs::write(root.join("src").join("main.rs"), "fn main() {}").unwrap();fs::write(root.join("src").join(".secret"), "nope").unwrap();- let rels: Vec<String> = walk(root, false, true)+ let rels: Vec<String> = walk(root, false, true, true).unwrap().0.into_iter()@@ -237,7 +255,7 @@ mod hidden_skip_tests {let root = dir.path().join(".notes");fs::create_dir(&root).unwrap();fs::write(root.join("a.md"), "x").unwrap();- let rels: Vec<String> = walk(&root, false, true)+ let rels: Vec<String> = walk(&root, false, true, true).unwrap().0.into_iter()@@ -278,7 +296,7 @@ mod generated_dir_tests {fn unity_marker_prunes_generated_dirs_and_records_them() {let dir = tempfile::TempDir::new().unwrap();make_unity_root(dir.path(), true);- let (files, skipped) = walk(dir.path(), false, true).unwrap();+ let (files, skipped) = walk(dir.path(), false, true, true).unwrap();let rels: Vec<&str> = files.iter().map(|e| e.rel.as_str()).collect();assert!(rels.contains(&"Assets/Player.cs"));assert!(@@ -295,7 +313,7 @@ mod generated_dir_tests {fn without_the_marker_nothing_is_pruned() {let dir = tempfile::TempDir::new().unwrap();make_unity_root(dir.path(), false);- let (files, skipped) = walk(dir.path(), false, true).unwrap();+ let (files, skipped) = walk(dir.path(), false, true, true).unwrap();let rels: Vec<&str> = files.iter().map(|e| e.rel.as_str()).collect();assert!(rels.iter().any(|r| r.starts_with("Library/")),@@ -308,7 +326,7 @@ mod generated_dir_tests {fn no_default_excludes_walks_everything() {let dir = tempfile::TempDir::new().unwrap();make_unity_root(dir.path(), true);- let (files, skipped) = walk(dir.path(), false, false).unwrap();+ let (files, skipped) = walk(dir.path(), false, false, true).unwrap();let rels: Vec<&str> = files.iter().map(|e| e.rel.as_str()).collect();assert!(rels.iter().any(|r| r.starts_with("Library/")));assert!(skipped.is_empty());@@ -319,7 +337,7 @@ mod generated_dir_tests {let dir = tempfile::TempDir::new().unwrap();make_unity_root(dir.path(), true);fs::write(dir.path().join("obj"), "a FILE named obj").unwrap();- let (files, _) = walk(dir.path(), false, true).unwrap();+ let (files, _) = walk(dir.path(), false, true, true).unwrap();assert!(files.iter().any(|e| e.rel == "obj"),"the prune applies to directories only"@@ -336,7 +354,7 @@ mod generated_dir_tests {fs::create_dir_all(&unity).unwrap();make_unity_root(&unity, true);fs::write(dir.path().join("README.md"), "monorepo").unwrap();- let (files, skipped) = walk(dir.path(), false, true).unwrap();+ let (files, skipped) = walk(dir.path(), false, true, true).unwrap();let rels: Vec<&str> = files.iter().map(|e| e.rel.as_str()).collect();assert!(rels.contains(&"apps/game/Assets/Player.cs"));assert!(!rels.iter().any(|r| r.contains("game/Library/")));@@ -354,7 +372,7 @@ mod generated_dir_tests {let bare = dir.path().join("data/node_modules");fs::create_dir_all(&bare).unwrap();fs::write(bare.join("notes.txt"), "not an npm tree").unwrap();- let (files, skipped) = walk(dir.path(), false, true).unwrap();+ let (files, skipped) = walk(dir.path(), false, true, true).unwrap();let rels: Vec<&str> = files.iter().map(|e| e.rel.as_str()).collect();assert!(rels.contains(&"web/index.js"));assert!(!rels.iter().any(|r| r.starts_with("web/node_modules")));@@ -366,6 +384,73 @@ mod generated_dir_tests {assert_eq!(skipped[0].rel, "web/node_modules");}+ #[test]+ fn gitignored_files_are_skipped_only_inside_a_git_repo() {+ let dir = tempfile::TempDir::new().unwrap();+ let root = dir.path();+ fs::create_dir(root.join(".git")).unwrap();+ fs::write(root.join(".gitignore"), "build/\n*.log\n").unwrap();+ fs::create_dir_all(root.join("build/intermediates")).unwrap();+ fs::write(root.join("build/intermediates/R.txt"), "generated").unwrap();+ fs::write(root.join("debug.log"), "noise").unwrap();+ fs::write(root.join("main.rs"), "fn main() {}").unwrap();+ fs::create_dir(root.join("src")).unwrap();+ fs::write(root.join("src/app.log"), "noise").unwrap();+ fs::write(root.join("src/lib.rs"), "pub fn f() {}").unwrap();++ let (files, _) = walk(root, false, true, true).unwrap();+ let rels: Vec<&str> = files.iter().map(|e| e.rel.as_str()).collect();+ assert!(rels.contains(&"main.rs"));+ assert!(rels.contains(&"src/lib.rs"));+ assert!(+ !rels.iter().any(|r| r.starts_with("build/")),+ "gitignored dir must be pruned: {rels:?}"+ );+ assert!(+ !rels.iter().any(|r| r.ends_with(".log")),+ "gitignored glob applies at every depth: {rels:?}"+ );++ // --no-gitignore restores the full walk.+ let (files, _) = walk(root, false, true, false).unwrap();+ let rels: Vec<&str> = files.iter().map(|e| e.rel.as_str()).collect();+ assert!(rels.contains(&"build/intermediates/R.txt"));+ assert!(rels.contains(&"debug.log"));+ }++ /// A `.gitignore` OUTSIDE a git repository is not an owner declaration —+ /// require_git keeps the rule evidence-based.+ #[test]+ fn a_gitignore_without_a_git_repo_is_inert() {+ let dir = tempfile::TempDir::new().unwrap();+ let root = dir.path();+ fs::write(root.join(".gitignore"), "*.log\n").unwrap();+ fs::write(root.join("debug.log"), "kept").unwrap();+ let (files, _) = walk(root, false, true, true).unwrap();+ assert!(+ files.iter().any(|e| e.rel == "debug.log"),+ "no .git dir, so gitignore semantics must not apply"+ );+ }++ /// Nested .gitignore files scope to their own subtree, like git itself.+ #[test]+ fn nested_gitignore_files_scope_to_their_subtree() {+ let dir = tempfile::TempDir::new().unwrap();+ let root = dir.path();+ fs::create_dir(root.join(".git")).unwrap();+ fs::create_dir(root.join("web")).unwrap();+ fs::write(root.join("web/.gitignore"), "dist/\n").unwrap();+ fs::create_dir_all(root.join("web/dist")).unwrap();+ fs::write(root.join("web/dist/bundle.js"), "x").unwrap();+ fs::create_dir_all(root.join("docs/dist")).unwrap();+ fs::write(root.join("docs/dist/manual.md"), "kept").unwrap();+ let (files, _) = walk(root, false, true, true).unwrap();+ let rels: Vec<&str> = files.iter().map(|e| e.rel.as_str()).collect();+ assert!(!rels.iter().any(|r| r.starts_with("web/dist/")));+ assert!(rels.contains(&"docs/dist/manual.md"));+ }+#[test]fn cargo_target_is_pruned_only_next_to_a_cargo_toml() {let dir = tempfile::TempDir::new().unwrap();@@ -377,7 +462,7 @@ mod generated_dir_tests {let plain = dir.path().join("shooting/target");fs::create_dir_all(&plain).unwrap();fs::write(plain.join("scores.csv"), "a,b\n1,2\n").unwrap();- let (files, skipped) = walk(dir.path(), false, true).unwrap();+ let (files, skipped) = walk(dir.path(), false, true, true).unwrap();let rels: Vec<&str> = files.iter().map(|e| e.rel.as_str()).collect();assert!(rels.contains(&"mycrate/main.rs"));assert!(!rels.iter().any(|r| r.starts_with("mycrate/target")));diff --git a/engine/crates/xerj-server/src/brain.rs b/engine/crates/xerj-server/src/brain.rsindex 8fd418ab9..4951aa6ba 100644--- a/engine/crates/xerj-server/src/brain.rs+++ b/engine/crates/xerj-server/src/brain.rs@@ -177,7 +177,7 @@ fn run(cfg: BrainCfg) -> Result<i32> {if !cfg.root.exists() {bail!("{} does not exist", cfg.root.display());}- let (files, _skipped_dirs) = walk::walk(&cfg.root, false, true)?;+ let (files, _skipped_dirs) = walk::walk(&cfg.root, false, true, true)?;if files.is_empty() {eprintln!("xerj brain: no files under {} — nothing to build a brain from.\n\@@ -433,6 +433,8 @@ fn index_cfg(cfg: &BrainCfg, brain: &str, api_key: Option<String>) -> IndexCfg {fresh: cfg.fresh,follow_symlinks: false,no_default_excludes: false,+ stub_globs: Vec::new(),+ no_gitignore: false,max_file_gb: 2,sample: 500,no_semantic: false,