Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

feat(autoindex): first-class Unity project indexing

Sergey Gonchar33147589838c
diff --git a/README.md b/README.md
index bd71298f0..1d6d12b9e 100644
--- a/README.md
+++ b/README.md
@@ -56,7 +56,12 @@ done in 158.1s, 593 datasets, 83103 records live, 790 junk records
Source files go through tree-sitter, so code arrives with its symbols and line
numbers instead of as flat text. CSV, JSON, JSONL, XML, YAML, SQLite, PDF, DOCX,
-HTML and common log formats are all handled.
+HTML and common log formats are all handled. Unity projects get first-class
+treatment: text-serialized scenes, prefabs and assets become one record per
+GameObject/Component, `.meta` files become a GUID↔path table, and MonoBehaviour
+records carry `script_class`/`script_path` so "which scenes use this script?"
+is a single query (binary-serialized assets need Force Text to be readable;
+generated dirs like `Library/` are auto-skipped and recorded).
## Search it
diff --git a/demo/usecases/autoindex/README.md b/demo/usecases/autoindex/README.md
index ce5526d4f..6e78f810f 100644
--- a/demo/usecases/autoindex/README.md
+++ b/demo/usecases/autoindex/README.md
@@ -104,8 +104,18 @@ walk → sniff → sample → infer → map → extract → bulk → correlate
hint at best, never trusted. Formats covered by streaming extractors:
JSON/JSONL, CSV (dialect detection: comma/semicolon/tab, decimal comma,
quoted multiline fields, BOM), structured logs (nginx CLF, app logs,
- syslog), SQL dumps, SQLite, PDF, DOCX, HTML, XML, YAML, plain text,
- gzip-compressed variants of the above.
+ syslog), SQL dumps, SQLite, PDF, DOCX, HTML, XML, YAML, Unity
+ text-serialized assets (scenes/prefabs/.asset/.mat/.anim → one record per
+ GameObject/Component with `unity_class`, `file_id`, guid references, and a
+ denormalized `script_class`/`script_path` on MonoBehaviours; `.meta`
+ sidecars → a guid↔asset-path table; detected by the `%TAG !u!` header,
+ never extension — binary-serialized Unity assets stay junk, enable Force
+ Text serialization; generated dirs are pruned and recorded when a sibling
+ marker proves them, at any nesting depth — Unity `Library/`, `Temp/`,
+ `obj/`, `Logs/`, `UserSettings/` next to
+ `ProjectSettings/ProjectVersion.txt`, `node_modules/` next to
+ `package.json`, `target/` next to `Cargo.toml`; `--no-default-excludes`
+ walks them), plain text, gzip-compressed variants of the above.
3. **Sample** — bounded per-file sampling (default 500 records,
`--sample N`); inference cost does not grow with file size.
4. **Infer** — field types (long/double/boolean/date/keyword/text), date
diff --git a/engine/crates/xerj-autoindex/src/cli.rs b/engine/crates/xerj-autoindex/src/cli.rs
index ee81b6a7b..efe4ac8ef 100644
--- a/engine/crates/xerj-autoindex/src/cli.rs
+++ b/engine/crates/xerj-autoindex/src/cli.rs
@@ -16,6 +16,8 @@ pub struct IndexCfg {
pub state_dir: Option<PathBuf>,
pub fresh: bool,
pub follow_symlinks: bool,
+ /// Disable the marker-gated generated-dir prune (e.g. Unity `Library/`).
+ pub no_default_excludes: bool,
pub max_file_gb: u64,
pub sample: usize,
pub no_semantic: bool,
@@ -76,6 +78,14 @@ pub fn print_help() {
--state-dir <PATH> resume journal location (default ~/.xerj/autoindex/<hash>/)\n\
--fresh ignore existing journal, restart (ids stay idempotent)\n\
--follow-symlinks follow symlinks (loop-safe); off by default\n\
+ --no-default-excludes\n\
+ walk generated/cache dirs too. By default,\n\
+ well-known generated dirs are skipped (and\n\
+ recorded) when a sibling marker proves them, at\n\
+ any depth: Unity Library/Temp/obj/Logs/\n\
+ UserSettings next to ProjectSettings/\n\
+ ProjectVersion.txt, node_modules next to\n\
+ package.json, target next to Cargo.toml\n\
--max-file-gb <N> skip+record oversized non-streamable files (default 2)\n\
--sample <N> records sampled per file for inference (default 500)\n\
--no-semantic skip semantic_text on body fields (pure BM25+keyword)\n\
@@ -137,6 +147,7 @@ pub fn parse(args: Vec<String>) -> Result<Cmd, String> {
let mut state_dir: Option<PathBuf> = None;
let mut fresh = false;
let mut follow_symlinks = false;
+ let mut no_default_excludes = false;
let mut max_file_gb = 2u64;
let mut sample = 500usize;
let mut no_semantic = false;
@@ -195,6 +206,7 @@ pub fn parse(args: Vec<String>) -> Result<Cmd, String> {
"--state-dir" => state_dir = it.next().map(PathBuf::from),
"--fresh" => fresh = true,
"--follow-symlinks" => follow_symlinks = true,
+ "--no-default-excludes" => no_default_excludes = true,
"--max-file-gb" => {
max_file_gb = it
.next()
@@ -270,6 +282,7 @@ pub fn parse(args: Vec<String>) -> Result<Cmd, String> {
state_dir,
fresh,
follow_symlinks,
+ no_default_excludes,
max_file_gb,
sample: sample.max(50),
no_semantic,
diff --git a/engine/crates/xerj-autoindex/src/content.rs b/engine/crates/xerj-autoindex/src/content.rs
index 4f865b47a..4111e7b22 100644
--- a/engine/crates/xerj-autoindex/src/content.rs
+++ b/engine/crates/xerj-autoindex/src/content.rs
@@ -187,7 +187,7 @@ mod tests {
let dir = tempfile::tempdir().unwrap();
fs::write(dir.path().join("a.txt"), b"same records\n").unwrap();
fs::write(dir.path().join("b.txt"), b"same records\n").unwrap();
- let inv = resolve(crate::walk::walk(dir.path(), false).unwrap()).unwrap();
+ let inv = resolve(crate::walk::walk(dir.path(), false, true).unwrap().0).unwrap();
assert_eq!(inv.files.len(), 1);
assert_eq!(inv.files[0].rel, "a.txt");
assert_eq!(inv.duplicates.len(), 1);
@@ -204,7 +204,7 @@ mod tests {
second[65_536] = b'b';
fs::write(dir.path().join("a.txt"), first).unwrap();
fs::write(dir.path().join("b.txt"), second).unwrap();
- let inv = resolve(crate::walk::walk(dir.path(), false).unwrap()).unwrap();
+ let inv = resolve(crate::walk::walk(dir.path(), false, true).unwrap().0).unwrap();
assert_eq!(inv.files.len(), 2);
assert!(inv.duplicates.is_empty());
assert_ne!(inv.keys[0], inv.keys[1]);
@@ -215,7 +215,7 @@ mod tests {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("a.txt");
fs::write(&path, b"content-a").unwrap();
- let inv = resolve(crate::walk::walk(dir.path(), false).unwrap()).unwrap();
+ let inv = resolve(crate::walk::walk(dir.path(), false, true).unwrap().0).unwrap();
fs::write(&path, b"content-b").unwrap();
assert!(verify(&path, 9, &inv.digests[0]).is_err());
}
@@ -225,7 +225,7 @@ mod tests {
let dir = tempfile::tempdir().unwrap();
fs::write(dir.path().join("a.txt"), b"unequal-a").unwrap();
fs::write(dir.path().join("b.txt"), b"unequal-b").unwrap();
- let files = crate::walk::walk(dir.path(), false).unwrap();
+ let files = crate::walk::walk(dir.path(), false, true).unwrap().0;
let forced = vec!["axf2-forced-9".to_string(); 2];
let inv = resolve_with_digests(files, forced).unwrap();
assert_eq!(inv.files.len(), 2);
@@ -242,7 +242,7 @@ mod tests {
let b = dir.path().join("b.txt");
fs::write(&a, b"hardlinked").unwrap();
fs::hard_link(&a, &b).unwrap();
- let inv = resolve(crate::walk::walk(dir.path(), false).unwrap()).unwrap();
+ let inv = resolve(crate::walk::walk(dir.path(), false, true).unwrap().0).unwrap();
assert_eq!(inv.files.len(), 1);
assert_eq!(inv.duplicates.len(), 1);
}
@@ -264,7 +264,7 @@ mod tests {
b"same",
)
.unwrap();
- let inv = resolve(crate::walk::walk(dir.path(), false).unwrap()).unwrap();
+ let inv = resolve(crate::walk::walk(dir.path(), false, true).unwrap().0).unwrap();
assert_eq!(inv.files.len(), 1);
assert_eq!(inv.duplicates.len(), 2);
assert_ne!(inv.duplicates[0].path_id, inv.duplicates[1].path_id);
@@ -278,7 +278,7 @@ mod tests {
let real = dir.path().join("z-real");
fs::write(&real, b"same").unwrap();
symlink(&real, dir.path().join("a-link")).unwrap();
- let inv = resolve(crate::walk::walk(dir.path(), true).unwrap()).unwrap();
+ let inv = resolve(crate::walk::walk(dir.path(), true, true).unwrap().0).unwrap();
assert_eq!(inv.files[0].rel, "z-real");
assert_eq!(inv.duplicates[0].rel, "a-link");
}
diff --git a/engine/crates/xerj-autoindex/src/detect/e2e.rs b/engine/crates/xerj-autoindex/src/detect/e2e.rs
index 7e76b671b..30b95dede 100644
--- a/engine/crates/xerj-autoindex/src/detect/e2e.rs
+++ b/engine/crates/xerj-autoindex/src/detect/e2e.rs
@@ -35,7 +35,12 @@ impl MockEs {
let (sd, ss) = (Arc::clone(&docs), Arc::clone(&stop));
let join = thread::spawn(move || loop {
match listener.accept() {
- Ok((stream, _)) => handle(stream, &sd),
+ Ok((stream, _)) => {
+ // BSD/macOS: accepted sockets inherit the listener's
+ // O_NONBLOCK; the handler does blocking reads.
+ stream.set_nonblocking(false).unwrap();
+ handle(stream, &sd)
+ }
Err(e) if e.kind() == std::io::ErrorKind::WouldBlock => {
if *ss.lock().unwrap() {
break;
@@ -265,6 +270,7 @@ fn cfg(root: &Path, state_dir: &Path, url: &str) -> IndexCfg {
state_dir: Some(state_dir.to_owned()),
fresh: false,
follow_symlinks: false,
+ no_default_excludes: false,
max_file_gb: 1,
sample: 50,
no_semantic: true,
diff --git a/engine/crates/xerj-autoindex/src/extract/mod.rs b/engine/crates/xerj-autoindex/src/extract/mod.rs
index b18e959f6..a3c8c6199 100644
--- a/engine/crates/xerj-autoindex/src/extract/mod.rs
+++ b/engine/crates/xerj-autoindex/src/extract/mod.rs
@@ -13,6 +13,7 @@ pub mod pdf;
pub mod sqldump;
pub mod sqlite_x;
pub mod txt;
+pub mod unity;
pub mod xml_x;
pub mod yaml_x;
@@ -139,6 +140,8 @@ pub fn extract(
Family::Sqlite => sqlite_x::extract(path, limit_bytes.map(|_| 500), sink),
Family::SqlDump => sqldump::extract(path, sn.gzip, limit_bytes, sink),
Family::Code => code::extract(path, sink),
+ Family::UnityYaml => unity::extract_unity(path, sn.gzip, limit_bytes, sink),
+ Family::UnityMeta => unity::extract_meta(path, sn.gzip, sink),
Family::Binary => Ok(ExtractStats::default()),
}
}
@@ -205,6 +208,46 @@ fn flatten_into(key: &str, v: Value, depth: usize, out: &mut Map<String, Value>)
}
}
+/// Convert a parsed YAML value into JSON (shared by the YAML and Unity
+/// extractors). Tagged values unwrap to their inner value; non-string mapping
+/// keys are stringified.
+pub(crate) fn yaml_to_json(v: serde_yaml::Value) -> Value {
+ match v {
+ serde_yaml::Value::Null => Value::Null,
+ serde_yaml::Value::Bool(b) => Value::Bool(b),
+ serde_yaml::Value::Number(n) => {
+ if let Some(i) = n.as_i64() {
+ Value::Number(i.into())
+ } else if let Some(u) = n.as_u64() {
+ Value::Number(u.into())
+ } else {
+ serde_json::Number::from_f64(n.as_f64().unwrap_or(0.0))
+ .map(Value::Number)
+ .unwrap_or(Value::Null)
+ }
+ }
+ serde_yaml::Value::String(s) => Value::String(s),
+ serde_yaml::Value::Sequence(seq) => {
+ Value::Array(seq.into_iter().map(yaml_to_json).collect())
+ }
+ serde_yaml::Value::Mapping(m) => {
+ let mut out = Map::new();
+ for (k, vv) in m {
+ let key = match k {
+ serde_yaml::Value::String(s) => s,
+ other => serde_yaml::to_string(&other)
+ .unwrap_or_else(|_| "key".into())
+ .trim()
+ .to_string(),
+ };
+ out.insert(key, yaml_to_json(vv));
+ }
+ Value::Object(out)
+ }
+ serde_yaml::Value::Tagged(t) => yaml_to_json(t.value),
+ }
+}
+
/// Sanitize a discovered field/column name to a safe ES field name.
pub fn sanitize_field_name(name: &str) -> String {
let mut s: String = name
diff --git a/engine/crates/xerj-autoindex/src/extract/unity.rs b/engine/crates/xerj-autoindex/src/extract/unity.rs
new file mode 100644
index 000000000..3903ca774
--- /dev/null
+++ b/engine/crates/xerj-autoindex/src/extract/unity.rs
@@ -0,0 +1,536 @@
+//! Unity text-serialized assets — streaming multi-document splitter — and
+//! `.meta` sidecars.
+//!
+//! Unity YAML is a multi-document stream where every document opens with
+//! `--- !u!<classId> &<fileID>` (optionally suffixed `stripped` for prefab
+//! instances). serde_yaml cannot be used on the whole stream: it resolves
+//! anchors internally and never exposes the anchor NAME, and the `fileID`
+//! that makes Unity objects joinable lives in that anchor. So the splitter
+//! owns the document boundaries (a cheap line prefix match) and hands each
+//! body — plain YAML once the `%YAML`/`%TAG` header is dropped — to
+//! serde_yaml individually. Owning the boundaries is also what makes it safe
+//! to `continue` past a corrupt document: unlike `yaml_x`, there is no
+//! iterator that would re-yield the same broken document forever.
+//!
+//! Files stream line-by-line (real scenes can exceed 200 MB; a single
+//! AnimationClip document can be tens of MB), with a 16 MiB per-document
+//! cap: over-cap documents are junked and skipped to the next boundary,
+//! the rest of the file survives.
+//!
+//! Emitted fields per object document:
+//! - `unity_class` body's top-level key (authoritative), else class-id table
+//! - `unity_class_id` from the `!u!<id>` tag
+//! - `file_id` from the `&<anchor>` — the join key for local refs
+//! - `stripped` only present (true) on stripped prefab-instance docs
+//! - `name` `m_Name` when present
+//! - `ref_guids` deduped guids of every `{fileID, guid, type}` reference
+//! - `script_guid` the `m_Script` reference's guid (MonoBehaviour → .cs)
+//! - flattened body fields
+//!
+//! `.meta` sidecars emit ONE record: `guid`, `asset_name`, `importer`, and
+//! flattened settings. The root-relative `asset_path` (and the
+//! `script_path`/`script_class` denormalization on MonoBehaviour records) is
+//! stamped by the pipeline, which knows the walk root — extractors stay
+//! per-file pure.
+
+use super::{
+ flatten_object, yaml_to_json, ExtractStats, FieldOrigin, RawRecord, Sink, MAX_LINE,
+};
+use anyhow::Result;
+use serde_json::{Map, Value};
+use std::path::Path;
+
+/// Per-document cap. One logical unit follows the `MAX_LINE` convention;
+/// 16 MiB covers large AnimationClip / font documents while bounding memory.
+/// (A 4 MiB cap was considered and rejected: it junks those documents.)
+const DOC_CAP: usize = 16 << 20;
+
+const META_CAP: u64 = 16 << 20;
+
+/// Field names this extractor invents on object records. A body field with
+/// the same name (possible on MonoBehaviour custom fields) is renamed
+/// `data_<name>` rather than dropped, mirroring the `ax_*` collision policy.
+const RESERVED: &[&str] = &[
+ "unity_class",
+ "unity_class_id",
+ "file_id",
+ "stripped",
+ "name",
+ "ref_guids",
+ "script_guid",
+ "script_path",
+ "script_class",
+];
+
+pub fn extract_unity(
+ path: &Path,
+ gzip: bool,
+ limit_bytes: Option<u64>,
+ sink: Sink,
+) -> Result<ExtractStats> {
+ let mut r = super::open_reader(path, gzip, limit_bytes)?;
+ let mut stats = ExtractStats::default();
+ let mut offset: u64 = 0;
+ let mut line: Vec<u8> = Vec::new();
+ let mut doc: Vec<u8> = Vec::new();
+ let mut header: Option<DocHeader> = None;
+ let mut skipping = false;
+ loop {
+ line.clear();
+ let n = super::jsonl::read_capped_line(&mut r, &mut line)?;
+ let eof = n == 0;
+ offset += n as u64;
+ let boundary = if eof { None } else { parse_boundary(&line) };
+ if eof || boundary.is_some() {
+ if let Some(h) = header.take() {
+ if skipping {
+ skipping = false;
+ } else {
+ let truncated =
+ eof && limit_bytes.is_some_and(|l| offset >= l);
+ if !emit_doc(&h, &doc, truncated, &mut stats, sink) {
+ return Ok(stats);
+ }
+ }
+ }
+ doc.clear();
+ header = boundary;
+ if eof {
+ break;
+ }
+ continue;
+ }
+ if header.is_none() || skipping {
+ continue; // %YAML/%TAG preamble, or discarding an over-cap doc
+ }
+ if doc.len() + line.len() > DOC_CAP {
+ stats.junk += 1;
+ skipping = true;
+ doc.clear();
+ continue;
+ }
+ doc.extend_from_slice(&line);
+ }
+ // Empty yield from a FULL read means the file was preamble-only (or
+ // junk-shaped throughout) — record it. A sampling read that truncated
+ // before the first document completed is not evidence of anything.
+ let truncated_run = limit_bytes.is_some_and(|l| offset >= l);
+ if stats.records == 0 && stats.junk == 0 && !truncated_run {
+ stats.junk += 1;
+ }
+ Ok(stats)
+}
+
+struct DocHeader {
+ class_id: i64,
+ /// Raw anchor text (handles negative 64-bit ids verbatim).
+ file_id: String,
+ stripped: bool,
+}
+
+/// `--- !u!<classId> &<fileID>[ stripped]` — cheap manual parse, no regex on
+/// the hot per-line path.
+fn parse_boundary(line: &[u8]) -> Option<DocHeader> {
+ let s = std::str::from_utf8(line).ok()?.trim_end();
+ let rest = s.strip_prefix("--- !u!")?;
+ let (id_part, rest) = rest.split_once(" &")?;
+ let class_id: i64 = id_part.trim().parse().ok()?;
+ let (anchor, tail) = match rest.split_once(' ') {
+ Some((a, t)) => (a, t.trim()),
+ None => (rest, ""),
+ };
+ let anchor = anchor.trim();
+ if anchor.is_empty()
+ || !anchor
+ .chars()
+ .all(|c| c.is_ascii_digit() || c == '-')
+ {
+ return None;
+ }
+ Some(DocHeader {
+ class_id,
+ file_id: anchor.to_string(),
+ stripped: tail == "stripped",
+ })
+}
+
+fn emit_doc(
+ h: &DocHeader,
+ body: &[u8],
+ truncated_by_sampling: bool,
+ stats: &mut ExtractStats,
+ sink: Sink,
+) -> bool {
+ let text = String::from_utf8_lossy(body);
+ let parsed: serde_yaml::Value = match serde_yaml::from_str(&text) {
+ Ok(v) => v,
+ Err(_) => {
+ if !truncated_by_sampling {
+ stats.junk += 1;
+ }
+ return true;
+ }
+ };
+ let jv = yaml_to_json(parsed);
+ let (class_from_body, inner) = match jv {
+ Value::Object(m) if m.len() == 1 => {
+ let (k, v) = m.into_iter().next().expect("len checked");
+ (Some(k), v)
+ }
+ other => (None, other),
+ };
+ let unity_class = class_from_body
+ .unwrap_or_else(|| class_name(h.class_id).map(str::to_string).unwrap_or_else(|| format!("class_{}", h.class_id)));
+
+ let mut refs: Vec<String> = Vec::new();
+ let mut script_guid: Option<String> = None;
+ collect_guid_refs(&inner, &mut refs);
+ if let Some(script) = inner.get("m_Script") {
+ if let Some(g) = script.get("guid").and_then(Value::as_str) {
+ script_guid = Some(g.to_string());
+ }
+ }
+
+ let mut fields = match inner {
+ Value::Object(m) => flatten_object(m),
+ Value::Null => Map::new(),
+ other => {
+ let mut m = Map::new();
+ m.insert("value".into(), other);
+ m
+ }
+ };
+ for k in RESERVED {
+ if let Some(v) = fields.remove(*k) {
+ fields.insert(format!("data_{k}"), v);
+ }
+ }
+ if let Some(Value::String(n)) = fields.get("m_Name") {
+ if !n.is_empty() {
+ fields.insert("name".into(), Value::String(n.clone()));
+ }
+ }
+ fields.insert("unity_class".into(), Value::String(unity_class.clone()));
+ fields.insert("unity_class_id".into(), Value::Number(h.class_id.into()));
+ fields.insert("file_id".into(), Value::String(h.file_id.clone()));
+ if h.stripped {
+ fields.insert("stripped".into(), Value::Bool(true));
+ }
+ if !refs.is_empty() {
+ let mut seen = std::collections::HashSet::new();
+ let deduped: Vec<Value> = refs
+ .into_iter()
+ .filter(|g| seen.insert(g.clone()))
+ .map(Value::String)
+ .collect();
+ fields.insert("ref_guids".into(), Value::Array(deduped));
+ }
+ if let Some(g) = script_guid {
+ fields.insert("script_guid".into(), Value::String(g));
+ }
+
+ stats.records += 1;
+ sink(RawRecord {
+ fields,
+ locator: format!("u{}", h.file_id),
+ group: Some(unity_class),
+ // Body field names are Unity's serialization vocabulary plus per-script
+ // custom fields; letting them cluster would split MonoBehaviours into
+ // schema-similarity datasets and re-home documents when scripts change
+ // (issue #178). Family+group clustering gives one stable dataset per
+ // Unity class instead.
+ origin: FieldOrigin::Extractor,
+ })
+}
+
+/// Walk the parsed document for `{fileID, guid, type}` reference mappings and
+/// collect every guid, including inside arrays (`m_Modifications` — prefab
+/// overrides — is where the highest-value references live).
+fn collect_guid_refs(v: &Value, out: &mut Vec<String>) {
+ match v {
+ Value::Object(m) => {
+ if m.contains_key("fileID") {
+ if let Some(g) = m.get("guid").and_then(Value::as_str) {
+ out.push(g.to_string());
+ }
+ }
+ for vv in m.values() {
+ collect_guid_refs(vv, out);
+ }
+ }
+ Value::Array(a) => {
+ for vv in a {
+ collect_guid_refs(vv, out);
+ }
+ }
+ _ => {}
+ }
+}
+
+pub fn extract_meta(path: &Path, gzip: bool, sink: Sink) -> Result<ExtractStats> {
+ let mut stats = ExtractStats::default();
+ let Some(bytes) = super::read_whole(path, gzip, META_CAP)? else {
+ stats.junk += 1;
+ return Ok(stats);
+ };
+ let (text, _) = crate::sniff::decode_text(&bytes);
+ let text = text.trim_start_matches('\u{feff}');
+ let parsed: serde_yaml::Value = match serde_yaml::from_str(text) {
+ Ok(v) => v,
+ Err(_) => {
+ stats.junk += 1;
+ return Ok(stats);
+ }
+ };
+ let Value::Object(m) = yaml_to_json(parsed) else {
+ stats.junk += 1;
+ return Ok(stats);
+ };
+ let guid = m.get("guid").and_then(Value::as_str).map(str::to_string);
+ let importer = m
+ .keys()
+ .find(|k| k.ends_with("Importer") || k.ends_with("importer"))
+ .cloned();
+ let mut fields = flatten_object(m);
+ if let Some(g) = guid {
+ fields.insert("guid".into(), Value::String(g));
+ }
+ if let Some(imp) = importer {
+ fields.insert("importer".into(), Value::String(imp));
+ }
+ if let Some(name) = path
+ .file_name()
+ .and_then(|n| n.to_str())
+ .and_then(|n| n.strip_suffix(".meta"))
+ {
+ fields.insert("asset_name".into(), Value::String(name.to_string()));
+ }
+ stats.records += 1;
+ sink(RawRecord {
+ fields,
+ locator: "meta".into(),
+ group: None,
+ origin: FieldOrigin::Extractor,
+ });
+ Ok(stats)
+}
+
+/// Fallback names for documents whose body failed to yield a top-level key
+/// (the body key is authoritative when present). Sourced from Unity's public
+/// class-id reference; unknown ids render `class_<id>`.
+fn class_name(id: i64) -> Option<&'static str> {
+ Some(match id {
+ 1 => "GameObject",
+ 2 => "Component",
+ 4 => "Transform",
+ 20 => "Camera",
+ 21 => "Material",
+ 23 => "MeshRenderer",
+ 25 => "Renderer",
+ 28 => "Texture2D",
+ 33 => "MeshFilter",
+ 43 => "Mesh",
+ 54 => "Rigidbody",
+ 61 => "BoxCollider2D",
+ 64 => "MeshCollider",
+ 65 => "BoxCollider",
+ 74 => "AnimationClip",
+ 81 => "AudioListener",
+ 82 => "AudioSource",
+ 90 => "Avatar",
+ 91 => "AnimatorController",
+ 95 => "Animator",
+ 104 => "RenderSettings",
+ 108 => "Light",
+ 114 => "MonoBehaviour",
+ 115 => "MonoScript",
+ 128 => "Font",
+ 135 => "SphereCollider",
+ 136 => "CapsuleCollider",
+ 137 => "SkinnedMeshRenderer",
+ 157 => "LightmapSettings",
+ 196 => "NavMeshSettings",
+ 198 => "ParticleSystem",
+ 199 => "ParticleSystemRenderer",
+ 212 => "SpriteRenderer",
+ 213 => "Sprite",
+ 224 => "RectTransform",
+ 320 => "PlayableDirector",
+ 850595691 => "LightingSettings",
+ 1001 => "PrefabInstance",
+ 1660057539 => "SceneRoots",
+ _ => return None,
+ })
+}
+
+// The `MAX_LINE` import is load-bearing indirectly: `read_capped_line` caps
+// each physical line at 16 MiB, and DOC_CAP matches it so a single-line
+// document cannot bypass the document cap.
+const _: () = assert!(DOC_CAP == MAX_LINE);
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ const SCENE: &str = "\u{feff}%YAML 1.1\n%TAG !u! tag:unity3d.com,2011:\n--- !u!1 &1234567890\nGameObject:\n m_ObjectHideFlags: 0\n m_Name: Player\n m_IsActive: 1\n--- !u!4 &1234567891\nTransform:\n m_GameObject: {fileID: 1234567890}\n m_LocalPosition: {x: 0, y: 1.5, z: 0}\n--- !u!114 &1234567892\nMonoBehaviour:\n m_GameObject: {fileID: 1234567890}\n m_Script: {fileID: 11500000, guid: abc123def456, type: 3}\n m_EditorClassIdentifier: \n speed: 4.5\n target: {fileID: 5555, guid: fedcba987654, type: 2}\n";
+
+ fn run(text: &str, limit: Option<u64>) -> (ExtractStats, Vec<RawRecord>) {
+ let dir = tempfile::tempdir().unwrap();
+ let path = dir.path().join("t.unity");
+ std::fs::write(&path, text).unwrap();
+ let mut recs = Vec::new();
+ let stats = extract_unity(&path, false, limit, &mut |r| {
+ recs.push(r);
+ true
+ })
+ .unwrap();
+ (stats, recs)
+ }
+
+ fn run_meta(text: &str, name: &str) -> (ExtractStats, Vec<RawRecord>) {
+ let dir = tempfile::tempdir().unwrap();
+ let path = dir.path().join(name);
+ std::fs::write(&path, text).unwrap();
+ let mut recs = Vec::new();
+ let stats = extract_meta(&path, false, &mut |r| {
+ recs.push(r);
+ true
+ })
+ .unwrap();
+ (stats, recs)
+ }
+
+ #[test]
+ fn each_document_becomes_one_record_with_class_and_file_id() {
+ let (stats, recs) = run(SCENE, None);
+ assert_eq!((stats.records, stats.junk), (3, 0));
+ assert_eq!(recs[0].fields["unity_class"], "GameObject");
+ assert_eq!(recs[0].fields["unity_class_id"], 1);
+ assert_eq!(recs[0].fields["file_id"], "1234567890");
+ assert_eq!(recs[0].fields["name"], "Player");
+ assert_eq!(recs[0].locator, "u1234567890");
+ assert_eq!(recs[0].group.as_deref(), Some("GameObject"));
+ assert_eq!(recs[1].fields["unity_class"], "Transform");
+ assert_eq!(
+ recs[1].fields["m_LocalPosition_y"], 1.5,
+ "nested mappings flatten like every other extractor"
+ );
+ }
+
+ #[test]
+ fn monobehaviour_carries_script_guid_and_deduped_ref_guids() {
+ let (_, recs) = run(SCENE, None);
+ let mb = &recs[2];
+ assert_eq!(mb.fields["unity_class"], "MonoBehaviour");
+ assert_eq!(mb.fields["script_guid"], "abc123def456");
+ assert_eq!(
+ mb.fields["ref_guids"],
+ serde_json::json!(["abc123def456", "fedcba987654"])
+ );
+ assert_eq!(mb.fields["speed"], 4.5, "custom script fields survive");
+ }
+
+ #[test]
+ fn stripped_and_negative_file_ids_parse() {
+ let text = "%YAML 1.1\n%TAG !u! tag:unity3d.com,2011:\n--- !u!4 &-8679921383154817045 stripped\nTransform:\n m_PrefabInstance: {fileID: 100100000, guid: 0123456789abcdef, type: 3}\n";
+ let (stats, recs) = run(text, None);
+ assert_eq!((stats.records, stats.junk), (1, 0));
+ assert_eq!(recs[0].fields["file_id"], "-8679921383154817045");
+ assert_eq!(recs[0].fields["stripped"], true);
+ assert_eq!(recs[0].locator, "u-8679921383154817045");
+ }
+
+ #[test]
+ fn crlf_line_endings_are_tolerated() {
+ let text = SCENE.replace('\n', "\r\n");
+ let (stats, recs) = run(&text, None);
+ assert_eq!((stats.records, stats.junk), (3, 0));
+ assert_eq!(recs[2].fields["script_guid"], "abc123def456");
+ }
+
+ #[test]
+ fn a_corrupt_document_junks_only_itself() {
+ let text = "%YAML 1.1\n%TAG !u! tag:unity3d.com,2011:\n--- !u!1 &1\nGameObject:\n m_Name: A\n--- !u!4 &2\n\t{{{ not yaml\n--- !u!1 &3\nGameObject:\n m_Name: B\n";
+ let (stats, recs) = run(text, None);
+ assert_eq!(stats.junk, 1);
+ assert_eq!(stats.records, 2);
+ assert_eq!(recs[1].fields["name"], "B");
+ }
+
+ #[test]
+ fn an_over_cap_document_is_skipped_and_the_stream_continues() {
+ let mut text = String::from("%YAML 1.1\n%TAG !u! tag:unity3d.com,2011:\n--- !u!74 &1\nAnimationClip:\n m_Name: huge\n data: ");
+ text.push_str(&"x".repeat(DOC_CAP + 1024));
+ text.push_str("\n--- !u!1 &2\nGameObject:\n m_Name: after\n");
+ let (stats, recs) = run(&text, None);
+ assert_eq!(stats.junk, 1, "the oversized doc is junked, not fatal");
+ assert_eq!(stats.records, 1);
+ assert_eq!(recs[0].fields["name"], "after");
+ }
+
+ #[test]
+ fn a_doc_cut_short_by_a_sampling_limit_is_not_junk() {
+ let (stats, _) = run(SCENE, Some(120));
+ assert_eq!(stats.junk, 0, "sampling truncation is not corruption");
+ assert!(stats.records <= 1);
+ }
+
+ #[test]
+ fn sampling_early_stop_via_sink_leaves_no_junk() {
+ let dir = tempfile::tempdir().unwrap();
+ let path = dir.path().join("t.unity");
+ std::fs::write(&path, SCENE).unwrap();
+ let mut n = 0;
+ let stats = extract_unity(&path, false, None, &mut |_r| {
+ n += 1;
+ n < 2
+ })
+ .unwrap();
+ assert_eq!(n, 2);
+ assert_eq!(stats.junk, 0);
+ }
+
+ #[test]
+ fn locators_are_idempotent_across_runs() {
+ let (_, a) = run(SCENE, None);
+ let (_, b) = run(SCENE, None);
+ let la: Vec<_> = a.iter().map(|r| r.locator.clone()).collect();
+ let lb: Vec<_> = b.iter().map(|r| r.locator.clone()).collect();
+ assert_eq!(la, lb);
+ }
+
+ #[test]
+ fn a_reserved_body_field_is_renamed_not_clobbered() {
+ let text = "%YAML 1.1\n%TAG !u! tag:unity3d.com,2011:\n--- !u!114 &1\nMonoBehaviour:\n name: custom-field\n m_Name: Real\n";
+ let (_, recs) = run(text, None);
+ assert_eq!(recs[0].fields["data_name"], "custom-field");
+ assert_eq!(recs[0].fields["name"], "Real");
+ }
+
+ #[test]
+ fn unknown_class_id_with_unparseable_body_still_names_the_class() {
+ let text = "%YAML 1.1\n%TAG !u! tag:unity3d.com,2011:\n--- !u!999999 &1\nsome scalar\n";
+ let (stats, recs) = run(text, None);
+ assert_eq!(stats.records, 1);
+ assert_eq!(recs[0].fields["unity_class"], "class_999999");
+ }
+
+ #[test]
+ fn meta_yields_guid_importer_and_asset_name() {
+ let text = "fileFormatVersion: 2\nguid: 9f1c4d0ab2e34f6\nMonoImporter:\n externalObjects: {}\n serializedVersion: 2\n";
+ let (stats, recs) = run_meta(text, "Player.cs.meta");
+ assert_eq!((stats.records, stats.junk), (1, 0));
+ assert_eq!(recs[0].fields["guid"], "9f1c4d0ab2e34f6");
+ assert_eq!(recs[0].fields["importer"], "MonoImporter");
+ assert_eq!(recs[0].fields["asset_name"], "Player.cs");
+ assert_eq!(recs[0].locator, "meta");
+ }
+
+ #[test]
+ fn a_file_with_only_preamble_is_junk() {
+ let (stats, recs) = run("%YAML 1.1\n%TAG !u! tag:unity3d.com,2011:\n", None);
+ assert_eq!(stats.records, 0);
+ assert!(stats.junk >= 1);
+ assert!(recs.is_empty());
+ }
+}
diff --git a/engine/crates/xerj-autoindex/src/extract/yaml_x.rs b/engine/crates/xerj-autoindex/src/extract/yaml_x.rs
index 0b17914d0..6b298d57e 100644
--- a/engine/crates/xerj-autoindex/src/extract/yaml_x.rs
+++ b/engine/crates/xerj-autoindex/src/extract/yaml_x.rs
@@ -1,7 +1,7 @@
//! YAML — multi-document streams; each doc becomes a record (a top-level
//! sequence of mappings becomes per-element records). 16MB file cap.
-use super::{flatten_object, ExtractStats, FieldOrigin, RawRecord, Sink};
+use super::{flatten_object, yaml_to_json, ExtractStats, FieldOrigin, RawRecord, Sink};
use anyhow::Result;
use serde_json::{Map, Value};
use std::path::Path;
@@ -83,43 +83,6 @@ pub fn extract(path: &Path, gzip: bool, sink: Sink) -> Result<ExtractStats> {
use serde::Deserialize;
-fn yaml_to_json(v: serde_yaml::Value) -> Value {
- match v {
- serde_yaml::Value::Null => Value::Null,
- serde_yaml::Value::Bool(b) => Value::Bool(b),
- serde_yaml::Value::Number(n) => {
- if let Some(i) = n.as_i64() {
- Value::Number(i.into())
- } else if let Some(u) = n.as_u64() {
- Value::Number(u.into())
- } else {
- serde_json::Number::from_f64(n.as_f64().unwrap_or(0.0))
- .map(Value::Number)
- .unwrap_or(Value::Null)
- }
- }
- serde_yaml::Value::String(s) => Value::String(s),
- serde_yaml::Value::Sequence(seq) => {
- Value::Array(seq.into_iter().map(yaml_to_json).collect())
- }
- serde_yaml::Value::Mapping(m) => {
- let mut out = Map::new();
- for (k, vv) in m {
- let key = match k {
- serde_yaml::Value::String(s) => s,
- other => serde_yaml::to_string(&other)
- .unwrap_or_else(|_| "key".into())
- .trim()
- .to_string(),
- };
- out.insert(key, yaml_to_json(vv));
- }
- Value::Object(out)
- }
- serde_yaml::Value::Tagged(t) => yaml_to_json(t.value),
- }
-}
-
#[cfg(test)]
mod tests {
use super::*;
diff --git a/engine/crates/xerj-autoindex/src/failure_resume_http_tests.rs b/engine/crates/xerj-autoindex/src/failure_resume_http_tests.rs
index c7d18289f..7be372a48 100644
--- a/engine/crates/xerj-autoindex/src/failure_resume_http_tests.rs
+++ b/engine/crates/xerj-autoindex/src/failure_resume_http_tests.rs
@@ -46,7 +46,12 @@ impl MockEndpoint {
let server_state = Arc::clone(&state);
let join = thread::spawn(move || loop {
match listener.accept() {
- Ok((stream, _)) => handle(stream, &server_state),
+ Ok((stream, _)) => {
+ // BSD/macOS: accepted sockets inherit the listener's
+ // O_NONBLOCK; the handler does blocking reads.
+ stream.set_nonblocking(false).unwrap();
+ handle(stream, &server_state)
+ }
Err(error) if error.kind() == std::io::ErrorKind::WouldBlock => {
if server_state.lock().unwrap().stop {
break;
@@ -199,6 +204,7 @@ fn cfg(root: &Path, state_dir: &Path, url: &str) -> IndexCfg {
state_dir: Some(state_dir.to_owned()),
fresh: false,
follow_symlinks: false,
+ no_default_excludes: false,
max_file_gb: 1,
sample: 50,
no_semantic: true,
diff --git a/engine/crates/xerj-autoindex/src/lib.rs b/engine/crates/xerj-autoindex/src/lib.rs
index da8614e6f..3aa83fe4f 100644
--- a/engine/crates/xerj-autoindex/src/lib.rs
+++ b/engine/crates/xerj-autoindex/src/lib.rs
@@ -233,7 +233,9 @@ fn scan_file(path: &Path, size: u64, sample: usize, max_file_gb: u64) -> FileSca
}
let limit = match sn.family {
Family::SqlDump => Some(SQLDUMP_SAMPLE_LIMIT),
- Family::Jsonl | Family::Logs | Family::Csv | Family::TxtLines => Some(SAMPLE_LIMIT_BYTES),
+ Family::Jsonl | Family::Logs | Family::Csv | Family::TxtLines | Family::UnityYaml => {
+ Some(SAMPLE_LIMIT_BYTES)
+ }
Family::Sqlite => Some(1), // signals per-table row cap inside the extractor
_ => None, // whole-file extractors cap themselves
};
@@ -243,7 +245,14 @@ fn scan_file(path: &Path, size: u64, sample: usize, max_file_gb: u64) -> FileSca
std::collections::HashSet<String>,
);
let mut groups: HashMap<Option<String>, GroupAcc> = HashMap::new();
- let grouped_family = matches!(sn.family, Family::SqlDump | Family::Sqlite);
+ // Grouped families keep reading past the per-group sample size: their
+ // groups (SQL tables, Unity classes) appear all through the file, and
+ // stopping at the first N records would leave later groups unsampled and
+ // untyped.
+ let grouped_family = matches!(
+ sn.family,
+ Family::SqlDump | Family::Sqlite | Family::UnityYaml
+ );
let mut sink = |rec: extract::RawRecord| -> bool {
let entry = groups.entry(rec.group.clone()).or_default();
if (entry.1 as usize) < sample {
@@ -392,6 +401,97 @@ pub const PROVENANCE_FIELDS: &[&str] = &[
"ax_format",
];
+/// Spec for a field the PIPELINE derives at index time (Unity script-link
+/// enrichment): typed keyword in the explicit mapping, zeroed sampling stats
+/// because phase-A inference never observes it.
+fn pipeline_keyword_spec(name: &str) -> infer::FieldSpec {
+ infer::FieldSpec {
+ name: name.into(),
+ es_type: "keyword".into(),
+ date_enc: None,
+ semantic: None,
+ cardinality_est: 0,
+ cardinality_overflow: false,
+ null_ratio: 0.0,
+ avg_len: 0.0,
+ coverage: 0.0,
+ examples: Vec::new(),
+ notes: vec!["pipeline-derived: resolved from the .meta guid map at index time".into()],
+ date_min: None,
+ date_max: None,
+ date_evidence: Vec::new(),
+ }
+}
+
+/// Unity script-link map: `.meta` guid → root-relative asset path. Metas are
+/// tiny (one small YAML doc), so re-extracting them here costs milliseconds
+/// and works identically on fresh and resumed runs — the map never has to be
+/// journaled.
+fn build_unity_guid_map(
+ files: &[walk::FileEntry],
+ plan: &Plan,
+) -> std::collections::HashMap<String, String> {
+ let by_rel: HashMap<&str, &Path> = files
+ .iter()
+ .map(|f| (f.rel.as_str(), f.path.as_path()))
+ .collect();
+ let mut map = std::collections::HashMap::new();
+ for fa in plan.files.values() {
+ if fa.family != "unity-meta" {
+ continue;
+ }
+ let Some(asset_rel) = fa.rel.strip_suffix(".meta") else {
+ continue;
+ };
+ let Some(path) = by_rel.get(fa.rel.as_str()) else {
+ continue;
+ };
+ let mut guid: Option<String> = None;
+ let _ = extract::unity::extract_meta(path, fa.gzip, &mut |rec| {
+ guid = rec
+ .fields
+ .get("guid")
+ .and_then(Value::as_str)
+ .map(str::to_string);
+ false
+ });
+ if let Some(g) = guid {
+ map.insert(g, asset_rel.to_string());
+ }
+ }
+ map
+}
+
+/// Stamp pipeline-derived Unity fields onto a record. MonoBehaviour records
+/// gain `script_path`/`script_class` when their `script_guid` resolves; meta
+/// records gain the root-relative `asset_path` their guid names. Denormalized
+/// for one-query answers — `script_guid` remains the authoritative join.
+fn enrich_unity_fields(
+ family: Family,
+ fields: &mut Map<String, Value>,
+ guid_map: &std::collections::HashMap<String, String>,
+ rel: &str,
+) {
+ match family {
+ Family::UnityYaml => {
+ let Some(g) = fields.get("script_guid").and_then(Value::as_str) else {
+ return;
+ };
+ let Some(p) = guid_map.get(g) else { return };
+ fields.insert("script_path".into(), Value::String(p.clone()));
+ if let Some(stem) = Path::new(p).file_stem().and_then(|s| s.to_str()) {
+ fields.insert("script_class".into(), Value::String(stem.to_string()));
+ }
+ }
+ Family::UnityMeta => {
+ if let Some(asset_rel) = rel.strip_suffix(".meta") {
+ fields.insert("asset_path".into(), Value::String(asset_rel.to_string()));
+ }
+ }
+ _ => {}
+ }
+}
+
fn build_mapping(specs: &[infer::FieldSpec]) -> Value {
let mut props = Map::new();
for s in specs {
@@ -575,7 +675,18 @@ pub fn run_index_report(cfg: IndexCfg) -> Result<(i32, Option<Value>)> {
let es = Es::with_bulk_timeout(&cfg.url, cfg.api_key.clone(), cfg.bulk_timeout_secs)?;
es.ping()?;
- let discovered_files = walk::walk(&cfg.root, cfg.follow_symlinks)?;
+ let (discovered_files, skipped_dirs) =
+ walk::walk(&cfg.root, cfg.follow_symlinks, !cfg.no_default_excludes)?;
+ if !skipped_dirs.is_empty() && !cfg.quiet {
+ let names: Vec<&str> = skipped_dirs.iter().map(|s| s.rel.as_str()).collect();
+ eprintln!(
+ "skipping {} generated director{} ({}): {} — pass --no-default-excludes to include",
+ skipped_dirs.len(),
+ if skipped_dirs.len() == 1 { "y" } else { "ies" },
+ skipped_dirs[0].reason,
+ names.join(", ")
+ );
+ }
if discovered_files.is_empty() {
println!("no files found under {}", cfg.root.display());
return Ok((0, None));
@@ -781,6 +892,16 @@ pub fn run_index_report(cfg: IndexCfg) -> Result<(i32, Option<Value>)> {
let rels: Vec<String> = files.iter().map(|f| f.rel.clone()).collect();
let mut sketches = Vec::new();
let mut junk_files = Vec::new();
+ for sd in &skipped_dirs {
+ junk_files.push(JunkFile {
+ file_key: format!("dir:{}", sd.rel),
+ rel: sd.rel.clone(),
+ format: "dir".into(),
+ status: "skipped".into(),
+ reason: format!("{}; --no-default-excludes to include", sd.reason),
+ bytes: 0,
+ });
+ }
for (i, sc) in scans.into_iter().enumerate() {
let family = sc
.sniffed
@@ -841,7 +962,17 @@ pub fn run_index_report(cfg: IndexCfg) -> Result<(i32, Option<Value>)> {
let mut datasets = Vec::new();
for c in &clusters {
- let specs = infer::infer_fields(&c.fields, c.records, cfg.no_semantic);
+ let mut specs = infer::infer_fields(&c.fields, c.records, cfg.no_semantic);
+ // Unity script-link enrichment fields are stamped by the phase-B
+ // pipeline (not the extractor), so inference never sees them —
+ // register them here or they would be dynamic-mapped coarsely.
+ if c.family == Family::UnityYaml && c.fields.contains_key("script_guid") {
+ specs.push(pipeline_keyword_spec("script_path"));
+ specs.push(pipeline_keyword_spec("script_class"));
+ }
+ if c.family == Family::UnityMeta {
+ specs.push(pipeline_keyword_spec("asset_path"));
+ }
let time_field = infer::elect_time_field(&specs);
let semantic_field = specs
.iter()
@@ -953,6 +1084,8 @@ pub fn run_index_report(cfg: IndexCfg) -> Result<(i32, Option<Value>)> {
}
replacement_failpoint(1).context("after durable replacement plan")?;
+ let unity_guid_map = build_unity_guid_map(&files, &plan);
+
// ── Phase B: full-stream extraction + bulk indexing ─────────────────
struct DsRt {
index: String,
@@ -1339,6 +1472,7 @@ pub fn run_index_report(cfg: IndexCfg) -> Result<(i32, Option<Value>)> {
if dropped > 0 {
rt.dropped.fetch_add(dropped as u64, Ordering::Relaxed);
}
+ enrich_unity_fields(sn.family, &mut fields, &unity_guid_map, &f.rel);
fields.insert("ax_path".into(), Value::String(f.rel.clone()));
fields.insert(
"ax_paths".into(),
@@ -2363,7 +2497,7 @@ mod duplicate_integration_tests {
b[65_536] = b'b';
fs::write(corpus.path().join("a.txt"), a).unwrap();
fs::write(corpus.path().join("b.txt"), b).unwrap();
- let files = walk::walk(corpus.path(), false).unwrap();
+ let files = walk::walk(corpus.path(), false, true).unwrap().0;
let inventory = content::resolve(files.clone()).unwrap();
let legacy = ids::file_key(&files[0].path, files[0].size).unwrap();
assert_eq!(
@@ -2400,7 +2534,7 @@ mod duplicate_integration_tests {
// with — so b.txt's content key IS the planned key a.txt claims by rel.
fs::write(corpus.path().join("a.txt"), b"rewritten content\n").unwrap();
fs::write(corpus.path().join("b.txt"), b"original planned content\n").unwrap();
- let inventory = content::resolve(walk::walk(corpus.path(), false).unwrap()).unwrap();
+ let inventory = content::resolve(walk::walk(corpus.path(), false, true).unwrap().0).unwrap();
let planned_key = inventory.keys[1].clone();
let mut plan = Plan::default();
plan.files
@@ -2470,7 +2604,7 @@ mod duplicate_integration_tests {
fs::write(corpus.path().join("report-original.txt"), body).unwrap();
fs::write(corpus.path().join("report-copy.txt"), body).unwrap();
- let discovered = walk::walk(corpus.path(), false).unwrap();
+ let discovered = walk::walk(corpus.path(), false, true).unwrap().0;
let inventory = content::resolve(discovered).unwrap();
assert_eq!(inventory.files.len(), 1);
assert_eq!(inventory.duplicates.len(), 1);
@@ -2543,7 +2677,7 @@ mod duplicate_integration_tests {
}
fs::write(&path, csv).unwrap();
- let inventory = content::resolve(walk::walk(corpus.path(), false).unwrap()).unwrap();
+ let inventory = content::resolve(walk::walk(corpus.path(), false, true).unwrap().0).unwrap();
let expected_size = inventory.files[0].size;
let expected_digest = inventory.digests[0].clone();
let sniffed = sniff::sniff(&path).unwrap();
diff --git a/engine/crates/xerj-autoindex/src/sniff.rs b/engine/crates/xerj-autoindex/src/sniff.rs
index a0d8c5485..0dbf3a489 100644
--- a/engine/crates/xerj-autoindex/src/sniff.rs
+++ b/engine/crates/xerj-autoindex/src/sniff.rs
@@ -24,6 +24,12 @@ pub enum Family {
SqlDump,
/// Source code — AST-parsed by the matching tree-sitter grammar.
Code,
+ /// Unity text-serialized asset (scene/prefab/.asset/.mat/.anim/…):
+ /// a `%YAML` + `%TAG !u! tag:unity3d.com` multi-document stream.
+ UnityYaml,
+ /// Unity `.meta` sidecar: plain YAML opening with `fileFormatVersion:`
+ /// and carrying the asset `guid` — the join key for everything Unity.
+ UnityMeta,
Binary,
}
@@ -44,6 +50,8 @@ impl Family {
Family::Sqlite => "sqlite",
Family::SqlDump => "sqldump",
Family::Code => "code",
+ Family::UnityYaml => "unity",
+ Family::UnityMeta => "unity-meta",
Family::Binary => "binary",
}
}
@@ -134,6 +142,11 @@ fn sniff_bytes(prefix: &[u8], path: &Path, gzip: bool) -> Result<Sniffed> {
s.binary_kind = Some("zip".into());
return Ok(s);
}
+ // Compressed image/audio/model payloads routinely pass the NUL and
+ // control-char heuristics below (windows-1252 decodes almost every byte
+ // to something printable), and a multi-MB PSD misread as prose costs
+ // ~100x its size in RAM through sectioning. Magic bytes are the reliable
+ // signal.
for (magic, kind) in [
(&b"\x89PNG"[..], "png"),
(&b"GIF8"[..], "gif"),
@@ -141,6 +154,15 @@ fn sniff_bytes(prefix: &[u8], path: &Path, gzip: bool) -> Result<Sniffed> {
(&b"\x7fELF"[..], "elf"),
(&b"BM"[..], "bmp"),
(&b"\x00\x00\x01\x00"[..], "ico"),
+ (&b"8BPS"[..], "psd"),
+ (&b"II*\x00"[..], "tiff"),
+ (&b"MM\x00*"[..], "tiff"),
+ (&b"RIFF"[..], "riff"),
+ (&b"OggS"[..], "ogg"),
+ (&b"fLaC"[..], "flac"),
+ (&b"ID3"[..], "mp3"),
+ (&b"Kaydara FBX Binary"[..], "fbx"),
+ (&b"\x76\x2f\x31\x01"[..], "exr"),
] {
if prefix.starts_with(magic) {
let mut s = mk(Family::Binary);
@@ -167,8 +189,44 @@ fn sniff_bytes(prefix: &[u8], path: &Path, gzip: bool) -> Result<Sniffed> {
s.binary_kind = Some("unknown".into());
return Ok(s);
}
+ // A prefix that only decoded via LOSSY windows-1252 and is majority
+ // high-byte is pixel/float soup, not prose in a legacy encoding: real
+ // windows-1252 text (accented European prose) runs well under 30%
+ // non-ASCII, while raw image channels run ~50%. Without this, a large
+ // TGA classified as txt-prose is amplified ~100x in RAM by sectioning.
+ if encoding != "utf-8" {
+ let total = text.chars().count().max(1);
+ let non_ascii = text.chars().filter(|c| !c.is_ascii()).count();
+ if non_ascii * 10 > total * 3 {
+ let mut s = mk(Family::Binary);
+ s.binary_kind = Some("unknown".into());
+ return Ok(s);
+ }
+ }
- // 2b. Source code: a known code extension whose content is text. We only
+ // 2b. Unity text serialization, detected by content (never extension):
+ // scenes/prefabs/assets open with `%YAML` and declare the Unity tag
+ // namespace; `.meta` sidecars open with `fileFormatVersion:` and carry a
+ // `guid:`. Binary-serialized Unity assets fail both checks and fall
+ // through to the binary/text heuristics as before.
+ {
+ let body = text.trim_start_matches('\u{feff}');
+ if body.starts_with("%YAML") && body.contains("%TAG !u! tag:unity3d.com") {
+ let mut s = mk(Family::UnityYaml);
+ s.encoding = encoding;
+ return Ok(s);
+ }
+ let first_line = body.lines().next().unwrap_or("");
+ if first_line.starts_with("fileFormatVersion:")
+ && body.lines().any(|l| l.starts_with("guid:"))
+ {
+ let mut s = mk(Family::UnityMeta);
+ s.encoding = encoding;
+ return Ok(s);
+ }
+ }
+
+ // 2c. Source code: a known code extension whose content is text. We only
// reach here after the binary guards above, so a text `.py`/`.rs`/`.go`/…
// routes to the tree-sitter AST extractor (crate::extract::code). Extension
// is the right signal — code vs prose is not reliably content-sniffable.
@@ -342,6 +400,26 @@ fn txt_kind(nonblank: &[&str]) -> Family {
if nonblank.is_empty() {
return Family::TxtLines;
}
+ // Raw pixel/float payloads without a container magic (TGA, bare .bytes
+ // dumps) decode via lossy windows-1252 into printable soup; every
+ // structured family above has already declined by the time control gets
+ // here, and treating the soup as prose costs ~100x the file size in RAM
+ // through sectioning. The discriminator is whitespace density:
+ // natural-language text runs ~15% whitespace and random bytes ~2%
+ // (0x09/0x0A/0x0D/0x20/0xA0 all decode to whitespace), so 5% cleanly
+ // separates them. Gated to large prefixes so short legitimate files keep
+ // their text families; dense CSVs are safe because the CSV sniffer
+ // claims them before this fallback.
+ let total_chars: usize = nonblank.iter().map(|l| l.chars().count()).sum();
+ if total_chars >= 4096 {
+ let ws: usize = nonblank
+ .iter()
+ .map(|l| l.chars().filter(|c| c.is_whitespace()).count())
+ .sum();
+ if ws * 20 < total_chars {
+ return Family::Binary;
+ }
+ }
let avg_len = nonblank.iter().map(|l| l.len()).sum::<usize>() as f64 / nonblank.len() as f64;
if avg_len > 60.0 {
return Family::TxtProse;
@@ -503,6 +581,102 @@ fn sniff_csv_dialect(nonblank: &[&str]) -> Option<CsvDialect> {
})
}
+#[cfg(test)]
+mod unity_sniff_tests {
+ use super::*;
+ use std::path::Path;
+
+ fn sniff_str(s: &str, name: &str) -> Family {
+ sniff_bytes(s.as_bytes(), Path::new(name), false)
+ .unwrap()
+ .family
+ }
+
+ #[test]
+ fn unity_tagged_yaml_is_detected_by_header_not_extension() {
+ let scene = "%YAML 1.1\n%TAG !u! tag:unity3d.com,2011:\n--- !u!1 &123\nGameObject:\n m_Name: X\n";
+ assert_eq!(sniff_str(scene, "Main.unity"), Family::UnityYaml);
+ assert_eq!(
+ sniff_str(scene, "renamed.txt"),
+ Family::UnityYaml,
+ "content decides, never the extension"
+ );
+ }
+
+ #[test]
+ fn a_bom_before_the_yaml_directive_is_tolerated() {
+ let scene =
+ "\u{feff}%YAML 1.1\n%TAG !u! tag:unity3d.com,2011:\n--- !u!1 &1\nGameObject:\n m_Name: X\n";
+ assert_eq!(sniff_str(scene, "Main.unity"), Family::UnityYaml);
+ }
+
+ #[test]
+ fn meta_needs_the_first_line_rule_and_a_guid() {
+ let meta = "fileFormatVersion: 2\nguid: 9f1c4d0ab2e34f6\nMonoImporter:\n serializedVersion: 2\n";
+ assert_eq!(sniff_str(meta, "Player.cs.meta"), Family::UnityMeta);
+ let stray = "config: true\nfileFormatVersion: 2\nguid: abc\n";
+ assert_ne!(
+ sniff_str(stray, "some.yaml"),
+ Family::UnityMeta,
+ "guid keys inside ordinary YAML must not reclassify it"
+ );
+ let no_guid = "fileFormatVersion: 2\nsettings:\n a: 1\n";
+ assert_ne!(sniff_str(no_guid, "x.meta"), Family::UnityMeta);
+ }
+
+ #[test]
+ fn plain_yaml_without_the_unity_tag_stays_yaml() {
+ let plain = "%YAML 1.2\n---\nkey: value\nother: 1\nnested:\n a: 2\n";
+ assert_ne!(sniff_str(plain, "doc.yaml"), Family::UnityYaml);
+ }
+
+ /// Regression: PSD image data decoded via lossy windows-1252 passed the
+ /// NUL/control-char heuristics and classified as txt-prose, turning a
+ /// multi-MB texture into ~100x its size of prose sections. Media magic
+ /// bytes must win before any text heuristic runs.
+ #[test]
+ fn media_containers_are_binary_by_magic_not_heuristics() {
+ for (name, head) in [
+ ("t.psd", &b"8BPS\x00\x01"[..]),
+ ("t.tif", &b"II*\x00\x08\x00"[..]),
+ ("t.tif2", &b"MM\x00*\x00\x08"[..]),
+ ("t.wav", &b"RIFF\x24\x08\x00\x00WAVE"[..]),
+ ("t.ogg", &b"OggS\x00\x02"[..]),
+ ("t.fbx", &b"Kaydara FBX Binary \x00"[..]),
+ ] {
+ let mut bytes = head.to_vec();
+ // A printable tail that WOULD pass the prose heuristics.
+ bytes.extend(b"lorem ipsum dolor sit amet, consectetur adipiscing elit. ".repeat(20));
+ let sn = sniff_bytes(&bytes, Path::new(name), false).unwrap();
+ assert_eq!(sn.family, Family::Binary, "{name} must be binary");
+ }
+ }
+
+ /// Regression: an uncompressed TGA (no magic bytes exist for TGA) decoded
+ /// via lossy windows-1252 into printable soup with near-zero whitespace
+ /// and classified txt-prose. Whitespace ratio is the discriminator.
+ #[test]
+ fn whitespace_free_printable_soup_is_binary_not_prose() {
+ let soup: String = (0..8192u32)
+ .map(|i| char::from_u32(0x21 + (i * 7) % 0x5d).unwrap())
+ .collect();
+ let sn = sniff_bytes(soup.as_bytes(), Path::new("texture.tga"), false).unwrap();
+ assert_eq!(sn.family, Family::Binary);
+ // Real prose of the same size keeps its family.
+ let prose = "The quick brown fox jumps over the lazy dog. ".repeat(200);
+ let sn = sniff_bytes(prose.as_bytes(), Path::new("note.txt"), false).unwrap();
+ assert_ne!(sn.family, Family::Binary, "real prose must stay text");
+ }
+
+ #[test]
+ fn binary_serialized_unity_assets_stay_binary() {
+ let mut bytes = b"UnityFS\x00\x00\x00\x00\x08".to_vec();
+ bytes.extend(std::iter::repeat(0u8).take(64));
+ let sn = sniff_bytes(&bytes, Path::new("scene.unity"), false).unwrap();
+ assert_eq!(sn.family, Family::Binary);
+ }
+}
+
#[cfg(test)]
mod tests {
use super::*;
diff --git a/engine/crates/xerj-autoindex/src/walk.rs b/engine/crates/xerj-autoindex/src/walk.rs
index 27278f2c5..e316aafc0 100644
--- a/engine/crates/xerj-autoindex/src/walk.rs
+++ b/engine/crates/xerj-autoindex/src/walk.rs
@@ -17,13 +17,72 @@ pub struct FileEntry {
pub size: u64,
}
-pub fn walk(root: &Path, follow_symlinks: bool) -> Result<Vec<FileEntry>> {
+/// A directory pruned whole by the generated-dir hygiene rule, recorded so
+/// the run can surface it (junk report / map) instead of skipping silently.
+#[derive(Debug, Clone)]
+pub struct SkippedDir {
+ pub rel: String,
+ pub reason: String,
+}
+
+/// Well-known generated/cache directories, pruned at ANY depth but ONLY when
+/// a sibling marker file proves the parent directory is that kind of project
+/// (a monorepo may nest e.g. `apps/game/Library`, `apps/web/node_modules`).
+/// Marker-gated like the dotfile rule (hygiene, evidence-based) — never
+/// extension guessing, never a per-corpus code path.
+/// `--no-default-excludes` disables the rule.
+///
+/// (dir name, marker path relative to the dir's PARENT, reason)
+const GENERATED_DIR_RULES: &[(&str, &str, &str)] = &[
+ (
+ "Library",
+ "ProjectSettings/ProjectVersion.txt",
+ "Unity generated/cache directory (marker: sibling ProjectSettings/ProjectVersion.txt)",
+ ),
+ (
+ "Temp",
+ "ProjectSettings/ProjectVersion.txt",
+ "Unity generated/cache directory (marker: sibling ProjectSettings/ProjectVersion.txt)",
+ ),
+ (
+ "obj",
+ "ProjectSettings/ProjectVersion.txt",
+ "Unity generated/cache directory (marker: sibling ProjectSettings/ProjectVersion.txt)",
+ ),
+ (
+ "Logs",
+ "ProjectSettings/ProjectVersion.txt",
+ "Unity generated/cache directory (marker: sibling ProjectSettings/ProjectVersion.txt)",
+ ),
+ (
+ "UserSettings",
+ "ProjectSettings/ProjectVersion.txt",
+ "Unity generated/cache directory (marker: sibling ProjectSettings/ProjectVersion.txt)",
+ ),
+ (
+ "node_modules",
+ "package.json",
+ "installed npm/yarn dependency tree (marker: sibling package.json)",
+ ),
+ (
+ "target",
+ "Cargo.toml",
+ "Cargo build directory (marker: sibling Cargo.toml)",
+ ),
+];
+
+pub fn walk(
+ root: &Path,
+ follow_symlinks: bool,
+ default_excludes: bool,
+) -> Result<(Vec<FileEntry>, Vec<SkippedDir>)> {
let root_canon = root
.canonicalize()
.with_context(|| format!("resolve root folder {}", root.display()))?;
if !root_canon.is_dir() {
anyhow::bail!("{} is not a directory", root_canon.display());
}
+ let skipped: std::rc::Rc<std::cell::RefCell<Vec<SkippedDir>>> = Default::default();
let mut out = Vec::new();
// SECURITY / hygiene: never index hidden files or descend into hidden
// directories. Without this the walker happily indexed `.env` (secrets,
@@ -32,11 +91,40 @@ pub fn walk(root: &Path, follow_symlinks: bool) -> Result<Vec<FileEntry>> {
// "point it at my project folder" use case. `filter_entry` prunes a hidden
// directory before descending, so `.git/` is skipped whole. The root itself
// (depth 0) is exempt so a brain over a dot-named folder still works.
+ let sk = std::rc::Rc::clone(&skipped);
+ let filter_root = root_canon.clone();
let walker = walkdir::WalkDir::new(&root_canon)
.follow_links(follow_symlinks)
.into_iter()
- .filter_entry(|e| {
- e.depth() == 0 || !e.file_name().to_str().is_some_and(|n| n.starts_with('.'))
+ .filter_entry(move |e| {
+ if e.depth() == 0 {
+ return true;
+ }
+ if e.file_name().to_str().is_some_and(|n| n.starts_with('.')) {
+ return false;
+ }
+ if default_excludes && e.file_type().is_dir() {
+ for (name, marker, reason) in GENERATED_DIR_RULES {
+ if e.file_name() == *name
+ && e.path()
+ .parent()
+ .is_some_and(|p| p.join(marker).is_file())
+ {
+ let rel = e
+ .path()
+ .strip_prefix(&filter_root)
+ .unwrap_or(e.path())
+ .to_string_lossy()
+ .replace('\\', "/");
+ sk.borrow_mut().push(SkippedDir {
+ rel,
+ reason: (*reason).to_string(),
+ });
+ return false;
+ }
+ }
+ }
+ true
});
for entry in walker {
let entry = match entry {
@@ -73,7 +161,10 @@ pub fn walk(root: &Path, follow_symlinks: bool) -> Result<Vec<FileEntry>> {
}
// Deterministic order for clustering / naming.
out.sort_by(|a, b| a.rel.cmp(&b.rel));
- Ok(out)
+ let skipped = std::rc::Rc::try_unwrap(skipped)
+ .expect("walker iterator consumed; sole owner remains")
+ .into_inner();
+ Ok((out, skipped))
}
#[cfg(unix)]
@@ -120,8 +211,9 @@ mod hidden_skip_tests {
fs::write(root.join("src").join("main.rs"), "fn main() {}").unwrap();
fs::write(root.join("src").join(".secret"), "nope").unwrap();
- let rels: Vec<String> = walk(root, false)
+ let rels: Vec<String> = walk(root, false, true)
.unwrap()
+ .0
.into_iter()
.map(|e| e.rel)
.collect();
@@ -145,8 +237,9 @@ mod hidden_skip_tests {
let root = dir.path().join(".notes");
fs::create_dir(&root).unwrap();
fs::write(root.join("a.md"), "x").unwrap();
- let rels: Vec<String> = walk(&root, false)
+ let rels: Vec<String> = walk(&root, false, true)
.unwrap()
+ .0
.into_iter()
.map(|e| e.rel)
.collect();
@@ -157,3 +250,142 @@ mod hidden_skip_tests {
);
}
}
+
+#[cfg(test)]
+mod generated_dir_tests {
+ use super::walk;
+ use std::fs;
+ use std::path::Path;
+
+ fn make_unity_root(root: &Path, with_marker: bool) {
+ fs::create_dir_all(root.join("Assets")).unwrap();
+ fs::write(root.join("Assets/Player.cs"), "class Player {}").unwrap();
+ fs::create_dir_all(root.join("Library/Artifacts")).unwrap();
+ fs::write(root.join("Library/Artifacts/blob.bin"), b"\x00\x01").unwrap();
+ fs::create_dir_all(root.join("Temp")).unwrap();
+ fs::write(root.join("Temp/scratch"), "x").unwrap();
+ if with_marker {
+ fs::create_dir_all(root.join("ProjectSettings")).unwrap();
+ fs::write(
+ root.join("ProjectSettings/ProjectVersion.txt"),
+ "m_EditorVersion: 2022.3.10f1\n",
+ )
+ .unwrap();
+ }
+ }
+
+ #[test]
+ fn unity_marker_prunes_generated_dirs_and_records_them() {
+ let dir = tempfile::TempDir::new().unwrap();
+ make_unity_root(dir.path(), true);
+ let (files, skipped) = walk(dir.path(), false, true).unwrap();
+ let rels: Vec<&str> = files.iter().map(|e| e.rel.as_str()).collect();
+ assert!(rels.contains(&"Assets/Player.cs"));
+ assert!(
+ !rels.iter().any(|r| r.starts_with("Library/")),
+ "Library/ must be pruned whole: {rels:?}"
+ );
+ assert!(!rels.iter().any(|r| r.starts_with("Temp/")));
+ let mut skipped_rels: Vec<&str> = skipped.iter().map(|s| s.rel.as_str()).collect();
+ skipped_rels.sort();
+ assert_eq!(skipped_rels, ["Library", "Temp"], "each pruned dir is recorded");
+ }
+
+ #[test]
+ fn without_the_marker_nothing_is_pruned() {
+ let dir = tempfile::TempDir::new().unwrap();
+ make_unity_root(dir.path(), false);
+ let (files, skipped) = walk(dir.path(), false, true).unwrap();
+ let rels: Vec<&str> = files.iter().map(|e| e.rel.as_str()).collect();
+ assert!(
+ rels.iter().any(|r| r.starts_with("Library/")),
+ "a folder merely NAMED Library is not evidence: {rels:?}"
+ );
+ assert!(skipped.is_empty());
+ }
+
+ #[test]
+ fn no_default_excludes_walks_everything() {
+ let dir = tempfile::TempDir::new().unwrap();
+ make_unity_root(dir.path(), true);
+ let (files, skipped) = walk(dir.path(), false, false).unwrap();
+ let rels: Vec<&str> = files.iter().map(|e| e.rel.as_str()).collect();
+ assert!(rels.iter().any(|r| r.starts_with("Library/")));
+ assert!(skipped.is_empty());
+ }
+
+ #[test]
+ fn a_file_named_like_a_generated_dir_is_not_pruned() {
+ let dir = tempfile::TempDir::new().unwrap();
+ make_unity_root(dir.path(), true);
+ fs::write(dir.path().join("obj"), "a FILE named obj").unwrap();
+ let (files, _) = walk(dir.path(), false, true).unwrap();
+ assert!(
+ files.iter().any(|e| e.rel == "obj"),
+ "the prune applies to directories only"
+ );
+ }
+
+ /// Monorepo case: the Unity project is a SUBFOLDER of the walk root. The
+ /// marker is checked against each candidate's parent, so nesting depth is
+ /// irrelevant.
+ #[test]
+ fn a_nested_unity_project_is_pruned_by_its_sibling_marker() {
+ let dir = tempfile::TempDir::new().unwrap();
+ let unity = dir.path().join("apps/game");
+ fs::create_dir_all(&unity).unwrap();
+ make_unity_root(&unity, true);
+ fs::write(dir.path().join("README.md"), "monorepo").unwrap();
+ let (files, skipped) = walk(dir.path(), false, true).unwrap();
+ let rels: Vec<&str> = files.iter().map(|e| e.rel.as_str()).collect();
+ assert!(rels.contains(&"apps/game/Assets/Player.cs"));
+ assert!(!rels.iter().any(|r| r.contains("game/Library/")));
+ assert!(skipped.iter().any(|s| s.rel == "apps/game/Library"));
+ }
+
+ #[test]
+ fn node_modules_is_pruned_only_next_to_a_package_json() {
+ let dir = tempfile::TempDir::new().unwrap();
+ let app = dir.path().join("web");
+ fs::create_dir_all(app.join("node_modules/lodash")).unwrap();
+ fs::write(app.join("node_modules/lodash/index.js"), "x").unwrap();
+ fs::write(app.join("package.json"), "{}").unwrap();
+ fs::write(app.join("index.js"), "x").unwrap();
+ let bare = dir.path().join("data/node_modules");
+ fs::create_dir_all(&bare).unwrap();
+ fs::write(bare.join("notes.txt"), "not an npm tree").unwrap();
+ let (files, skipped) = walk(dir.path(), false, true).unwrap();
+ let rels: Vec<&str> = files.iter().map(|e| e.rel.as_str()).collect();
+ assert!(rels.contains(&"web/index.js"));
+ assert!(!rels.iter().any(|r| r.starts_with("web/node_modules")));
+ assert!(
+ rels.contains(&"data/node_modules/notes.txt"),
+ "without the package.json marker there is no evidence: {rels:?}"
+ );
+ assert_eq!(skipped.len(), 1);
+ assert_eq!(skipped[0].rel, "web/node_modules");
+ }
+
+ #[test]
+ fn cargo_target_is_pruned_only_next_to_a_cargo_toml() {
+ let dir = tempfile::TempDir::new().unwrap();
+ let crate_dir = dir.path().join("mycrate");
+ fs::create_dir_all(crate_dir.join("target/debug")).unwrap();
+ fs::write(crate_dir.join("target/debug/build.log"), "x").unwrap();
+ fs::write(crate_dir.join("Cargo.toml"), "[package]").unwrap();
+ fs::write(crate_dir.join("main.rs"), "fn main() {}").unwrap();
+ let plain = dir.path().join("shooting/target");
+ fs::create_dir_all(&plain).unwrap();
+ fs::write(plain.join("scores.csv"), "a,b\n1,2\n").unwrap();
+ let (files, skipped) = walk(dir.path(), false, true).unwrap();
+ let rels: Vec<&str> = files.iter().map(|e| e.rel.as_str()).collect();
+ assert!(rels.contains(&"mycrate/main.rs"));
+ assert!(!rels.iter().any(|r| r.starts_with("mycrate/target")));
+ assert!(
+ rels.contains(&"shooting/target/scores.csv"),
+ "a folder merely NAMED target is data, not a build dir: {rels:?}"
+ );
+ assert_eq!(skipped.len(), 1);
+ assert_eq!(skipped[0].rel, "mycrate/target");
+ }
+}
diff --git a/engine/crates/xerj-server/src/brain.rs b/engine/crates/xerj-server/src/brain.rs
index 68f2a25d9..8fd418ab9 100644
--- a/engine/crates/xerj-server/src/brain.rs
+++ b/engine/crates/xerj-server/src/brain.rs
@@ -177,7 +177,7 @@ fn run(cfg: BrainCfg) -> Result<i32> {
if !cfg.root.exists() {
bail!("{} does not exist", cfg.root.display());
}
- let files = walk::walk(&cfg.root, false)?;
+ let (files, _skipped_dirs) = walk::walk(&cfg.root, false, true)?;
if files.is_empty() {
eprintln!(
"xerj brain: no files under {} — nothing to build a brain from.\n\
@@ -432,6 +432,7 @@ fn index_cfg(cfg: &BrainCfg, brain: &str, api_key: Option<String>) -> IndexCfg {
state_dir: None,
fresh: cfg.fresh,
follow_symlinks: false,
+ no_default_excludes: false,
max_file_gb: 2,
sample: 500,
no_semantic: false,