AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
feat(autoindex): first-class Unity project indexing
Sergey Gonchar33147589838c
diff --git a/README.md b/README.mdindex bd71298f0..1d6d12b9e 100644--- a/README.md+++ b/README.md@@ -56,7 +56,12 @@ done in 158.1s, 593 datasets, 83103 records live, 790 junk recordsSource files go through tree-sitter, so code arrives with its symbols and linenumbers instead of as flat text. CSV, JSON, JSONL, XML, YAML, SQLite, PDF, DOCX,-HTML and common log formats are all handled.+HTML and common log formats are all handled. Unity projects get first-class+treatment: text-serialized scenes, prefabs and assets become one record per+GameObject/Component, `.meta` files become a GUID↔path table, and MonoBehaviour+records carry `script_class`/`script_path` so "which scenes use this script?"+is a single query (binary-serialized assets need Force Text to be readable;+generated dirs like `Library/` are auto-skipped and recorded).## Search itdiff --git a/demo/usecases/autoindex/README.md b/demo/usecases/autoindex/README.mdindex ce5526d4f..6e78f810f 100644--- a/demo/usecases/autoindex/README.md+++ b/demo/usecases/autoindex/README.md@@ -104,8 +104,18 @@ walk → sniff → sample → infer → map → extract → bulk → correlatehint at best, never trusted. Formats covered by streaming extractors:JSON/JSONL, CSV (dialect detection: comma/semicolon/tab, decimal comma,quoted multiline fields, BOM), structured logs (nginx CLF, app logs,- syslog), SQL dumps, SQLite, PDF, DOCX, HTML, XML, YAML, plain text,- gzip-compressed variants of the above.+ syslog), SQL dumps, SQLite, PDF, DOCX, HTML, XML, YAML, Unity+ text-serialized assets (scenes/prefabs/.asset/.mat/.anim → one record per+ GameObject/Component with `unity_class`, `file_id`, guid references, and a+ denormalized `script_class`/`script_path` on MonoBehaviours; `.meta`+ sidecars → a guid↔asset-path table; detected by the `%TAG !u!` header,+ never extension — binary-serialized Unity assets stay junk, enable Force+ Text serialization; generated dirs are pruned and recorded when a sibling+ marker proves them, at any nesting depth — Unity `Library/`, `Temp/`,+ `obj/`, `Logs/`, `UserSettings/` next to+ `ProjectSettings/ProjectVersion.txt`, `node_modules/` next to+ `package.json`, `target/` next to `Cargo.toml`; `--no-default-excludes`+ walks them), plain text, gzip-compressed variants of the above.3. **Sample** — bounded per-file sampling (default 500 records,`--sample N`); inference cost does not grow with file size.4. **Infer** — field types (long/double/boolean/date/keyword/text), datediff --git a/engine/crates/xerj-autoindex/src/cli.rs b/engine/crates/xerj-autoindex/src/cli.rsindex ee81b6a7b..efe4ac8ef 100644--- a/engine/crates/xerj-autoindex/src/cli.rs+++ b/engine/crates/xerj-autoindex/src/cli.rs@@ -16,6 +16,8 @@ pub struct IndexCfg {pub state_dir: Option<PathBuf>,pub fresh: bool,pub follow_symlinks: bool,+ /// Disable the marker-gated generated-dir prune (e.g. Unity `Library/`).+ pub no_default_excludes: bool,pub max_file_gb: u64,pub sample: usize,pub no_semantic: bool,@@ -76,6 +78,14 @@ pub fn print_help() {--state-dir <PATH> resume journal location (default ~/.xerj/autoindex/<hash>/)\n\--fresh ignore existing journal, restart (ids stay idempotent)\n\--follow-symlinks follow symlinks (loop-safe); off by default\n\+ --no-default-excludes\n\+ walk generated/cache dirs too. By default,\n\+ well-known generated dirs are skipped (and\n\+ recorded) when a sibling marker proves them, at\n\+ any depth: Unity Library/Temp/obj/Logs/\n\+ UserSettings next to ProjectSettings/\n\+ ProjectVersion.txt, node_modules next to\n\+ package.json, target next to Cargo.toml\n\--max-file-gb <N> skip+record oversized non-streamable files (default 2)\n\--sample <N> records sampled per file for inference (default 500)\n\--no-semantic skip semantic_text on body fields (pure BM25+keyword)\n\@@ -137,6 +147,7 @@ pub fn parse(args: Vec<String>) -> Result<Cmd, String> {let mut state_dir: Option<PathBuf> = None;let mut fresh = false;let mut follow_symlinks = false;+ let mut no_default_excludes = false;let mut max_file_gb = 2u64;let mut sample = 500usize;let mut no_semantic = false;@@ -195,6 +206,7 @@ pub fn parse(args: Vec<String>) -> Result<Cmd, String> {"--state-dir" => state_dir = it.next().map(PathBuf::from),"--fresh" => fresh = true,"--follow-symlinks" => follow_symlinks = true,+ "--no-default-excludes" => no_default_excludes = true,"--max-file-gb" => {max_file_gb = it.next()@@ -270,6 +282,7 @@ pub fn parse(args: Vec<String>) -> Result<Cmd, String> {state_dir,fresh,follow_symlinks,+ no_default_excludes,max_file_gb,sample: sample.max(50),no_semantic,diff --git a/engine/crates/xerj-autoindex/src/content.rs b/engine/crates/xerj-autoindex/src/content.rsindex 4f865b47a..4111e7b22 100644--- a/engine/crates/xerj-autoindex/src/content.rs+++ b/engine/crates/xerj-autoindex/src/content.rs@@ -187,7 +187,7 @@ mod tests {let dir = tempfile::tempdir().unwrap();fs::write(dir.path().join("a.txt"), b"same records\n").unwrap();fs::write(dir.path().join("b.txt"), b"same records\n").unwrap();- let inv = resolve(crate::walk::walk(dir.path(), false).unwrap()).unwrap();+ let inv = resolve(crate::walk::walk(dir.path(), false, true).unwrap().0).unwrap();assert_eq!(inv.files.len(), 1);assert_eq!(inv.files[0].rel, "a.txt");assert_eq!(inv.duplicates.len(), 1);@@ -204,7 +204,7 @@ mod tests {second[65_536] = b'b';fs::write(dir.path().join("a.txt"), first).unwrap();fs::write(dir.path().join("b.txt"), second).unwrap();- let inv = resolve(crate::walk::walk(dir.path(), false).unwrap()).unwrap();+ let inv = resolve(crate::walk::walk(dir.path(), false, true).unwrap().0).unwrap();assert_eq!(inv.files.len(), 2);assert!(inv.duplicates.is_empty());assert_ne!(inv.keys[0], inv.keys[1]);@@ -215,7 +215,7 @@ mod tests {let dir = tempfile::tempdir().unwrap();let path = dir.path().join("a.txt");fs::write(&path, b"content-a").unwrap();- let inv = resolve(crate::walk::walk(dir.path(), false).unwrap()).unwrap();+ let inv = resolve(crate::walk::walk(dir.path(), false, true).unwrap().0).unwrap();fs::write(&path, b"content-b").unwrap();assert!(verify(&path, 9, &inv.digests[0]).is_err());}@@ -225,7 +225,7 @@ mod tests {let dir = tempfile::tempdir().unwrap();fs::write(dir.path().join("a.txt"), b"unequal-a").unwrap();fs::write(dir.path().join("b.txt"), b"unequal-b").unwrap();- let files = crate::walk::walk(dir.path(), false).unwrap();+ let files = crate::walk::walk(dir.path(), false, true).unwrap().0;let forced = vec!["axf2-forced-9".to_string(); 2];let inv = resolve_with_digests(files, forced).unwrap();assert_eq!(inv.files.len(), 2);@@ -242,7 +242,7 @@ mod tests {let b = dir.path().join("b.txt");fs::write(&a, b"hardlinked").unwrap();fs::hard_link(&a, &b).unwrap();- let inv = resolve(crate::walk::walk(dir.path(), false).unwrap()).unwrap();+ let inv = resolve(crate::walk::walk(dir.path(), false, true).unwrap().0).unwrap();assert_eq!(inv.files.len(), 1);assert_eq!(inv.duplicates.len(), 1);}@@ -264,7 +264,7 @@ mod tests {b"same",).unwrap();- let inv = resolve(crate::walk::walk(dir.path(), false).unwrap()).unwrap();+ let inv = resolve(crate::walk::walk(dir.path(), false, true).unwrap().0).unwrap();assert_eq!(inv.files.len(), 1);assert_eq!(inv.duplicates.len(), 2);assert_ne!(inv.duplicates[0].path_id, inv.duplicates[1].path_id);@@ -278,7 +278,7 @@ mod tests {let real = dir.path().join("z-real");fs::write(&real, b"same").unwrap();symlink(&real, dir.path().join("a-link")).unwrap();- let inv = resolve(crate::walk::walk(dir.path(), true).unwrap()).unwrap();+ let inv = resolve(crate::walk::walk(dir.path(), true, true).unwrap().0).unwrap();assert_eq!(inv.files[0].rel, "z-real");assert_eq!(inv.duplicates[0].rel, "a-link");}diff --git a/engine/crates/xerj-autoindex/src/detect/e2e.rs b/engine/crates/xerj-autoindex/src/detect/e2e.rsindex 7e76b671b..30b95dede 100644--- a/engine/crates/xerj-autoindex/src/detect/e2e.rs+++ b/engine/crates/xerj-autoindex/src/detect/e2e.rs@@ -35,7 +35,12 @@ impl MockEs {let (sd, ss) = (Arc::clone(&docs), Arc::clone(&stop));let join = thread::spawn(move || loop {match listener.accept() {- Ok((stream, _)) => handle(stream, &sd),+ Ok((stream, _)) => {+ // BSD/macOS: accepted sockets inherit the listener's+ // O_NONBLOCK; the handler does blocking reads.+ stream.set_nonblocking(false).unwrap();+ handle(stream, &sd)+ }Err(e) if e.kind() == std::io::ErrorKind::WouldBlock => {if *ss.lock().unwrap() {break;@@ -265,6 +270,7 @@ fn cfg(root: &Path, state_dir: &Path, url: &str) -> IndexCfg {state_dir: Some(state_dir.to_owned()),fresh: false,follow_symlinks: false,+ no_default_excludes: false,max_file_gb: 1,sample: 50,no_semantic: true,diff --git a/engine/crates/xerj-autoindex/src/extract/mod.rs b/engine/crates/xerj-autoindex/src/extract/mod.rsindex b18e959f6..a3c8c6199 100644--- a/engine/crates/xerj-autoindex/src/extract/mod.rs+++ b/engine/crates/xerj-autoindex/src/extract/mod.rs@@ -13,6 +13,7 @@ pub mod pdf;pub mod sqldump;pub mod sqlite_x;pub mod txt;+pub mod unity;pub mod xml_x;pub mod yaml_x;@@ -139,6 +140,8 @@ pub fn extract(Family::Sqlite => sqlite_x::extract(path, limit_bytes.map(|_| 500), sink),Family::SqlDump => sqldump::extract(path, sn.gzip, limit_bytes, sink),Family::Code => code::extract(path, sink),+ Family::UnityYaml => unity::extract_unity(path, sn.gzip, limit_bytes, sink),+ Family::UnityMeta => unity::extract_meta(path, sn.gzip, sink),Family::Binary => Ok(ExtractStats::default()),}}@@ -205,6 +208,46 @@ fn flatten_into(key: &str, v: Value, depth: usize, out: &mut Map<String, Value>)}}+/// Convert a parsed YAML value into JSON (shared by the YAML and Unity+/// extractors). Tagged values unwrap to their inner value; non-string mapping+/// keys are stringified.+pub(crate) fn yaml_to_json(v: serde_yaml::Value) -> Value {+ match v {+ serde_yaml::Value::Null => Value::Null,+ serde_yaml::Value::Bool(b) => Value::Bool(b),+ serde_yaml::Value::Number(n) => {+ if let Some(i) = n.as_i64() {+ Value::Number(i.into())+ } else if let Some(u) = n.as_u64() {+ Value::Number(u.into())+ } else {+ serde_json::Number::from_f64(n.as_f64().unwrap_or(0.0))+ .map(Value::Number)+ .unwrap_or(Value::Null)+ }+ }+ serde_yaml::Value::String(s) => Value::String(s),+ serde_yaml::Value::Sequence(seq) => {+ Value::Array(seq.into_iter().map(yaml_to_json).collect())+ }+ serde_yaml::Value::Mapping(m) => {+ let mut out = Map::new();+ for (k, vv) in m {+ let key = match k {+ serde_yaml::Value::String(s) => s,+ other => serde_yaml::to_string(&other)+ .unwrap_or_else(|_| "key".into())+ .trim()+ .to_string(),+ };+ out.insert(key, yaml_to_json(vv));+ }+ Value::Object(out)+ }+ serde_yaml::Value::Tagged(t) => yaml_to_json(t.value),+ }+}+/// Sanitize a discovered field/column name to a safe ES field name.pub fn sanitize_field_name(name: &str) -> String {let mut s: String = namediff --git a/engine/crates/xerj-autoindex/src/extract/unity.rs b/engine/crates/xerj-autoindex/src/extract/unity.rsnew file mode 100644index 000000000..3903ca774--- /dev/null+++ b/engine/crates/xerj-autoindex/src/extract/unity.rs@@ -0,0 +1,536 @@+//! Unity text-serialized assets — streaming multi-document splitter — and+//! `.meta` sidecars.+//!+//! Unity YAML is a multi-document stream where every document opens with+//! `--- !u!<classId> &<fileID>` (optionally suffixed `stripped` for prefab+//! instances). serde_yaml cannot be used on the whole stream: it resolves+//! anchors internally and never exposes the anchor NAME, and the `fileID`+//! that makes Unity objects joinable lives in that anchor. So the splitter+//! owns the document boundaries (a cheap line prefix match) and hands each+//! body — plain YAML once the `%YAML`/`%TAG` header is dropped — to+//! serde_yaml individually. Owning the boundaries is also what makes it safe+//! to `continue` past a corrupt document: unlike `yaml_x`, there is no+//! iterator that would re-yield the same broken document forever.+//!+//! Files stream line-by-line (real scenes can exceed 200 MB; a single+//! AnimationClip document can be tens of MB), with a 16 MiB per-document+//! cap: over-cap documents are junked and skipped to the next boundary,+//! the rest of the file survives.+//!+//! Emitted fields per object document:+//! - `unity_class` body's top-level key (authoritative), else class-id table+//! - `unity_class_id` from the `!u!<id>` tag+//! - `file_id` from the `&<anchor>` — the join key for local refs+//! - `stripped` only present (true) on stripped prefab-instance docs+//! - `name` `m_Name` when present+//! - `ref_guids` deduped guids of every `{fileID, guid, type}` reference+//! - `script_guid` the `m_Script` reference's guid (MonoBehaviour → .cs)+//! - flattened body fields+//!+//! `.meta` sidecars emit ONE record: `guid`, `asset_name`, `importer`, and+//! flattened settings. The root-relative `asset_path` (and the+//! `script_path`/`script_class` denormalization on MonoBehaviour records) is+//! stamped by the pipeline, which knows the walk root — extractors stay+//! per-file pure.++use super::{+ flatten_object, yaml_to_json, ExtractStats, FieldOrigin, RawRecord, Sink, MAX_LINE,+};+use anyhow::Result;+use serde_json::{Map, Value};+use std::path::Path;++/// Per-document cap. One logical unit follows the `MAX_LINE` convention;+/// 16 MiB covers large AnimationClip / font documents while bounding memory.+/// (A 4 MiB cap was considered and rejected: it junks those documents.)+const DOC_CAP: usize = 16 << 20;++const META_CAP: u64 = 16 << 20;++/// Field names this extractor invents on object records. A body field with+/// the same name (possible on MonoBehaviour custom fields) is renamed+/// `data_<name>` rather than dropped, mirroring the `ax_*` collision policy.+const RESERVED: &[&str] = &[+ "unity_class",+ "unity_class_id",+ "file_id",+ "stripped",+ "name",+ "ref_guids",+ "script_guid",+ "script_path",+ "script_class",+];++pub fn extract_unity(+ path: &Path,+ gzip: bool,+ limit_bytes: Option<u64>,+ sink: Sink,+) -> Result<ExtractStats> {+ let mut r = super::open_reader(path, gzip, limit_bytes)?;+ let mut stats = ExtractStats::default();+ let mut offset: u64 = 0;+ let mut line: Vec<u8> = Vec::new();+ let mut doc: Vec<u8> = Vec::new();+ let mut header: Option<DocHeader> = None;+ let mut skipping = false;+ loop {+ line.clear();+ let n = super::jsonl::read_capped_line(&mut r, &mut line)?;+ let eof = n == 0;+ offset += n as u64;+ let boundary = if eof { None } else { parse_boundary(&line) };+ if eof || boundary.is_some() {+ if let Some(h) = header.take() {+ if skipping {+ skipping = false;+ } else {+ let truncated =+ eof && limit_bytes.is_some_and(|l| offset >= l);+ if !emit_doc(&h, &doc, truncated, &mut stats, sink) {+ return Ok(stats);+ }+ }+ }+ doc.clear();+ header = boundary;+ if eof {+ break;+ }+ continue;+ }+ if header.is_none() || skipping {+ continue; // %YAML/%TAG preamble, or discarding an over-cap doc+ }+ if doc.len() + line.len() > DOC_CAP {+ stats.junk += 1;+ skipping = true;+ doc.clear();+ continue;+ }+ doc.extend_from_slice(&line);+ }+ // Empty yield from a FULL read means the file was preamble-only (or+ // junk-shaped throughout) — record it. A sampling read that truncated+ // before the first document completed is not evidence of anything.+ let truncated_run = limit_bytes.is_some_and(|l| offset >= l);+ if stats.records == 0 && stats.junk == 0 && !truncated_run {+ stats.junk += 1;+ }+ Ok(stats)+}++struct DocHeader {+ class_id: i64,+ /// Raw anchor text (handles negative 64-bit ids verbatim).+ file_id: String,+ stripped: bool,+}++/// `--- !u!<classId> &<fileID>[ stripped]` — cheap manual parse, no regex on+/// the hot per-line path.+fn parse_boundary(line: &[u8]) -> Option<DocHeader> {+ let s = std::str::from_utf8(line).ok()?.trim_end();+ let rest = s.strip_prefix("--- !u!")?;+ let (id_part, rest) = rest.split_once(" &")?;+ let class_id: i64 = id_part.trim().parse().ok()?;+ let (anchor, tail) = match rest.split_once(' ') {+ Some((a, t)) => (a, t.trim()),+ None => (rest, ""),+ };+ let anchor = anchor.trim();+ if anchor.is_empty()+ || !anchor+ .chars()+ .all(|c| c.is_ascii_digit() || c == '-')+ {+ return None;+ }+ Some(DocHeader {+ class_id,+ file_id: anchor.to_string(),+ stripped: tail == "stripped",+ })+}++fn emit_doc(+ h: &DocHeader,+ body: &[u8],+ truncated_by_sampling: bool,+ stats: &mut ExtractStats,+ sink: Sink,+) -> bool {+ let text = String::from_utf8_lossy(body);+ let parsed: serde_yaml::Value = match serde_yaml::from_str(&text) {+ Ok(v) => v,+ Err(_) => {+ if !truncated_by_sampling {+ stats.junk += 1;+ }+ return true;+ }+ };+ let jv = yaml_to_json(parsed);+ let (class_from_body, inner) = match jv {+ Value::Object(m) if m.len() == 1 => {+ let (k, v) = m.into_iter().next().expect("len checked");+ (Some(k), v)+ }+ other => (None, other),+ };+ let unity_class = class_from_body+ .unwrap_or_else(|| class_name(h.class_id).map(str::to_string).unwrap_or_else(|| format!("class_{}", h.class_id)));++ let mut refs: Vec<String> = Vec::new();+ let mut script_guid: Option<String> = None;+ collect_guid_refs(&inner, &mut refs);+ if let Some(script) = inner.get("m_Script") {+ if let Some(g) = script.get("guid").and_then(Value::as_str) {+ script_guid = Some(g.to_string());+ }+ }++ let mut fields = match inner {+ Value::Object(m) => flatten_object(m),+ Value::Null => Map::new(),+ other => {+ let mut m = Map::new();+ m.insert("value".into(), other);+ m+ }+ };+ for k in RESERVED {+ if let Some(v) = fields.remove(*k) {+ fields.insert(format!("data_{k}"), v);+ }+ }+ if let Some(Value::String(n)) = fields.get("m_Name") {+ if !n.is_empty() {+ fields.insert("name".into(), Value::String(n.clone()));+ }+ }+ fields.insert("unity_class".into(), Value::String(unity_class.clone()));+ fields.insert("unity_class_id".into(), Value::Number(h.class_id.into()));+ fields.insert("file_id".into(), Value::String(h.file_id.clone()));+ if h.stripped {+ fields.insert("stripped".into(), Value::Bool(true));+ }+ if !refs.is_empty() {+ let mut seen = std::collections::HashSet::new();+ let deduped: Vec<Value> = refs+ .into_iter()+ .filter(|g| seen.insert(g.clone()))+ .map(Value::String)+ .collect();+ fields.insert("ref_guids".into(), Value::Array(deduped));+ }+ if let Some(g) = script_guid {+ fields.insert("script_guid".into(), Value::String(g));+ }++ stats.records += 1;+ sink(RawRecord {+ fields,+ locator: format!("u{}", h.file_id),+ group: Some(unity_class),+ // Body field names are Unity's serialization vocabulary plus per-script+ // custom fields; letting them cluster would split MonoBehaviours into+ // schema-similarity datasets and re-home documents when scripts change+ // (issue #178). Family+group clustering gives one stable dataset per+ // Unity class instead.+ origin: FieldOrigin::Extractor,+ })+}++/// Walk the parsed document for `{fileID, guid, type}` reference mappings and+/// collect every guid, including inside arrays (`m_Modifications` — prefab+/// overrides — is where the highest-value references live).+fn collect_guid_refs(v: &Value, out: &mut Vec<String>) {+ match v {+ Value::Object(m) => {+ if m.contains_key("fileID") {+ if let Some(g) = m.get("guid").and_then(Value::as_str) {+ out.push(g.to_string());+ }+ }+ for vv in m.values() {+ collect_guid_refs(vv, out);+ }+ }+ Value::Array(a) => {+ for vv in a {+ collect_guid_refs(vv, out);+ }+ }+ _ => {}+ }+}++pub fn extract_meta(path: &Path, gzip: bool, sink: Sink) -> Result<ExtractStats> {+ let mut stats = ExtractStats::default();+ let Some(bytes) = super::read_whole(path, gzip, META_CAP)? else {+ stats.junk += 1;+ return Ok(stats);+ };+ let (text, _) = crate::sniff::decode_text(&bytes);+ let text = text.trim_start_matches('\u{feff}');+ let parsed: serde_yaml::Value = match serde_yaml::from_str(text) {+ Ok(v) => v,+ Err(_) => {+ stats.junk += 1;+ return Ok(stats);+ }+ };+ let Value::Object(m) = yaml_to_json(parsed) else {+ stats.junk += 1;+ return Ok(stats);+ };+ let guid = m.get("guid").and_then(Value::as_str).map(str::to_string);+ let importer = m+ .keys()+ .find(|k| k.ends_with("Importer") || k.ends_with("importer"))+ .cloned();+ let mut fields = flatten_object(m);+ if let Some(g) = guid {+ fields.insert("guid".into(), Value::String(g));+ }+ if let Some(imp) = importer {+ fields.insert("importer".into(), Value::String(imp));+ }+ if let Some(name) = path+ .file_name()+ .and_then(|n| n.to_str())+ .and_then(|n| n.strip_suffix(".meta"))+ {+ fields.insert("asset_name".into(), Value::String(name.to_string()));+ }+ stats.records += 1;+ sink(RawRecord {+ fields,+ locator: "meta".into(),+ group: None,+ origin: FieldOrigin::Extractor,+ });+ Ok(stats)+}++/// Fallback names for documents whose body failed to yield a top-level key+/// (the body key is authoritative when present). Sourced from Unity's public+/// class-id reference; unknown ids render `class_<id>`.+fn class_name(id: i64) -> Option<&'static str> {+ Some(match id {+ 1 => "GameObject",+ 2 => "Component",+ 4 => "Transform",+ 20 => "Camera",+ 21 => "Material",+ 23 => "MeshRenderer",+ 25 => "Renderer",+ 28 => "Texture2D",+ 33 => "MeshFilter",+ 43 => "Mesh",+ 54 => "Rigidbody",+ 61 => "BoxCollider2D",+ 64 => "MeshCollider",+ 65 => "BoxCollider",+ 74 => "AnimationClip",+ 81 => "AudioListener",+ 82 => "AudioSource",+ 90 => "Avatar",+ 91 => "AnimatorController",+ 95 => "Animator",+ 104 => "RenderSettings",+ 108 => "Light",+ 114 => "MonoBehaviour",+ 115 => "MonoScript",+ 128 => "Font",+ 135 => "SphereCollider",+ 136 => "CapsuleCollider",+ 137 => "SkinnedMeshRenderer",+ 157 => "LightmapSettings",+ 196 => "NavMeshSettings",+ 198 => "ParticleSystem",+ 199 => "ParticleSystemRenderer",+ 212 => "SpriteRenderer",+ 213 => "Sprite",+ 224 => "RectTransform",+ 320 => "PlayableDirector",+ 850595691 => "LightingSettings",+ 1001 => "PrefabInstance",+ 1660057539 => "SceneRoots",+ _ => return None,+ })+}++// The `MAX_LINE` import is load-bearing indirectly: `read_capped_line` caps+// each physical line at 16 MiB, and DOC_CAP matches it so a single-line+// document cannot bypass the document cap.+const _: () = assert!(DOC_CAP == MAX_LINE);++#[cfg(test)]+mod tests {+ use super::*;++ const SCENE: &str = "\u{feff}%YAML 1.1\n%TAG !u! tag:unity3d.com,2011:\n--- !u!1 &1234567890\nGameObject:\n m_ObjectHideFlags: 0\n m_Name: Player\n m_IsActive: 1\n--- !u!4 &1234567891\nTransform:\n m_GameObject: {fileID: 1234567890}\n m_LocalPosition: {x: 0, y: 1.5, z: 0}\n--- !u!114 &1234567892\nMonoBehaviour:\n m_GameObject: {fileID: 1234567890}\n m_Script: {fileID: 11500000, guid: abc123def456, type: 3}\n m_EditorClassIdentifier: \n speed: 4.5\n target: {fileID: 5555, guid: fedcba987654, type: 2}\n";++ fn run(text: &str, limit: Option<u64>) -> (ExtractStats, Vec<RawRecord>) {+ let dir = tempfile::tempdir().unwrap();+ let path = dir.path().join("t.unity");+ std::fs::write(&path, text).unwrap();+ let mut recs = Vec::new();+ let stats = extract_unity(&path, false, limit, &mut |r| {+ recs.push(r);+ true+ })+ .unwrap();+ (stats, recs)+ }++ fn run_meta(text: &str, name: &str) -> (ExtractStats, Vec<RawRecord>) {+ let dir = tempfile::tempdir().unwrap();+ let path = dir.path().join(name);+ std::fs::write(&path, text).unwrap();+ let mut recs = Vec::new();+ let stats = extract_meta(&path, false, &mut |r| {+ recs.push(r);+ true+ })+ .unwrap();+ (stats, recs)+ }++ #[test]+ fn each_document_becomes_one_record_with_class_and_file_id() {+ let (stats, recs) = run(SCENE, None);+ assert_eq!((stats.records, stats.junk), (3, 0));+ assert_eq!(recs[0].fields["unity_class"], "GameObject");+ assert_eq!(recs[0].fields["unity_class_id"], 1);+ assert_eq!(recs[0].fields["file_id"], "1234567890");+ assert_eq!(recs[0].fields["name"], "Player");+ assert_eq!(recs[0].locator, "u1234567890");+ assert_eq!(recs[0].group.as_deref(), Some("GameObject"));+ assert_eq!(recs[1].fields["unity_class"], "Transform");+ assert_eq!(+ recs[1].fields["m_LocalPosition_y"], 1.5,+ "nested mappings flatten like every other extractor"+ );+ }++ #[test]+ fn monobehaviour_carries_script_guid_and_deduped_ref_guids() {+ let (_, recs) = run(SCENE, None);+ let mb = &recs[2];+ assert_eq!(mb.fields["unity_class"], "MonoBehaviour");+ assert_eq!(mb.fields["script_guid"], "abc123def456");+ assert_eq!(+ mb.fields["ref_guids"],+ serde_json::json!(["abc123def456", "fedcba987654"])+ );+ assert_eq!(mb.fields["speed"], 4.5, "custom script fields survive");+ }++ #[test]+ fn stripped_and_negative_file_ids_parse() {+ let text = "%YAML 1.1\n%TAG !u! tag:unity3d.com,2011:\n--- !u!4 &-8679921383154817045 stripped\nTransform:\n m_PrefabInstance: {fileID: 100100000, guid: 0123456789abcdef, type: 3}\n";+ let (stats, recs) = run(text, None);+ assert_eq!((stats.records, stats.junk), (1, 0));+ assert_eq!(recs[0].fields["file_id"], "-8679921383154817045");+ assert_eq!(recs[0].fields["stripped"], true);+ assert_eq!(recs[0].locator, "u-8679921383154817045");+ }++ #[test]+ fn crlf_line_endings_are_tolerated() {+ let text = SCENE.replace('\n', "\r\n");+ let (stats, recs) = run(&text, None);+ assert_eq!((stats.records, stats.junk), (3, 0));+ assert_eq!(recs[2].fields["script_guid"], "abc123def456");+ }++ #[test]+ fn a_corrupt_document_junks_only_itself() {+ let text = "%YAML 1.1\n%TAG !u! tag:unity3d.com,2011:\n--- !u!1 &1\nGameObject:\n m_Name: A\n--- !u!4 &2\n\t{{{ not yaml\n--- !u!1 &3\nGameObject:\n m_Name: B\n";+ let (stats, recs) = run(text, None);+ assert_eq!(stats.junk, 1);+ assert_eq!(stats.records, 2);+ assert_eq!(recs[1].fields["name"], "B");+ }++ #[test]+ fn an_over_cap_document_is_skipped_and_the_stream_continues() {+ let mut text = String::from("%YAML 1.1\n%TAG !u! tag:unity3d.com,2011:\n--- !u!74 &1\nAnimationClip:\n m_Name: huge\n data: ");+ text.push_str(&"x".repeat(DOC_CAP + 1024));+ text.push_str("\n--- !u!1 &2\nGameObject:\n m_Name: after\n");+ let (stats, recs) = run(&text, None);+ assert_eq!(stats.junk, 1, "the oversized doc is junked, not fatal");+ assert_eq!(stats.records, 1);+ assert_eq!(recs[0].fields["name"], "after");+ }++ #[test]+ fn a_doc_cut_short_by_a_sampling_limit_is_not_junk() {+ let (stats, _) = run(SCENE, Some(120));+ assert_eq!(stats.junk, 0, "sampling truncation is not corruption");+ assert!(stats.records <= 1);+ }++ #[test]+ fn sampling_early_stop_via_sink_leaves_no_junk() {+ let dir = tempfile::tempdir().unwrap();+ let path = dir.path().join("t.unity");+ std::fs::write(&path, SCENE).unwrap();+ let mut n = 0;+ let stats = extract_unity(&path, false, None, &mut |_r| {+ n += 1;+ n < 2+ })+ .unwrap();+ assert_eq!(n, 2);+ assert_eq!(stats.junk, 0);+ }++ #[test]+ fn locators_are_idempotent_across_runs() {+ let (_, a) = run(SCENE, None);+ let (_, b) = run(SCENE, None);+ let la: Vec<_> = a.iter().map(|r| r.locator.clone()).collect();+ let lb: Vec<_> = b.iter().map(|r| r.locator.clone()).collect();+ assert_eq!(la, lb);+ }++ #[test]+ fn a_reserved_body_field_is_renamed_not_clobbered() {+ let text = "%YAML 1.1\n%TAG !u! tag:unity3d.com,2011:\n--- !u!114 &1\nMonoBehaviour:\n name: custom-field\n m_Name: Real\n";+ let (_, recs) = run(text, None);+ assert_eq!(recs[0].fields["data_name"], "custom-field");+ assert_eq!(recs[0].fields["name"], "Real");+ }++ #[test]+ fn unknown_class_id_with_unparseable_body_still_names_the_class() {+ let text = "%YAML 1.1\n%TAG !u! tag:unity3d.com,2011:\n--- !u!999999 &1\nsome scalar\n";+ let (stats, recs) = run(text, None);+ assert_eq!(stats.records, 1);+ assert_eq!(recs[0].fields["unity_class"], "class_999999");+ }++ #[test]+ fn meta_yields_guid_importer_and_asset_name() {+ let text = "fileFormatVersion: 2\nguid: 9f1c4d0ab2e34f6\nMonoImporter:\n externalObjects: {}\n serializedVersion: 2\n";+ let (stats, recs) = run_meta(text, "Player.cs.meta");+ assert_eq!((stats.records, stats.junk), (1, 0));+ assert_eq!(recs[0].fields["guid"], "9f1c4d0ab2e34f6");+ assert_eq!(recs[0].fields["importer"], "MonoImporter");+ assert_eq!(recs[0].fields["asset_name"], "Player.cs");+ assert_eq!(recs[0].locator, "meta");+ }++ #[test]+ fn a_file_with_only_preamble_is_junk() {+ let (stats, recs) = run("%YAML 1.1\n%TAG !u! tag:unity3d.com,2011:\n", None);+ assert_eq!(stats.records, 0);+ assert!(stats.junk >= 1);+ assert!(recs.is_empty());+ }+}diff --git a/engine/crates/xerj-autoindex/src/extract/yaml_x.rs b/engine/crates/xerj-autoindex/src/extract/yaml_x.rsindex 0b17914d0..6b298d57e 100644--- a/engine/crates/xerj-autoindex/src/extract/yaml_x.rs+++ b/engine/crates/xerj-autoindex/src/extract/yaml_x.rs@@ -1,7 +1,7 @@//! YAML — multi-document streams; each doc becomes a record (a top-level//! sequence of mappings becomes per-element records). 16MB file cap.-use super::{flatten_object, ExtractStats, FieldOrigin, RawRecord, Sink};+use super::{flatten_object, yaml_to_json, ExtractStats, FieldOrigin, RawRecord, Sink};use anyhow::Result;use serde_json::{Map, Value};use std::path::Path;@@ -83,43 +83,6 @@ pub fn extract(path: &Path, gzip: bool, sink: Sink) -> Result<ExtractStats> {use serde::Deserialize;-fn yaml_to_json(v: serde_yaml::Value) -> Value {- match v {- serde_yaml::Value::Null => Value::Null,- serde_yaml::Value::Bool(b) => Value::Bool(b),- serde_yaml::Value::Number(n) => {- if let Some(i) = n.as_i64() {- Value::Number(i.into())- } else if let Some(u) = n.as_u64() {- Value::Number(u.into())- } else {- serde_json::Number::from_f64(n.as_f64().unwrap_or(0.0))- .map(Value::Number)- .unwrap_or(Value::Null)- }- }- serde_yaml::Value::String(s) => Value::String(s),- serde_yaml::Value::Sequence(seq) => {- Value::Array(seq.into_iter().map(yaml_to_json).collect())- }- serde_yaml::Value::Mapping(m) => {- let mut out = Map::new();- for (k, vv) in m {- let key = match k {- serde_yaml::Value::String(s) => s,- other => serde_yaml::to_string(&other)- .unwrap_or_else(|_| "key".into())- .trim()- .to_string(),- };- out.insert(key, yaml_to_json(vv));- }- Value::Object(out)- }- serde_yaml::Value::Tagged(t) => yaml_to_json(t.value),- }-}-#[cfg(test)]mod tests {use super::*;diff --git a/engine/crates/xerj-autoindex/src/failure_resume_http_tests.rs b/engine/crates/xerj-autoindex/src/failure_resume_http_tests.rsindex c7d18289f..7be372a48 100644--- a/engine/crates/xerj-autoindex/src/failure_resume_http_tests.rs+++ b/engine/crates/xerj-autoindex/src/failure_resume_http_tests.rs@@ -46,7 +46,12 @@ impl MockEndpoint {let server_state = Arc::clone(&state);let join = thread::spawn(move || loop {match listener.accept() {- Ok((stream, _)) => handle(stream, &server_state),+ Ok((stream, _)) => {+ // BSD/macOS: accepted sockets inherit the listener's+ // O_NONBLOCK; the handler does blocking reads.+ stream.set_nonblocking(false).unwrap();+ handle(stream, &server_state)+ }Err(error) if error.kind() == std::io::ErrorKind::WouldBlock => {if server_state.lock().unwrap().stop {break;@@ -199,6 +204,7 @@ fn cfg(root: &Path, state_dir: &Path, url: &str) -> IndexCfg {state_dir: Some(state_dir.to_owned()),fresh: false,follow_symlinks: false,+ no_default_excludes: false,max_file_gb: 1,sample: 50,no_semantic: true,diff --git a/engine/crates/xerj-autoindex/src/lib.rs b/engine/crates/xerj-autoindex/src/lib.rsindex da8614e6f..3aa83fe4f 100644--- a/engine/crates/xerj-autoindex/src/lib.rs+++ b/engine/crates/xerj-autoindex/src/lib.rs@@ -233,7 +233,9 @@ fn scan_file(path: &Path, size: u64, sample: usize, max_file_gb: u64) -> FileSca}let limit = match sn.family {Family::SqlDump => Some(SQLDUMP_SAMPLE_LIMIT),- Family::Jsonl | Family::Logs | Family::Csv | Family::TxtLines => Some(SAMPLE_LIMIT_BYTES),+ Family::Jsonl | Family::Logs | Family::Csv | Family::TxtLines | Family::UnityYaml => {+ Some(SAMPLE_LIMIT_BYTES)+ }Family::Sqlite => Some(1), // signals per-table row cap inside the extractor_ => None, // whole-file extractors cap themselves};@@ -243,7 +245,14 @@ fn scan_file(path: &Path, size: u64, sample: usize, max_file_gb: u64) -> FileScastd::collections::HashSet<String>,);let mut groups: HashMap<Option<String>, GroupAcc> = HashMap::new();- let grouped_family = matches!(sn.family, Family::SqlDump | Family::Sqlite);+ // Grouped families keep reading past the per-group sample size: their+ // groups (SQL tables, Unity classes) appear all through the file, and+ // stopping at the first N records would leave later groups unsampled and+ // untyped.+ let grouped_family = matches!(+ sn.family,+ Family::SqlDump | Family::Sqlite | Family::UnityYaml+ );let mut sink = |rec: extract::RawRecord| -> bool {let entry = groups.entry(rec.group.clone()).or_default();if (entry.1 as usize) < sample {@@ -392,6 +401,97 @@ pub const PROVENANCE_FIELDS: &[&str] = &["ax_format",];+/// Spec for a field the PIPELINE derives at index time (Unity script-link+/// enrichment): typed keyword in the explicit mapping, zeroed sampling stats+/// because phase-A inference never observes it.+fn pipeline_keyword_spec(name: &str) -> infer::FieldSpec {+ infer::FieldSpec {+ name: name.into(),+ es_type: "keyword".into(),+ date_enc: None,+ semantic: None,+ cardinality_est: 0,+ cardinality_overflow: false,+ null_ratio: 0.0,+ avg_len: 0.0,+ coverage: 0.0,+ examples: Vec::new(),+ notes: vec!["pipeline-derived: resolved from the .meta guid map at index time".into()],+ date_min: None,+ date_max: None,+ date_evidence: Vec::new(),+ }+}++/// Unity script-link map: `.meta` guid → root-relative asset path. Metas are+/// tiny (one small YAML doc), so re-extracting them here costs milliseconds+/// and works identically on fresh and resumed runs — the map never has to be+/// journaled.+fn build_unity_guid_map(+ files: &[walk::FileEntry],+ plan: &Plan,+) -> std::collections::HashMap<String, String> {+ let by_rel: HashMap<&str, &Path> = files+ .iter()+ .map(|f| (f.rel.as_str(), f.path.as_path()))+ .collect();+ let mut map = std::collections::HashMap::new();+ for fa in plan.files.values() {+ if fa.family != "unity-meta" {+ continue;+ }+ let Some(asset_rel) = fa.rel.strip_suffix(".meta") else {+ continue;+ };+ let Some(path) = by_rel.get(fa.rel.as_str()) else {+ continue;+ };+ let mut guid: Option<String> = None;+ let _ = extract::unity::extract_meta(path, fa.gzip, &mut |rec| {+ guid = rec+ .fields+ .get("guid")+ .and_then(Value::as_str)+ .map(str::to_string);+ false+ });+ if let Some(g) = guid {+ map.insert(g, asset_rel.to_string());+ }+ }+ map+}++/// Stamp pipeline-derived Unity fields onto a record. MonoBehaviour records+/// gain `script_path`/`script_class` when their `script_guid` resolves; meta+/// records gain the root-relative `asset_path` their guid names. Denormalized+/// for one-query answers — `script_guid` remains the authoritative join.+fn enrich_unity_fields(+ family: Family,+ fields: &mut Map<String, Value>,+ guid_map: &std::collections::HashMap<String, String>,+ rel: &str,+) {+ match family {+ Family::UnityYaml => {+ let Some(g) = fields.get("script_guid").and_then(Value::as_str) else {+ return;+ };+ let Some(p) = guid_map.get(g) else { return };+ fields.insert("script_path".into(), Value::String(p.clone()));+ if let Some(stem) = Path::new(p).file_stem().and_then(|s| s.to_str()) {+ fields.insert("script_class".into(), Value::String(stem.to_string()));+ }+ }+ Family::UnityMeta => {+ if let Some(asset_rel) = rel.strip_suffix(".meta") {+ fields.insert("asset_path".into(), Value::String(asset_rel.to_string()));+ }+ }+ _ => {}+ }+}+fn build_mapping(specs: &[infer::FieldSpec]) -> Value {let mut props = Map::new();for s in specs {@@ -575,7 +675,18 @@ pub fn run_index_report(cfg: IndexCfg) -> Result<(i32, Option<Value>)> {let es = Es::with_bulk_timeout(&cfg.url, cfg.api_key.clone(), cfg.bulk_timeout_secs)?;es.ping()?;- let discovered_files = walk::walk(&cfg.root, cfg.follow_symlinks)?;+ let (discovered_files, skipped_dirs) =+ walk::walk(&cfg.root, cfg.follow_symlinks, !cfg.no_default_excludes)?;+ if !skipped_dirs.is_empty() && !cfg.quiet {+ let names: Vec<&str> = skipped_dirs.iter().map(|s| s.rel.as_str()).collect();+ eprintln!(+ "skipping {} generated director{} ({}): {} — pass --no-default-excludes to include",+ skipped_dirs.len(),+ if skipped_dirs.len() == 1 { "y" } else { "ies" },+ skipped_dirs[0].reason,+ names.join(", ")+ );+ }if discovered_files.is_empty() {println!("no files found under {}", cfg.root.display());return Ok((0, None));@@ -781,6 +892,16 @@ pub fn run_index_report(cfg: IndexCfg) -> Result<(i32, Option<Value>)> {let rels: Vec<String> = files.iter().map(|f| f.rel.clone()).collect();let mut sketches = Vec::new();let mut junk_files = Vec::new();+ for sd in &skipped_dirs {+ junk_files.push(JunkFile {+ file_key: format!("dir:{}", sd.rel),+ rel: sd.rel.clone(),+ format: "dir".into(),+ status: "skipped".into(),+ reason: format!("{}; --no-default-excludes to include", sd.reason),+ bytes: 0,+ });+ }for (i, sc) in scans.into_iter().enumerate() {let family = sc.sniffed@@ -841,7 +962,17 @@ pub fn run_index_report(cfg: IndexCfg) -> Result<(i32, Option<Value>)> {let mut datasets = Vec::new();for c in &clusters {- let specs = infer::infer_fields(&c.fields, c.records, cfg.no_semantic);+ let mut specs = infer::infer_fields(&c.fields, c.records, cfg.no_semantic);+ // Unity script-link enrichment fields are stamped by the phase-B+ // pipeline (not the extractor), so inference never sees them —+ // register them here or they would be dynamic-mapped coarsely.+ if c.family == Family::UnityYaml && c.fields.contains_key("script_guid") {+ specs.push(pipeline_keyword_spec("script_path"));+ specs.push(pipeline_keyword_spec("script_class"));+ }+ if c.family == Family::UnityMeta {+ specs.push(pipeline_keyword_spec("asset_path"));+ }let time_field = infer::elect_time_field(&specs);let semantic_field = specs.iter()@@ -953,6 +1084,8 @@ pub fn run_index_report(cfg: IndexCfg) -> Result<(i32, Option<Value>)> {}replacement_failpoint(1).context("after durable replacement plan")?;+ let unity_guid_map = build_unity_guid_map(&files, &plan);+// ── Phase B: full-stream extraction + bulk indexing ─────────────────struct DsRt {index: String,@@ -1339,6 +1472,7 @@ pub fn run_index_report(cfg: IndexCfg) -> Result<(i32, Option<Value>)> {if dropped > 0 {rt.dropped.fetch_add(dropped as u64, Ordering::Relaxed);}+ enrich_unity_fields(sn.family, &mut fields, &unity_guid_map, &f.rel);fields.insert("ax_path".into(), Value::String(f.rel.clone()));fields.insert("ax_paths".into(),@@ -2363,7 +2497,7 @@ mod duplicate_integration_tests {b[65_536] = b'b';fs::write(corpus.path().join("a.txt"), a).unwrap();fs::write(corpus.path().join("b.txt"), b).unwrap();- let files = walk::walk(corpus.path(), false).unwrap();+ let files = walk::walk(corpus.path(), false, true).unwrap().0;let inventory = content::resolve(files.clone()).unwrap();let legacy = ids::file_key(&files[0].path, files[0].size).unwrap();assert_eq!(@@ -2400,7 +2534,7 @@ mod duplicate_integration_tests {// with — so b.txt's content key IS the planned key a.txt claims by rel.fs::write(corpus.path().join("a.txt"), b"rewritten content\n").unwrap();fs::write(corpus.path().join("b.txt"), b"original planned content\n").unwrap();- let inventory = content::resolve(walk::walk(corpus.path(), false).unwrap()).unwrap();+ let inventory = content::resolve(walk::walk(corpus.path(), false, true).unwrap().0).unwrap();let planned_key = inventory.keys[1].clone();let mut plan = Plan::default();plan.files@@ -2470,7 +2604,7 @@ mod duplicate_integration_tests {fs::write(corpus.path().join("report-original.txt"), body).unwrap();fs::write(corpus.path().join("report-copy.txt"), body).unwrap();- let discovered = walk::walk(corpus.path(), false).unwrap();+ let discovered = walk::walk(corpus.path(), false, true).unwrap().0;let inventory = content::resolve(discovered).unwrap();assert_eq!(inventory.files.len(), 1);assert_eq!(inventory.duplicates.len(), 1);@@ -2543,7 +2677,7 @@ mod duplicate_integration_tests {}fs::write(&path, csv).unwrap();- let inventory = content::resolve(walk::walk(corpus.path(), false).unwrap()).unwrap();+ let inventory = content::resolve(walk::walk(corpus.path(), false, true).unwrap().0).unwrap();let expected_size = inventory.files[0].size;let expected_digest = inventory.digests[0].clone();let sniffed = sniff::sniff(&path).unwrap();diff --git a/engine/crates/xerj-autoindex/src/sniff.rs b/engine/crates/xerj-autoindex/src/sniff.rsindex a0d8c5485..0dbf3a489 100644--- a/engine/crates/xerj-autoindex/src/sniff.rs+++ b/engine/crates/xerj-autoindex/src/sniff.rs@@ -24,6 +24,12 @@ pub enum Family {SqlDump,/// Source code — AST-parsed by the matching tree-sitter grammar.Code,+ /// Unity text-serialized asset (scene/prefab/.asset/.mat/.anim/…):+ /// a `%YAML` + `%TAG !u! tag:unity3d.com` multi-document stream.+ UnityYaml,+ /// Unity `.meta` sidecar: plain YAML opening with `fileFormatVersion:`+ /// and carrying the asset `guid` — the join key for everything Unity.+ UnityMeta,Binary,}@@ -44,6 +50,8 @@ impl Family {Family::Sqlite => "sqlite",Family::SqlDump => "sqldump",Family::Code => "code",+ Family::UnityYaml => "unity",+ Family::UnityMeta => "unity-meta",Family::Binary => "binary",}}@@ -134,6 +142,11 @@ fn sniff_bytes(prefix: &[u8], path: &Path, gzip: bool) -> Result<Sniffed> {s.binary_kind = Some("zip".into());return Ok(s);}+ // Compressed image/audio/model payloads routinely pass the NUL and+ // control-char heuristics below (windows-1252 decodes almost every byte+ // to something printable), and a multi-MB PSD misread as prose costs+ // ~100x its size in RAM through sectioning. Magic bytes are the reliable+ // signal.for (magic, kind) in [(&b"\x89PNG"[..], "png"),(&b"GIF8"[..], "gif"),@@ -141,6 +154,15 @@ fn sniff_bytes(prefix: &[u8], path: &Path, gzip: bool) -> Result<Sniffed> {(&b"\x7fELF"[..], "elf"),(&b"BM"[..], "bmp"),(&b"\x00\x00\x01\x00"[..], "ico"),+ (&b"8BPS"[..], "psd"),+ (&b"II*\x00"[..], "tiff"),+ (&b"MM\x00*"[..], "tiff"),+ (&b"RIFF"[..], "riff"),+ (&b"OggS"[..], "ogg"),+ (&b"fLaC"[..], "flac"),+ (&b"ID3"[..], "mp3"),+ (&b"Kaydara FBX Binary"[..], "fbx"),+ (&b"\x76\x2f\x31\x01"[..], "exr"),] {if prefix.starts_with(magic) {let mut s = mk(Family::Binary);@@ -167,8 +189,44 @@ fn sniff_bytes(prefix: &[u8], path: &Path, gzip: bool) -> Result<Sniffed> {s.binary_kind = Some("unknown".into());return Ok(s);}+ // A prefix that only decoded via LOSSY windows-1252 and is majority+ // high-byte is pixel/float soup, not prose in a legacy encoding: real+ // windows-1252 text (accented European prose) runs well under 30%+ // non-ASCII, while raw image channels run ~50%. Without this, a large+ // TGA classified as txt-prose is amplified ~100x in RAM by sectioning.+ if encoding != "utf-8" {+ let total = text.chars().count().max(1);+ let non_ascii = text.chars().filter(|c| !c.is_ascii()).count();+ if non_ascii * 10 > total * 3 {+ let mut s = mk(Family::Binary);+ s.binary_kind = Some("unknown".into());+ return Ok(s);+ }+ }- // 2b. Source code: a known code extension whose content is text. We only+ // 2b. Unity text serialization, detected by content (never extension):+ // scenes/prefabs/assets open with `%YAML` and declare the Unity tag+ // namespace; `.meta` sidecars open with `fileFormatVersion:` and carry a+ // `guid:`. Binary-serialized Unity assets fail both checks and fall+ // through to the binary/text heuristics as before.+ {+ let body = text.trim_start_matches('\u{feff}');+ if body.starts_with("%YAML") && body.contains("%TAG !u! tag:unity3d.com") {+ let mut s = mk(Family::UnityYaml);+ s.encoding = encoding;+ return Ok(s);+ }+ let first_line = body.lines().next().unwrap_or("");+ if first_line.starts_with("fileFormatVersion:")+ && body.lines().any(|l| l.starts_with("guid:"))+ {+ let mut s = mk(Family::UnityMeta);+ s.encoding = encoding;+ return Ok(s);+ }+ }++ // 2c. Source code: a known code extension whose content is text. We only// reach here after the binary guards above, so a text `.py`/`.rs`/`.go`/…// routes to the tree-sitter AST extractor (crate::extract::code). Extension// is the right signal — code vs prose is not reliably content-sniffable.@@ -342,6 +400,26 @@ fn txt_kind(nonblank: &[&str]) -> Family {if nonblank.is_empty() {return Family::TxtLines;}+ // Raw pixel/float payloads without a container magic (TGA, bare .bytes+ // dumps) decode via lossy windows-1252 into printable soup; every+ // structured family above has already declined by the time control gets+ // here, and treating the soup as prose costs ~100x the file size in RAM+ // through sectioning. The discriminator is whitespace density:+ // natural-language text runs ~15% whitespace and random bytes ~2%+ // (0x09/0x0A/0x0D/0x20/0xA0 all decode to whitespace), so 5% cleanly+ // separates them. Gated to large prefixes so short legitimate files keep+ // their text families; dense CSVs are safe because the CSV sniffer+ // claims them before this fallback.+ let total_chars: usize = nonblank.iter().map(|l| l.chars().count()).sum();+ if total_chars >= 4096 {+ let ws: usize = nonblank+ .iter()+ .map(|l| l.chars().filter(|c| c.is_whitespace()).count())+ .sum();+ if ws * 20 < total_chars {+ return Family::Binary;+ }+ }let avg_len = nonblank.iter().map(|l| l.len()).sum::<usize>() as f64 / nonblank.len() as f64;if avg_len > 60.0 {return Family::TxtProse;@@ -503,6 +581,102 @@ fn sniff_csv_dialect(nonblank: &[&str]) -> Option<CsvDialect> {})}+#[cfg(test)]+mod unity_sniff_tests {+ use super::*;+ use std::path::Path;++ fn sniff_str(s: &str, name: &str) -> Family {+ sniff_bytes(s.as_bytes(), Path::new(name), false)+ .unwrap()+ .family+ }++ #[test]+ fn unity_tagged_yaml_is_detected_by_header_not_extension() {+ let scene = "%YAML 1.1\n%TAG !u! tag:unity3d.com,2011:\n--- !u!1 &123\nGameObject:\n m_Name: X\n";+ assert_eq!(sniff_str(scene, "Main.unity"), Family::UnityYaml);+ assert_eq!(+ sniff_str(scene, "renamed.txt"),+ Family::UnityYaml,+ "content decides, never the extension"+ );+ }++ #[test]+ fn a_bom_before_the_yaml_directive_is_tolerated() {+ let scene =+ "\u{feff}%YAML 1.1\n%TAG !u! tag:unity3d.com,2011:\n--- !u!1 &1\nGameObject:\n m_Name: X\n";+ assert_eq!(sniff_str(scene, "Main.unity"), Family::UnityYaml);+ }++ #[test]+ fn meta_needs_the_first_line_rule_and_a_guid() {+ let meta = "fileFormatVersion: 2\nguid: 9f1c4d0ab2e34f6\nMonoImporter:\n serializedVersion: 2\n";+ assert_eq!(sniff_str(meta, "Player.cs.meta"), Family::UnityMeta);+ let stray = "config: true\nfileFormatVersion: 2\nguid: abc\n";+ assert_ne!(+ sniff_str(stray, "some.yaml"),+ Family::UnityMeta,+ "guid keys inside ordinary YAML must not reclassify it"+ );+ let no_guid = "fileFormatVersion: 2\nsettings:\n a: 1\n";+ assert_ne!(sniff_str(no_guid, "x.meta"), Family::UnityMeta);+ }++ #[test]+ fn plain_yaml_without_the_unity_tag_stays_yaml() {+ let plain = "%YAML 1.2\n---\nkey: value\nother: 1\nnested:\n a: 2\n";+ assert_ne!(sniff_str(plain, "doc.yaml"), Family::UnityYaml);+ }++ /// Regression: PSD image data decoded via lossy windows-1252 passed the+ /// NUL/control-char heuristics and classified as txt-prose, turning a+ /// multi-MB texture into ~100x its size of prose sections. Media magic+ /// bytes must win before any text heuristic runs.+ #[test]+ fn media_containers_are_binary_by_magic_not_heuristics() {+ for (name, head) in [+ ("t.psd", &b"8BPS\x00\x01"[..]),+ ("t.tif", &b"II*\x00\x08\x00"[..]),+ ("t.tif2", &b"MM\x00*\x00\x08"[..]),+ ("t.wav", &b"RIFF\x24\x08\x00\x00WAVE"[..]),+ ("t.ogg", &b"OggS\x00\x02"[..]),+ ("t.fbx", &b"Kaydara FBX Binary \x00"[..]),+ ] {+ let mut bytes = head.to_vec();+ // A printable tail that WOULD pass the prose heuristics.+ bytes.extend(b"lorem ipsum dolor sit amet, consectetur adipiscing elit. ".repeat(20));+ let sn = sniff_bytes(&bytes, Path::new(name), false).unwrap();+ assert_eq!(sn.family, Family::Binary, "{name} must be binary");+ }+ }++ /// Regression: an uncompressed TGA (no magic bytes exist for TGA) decoded+ /// via lossy windows-1252 into printable soup with near-zero whitespace+ /// and classified txt-prose. Whitespace ratio is the discriminator.+ #[test]+ fn whitespace_free_printable_soup_is_binary_not_prose() {+ let soup: String = (0..8192u32)+ .map(|i| char::from_u32(0x21 + (i * 7) % 0x5d).unwrap())+ .collect();+ let sn = sniff_bytes(soup.as_bytes(), Path::new("texture.tga"), false).unwrap();+ assert_eq!(sn.family, Family::Binary);+ // Real prose of the same size keeps its family.+ let prose = "The quick brown fox jumps over the lazy dog. ".repeat(200);+ let sn = sniff_bytes(prose.as_bytes(), Path::new("note.txt"), false).unwrap();+ assert_ne!(sn.family, Family::Binary, "real prose must stay text");+ }++ #[test]+ fn binary_serialized_unity_assets_stay_binary() {+ let mut bytes = b"UnityFS\x00\x00\x00\x00\x08".to_vec();+ bytes.extend(std::iter::repeat(0u8).take(64));+ let sn = sniff_bytes(&bytes, Path::new("scene.unity"), false).unwrap();+ assert_eq!(sn.family, Family::Binary);+ }+}+#[cfg(test)]mod tests {use super::*;diff --git a/engine/crates/xerj-autoindex/src/walk.rs b/engine/crates/xerj-autoindex/src/walk.rsindex 27278f2c5..e316aafc0 100644--- a/engine/crates/xerj-autoindex/src/walk.rs+++ b/engine/crates/xerj-autoindex/src/walk.rs@@ -17,13 +17,72 @@ pub struct FileEntry {pub size: u64,}-pub fn walk(root: &Path, follow_symlinks: bool) -> Result<Vec<FileEntry>> {+/// A directory pruned whole by the generated-dir hygiene rule, recorded so+/// the run can surface it (junk report / map) instead of skipping silently.+#[derive(Debug, Clone)]+pub struct SkippedDir {+ pub rel: String,+ pub reason: String,+}++/// Well-known generated/cache directories, pruned at ANY depth but ONLY when+/// a sibling marker file proves the parent directory is that kind of project+/// (a monorepo may nest e.g. `apps/game/Library`, `apps/web/node_modules`).+/// Marker-gated like the dotfile rule (hygiene, evidence-based) — never+/// extension guessing, never a per-corpus code path.+/// `--no-default-excludes` disables the rule.+///+/// (dir name, marker path relative to the dir's PARENT, reason)+const GENERATED_DIR_RULES: &[(&str, &str, &str)] = &[+ (+ "Library",+ "ProjectSettings/ProjectVersion.txt",+ "Unity generated/cache directory (marker: sibling ProjectSettings/ProjectVersion.txt)",+ ),+ (+ "Temp",+ "ProjectSettings/ProjectVersion.txt",+ "Unity generated/cache directory (marker: sibling ProjectSettings/ProjectVersion.txt)",+ ),+ (+ "obj",+ "ProjectSettings/ProjectVersion.txt",+ "Unity generated/cache directory (marker: sibling ProjectSettings/ProjectVersion.txt)",+ ),+ (+ "Logs",+ "ProjectSettings/ProjectVersion.txt",+ "Unity generated/cache directory (marker: sibling ProjectSettings/ProjectVersion.txt)",+ ),+ (+ "UserSettings",+ "ProjectSettings/ProjectVersion.txt",+ "Unity generated/cache directory (marker: sibling ProjectSettings/ProjectVersion.txt)",+ ),+ (+ "node_modules",+ "package.json",+ "installed npm/yarn dependency tree (marker: sibling package.json)",+ ),+ (+ "target",+ "Cargo.toml",+ "Cargo build directory (marker: sibling Cargo.toml)",+ ),+];++pub fn walk(+ root: &Path,+ follow_symlinks: bool,+ default_excludes: bool,+) -> Result<(Vec<FileEntry>, Vec<SkippedDir>)> {let root_canon = root.canonicalize().with_context(|| format!("resolve root folder {}", root.display()))?;if !root_canon.is_dir() {anyhow::bail!("{} is not a directory", root_canon.display());}+ let skipped: std::rc::Rc<std::cell::RefCell<Vec<SkippedDir>>> = Default::default();let mut out = Vec::new();// SECURITY / hygiene: never index hidden files or descend into hidden// directories. Without this the walker happily indexed `.env` (secrets,@@ -32,11 +91,40 @@ pub fn walk(root: &Path, follow_symlinks: bool) -> Result<Vec<FileEntry>> {// "point it at my project folder" use case. `filter_entry` prunes a hidden// directory before descending, so `.git/` is skipped whole. The root itself// (depth 0) is exempt so a brain over a dot-named folder still works.+ let sk = std::rc::Rc::clone(&skipped);+ let filter_root = root_canon.clone();let walker = walkdir::WalkDir::new(&root_canon).follow_links(follow_symlinks).into_iter()- .filter_entry(|e| {- e.depth() == 0 || !e.file_name().to_str().is_some_and(|n| n.starts_with('.'))+ .filter_entry(move |e| {+ if e.depth() == 0 {+ return true;+ }+ if e.file_name().to_str().is_some_and(|n| n.starts_with('.')) {+ return false;+ }+ if default_excludes && e.file_type().is_dir() {+ for (name, marker, reason) in GENERATED_DIR_RULES {+ if e.file_name() == *name+ && e.path()+ .parent()+ .is_some_and(|p| p.join(marker).is_file())+ {+ let rel = e+ .path()+ .strip_prefix(&filter_root)+ .unwrap_or(e.path())+ .to_string_lossy()+ .replace('\\', "/");+ sk.borrow_mut().push(SkippedDir {+ rel,+ reason: (*reason).to_string(),+ });+ return false;+ }+ }+ }+ true});for entry in walker {let entry = match entry {@@ -73,7 +161,10 @@ pub fn walk(root: &Path, follow_symlinks: bool) -> Result<Vec<FileEntry>> {}// Deterministic order for clustering / naming.out.sort_by(|a, b| a.rel.cmp(&b.rel));- Ok(out)+ let skipped = std::rc::Rc::try_unwrap(skipped)+ .expect("walker iterator consumed; sole owner remains")+ .into_inner();+ Ok((out, skipped))}#[cfg(unix)]@@ -120,8 +211,9 @@ mod hidden_skip_tests {fs::write(root.join("src").join("main.rs"), "fn main() {}").unwrap();fs::write(root.join("src").join(".secret"), "nope").unwrap();- let rels: Vec<String> = walk(root, false)+ let rels: Vec<String> = walk(root, false, true).unwrap()+ .0.into_iter().map(|e| e.rel).collect();@@ -145,8 +237,9 @@ mod hidden_skip_tests {let root = dir.path().join(".notes");fs::create_dir(&root).unwrap();fs::write(root.join("a.md"), "x").unwrap();- let rels: Vec<String> = walk(&root, false)+ let rels: Vec<String> = walk(&root, false, true).unwrap()+ .0.into_iter().map(|e| e.rel).collect();@@ -157,3 +250,142 @@ mod hidden_skip_tests {);}}++#[cfg(test)]+mod generated_dir_tests {+ use super::walk;+ use std::fs;+ use std::path::Path;++ fn make_unity_root(root: &Path, with_marker: bool) {+ fs::create_dir_all(root.join("Assets")).unwrap();+ fs::write(root.join("Assets/Player.cs"), "class Player {}").unwrap();+ fs::create_dir_all(root.join("Library/Artifacts")).unwrap();+ fs::write(root.join("Library/Artifacts/blob.bin"), b"\x00\x01").unwrap();+ fs::create_dir_all(root.join("Temp")).unwrap();+ fs::write(root.join("Temp/scratch"), "x").unwrap();+ if with_marker {+ fs::create_dir_all(root.join("ProjectSettings")).unwrap();+ fs::write(+ root.join("ProjectSettings/ProjectVersion.txt"),+ "m_EditorVersion: 2022.3.10f1\n",+ )+ .unwrap();+ }+ }++ #[test]+ fn unity_marker_prunes_generated_dirs_and_records_them() {+ let dir = tempfile::TempDir::new().unwrap();+ make_unity_root(dir.path(), true);+ let (files, skipped) = walk(dir.path(), false, true).unwrap();+ let rels: Vec<&str> = files.iter().map(|e| e.rel.as_str()).collect();+ assert!(rels.contains(&"Assets/Player.cs"));+ assert!(+ !rels.iter().any(|r| r.starts_with("Library/")),+ "Library/ must be pruned whole: {rels:?}"+ );+ assert!(!rels.iter().any(|r| r.starts_with("Temp/")));+ let mut skipped_rels: Vec<&str> = skipped.iter().map(|s| s.rel.as_str()).collect();+ skipped_rels.sort();+ assert_eq!(skipped_rels, ["Library", "Temp"], "each pruned dir is recorded");+ }++ #[test]+ fn without_the_marker_nothing_is_pruned() {+ let dir = tempfile::TempDir::new().unwrap();+ make_unity_root(dir.path(), false);+ let (files, skipped) = walk(dir.path(), false, true).unwrap();+ let rels: Vec<&str> = files.iter().map(|e| e.rel.as_str()).collect();+ assert!(+ rels.iter().any(|r| r.starts_with("Library/")),+ "a folder merely NAMED Library is not evidence: {rels:?}"+ );+ assert!(skipped.is_empty());+ }++ #[test]+ fn no_default_excludes_walks_everything() {+ let dir = tempfile::TempDir::new().unwrap();+ make_unity_root(dir.path(), true);+ let (files, skipped) = walk(dir.path(), false, false).unwrap();+ let rels: Vec<&str> = files.iter().map(|e| e.rel.as_str()).collect();+ assert!(rels.iter().any(|r| r.starts_with("Library/")));+ assert!(skipped.is_empty());+ }++ #[test]+ fn a_file_named_like_a_generated_dir_is_not_pruned() {+ let dir = tempfile::TempDir::new().unwrap();+ make_unity_root(dir.path(), true);+ fs::write(dir.path().join("obj"), "a FILE named obj").unwrap();+ let (files, _) = walk(dir.path(), false, true).unwrap();+ assert!(+ files.iter().any(|e| e.rel == "obj"),+ "the prune applies to directories only"+ );+ }++ /// Monorepo case: the Unity project is a SUBFOLDER of the walk root. The+ /// marker is checked against each candidate's parent, so nesting depth is+ /// irrelevant.+ #[test]+ fn a_nested_unity_project_is_pruned_by_its_sibling_marker() {+ let dir = tempfile::TempDir::new().unwrap();+ let unity = dir.path().join("apps/game");+ fs::create_dir_all(&unity).unwrap();+ make_unity_root(&unity, true);+ fs::write(dir.path().join("README.md"), "monorepo").unwrap();+ let (files, skipped) = walk(dir.path(), false, true).unwrap();+ let rels: Vec<&str> = files.iter().map(|e| e.rel.as_str()).collect();+ assert!(rels.contains(&"apps/game/Assets/Player.cs"));+ assert!(!rels.iter().any(|r| r.contains("game/Library/")));+ assert!(skipped.iter().any(|s| s.rel == "apps/game/Library"));+ }++ #[test]+ fn node_modules_is_pruned_only_next_to_a_package_json() {+ let dir = tempfile::TempDir::new().unwrap();+ let app = dir.path().join("web");+ fs::create_dir_all(app.join("node_modules/lodash")).unwrap();+ fs::write(app.join("node_modules/lodash/index.js"), "x").unwrap();+ fs::write(app.join("package.json"), "{}").unwrap();+ fs::write(app.join("index.js"), "x").unwrap();+ let bare = dir.path().join("data/node_modules");+ fs::create_dir_all(&bare).unwrap();+ fs::write(bare.join("notes.txt"), "not an npm tree").unwrap();+ let (files, skipped) = walk(dir.path(), false, true).unwrap();+ let rels: Vec<&str> = files.iter().map(|e| e.rel.as_str()).collect();+ assert!(rels.contains(&"web/index.js"));+ assert!(!rels.iter().any(|r| r.starts_with("web/node_modules")));+ assert!(+ rels.contains(&"data/node_modules/notes.txt"),+ "without the package.json marker there is no evidence: {rels:?}"+ );+ assert_eq!(skipped.len(), 1);+ assert_eq!(skipped[0].rel, "web/node_modules");+ }++ #[test]+ fn cargo_target_is_pruned_only_next_to_a_cargo_toml() {+ let dir = tempfile::TempDir::new().unwrap();+ let crate_dir = dir.path().join("mycrate");+ fs::create_dir_all(crate_dir.join("target/debug")).unwrap();+ fs::write(crate_dir.join("target/debug/build.log"), "x").unwrap();+ fs::write(crate_dir.join("Cargo.toml"), "[package]").unwrap();+ fs::write(crate_dir.join("main.rs"), "fn main() {}").unwrap();+ let plain = dir.path().join("shooting/target");+ fs::create_dir_all(&plain).unwrap();+ fs::write(plain.join("scores.csv"), "a,b\n1,2\n").unwrap();+ let (files, skipped) = walk(dir.path(), false, true).unwrap();+ let rels: Vec<&str> = files.iter().map(|e| e.rel.as_str()).collect();+ assert!(rels.contains(&"mycrate/main.rs"));+ assert!(!rels.iter().any(|r| r.starts_with("mycrate/target")));+ assert!(+ rels.contains(&"shooting/target/scores.csv"),+ "a folder merely NAMED target is data, not a build dir: {rels:?}"+ );+ assert_eq!(skipped.len(), 1);+ assert_eq!(skipped[0].rel, "mycrate/target");+ }+}diff --git a/engine/crates/xerj-server/src/brain.rs b/engine/crates/xerj-server/src/brain.rsindex 68f2a25d9..8fd418ab9 100644--- a/engine/crates/xerj-server/src/brain.rs+++ b/engine/crates/xerj-server/src/brain.rs@@ -177,7 +177,7 @@ fn run(cfg: BrainCfg) -> Result<i32> {if !cfg.root.exists() {bail!("{} does not exist", cfg.root.display());}- let files = walk::walk(&cfg.root, false)?;+ let (files, _skipped_dirs) = walk::walk(&cfg.root, false, true)?;if files.is_empty() {eprintln!("xerj brain: no files under {} — nothing to build a brain from.\n\@@ -432,6 +432,7 @@ fn index_cfg(cfg: &BrainCfg, brain: &str, api_key: Option<String>) -> IndexCfg {state_dir: None,fresh: cfg.fresh,follow_symlinks: false,+ no_default_excludes: false,max_file_gb: 2,sample: 500,no_semantic: false,