AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
fix(autoindex): repository-scoped dataset grouping — a 2-repo tree plans 13 datasets, not 404
xerj-orga246328b366d
diff --git a/engine/crates/xerj-autoindex/src/dataset.rs b/engine/crates/xerj-autoindex/src/dataset.rsindex 31065a65d..2e85d58af 100644--- a/engine/crates/xerj-autoindex/src/dataset.rs+++ b/engine/crates/xerj-autoindex/src/dataset.rs@@ -1,21 +1,46 @@-//! Dataset clustering: files (or per-table groups within files) merge into-//! datasets by schema fingerprint (Jaccard on field-name sets ≥ 0.7) within-//! the same format family. Path family is a NAMING hint only — no hardcoded-//! directory semantics.+//! Dataset clustering: what a "dataset" means for a folder of files.//!-//! Only the DATA-derived field names take part (`Sketch::key_fields`, fed from-//! `extract::FieldOrigin`). A name the extractor invented — `defs`, `symbols`,-//! `title`, `page` — is not evidence about the file, and letting one decide-//! membership made every extractor improvement re-home files: the dataset slug-//! is an ingredient of `ids::doc_id`, so a moved file is re-indexed under a new-//! `_id` in a new index while its old document survives, unreferenced, in the-//! old one (issue #178). Files whose names are all extractor-invented (source-//! code, prose documents) therefore cluster by format family and group alone,-//! which is what their sniffed shape actually says about them.+//! Two kinds of dataset come out of a tree (#173/#196):+//!+//! **Documents.** Files whose field names are all extractor-invented (source+//! code, prose, PDFs, line-oriented text, HTML pages) carry no schema of their+//! own, so schema inference has nothing legitimate to split them on. They merge+//! into ONE dataset per *scope* — the nearest enclosing repository root+//! (a directory with a `.git` entry), or the autoindex root when the file is+//! under no repository. Splitting them per format family produced indices no+//! user recognised as datasets (#196: one Rust workspace → four indices), and+//! per-schema clustering of their incidental shapes shattered a two-repo tree+//! into hundreds (#173). One folder → one searchable corpus per repository.+//!+//! **Data.** Files with data-derived field names (`Sketch::key_fields`, fed+//! from `extract::FieldOrigin`: CSV headers, JSON keys, SQL columns) still+//! cluster by schema fingerprint (Jaccard on field-name sets ≥ 0.7) within the+//! same format family AND scope — a real schema that recurs across files is a+//! dataset. But a self-describing config blob is not: a small cluster of+//! `json`/`yaml`/`xml` files that each produced exactly one record is *demoted*+//! to the scope's document dataset and indexed as a document (title/body),+//! because its one-off key set is configuration, not a collection+//! (#173: valkey's per-command JSON files made 382 such "datasets").+//!+//! Only DATA-derived names take part in schema clustering. A name the+//! extractor invented — `defs`, `symbols`, `title`, `page` — is not evidence+//! about the file, and letting one decide membership made every extractor+//! improvement re-home files: the dataset slug is an ingredient of+//! `ids::doc_id`, so a moved file is re-indexed under a new `_id` in a new+//! index while its old document survives, unreferenced, in the old one+//! (issue #178). Scope-based document grouping strengthens that stability:+//! a document's dataset now depends only on its own path and the repository+//! layout, never on which other files exist.use crate::infer::FieldAcc;use crate::sniff::Family;-use std::collections::{HashMap, HashSet};+use std::collections::{BTreeMap, HashMap, HashSet};++/// A schema-clustered `json`/`yaml`/`xml` group at most this many files large,+/// in which every file produced exactly one record, is configuration — demote+/// it to the scope's document dataset. A schema that recurs across more files+/// than this is treated as a real single-record-per-file collection.+pub const DOC_DEMOTE_MAX_FILES: usize = 8;#[derive(Debug)]pub struct Sketch {@@ -24,7 +49,7 @@ pub struct Sketch {pub family: Family,pub fields: HashMap<String, FieldAcc>,/// The subset of `fields` whose NAMES were read out of the file. This, not- /// `fields`, is the clustering key.+ /// `fields`, is the schema-clustering key; empty means "document".pub key_fields: HashSet<String>,pub records: u64,}@@ -39,6 +64,14 @@ pub struct Cluster {pub key_fields: HashSet<String>,pub records: u64,pub slug: String,+ /// Repository scope this cluster lives in ("" = the autoindex root).+ pub scope: String,+ /// A per-scope document dataset (code/prose/config), not a schema dataset.+ pub is_docs: bool,+ /// Members demoted from one-off config clusters. Their sampled fields were+ /// discarded (config keys are not document fields); the caller re-samples+ /// them through the document renderer and must index them as documents.+ pub demoted: Vec<usize>,}fn jaccard(a: &HashSet<&str>, b: &HashSet<&str>) -> f64 {@@ -50,13 +83,58 @@ fn jaccard(a: &HashSet<&str>, b: &HashSet<&str>) -> f64 {inter as f64 / union.max(1) as f64}-pub fn cluster(sketches: Vec<Sketch>, rels: &[String]) -> Vec<Cluster> {- let mut clusters: Vec<Cluster> = Vec::new();+/// May this data cluster be demoted to documents if it stays a small fleet of+/// single-record files? Self-describing structured formats only: CSV is+/// tabular by nature, logs/jsonl are collections by nature, and sql groups+/// carry a real table schema.+fn demotable_family(f: Family) -> bool {+ matches!(f, Family::Json | Family::Yaml | Family::Xml)+}++/// Cluster sketches into datasets. `rels` and `scopes` are indexed by+/// `Sketch::file_idx`: `rels` are root-relative paths (slug naming), `scopes`+/// the repository scope of each file ("" = root) — see `module` docs.+pub fn cluster(sketches: Vec<Sketch>, rels: &[String], scopes: &[String]) -> Vec<Cluster> {+ // scope → docs cluster (BTreeMap: deterministic scope order).+ let mut docs: BTreeMap<String, Cluster> = BTreeMap::new();+ let mut data: Vec<Cluster> = Vec::new();+ fn docs_cluster<'a>(docs: &'a mut BTreeMap<String, Cluster>, scope: &str) -> &'a mut Cluster {+ docs.entry(scope.to_string()).or_insert_with(|| Cluster {+ family: Family::Code,+ group: None,+ members: Vec::new(),+ fields: HashMap::new(),+ key_fields: HashSet::new(),+ records: 0,+ slug: String::new(),+ scope: scope.to_string(),+ is_docs: true,+ demoted: Vec::new(),+ })+ }+for sk in sketches {+ let scope = scopes[sk.file_idx].as_str();+ // Documents: no data-derived names, no sub-file group.+ if sk.key_fields.is_empty() && sk.group.is_none() {+ let c = docs_cluster(&mut docs, scope);+ c.members.push(sk.file_idx);+ c.records += sk.records;+ for (k, acc) in sk.fields {+ match c.fields.get_mut(&k) {+ Some(existing) => existing.merge(&acc),+ None => {+ c.fields.insert(k, acc);+ }+ }+ }+ continue;+ }+ // Data: schema clustering within (scope, family, group).let names: HashSet<&str> = sk.key_fields.iter().map(|s| s.as_str()).collect();let mut best: Option<(usize, f64)> = None;- for (ci, c) in clusters.iter().enumerate() {- if c.family != sk.family || c.group != sk.group {+ for (ci, c) in data.iter().enumerate() {+ if c.family != sk.family || c.group != sk.group || c.scope != scope {continue;}let cnames: HashSet<&str> = c.key_fields.iter().map(|s| s.as_str()).collect();@@ -68,7 +146,7 @@ pub fn cluster(sketches: Vec<Sketch>, rels: &[String]) -> Vec<Cluster> {}match best {Some((ci, _)) => {- let c = &mut clusters[ci];+ let c = &mut data[ci];c.members.push(sk.file_idx);c.records += sk.records;c.key_fields.extend(sk.key_fields);@@ -81,7 +159,7 @@ pub fn cluster(sketches: Vec<Sketch>, rels: &[String]) -> Vec<Cluster> {}}}- None => clusters.push(Cluster {+ None => data.push(Cluster {family: sk.family,group: sk.group,members: vec![sk.file_idx],@@ -89,10 +167,58 @@ pub fn cluster(sketches: Vec<Sketch>, rels: &[String]) -> Vec<Cluster> {key_fields: sk.key_fields,records: sk.records,slug: String::new(),+ scope: scope.to_string(),+ is_docs: false,+ demoted: Vec::new(),}),}}- assign_slugs(&mut clusters, rels);++ // Demote one-off config clusters to their scope's document dataset. Their+ // sampled fields are dropped — the caller re-samples the demoted files+ // through the document renderer, which is also what indexes them.+ let mut kept: Vec<Cluster> = Vec::with_capacity(data.len());+ for c in data {+ let one_record_each = c.records == c.members.len() as u64;+ if demotable_family(c.family)+ && c.group.is_none()+ && c.members.len() <= DOC_DEMOTE_MAX_FILES+ && one_record_each+ {+ let d = docs_cluster(&mut docs, &c.scope);+ d.members.extend(c.members.iter().copied());+ d.demoted.extend(c.members);+ continue;+ }+ kept.push(c);+ }++ // Docs slugs are scope-derived — independent of which other files exist,+ // so adding a file can never rename (and re-home, #178) a docs dataset.+ let mut clusters: Vec<Cluster> = Vec::with_capacity(docs.len() + kept.len());+ let mut used: HashSet<String> = HashSet::new();+ for (scope, mut c) in docs {+ let base = if scope.is_empty() {+ "docs".to_string()+ } else {+ let s = sanitize_slug(&scope);+ if s.is_empty() {+ "docs".to_string()+ } else {+ format!("{s}-docs")+ }+ };+ let mut slug = base.clone();+ let mut k = 2;+ while !used.insert(slug.clone()) {+ slug = format!("{base}-{k}");+ k += 1;+ }+ c.slug = slug;+ clusters.push(c);+ }+ assign_slugs(&mut kept, rels, &mut used);+ clusters.extend(kept);clusters}@@ -131,7 +257,7 @@ fn path_candidate(rel: &str) -> String {meaningful.join("-")}-fn assign_slugs(clusters: &mut [Cluster], rels: &[String]) {+fn assign_slugs(clusters: &mut [Cluster], rels: &[String], used: &mut HashSet<String>) {// deterministic cluster order: by first member rellet mut order: Vec<usize> = (0..clusters.len()).collect();order.sort_by_key(|&i| {@@ -226,7 +352,6 @@ fn assign_slugs(clusters: &mut [Cluster], rels: &[String]) {}// final dedup with -2/-3 …- let mut used: HashSet<String> = HashSet::new();for &i in &order {let mut slug = bases[i].clone();let mut k = 2;@@ -271,8 +396,27 @@ mod tests {}}+ /// A prose file: same extractor-owned vocabulary situation as code.+ fn prose_sketch(file_idx: usize) -> Sketch {+ let mut fields = HashMap::new();+ fields.insert("title".to_string(), acc("README"));+ fields.insert("body".to_string(), acc("This project does things."));+ Sketch {+ file_idx,+ group: None,+ family: Family::TxtProse,+ fields,+ key_fields: HashSet::new(),+ records: 1,+ }+ }+/// A data file: the names are the file's own, so they are the key.fn data_sketch(file_idx: usize, family: Family, names: &[&str]) -> Sketch {+ data_sketch_n(file_idx, family, names, 1)+ }++ fn data_sketch_n(file_idx: usize, family: Family, names: &[&str], records: u64) -> Sketch {let mut fields = HashMap::new();let mut key_fields = HashSet::new();for n in names {@@ -285,10 +429,14 @@ mod tests {family,fields,key_fields,- records: 1,+ records,}}+ fn root_scopes(n: usize) -> Vec<String> {+ vec![String::new(); n]+ }+fn doc_ids(clusters: &[Cluster], rels: &[String]) -> Vec<String> {let mut out = Vec::new();for c in clusters {@@ -309,11 +457,13 @@ mod tests {#[test]fn a_better_extractor_does_not_re_home_a_single_file() {let rels: Vec<String> = (0..6).map(|i| format!("src/mod{i}/f.rs")).collect();+ let scopes = root_scopes(6);// Before: files 4 and 5 parsed to zero symbols.let before = cluster((0..6).map(|i| code_sketch(i, i < 4)).collect::<Vec<_>>(),&rels,+ &scopes,);// The whole point: one dataset, not one per symbol-presence variant.assert_eq!(before.len(), 1, "{before:#?}");@@ -323,6 +473,7 @@ mod tests {let after = cluster((0..6).map(|i| code_sketch(i, true)).collect::<Vec<_>>(),&rels,+ &scopes,);let (b, a) = (doc_ids(&before, &rels), doc_ids(&after, &rels));@@ -340,16 +491,17 @@ mod tests {"data/events.csv".into(),"data/users.csv".into(),];+ let scopes = root_scopes(4);let mixed = |symbols_in_b: bool| {vec![code_sketch(0, true),code_sketch(1, symbols_in_b),- data_sketch(2, Family::Csv, &["ts", "level", "msg"]),- data_sketch(3, Family::Csv, &["id", "email", "name"]),+ data_sketch_n(2, Family::Csv, &["ts", "level", "msg"], 40),+ data_sketch_n(3, Family::Csv, &["id", "email", "name"], 40),]};- let before = cluster(mixed(false), &rels);- let after = cluster(mixed(true), &rels);+ let before = cluster(mixed(false), &rels, &scopes);+ let after = cluster(mixed(true), &rels, &scopes);let slugs = |cs: &[Cluster]| {let mut v: Vec<(usize, String)> = cs.iter()@@ -359,26 +511,28 @@ mod tests {v};assert_eq!(slugs(&before), slugs(&after));- // 1 code dataset + 2 unrelated CSV schemas.+ // 1 document dataset + 2 unrelated CSV schemas.assert_eq!(before.len(), 3, "{before:#?}");}- /// The fix must not blunt clustering: unrelated schemas stay apart, near- /// -identical ones still merge, and formats never mix.+ /// The fix must not blunt clustering: unrelated data schemas stay apart,+ /// near-identical ones still merge, and data formats never mix.#[test]- fn genuinely_different_content_still_separates() {+ fn genuinely_different_data_still_separates() {let rels: Vec<String> = (0..5).map(|i| format!("d/f{i}")).collect();+ let scopes = root_scopes(5);let clusters = cluster(vec![- data_sketch(0, Family::Csv, &["ts", "level", "msg"]),+ data_sketch_n(0, Family::Csv, &["ts", "level", "msg"], 40),// one extra column out of four — still the same schema- data_sketch(1, Family::Csv, &["ts", "level", "msg", "host"]),- data_sketch(2, Family::Csv, &["id", "email", "name"]),+ data_sketch_n(1, Family::Csv, &["ts", "level", "msg", "host"], 40),+ data_sketch_n(2, Family::Csv, &["id", "email", "name"], 40),// same names, different format family- data_sketch(3, Family::Jsonl, &["ts", "level", "msg"]),+ data_sketch_n(3, Family::Jsonl, &["ts", "level", "msg"], 40),code_sketch(4, true),],&rels,+ &scopes,);assert_eq!(clusters.len(), 4, "{clusters:#?}");let members: Vec<Vec<usize>> = clusters.iter().map(|c| c.members.clone()).collect();@@ -387,4 +541,170 @@ mod tests {assert!(members.contains(&vec![3]), "{members:?}");assert!(members.contains(&vec![4]), "{members:?}");}++ /// #196: one workspace, several format families — still ONE dataset.+ /// Code and prose carry no schema, so format family is not a boundary.+ #[test]+ fn one_workspace_of_code_and_prose_is_one_dataset() {+ let rels: Vec<String> = vec![+ "src/a.rs".into(),+ "src/b.rs".into(),+ "README.md".into(),+ "docs/guide.md".into(),+ ];+ let scopes = root_scopes(4);+ let clusters = cluster(+ vec![+ code_sketch(0, true),+ code_sketch(1, false),+ prose_sketch(2),+ prose_sketch(3),+ ],+ &rels,+ &scopes,+ );+ assert_eq!(clusters.len(), 1, "{clusters:#?}");+ assert_eq!(clusters[0].members.len(), 4);+ assert!(clusters[0].is_docs);+ assert_eq!(clusters[0].slug, "docs");+ // the docs mapping holds both vocabularies+ assert!(clusters[0].fields.contains_key("language"));+ assert!(clusters[0].fields.contains_key("body"));+ }++ /// #173: two repositories in one tree — one document dataset per repo,+ /// named after the repo, regardless of how many incidental shapes the+ /// source files take.+ #[test]+ fn each_repository_is_its_own_document_dataset() {+ let rels: Vec<String> = vec![+ "valkey/src/a.c".into(),+ "valkey/README.md".into(),+ "memcached/src/b.c".into(),+ "memcached/doc/notes.md".into(),+ ];+ let scopes: Vec<String> = vec![+ "valkey".into(),+ "valkey".into(),+ "memcached".into(),+ "memcached".into(),+ ];+ let clusters = cluster(+ vec![+ code_sketch(0, true),+ prose_sketch(1),+ code_sketch(2, true),+ prose_sketch(3),+ ],+ &rels,+ &scopes,+ );+ assert_eq!(clusters.len(), 2, "{clusters:#?}");+ let mut slugs: Vec<&str> = clusters.iter().map(|c| c.slug.as_str()).collect();+ slugs.sort();+ assert_eq!(slugs, ["memcached-docs", "valkey-docs"]);+ for c in &clusters {+ assert!(c.is_docs);+ assert_eq!(c.members.len(), 2);+ }+ }++ /// #173's 382-cluster mechanism: single-record structured files with+ /// one-off key sets are configuration, not datasets — they demote into+ /// the scope's document dataset and their config keys never reach its+ /// mapping. A recurring schema (many files) stays a data dataset, and a+ /// multi-record structured file is a collection whatever its size.+ #[test]+ fn one_off_config_files_demote_to_documents() {+ let mut sketches = Vec::new();+ let mut rels = Vec::new();+ // one code file so the scope has a docs dataset already+ rels.push("src/main.c".to_string());+ sketches.push(code_sketch(0, true));+ // 30 per-command config JSONs, each its own key set, one record each+ for i in 1..=30 {+ rels.push(format!("commands/cmd{i}.json"));+ let summary = format!("CMD{i}_summary");+ let arity = format!("CMD{i}_arity");+ sketches.push(data_sketch(+ i,+ Family::Json,+ &[summary.as_str(), arity.as_str()],+ ));+ }+ // a recurring schema: 12 files sharing the same keys, one record each+ for i in 31..=42 {+ rels.push(format!("profiles/p{i}.json"));+ sketches.push(data_sketch(i, Family::Json, &["name", "email", "age"]));+ }+ // a multi-record JSON collection (array file) with a one-off schema+ rels.push("data/events.json".to_string());+ sketches.push(data_sketch_n(43, Family::Json, &["evt", "ts_ms"], 500));+ let scopes = root_scopes(rels.len());+ let clusters = cluster(sketches, &rels, &scopes);++ let docs: Vec<&Cluster> = clusters.iter().filter(|c| c.is_docs).collect();+ assert_eq!(docs.len(), 1, "{clusters:#?}");+ let d = docs[0];+ assert_eq!(d.members.len(), 31, "code + 30 demoted configs");+ assert_eq!(d.demoted.len(), 30);+ assert!(+ !d.fields.keys().any(|k| k.starts_with("CMD")),+ "config keys leaked into the document mapping: {:?}",+ d.fields.keys()+ );++ let data: Vec<&Cluster> = clusters.iter().filter(|c| !c.is_docs).collect();+ assert_eq!(data.len(), 2, "{clusters:#?}");+ assert!(+ data.iter()+ .any(|c| c.members.len() == 12 && c.key_fields.contains("email")),+ "the recurring profile schema must stay a data dataset"+ );+ assert!(+ data.iter()+ .any(|c| c.records == 500 && c.key_fields.contains("evt")),+ "a multi-record collection must stay a data dataset"+ );+ }++ /// Scope is a hard boundary for data schemas too: the same CSV schema in+ /// two repositories is two datasets (per-repo corpora stay self-contained).+ #[test]+ fn the_same_schema_in_two_repositories_stays_apart() {+ let rels: Vec<String> = vec!["a/x.csv".into(), "b/y.csv".into()];+ let scopes: Vec<String> = vec!["a".into(), "b".into()];+ let clusters = cluster(+ vec![+ data_sketch_n(0, Family::Csv, &["ts", "level", "msg"], 40),+ data_sketch_n(1, Family::Csv, &["ts", "level", "msg"], 40),+ ],+ &rels,+ &scopes,+ );+ assert_eq!(clusters.len(), 2, "{clusters:#?}");+ }++ /// Adding a file to the tree must never rename the docs dataset (its slug+ /// is scope-derived, not content-derived) — the #178 property extended to+ /// the new grouping.+ #[test]+ fn adding_a_file_never_renames_the_docs_dataset() {+ let rels3: Vec<String> = vec!["src/a.rs".into(), "src/b.rs".into(), "lib/c.rs".into()];+ let before = cluster(+ vec![code_sketch(0, true), code_sketch(1, true)],+ &rels3,+ &root_scopes(3),+ );+ let after = cluster(+ vec![+ code_sketch(0, true),+ code_sketch(1, true),+ code_sketch(2, true),+ ],+ &rels3,+ &root_scopes(3),+ );+ assert_eq!(before[0].slug, after[0].slug);+ }}diff --git a/engine/crates/xerj-autoindex/src/extract/mod.rs b/engine/crates/xerj-autoindex/src/extract/mod.rsindex b18e959f6..adad65ff0 100644--- a/engine/crates/xerj-autoindex/src/extract/mod.rs+++ b/engine/crates/xerj-autoindex/src/extract/mod.rs@@ -116,6 +116,31 @@ pub fn read_whole(path: &Path, gzip: bool, cap: u64) -> Result<Option<Vec<u8>>>Ok(Some(buf))}+/// Render a file as a DOCUMENT (title = file stem, body = decoded text,+/// section-split), regardless of its format family. This is how demoted+/// one-off config files (`dataset` module docs, #173) are indexed: their+/// key sets are configuration, not a schema, so their retrievable value is+/// the text itself. Emits the same `title`/`body`/`section` vocabulary as+/// every other document extractor (`FieldOrigin::Extractor`).+pub fn extract_as_document(path: &Path, gzip: bool, sink: Sink) -> Result<ExtractStats> {+ let mut stats = ExtractStats::default();+ let Some(bytes) = read_whole(path, gzip, MAX_WHOLE_FILE)? else {+ stats.junk += 1;+ return Ok(stats);+ };+ let (text, _) = crate::sniff::decode_text(&bytes);+ if text.trim().is_empty() {+ stats.junk += 1;+ return Ok(stats);+ }+ let title = path+ .file_stem()+ .map(|s| s.to_string_lossy().to_string())+ .unwrap_or_else(|| "untitled".into());+ emit_document(&title, &[], text.trim(), sink, &mut stats);+ Ok(stats)+}+/// Dispatch to the family extractor. `limit_bytes` bounds SAMPLING reads;/// `None` = full stream.pub fn extract(diff --git a/engine/crates/xerj-autoindex/src/infer/mod.rs b/engine/crates/xerj-autoindex/src/infer/mod.rsindex 6491db9e1..430db4764 100644--- a/engine/crates/xerj-autoindex/src/infer/mod.rs+++ b/engine/crates/xerj-autoindex/src/infer/mod.rs@@ -15,6 +15,13 @@ pub const DISTINCT_CAP: usize = 8192;pub const RAW_CAP: usize = 8192;pub const MAX_FIELDS_PER_DATASET: usize = 512;+/// The extractor-owned fields that carry a document's text. Every document+/// extractor puts its retrievable content in exactly one of these+/// (`extract::emit_document`, `extract::code`, `extract::txt`), which is what+/// lets a document dataset elect them ALL as `semantic_text` without ever+/// embedding one record twice.+pub const DOC_BODY_FIELDS: &[&str] = &["body", "text"];+/// 256-slot byte histogram with a `Default` impl (`[u32; 256]` has none).#[derive(Debug, Clone)]pub struct ByteHist(pub [u32; 256]);@@ -387,6 +394,34 @@ pub fn infer_fields(fields: &HashMap<String, FieldAcc>,records: u64,no_semantic: bool,+) -> Vec<FieldSpec> {+ infer_fields_with_policy(fields, records, no_semantic, false)+}++/// `infer_fields` with the semantic election policy explicit.+///+/// `semantic_all = false` (data datasets): elect at most ONE semantic body —+/// the longest qualifying field. A data record usually carries several long+/// text columns holding the SAME entity, and embedding every one multiplies+/// cost (the built-in neural backend measures ~3 docs/s) for no retrieval+/// gain.+///+/// `semantic_all = true` (document datasets): elect every qualifying BODY+/// CARRIER (`DOC_BODY_FIELDS` — extractor-owned names, so this is a contract+/// of this crate, not a heuristic). A document dataset merges several+/// extractor vocabularies (`body` from code/prose/pdf/html, `text` from+/// line-oriented chunks) and each record populates exactly one carrier, so+/// electing all of them costs the same embedding work as electing one —+/// while electing one silently cuts entire format families out of the+/// vector arm (#173: 98.8% of the corpus had no `semantic_text` at all).+/// Non-carrier text fields (`defs` symbol lists) stay lexical: they ride on+/// the same record as a carrier, so electing them would embed every code+/// record twice.+pub fn infer_fields_with_policy(+ fields: &HashMap<String, FieldAcc>,+ records: u64,+ no_semantic: bool,+ semantic_all: bool,) -> Vec<FieldSpec> {let mut specs: Vec<FieldSpec> = Vec::new();let mut date_ranges: Vec<(chrono::DateTime<chrono::Utc>, chrono::DateTime<chrono::Utc>)> =@@ -548,7 +583,7 @@ pub fn infer_fields(// source code. The length floor is kept but relaxed, because the// language test now does the discriminating.if !no_semantic {- let best = specs+ let qualifying: Vec<usize> = specs.iter().enumerate().filter(|(_i, s)| {@@ -559,8 +594,21 @@ pub fn infer_fields(.map(|a| a.looks_natural_language()).unwrap_or(false)})- .max_by(|a, b| a.1.avg_len.partial_cmp(&b.1.avg_len).unwrap());- if let Some((i, _)) = best {+ .map(|(i, _)| i)+ .collect();+ let elected: Vec<usize> = if semantic_all {+ qualifying+ .into_iter()+ .filter(|&i| DOC_BODY_FIELDS.contains(&specs[i].name.as_str()))+ .collect()+ } else {+ qualifying+ .into_iter()+ .max_by(|&a, &b| specs[a].avg_len.partial_cmp(&specs[b].avg_len).unwrap())+ .into_iter()+ .collect()+ };+ for i in elected {let a = fields.get(&specs[i].name);specs[i].es_type = "semantic_text".into();specs[i].notes.push(format!(@@ -720,6 +768,45 @@ mod tests {}}+ /// #173: a document dataset merges extractor vocabularies whose records+ /// each populate exactly one body carrier (`body` for code/prose, `text`+ /// for line chunks). The docs policy must elect EVERY qualifying carrier+ /// — electing only the longest silently cut whole format families out of+ /// the vector arm — while a non-carrier text field (`defs`) stays+ /// lexical, because it rides on the same record as a carrier and electing+ /// it would embed every code record twice.+ #[test]+ fn a_document_dataset_elects_every_body_carrier_and_only_the_carriers() {+ let prose = "The connection pool retries every failed handshake with backoff. \+ Each worker owns one socket and never shares it across threads.";+ let mut fields = HashMap::new();+ for name in ["body", "text", "defs"] {+ let mut acc = FieldAcc::default();+ for _ in 0..20 {+ acc.add(&Value::String(prose.to_string()));+ }+ fields.insert(name.to_string(), acc);+ }++ // Data policy (semantic_all = false): at most one semantic field.+ let single = infer_fields_with_policy(&fields, 60, false, false);+ assert_eq!(+ single+ .iter()+ .filter(|s| s.es_type == "semantic_text")+ .count(),+ 1,+ "{single:#?}"+ );++ // Docs policy: both carriers elected, the non-carrier stays text.+ let docs = infer_fields_with_policy(&fields, 60, false, true);+ let by_name = |n: &str| docs.iter().find(|s| s.name == n).unwrap();+ assert_eq!(by_name("body").es_type, "semantic_text", "{docs:#?}");+ assert_eq!(by_name("text").es_type, "semantic_text", "{docs:#?}");+ assert_eq!(by_name("defs").es_type, "text", "{docs:#?}");+ }+/// Pins the reason the entity tie is latent rather than live: the ≥90%/// supermajority gate rejects any tied field, so the winner never reaches/// the spec. If this test starts failing because the gate was relaxed, thediff --git a/engine/crates/xerj-autoindex/src/lib.rs b/engine/crates/xerj-autoindex/src/lib.rsindex da8614e6f..351d381b5 100644--- a/engine/crates/xerj-autoindex/src/lib.rs+++ b/engine/crates/xerj-autoindex/src/lib.rs@@ -33,7 +33,7 @@ use cli::{Cmd, IndexCfg, MapCfg, StatusCfg};use detect::EdgeDetector as _;use esclient::Es;use sniff::{Family, Sniffed};-use state::{FileAssignment, FileDone, JunkFile, Plan, PlanDataset};+use state::{DuplicateFile, FileAssignment, FileDone, JunkFile, Plan, PlanDataset};/// Entry point for the `xerj autoindex` subcommand (blocking; the server/// binary calls this via spawn_blocking). Returns the process exit code.@@ -358,7 +358,8 @@ mod clustering_key_tests {records: s.sketches[0].records,}).collect();- let clusters = dataset::cluster(sketches, &rels);+ let scopes = vec![String::new(); rels.len()];+ let clusters = dataset::cluster(sketches, &rels, &scopes);assert_eq!(clusters.len(), 1, "{clusters:#?}");}@@ -380,6 +381,348 @@ mod clustering_key_tests {}}+#[cfg(test)]+mod phase_a_grouping_tests {+ use super::*;++ fn cfg_for(root: &Path) -> IndexCfg {+ IndexCfg {+ root: root.to_path_buf(),+ url: "http://unused.invalid".into(),+ api_key: None,+ workers: 1,+ pdf_workers: 1,+ pdf_timeout_secs: 10,+ bulk_mb: 1,+ bulk_timeout_secs: 10,+ prefix: "t".into(),+ state_dir: None,+ fresh: true,+ follow_symlinks: false,+ max_file_gb: 2,+ sample: 500,+ no_semantic: false,+ brain: None,+ no_graph: true,+ dry_run: true,+ json: false,+ quiet: true,+ }+ }++ fn plan_for(root: &Path) -> Plan {+ let files = walk::walk(root, false).unwrap();+ let keys: Vec<String> = files+ .iter()+ .map(|f| ids::file_key(&f.path, f.size).unwrap())+ .collect();+ let digests: Vec<String> = (0..files.len()).map(|i| format!("d{i}")).collect();+ let (plan, _) = build_phase_a(root, &files, &keys, &digests, Vec::new(), &cfg_for(root));+ plan+ }++ const CODE: &str = "// The event loop dispatches every ready connection to a worker.\n\+ // Each worker drains its queue before polling for more sockets.\n\+ static void dispatch_ready_connections(struct event_loop *loop) {\n\+ for (int index = 0; index < loop->ready_count; index++) {\n\+ worker_submit(loop->workers, loop->ready[index]);\n\+ }\n\+ }\n";++ const PROSE: &str = "# Overview\n\nThis server accepts client connections and stores \+ keys in memory. Every command travels through the same parser before the \+ dispatcher routes it to the matching handler function.\n";++ /// #173 end to end through the real planner: a tree holding two+ /// repositories of source, prose and one-off config JSON yields one+ /// document dataset per repository — with `body` elected `semantic_text`+ /// — plus the genuine data dataset, instead of one dataset per incidental+ /// config schema with no vector arm.+ #[test]+ fn a_two_repo_tree_plans_one_document_dataset_per_repo() {+ let dir = tempfile::tempdir().unwrap();+ let root = dir.path();+ for repo in ["valkey", "memcached"] {+ std::fs::create_dir_all(root.join(repo).join(".git")).unwrap();+ std::fs::write(root.join(repo).join(".git").join("HEAD"), "ref: x").unwrap();+ }+ std::fs::create_dir_all(root.join("valkey/src")).unwrap();+ std::fs::create_dir_all(root.join("valkey/commands")).unwrap();+ std::fs::create_dir_all(root.join("memcached/data")).unwrap();+ std::fs::write(root.join("valkey/src/server.c"), CODE).unwrap();+ std::fs::write(root.join("valkey/README.md"), PROSE).unwrap();+ // one-off configs: single-record JSON, each with its own key set+ std::fs::write(+ root.join("valkey/commands/get.json"),+ r#"{"GET": {"summary": "Return the string value stored at the given key.", "arity": 2}}"#,+ )+ .unwrap();+ std::fs::write(+ root.join("valkey/commands/set.json"),+ r#"{"SET": {"summary": "Store the given string value under the given key.", "arity": 3}}"#,+ )+ .unwrap();+ // distinct bytes — byte-identical files alias into one canonical copy+ std::fs::write(root.join("memcached/proto.c"), format!("{CODE}\n// v2\n")).unwrap();+ // a genuine data file: recurring rows, real schema+ std::fs::write(+ root.join("memcached/data/events.csv"),+ )+ .unwrap();++ let plan = plan_for(root);+ let mut slugs: Vec<&str> = plan.datasets.iter().map(|d| d.slug.as_str()).collect();+ slugs.sort();+ assert_eq!(+ slugs,+ ["memcached-data", "memcached-docs", "valkey-docs"],+ "{:#?}",+ plan.datasets+ );++ // every docs dataset carries the vector arm on body+ for d in plan.datasets.iter().filter(|d| d.family == "docs") {+ let body = d.specs.iter().find(|s| s.name == "body").unwrap();+ assert_eq!(body.es_type, "semantic_text", "{}: {:#?}", d.slug, d.specs);+ assert_eq!(d.semantic_field.as_deref(), Some("body"), "{}", d.slug);+ }++ // the one-off configs were demoted: document-rendered, and their+ // config keys never reached the docs mapping+ let by_rel: HashMap<&str, &FileAssignment> =+ plan.files.values().map(|f| (f.rel.as_str(), f)).collect();+ assert!(by_rel["valkey/commands/get.json"].as_document);+ assert!(by_rel["valkey/commands/set.json"].as_document);+ assert!(!by_rel["valkey/src/server.c"].as_document);+ assert!(!by_rel["memcached/data/events.csv"].as_document);+ let vdocs = plan+ .datasets+ .iter()+ .find(|d| d.slug == "valkey-docs")+ .unwrap();+ assert!(+ !vdocs.specs.iter().any(|s| s.name.contains("summary")),+ "config keys leaked into the docs mapping: {:#?}",+ vdocs.specs+ );+ assert_eq!(vdocs.file_count, 4, "code + prose + 2 demoted configs");++ // the CSV kept its real schema+ let data = plan+ .datasets+ .iter()+ .find(|d| d.slug == "memcached-data")+ .unwrap();+ assert!(data.specs.iter().any(|s| s.name == "email"), "{data:#?}");+ }++ /// #196: the same tree WITHOUT nested `.git` markers (or with one at the+ /// root — a workspace) is ONE scope: a single document corpus.+ #[test]+ fn a_single_workspace_plans_one_document_dataset() {+ let dir = tempfile::tempdir().unwrap();+ let root = dir.path();+ std::fs::create_dir_all(root.join(".git")).unwrap();+ std::fs::create_dir_all(root.join("src")).unwrap();+ std::fs::create_dir_all(root.join("wasm/examples")).unwrap();+ std::fs::write(root.join("src/a.c"), CODE).unwrap();+ std::fs::write(root.join("src/b.c"), CODE).unwrap();+ std::fs::write(root.join("README.md"), PROSE).unwrap();+ std::fs::write(root.join("wasm/examples/demo.md"), PROSE).unwrap();++ let plan = plan_for(root);+ assert_eq!(+ plan.datasets.len(),+ 1,+ "one workspace, one corpus: {:#?}",+ plan.datasets+ );+ assert_eq!(plan.datasets[0].slug, "docs");+ assert_eq!(plan.datasets[0].file_count, 4);+ }+}++// ─── Phase A plan building (pure: no server contact) ─────────────────────++/// Repository scope per file: the deepest ancestor directory (root-relative,+/// "/"-separated) containing a `.git` entry, or "" when the file is under+/// none. The walk never descends into `.git` itself, but the marker is still+/// on disk. A `.git` at the autoindex root yields "" for every file — the+/// whole tree is one scope — and so does a tree with no `.git` at all, which+/// is exactly the #173/#196 property: one autoindex of one folder yields a+/// corpus searchable as one corpus, split only at nested repository roots.+fn compute_scopes(root: &Path, rels: &[String]) -> Vec<String> {+ let mut cache: HashMap<String, bool> = HashMap::new();+ rels.iter()+ .map(|rel| {+ let mut scope = String::new();+ let mut prefix = String::new();+ let mut segs = rel.split('/').peekable();+ while let Some(seg) = segs.next() {+ if segs.peek().is_none() {+ break; // final segment is the file name+ }+ if !prefix.is_empty() {+ prefix.push('/');+ }+ prefix.push_str(seg);+ let repo = *cache+ .entry(prefix.clone())+ .or_insert_with(|| root.join(&prefix).join(".git").exists());+ if repo {+ scope = prefix.clone();+ }+ }+ scope+ })+ .collect()+}++/// Sniff + sample every file, cluster into datasets, and assemble the plan.+/// Pure planning: reads the tree, never the server — which is what makes the+/// #173/#196 grouping behaviour testable end-to-end without a cluster.+fn build_phase_a(+ root: &Path,+ files: &[walk::FileEntry],+ keys: &[String],+ digests: &[String],+ duplicate_files: Vec<DuplicateFile>,+ cfg: &IndexCfg,+) -> (Plan, Vec<dataset::Cluster>) {+ use rayon::prelude::*;+ let scans: Vec<FileScan> = files+ .par_iter()+ .map(|f| scan_file(&f.path, f.size, cfg.sample, cfg.max_file_gb))+ .collect();++ let rels: Vec<String> = files.iter().map(|f| f.rel.clone()).collect();+ let scopes = compute_scopes(root, &rels);+ // (family, gzip) per file, from the scan's sniff — reused for assignments+ // and demoted-file re-sampling instead of re-sniffing every file.+ let file_meta: Vec<Option<(Family, bool)>> = scans+ .iter()+ .map(|sc| sc.sniffed.as_ref().map(|s| (s.family, s.gzip)))+ .collect();+ let mut sketches = Vec::new();+ let mut junk_files = Vec::new();+ for (i, sc) in scans.into_iter().enumerate() {+ let family = sc+ .sniffed+ .as_ref()+ .map(|s| s.family)+ .unwrap_or(Family::Binary);+ if let Some((status, reason)) = sc.junk {+ junk_files.push(JunkFile {+ file_key: keys[i].clone(),+ rel: files[i].rel.clone(),+ format: format_str(sc.sniffed.as_ref()),+ status,+ reason,+ bytes: files[i].size,+ });+ continue;+ }+ for gs in sc.sketches {+ sketches.push(dataset::Sketch {+ file_idx: i,+ group: gs.group,+ family,+ fields: gs.fields,+ key_fields: gs.key_fields,+ records: gs.records,+ });+ }+ }+ let mut clusters = dataset::cluster(sketches, &rels, &scopes);++ // Demoted one-off config files (#173) were sampled as flattened records;+ // phase B will index them as documents. Re-sample them through the same+ // document renderer so the docs dataset's mapping and stats describe what+ // actually gets indexed.+ for c in clusters.iter_mut().filter(|c| !c.demoted.is_empty()) {+ let demoted = c.demoted.clone();+ let fields = &mut c.fields;+ let mut sampled_total = 0u64;+ for m in demoted {+ let gzip = file_meta[m].map(|(_, g)| g).unwrap_or(false);+ let mut sampled = 0u64;+ let mut sink = |rec: extract::RawRecord| -> bool {+ if (sampled as usize) < cfg.sample {+ for (k, v) in &rec.fields {+ fields.entry(k.clone()).or_default().add(v);+ }+ }+ sampled += 1;+ (sampled as usize) < cfg.sample+ };+ // An unreadable file junks at phase B like any other; the plan+ // keeps its membership either way.+ let _ = extract::extract_as_document(&files[m].path, gzip, &mut sink);+ sampled_total += sampled;+ }+ c.records += sampled_total;+ }++ // per-file assignments+ let mut file_assignments: HashMap<String, FileAssignment> = HashMap::new();+ for (ci, c) in clusters.iter().enumerate() {+ let demoted: std::collections::HashSet<usize> = c.demoted.iter().copied().collect();+ for &m in &c.members {+ let key = &keys[m];+ let (family, gzip) = file_meta[m].unwrap_or((c.family, false));+ let fa = file_assignments+ .entry(key.clone())+ .or_insert_with(|| FileAssignment {+ rel: files[m].rel.clone(),+ path_id: files[m].rel_id.clone(),+ family: family.as_str().to_string(),+ gzip,+ content_digest: Some(digests[m].clone()),+ assignments: Vec::new(),+ as_document: false,+ });+ fa.as_document |= demoted.contains(&m);+ fa.assignments+ .push((c.group.clone(), clusters[ci].slug.clone()));+ }+ }++ let mut datasets = Vec::new();+ for c in &clusters {+ let specs =+ infer::infer_fields_with_policy(&c.fields, c.records, cfg.no_semantic, c.is_docs);+ let time_field = infer::elect_time_field(&specs);+ let semantic_field = specs+ .iter()+ .find(|s| s.es_type == "semantic_text")+ .map(|s| s.name.clone());+ datasets.push(PlanDataset {+ slug: c.slug.clone(),+ index: format!("{}-{}", cfg.prefix, c.slug),+ family: if c.is_docs {+ "docs".to_string()+ } else {+ c.family.as_str().to_string()+ },+ group: c.group.clone(),+ specs,+ time_field,+ semantic_field,+ sampled_records: c.records,+ file_count: c.members.len(),+ });+ }+ let plan = Plan {+ datasets,+ files: file_assignments,+ junk_files,+ duplicate_files,+ alias_paths_indexed: true,+ };+ (plan, clusters)+}+// ─── mapping builder ─────────────────────────────────────────────────────pub const PROVENANCE_FIELDS: &[&str] = &[@@ -564,7 +907,6 @@ fn run_index(cfg: IndexCfg) -> Result<i32> {pub fn run_index_report(cfg: IndexCfg) -> Result<(i32, Option<Value>)> {extract::pdf::configure_workers(cfg.pdf_workers);extract::pdf::configure_timeout(cfg.pdf_timeout_secs);- use rayon::prelude::*;let t0 = Instant::now();if !cfg.quiet {eprintln!(@@ -773,99 +1115,21 @@ pub fn run_index_report(cfg: IndexCfg) -> Result<(i32, Option<Value>)> {if !cfg.quiet {eprintln!("phase A: sniffing + sampling {} files…", files.len());}- let scans: Vec<FileScan> = files- .par_iter()- .map(|f| scan_file(&f.path, f.size, cfg.sample, cfg.max_file_gb))- .collect();-- let rels: Vec<String> = files.iter().map(|f| f.rel.clone()).collect();- let mut sketches = Vec::new();- let mut junk_files = Vec::new();- for (i, sc) in scans.into_iter().enumerate() {- let family = sc- .sniffed- .as_ref()- .map(|s| s.family)- .unwrap_or(Family::Binary);- if let Some((status, reason)) = sc.junk {- junk_files.push(JunkFile {- file_key: keys[i].clone(),- rel: files[i].rel.clone(),- format: format_str(sc.sniffed.as_ref()),- status,- reason,- bytes: files[i].size,- });- continue;- }- for gs in sc.sketches {- sketches.push(dataset::Sketch {- file_idx: i,- group: gs.group,- family,- fields: gs.fields,- key_fields: gs.key_fields,- records: gs.records,- });- }- }- let clusters = dataset::cluster(sketches, &rels);+ let (plan, clusters) = build_phase_a(+ &cfg.root,+ &files,+ &keys,+ &digests,+ duplicate_files.clone(),+ &cfg,+ );if !cfg.quiet {eprintln!("phase A: {} datasets inferred, {} junk/skipped files",- clusters.len(),- junk_files.len()+ plan.datasets.len(),+ plan.junk_files.len());}-- // per-file assignments- let mut file_assignments: HashMap<String, FileAssignment> = HashMap::new();- for (ci, c) in clusters.iter().enumerate() {- for &m in &c.members {- let key = &keys[m];- let sn = sniff::sniff(&files[m].path).ok();- let fa = file_assignments- .entry(key.clone())- .or_insert_with(|| FileAssignment {- rel: files[m].rel.clone(),- path_id: files[m].rel_id.clone(),- family: c.family.as_str().to_string(),- gzip: sn.map(|s| s.gzip).unwrap_or(false),- content_digest: Some(digests[m].clone()),- assignments: Vec::new(),- });- fa.assignments- .push((c.group.clone(), clusters[ci].slug.clone()));- }- }-- let mut datasets = Vec::new();- for c in &clusters {- let specs = infer::infer_fields(&c.fields, c.records, cfg.no_semantic);- let time_field = infer::elect_time_field(&specs);- let semantic_field = specs- .iter()- .find(|s| s.es_type == "semantic_text")- .map(|s| s.name.clone());- datasets.push(PlanDataset {- slug: c.slug.clone(),- index: format!("{}-{}", cfg.prefix, c.slug),- family: c.family.as_str().to_string(),- group: c.group.clone(),- specs,- time_field,- semantic_field,- sampled_records: c.records,- file_count: c.members.len(),- });- }- let plan = Plan {- datasets,- files: file_assignments,- junk_files,- duplicate_files,- alias_paths_indexed: true,- };clusters_rt = Some(clusters);plan};@@ -1401,7 +1665,14 @@ pub fn run_index_report(cfg: IndexCfg) -> Result<(i32, Option<Value>)> {}true};- let res = extract::extract(&f.path, &sn, None, &mut sink);+ // Demoted one-off config files (#173) index as+ // documents — their key sets are configuration, not a+ // schema (see `dataset` module docs).+ let res = if fa.as_document {+ extract::extract_as_document(&f.path, sn.gzip, &mut sink)+ } else {+ extract::extract(&f.path, &sn, None, &mut sink)+ };match res {Ok(stats) => {file_junk += stats.junk;@@ -2351,6 +2622,7 @@ mod duplicate_integration_tests {gzip: false,content_digest: None,assignments: vec![(None, "text".to_string())],+ as_document: false,}}diff --git a/engine/crates/xerj-autoindex/src/state.rs b/engine/crates/xerj-autoindex/src/state.rsindex 5df8fdd60..50487c15f 100644--- a/engine/crates/xerj-autoindex/src/state.rs+++ b/engine/crates/xerj-autoindex/src/state.rs@@ -193,6 +193,11 @@ pub struct FileAssignment {pub content_digest: Option<String>,/// group (None = whole file) → dataset slugpub assignments: Vec<(Option<String>, String)>,+ /// Demoted one-off config file (#173): index it through the document+ /// renderer (`extract::extract_as_document`), not its family extractor.+ /// Defaults false so frozen plans from earlier versions resume unchanged.+ #[serde(default, skip_serializing_if = "std::ops::Not::not")]+ pub as_document: bool,}#[derive(Debug, Clone, Serialize, Deserialize)]