Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

fix(autoindex): repository-scoped dataset grouping — a 2-repo tree plans 13 datasets, not 404

xerj-orga246328b366d
diff --git a/engine/crates/xerj-autoindex/src/dataset.rs b/engine/crates/xerj-autoindex/src/dataset.rs
index 31065a65d..2e85d58af 100644
--- a/engine/crates/xerj-autoindex/src/dataset.rs
+++ b/engine/crates/xerj-autoindex/src/dataset.rs
@@ -1,21 +1,46 @@
-//! Dataset clustering: files (or per-table groups within files) merge into
-//! datasets by schema fingerprint (Jaccard on field-name sets ≥ 0.7) within
-//! the same format family. Path family is a NAMING hint only — no hardcoded
-//! directory semantics.
+//! Dataset clustering: what a "dataset" means for a folder of files.
//!
-//! Only the DATA-derived field names take part (`Sketch::key_fields`, fed from
-//! `extract::FieldOrigin`). A name the extractor invented — `defs`, `symbols`,
-//! `title`, `page` — is not evidence about the file, and letting one decide
-//! membership made every extractor improvement re-home files: the dataset slug
-//! is an ingredient of `ids::doc_id`, so a moved file is re-indexed under a new
-//! `_id` in a new index while its old document survives, unreferenced, in the
-//! old one (issue #178). Files whose names are all extractor-invented (source
-//! code, prose documents) therefore cluster by format family and group alone,
-//! which is what their sniffed shape actually says about them.
+//! Two kinds of dataset come out of a tree (#173/#196):
+//!
+//! **Documents.** Files whose field names are all extractor-invented (source
+//! code, prose, PDFs, line-oriented text, HTML pages) carry no schema of their
+//! own, so schema inference has nothing legitimate to split them on. They merge
+//! into ONE dataset per *scope* — the nearest enclosing repository root
+//! (a directory with a `.git` entry), or the autoindex root when the file is
+//! under no repository. Splitting them per format family produced indices no
+//! user recognised as datasets (#196: one Rust workspace → four indices), and
+//! per-schema clustering of their incidental shapes shattered a two-repo tree
+//! into hundreds (#173). One folder → one searchable corpus per repository.
+//!
+//! **Data.** Files with data-derived field names (`Sketch::key_fields`, fed
+//! from `extract::FieldOrigin`: CSV headers, JSON keys, SQL columns) still
+//! cluster by schema fingerprint (Jaccard on field-name sets ≥ 0.7) within the
+//! same format family AND scope — a real schema that recurs across files is a
+//! dataset. But a self-describing config blob is not: a small cluster of
+//! `json`/`yaml`/`xml` files that each produced exactly one record is *demoted*
+//! to the scope's document dataset and indexed as a document (title/body),
+//! because its one-off key set is configuration, not a collection
+//! (#173: valkey's per-command JSON files made 382 such "datasets").
+//!
+//! Only DATA-derived names take part in schema clustering. A name the
+//! extractor invented — `defs`, `symbols`, `title`, `page` — is not evidence
+//! about the file, and letting one decide membership made every extractor
+//! improvement re-home files: the dataset slug is an ingredient of
+//! `ids::doc_id`, so a moved file is re-indexed under a new `_id` in a new
+//! index while its old document survives, unreferenced, in the old one
+//! (issue #178). Scope-based document grouping strengthens that stability:
+//! a document's dataset now depends only on its own path and the repository
+//! layout, never on which other files exist.
use crate::infer::FieldAcc;
use crate::sniff::Family;
-use std::collections::{HashMap, HashSet};
+use std::collections::{BTreeMap, HashMap, HashSet};
+
+/// A schema-clustered `json`/`yaml`/`xml` group at most this many files large,
+/// in which every file produced exactly one record, is configuration — demote
+/// it to the scope's document dataset. A schema that recurs across more files
+/// than this is treated as a real single-record-per-file collection.
+pub const DOC_DEMOTE_MAX_FILES: usize = 8;
#[derive(Debug)]
pub struct Sketch {
@@ -24,7 +49,7 @@ pub struct Sketch {
pub family: Family,
pub fields: HashMap<String, FieldAcc>,
/// The subset of `fields` whose NAMES were read out of the file. This, not
- /// `fields`, is the clustering key.
+ /// `fields`, is the schema-clustering key; empty means "document".
pub key_fields: HashSet<String>,
pub records: u64,
}
@@ -39,6 +64,14 @@ pub struct Cluster {
pub key_fields: HashSet<String>,
pub records: u64,
pub slug: String,
+ /// Repository scope this cluster lives in ("" = the autoindex root).
+ pub scope: String,
+ /// A per-scope document dataset (code/prose/config), not a schema dataset.
+ pub is_docs: bool,
+ /// Members demoted from one-off config clusters. Their sampled fields were
+ /// discarded (config keys are not document fields); the caller re-samples
+ /// them through the document renderer and must index them as documents.
+ pub demoted: Vec<usize>,
}
fn jaccard(a: &HashSet<&str>, b: &HashSet<&str>) -> f64 {
@@ -50,13 +83,58 @@ fn jaccard(a: &HashSet<&str>, b: &HashSet<&str>) -> f64 {
inter as f64 / union.max(1) as f64
}
-pub fn cluster(sketches: Vec<Sketch>, rels: &[String]) -> Vec<Cluster> {
- let mut clusters: Vec<Cluster> = Vec::new();
+/// May this data cluster be demoted to documents if it stays a small fleet of
+/// single-record files? Self-describing structured formats only: CSV is
+/// tabular by nature, logs/jsonl are collections by nature, and sql groups
+/// carry a real table schema.
+fn demotable_family(f: Family) -> bool {
+ matches!(f, Family::Json | Family::Yaml | Family::Xml)
+}
+
+/// Cluster sketches into datasets. `rels` and `scopes` are indexed by
+/// `Sketch::file_idx`: `rels` are root-relative paths (slug naming), `scopes`
+/// the repository scope of each file ("" = root) — see `module` docs.
+pub fn cluster(sketches: Vec<Sketch>, rels: &[String], scopes: &[String]) -> Vec<Cluster> {
+ // scope → docs cluster (BTreeMap: deterministic scope order).
+ let mut docs: BTreeMap<String, Cluster> = BTreeMap::new();
+ let mut data: Vec<Cluster> = Vec::new();
+ fn docs_cluster<'a>(docs: &'a mut BTreeMap<String, Cluster>, scope: &str) -> &'a mut Cluster {
+ docs.entry(scope.to_string()).or_insert_with(|| Cluster {
+ family: Family::Code,
+ group: None,
+ members: Vec::new(),
+ fields: HashMap::new(),
+ key_fields: HashSet::new(),
+ records: 0,
+ slug: String::new(),
+ scope: scope.to_string(),
+ is_docs: true,
+ demoted: Vec::new(),
+ })
+ }
+
for sk in sketches {
+ let scope = scopes[sk.file_idx].as_str();
+ // Documents: no data-derived names, no sub-file group.
+ if sk.key_fields.is_empty() && sk.group.is_none() {
+ let c = docs_cluster(&mut docs, scope);
+ c.members.push(sk.file_idx);
+ c.records += sk.records;
+ for (k, acc) in sk.fields {
+ match c.fields.get_mut(&k) {
+ Some(existing) => existing.merge(&acc),
+ None => {
+ c.fields.insert(k, acc);
+ }
+ }
+ }
+ continue;
+ }
+ // Data: schema clustering within (scope, family, group).
let names: HashSet<&str> = sk.key_fields.iter().map(|s| s.as_str()).collect();
let mut best: Option<(usize, f64)> = None;
- for (ci, c) in clusters.iter().enumerate() {
- if c.family != sk.family || c.group != sk.group {
+ for (ci, c) in data.iter().enumerate() {
+ if c.family != sk.family || c.group != sk.group || c.scope != scope {
continue;
}
let cnames: HashSet<&str> = c.key_fields.iter().map(|s| s.as_str()).collect();
@@ -68,7 +146,7 @@ pub fn cluster(sketches: Vec<Sketch>, rels: &[String]) -> Vec<Cluster> {
}
match best {
Some((ci, _)) => {
- let c = &mut clusters[ci];
+ let c = &mut data[ci];
c.members.push(sk.file_idx);
c.records += sk.records;
c.key_fields.extend(sk.key_fields);
@@ -81,7 +159,7 @@ pub fn cluster(sketches: Vec<Sketch>, rels: &[String]) -> Vec<Cluster> {
}
}
}
- None => clusters.push(Cluster {
+ None => data.push(Cluster {
family: sk.family,
group: sk.group,
members: vec![sk.file_idx],
@@ -89,10 +167,58 @@ pub fn cluster(sketches: Vec<Sketch>, rels: &[String]) -> Vec<Cluster> {
key_fields: sk.key_fields,
records: sk.records,
slug: String::new(),
+ scope: scope.to_string(),
+ is_docs: false,
+ demoted: Vec::new(),
}),
}
}
- assign_slugs(&mut clusters, rels);
+
+ // Demote one-off config clusters to their scope's document dataset. Their
+ // sampled fields are dropped — the caller re-samples the demoted files
+ // through the document renderer, which is also what indexes them.
+ let mut kept: Vec<Cluster> = Vec::with_capacity(data.len());
+ for c in data {
+ let one_record_each = c.records == c.members.len() as u64;
+ if demotable_family(c.family)
+ && c.group.is_none()
+ && c.members.len() <= DOC_DEMOTE_MAX_FILES
+ && one_record_each
+ {
+ let d = docs_cluster(&mut docs, &c.scope);
+ d.members.extend(c.members.iter().copied());
+ d.demoted.extend(c.members);
+ continue;
+ }
+ kept.push(c);
+ }
+
+ // Docs slugs are scope-derived — independent of which other files exist,
+ // so adding a file can never rename (and re-home, #178) a docs dataset.
+ let mut clusters: Vec<Cluster> = Vec::with_capacity(docs.len() + kept.len());
+ let mut used: HashSet<String> = HashSet::new();
+ for (scope, mut c) in docs {
+ let base = if scope.is_empty() {
+ "docs".to_string()
+ } else {
+ let s = sanitize_slug(&scope);
+ if s.is_empty() {
+ "docs".to_string()
+ } else {
+ format!("{s}-docs")
+ }
+ };
+ let mut slug = base.clone();
+ let mut k = 2;
+ while !used.insert(slug.clone()) {
+ slug = format!("{base}-{k}");
+ k += 1;
+ }
+ c.slug = slug;
+ clusters.push(c);
+ }
+ assign_slugs(&mut kept, rels, &mut used);
+ clusters.extend(kept);
clusters
}
@@ -131,7 +257,7 @@ fn path_candidate(rel: &str) -> String {
meaningful.join("-")
}
-fn assign_slugs(clusters: &mut [Cluster], rels: &[String]) {
+fn assign_slugs(clusters: &mut [Cluster], rels: &[String], used: &mut HashSet<String>) {
// deterministic cluster order: by first member rel
let mut order: Vec<usize> = (0..clusters.len()).collect();
order.sort_by_key(|&i| {
@@ -226,7 +352,6 @@ fn assign_slugs(clusters: &mut [Cluster], rels: &[String]) {
}
// final dedup with -2/-3 …
- let mut used: HashSet<String> = HashSet::new();
for &i in &order {
let mut slug = bases[i].clone();
let mut k = 2;
@@ -271,8 +396,27 @@ mod tests {
}
}
+ /// A prose file: same extractor-owned vocabulary situation as code.
+ fn prose_sketch(file_idx: usize) -> Sketch {
+ let mut fields = HashMap::new();
+ fields.insert("title".to_string(), acc("README"));
+ fields.insert("body".to_string(), acc("This project does things."));
+ Sketch {
+ file_idx,
+ group: None,
+ family: Family::TxtProse,
+ fields,
+ key_fields: HashSet::new(),
+ records: 1,
+ }
+ }
+
/// A data file: the names are the file's own, so they are the key.
fn data_sketch(file_idx: usize, family: Family, names: &[&str]) -> Sketch {
+ data_sketch_n(file_idx, family, names, 1)
+ }
+
+ fn data_sketch_n(file_idx: usize, family: Family, names: &[&str], records: u64) -> Sketch {
let mut fields = HashMap::new();
let mut key_fields = HashSet::new();
for n in names {
@@ -285,10 +429,14 @@ mod tests {
family,
fields,
key_fields,
- records: 1,
+ records,
}
}
+ fn root_scopes(n: usize) -> Vec<String> {
+ vec![String::new(); n]
+ }
+
fn doc_ids(clusters: &[Cluster], rels: &[String]) -> Vec<String> {
let mut out = Vec::new();
for c in clusters {
@@ -309,11 +457,13 @@ mod tests {
#[test]
fn a_better_extractor_does_not_re_home_a_single_file() {
let rels: Vec<String> = (0..6).map(|i| format!("src/mod{i}/f.rs")).collect();
+ let scopes = root_scopes(6);
// Before: files 4 and 5 parsed to zero symbols.
let before = cluster(
(0..6).map(|i| code_sketch(i, i < 4)).collect::<Vec<_>>(),
&rels,
+ &scopes,
);
// The whole point: one dataset, not one per symbol-presence variant.
assert_eq!(before.len(), 1, "{before:#?}");
@@ -323,6 +473,7 @@ mod tests {
let after = cluster(
(0..6).map(|i| code_sketch(i, true)).collect::<Vec<_>>(),
&rels,
+ &scopes,
);
let (b, a) = (doc_ids(&before, &rels), doc_ids(&after, &rels));
@@ -340,16 +491,17 @@ mod tests {
"data/events.csv".into(),
"data/users.csv".into(),
];
+ let scopes = root_scopes(4);
let mixed = |symbols_in_b: bool| {
vec![
code_sketch(0, true),
code_sketch(1, symbols_in_b),
- data_sketch(2, Family::Csv, &["ts", "level", "msg"]),
- data_sketch(3, Family::Csv, &["id", "email", "name"]),
+ data_sketch_n(2, Family::Csv, &["ts", "level", "msg"], 40),
+ data_sketch_n(3, Family::Csv, &["id", "email", "name"], 40),
]
};
- let before = cluster(mixed(false), &rels);
- let after = cluster(mixed(true), &rels);
+ let before = cluster(mixed(false), &rels, &scopes);
+ let after = cluster(mixed(true), &rels, &scopes);
let slugs = |cs: &[Cluster]| {
let mut v: Vec<(usize, String)> = cs
.iter()
@@ -359,26 +511,28 @@ mod tests {
v
};
assert_eq!(slugs(&before), slugs(&after));
- // 1 code dataset + 2 unrelated CSV schemas.
+ // 1 document dataset + 2 unrelated CSV schemas.
assert_eq!(before.len(), 3, "{before:#?}");
}
- /// The fix must not blunt clustering: unrelated schemas stay apart, near
- /// -identical ones still merge, and formats never mix.
+ /// The fix must not blunt clustering: unrelated data schemas stay apart,
+ /// near-identical ones still merge, and data formats never mix.
#[test]
- fn genuinely_different_content_still_separates() {
+ fn genuinely_different_data_still_separates() {
let rels: Vec<String> = (0..5).map(|i| format!("d/f{i}")).collect();
+ let scopes = root_scopes(5);
let clusters = cluster(
vec![
- data_sketch(0, Family::Csv, &["ts", "level", "msg"]),
+ data_sketch_n(0, Family::Csv, &["ts", "level", "msg"], 40),
// one extra column out of four — still the same schema
- data_sketch(1, Family::Csv, &["ts", "level", "msg", "host"]),
- data_sketch(2, Family::Csv, &["id", "email", "name"]),
+ data_sketch_n(1, Family::Csv, &["ts", "level", "msg", "host"], 40),
+ data_sketch_n(2, Family::Csv, &["id", "email", "name"], 40),
// same names, different format family
- data_sketch(3, Family::Jsonl, &["ts", "level", "msg"]),
+ data_sketch_n(3, Family::Jsonl, &["ts", "level", "msg"], 40),
code_sketch(4, true),
],
&rels,
+ &scopes,
);
assert_eq!(clusters.len(), 4, "{clusters:#?}");
let members: Vec<Vec<usize>> = clusters.iter().map(|c| c.members.clone()).collect();
@@ -387,4 +541,170 @@ mod tests {
assert!(members.contains(&vec![3]), "{members:?}");
assert!(members.contains(&vec![4]), "{members:?}");
}
+
+ /// #196: one workspace, several format families — still ONE dataset.
+ /// Code and prose carry no schema, so format family is not a boundary.
+ #[test]
+ fn one_workspace_of_code_and_prose_is_one_dataset() {
+ let rels: Vec<String> = vec![
+ "src/a.rs".into(),
+ "src/b.rs".into(),
+ "README.md".into(),
+ "docs/guide.md".into(),
+ ];
+ let scopes = root_scopes(4);
+ let clusters = cluster(
+ vec![
+ code_sketch(0, true),
+ code_sketch(1, false),
+ prose_sketch(2),
+ prose_sketch(3),
+ ],
+ &rels,
+ &scopes,
+ );
+ assert_eq!(clusters.len(), 1, "{clusters:#?}");
+ assert_eq!(clusters[0].members.len(), 4);
+ assert!(clusters[0].is_docs);
+ assert_eq!(clusters[0].slug, "docs");
+ // the docs mapping holds both vocabularies
+ assert!(clusters[0].fields.contains_key("language"));
+ assert!(clusters[0].fields.contains_key("body"));
+ }
+
+ /// #173: two repositories in one tree — one document dataset per repo,
+ /// named after the repo, regardless of how many incidental shapes the
+ /// source files take.
+ #[test]
+ fn each_repository_is_its_own_document_dataset() {
+ let rels: Vec<String> = vec![
+ "valkey/src/a.c".into(),
+ "valkey/README.md".into(),
+ "memcached/src/b.c".into(),
+ "memcached/doc/notes.md".into(),
+ ];
+ let scopes: Vec<String> = vec![
+ "valkey".into(),
+ "valkey".into(),
+ "memcached".into(),
+ "memcached".into(),
+ ];
+ let clusters = cluster(
+ vec![
+ code_sketch(0, true),
+ prose_sketch(1),
+ code_sketch(2, true),
+ prose_sketch(3),
+ ],
+ &rels,
+ &scopes,
+ );
+ assert_eq!(clusters.len(), 2, "{clusters:#?}");
+ let mut slugs: Vec<&str> = clusters.iter().map(|c| c.slug.as_str()).collect();
+ slugs.sort();
+ assert_eq!(slugs, ["memcached-docs", "valkey-docs"]);
+ for c in &clusters {
+ assert!(c.is_docs);
+ assert_eq!(c.members.len(), 2);
+ }
+ }
+
+ /// #173's 382-cluster mechanism: single-record structured files with
+ /// one-off key sets are configuration, not datasets — they demote into
+ /// the scope's document dataset and their config keys never reach its
+ /// mapping. A recurring schema (many files) stays a data dataset, and a
+ /// multi-record structured file is a collection whatever its size.
+ #[test]
+ fn one_off_config_files_demote_to_documents() {
+ let mut sketches = Vec::new();
+ let mut rels = Vec::new();
+ // one code file so the scope has a docs dataset already
+ rels.push("src/main.c".to_string());
+ sketches.push(code_sketch(0, true));
+ // 30 per-command config JSONs, each its own key set, one record each
+ for i in 1..=30 {
+ rels.push(format!("commands/cmd{i}.json"));
+ let summary = format!("CMD{i}_summary");
+ let arity = format!("CMD{i}_arity");
+ sketches.push(data_sketch(
+ i,
+ Family::Json,
+ &[summary.as_str(), arity.as_str()],
+ ));
+ }
+ // a recurring schema: 12 files sharing the same keys, one record each
+ for i in 31..=42 {
+ rels.push(format!("profiles/p{i}.json"));
+ sketches.push(data_sketch(i, Family::Json, &["name", "email", "age"]));
+ }
+ // a multi-record JSON collection (array file) with a one-off schema
+ rels.push("data/events.json".to_string());
+ sketches.push(data_sketch_n(43, Family::Json, &["evt", "ts_ms"], 500));
+ let scopes = root_scopes(rels.len());
+ let clusters = cluster(sketches, &rels, &scopes);
+
+ let docs: Vec<&Cluster> = clusters.iter().filter(|c| c.is_docs).collect();
+ assert_eq!(docs.len(), 1, "{clusters:#?}");
+ let d = docs[0];
+ assert_eq!(d.members.len(), 31, "code + 30 demoted configs");
+ assert_eq!(d.demoted.len(), 30);
+ assert!(
+ !d.fields.keys().any(|k| k.starts_with("CMD")),
+ "config keys leaked into the document mapping: {:?}",
+ d.fields.keys()
+ );
+
+ let data: Vec<&Cluster> = clusters.iter().filter(|c| !c.is_docs).collect();
+ assert_eq!(data.len(), 2, "{clusters:#?}");
+ assert!(
+ data.iter()
+ .any(|c| c.members.len() == 12 && c.key_fields.contains("email")),
+ "the recurring profile schema must stay a data dataset"
+ );
+ assert!(
+ data.iter()
+ .any(|c| c.records == 500 && c.key_fields.contains("evt")),
+ "a multi-record collection must stay a data dataset"
+ );
+ }
+
+ /// Scope is a hard boundary for data schemas too: the same CSV schema in
+ /// two repositories is two datasets (per-repo corpora stay self-contained).
+ #[test]
+ fn the_same_schema_in_two_repositories_stays_apart() {
+ let rels: Vec<String> = vec!["a/x.csv".into(), "b/y.csv".into()];
+ let scopes: Vec<String> = vec!["a".into(), "b".into()];
+ let clusters = cluster(
+ vec![
+ data_sketch_n(0, Family::Csv, &["ts", "level", "msg"], 40),
+ data_sketch_n(1, Family::Csv, &["ts", "level", "msg"], 40),
+ ],
+ &rels,
+ &scopes,
+ );
+ assert_eq!(clusters.len(), 2, "{clusters:#?}");
+ }
+
+ /// Adding a file to the tree must never rename the docs dataset (its slug
+ /// is scope-derived, not content-derived) — the #178 property extended to
+ /// the new grouping.
+ #[test]
+ fn adding_a_file_never_renames_the_docs_dataset() {
+ let rels3: Vec<String> = vec!["src/a.rs".into(), "src/b.rs".into(), "lib/c.rs".into()];
+ let before = cluster(
+ vec![code_sketch(0, true), code_sketch(1, true)],
+ &rels3,
+ &root_scopes(3),
+ );
+ let after = cluster(
+ vec![
+ code_sketch(0, true),
+ code_sketch(1, true),
+ code_sketch(2, true),
+ ],
+ &rels3,
+ &root_scopes(3),
+ );
+ assert_eq!(before[0].slug, after[0].slug);
+ }
}
diff --git a/engine/crates/xerj-autoindex/src/extract/mod.rs b/engine/crates/xerj-autoindex/src/extract/mod.rs
index b18e959f6..adad65ff0 100644
--- a/engine/crates/xerj-autoindex/src/extract/mod.rs
+++ b/engine/crates/xerj-autoindex/src/extract/mod.rs
@@ -116,6 +116,31 @@ pub fn read_whole(path: &Path, gzip: bool, cap: u64) -> Result<Option<Vec<u8>>>
Ok(Some(buf))
}
+/// Render a file as a DOCUMENT (title = file stem, body = decoded text,
+/// section-split), regardless of its format family. This is how demoted
+/// one-off config files (`dataset` module docs, #173) are indexed: their
+/// key sets are configuration, not a schema, so their retrievable value is
+/// the text itself. Emits the same `title`/`body`/`section` vocabulary as
+/// every other document extractor (`FieldOrigin::Extractor`).
+pub fn extract_as_document(path: &Path, gzip: bool, sink: Sink) -> Result<ExtractStats> {
+ let mut stats = ExtractStats::default();
+ let Some(bytes) = read_whole(path, gzip, MAX_WHOLE_FILE)? else {
+ stats.junk += 1;
+ return Ok(stats);
+ };
+ let (text, _) = crate::sniff::decode_text(&bytes);
+ if text.trim().is_empty() {
+ stats.junk += 1;
+ return Ok(stats);
+ }
+ let title = path
+ .file_stem()
+ .map(|s| s.to_string_lossy().to_string())
+ .unwrap_or_else(|| "untitled".into());
+ emit_document(&title, &[], text.trim(), sink, &mut stats);
+ Ok(stats)
+}
+
/// Dispatch to the family extractor. `limit_bytes` bounds SAMPLING reads;
/// `None` = full stream.
pub fn extract(
diff --git a/engine/crates/xerj-autoindex/src/infer/mod.rs b/engine/crates/xerj-autoindex/src/infer/mod.rs
index 6491db9e1..430db4764 100644
--- a/engine/crates/xerj-autoindex/src/infer/mod.rs
+++ b/engine/crates/xerj-autoindex/src/infer/mod.rs
@@ -15,6 +15,13 @@ pub const DISTINCT_CAP: usize = 8192;
pub const RAW_CAP: usize = 8192;
pub const MAX_FIELDS_PER_DATASET: usize = 512;
+/// The extractor-owned fields that carry a document's text. Every document
+/// extractor puts its retrievable content in exactly one of these
+/// (`extract::emit_document`, `extract::code`, `extract::txt`), which is what
+/// lets a document dataset elect them ALL as `semantic_text` without ever
+/// embedding one record twice.
+pub const DOC_BODY_FIELDS: &[&str] = &["body", "text"];
+
/// 256-slot byte histogram with a `Default` impl (`[u32; 256]` has none).
#[derive(Debug, Clone)]
pub struct ByteHist(pub [u32; 256]);
@@ -387,6 +394,34 @@ pub fn infer_fields(
fields: &HashMap<String, FieldAcc>,
records: u64,
no_semantic: bool,
+) -> Vec<FieldSpec> {
+ infer_fields_with_policy(fields, records, no_semantic, false)
+}
+
+/// `infer_fields` with the semantic election policy explicit.
+///
+/// `semantic_all = false` (data datasets): elect at most ONE semantic body —
+/// the longest qualifying field. A data record usually carries several long
+/// text columns holding the SAME entity, and embedding every one multiplies
+/// cost (the built-in neural backend measures ~3 docs/s) for no retrieval
+/// gain.
+///
+/// `semantic_all = true` (document datasets): elect every qualifying BODY
+/// CARRIER (`DOC_BODY_FIELDS` — extractor-owned names, so this is a contract
+/// of this crate, not a heuristic). A document dataset merges several
+/// extractor vocabularies (`body` from code/prose/pdf/html, `text` from
+/// line-oriented chunks) and each record populates exactly one carrier, so
+/// electing all of them costs the same embedding work as electing one —
+/// while electing one silently cuts entire format families out of the
+/// vector arm (#173: 98.8% of the corpus had no `semantic_text` at all).
+/// Non-carrier text fields (`defs` symbol lists) stay lexical: they ride on
+/// the same record as a carrier, so electing them would embed every code
+/// record twice.
+pub fn infer_fields_with_policy(
+ fields: &HashMap<String, FieldAcc>,
+ records: u64,
+ no_semantic: bool,
+ semantic_all: bool,
) -> Vec<FieldSpec> {
let mut specs: Vec<FieldSpec> = Vec::new();
let mut date_ranges: Vec<(chrono::DateTime<chrono::Utc>, chrono::DateTime<chrono::Utc>)> =
@@ -548,7 +583,7 @@ pub fn infer_fields(
// source code. The length floor is kept but relaxed, because the
// language test now does the discriminating.
if !no_semantic {
- let best = specs
+ let qualifying: Vec<usize> = specs
.iter()
.enumerate()
.filter(|(_i, s)| {
@@ -559,8 +594,21 @@ pub fn infer_fields(
.map(|a| a.looks_natural_language())
.unwrap_or(false)
})
- .max_by(|a, b| a.1.avg_len.partial_cmp(&b.1.avg_len).unwrap());
- if let Some((i, _)) = best {
+ .map(|(i, _)| i)
+ .collect();
+ let elected: Vec<usize> = if semantic_all {
+ qualifying
+ .into_iter()
+ .filter(|&i| DOC_BODY_FIELDS.contains(&specs[i].name.as_str()))
+ .collect()
+ } else {
+ qualifying
+ .into_iter()
+ .max_by(|&a, &b| specs[a].avg_len.partial_cmp(&specs[b].avg_len).unwrap())
+ .into_iter()
+ .collect()
+ };
+ for i in elected {
let a = fields.get(&specs[i].name);
specs[i].es_type = "semantic_text".into();
specs[i].notes.push(format!(
@@ -720,6 +768,45 @@ mod tests {
}
}
+ /// #173: a document dataset merges extractor vocabularies whose records
+ /// each populate exactly one body carrier (`body` for code/prose, `text`
+ /// for line chunks). The docs policy must elect EVERY qualifying carrier
+ /// — electing only the longest silently cut whole format families out of
+ /// the vector arm — while a non-carrier text field (`defs`) stays
+ /// lexical, because it rides on the same record as a carrier and electing
+ /// it would embed every code record twice.
+ #[test]
+ fn a_document_dataset_elects_every_body_carrier_and_only_the_carriers() {
+ let prose = "The connection pool retries every failed handshake with backoff. \
+ Each worker owns one socket and never shares it across threads.";
+ let mut fields = HashMap::new();
+ for name in ["body", "text", "defs"] {
+ let mut acc = FieldAcc::default();
+ for _ in 0..20 {
+ acc.add(&Value::String(prose.to_string()));
+ }
+ fields.insert(name.to_string(), acc);
+ }
+
+ // Data policy (semantic_all = false): at most one semantic field.
+ let single = infer_fields_with_policy(&fields, 60, false, false);
+ assert_eq!(
+ single
+ .iter()
+ .filter(|s| s.es_type == "semantic_text")
+ .count(),
+ 1,
+ "{single:#?}"
+ );
+
+ // Docs policy: both carriers elected, the non-carrier stays text.
+ let docs = infer_fields_with_policy(&fields, 60, false, true);
+ let by_name = |n: &str| docs.iter().find(|s| s.name == n).unwrap();
+ assert_eq!(by_name("body").es_type, "semantic_text", "{docs:#?}");
+ assert_eq!(by_name("text").es_type, "semantic_text", "{docs:#?}");
+ assert_eq!(by_name("defs").es_type, "text", "{docs:#?}");
+ }
+
/// Pins the reason the entity tie is latent rather than live: the ≥90%
/// supermajority gate rejects any tied field, so the winner never reaches
/// the spec. If this test starts failing because the gate was relaxed, the
diff --git a/engine/crates/xerj-autoindex/src/lib.rs b/engine/crates/xerj-autoindex/src/lib.rs
index da8614e6f..351d381b5 100644
--- a/engine/crates/xerj-autoindex/src/lib.rs
+++ b/engine/crates/xerj-autoindex/src/lib.rs
@@ -33,7 +33,7 @@ use cli::{Cmd, IndexCfg, MapCfg, StatusCfg};
use detect::EdgeDetector as _;
use esclient::Es;
use sniff::{Family, Sniffed};
-use state::{FileAssignment, FileDone, JunkFile, Plan, PlanDataset};
+use state::{DuplicateFile, FileAssignment, FileDone, JunkFile, Plan, PlanDataset};
/// Entry point for the `xerj autoindex` subcommand (blocking; the server
/// binary calls this via spawn_blocking). Returns the process exit code.
@@ -358,7 +358,8 @@ mod clustering_key_tests {
records: s.sketches[0].records,
})
.collect();
- let clusters = dataset::cluster(sketches, &rels);
+ let scopes = vec![String::new(); rels.len()];
+ let clusters = dataset::cluster(sketches, &rels, &scopes);
assert_eq!(clusters.len(), 1, "{clusters:#?}");
}
@@ -380,6 +381,348 @@ mod clustering_key_tests {
}
}
+#[cfg(test)]
+mod phase_a_grouping_tests {
+ use super::*;
+
+ fn cfg_for(root: &Path) -> IndexCfg {
+ IndexCfg {
+ root: root.to_path_buf(),
+ url: "http://unused.invalid".into(),
+ api_key: None,
+ workers: 1,
+ pdf_workers: 1,
+ pdf_timeout_secs: 10,
+ bulk_mb: 1,
+ bulk_timeout_secs: 10,
+ prefix: "t".into(),
+ state_dir: None,
+ fresh: true,
+ follow_symlinks: false,
+ max_file_gb: 2,
+ sample: 500,
+ no_semantic: false,
+ brain: None,
+ no_graph: true,
+ dry_run: true,
+ json: false,
+ quiet: true,
+ }
+ }
+
+ fn plan_for(root: &Path) -> Plan {
+ let files = walk::walk(root, false).unwrap();
+ let keys: Vec<String> = files
+ .iter()
+ .map(|f| ids::file_key(&f.path, f.size).unwrap())
+ .collect();
+ let digests: Vec<String> = (0..files.len()).map(|i| format!("d{i}")).collect();
+ let (plan, _) = build_phase_a(root, &files, &keys, &digests, Vec::new(), &cfg_for(root));
+ plan
+ }
+
+ const CODE: &str = "// The event loop dispatches every ready connection to a worker.\n\
+ // Each worker drains its queue before polling for more sockets.\n\
+ static void dispatch_ready_connections(struct event_loop *loop) {\n\
+ for (int index = 0; index < loop->ready_count; index++) {\n\
+ worker_submit(loop->workers, loop->ready[index]);\n\
+ }\n\
+ }\n";
+
+ const PROSE: &str = "# Overview\n\nThis server accepts client connections and stores \
+ keys in memory. Every command travels through the same parser before the \
+ dispatcher routes it to the matching handler function.\n";
+
+ /// #173 end to end through the real planner: a tree holding two
+ /// repositories of source, prose and one-off config JSON yields one
+ /// document dataset per repository — with `body` elected `semantic_text`
+ /// — plus the genuine data dataset, instead of one dataset per incidental
+ /// config schema with no vector arm.
+ #[test]
+ fn a_two_repo_tree_plans_one_document_dataset_per_repo() {
+ let dir = tempfile::tempdir().unwrap();
+ let root = dir.path();
+ for repo in ["valkey", "memcached"] {
+ std::fs::create_dir_all(root.join(repo).join(".git")).unwrap();
+ std::fs::write(root.join(repo).join(".git").join("HEAD"), "ref: x").unwrap();
+ }
+ std::fs::create_dir_all(root.join("valkey/src")).unwrap();
+ std::fs::create_dir_all(root.join("valkey/commands")).unwrap();
+ std::fs::create_dir_all(root.join("memcached/data")).unwrap();
+ std::fs::write(root.join("valkey/src/server.c"), CODE).unwrap();
+ std::fs::write(root.join("valkey/README.md"), PROSE).unwrap();
+ // one-off configs: single-record JSON, each with its own key set
+ std::fs::write(
+ root.join("valkey/commands/get.json"),
+ r#"{"GET": {"summary": "Return the string value stored at the given key.", "arity": 2}}"#,
+ )
+ .unwrap();
+ std::fs::write(
+ root.join("valkey/commands/set.json"),
+ r#"{"SET": {"summary": "Store the given string value under the given key.", "arity": 3}}"#,
+ )
+ .unwrap();
+ // distinct bytes — byte-identical files alias into one canonical copy
+ std::fs::write(root.join("memcached/proto.c"), format!("{CODE}\n// v2\n")).unwrap();
+ // a genuine data file: recurring rows, real schema
+ std::fs::write(
+ root.join("memcached/data/events.csv"),
+ "id,email,level\n1,[email protected],info\n2,[email protected],warn\n3,[email protected],info\n",
+ )
+ .unwrap();
+
+ let plan = plan_for(root);
+ let mut slugs: Vec<&str> = plan.datasets.iter().map(|d| d.slug.as_str()).collect();
+ slugs.sort();
+ assert_eq!(
+ slugs,
+ ["memcached-data", "memcached-docs", "valkey-docs"],
+ "{:#?}",
+ plan.datasets
+ );
+
+ // every docs dataset carries the vector arm on body
+ for d in plan.datasets.iter().filter(|d| d.family == "docs") {
+ let body = d.specs.iter().find(|s| s.name == "body").unwrap();
+ assert_eq!(body.es_type, "semantic_text", "{}: {:#?}", d.slug, d.specs);
+ assert_eq!(d.semantic_field.as_deref(), Some("body"), "{}", d.slug);
+ }
+
+ // the one-off configs were demoted: document-rendered, and their
+ // config keys never reached the docs mapping
+ let by_rel: HashMap<&str, &FileAssignment> =
+ plan.files.values().map(|f| (f.rel.as_str(), f)).collect();
+ assert!(by_rel["valkey/commands/get.json"].as_document);
+ assert!(by_rel["valkey/commands/set.json"].as_document);
+ assert!(!by_rel["valkey/src/server.c"].as_document);
+ assert!(!by_rel["memcached/data/events.csv"].as_document);
+ let vdocs = plan
+ .datasets
+ .iter()
+ .find(|d| d.slug == "valkey-docs")
+ .unwrap();
+ assert!(
+ !vdocs.specs.iter().any(|s| s.name.contains("summary")),
+ "config keys leaked into the docs mapping: {:#?}",
+ vdocs.specs
+ );
+ assert_eq!(vdocs.file_count, 4, "code + prose + 2 demoted configs");
+
+ // the CSV kept its real schema
+ let data = plan
+ .datasets
+ .iter()
+ .find(|d| d.slug == "memcached-data")
+ .unwrap();
+ assert!(data.specs.iter().any(|s| s.name == "email"), "{data:#?}");
+ }
+
+ /// #196: the same tree WITHOUT nested `.git` markers (or with one at the
+ /// root — a workspace) is ONE scope: a single document corpus.
+ #[test]
+ fn a_single_workspace_plans_one_document_dataset() {
+ let dir = tempfile::tempdir().unwrap();
+ let root = dir.path();
+ std::fs::create_dir_all(root.join(".git")).unwrap();
+ std::fs::create_dir_all(root.join("src")).unwrap();
+ std::fs::create_dir_all(root.join("wasm/examples")).unwrap();
+ std::fs::write(root.join("src/a.c"), CODE).unwrap();
+ std::fs::write(root.join("src/b.c"), CODE).unwrap();
+ std::fs::write(root.join("README.md"), PROSE).unwrap();
+ std::fs::write(root.join("wasm/examples/demo.md"), PROSE).unwrap();
+
+ let plan = plan_for(root);
+ assert_eq!(
+ plan.datasets.len(),
+ 1,
+ "one workspace, one corpus: {:#?}",
+ plan.datasets
+ );
+ assert_eq!(plan.datasets[0].slug, "docs");
+ assert_eq!(plan.datasets[0].file_count, 4);
+ }
+}
+
+// ─── Phase A plan building (pure: no server contact) ─────────────────────
+
+/// Repository scope per file: the deepest ancestor directory (root-relative,
+/// "/"-separated) containing a `.git` entry, or "" when the file is under
+/// none. The walk never descends into `.git` itself, but the marker is still
+/// on disk. A `.git` at the autoindex root yields "" for every file — the
+/// whole tree is one scope — and so does a tree with no `.git` at all, which
+/// is exactly the #173/#196 property: one autoindex of one folder yields a
+/// corpus searchable as one corpus, split only at nested repository roots.
+fn compute_scopes(root: &Path, rels: &[String]) -> Vec<String> {
+ let mut cache: HashMap<String, bool> = HashMap::new();
+ rels.iter()
+ .map(|rel| {
+ let mut scope = String::new();
+ let mut prefix = String::new();
+ let mut segs = rel.split('/').peekable();
+ while let Some(seg) = segs.next() {
+ if segs.peek().is_none() {
+ break; // final segment is the file name
+ }
+ if !prefix.is_empty() {
+ prefix.push('/');
+ }
+ prefix.push_str(seg);
+ let repo = *cache
+ .entry(prefix.clone())
+ .or_insert_with(|| root.join(&prefix).join(".git").exists());
+ if repo {
+ scope = prefix.clone();
+ }
+ }
+ scope
+ })
+ .collect()
+}
+
+/// Sniff + sample every file, cluster into datasets, and assemble the plan.
+/// Pure planning: reads the tree, never the server — which is what makes the
+/// #173/#196 grouping behaviour testable end-to-end without a cluster.
+fn build_phase_a(
+ root: &Path,
+ files: &[walk::FileEntry],
+ keys: &[String],
+ digests: &[String],
+ duplicate_files: Vec<DuplicateFile>,
+ cfg: &IndexCfg,
+) -> (Plan, Vec<dataset::Cluster>) {
+ use rayon::prelude::*;
+ let scans: Vec<FileScan> = files
+ .par_iter()
+ .map(|f| scan_file(&f.path, f.size, cfg.sample, cfg.max_file_gb))
+ .collect();
+
+ let rels: Vec<String> = files.iter().map(|f| f.rel.clone()).collect();
+ let scopes = compute_scopes(root, &rels);
+ // (family, gzip) per file, from the scan's sniff — reused for assignments
+ // and demoted-file re-sampling instead of re-sniffing every file.
+ let file_meta: Vec<Option<(Family, bool)>> = scans
+ .iter()
+ .map(|sc| sc.sniffed.as_ref().map(|s| (s.family, s.gzip)))
+ .collect();
+ let mut sketches = Vec::new();
+ let mut junk_files = Vec::new();
+ for (i, sc) in scans.into_iter().enumerate() {
+ let family = sc
+ .sniffed
+ .as_ref()
+ .map(|s| s.family)
+ .unwrap_or(Family::Binary);
+ if let Some((status, reason)) = sc.junk {
+ junk_files.push(JunkFile {
+ file_key: keys[i].clone(),
+ rel: files[i].rel.clone(),
+ format: format_str(sc.sniffed.as_ref()),
+ status,
+ reason,
+ bytes: files[i].size,
+ });
+ continue;
+ }
+ for gs in sc.sketches {
+ sketches.push(dataset::Sketch {
+ file_idx: i,
+ group: gs.group,
+ family,
+ fields: gs.fields,
+ key_fields: gs.key_fields,
+ records: gs.records,
+ });
+ }
+ }
+ let mut clusters = dataset::cluster(sketches, &rels, &scopes);
+
+ // Demoted one-off config files (#173) were sampled as flattened records;
+ // phase B will index them as documents. Re-sample them through the same
+ // document renderer so the docs dataset's mapping and stats describe what
+ // actually gets indexed.
+ for c in clusters.iter_mut().filter(|c| !c.demoted.is_empty()) {
+ let demoted = c.demoted.clone();
+ let fields = &mut c.fields;
+ let mut sampled_total = 0u64;
+ for m in demoted {
+ let gzip = file_meta[m].map(|(_, g)| g).unwrap_or(false);
+ let mut sampled = 0u64;
+ let mut sink = |rec: extract::RawRecord| -> bool {
+ if (sampled as usize) < cfg.sample {
+ for (k, v) in &rec.fields {
+ fields.entry(k.clone()).or_default().add(v);
+ }
+ }
+ sampled += 1;
+ (sampled as usize) < cfg.sample
+ };
+ // An unreadable file junks at phase B like any other; the plan
+ // keeps its membership either way.
+ let _ = extract::extract_as_document(&files[m].path, gzip, &mut sink);
+ sampled_total += sampled;
+ }
+ c.records += sampled_total;
+ }
+
+ // per-file assignments
+ let mut file_assignments: HashMap<String, FileAssignment> = HashMap::new();
+ for (ci, c) in clusters.iter().enumerate() {
+ let demoted: std::collections::HashSet<usize> = c.demoted.iter().copied().collect();
+ for &m in &c.members {
+ let key = &keys[m];
+ let (family, gzip) = file_meta[m].unwrap_or((c.family, false));
+ let fa = file_assignments
+ .entry(key.clone())
+ .or_insert_with(|| FileAssignment {
+ rel: files[m].rel.clone(),
+ path_id: files[m].rel_id.clone(),
+ family: family.as_str().to_string(),
+ gzip,
+ content_digest: Some(digests[m].clone()),
+ assignments: Vec::new(),
+ as_document: false,
+ });
+ fa.as_document |= demoted.contains(&m);
+ fa.assignments
+ .push((c.group.clone(), clusters[ci].slug.clone()));
+ }
+ }
+
+ let mut datasets = Vec::new();
+ for c in &clusters {
+ let specs =
+ infer::infer_fields_with_policy(&c.fields, c.records, cfg.no_semantic, c.is_docs);
+ let time_field = infer::elect_time_field(&specs);
+ let semantic_field = specs
+ .iter()
+ .find(|s| s.es_type == "semantic_text")
+ .map(|s| s.name.clone());
+ datasets.push(PlanDataset {
+ slug: c.slug.clone(),
+ index: format!("{}-{}", cfg.prefix, c.slug),
+ family: if c.is_docs {
+ "docs".to_string()
+ } else {
+ c.family.as_str().to_string()
+ },
+ group: c.group.clone(),
+ specs,
+ time_field,
+ semantic_field,
+ sampled_records: c.records,
+ file_count: c.members.len(),
+ });
+ }
+ let plan = Plan {
+ datasets,
+ files: file_assignments,
+ junk_files,
+ duplicate_files,
+ alias_paths_indexed: true,
+ };
+ (plan, clusters)
+}
+
// ─── mapping builder ─────────────────────────────────────────────────────
pub const PROVENANCE_FIELDS: &[&str] = &[
@@ -564,7 +907,6 @@ fn run_index(cfg: IndexCfg) -> Result<i32> {
pub fn run_index_report(cfg: IndexCfg) -> Result<(i32, Option<Value>)> {
extract::pdf::configure_workers(cfg.pdf_workers);
extract::pdf::configure_timeout(cfg.pdf_timeout_secs);
- use rayon::prelude::*;
let t0 = Instant::now();
if !cfg.quiet {
eprintln!(
@@ -773,99 +1115,21 @@ pub fn run_index_report(cfg: IndexCfg) -> Result<(i32, Option<Value>)> {
if !cfg.quiet {
eprintln!("phase A: sniffing + sampling {} files…", files.len());
}
- let scans: Vec<FileScan> = files
- .par_iter()
- .map(|f| scan_file(&f.path, f.size, cfg.sample, cfg.max_file_gb))
- .collect();
-
- let rels: Vec<String> = files.iter().map(|f| f.rel.clone()).collect();
- let mut sketches = Vec::new();
- let mut junk_files = Vec::new();
- for (i, sc) in scans.into_iter().enumerate() {
- let family = sc
- .sniffed
- .as_ref()
- .map(|s| s.family)
- .unwrap_or(Family::Binary);
- if let Some((status, reason)) = sc.junk {
- junk_files.push(JunkFile {
- file_key: keys[i].clone(),
- rel: files[i].rel.clone(),
- format: format_str(sc.sniffed.as_ref()),
- status,
- reason,
- bytes: files[i].size,
- });
- continue;
- }
- for gs in sc.sketches {
- sketches.push(dataset::Sketch {
- file_idx: i,
- group: gs.group,
- family,
- fields: gs.fields,
- key_fields: gs.key_fields,
- records: gs.records,
- });
- }
- }
- let clusters = dataset::cluster(sketches, &rels);
+ let (plan, clusters) = build_phase_a(
+ &cfg.root,
+ &files,
+ &keys,
+ &digests,
+ duplicate_files.clone(),
+ &cfg,
+ );
if !cfg.quiet {
eprintln!(
"phase A: {} datasets inferred, {} junk/skipped files",
- clusters.len(),
- junk_files.len()
+ plan.datasets.len(),
+ plan.junk_files.len()
);
}
-
- // per-file assignments
- let mut file_assignments: HashMap<String, FileAssignment> = HashMap::new();
- for (ci, c) in clusters.iter().enumerate() {
- for &m in &c.members {
- let key = &keys[m];
- let sn = sniff::sniff(&files[m].path).ok();
- let fa = file_assignments
- .entry(key.clone())
- .or_insert_with(|| FileAssignment {
- rel: files[m].rel.clone(),
- path_id: files[m].rel_id.clone(),
- family: c.family.as_str().to_string(),
- gzip: sn.map(|s| s.gzip).unwrap_or(false),
- content_digest: Some(digests[m].clone()),
- assignments: Vec::new(),
- });
- fa.assignments
- .push((c.group.clone(), clusters[ci].slug.clone()));
- }
- }
-
- let mut datasets = Vec::new();
- for c in &clusters {
- let specs = infer::infer_fields(&c.fields, c.records, cfg.no_semantic);
- let time_field = infer::elect_time_field(&specs);
- let semantic_field = specs
- .iter()
- .find(|s| s.es_type == "semantic_text")
- .map(|s| s.name.clone());
- datasets.push(PlanDataset {
- slug: c.slug.clone(),
- index: format!("{}-{}", cfg.prefix, c.slug),
- family: c.family.as_str().to_string(),
- group: c.group.clone(),
- specs,
- time_field,
- semantic_field,
- sampled_records: c.records,
- file_count: c.members.len(),
- });
- }
- let plan = Plan {
- datasets,
- files: file_assignments,
- junk_files,
- duplicate_files,
- alias_paths_indexed: true,
- };
clusters_rt = Some(clusters);
plan
};
@@ -1401,7 +1665,14 @@ pub fn run_index_report(cfg: IndexCfg) -> Result<(i32, Option<Value>)> {
}
true
};
- let res = extract::extract(&f.path, &sn, None, &mut sink);
+ // Demoted one-off config files (#173) index as
+ // documents — their key sets are configuration, not a
+ // schema (see `dataset` module docs).
+ let res = if fa.as_document {
+ extract::extract_as_document(&f.path, sn.gzip, &mut sink)
+ } else {
+ extract::extract(&f.path, &sn, None, &mut sink)
+ };
match res {
Ok(stats) => {
file_junk += stats.junk;
@@ -2351,6 +2622,7 @@ mod duplicate_integration_tests {
gzip: false,
content_digest: None,
assignments: vec![(None, "text".to_string())],
+ as_document: false,
}
}
diff --git a/engine/crates/xerj-autoindex/src/state.rs b/engine/crates/xerj-autoindex/src/state.rs
index 5df8fdd60..50487c15f 100644
--- a/engine/crates/xerj-autoindex/src/state.rs
+++ b/engine/crates/xerj-autoindex/src/state.rs
@@ -193,6 +193,11 @@ pub struct FileAssignment {
pub content_digest: Option<String>,
/// group (None = whole file) → dataset slug
pub assignments: Vec<(Option<String>, String)>,
+ /// Demoted one-off config file (#173): index it through the document
+ /// renderer (`extract::extract_as_document`), not its family extractor.
+ /// Defaults false so frozen plans from earlier versions resume unchanged.
+ #[serde(default, skip_serializing_if = "std::ops::Not::not")]
+ pub as_document: bool,
}
#[derive(Debug, Clone, Serialize, Deserialize)]