Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

autoindex: bound phase-A memory and cap long dataset names (part of #1122) (#2417)

xerj-org6cdb32e29226
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 0a01cec3b..044d32fb8 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -79,6 +79,18 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
### Fixed
+- `xerj autoindex` no longer runs out of memory while planning a large folder.
+ It used to keep the sampled values of every file in memory at once before
+ grouping them into datasets, and kept them all until the end of the run;
+ on a 5 GB folder of about 146,000 files that was several GB on the client
+ before anything was indexed. Files are now grouped as they are read, and only
+ what the end-of-run key correlation needs is kept. The datasets and index
+ names it plans are unchanged (part of #1122).
+- `xerj autoindex` no longer fails a run because a dataset name built from a
+ long path (for example a Go module path) is over the 255-character index name
+ limit. Names longer than 128 characters are shortened and end in a short
+ hash, so they stay the same on every run and never clash (part of #1122).
+
- Server startup now reports the resolved data directory, its configuration
source and whether this launch created it. New-directory guidance points
operators back to their previous `--data-dir` or `--config` when they expected
diff --git a/engine/crates/xerj-autoindex/src/correlate.rs b/engine/crates/xerj-autoindex/src/correlate.rs
index a3838abac..4ea8ee5e5 100644
--- a/engine/crates/xerj-autoindex/src/correlate.rs
+++ b/engine/crates/xerj-autoindex/src/correlate.rs
@@ -7,7 +7,7 @@
use crate::esclient::Es;
use anyhow::Result;
use serde_json::{json, Value};
-use std::collections::HashSet;
+use std::collections::{HashMap, HashSet};
pub struct Candidate {
pub slug: String,
@@ -70,6 +70,90 @@ impl KeyCorr {
}
}
+/// What end-of-run key correlation needs from one sampled field, kept in
+/// place of the field's full phase-A accumulator.
+#[derive(Debug, Clone, PartialEq)]
+pub struct KeyFieldStats {
+ pub n: u64,
+ pub distinct: usize,
+ pub overflow: bool,
+ pub int_range: Option<(i64, i64)>,
+ pub values: HashSet<String>,
+}
+
+/// dataset slug → field name → stats.
+pub type KeyStats = HashMap<String, HashMap<String, KeyFieldStats>>;
+
+/// Fewest distinct sampled values [`is_candidate`] accepts for any type.
+const MIN_CANDIDATE_DISTINCT: usize = 10;
+
+/// Keep only the correlation inputs of every cluster's fields, so the phase-A
+/// accumulators (byte histograms, length/token samples, hashed distinct sets)
+/// can be dropped once the plan is built instead of living to the end of the
+/// run. Fields with too few distinct values to ever be a candidate are left
+/// out; the type-dependent checks run at the end against the final specs.
+pub fn key_stats(clusters: &[crate::dataset::Cluster]) -> KeyStats {
+ let mut out = KeyStats::new();
+ for c in clusters {
+ let fields: HashMap<String, KeyFieldStats> = c
+ .fields
+ .iter()
+ .filter(|(_, acc)| acc.distinct.len() >= MIN_CANDIDATE_DISTINCT)
+ .map(|(name, acc)| {
+ (
+ name.clone(),
+ KeyFieldStats {
+ n: acc.n,
+ distinct: acc.distinct.len(),
+ overflow: acc.distinct_overflow,
+ int_range: (acc.long_ok > 0).then_some((acc.int_min, acc.int_max)),
+ values: acc.raw_values.clone(),
+ },
+ )
+ })
+ .collect();
+ if !fields.is_empty() {
+ out.insert(c.slug.clone(), fields);
+ }
+ }
+ out
+}
+
+/// Key-overlap candidates among `datasets`' fields, from stats captured by
+/// [`key_stats`]. Looked up by slug, so datasets refused after planning
+/// simply drop out.
+pub fn candidates(stats: &KeyStats, datasets: &[crate::state::PlanDataset]) -> Vec<Candidate> {
+ let mut cands = Vec::new();
+ for d in datasets {
+ let Some(fields) = stats.get(&d.slug) else {
+ continue;
+ };
+ for spec in &d.specs {
+ let Some(st) = fields.get(&spec.name) else {
+ continue;
+ };
+ if is_candidate(
+ &spec.es_type,
+ spec.semantic.as_deref(),
+ st.distinct,
+ st.n,
+ st.overflow,
+ st.int_range,
+ ) {
+ cands.push(Candidate {
+ slug: d.slug.clone(),
+ index: d.index.clone(),
+ field: spec.name.clone(),
+ kind: spec.es_type.clone(),
+ values: st.values.clone(),
+ sampled_n: st.n,
+ });
+ }
+ }
+ }
+ cands
+}
+
/// Key-like candidate filter. Deviations from the design draft (recorded):
/// - keyword fields qualify at cardinality ≥ 20 regardless of distinct/n
/// ratio (join keys like user ids have LOW sample ratios in big datasets);
@@ -351,6 +435,110 @@ pub fn time_alignment(series: &[TimeSeries]) -> Vec<Value> {
mod tests {
use super::*;
+ fn spec(name: &str, es_type: &str) -> crate::infer::FieldSpec {
+ crate::infer::FieldSpec {
+ name: name.into(),
+ es_type: es_type.into(),
+ date_enc: None,
+ semantic: None,
+ cardinality_est: 0,
+ cardinality_overflow: false,
+ null_ratio: 0.0,
+ avg_len: 0.0,
+ coverage: 1.0,
+ examples: vec![],
+ notes: vec![],
+ date_min: None,
+ date_max: None,
+ date_evidence: vec![],
+ num_min: None,
+ num_max: None,
+ }
+ }
+
+ fn plan_dataset(slug: &str, specs: Vec<crate::infer::FieldSpec>) -> crate::state::PlanDataset {
+ crate::state::PlanDataset {
+ slug: slug.into(),
+ index: format!("ax-{slug}"),
+ family: "jsonl".into(),
+ group: None,
+ specs,
+ time_field: None,
+ semantic_field: None,
+ text_analyzer: None,
+ scope: None,
+ sampled_records: 1,
+ file_count: 1,
+ }
+ }
+
+ fn cluster(
+ slug: &str,
+ field: &str,
+ values: impl Iterator<Item = String>,
+ ) -> crate::dataset::Cluster {
+ let mut acc = crate::infer::FieldAcc::default();
+ for v in values {
+ acc.add(&serde_json::Value::String(v));
+ }
+ let mut few = crate::infer::FieldAcc::default();
+ few.add(&serde_json::Value::String("only".into()));
+ crate::dataset::Cluster {
+ family: crate::sniff::Family::Jsonl,
+ group: None,
+ members: vec![0],
+ fields: [(field.to_string(), acc), ("level".to_string(), few)]
+ .into_iter()
+ .collect(),
+ key_fields: Default::default(),
+ records: 100,
+ population: 100,
+ slug: slug.into(),
+ scope: String::new(),
+ is_docs: false,
+ demoted: Vec::new(),
+ }
+ }
+
+ /// The compact stats give the same candidates the full accumulators did,
+ /// drop fields that can never be keys, and follow datasets by slug — so
+ /// a dataset refused after planning does not shift its neighbours' fields.
+ #[test]
+ fn key_stats_match_the_accumulators_and_follow_datasets_by_slug() {
+ let ids = || (0..50).map(|i| format!("user-{i}"));
+ let clusters = vec![
+ cluster("refused", "user_id", ids()),
+ cluster("orders", "user_id", ids()),
+ cluster("users", "id", ids()),
+ ];
+ let stats = key_stats(&clusters);
+ assert!(
+ !stats["orders"].contains_key("level"),
+ "1 distinct value is never a key"
+ );
+ let acc = &clusters[1].fields["user_id"];
+ let st = &stats["orders"]["user_id"];
+ assert_eq!(st.n, acc.n);
+ assert_eq!(st.distinct, acc.distinct.len());
+ assert_eq!(st.values, acc.raw_values);
+
+ // "refused" is no longer in the plan.
+ let datasets = vec![
+ plan_dataset(
+ "orders",
+ vec![spec("user_id", "keyword"), spec("level", "keyword")],
+ ),
+ plan_dataset("users", vec![spec("id", "keyword")]),
+ ];
+ let cands = candidates(&stats, &datasets);
+ let got: Vec<(&str, &str)> = cands
+ .iter()
+ .map(|c| (c.slug.as_str(), c.field.as_str()))
+ .collect();
+ assert_eq!(got, vec![("orders", "user_id"), ("users", "id")]);
+ assert_eq!(key_overlaps(&cands).len(), 1);
+ }
+
fn sample_corr() -> KeyCorr {
KeyCorr {
a_slug: "reports".into(),
diff --git a/engine/crates/xerj-autoindex/src/dataset.rs b/engine/crates/xerj-autoindex/src/dataset.rs
index 11c80d4a6..40eac1053 100644
--- a/engine/crates/xerj-autoindex/src/dataset.rs
+++ b/engine/crates/xerj-autoindex/src/dataset.rs
@@ -135,31 +135,61 @@ pub(crate) fn demotable_family(f: Family) -> bool {
/// Cluster sketches into datasets. `rels` and `scopes` are indexed by
/// `Sketch::file_idx`: `rels` are root-relative paths (slug naming), `scopes`
/// the repository scope of each file ("" = root) — see `module` docs.
+///
+/// Equivalent to feeding every sketch, in order, to a [`Clusterer`] and
+/// calling [`Clusterer::finish`].
pub fn cluster(sketches: Vec<Sketch>, rels: &[String], scopes: &[String]) -> Vec<Cluster> {
- // scope → docs cluster (BTreeMap: deterministic scope order).
- let mut docs: BTreeMap<String, Cluster> = BTreeMap::new();
- let mut data: Vec<Cluster> = Vec::new();
- fn docs_cluster<'a>(docs: &'a mut BTreeMap<String, Cluster>, scope: &str) -> &'a mut Cluster {
- docs.entry(scope.to_string()).or_insert_with(|| Cluster {
- family: Family::Code,
- group: None,
- members: Vec::new(),
- fields: HashMap::new(),
- key_fields: HashSet::new(),
- records: 0,
- population: 0,
- slug: String::new(),
- scope: scope.to_string(),
- is_docs: true,
- demoted: Vec::new(),
- })
- }
-
+ let mut clusterer = Clusterer::default();
for sk in sketches {
let scope = scopes[sk.file_idx].as_str();
+ clusterer.push(sk, scope);
+ }
+ clusterer.finish(rels)
+}
+
+fn docs_cluster<'a>(docs: &'a mut BTreeMap<String, Cluster>, scope: &str) -> &'a mut Cluster {
+ docs.entry(scope.to_string()).or_insert_with(|| Cluster {
+ family: Family::Code,
+ group: None,
+ members: Vec::new(),
+ fields: HashMap::new(),
+ key_fields: HashSet::new(),
+ records: 0,
+ population: 0,
+ slug: String::new(),
+ scope: scope.to_string(),
+ is_docs: true,
+ demoted: Vec::new(),
+ })
+}
+
+/// Incremental form of [`cluster`]: sketches are folded into their dataset as
+/// they arrive, so a caller never has to hold every file's sketch at once.
+///
+/// A sketch carries one [`FieldAcc`] per sampled field (a 1 KiB byte histogram,
+/// up to 512 length samples and up to [`crate::infer::RAW_CAP`] raw values
+/// each). Phase A used to collect every file's sketches before clustering:
+/// tens of KiB per file, and 4.4 GB of client RSS on a 146k-file folder before
+/// a single document was sent. Folding them as they are produced bounds that
+/// to the clusters themselves. Sketches must be pushed in file order for the
+/// result to match [`cluster`] exactly (clustering is greedy and
+/// order-dependent).
+#[derive(Default)]
+pub struct Clusterer {
+ // scope → docs cluster (BTreeMap: deterministic scope order).
+ docs: BTreeMap<String, Cluster>,
+ data: Vec<Cluster>,
+}
+
+impl Clusterer {
+ /// Fold one sketch into its dataset. `scope` is the repository scope of
+ /// `sk.file_idx`.
+ pub fn push(&mut self, sk: Sketch, scope: &str) {
+ let docs = &mut self.docs;
+ let data = &mut self.data;
// Documents: no data-derived names, no sub-file group.
if sk.key_fields.is_empty() && sk.group.is_none() {
- let c = docs_cluster(&mut docs, scope);
+ let c = docs_cluster(docs, scope);
c.members.push(sk.file_idx);
c.records += sk.records;
c.population += sk.population;
@@ -171,7 +201,7 @@ pub fn cluster(sketches: Vec<Sketch>, rels: &[String], scopes: &[String]) -> Vec
}
}
}
- continue;
+ return;
}
// Data: schema clustering within (scope, family, group).
let names: HashSet<&str> = sk.key_fields.iter().map(|s| s.as_str()).collect();
@@ -219,92 +249,96 @@ pub fn cluster(sketches: Vec<Sketch>, rels: &[String], scopes: &[String]) -> Vec
}
}
- // Demote small config clusters — one record each, or a short multi-doc
- // YAML/JSON/XML file — to their scope's document dataset. Their sampled
- // fields are dropped — the caller re-samples the demoted files through
- // the document renderer, which is also what indexes them.
- //
- // A file that also produced a grouped dataset (a second top-level JSON
- // array) must not be folded: folding marks the whole file `as_document`
- // and phase B then drops the grouped rows.
- let grouped_files: HashSet<usize> = data
- .iter()
- .filter(|c| c.group.is_some())
- .flat_map(|c| c.members.iter().copied())
- .collect();
- let mut kept: Vec<Cluster> = Vec::with_capacity(data.len());
- let mut capped: BTreeMap<String, Vec<Cluster>> = BTreeMap::new();
- for c in data {
- let shares_grouped_sibling = c.members.iter().any(|m| grouped_files.contains(m));
- if small_structured_config(&c) && !shares_grouped_sibling {
- let d = docs_cluster(&mut docs, &c.scope);
- d.members.extend(c.members.iter().copied());
- d.demoted.extend(c.members);
- continue;
- }
- // Candidates for the per-scope cap: structured formats, no sub-file
- // group, and not a large collection. Everything else stays.
- if demotable_family(c.family)
- && c.group.is_none()
- && c.population < DATASET_CAP_PROTECT_RECORDS
- && !shares_grouped_sibling
- {
- capped.entry(c.scope.clone()).or_default().push(c);
- continue;
+ /// Demote, cap and name the clusters. `rels` is indexed by file index.
+ pub fn finish(self, rels: &[String]) -> Vec<Cluster> {
+ let Clusterer { mut docs, data } = self;
+ // Demote small config clusters — one record each, or a short multi-doc
+ // YAML/JSON/XML file — to their scope's document dataset. Their sampled
+ // fields are dropped — the caller re-samples the demoted files through
+ // the document renderer, which is also what indexes them.
+ //
+ // A file that also produced a grouped dataset (a second top-level JSON
+ // array) must not be folded: folding marks the whole file `as_document`
+ // and phase B then drops the grouped rows.
+ let grouped_files: HashSet<usize> = data
+ .iter()
+ .filter(|c| c.group.is_some())
+ .flat_map(|c| c.members.iter().copied())
+ .collect();
+ let mut kept: Vec<Cluster> = Vec::with_capacity(data.len());
+ let mut capped: BTreeMap<String, Vec<Cluster>> = BTreeMap::new();
+ for c in data {
+ let shares_grouped_sibling = c.members.iter().any(|m| grouped_files.contains(m));
+ if small_structured_config(&c) && !shares_grouped_sibling {
+ let d = docs_cluster(&mut docs, &c.scope);
+ d.members.extend(c.members.iter().copied());
+ d.demoted.extend(c.members);
+ continue;
+ }
+ // Candidates for the per-scope cap: structured formats, no sub-file
+ // group, and not a large collection. Everything else stays.
+ if demotable_family(c.family)
+ && c.group.is_none()
+ && c.population < DATASET_CAP_PROTECT_RECORDS
+ && !shares_grouped_sibling
+ {
+ capped.entry(c.scope.clone()).or_default().push(c);
+ continue;
+ }
+ kept.push(c);
}
- kept.push(c);
- }
- for (scope, mut group) in capped {
- // Largest collections stay. Ties break toward the earlier file so
- // the fold does not depend on hash order. Size is the real row
- // count, so a file sampled at 500 does not tie with a 100-row file
- // that happened to fit in the sample.
- group.sort_by(|a, b| {
- b.population
- .cmp(&a.population)
- .then(b.members.len().cmp(&a.members.len()))
- .then_with(|| {
- let am = a.members.iter().copied().min().unwrap_or(usize::MAX);
- let bm = b.members.iter().copied().min().unwrap_or(usize::MAX);
- am.cmp(&bm)
- })
- });
- let keep_n = MAX_STRUCTURED_DATASETS_PER_SCOPE.min(group.len());
- for c in group.drain(keep_n..) {
- let d = docs_cluster(&mut docs, &scope);
- d.members.extend(c.members.iter().copied());
- d.demoted.extend(c.members);
+ for (scope, mut group) in capped {
+ // Largest collections stay. Ties break toward the earlier file so
+ // the fold does not depend on hash order. Size is the real row
+ // count, so a file sampled at 500 does not tie with a 100-row file
+ // that happened to fit in the sample.
+ group.sort_by(|a, b| {
+ b.population
+ .cmp(&a.population)
+ .then(b.members.len().cmp(&a.members.len()))
+ .then_with(|| {
+ let am = a.members.iter().copied().min().unwrap_or(usize::MAX);
+ let bm = b.members.iter().copied().min().unwrap_or(usize::MAX);
+ am.cmp(&bm)
+ })
+ });
+ let keep_n = MAX_STRUCTURED_DATASETS_PER_SCOPE.min(group.len());
+ for c in group.drain(keep_n..) {
+ let d = docs_cluster(&mut docs, &scope);
+ d.members.extend(c.members.iter().copied());
+ d.demoted.extend(c.members);
+ }
+ kept.extend(group);
}
- kept.extend(group);
- }
- // Docs slugs are scope-derived — independent of which other files exist,
- // so adding a file can never rename (and re-home, #178) a docs dataset.
- let mut clusters: Vec<Cluster> = Vec::with_capacity(docs.len() + kept.len());
- let mut used: HashSet<String> = HashSet::new();
- for (scope, mut c) in docs {
- let base = if scope.is_empty() {
- "docs".to_string()
- } else {
- let s = sanitize_slug(&scope);
- if s.is_empty() {
+ // Docs slugs are scope-derived — independent of which other files exist,
+ // so adding a file can never rename (and re-home, #178) a docs dataset.
+ let mut clusters: Vec<Cluster> = Vec::with_capacity(docs.len() + kept.len());
+ let mut used: HashSet<String> = HashSet::new();
+ for (scope, mut c) in docs {
+ let base = if scope.is_empty() {
"docs".to_string()
} else {
- format!("{s}-docs")
+ let s = sanitize_slug(&scope);
+ if s.is_empty() {
+ "docs".to_string()
+ } else {
+ format!("{s}-docs")
+ }
+ };
+ let mut slug = base.clone();
+ let mut k = 2;
+ while !used.insert(slug.clone()) {
+ slug = format!("{base}-{k}");
+ k += 1;
}
- };
- let mut slug = base.clone();
- let mut k = 2;
- while !used.insert(slug.clone()) {
- slug = format!("{base}-{k}");
- k += 1;
+ c.slug = slug;
+ clusters.push(c);
}
- c.slug = slug;
- clusters.push(c);
+ assign_slugs(&mut kept, rels, &mut used);
+ clusters.extend(kept);
+ clusters
}
- assign_slugs(&mut kept, rels, &mut used);
- clusters.extend(kept);
- clusters
}
pub fn sanitize_slug(s: &str) -> String {
@@ -543,10 +577,114 @@ pub fn drop_repeated_prefix(prefix: &str, clusters: &mut [Cluster]) {
}
}
+/// Longest dataset slug autoindex derives. Slugs come from file paths and
+/// group names with no natural bound — a Go module path produced a 329-char
+/// index name that the server refused (index names are capped at 255 bytes),
+/// failing the run (part of #1122). 128 leaves room for the `--prefix` and for
+/// the suffixes rebuilds add to it (`-reindex`, `-generation-v1`).
+pub const MAX_SLUG_LEN: usize = 128;
+
+/// Shorten slugs longer than [`MAX_SLUG_LEN`]: keep the leading part and
+/// append `-` plus 8 hex digits of a hash of the full slug, so the name stays
+/// readable, is the same on every run, and two long slugs sharing a long
+/// beginning still get different names. Slugs within the limit are untouched.
+pub fn cap_slug_len(clusters: &mut [Cluster]) {
+ if clusters.iter().all(|c| c.slug.len() <= MAX_SLUG_LEN) {
+ return;
+ }
+ let mut used: HashSet<String> = clusters
+ .iter()
+ .filter(|c| c.slug.len() <= MAX_SLUG_LEN)
+ .map(|c| c.slug.clone())
+ .collect();
+ let mut order: Vec<usize> = (0..clusters.len())
+ .filter(|&i| clusters[i].slug.len() > MAX_SLUG_LEN)
+ .collect();
+ order.sort_by_key(|&i| clusters[i].slug.clone());
+ for i in order {
+ let full = clusters[i].slug.clone();
+ let mut salt = 0u64;
+ loop {
+ let slug = capped_slug(&full, salt);
+ if used.insert(slug.clone()) {
+ clusters[i].slug = slug;
+ break;
+ }
+ salt += 1;
+ }
+ }
+}
+
+fn capped_slug(full: &str, salt: u64) -> String {
+ let hash = xxhash_rust::xxh3::xxh3_64_with_seed(full.as_bytes(), salt) as u32;
+ let suffix = format!("-{hash:08x}");
+ // Slugs are ASCII (`sanitize_slug`); the boundary check keeps this safe
+ // for any other caller.
+ let mut cut = MAX_SLUG_LEN - suffix.len();
+ while !full.is_char_boundary(cut) {
+ cut -= 1;
+ }
+ let head = full[..cut].trim_end_matches('-');
+ format!("{head}{suffix}")
+}
+
#[cfg(test)]
mod tests {
use super::*;
+ fn named(slug: &str) -> Cluster {
+ Cluster {
+ family: Family::Json,
+ group: None,
+ members: Vec::new(),
+ fields: HashMap::new(),
+ key_fields: HashSet::new(),
+ records: 0,
+ population: 0,
+ slug: slug.to_string(),
+ scope: String::new(),
+ is_docs: false,
+ demoted: Vec::new(),
+ }
+ }
+
+ #[test]
+ fn long_slugs_are_capped_with_a_stable_hash_suffix() {
+ // The real 329-char index name from a Go module tree (part of #1122).
+ let long = format!(
+ "gomod-github-com-paths-v1-example-a-bit-of-everything-floatvalue-{}",
+ "nested-message-repeated-string-value-".repeat(7)
+ );
+ let sibling = format!("{long}other");
+ let mut clusters = vec![named("short"), named(&long), named(&sibling)];
+ cap_slug_len(&mut clusters);
+ assert_eq!(clusters[0].slug, "short");
+ for c in &clusters[1..] {
+ assert!(c.slug.len() <= MAX_SLUG_LEN, "{} chars", c.slug.len());
+ assert!(c.slug.starts_with("gomod-github-com-paths-v1-example"));
+ assert_eq!(c.slug, sanitize_slug(&c.slug), "still a valid slug");
+ assert!(xerj_common::IndexName::validate(&format!("b5-{}", c.slug)).is_ok());
+ }
+ assert_ne!(clusters[1].slug, clusters[2].slug);
+ // Deterministic: the same input names the same index on every run.
+ let mut again = vec![named("short"), named(&long), named(&sibling)];
+ cap_slug_len(&mut again);
+ let a: Vec<_> = clusters.iter().map(|c| c.slug.clone()).collect();
+ let b: Vec<_> = again.iter().map(|c| c.slug.clone()).collect();
+ assert_eq!(a, b);
+ }
+
+ #[test]
+ fn capped_slug_never_collides_with_an_existing_one() {
+ let long = "a".repeat(MAX_SLUG_LEN + 50);
+ let taken = capped_slug(&long, 0);
+ let mut clusters = vec![named(&taken), named(&long)];
+ cap_slug_len(&mut clusters);
+ assert_eq!(clusters[0].slug, taken);
+ assert_ne!(clusters[1].slug, taken);
+ assert!(clusters[1].slug.len() <= MAX_SLUG_LEN);
+ }
+
#[test]
fn a_digit_leading_prefix_is_rejected_before_any_walk() {
let err = index_prefix_error("2042").expect("2042-docs is not an index name");
diff --git a/engine/crates/xerj-autoindex/src/lib.rs b/engine/crates/xerj-autoindex/src/lib.rs
index 367943c25..4e9be1d43 100644
--- a/engine/crates/xerj-autoindex/src/lib.rs
+++ b/engine/crates/xerj-autoindex/src/lib.rs
@@ -5247,18 +5247,17 @@ struct PhaseA {
/// `--progress json` stays a single parseable stream (#241).
fn report_pdf_spool_fallbacks(
files: &[walk::FileEntry],
- scans: &[FileScan],
+ fallbacks: &[(usize, Vec<extract::pdf::SpoolFallback>)],
ctx: &PhaseAContext<'_>,
) {
let pdf_spool_budget = ctx.budget;
let pr = ctx.progress;
- let reasons: Vec<(&str, &extract::pdf::SpoolFallback)> = scans
+ let reasons: Vec<(&str, &extract::pdf::SpoolFallback)> = fallbacks
.iter()
- .enumerate()
- .flat_map(|(index, scan)| {
- scan.pdf_spool_fallbacks
+ .flat_map(|(index, per_file)| {
+ per_file
.iter()
- .map(move |fallback| (files[index].rel.as_str(), fallback))
+ .map(move |fallback| (files[*index].rel.as_str(), fallback))
})
.collect();
for (path, fallback) in &reasons {
@@ -5296,6 +5295,13 @@ fn report_pdf_spool_fallbacks(
}
}
+/// Files phase A scans before folding their sketches into clusters. Large
+/// enough that the scan pool stays busy across a chunk, small enough that the
+/// sketches held at once stay bounded whatever the folder's file count.
+// Tests use a tiny chunk so every multi-file phase-A test crosses chunk
+// boundaries.
+const PHASE_A_CHUNK_FILES: usize = if cfg!(test) { 3 } else { 2048 };
+
/// Sniff + sample every file, cluster into datasets, and assemble the plan.
/// Pure planning: reads the tree, never the server — which is what makes the
/// #173/#196 grouping behaviour testable end-to-end without a cluster.
@@ -5320,77 +5326,95 @@ fn build_phase_a(
// Patterns were validated (loudly) at run start; a failure here would be
// a programming error, not user input.
let stub_matcher = StubMatcher::compile(&cfg.stub_globs).expect("validated at startup");
- let scans: Vec<FileScan> = crate::pool::install(|| {
- files
- .par_iter()
- .zip(digests.par_iter())
- .map(|(f, digest)| {
- let _in_flight = pr.file(&f.rel, f.size);
- scan_file(
- &f.path,
- f.size,
- digest,
- ctx,
- cfg.sample,
- cfg.max_file_bytes,
- stub_matcher.matches(&f.rel),
- )
- })
- .collect()
- });
-
- report_pdf_spool_fallbacks(files, &scans, ctx);
- note_worker_address_space_cap(pr);
-
let rels: Vec<String> = files.iter().map(|f| f.rel.clone()).collect();
let scopes = compute_scopes(root, &rels);
// (family, gzip) per file, from the scan's sniff — reused for assignments
// and demoted-file re-sampling instead of re-sniffing every file.
- let file_meta: Vec<Option<(Family, bool)>> = scans
- .iter()
- .map(|sc| sc.sniffed.as_ref().map(|s| (s.family, s.gzip)))
- .collect();
- let mut sketches = Vec::new();
+ let mut file_meta: Vec<Option<(Family, bool)>> = Vec::with_capacity(files.len());
let mut junk_files = Vec::new();
let mut pdf_spools: Vec<Option<extract::pdf::ExtractionSpool>> =
(0..files.len()).map(|_| None).collect();
- for (i, mut sc) in scans.into_iter().enumerate() {
- let family = sc
- .sniffed
- .as_ref()
- .map(|s| s.family)
- .unwrap_or(Family::Binary);
- // A file that phase A junks is never indexed, so its artifact is
- // refunded here rather than held to the phase A→B boundary.
- pdf_spools[i] =
- take_pdf_spool_if_indexable(&mut sc.pdf_spool, sc.junk.is_some(), pdf_spool_budget);
- if let Some((status, reason)) = sc.junk {
- junk_files.push(JunkFile {
- file_key: keys[i].clone(),
- rel: files[i].rel.clone(),
- format: format_str(sc.sniffed.as_ref()),
- status,
- reason,
- bytes: files[i].size,
- });
- continue;
- }
- for gs in sc.sketches {
- sketches.push(dataset::Sketch {
- file_idx: i,
- group: gs.group,
- family,
- fields: gs.fields,
- key_fields: gs.key_fields,
- records: gs.records,
- population: gs.population,
- });
+ let mut pdf_fallbacks: Vec<(usize, Vec<extract::pdf::SpoolFallback>)> = Vec::new();
+ // Scan in file-ordered chunks and fold each chunk's sketches into the
+ // clusters before the next chunk is scanned. Collecting every file's scan
+ // first held one set of field accumulators per file for the whole folder
+ // at once: 4.4 GB of client RSS on a 146k-file tree before anything was
+ // indexed (part of #1122). Folding in file order keeps the clustering
+ // identical to the collect-all version (see `dataset::Clusterer`).
+ let mut clusterer = dataset::Clusterer::default();
+ let mut start = 0usize;
+ while start < files.len() {
+ let end = (start + PHASE_A_CHUNK_FILES).min(files.len());
+ let scans: Vec<FileScan> = crate::pool::install(|| {
+ files[start..end]
+ .par_iter()
+ .zip(digests[start..end].par_iter())
+ .map(|(f, digest)| {
+ let _in_flight = pr.file(&f.rel, f.size);
+ scan_file(
+ &f.path,
+ f.size,
+ digest,
+ ctx,
+ cfg.sample,
+ cfg.max_file_bytes,
+ stub_matcher.matches(&f.rel),
+ )
+ })
+ .collect()
+ });
+ for (offset, mut sc) in scans.into_iter().enumerate() {
+ let i = start + offset;
+ file_meta.push(sc.sniffed.as_ref().map(|s| (s.family, s.gzip)));
+ if !sc.pdf_spool_fallbacks.is_empty() {
+ pdf_fallbacks.push((i, std::mem::take(&mut sc.pdf_spool_fallbacks)));
+ }
+ let family = sc
+ .sniffed
+ .as_ref()
+ .map(|s| s.family)
+ .unwrap_or(Family::Binary);
+ // A file that phase A junks is never indexed, so its artifact is
+ // refunded here rather than held to the phase A→B boundary.
+ pdf_spools[i] =
+ take_pdf_spool_if_indexable(&mut sc.pdf_spool, sc.junk.is_some(), pdf_spool_budget);
+ if let Some((status, reason)) = sc.junk {
+ junk_files.push(JunkFile {
+ file_key: keys[i].clone(),
+ rel: files[i].rel.clone(),
+ format: format_str(sc.sniffed.as_ref()),
+ status,
+ reason,
+ bytes: files[i].size,
+ });
+ continue;
+ }
+ for gs in sc.sketches {
+ clusterer.push(
+ dataset::Sketch {
+ file_idx: i,
+ group: gs.group,
+ family,
+ fields: gs.fields,
+ key_fields: gs.key_fields,
+ records: gs.records,
+ population: gs.population,
+ },
+ &scopes[i],
+ );
+ }
}
+ start = end;
}
- let mut clusters = dataset::cluster(sketches, &rels, &scopes);
+
+ report_pdf_spool_fallbacks(files, &pdf_fallbacks, ctx);
+ note_worker_address_space_cap(pr);
+
+ let mut clusters = clusterer.finish(&rels);
// `{prefix}-{slug}` must not stutter (`logs-logs-syslog`) when the slug's
// first segment is the prefix the operator already chose.
dataset::drop_repeated_prefix(&cfg.prefix, &mut clusters);
+ dataset::cap_slug_len(&mut clusters);
// Demoted one-off config files (#173) were sampled as flattened records;
// phase B will index them as documents. Re-sample them through the same
@@ -9563,7 +9587,9 @@ fn run_index_report_inner(
}
// ── Phase A: inference (skipped when a frozen plan exists) ──────────
- let mut clusters_rt: Option<Vec<dataset::Cluster>> = None;
+ // Only what end-of-run correlation needs; the phase-A clusters (every
+ // field's full accumulator) are dropped as soon as the plan is built.
+ let mut key_stats: Option<correlate::KeyStats> = None;
let mut pdf_spools: Vec<Option<extract::pdf::ExtractionSpool>> =
(0..files.len()).map(|_| None).collect();
// Both worker widths come from the one resource policy (#240), and since
@@ -9628,7 +9654,8 @@ fn run_index_report_inner(
plan.datasets.len() as u64,
plan.junk_files.len() as u64,
));
- clusters_rt = Some(clusters);
+ key_stats = Some(correlate::key_stats(&clusters));
+ drop(clusters);
plan
};
let unreadable_keys: HashSet<String> = inventory
@@ -11667,32 +11694,8 @@ fn run_index_report_inner(
// correlations
let mut key_corrs: Vec<correlate::KeyCorr> = Vec::new();
- if let Some(clusters) = &clusters_rt {
- let mut cands = Vec::new();
- for (c, d) in clusters.iter().zip(plan.datasets.iter()) {
- for spec in &d.specs {
- let Some(acc) = c.fields.get(&spec.name) else {
- continue;
- };
- if correlate::is_candidate(
- &spec.es_type,
- spec.semantic.as_deref(),
- acc.distinct.len(),
- acc.n,
- acc.distinct_overflow,
- (acc.long_ok > 0).then_some((acc.int_min, acc.int_max)),
- ) {
- cands.push(correlate::Candidate {
- slug: d.slug.clone(),
- index: d.index.clone(),
- field: spec.name.clone(),
- kind: spec.es_type.clone(),
- values: acc.raw_values.clone(),
- sampled_n: acc.n,
- });
- }
- }
- }
+ if let Some(stats) = &key_stats {
+ let cands = correlate::candidates(stats, &plan.datasets);
key_corrs = correlate::key_overlaps(&cands);
// One live query per candidate overlap, and the bound is known before
// the loop starts — `phase=finalize-correlate items=37/135` needs no