AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
autoindex: bound phase-A memory and cap long dataset names (part of #1122) (#2417)
xerj-org6cdb32e29226
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 0a01cec3b..044d32fb8 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -79,6 +79,18 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0### Fixed+- `xerj autoindex` no longer runs out of memory while planning a large folder.+ It used to keep the sampled values of every file in memory at once before+ grouping them into datasets, and kept them all until the end of the run;+ on a 5 GB folder of about 146,000 files that was several GB on the client+ before anything was indexed. Files are now grouped as they are read, and only+ what the end-of-run key correlation needs is kept. The datasets and index+ names it plans are unchanged (part of #1122).+- `xerj autoindex` no longer fails a run because a dataset name built from a+ long path (for example a Go module path) is over the 255-character index name+ limit. Names longer than 128 characters are shortened and end in a short+ hash, so they stay the same on every run and never clash (part of #1122).+- Server startup now reports the resolved data directory, its configurationsource and whether this launch created it. New-directory guidance pointsoperators back to their previous `--data-dir` or `--config` when they expecteddiff --git a/engine/crates/xerj-autoindex/src/correlate.rs b/engine/crates/xerj-autoindex/src/correlate.rsindex a3838abac..4ea8ee5e5 100644--- a/engine/crates/xerj-autoindex/src/correlate.rs+++ b/engine/crates/xerj-autoindex/src/correlate.rs@@ -7,7 +7,7 @@use crate::esclient::Es;use anyhow::Result;use serde_json::{json, Value};-use std::collections::HashSet;+use std::collections::{HashMap, HashSet};pub struct Candidate {pub slug: String,@@ -70,6 +70,90 @@ impl KeyCorr {}}+/// What end-of-run key correlation needs from one sampled field, kept in+/// place of the field's full phase-A accumulator.+#[derive(Debug, Clone, PartialEq)]+pub struct KeyFieldStats {+ pub n: u64,+ pub distinct: usize,+ pub overflow: bool,+ pub int_range: Option<(i64, i64)>,+ pub values: HashSet<String>,+}++/// dataset slug → field name → stats.+pub type KeyStats = HashMap<String, HashMap<String, KeyFieldStats>>;++/// Fewest distinct sampled values [`is_candidate`] accepts for any type.+const MIN_CANDIDATE_DISTINCT: usize = 10;++/// Keep only the correlation inputs of every cluster's fields, so the phase-A+/// accumulators (byte histograms, length/token samples, hashed distinct sets)+/// can be dropped once the plan is built instead of living to the end of the+/// run. Fields with too few distinct values to ever be a candidate are left+/// out; the type-dependent checks run at the end against the final specs.+pub fn key_stats(clusters: &[crate::dataset::Cluster]) -> KeyStats {+ let mut out = KeyStats::new();+ for c in clusters {+ let fields: HashMap<String, KeyFieldStats> = c+ .fields+ .iter()+ .filter(|(_, acc)| acc.distinct.len() >= MIN_CANDIDATE_DISTINCT)+ .map(|(name, acc)| {+ (+ name.clone(),+ KeyFieldStats {+ n: acc.n,+ distinct: acc.distinct.len(),+ overflow: acc.distinct_overflow,+ int_range: (acc.long_ok > 0).then_some((acc.int_min, acc.int_max)),+ values: acc.raw_values.clone(),+ },+ )+ })+ .collect();+ if !fields.is_empty() {+ out.insert(c.slug.clone(), fields);+ }+ }+ out+}++/// Key-overlap candidates among `datasets`' fields, from stats captured by+/// [`key_stats`]. Looked up by slug, so datasets refused after planning+/// simply drop out.+pub fn candidates(stats: &KeyStats, datasets: &[crate::state::PlanDataset]) -> Vec<Candidate> {+ let mut cands = Vec::new();+ for d in datasets {+ let Some(fields) = stats.get(&d.slug) else {+ continue;+ };+ for spec in &d.specs {+ let Some(st) = fields.get(&spec.name) else {+ continue;+ };+ if is_candidate(+ &spec.es_type,+ spec.semantic.as_deref(),+ st.distinct,+ st.n,+ st.overflow,+ st.int_range,+ ) {+ cands.push(Candidate {+ slug: d.slug.clone(),+ index: d.index.clone(),+ field: spec.name.clone(),+ kind: spec.es_type.clone(),+ values: st.values.clone(),+ sampled_n: st.n,+ });+ }+ }+ }+ cands+}+/// Key-like candidate filter. Deviations from the design draft (recorded):/// - keyword fields qualify at cardinality ≥ 20 regardless of distinct/n/// ratio (join keys like user ids have LOW sample ratios in big datasets);@@ -351,6 +435,110 @@ pub fn time_alignment(series: &[TimeSeries]) -> Vec<Value> {mod tests {use super::*;+ fn spec(name: &str, es_type: &str) -> crate::infer::FieldSpec {+ crate::infer::FieldSpec {+ name: name.into(),+ es_type: es_type.into(),+ date_enc: None,+ semantic: None,+ cardinality_est: 0,+ cardinality_overflow: false,+ null_ratio: 0.0,+ avg_len: 0.0,+ coverage: 1.0,+ examples: vec![],+ notes: vec![],+ date_min: None,+ date_max: None,+ date_evidence: vec![],+ num_min: None,+ num_max: None,+ }+ }++ fn plan_dataset(slug: &str, specs: Vec<crate::infer::FieldSpec>) -> crate::state::PlanDataset {+ crate::state::PlanDataset {+ slug: slug.into(),+ index: format!("ax-{slug}"),+ family: "jsonl".into(),+ group: None,+ specs,+ time_field: None,+ semantic_field: None,+ text_analyzer: None,+ scope: None,+ sampled_records: 1,+ file_count: 1,+ }+ }++ fn cluster(+ slug: &str,+ field: &str,+ values: impl Iterator<Item = String>,+ ) -> crate::dataset::Cluster {+ let mut acc = crate::infer::FieldAcc::default();+ for v in values {+ acc.add(&serde_json::Value::String(v));+ }+ let mut few = crate::infer::FieldAcc::default();+ few.add(&serde_json::Value::String("only".into()));+ crate::dataset::Cluster {+ family: crate::sniff::Family::Jsonl,+ group: None,+ members: vec![0],+ fields: [(field.to_string(), acc), ("level".to_string(), few)]+ .into_iter()+ .collect(),+ key_fields: Default::default(),+ records: 100,+ population: 100,+ slug: slug.into(),+ scope: String::new(),+ is_docs: false,+ demoted: Vec::new(),+ }+ }++ /// The compact stats give the same candidates the full accumulators did,+ /// drop fields that can never be keys, and follow datasets by slug — so+ /// a dataset refused after planning does not shift its neighbours' fields.+ #[test]+ fn key_stats_match_the_accumulators_and_follow_datasets_by_slug() {+ let ids = || (0..50).map(|i| format!("user-{i}"));+ let clusters = vec![+ cluster("refused", "user_id", ids()),+ cluster("orders", "user_id", ids()),+ cluster("users", "id", ids()),+ ];+ let stats = key_stats(&clusters);+ assert!(+ !stats["orders"].contains_key("level"),+ "1 distinct value is never a key"+ );+ let acc = &clusters[1].fields["user_id"];+ let st = &stats["orders"]["user_id"];+ assert_eq!(st.n, acc.n);+ assert_eq!(st.distinct, acc.distinct.len());+ assert_eq!(st.values, acc.raw_values);++ // "refused" is no longer in the plan.+ let datasets = vec![+ plan_dataset(+ "orders",+ vec![spec("user_id", "keyword"), spec("level", "keyword")],+ ),+ plan_dataset("users", vec![spec("id", "keyword")]),+ ];+ let cands = candidates(&stats, &datasets);+ let got: Vec<(&str, &str)> = cands+ .iter()+ .map(|c| (c.slug.as_str(), c.field.as_str()))+ .collect();+ assert_eq!(got, vec![("orders", "user_id"), ("users", "id")]);+ assert_eq!(key_overlaps(&cands).len(), 1);+ }+fn sample_corr() -> KeyCorr {KeyCorr {a_slug: "reports".into(),diff --git a/engine/crates/xerj-autoindex/src/dataset.rs b/engine/crates/xerj-autoindex/src/dataset.rsindex 11c80d4a6..40eac1053 100644--- a/engine/crates/xerj-autoindex/src/dataset.rs+++ b/engine/crates/xerj-autoindex/src/dataset.rs@@ -135,31 +135,61 @@ pub(crate) fn demotable_family(f: Family) -> bool {/// Cluster sketches into datasets. `rels` and `scopes` are indexed by/// `Sketch::file_idx`: `rels` are root-relative paths (slug naming), `scopes`/// the repository scope of each file ("" = root) — see `module` docs.+///+/// Equivalent to feeding every sketch, in order, to a [`Clusterer`] and+/// calling [`Clusterer::finish`].pub fn cluster(sketches: Vec<Sketch>, rels: &[String], scopes: &[String]) -> Vec<Cluster> {- // scope → docs cluster (BTreeMap: deterministic scope order).- let mut docs: BTreeMap<String, Cluster> = BTreeMap::new();- let mut data: Vec<Cluster> = Vec::new();- fn docs_cluster<'a>(docs: &'a mut BTreeMap<String, Cluster>, scope: &str) -> &'a mut Cluster {- docs.entry(scope.to_string()).or_insert_with(|| Cluster {- family: Family::Code,- group: None,- members: Vec::new(),- fields: HashMap::new(),- key_fields: HashSet::new(),- records: 0,- population: 0,- slug: String::new(),- scope: scope.to_string(),- is_docs: true,- demoted: Vec::new(),- })- }-+ let mut clusterer = Clusterer::default();for sk in sketches {let scope = scopes[sk.file_idx].as_str();+ clusterer.push(sk, scope);+ }+ clusterer.finish(rels)+}++fn docs_cluster<'a>(docs: &'a mut BTreeMap<String, Cluster>, scope: &str) -> &'a mut Cluster {+ docs.entry(scope.to_string()).or_insert_with(|| Cluster {+ family: Family::Code,+ group: None,+ members: Vec::new(),+ fields: HashMap::new(),+ key_fields: HashSet::new(),+ records: 0,+ population: 0,+ slug: String::new(),+ scope: scope.to_string(),+ is_docs: true,+ demoted: Vec::new(),+ })+}++/// Incremental form of [`cluster`]: sketches are folded into their dataset as+/// they arrive, so a caller never has to hold every file's sketch at once.+///+/// A sketch carries one [`FieldAcc`] per sampled field (a 1 KiB byte histogram,+/// up to 512 length samples and up to [`crate::infer::RAW_CAP`] raw values+/// each). Phase A used to collect every file's sketches before clustering:+/// tens of KiB per file, and 4.4 GB of client RSS on a 146k-file folder before+/// a single document was sent. Folding them as they are produced bounds that+/// to the clusters themselves. Sketches must be pushed in file order for the+/// result to match [`cluster`] exactly (clustering is greedy and+/// order-dependent).+#[derive(Default)]+pub struct Clusterer {+ // scope → docs cluster (BTreeMap: deterministic scope order).+ docs: BTreeMap<String, Cluster>,+ data: Vec<Cluster>,+}++impl Clusterer {+ /// Fold one sketch into its dataset. `scope` is the repository scope of+ /// `sk.file_idx`.+ pub fn push(&mut self, sk: Sketch, scope: &str) {+ let docs = &mut self.docs;+ let data = &mut self.data;// Documents: no data-derived names, no sub-file group.if sk.key_fields.is_empty() && sk.group.is_none() {- let c = docs_cluster(&mut docs, scope);+ let c = docs_cluster(docs, scope);c.members.push(sk.file_idx);c.records += sk.records;c.population += sk.population;@@ -171,7 +201,7 @@ pub fn cluster(sketches: Vec<Sketch>, rels: &[String], scopes: &[String]) -> Vec}}}- continue;+ return;}// Data: schema clustering within (scope, family, group).let names: HashSet<&str> = sk.key_fields.iter().map(|s| s.as_str()).collect();@@ -219,92 +249,96 @@ pub fn cluster(sketches: Vec<Sketch>, rels: &[String], scopes: &[String]) -> Vec}}- // Demote small config clusters — one record each, or a short multi-doc- // YAML/JSON/XML file — to their scope's document dataset. Their sampled- // fields are dropped — the caller re-samples the demoted files through- // the document renderer, which is also what indexes them.- //- // A file that also produced a grouped dataset (a second top-level JSON- // array) must not be folded: folding marks the whole file `as_document`- // and phase B then drops the grouped rows.- let grouped_files: HashSet<usize> = data- .iter()- .filter(|c| c.group.is_some())- .flat_map(|c| c.members.iter().copied())- .collect();- let mut kept: Vec<Cluster> = Vec::with_capacity(data.len());- let mut capped: BTreeMap<String, Vec<Cluster>> = BTreeMap::new();- for c in data {- let shares_grouped_sibling = c.members.iter().any(|m| grouped_files.contains(m));- if small_structured_config(&c) && !shares_grouped_sibling {- let d = docs_cluster(&mut docs, &c.scope);- d.members.extend(c.members.iter().copied());- d.demoted.extend(c.members);- continue;- }- // Candidates for the per-scope cap: structured formats, no sub-file- // group, and not a large collection. Everything else stays.- if demotable_family(c.family)- && c.group.is_none()- && c.population < DATASET_CAP_PROTECT_RECORDS- && !shares_grouped_sibling- {- capped.entry(c.scope.clone()).or_default().push(c);- continue;+ /// Demote, cap and name the clusters. `rels` is indexed by file index.+ pub fn finish(self, rels: &[String]) -> Vec<Cluster> {+ let Clusterer { mut docs, data } = self;+ // Demote small config clusters — one record each, or a short multi-doc+ // YAML/JSON/XML file — to their scope's document dataset. Their sampled+ // fields are dropped — the caller re-samples the demoted files through+ // the document renderer, which is also what indexes them.+ //+ // A file that also produced a grouped dataset (a second top-level JSON+ // array) must not be folded: folding marks the whole file `as_document`+ // and phase B then drops the grouped rows.+ let grouped_files: HashSet<usize> = data+ .iter()+ .filter(|c| c.group.is_some())+ .flat_map(|c| c.members.iter().copied())+ .collect();+ let mut kept: Vec<Cluster> = Vec::with_capacity(data.len());+ let mut capped: BTreeMap<String, Vec<Cluster>> = BTreeMap::new();+ for c in data {+ let shares_grouped_sibling = c.members.iter().any(|m| grouped_files.contains(m));+ if small_structured_config(&c) && !shares_grouped_sibling {+ let d = docs_cluster(&mut docs, &c.scope);+ d.members.extend(c.members.iter().copied());+ d.demoted.extend(c.members);+ continue;+ }+ // Candidates for the per-scope cap: structured formats, no sub-file+ // group, and not a large collection. Everything else stays.+ if demotable_family(c.family)+ && c.group.is_none()+ && c.population < DATASET_CAP_PROTECT_RECORDS+ && !shares_grouped_sibling+ {+ capped.entry(c.scope.clone()).or_default().push(c);+ continue;+ }+ kept.push(c);}- kept.push(c);- }- for (scope, mut group) in capped {- // Largest collections stay. Ties break toward the earlier file so- // the fold does not depend on hash order. Size is the real row- // count, so a file sampled at 500 does not tie with a 100-row file- // that happened to fit in the sample.- group.sort_by(|a, b| {- b.population- .cmp(&a.population)- .then(b.members.len().cmp(&a.members.len()))- .then_with(|| {- let am = a.members.iter().copied().min().unwrap_or(usize::MAX);- let bm = b.members.iter().copied().min().unwrap_or(usize::MAX);- am.cmp(&bm)- })- });- let keep_n = MAX_STRUCTURED_DATASETS_PER_SCOPE.min(group.len());- for c in group.drain(keep_n..) {- let d = docs_cluster(&mut docs, &scope);- d.members.extend(c.members.iter().copied());- d.demoted.extend(c.members);+ for (scope, mut group) in capped {+ // Largest collections stay. Ties break toward the earlier file so+ // the fold does not depend on hash order. Size is the real row+ // count, so a file sampled at 500 does not tie with a 100-row file+ // that happened to fit in the sample.+ group.sort_by(|a, b| {+ b.population+ .cmp(&a.population)+ .then(b.members.len().cmp(&a.members.len()))+ .then_with(|| {+ let am = a.members.iter().copied().min().unwrap_or(usize::MAX);+ let bm = b.members.iter().copied().min().unwrap_or(usize::MAX);+ am.cmp(&bm)+ })+ });+ let keep_n = MAX_STRUCTURED_DATASETS_PER_SCOPE.min(group.len());+ for c in group.drain(keep_n..) {+ let d = docs_cluster(&mut docs, &scope);+ d.members.extend(c.members.iter().copied());+ d.demoted.extend(c.members);+ }+ kept.extend(group);}- kept.extend(group);- }- // Docs slugs are scope-derived — independent of which other files exist,- // so adding a file can never rename (and re-home, #178) a docs dataset.- let mut clusters: Vec<Cluster> = Vec::with_capacity(docs.len() + kept.len());- let mut used: HashSet<String> = HashSet::new();- for (scope, mut c) in docs {- let base = if scope.is_empty() {- "docs".to_string()- } else {- let s = sanitize_slug(&scope);- if s.is_empty() {+ // Docs slugs are scope-derived — independent of which other files exist,+ // so adding a file can never rename (and re-home, #178) a docs dataset.+ let mut clusters: Vec<Cluster> = Vec::with_capacity(docs.len() + kept.len());+ let mut used: HashSet<String> = HashSet::new();+ for (scope, mut c) in docs {+ let base = if scope.is_empty() {"docs".to_string()} else {- format!("{s}-docs")+ let s = sanitize_slug(&scope);+ if s.is_empty() {+ "docs".to_string()+ } else {+ format!("{s}-docs")+ }+ };+ let mut slug = base.clone();+ let mut k = 2;+ while !used.insert(slug.clone()) {+ slug = format!("{base}-{k}");+ k += 1;}- };- let mut slug = base.clone();- let mut k = 2;- while !used.insert(slug.clone()) {- slug = format!("{base}-{k}");- k += 1;+ c.slug = slug;+ clusters.push(c);}- c.slug = slug;- clusters.push(c);+ assign_slugs(&mut kept, rels, &mut used);+ clusters.extend(kept);+ clusters}- assign_slugs(&mut kept, rels, &mut used);- clusters.extend(kept);- clusters}pub fn sanitize_slug(s: &str) -> String {@@ -543,10 +577,114 @@ pub fn drop_repeated_prefix(prefix: &str, clusters: &mut [Cluster]) {}}+/// Longest dataset slug autoindex derives. Slugs come from file paths and+/// group names with no natural bound — a Go module path produced a 329-char+/// index name that the server refused (index names are capped at 255 bytes),+/// failing the run (part of #1122). 128 leaves room for the `--prefix` and for+/// the suffixes rebuilds add to it (`-reindex`, `-generation-v1`).+pub const MAX_SLUG_LEN: usize = 128;++/// Shorten slugs longer than [`MAX_SLUG_LEN`]: keep the leading part and+/// append `-` plus 8 hex digits of a hash of the full slug, so the name stays+/// readable, is the same on every run, and two long slugs sharing a long+/// beginning still get different names. Slugs within the limit are untouched.+pub fn cap_slug_len(clusters: &mut [Cluster]) {+ if clusters.iter().all(|c| c.slug.len() <= MAX_SLUG_LEN) {+ return;+ }+ let mut used: HashSet<String> = clusters+ .iter()+ .filter(|c| c.slug.len() <= MAX_SLUG_LEN)+ .map(|c| c.slug.clone())+ .collect();+ let mut order: Vec<usize> = (0..clusters.len())+ .filter(|&i| clusters[i].slug.len() > MAX_SLUG_LEN)+ .collect();+ order.sort_by_key(|&i| clusters[i].slug.clone());+ for i in order {+ let full = clusters[i].slug.clone();+ let mut salt = 0u64;+ loop {+ let slug = capped_slug(&full, salt);+ if used.insert(slug.clone()) {+ clusters[i].slug = slug;+ break;+ }+ salt += 1;+ }+ }+}++fn capped_slug(full: &str, salt: u64) -> String {+ let hash = xxhash_rust::xxh3::xxh3_64_with_seed(full.as_bytes(), salt) as u32;+ let suffix = format!("-{hash:08x}");+ // Slugs are ASCII (`sanitize_slug`); the boundary check keeps this safe+ // for any other caller.+ let mut cut = MAX_SLUG_LEN - suffix.len();+ while !full.is_char_boundary(cut) {+ cut -= 1;+ }+ let head = full[..cut].trim_end_matches('-');+ format!("{head}{suffix}")+}+#[cfg(test)]mod tests {use super::*;+ fn named(slug: &str) -> Cluster {+ Cluster {+ family: Family::Json,+ group: None,+ members: Vec::new(),+ fields: HashMap::new(),+ key_fields: HashSet::new(),+ records: 0,+ population: 0,+ slug: slug.to_string(),+ scope: String::new(),+ is_docs: false,+ demoted: Vec::new(),+ }+ }++ #[test]+ fn long_slugs_are_capped_with_a_stable_hash_suffix() {+ // The real 329-char index name from a Go module tree (part of #1122).+ let long = format!(+ "gomod-github-com-paths-v1-example-a-bit-of-everything-floatvalue-{}",+ "nested-message-repeated-string-value-".repeat(7)+ );+ let sibling = format!("{long}other");+ let mut clusters = vec![named("short"), named(&long), named(&sibling)];+ cap_slug_len(&mut clusters);+ assert_eq!(clusters[0].slug, "short");+ for c in &clusters[1..] {+ assert!(c.slug.len() <= MAX_SLUG_LEN, "{} chars", c.slug.len());+ assert!(c.slug.starts_with("gomod-github-com-paths-v1-example"));+ assert_eq!(c.slug, sanitize_slug(&c.slug), "still a valid slug");+ assert!(xerj_common::IndexName::validate(&format!("b5-{}", c.slug)).is_ok());+ }+ assert_ne!(clusters[1].slug, clusters[2].slug);+ // Deterministic: the same input names the same index on every run.+ let mut again = vec![named("short"), named(&long), named(&sibling)];+ cap_slug_len(&mut again);+ let a: Vec<_> = clusters.iter().map(|c| c.slug.clone()).collect();+ let b: Vec<_> = again.iter().map(|c| c.slug.clone()).collect();+ assert_eq!(a, b);+ }++ #[test]+ fn capped_slug_never_collides_with_an_existing_one() {+ let long = "a".repeat(MAX_SLUG_LEN + 50);+ let taken = capped_slug(&long, 0);+ let mut clusters = vec![named(&taken), named(&long)];+ cap_slug_len(&mut clusters);+ assert_eq!(clusters[0].slug, taken);+ assert_ne!(clusters[1].slug, taken);+ assert!(clusters[1].slug.len() <= MAX_SLUG_LEN);+ }+#[test]fn a_digit_leading_prefix_is_rejected_before_any_walk() {let err = index_prefix_error("2042").expect("2042-docs is not an index name");diff --git a/engine/crates/xerj-autoindex/src/lib.rs b/engine/crates/xerj-autoindex/src/lib.rsindex 367943c25..4e9be1d43 100644--- a/engine/crates/xerj-autoindex/src/lib.rs+++ b/engine/crates/xerj-autoindex/src/lib.rs@@ -5247,18 +5247,17 @@ struct PhaseA {/// `--progress json` stays a single parseable stream (#241).fn report_pdf_spool_fallbacks(files: &[walk::FileEntry],- scans: &[FileScan],+ fallbacks: &[(usize, Vec<extract::pdf::SpoolFallback>)],ctx: &PhaseAContext<'_>,) {let pdf_spool_budget = ctx.budget;let pr = ctx.progress;- let reasons: Vec<(&str, &extract::pdf::SpoolFallback)> = scans+ let reasons: Vec<(&str, &extract::pdf::SpoolFallback)> = fallbacks.iter()- .enumerate()- .flat_map(|(index, scan)| {- scan.pdf_spool_fallbacks+ .flat_map(|(index, per_file)| {+ per_file.iter()- .map(move |fallback| (files[index].rel.as_str(), fallback))+ .map(move |fallback| (files[*index].rel.as_str(), fallback))}).collect();for (path, fallback) in &reasons {@@ -5296,6 +5295,13 @@ fn report_pdf_spool_fallbacks(}}+/// Files phase A scans before folding their sketches into clusters. Large+/// enough that the scan pool stays busy across a chunk, small enough that the+/// sketches held at once stay bounded whatever the folder's file count.+// Tests use a tiny chunk so every multi-file phase-A test crosses chunk+// boundaries.+const PHASE_A_CHUNK_FILES: usize = if cfg!(test) { 3 } else { 2048 };+/// Sniff + sample every file, cluster into datasets, and assemble the plan./// Pure planning: reads the tree, never the server — which is what makes the/// #173/#196 grouping behaviour testable end-to-end without a cluster.@@ -5320,77 +5326,95 @@ fn build_phase_a(// Patterns were validated (loudly) at run start; a failure here would be// a programming error, not user input.let stub_matcher = StubMatcher::compile(&cfg.stub_globs).expect("validated at startup");- let scans: Vec<FileScan> = crate::pool::install(|| {- files- .par_iter()- .zip(digests.par_iter())- .map(|(f, digest)| {- let _in_flight = pr.file(&f.rel, f.size);- scan_file(- &f.path,- f.size,- digest,- ctx,- cfg.sample,- cfg.max_file_bytes,- stub_matcher.matches(&f.rel),- )- })- .collect()- });-- report_pdf_spool_fallbacks(files, &scans, ctx);- note_worker_address_space_cap(pr);-let rels: Vec<String> = files.iter().map(|f| f.rel.clone()).collect();let scopes = compute_scopes(root, &rels);// (family, gzip) per file, from the scan's sniff — reused for assignments// and demoted-file re-sampling instead of re-sniffing every file.- let file_meta: Vec<Option<(Family, bool)>> = scans- .iter()- .map(|sc| sc.sniffed.as_ref().map(|s| (s.family, s.gzip)))- .collect();- let mut sketches = Vec::new();+ let mut file_meta: Vec<Option<(Family, bool)>> = Vec::with_capacity(files.len());let mut junk_files = Vec::new();let mut pdf_spools: Vec<Option<extract::pdf::ExtractionSpool>> =(0..files.len()).map(|_| None).collect();- for (i, mut sc) in scans.into_iter().enumerate() {- let family = sc- .sniffed- .as_ref()- .map(|s| s.family)- .unwrap_or(Family::Binary);- // A file that phase A junks is never indexed, so its artifact is- // refunded here rather than held to the phase A→B boundary.- pdf_spools[i] =- take_pdf_spool_if_indexable(&mut sc.pdf_spool, sc.junk.is_some(), pdf_spool_budget);- if let Some((status, reason)) = sc.junk {- junk_files.push(JunkFile {- file_key: keys[i].clone(),- rel: files[i].rel.clone(),- format: format_str(sc.sniffed.as_ref()),- status,- reason,- bytes: files[i].size,- });- continue;- }- for gs in sc.sketches {- sketches.push(dataset::Sketch {- file_idx: i,- group: gs.group,- family,- fields: gs.fields,- key_fields: gs.key_fields,- records: gs.records,- population: gs.population,- });+ let mut pdf_fallbacks: Vec<(usize, Vec<extract::pdf::SpoolFallback>)> = Vec::new();+ // Scan in file-ordered chunks and fold each chunk's sketches into the+ // clusters before the next chunk is scanned. Collecting every file's scan+ // first held one set of field accumulators per file for the whole folder+ // at once: 4.4 GB of client RSS on a 146k-file tree before anything was+ // indexed (part of #1122). Folding in file order keeps the clustering+ // identical to the collect-all version (see `dataset::Clusterer`).+ let mut clusterer = dataset::Clusterer::default();+ let mut start = 0usize;+ while start < files.len() {+ let end = (start + PHASE_A_CHUNK_FILES).min(files.len());+ let scans: Vec<FileScan> = crate::pool::install(|| {+ files[start..end]+ .par_iter()+ .zip(digests[start..end].par_iter())+ .map(|(f, digest)| {+ let _in_flight = pr.file(&f.rel, f.size);+ scan_file(+ &f.path,+ f.size,+ digest,+ ctx,+ cfg.sample,+ cfg.max_file_bytes,+ stub_matcher.matches(&f.rel),+ )+ })+ .collect()+ });+ for (offset, mut sc) in scans.into_iter().enumerate() {+ let i = start + offset;+ file_meta.push(sc.sniffed.as_ref().map(|s| (s.family, s.gzip)));+ if !sc.pdf_spool_fallbacks.is_empty() {+ pdf_fallbacks.push((i, std::mem::take(&mut sc.pdf_spool_fallbacks)));+ }+ let family = sc+ .sniffed+ .as_ref()+ .map(|s| s.family)+ .unwrap_or(Family::Binary);+ // A file that phase A junks is never indexed, so its artifact is+ // refunded here rather than held to the phase A→B boundary.+ pdf_spools[i] =+ take_pdf_spool_if_indexable(&mut sc.pdf_spool, sc.junk.is_some(), pdf_spool_budget);+ if let Some((status, reason)) = sc.junk {+ junk_files.push(JunkFile {+ file_key: keys[i].clone(),+ rel: files[i].rel.clone(),+ format: format_str(sc.sniffed.as_ref()),+ status,+ reason,+ bytes: files[i].size,+ });+ continue;+ }+ for gs in sc.sketches {+ clusterer.push(+ dataset::Sketch {+ file_idx: i,+ group: gs.group,+ family,+ fields: gs.fields,+ key_fields: gs.key_fields,+ records: gs.records,+ population: gs.population,+ },+ &scopes[i],+ );+ }}+ start = end;}- let mut clusters = dataset::cluster(sketches, &rels, &scopes);++ report_pdf_spool_fallbacks(files, &pdf_fallbacks, ctx);+ note_worker_address_space_cap(pr);++ let mut clusters = clusterer.finish(&rels);// `{prefix}-{slug}` must not stutter (`logs-logs-syslog`) when the slug's// first segment is the prefix the operator already chose.dataset::drop_repeated_prefix(&cfg.prefix, &mut clusters);+ dataset::cap_slug_len(&mut clusters);// Demoted one-off config files (#173) were sampled as flattened records;// phase B will index them as documents. Re-sample them through the same@@ -9563,7 +9587,9 @@ fn run_index_report_inner(}// ── Phase A: inference (skipped when a frozen plan exists) ──────────- let mut clusters_rt: Option<Vec<dataset::Cluster>> = None;+ // Only what end-of-run correlation needs; the phase-A clusters (every+ // field's full accumulator) are dropped as soon as the plan is built.+ let mut key_stats: Option<correlate::KeyStats> = None;let mut pdf_spools: Vec<Option<extract::pdf::ExtractionSpool>> =(0..files.len()).map(|_| None).collect();// Both worker widths come from the one resource policy (#240), and since@@ -9628,7 +9654,8 @@ fn run_index_report_inner(plan.datasets.len() as u64,plan.junk_files.len() as u64,));- clusters_rt = Some(clusters);+ key_stats = Some(correlate::key_stats(&clusters));+ drop(clusters);plan};let unreadable_keys: HashSet<String> = inventory@@ -11667,32 +11694,8 @@ fn run_index_report_inner(// correlationslet mut key_corrs: Vec<correlate::KeyCorr> = Vec::new();- if let Some(clusters) = &clusters_rt {- let mut cands = Vec::new();- for (c, d) in clusters.iter().zip(plan.datasets.iter()) {- for spec in &d.specs {- let Some(acc) = c.fields.get(&spec.name) else {- continue;- };- if correlate::is_candidate(- &spec.es_type,- spec.semantic.as_deref(),- acc.distinct.len(),- acc.n,- acc.distinct_overflow,- (acc.long_ok > 0).then_some((acc.int_min, acc.int_max)),- ) {- cands.push(correlate::Candidate {- slug: d.slug.clone(),- index: d.index.clone(),- field: spec.name.clone(),- kind: spec.es_type.clone(),- values: acc.raw_values.clone(),- sampled_n: acc.n,- });- }- }- }+ if let Some(stats) = &key_stats {+ let cands = correlate::candidates(stats, &plan.datasets);key_corrs = correlate::key_overlaps(&cands);// One live query per candidate overlap, and the bound is known before// the loop starts — `phase=finalize-correlate items=37/135` needs no