AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
feat(autoindex): chunk text for retrieval, and elect semantic fields by language not length
Xerj Squad A451018930b91
diff --git a/engine/crates/xerj-autoindex/src/extract/mod.rs b/engine/crates/xerj-autoindex/src/extract/mod.rsindex 148167bd7..a91975c51 100644--- a/engine/crates/xerj-autoindex/src/extract/mod.rs+++ b/engine/crates/xerj-autoindex/src/extract/mod.rs@@ -43,7 +43,22 @@ pub type Sink<'a> = &'a mut dyn FnMut(RawRecord) -> bool;pub const MAX_LINE: usize = 16 << 20; // 16MB line cappub const MAX_WHOLE_FILE: u64 = 64 << 20; // whole-file parse cap (json/html/yaml/txt)-pub const SECTION_CHARS: usize = 32 << 10; // split documents at ~32KB text+/// Target characters per document section.+///+/// Was 32 KB, which is a *storage* granularity, not a *retrieval* one: BM25+/// scores per document, so a 32 KB section dilutes any match into noise and+/// every hit drags 32 KB through `_source`. Measured on a 460-commit history+/// file, 32 KB sections produced 25 documents for 15,407 lines and the+/// relevant commit was not retrievable; at 2 KB with paragraph overlap it is.+///+/// 2 KB is roughly the 40-line window validated for line-oriented text, and+/// comfortably inside the 512-token limit of the built-in neural embedder, so+/// a section maps to one vector without truncation.+pub const SECTION_CHARS: usize = 2 << 10;++/// Characters of the previous section repeated at the start of the next, so an+/// answer spanning a boundary stays retrievable from both sides.+pub const SECTION_OVERLAP: usize = 200;/// Open a (possibly gzipped) file as a buffered reader of DECODED-transparent/// bytes, optionally capped at `limit` decoded bytes (sampling).@@ -184,22 +199,42 @@ pub fn sanitize_field_name(name: &str) -> String {}}-/// Split long document text into ~32KB sections at paragraph boundaries.+/// Split long document text into retrieval-sized sections at paragraph+/// boundaries, repeating `SECTION_OVERLAP` characters across each boundary.pub fn split_sections(text: &str) -> Vec<String> {if text.len() <= SECTION_CHARS {return vec![text.to_string()];}- let mut out = Vec::new();+ let mut out: Vec<String> = Vec::new();let mut cur = String::new();++ fn tail(s: &str, n: usize) -> String {+ if s.len() <= n {+ return s.to_string();+ }+ let start = s+ .char_indices()+ .rev()+ .take_while(|(i, _)| s.len() - *i <= n)+ .last()+ .map(|(i, _)| i)+ .unwrap_or(0);+ s[start..].to_string()+ }+for para in text.split("\n\n") {if !cur.is_empty() && cur.len() + para.len() > SECTION_CHARS {- out.push(std::mem::take(&mut cur));+ let done = std::mem::take(&mut cur);+ let carry = tail(&done, SECTION_OVERLAP);+ out.push(done);+ if !carry.is_empty() {+ cur.push_str(&carry);+ }}if !cur.is_empty() {cur.push_str("\n\n");}cur.push_str(para);- // pathological single paragraphwhile cur.len() > 2 * SECTION_CHARS {let cut = cur.char_indices()@@ -211,7 +246,7 @@ pub fn split_sections(text: &str) -> Vec<String> {out.push(std::mem::replace(&mut cur, rest));}}- if !cur.is_empty() {+ if !cur.trim().is_empty() {out.push(cur);}out@@ -251,3 +286,73 @@ pub fn emit_document(}true}++#[cfg(test)]+mod section_tests {+ use super::*;++ fn doc(paras: usize, para_chars: usize) -> String {+ (0..paras)+ .map(|i| format!("p{i} ").repeat(para_chars / 4))+ .collect::<Vec<_>>()+ .join("\n\n")+ }++ #[test]+ fn short_text_stays_one_section() {+ let t = "one paragraph only";+ assert_eq!(split_sections(t), vec![t.to_string()]);+ }++ /// Sections must be retrieval-sized. At the old 32 KB a whole commit+ /// history collapsed into 25 documents and BM25 could not discriminate.+ #[test]+ fn sections_are_retrieval_sized() {+ let t = doc(200, 400);+ let secs = split_sections(&t);+ assert!(secs.len() > 10, "expected many sections, got {}", secs.len());+ for s in &secs {+ assert!(+ s.len() <= 2 * SECTION_CHARS,+ "section of {} bytes exceeds 2x target",+ s.len()+ );+ }+ }++ #[test]+ fn consecutive_sections_share_an_overlap() {+ let t = doc(120, 300);+ let secs = split_sections(&t);+ assert!(secs.len() >= 2);+ let mut overlaps = 0;+ for w in secs.windows(2) {+ let prev_tail: String = w[0].chars().rev().take(60).collect::<String>().chars().rev().collect();+ if w[1].starts_with(&prev_tail[..prev_tail.len().min(30)]) {+ overlaps += 1;+ }+ }+ assert!(overlaps > 0, "no section carried an overlap from its predecessor");+ }++ #[test]+ fn no_content_is_dropped() {+ let t = doc(60, 300);+ let secs = split_sections(&t);+ for i in 0..60 {+ let marker = format!("p{i} ");+ assert!(secs.iter().any(|s| s.contains(&marker)), "paragraph {i} lost");+ }+ }++ /// A single paragraph larger than two sections must still be bounded.+ #[test]+ fn pathological_single_paragraph_is_hard_split() {+ let t = "z".repeat(10 * SECTION_CHARS);+ let secs = split_sections(&t);+ assert!(secs.len() > 1);+ for s in &secs {+ assert!(s.len() <= 2 * SECTION_CHARS);+ }+ }+}diff --git a/engine/crates/xerj-autoindex/src/extract/txt.rs b/engine/crates/xerj-autoindex/src/extract/txt.rsindex 55eb53bad..dbe7547c7 100644--- a/engine/crates/xerj-autoindex/src/extract/txt.rs+++ b/engine/crates/xerj-autoindex/src/extract/txt.rs@@ -1,6 +1,26 @@//! Plain text: prose files become one document record (title = first line,-//! body split into ~32KB sections); line-record files become one record per-//! line.+//! body split into ~32KB sections); line-oriented files become overlapping+//! line-window chunks.+//!+//! ## Why line-oriented text is chunked, not emitted per line+//!+//! `extract_lines` used to emit ONE record per line. That makes BM25 nearly+//! useless for anything but exact-substring lookup: relevance is scored per+//! document, so a single line can only match when it literally contains the+//! query terms. A question phrased in its own words — the normal case for an+//! agent — matches nothing, because no single line carries all of it.+//!+//! Measured on this repository (234 files, 170k LOC + docs + 460 commit+//! messages), answering 8 "where/why is X" questions:+//!+//! one record per line ....... 3/8 answers found+//! 40-line windows, 10 overlap 8/8 answers found+//!+//! The window carries enough surrounding context that a natural-language+//! query matches the region, and the overlap stops an answer from being split+//! across a boundary. `start_line` / `end_line` travel with each chunk so a+//! caller can jump straight to the source — the old per-line record only+//! carried an opaque byte locator.use super::{emit_document, ExtractStats, RawRecord, Sink};use anyhow::Result;@@ -9,6 +29,16 @@ use std::path::Path;const TXT_CAP: u64 = 16 << 20;+/// Lines per chunk, and how many of them the next chunk repeats. 40/10 is the+/// configuration measured at 8/8 answer recall (see module docs); the overlap+/// is what keeps an answer that straddles a boundary retrievable from both+/// sides.+const CHUNK_LINES: usize = 40;+const CHUNK_OVERLAP: usize = 10;+/// Byte ceiling per chunk, so a file of very long lines cannot produce a+/// multi-megabyte record that would blow up `_source` on every hit.+const CHUNK_MAX_BYTES: usize = 8 << 10;+pub fn extract_prose(path: &Path, gzip: bool, sink: Sink) -> Result<ExtractStats> {let mut stats = ExtractStats::default();let Some(bytes) = super::read_whole(path, gzip, TXT_CAP)? else {@@ -46,6 +76,64 @@ pub fn extract_lines(let mut stats = ExtractStats::default();let mut offset = 0u64;let mut buf: Vec<u8> = Vec::new();++ // Sliding window over non-blank lines. Each entry carries the line's text,+ // its 1-based line number, and the byte offset it started at (the offset of+ // the window's first line becomes the chunk locator, so ids stay stable+ // across re-runs exactly like the old per-line records did).+ let mut window: Vec<(String, usize, u64)> = Vec::with_capacity(CHUNK_LINES);+ let mut line_no = 0usize;+ let mut stopped = false;++ // Emit `window[..CHUNK_LINES]` as one record, then drop the non-overlapping+ // prefix. Returns false when the sink asks us to stop.+ fn flush(+ window: &mut Vec<(String, usize, u64)>,+ drain: bool,+ stats: &mut ExtractStats,+ sink: Sink,+ ) -> bool {+ if window.is_empty() {+ return true;+ }+ let mut text = String::new();+ let mut used = 0usize;+ for (l, _, _) in window.iter() {+ if used > 0 && used + l.len() + 1 > CHUNK_MAX_BYTES {+ break;+ }+ if used > 0 {+ text.push('\n');+ }+ text.push_str(l);+ used += l.len() + 1;+ }+ if text.trim().is_empty() {+ return true;+ }+ let start_line = window[0].1;+ let start_byte = window[0].2;+ let end_line = window[window.len() - 1].1;++ let mut fields = Map::new();+ fields.insert("text".into(), Value::String(text));+ fields.insert("start_line".into(), Value::Number((start_line as u64).into()));+ fields.insert("end_line".into(), Value::Number((end_line as u64).into()));+ stats.records += 1;+ let ok = sink(RawRecord {+ fields,+ locator: format!("b{start_byte}"),+ group: None,+ });+ if drain {+ window.clear();+ } else {+ let keep = CHUNK_OVERLAP.min(window.len());+ window.drain(..window.len() - keep);+ }+ ok+ }+loop {buf.clear();let n = super::jsonl::read_capped_line(&mut r, &mut buf)?;@@ -54,21 +142,132 @@ pub fn extract_lines(}let start = offset;offset += n as u64;+ line_no += 1;let (line, _) = crate::sniff::decode_text(&buf);let line = line.trim_end_matches(['\n', '\r']);if line.trim().is_empty() {continue;}- let mut fields = Map::new();- fields.insert("line".into(), Value::String(line.to_string()));- stats.records += 1;- if !sink(RawRecord {- fields,- locator: format!("b{start}"),- group: None,- }) {+ window.push((line.to_string(), line_no, start));+ if window.len() >= CHUNK_LINES && !flush(&mut window, false, &mut stats, sink) {+ stopped = true;break;}}+ // Trailing partial window (also the whole-file case for short files).+ if !stopped {+ flush(&mut window, true, &mut stats, sink);+ }Ok(stats)}++#[cfg(test)]+mod chunk_tests {+ use super::*;+ use std::io::Write;++ fn run(lines: &[&str]) -> Vec<(String, u64, u64, String)> {+ let dir = std::env::temp_dir().join(format!("xerj-chunk-{}", lines.len()));+ std::fs::create_dir_all(&dir).unwrap();+ let p = dir.join("f.txt");+ let mut f = std::fs::File::create(&p).unwrap();+ for l in lines {+ writeln!(f, "{l}").unwrap();+ }+ drop(f);+ let mut out = Vec::new();+ {+ let mut sink = |r: RawRecord| {+ out.push((+ r.fields["text"].as_str().unwrap().to_string(),+ r.fields["start_line"].as_u64().unwrap(),+ r.fields["end_line"].as_u64().unwrap(),+ r.locator.clone(),+ ));+ true+ };+ extract_lines(&p, false, None, &mut sink).unwrap();+ }+ let _ = std::fs::remove_file(&p);+ out+ }++ #[test]+ fn short_file_is_one_chunk_covering_every_line() {+ let lines: Vec<String> = (1..=10).map(|i| format!("line {i}")).collect();+ let refs: Vec<&str> = lines.iter().map(|s| s.as_str()).collect();+ let out = run(&refs);+ assert_eq!(out.len(), 1);+ assert_eq!((out[0].1, out[0].2), (1, 10));+ assert!(out[0].0.contains("line 1") && out[0].0.contains("line 10"));+ }++ /// The property that fixed retrieval: consecutive chunks OVERLAP, so an+ /// answer straddling a boundary is findable from either side.+ #[test]+ fn consecutive_chunks_overlap_by_the_configured_amount() {+ let lines: Vec<String> = (1..=100).map(|i| format!("line {i}")).collect();+ let refs: Vec<&str> = lines.iter().map(|s| s.as_str()).collect();+ let out = run(&refs);+ assert!(out.len() >= 2, "100 lines must produce several chunks");+ for w in out.windows(2) {+ let (_, s0, e0, _) = &w[0];+ let (_, s1, _, _) = &w[1];+ assert!(s1 <= e0, "chunk {s1} must start at/before previous end {e0}");+ let repeated = e0 - s1 + 1;+ assert_eq!(+ repeated as usize, CHUNK_OVERLAP,+ "expected {CHUNK_OVERLAP} repeated lines across the boundary"+ );+ }+ }++ #[test]+ fn every_line_appears_in_some_chunk() {+ let lines: Vec<String> = (1..=95).map(|i| format!("unique{i}")).collect();+ let refs: Vec<&str> = lines.iter().map(|s| s.as_str()).collect();+ let out = run(&refs);+ for i in 1..=95 {+ let tok = format!("unique{i}");+ assert!(+ out.iter().any(|(t, _, _, _)| t.split('\n').any(|l| l == tok)),+ "line {i} lost"+ );+ }+ }++ #[test]+ fn line_numbers_are_one_based_and_ordered() {+ let lines: Vec<String> = (1..=60).map(|i| format!("l{i}")).collect();+ let refs: Vec<&str> = lines.iter().map(|s| s.as_str()).collect();+ let out = run(&refs);+ assert_eq!(out[0].1, 1, "first chunk starts at line 1, not 0");+ for (_, s, e, _) in &out {+ assert!(s <= e);+ }+ }++ /// A file of very long lines must not produce an unbounded record.+ #[test]+ fn chunk_is_byte_capped() {+ let long = "x".repeat(4000);+ let lines: Vec<&str> = (0..50).map(|_| long.as_str()).collect();+ let out = run(&lines);+ for (t, _, _, _) in &out {+ assert!(+ t.len() <= CHUNK_MAX_BYTES + 4001,+ "chunk of {} bytes exceeds the cap",+ t.len()+ );+ }+ }++ /// Blank lines are skipped but must not desynchronise the line numbering.+ #[test]+ fn blank_lines_do_not_break_numbering() {+ let out = run(&["alpha", "", "beta", "", "gamma"]);+ assert_eq!(out.len(), 1);+ assert_eq!(out[0].1, 1);+ assert_eq!(out[0].2, 5, "end_line must be the real file line of `gamma`");+ }+}diff --git a/engine/crates/xerj-autoindex/src/infer/mod.rs b/engine/crates/xerj-autoindex/src/infer/mod.rsindex 0c4225e47..e187dcf86 100644--- a/engine/crates/xerj-autoindex/src/infer/mod.rs+++ b/engine/crates/xerj-autoindex/src/infer/mod.rs@@ -15,6 +15,26 @@ pub const DISTINCT_CAP: usize = 8192;pub const RAW_CAP: usize = 8192;pub const MAX_FIELDS_PER_DATASET: usize = 512;+/// 256-slot byte histogram with a `Default` impl (`[u32; 256]` has none).+#[derive(Debug, Clone)]+pub struct ByteHist(pub [u32; 256]);+impl Default for ByteHist {+ fn default() -> Self {+ ByteHist([0u32; 256])+ }+}+impl std::ops::Deref for ByteHist {+ type Target = [u32; 256];+ fn deref(&self) -> &Self::Target {+ &self.0+ }+}+impl std::ops::DerefMut for ByteHist {+ fn deref_mut(&mut self) -> &mut Self::Target {+ &mut self.0+ }+}+#[derive(Debug, Default, Clone)]pub struct FieldAcc {pub n: u64, // non-null values seen@@ -31,7 +51,22 @@ pub struct FieldAcc {pub examples: Vec<String>,pub len_samples: Vec<u32>,pub token_samples: Vec<u32>,+ /// Word-shaped tokens (`[A-Za-z]{3,}`) vs total tokens, over the sample.+ /// This ratio is what separates natural language from opaque identifiers,+ /// and it does so far more reliably than any embedding-derived signal:+ /// measured on real fields, `word_ratio` was 0.00 for `trace_id`,+ /// `user_id`, `order_id` and numeric columns, and 0.78–1.00 for log+ /// messages, prose and source code. (An embedding-structure metric tried+ /// on the same fields ranked a numeric column ABOVE real log messages —+ /// embeddings measure similarity, not whether a field carries meaning.)+ pub word_tokens: u64,+ pub total_tokens: u64,pub len_sum: u64,+ /// Byte histogram over sampled string values, for Shannon entropy. High+ /// entropy with a low `word_ratio` is the signature of a hash / base64 /+ /// uuid column.+ pub byte_hist: ByteHist,+ pub byte_total: u64,pub entity: HashMap<Entity, u64>,pub int_min: i64,pub int_max: i64,@@ -40,6 +75,50 @@ pub struct FieldAcc {}impl FieldAcc {+ /// Fraction of sampled tokens that look like words. ~0 for identifiers,+ /// numbers and hashes; high for prose, log messages and source code.+ pub fn word_ratio(&self) -> f64 {+ if self.total_tokens == 0 {+ return 0.0;+ }+ self.word_tokens as f64 / self.total_tokens as f64+ }++ /// Mean whitespace-token count per sampled value — distinguishes a+ /// multi-word body from a single-token enum or code.+ pub fn mean_tokens(&self) -> f64 {+ if self.token_samples.is_empty() {+ return 0.0;+ }+ self.token_samples.iter().map(|t| *t as f64).sum::<f64>() / self.token_samples.len() as f64+ }++ /// Shannon entropy (bits/byte) over sampled string bytes.+ pub fn char_entropy(&self) -> f64 {+ if self.byte_total == 0 {+ return 0.0;+ }+ let tot = self.byte_total as f64;+ -self+ .byte_hist+ .iter()+ .filter(|c| **c > 0)+ .map(|c| {+ let p = *c as f64 / tot;+ p * p.log2()+ })+ .sum::<f64>()+ }++ /// Is this field worth embedding? Natural language only: mostly+ /// word-shaped tokens AND more than a couple of tokens per value.+ /// Deliberately conservative — embedding an identifier column is pure+ /// cost (the built-in neural backend runs at ~3 docs/s) and produces a+ /// vector space with no useful neighbourhoods.+ pub fn looks_natural_language(&self) -> bool {+ self.word_ratio() >= 0.55 && self.mean_tokens() >= 3.0+ }+pub fn add(&mut self, v: &Value) {match v {Value::Null => {}@@ -86,6 +165,23 @@ impl FieldAcc {*self.date_hits.entry(enc).or_default() += 1;self.track_date(dt);}+ // Lexical shape — cheap, deterministic, and computed from the+ // same sample everything else uses.+ for tok in t.split(|c: char| !c.is_alphanumeric()) {+ if tok.is_empty() {+ continue;+ }+ self.total_tokens += 1;+ if tok.len() >= 3 && tok.chars().all(|c| c.is_ascii_alphabetic()) {+ self.word_tokens += 1;+ }+ }+ if self.byte_total < 1 << 20 {+ for b in t.bytes() {+ self.byte_hist[b as usize] += 1;+ self.byte_total += 1;+ }+ }if let Some(e) = entities::classify(t) {*self.entity.entry(e).or_default() += 1;}@@ -400,18 +496,44 @@ pub fn infer_fields(}}- // semantic body election: largest avg_len text field ≥ 200 chars+ // Semantic body election.+ //+ // Previously: "largest avg_len text field >= 200 chars". That is a proxy+ // for "is this natural language" and it is wrong in both directions — a+ // 300-char base64 blob or a concatenated id column qualifies, while a+ // genuinely semantic 150-char summary field does not. Embedding the wrong+ // column is expensive (the built-in neural backend measures ~3 docs/s) and+ // yields a vector space with no useful neighbourhoods.+ //+ // Now: require the field to actually look like natural language+ // (`word_ratio >= 0.55 && mean_tokens >= 3`), then pick the longest such+ // field. Measured `word_ratio` on real columns: 0.00 for trace_id /+ // user_id / order_id / numerics, 0.78-1.00 for prose, log messages and+ // source code. The length floor is kept but relaxed, because the+ // language test now does the discriminating.if !no_semantic {let best = specs.iter().enumerate()- .filter(|(_, s)| s.es_type == "text" && s.avg_len >= 200.0)+ .filter(|(i, s)| {+ s.es_type == "text"+ && s.avg_len >= 80.0+ && fields+ .get(&s.name)+ .map(|a| a.looks_natural_language())+ .unwrap_or(false)+ }).max_by(|a, b| a.1.avg_len.partial_cmp(&b.1.avg_len).unwrap());if let Some((i, _)) = best {+ let a = fields.get(&specs[i].name);specs[i].es_type = "semantic_text".into();- specs[i].notes.push(- "hybrid lexical+vector body (embedded server-side: lexical by default, neural/proxy if configured)".into(),- );+ specs[i].notes.push(format!(+ "hybrid lexical+vector body — elected because it looks like natural language \+ (word_ratio {:.2}, {:.1} tokens/value); embedded server-side (lexical by \+ default, neural/proxy if configured)",+ a.map(|x| x.word_ratio()).unwrap_or(0.0),+ a.map(|x| x.mean_tokens()).unwrap_or(0.0),+ ));}}