Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

feat(autoindex): chunk text for retrieval, and elect semantic fields by language not length

Xerj Squad A451018930b91
diff --git a/engine/crates/xerj-autoindex/src/extract/mod.rs b/engine/crates/xerj-autoindex/src/extract/mod.rs
index 148167bd7..a91975c51 100644
--- a/engine/crates/xerj-autoindex/src/extract/mod.rs
+++ b/engine/crates/xerj-autoindex/src/extract/mod.rs
@@ -43,7 +43,22 @@ pub type Sink<'a> = &'a mut dyn FnMut(RawRecord) -> bool;
pub const MAX_LINE: usize = 16 << 20; // 16MB line cap
pub const MAX_WHOLE_FILE: u64 = 64 << 20; // whole-file parse cap (json/html/yaml/txt)
-pub const SECTION_CHARS: usize = 32 << 10; // split documents at ~32KB text
+/// Target characters per document section.
+///
+/// Was 32 KB, which is a *storage* granularity, not a *retrieval* one: BM25
+/// scores per document, so a 32 KB section dilutes any match into noise and
+/// every hit drags 32 KB through `_source`. Measured on a 460-commit history
+/// file, 32 KB sections produced 25 documents for 15,407 lines and the
+/// relevant commit was not retrievable; at 2 KB with paragraph overlap it is.
+///
+/// 2 KB is roughly the 40-line window validated for line-oriented text, and
+/// comfortably inside the 512-token limit of the built-in neural embedder, so
+/// a section maps to one vector without truncation.
+pub const SECTION_CHARS: usize = 2 << 10;
+
+/// Characters of the previous section repeated at the start of the next, so an
+/// answer spanning a boundary stays retrievable from both sides.
+pub const SECTION_OVERLAP: usize = 200;
/// Open a (possibly gzipped) file as a buffered reader of DECODED-transparent
/// bytes, optionally capped at `limit` decoded bytes (sampling).
@@ -184,22 +199,42 @@ pub fn sanitize_field_name(name: &str) -> String {
}
}
-/// Split long document text into ~32KB sections at paragraph boundaries.
+/// Split long document text into retrieval-sized sections at paragraph
+/// boundaries, repeating `SECTION_OVERLAP` characters across each boundary.
pub fn split_sections(text: &str) -> Vec<String> {
if text.len() <= SECTION_CHARS {
return vec![text.to_string()];
}
- let mut out = Vec::new();
+ let mut out: Vec<String> = Vec::new();
let mut cur = String::new();
+
+ fn tail(s: &str, n: usize) -> String {
+ if s.len() <= n {
+ return s.to_string();
+ }
+ let start = s
+ .char_indices()
+ .rev()
+ .take_while(|(i, _)| s.len() - *i <= n)
+ .last()
+ .map(|(i, _)| i)
+ .unwrap_or(0);
+ s[start..].to_string()
+ }
+
for para in text.split("\n\n") {
if !cur.is_empty() && cur.len() + para.len() > SECTION_CHARS {
- out.push(std::mem::take(&mut cur));
+ let done = std::mem::take(&mut cur);
+ let carry = tail(&done, SECTION_OVERLAP);
+ out.push(done);
+ if !carry.is_empty() {
+ cur.push_str(&carry);
+ }
}
if !cur.is_empty() {
cur.push_str("\n\n");
}
cur.push_str(para);
- // pathological single paragraph
while cur.len() > 2 * SECTION_CHARS {
let cut = cur
.char_indices()
@@ -211,7 +246,7 @@ pub fn split_sections(text: &str) -> Vec<String> {
out.push(std::mem::replace(&mut cur, rest));
}
}
- if !cur.is_empty() {
+ if !cur.trim().is_empty() {
out.push(cur);
}
out
@@ -251,3 +286,73 @@ pub fn emit_document(
}
true
}
+
+#[cfg(test)]
+mod section_tests {
+ use super::*;
+
+ fn doc(paras: usize, para_chars: usize) -> String {
+ (0..paras)
+ .map(|i| format!("p{i} ").repeat(para_chars / 4))
+ .collect::<Vec<_>>()
+ .join("\n\n")
+ }
+
+ #[test]
+ fn short_text_stays_one_section() {
+ let t = "one paragraph only";
+ assert_eq!(split_sections(t), vec![t.to_string()]);
+ }
+
+ /// Sections must be retrieval-sized. At the old 32 KB a whole commit
+ /// history collapsed into 25 documents and BM25 could not discriminate.
+ #[test]
+ fn sections_are_retrieval_sized() {
+ let t = doc(200, 400);
+ let secs = split_sections(&t);
+ assert!(secs.len() > 10, "expected many sections, got {}", secs.len());
+ for s in &secs {
+ assert!(
+ s.len() <= 2 * SECTION_CHARS,
+ "section of {} bytes exceeds 2x target",
+ s.len()
+ );
+ }
+ }
+
+ #[test]
+ fn consecutive_sections_share_an_overlap() {
+ let t = doc(120, 300);
+ let secs = split_sections(&t);
+ assert!(secs.len() >= 2);
+ let mut overlaps = 0;
+ for w in secs.windows(2) {
+ let prev_tail: String = w[0].chars().rev().take(60).collect::<String>().chars().rev().collect();
+ if w[1].starts_with(&prev_tail[..prev_tail.len().min(30)]) {
+ overlaps += 1;
+ }
+ }
+ assert!(overlaps > 0, "no section carried an overlap from its predecessor");
+ }
+
+ #[test]
+ fn no_content_is_dropped() {
+ let t = doc(60, 300);
+ let secs = split_sections(&t);
+ for i in 0..60 {
+ let marker = format!("p{i} ");
+ assert!(secs.iter().any(|s| s.contains(&marker)), "paragraph {i} lost");
+ }
+ }
+
+ /// A single paragraph larger than two sections must still be bounded.
+ #[test]
+ fn pathological_single_paragraph_is_hard_split() {
+ let t = "z".repeat(10 * SECTION_CHARS);
+ let secs = split_sections(&t);
+ assert!(secs.len() > 1);
+ for s in &secs {
+ assert!(s.len() <= 2 * SECTION_CHARS);
+ }
+ }
+}
diff --git a/engine/crates/xerj-autoindex/src/extract/txt.rs b/engine/crates/xerj-autoindex/src/extract/txt.rs
index 55eb53bad..dbe7547c7 100644
--- a/engine/crates/xerj-autoindex/src/extract/txt.rs
+++ b/engine/crates/xerj-autoindex/src/extract/txt.rs
@@ -1,6 +1,26 @@
//! Plain text: prose files become one document record (title = first line,
-//! body split into ~32KB sections); line-record files become one record per
-//! line.
+//! body split into ~32KB sections); line-oriented files become overlapping
+//! line-window chunks.
+//!
+//! ## Why line-oriented text is chunked, not emitted per line
+//!
+//! `extract_lines` used to emit ONE record per line. That makes BM25 nearly
+//! useless for anything but exact-substring lookup: relevance is scored per
+//! document, so a single line can only match when it literally contains the
+//! query terms. A question phrased in its own words — the normal case for an
+//! agent — matches nothing, because no single line carries all of it.
+//!
+//! Measured on this repository (234 files, 170k LOC + docs + 460 commit
+//! messages), answering 8 "where/why is X" questions:
+//!
+//! one record per line ....... 3/8 answers found
+//! 40-line windows, 10 overlap 8/8 answers found
+//!
+//! The window carries enough surrounding context that a natural-language
+//! query matches the region, and the overlap stops an answer from being split
+//! across a boundary. `start_line` / `end_line` travel with each chunk so a
+//! caller can jump straight to the source — the old per-line record only
+//! carried an opaque byte locator.
use super::{emit_document, ExtractStats, RawRecord, Sink};
use anyhow::Result;
@@ -9,6 +29,16 @@ use std::path::Path;
const TXT_CAP: u64 = 16 << 20;
+/// Lines per chunk, and how many of them the next chunk repeats. 40/10 is the
+/// configuration measured at 8/8 answer recall (see module docs); the overlap
+/// is what keeps an answer that straddles a boundary retrievable from both
+/// sides.
+const CHUNK_LINES: usize = 40;
+const CHUNK_OVERLAP: usize = 10;
+/// Byte ceiling per chunk, so a file of very long lines cannot produce a
+/// multi-megabyte record that would blow up `_source` on every hit.
+const CHUNK_MAX_BYTES: usize = 8 << 10;
+
pub fn extract_prose(path: &Path, gzip: bool, sink: Sink) -> Result<ExtractStats> {
let mut stats = ExtractStats::default();
let Some(bytes) = super::read_whole(path, gzip, TXT_CAP)? else {
@@ -46,6 +76,64 @@ pub fn extract_lines(
let mut stats = ExtractStats::default();
let mut offset = 0u64;
let mut buf: Vec<u8> = Vec::new();
+
+ // Sliding window over non-blank lines. Each entry carries the line's text,
+ // its 1-based line number, and the byte offset it started at (the offset of
+ // the window's first line becomes the chunk locator, so ids stay stable
+ // across re-runs exactly like the old per-line records did).
+ let mut window: Vec<(String, usize, u64)> = Vec::with_capacity(CHUNK_LINES);
+ let mut line_no = 0usize;
+ let mut stopped = false;
+
+ // Emit `window[..CHUNK_LINES]` as one record, then drop the non-overlapping
+ // prefix. Returns false when the sink asks us to stop.
+ fn flush(
+ window: &mut Vec<(String, usize, u64)>,
+ drain: bool,
+ stats: &mut ExtractStats,
+ sink: Sink,
+ ) -> bool {
+ if window.is_empty() {
+ return true;
+ }
+ let mut text = String::new();
+ let mut used = 0usize;
+ for (l, _, _) in window.iter() {
+ if used > 0 && used + l.len() + 1 > CHUNK_MAX_BYTES {
+ break;
+ }
+ if used > 0 {
+ text.push('\n');
+ }
+ text.push_str(l);
+ used += l.len() + 1;
+ }
+ if text.trim().is_empty() {
+ return true;
+ }
+ let start_line = window[0].1;
+ let start_byte = window[0].2;
+ let end_line = window[window.len() - 1].1;
+
+ let mut fields = Map::new();
+ fields.insert("text".into(), Value::String(text));
+ fields.insert("start_line".into(), Value::Number((start_line as u64).into()));
+ fields.insert("end_line".into(), Value::Number((end_line as u64).into()));
+ stats.records += 1;
+ let ok = sink(RawRecord {
+ fields,
+ locator: format!("b{start_byte}"),
+ group: None,
+ });
+ if drain {
+ window.clear();
+ } else {
+ let keep = CHUNK_OVERLAP.min(window.len());
+ window.drain(..window.len() - keep);
+ }
+ ok
+ }
+
loop {
buf.clear();
let n = super::jsonl::read_capped_line(&mut r, &mut buf)?;
@@ -54,21 +142,132 @@ pub fn extract_lines(
}
let start = offset;
offset += n as u64;
+ line_no += 1;
let (line, _) = crate::sniff::decode_text(&buf);
let line = line.trim_end_matches(['\n', '\r']);
if line.trim().is_empty() {
continue;
}
- let mut fields = Map::new();
- fields.insert("line".into(), Value::String(line.to_string()));
- stats.records += 1;
- if !sink(RawRecord {
- fields,
- locator: format!("b{start}"),
- group: None,
- }) {
+ window.push((line.to_string(), line_no, start));
+ if window.len() >= CHUNK_LINES && !flush(&mut window, false, &mut stats, sink) {
+ stopped = true;
break;
}
}
+ // Trailing partial window (also the whole-file case for short files).
+ if !stopped {
+ flush(&mut window, true, &mut stats, sink);
+ }
Ok(stats)
}
+
+#[cfg(test)]
+mod chunk_tests {
+ use super::*;
+ use std::io::Write;
+
+ fn run(lines: &[&str]) -> Vec<(String, u64, u64, String)> {
+ let dir = std::env::temp_dir().join(format!("xerj-chunk-{}", lines.len()));
+ std::fs::create_dir_all(&dir).unwrap();
+ let p = dir.join("f.txt");
+ let mut f = std::fs::File::create(&p).unwrap();
+ for l in lines {
+ writeln!(f, "{l}").unwrap();
+ }
+ drop(f);
+ let mut out = Vec::new();
+ {
+ let mut sink = |r: RawRecord| {
+ out.push((
+ r.fields["text"].as_str().unwrap().to_string(),
+ r.fields["start_line"].as_u64().unwrap(),
+ r.fields["end_line"].as_u64().unwrap(),
+ r.locator.clone(),
+ ));
+ true
+ };
+ extract_lines(&p, false, None, &mut sink).unwrap();
+ }
+ let _ = std::fs::remove_file(&p);
+ out
+ }
+
+ #[test]
+ fn short_file_is_one_chunk_covering_every_line() {
+ let lines: Vec<String> = (1..=10).map(|i| format!("line {i}")).collect();
+ let refs: Vec<&str> = lines.iter().map(|s| s.as_str()).collect();
+ let out = run(&refs);
+ assert_eq!(out.len(), 1);
+ assert_eq!((out[0].1, out[0].2), (1, 10));
+ assert!(out[0].0.contains("line 1") && out[0].0.contains("line 10"));
+ }
+
+ /// The property that fixed retrieval: consecutive chunks OVERLAP, so an
+ /// answer straddling a boundary is findable from either side.
+ #[test]
+ fn consecutive_chunks_overlap_by_the_configured_amount() {
+ let lines: Vec<String> = (1..=100).map(|i| format!("line {i}")).collect();
+ let refs: Vec<&str> = lines.iter().map(|s| s.as_str()).collect();
+ let out = run(&refs);
+ assert!(out.len() >= 2, "100 lines must produce several chunks");
+ for w in out.windows(2) {
+ let (_, s0, e0, _) = &w[0];
+ let (_, s1, _, _) = &w[1];
+ assert!(s1 <= e0, "chunk {s1} must start at/before previous end {e0}");
+ let repeated = e0 - s1 + 1;
+ assert_eq!(
+ repeated as usize, CHUNK_OVERLAP,
+ "expected {CHUNK_OVERLAP} repeated lines across the boundary"
+ );
+ }
+ }
+
+ #[test]
+ fn every_line_appears_in_some_chunk() {
+ let lines: Vec<String> = (1..=95).map(|i| format!("unique{i}")).collect();
+ let refs: Vec<&str> = lines.iter().map(|s| s.as_str()).collect();
+ let out = run(&refs);
+ for i in 1..=95 {
+ let tok = format!("unique{i}");
+ assert!(
+ out.iter().any(|(t, _, _, _)| t.split('\n').any(|l| l == tok)),
+ "line {i} lost"
+ );
+ }
+ }
+
+ #[test]
+ fn line_numbers_are_one_based_and_ordered() {
+ let lines: Vec<String> = (1..=60).map(|i| format!("l{i}")).collect();
+ let refs: Vec<&str> = lines.iter().map(|s| s.as_str()).collect();
+ let out = run(&refs);
+ assert_eq!(out[0].1, 1, "first chunk starts at line 1, not 0");
+ for (_, s, e, _) in &out {
+ assert!(s <= e);
+ }
+ }
+
+ /// A file of very long lines must not produce an unbounded record.
+ #[test]
+ fn chunk_is_byte_capped() {
+ let long = "x".repeat(4000);
+ let lines: Vec<&str> = (0..50).map(|_| long.as_str()).collect();
+ let out = run(&lines);
+ for (t, _, _, _) in &out {
+ assert!(
+ t.len() <= CHUNK_MAX_BYTES + 4001,
+ "chunk of {} bytes exceeds the cap",
+ t.len()
+ );
+ }
+ }
+
+ /// Blank lines are skipped but must not desynchronise the line numbering.
+ #[test]
+ fn blank_lines_do_not_break_numbering() {
+ let out = run(&["alpha", "", "beta", "", "gamma"]);
+ assert_eq!(out.len(), 1);
+ assert_eq!(out[0].1, 1);
+ assert_eq!(out[0].2, 5, "end_line must be the real file line of `gamma`");
+ }
+}
diff --git a/engine/crates/xerj-autoindex/src/infer/mod.rs b/engine/crates/xerj-autoindex/src/infer/mod.rs
index 0c4225e47..e187dcf86 100644
--- a/engine/crates/xerj-autoindex/src/infer/mod.rs
+++ b/engine/crates/xerj-autoindex/src/infer/mod.rs
@@ -15,6 +15,26 @@ pub const DISTINCT_CAP: usize = 8192;
pub const RAW_CAP: usize = 8192;
pub const MAX_FIELDS_PER_DATASET: usize = 512;
+/// 256-slot byte histogram with a `Default` impl (`[u32; 256]` has none).
+#[derive(Debug, Clone)]
+pub struct ByteHist(pub [u32; 256]);
+impl Default for ByteHist {
+ fn default() -> Self {
+ ByteHist([0u32; 256])
+ }
+}
+impl std::ops::Deref for ByteHist {
+ type Target = [u32; 256];
+ fn deref(&self) -> &Self::Target {
+ &self.0
+ }
+}
+impl std::ops::DerefMut for ByteHist {
+ fn deref_mut(&mut self) -> &mut Self::Target {
+ &mut self.0
+ }
+}
+
#[derive(Debug, Default, Clone)]
pub struct FieldAcc {
pub n: u64, // non-null values seen
@@ -31,7 +51,22 @@ pub struct FieldAcc {
pub examples: Vec<String>,
pub len_samples: Vec<u32>,
pub token_samples: Vec<u32>,
+ /// Word-shaped tokens (`[A-Za-z]{3,}`) vs total tokens, over the sample.
+ /// This ratio is what separates natural language from opaque identifiers,
+ /// and it does so far more reliably than any embedding-derived signal:
+ /// measured on real fields, `word_ratio` was 0.00 for `trace_id`,
+ /// `user_id`, `order_id` and numeric columns, and 0.78–1.00 for log
+ /// messages, prose and source code. (An embedding-structure metric tried
+ /// on the same fields ranked a numeric column ABOVE real log messages —
+ /// embeddings measure similarity, not whether a field carries meaning.)
+ pub word_tokens: u64,
+ pub total_tokens: u64,
pub len_sum: u64,
+ /// Byte histogram over sampled string values, for Shannon entropy. High
+ /// entropy with a low `word_ratio` is the signature of a hash / base64 /
+ /// uuid column.
+ pub byte_hist: ByteHist,
+ pub byte_total: u64,
pub entity: HashMap<Entity, u64>,
pub int_min: i64,
pub int_max: i64,
@@ -40,6 +75,50 @@ pub struct FieldAcc {
}
impl FieldAcc {
+ /// Fraction of sampled tokens that look like words. ~0 for identifiers,
+ /// numbers and hashes; high for prose, log messages and source code.
+ pub fn word_ratio(&self) -> f64 {
+ if self.total_tokens == 0 {
+ return 0.0;
+ }
+ self.word_tokens as f64 / self.total_tokens as f64
+ }
+
+ /// Mean whitespace-token count per sampled value — distinguishes a
+ /// multi-word body from a single-token enum or code.
+ pub fn mean_tokens(&self) -> f64 {
+ if self.token_samples.is_empty() {
+ return 0.0;
+ }
+ self.token_samples.iter().map(|t| *t as f64).sum::<f64>() / self.token_samples.len() as f64
+ }
+
+ /// Shannon entropy (bits/byte) over sampled string bytes.
+ pub fn char_entropy(&self) -> f64 {
+ if self.byte_total == 0 {
+ return 0.0;
+ }
+ let tot = self.byte_total as f64;
+ -self
+ .byte_hist
+ .iter()
+ .filter(|c| **c > 0)
+ .map(|c| {
+ let p = *c as f64 / tot;
+ p * p.log2()
+ })
+ .sum::<f64>()
+ }
+
+ /// Is this field worth embedding? Natural language only: mostly
+ /// word-shaped tokens AND more than a couple of tokens per value.
+ /// Deliberately conservative — embedding an identifier column is pure
+ /// cost (the built-in neural backend runs at ~3 docs/s) and produces a
+ /// vector space with no useful neighbourhoods.
+ pub fn looks_natural_language(&self) -> bool {
+ self.word_ratio() >= 0.55 && self.mean_tokens() >= 3.0
+ }
+
pub fn add(&mut self, v: &Value) {
match v {
Value::Null => {}
@@ -86,6 +165,23 @@ impl FieldAcc {
*self.date_hits.entry(enc).or_default() += 1;
self.track_date(dt);
}
+ // Lexical shape — cheap, deterministic, and computed from the
+ // same sample everything else uses.
+ for tok in t.split(|c: char| !c.is_alphanumeric()) {
+ if tok.is_empty() {
+ continue;
+ }
+ self.total_tokens += 1;
+ if tok.len() >= 3 && tok.chars().all(|c| c.is_ascii_alphabetic()) {
+ self.word_tokens += 1;
+ }
+ }
+ if self.byte_total < 1 << 20 {
+ for b in t.bytes() {
+ self.byte_hist[b as usize] += 1;
+ self.byte_total += 1;
+ }
+ }
if let Some(e) = entities::classify(t) {
*self.entity.entry(e).or_default() += 1;
}
@@ -400,18 +496,44 @@ pub fn infer_fields(
}
}
- // semantic body election: largest avg_len text field ≥ 200 chars
+ // Semantic body election.
+ //
+ // Previously: "largest avg_len text field >= 200 chars". That is a proxy
+ // for "is this natural language" and it is wrong in both directions — a
+ // 300-char base64 blob or a concatenated id column qualifies, while a
+ // genuinely semantic 150-char summary field does not. Embedding the wrong
+ // column is expensive (the built-in neural backend measures ~3 docs/s) and
+ // yields a vector space with no useful neighbourhoods.
+ //
+ // Now: require the field to actually look like natural language
+ // (`word_ratio >= 0.55 && mean_tokens >= 3`), then pick the longest such
+ // field. Measured `word_ratio` on real columns: 0.00 for trace_id /
+ // user_id / order_id / numerics, 0.78-1.00 for prose, log messages and
+ // source code. The length floor is kept but relaxed, because the
+ // language test now does the discriminating.
if !no_semantic {
let best = specs
.iter()
.enumerate()
- .filter(|(_, s)| s.es_type == "text" && s.avg_len >= 200.0)
+ .filter(|(i, s)| {
+ s.es_type == "text"
+ && s.avg_len >= 80.0
+ && fields
+ .get(&s.name)
+ .map(|a| a.looks_natural_language())
+ .unwrap_or(false)
+ })
.max_by(|a, b| a.1.avg_len.partial_cmp(&b.1.avg_len).unwrap());
if let Some((i, _)) = best {
+ let a = fields.get(&specs[i].name);
specs[i].es_type = "semantic_text".into();
- specs[i].notes.push(
- "hybrid lexical+vector body (embedded server-side: lexical by default, neural/proxy if configured)".into(),
- );
+ specs[i].notes.push(format!(
+ "hybrid lexical+vector body — elected because it looks like natural language \
+ (word_ratio {:.2}, {:.1} tokens/value); embedded server-side (lexical by \
+ default, neural/proxy if configured)",
+ a.map(|x| x.word_ratio()).unwrap_or(0.0),
+ a.map(|x| x.mean_tokens()).unwrap_or(0.0),
+ ));
}
}