AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
Revert #1623 + #1643 (thrash-era regressions) to unblock a red main (#1730)
xerj-org46f6b0c2aede
diff --git a/engine/crates/xerj-autoindex/src/cli.rs b/engine/crates/xerj-autoindex/src/cli.rsindex 1beb0a4df..ac9e91bb3 100644--- a/engine/crates/xerj-autoindex/src/cli.rs+++ b/engine/crates/xerj-autoindex/src/cli.rs@@ -629,9 +629,7 @@ pub fn help_text_with(feedback: bool) -> String {\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20journal committed; only the catalog's duplicate-alias cleanup\n\\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20could not run. Read `reason` on the xerj-done line to tell the\n\\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20two apart, and rerun the same command once the reported server\n\- \x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20condition clears. ok=true with exit=3 is success: the next\n\- \x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20line is a xerj-note that says so (--progress json sets\n\- \x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20exit_meaning on the done object);\n\+ \x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20condition clears;\n\\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x204 NEEDS A DECISION — the estimate exceeded --max-minutes, or a\n\\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20--watch poll cost more than --max-monthly-ops; nothing was\n\\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20indexed and a JSON decision request is on stdout;\n\diff --git a/engine/crates/xerj-autoindex/src/extract/mod.rs b/engine/crates/xerj-autoindex/src/extract/mod.rsindex 4ad292db8..7e3162691 100644--- a/engine/crates/xerj-autoindex/src/extract/mod.rs+++ b/engine/crates/xerj-autoindex/src/extract/mod.rs@@ -117,12 +117,6 @@ pub const MAX_RECORDS_PER_FILE: usize = 4 << 10; // 4096/// answer spanning a boundary stays retrievable from both sides.pub const SECTION_OVERLAP: usize = 200;-/// How far a hard split may move back from `SECTION_CHARS` to land on-/// whitespace. A run with no whitespace inside the window keeps the char-/// boundary, so multi-byte scalars and unbroken tokens are never severed-/// just to chase a word that is not there.-const WORD_SNAP_BYTES: usize = 64;-/// Open a (possibly gzipped) file as a buffered reader of DECODED-transparent/// bytes, optionally capped at `limit` decoded bytes (sampling).pub fn open_reader(path: &Path, gzip: bool, limit: Option<u64>) -> Result<Box<dyn BufRead + Send>> {@@ -189,10 +183,7 @@ pub fn extract_as_document_with_name(.file_stem().map(|s| s.to_string_lossy().to_string()).unwrap_or_else(|| "untitled".into());- let skipped = text.len() - text.trim_start().len();- let skipped_lines = text[..skipped].bytes().filter(|b| *b == b'\n').count() as u64;- emit_document_from(- skipped_lines,+ emit_document(&title,&[],text.trim(),@@ -594,34 +585,6 @@ fn floor_boundary(s: &str, i: usize) -> usize {i}-/// Cut `cur` near `target` bytes. Prefer the nearest whitespace at or before-/// `target` within [`WORD_SNAP_BYTES`], and cut *after* that whitespace so-/// the next section starts on a word. No whitespace in the window → the char-/// boundary at `target`, which never splits a scalar.-fn cut_on_word_or_char(cur: &str, target: usize) -> usize {- let boundary = ceil_boundary(cur, target.min(cur.len()));- if boundary == 0 || boundary >= cur.len() {- return boundary;- }- let look = floor_boundary(cur, boundary.saturating_sub(WORD_SNAP_BYTES));- let window = &cur[look..boundary];- if let Some(rel) = window.rfind(char::is_whitespace) {- let at = look + rel;- let ws_len = cur[at..].chars().next().map(|c| c.len_utf8()).unwrap_or(1);- let after = at + ws_len;- if after > 0 && after < cur.len() {- return after;- }- }- boundary-}--/// 1-based line number of `text` at byte offset `byte`.-fn line_number_at(text: &str, byte: usize) -> u64 {- let byte = byte.min(text.len());- 1 + text[..byte].bytes().filter(|b| *b == b'\n').count() as u64-}-/// Top up `cur` from the front of `rest` until it holds `want` bytes, moving/// whole chars only. Guarantees progress: if `want - cur.len()` lands inside/// the next char, that whole char moves anyway.@@ -655,23 +618,13 @@ fn fill_window(cur: &mut String, rest: &mut &str, want: usize) {/// that never exceeds `2 * SECTION_CHARS + 4` bytes, and the paragraph itself/// stays borrowed from `text`.pub fn for_each_section(text: &str, emit: &mut dyn FnMut(String) -> bool) -> bool {- for_each_section_spans(text, &mut |_at, section| emit(section))-}--/// [`for_each_section`] that also reports the byte offset in `text` where each-/// section begins. Paragraph overlap is included: a section that repeats the-/// previous tail starts at that tail.-fn for_each_section_spans(text: &str, emit: &mut dyn FnMut(usize, String) -> bool) -> bool {if text.len() <= SECTION_CHARS {- return emit(0, text.to_string());+ return emit(text.to_string());}// One byte past the largest section the loop below will ever hold, so a// full window always trips the hard-split condition.let window = 2 * SECTION_CHARS + 1;let mut cur = String::new();- // Byte offset in `text` of `cur`'s first byte once `cur` is non-empty.- let mut origin = 0usize;- let mut pos = 0usize;for para in text.split("\n\n") {// `cur.len() > SECTION_OVERLAP`: a shorter section would be carried@@ -681,20 +634,16 @@ fn for_each_section_spans(text: &str, emit: &mut dyn FnMut(usize, String) -> booif !cur.is_empty() && cur.len() + para.len() > SECTION_CHARS && cur.len() > SECTION_OVERLAP{let done = std::mem::take(&mut cur);- let start = origin;let carry = tail(&done, SECTION_OVERLAP);- origin = start + done.len() - carry.len();+ if !emit(done) {+ return false;+ }if !carry.is_empty() {cur.push_str(&carry);}- if !emit(start, done) {- return false;- }}if !cur.is_empty() {cur.push_str("\n\n");- } else {- origin = pos;}// `rest` is the not-yet-buffered remainder of this paragraph, borrowed// from `text` — it is never copied wholesale into `cur`.@@ -706,23 +655,16 @@ fn for_each_section_spans(text: &str, emit: &mut dyn FnMut(usize, String) -> booif cur.len() + rest.len() <= 2 * SECTION_CHARS {break;}- let cut = cut_on_word_or_char(&cur, SECTION_CHARS);+ let cut = ceil_boundary(&cur, SECTION_CHARS);let section = cur[..cut].to_string();- let start = origin;cur.drain(..cut);- origin += cut;- if !emit(start, section) {+ if !emit(section) {return false;}}- pos += para.len();- if pos < text.len() {- // The `\n\n` separator `split` consumed.- pos += 2;- }}if !cur.trim().is_empty() {- return emit(origin, cur);+ return emit(cur);}true}@@ -744,7 +686,6 @@ fn section_record(i: usize,multi: bool,section: String,- start_line: u64,) -> RawRecord {let mut fields = Map::new();fields.insert("title".into(), Value::String(title.to_string()));@@ -758,13 +699,12 @@ fn section_record(fields.insert("section".into(), Value::Number((i as u64).into()));}fields.insert("body".into(), Value::String(section));- fields.insert("start_line".into(), Value::Number(start_line.into()));RawRecord {fields,locator: format!("s{i}"),group: None,- // title/headings/section/body/start_line are this function's vocabulary,- // and `headings`/`section` come and go — never a clustering key.+ // title/headings/section/body are this function's vocabulary, and+ // `headings`/`section` come and go — never a clustering key.origin: FieldOrigin::Extractor,}}@@ -780,30 +720,14 @@ pub fn emit_document(max_records: usize,sink: Sink,stats: &mut ExtractStats,-) -> bool {- emit_document_from(0, title, headings, body, max_records, sink, stats)-}--/// [`emit_document`] whose `start_line` values are shifted by `skipped_lines`.-///-/// Callers that strip leading whitespace before sectioning (prose) pass the-/// number of newlines they removed, so `start_line` stays a file line.-pub(crate) fn emit_document_from(- skipped_lines: u64,- title: &str,- headings: &[String],- body: &str,- max_records: usize,- sink: Sink,- stats: &mut ExtractStats,) -> bool {// One section of lookahead: the `section` field is only present when the// document actually splits, which is not known until a second one arrives.- let mut pending: Option<(usize, String)> = None;+ let mut pending: Option<String> = None;let mut next: usize = 0;let mut alive = true;- for_each_section_spans(body, &mut |at, sec| {- let Some((prev_at, prev)) = pending.replace((at, sec)) else {+ for_each_section(body, &mut |sec| {+ let Some(prev) = pending.replace(sec) else {return true;};// #381: bound noise from a misclassified payload. Stopping the split@@ -815,8 +739,7 @@ pub(crate) fn emit_document_from(let i = next;next += 1;stats.records += 1;- let line = skipped_lines + line_number_at(body, prev_at);- alive = sink(section_record(title, headings, i, true, prev, line));+ alive = sink(section_record(title, headings, i, true, prev));alive});// A sink that stopped early (sampling) sets `alive = false`; the cap leaves@@ -829,14 +752,13 @@ pub(crate) fn emit_document_from(return true;}match pending {- Some((at, last)) => {+ Some(last) => {if next >= max_records {stats.truncated = true;return true;}stats.records += 1;- let line = skipped_lines + line_number_at(body, at);- sink(section_record(title, headings, next, next > 0, last, line))+ sink(section_record(title, headings, next, next > 0, last))}None => true,}@@ -859,9 +781,9 @@ pub(crate) fn emit_document_with_fields(// Collect sections first so we know whether to stamp a `section` field.// One past the cap is collected so that "exactly at the cap" and "over the// cap" are distinguishable: only the latter dropped anything (#381).- let mut secs: Vec<(usize, String)> = Vec::new();- for_each_section_spans(body, &mut |at, s| {- secs.push((at, s));+ let mut secs: Vec<String> = Vec::new();+ for_each_section(body, &mut |s| {+ secs.push(s);secs.len() <= MAX_RECORDS_PER_FILE});if secs.len() > MAX_RECORDS_PER_FILE {@@ -869,17 +791,13 @@ pub(crate) fn emit_document_with_fields(stats.truncated = true;}if secs.is_empty() {- secs.push((0, String::new()));+ secs.push(String::new());}let multi = secs.len() > 1;- for (i, (at, sec)) in secs.into_iter().enumerate() {+ for (i, sec) in secs.into_iter().enumerate() {let mut fields = base_fields.clone();fields.insert("title".into(), Value::String(title.to_string()));fields.insert("body".into(), Value::String(sec));- fields.insert(- "start_line".into(),- Value::Number(line_number_at(body, at).into()),- );if multi {fields.insert("section".into(), Value::Number((i as u64).into()));}@@ -1008,75 +926,6 @@ mod section_tests {);}- /// A single spaced paragraph is cut after a word, and the pieces still- /// partition the input: nothing dropped, nothing repeated. A hard split- /// does not overlap (overlap is the paragraph-boundary path).- #[test]- fn hard_split_of_spaced_prose_cuts_on_a_word_boundary() {- let text = "alpha ".repeat(SECTION_CHARS);- assert!(!text.contains('\n'));- let secs = split_sections(&text);- assert!(secs.len() > 2, "expected a hard split, got {}", secs.len());- let mut pos = 0usize;- for (i, section) in secs.iter().enumerate() {- assert_eq!(- &text[pos..pos + section.len()],- section.as_str(),- "section {i} dropped or duplicated content"- );- if i + 1 < secs.len() {- assert!(- section.ends_with(' '),- "section {i} ends mid-word: {:?}",- §ion[section.len().saturating_sub(24)..]- );- assert!(- secs[i + 1].starts_with("alpha"),- "section {} starts mid-word: {:?}",- i + 1,- &secs[i + 1][..secs[i + 1].len().min(24)]- );- }- pos += section.len();- }- assert_eq!(pos, text.len(), "sections did not cover the input");- }-- /// `start_line` is the file line of the section's first byte, not the- /// passage ordinal and not always 1.- #[test]- fn section_records_carry_the_source_line() {- let body = "word\n".repeat(SECTION_CHARS);- assert!(!body.contains("\n\n"));- let mut lines = Vec::new();- let mut stats = ExtractStats::default();- let mut sink = |r: RawRecord| {- lines.push(r.fields["start_line"].as_u64().unwrap());- true- };- assert!(emit_document(- "t",- &[],- &body,- MAX_RECORDS_PER_FILE,- &mut sink,- &mut stats- ));- let secs = split_sections(&body);- assert_eq!(lines.len(), secs.len());- assert!(secs.len() > 1);- let mut at = 0usize;- for (i, section) in secs.iter().enumerate() {- let expect = 1 + body[..at].bytes().filter(|b| *b == b'\n').count() as u64;- assert_eq!(lines[i], expect, "section {i}");- if i > 0 {- assert!(lines[i] > lines[i - 1]);- }- at += section.len();- }- assert_eq!(lines[0], 1);- }-/// A single paragraph larger than two sections must still be bounded.#[test]fn pathological_single_paragraph_is_hard_split() {@@ -1133,11 +982,9 @@ mod section_tests {out}- /// #239 rewrote the splitter for cost, not for behaviour: every input that- /// does not hard-split on whitespace must still produce byte-identical- /// sections. Multi-byte cases are the ones a hand-rolled boundary walk- /// gets wrong, so they are all in here. Spaced hard splits snap to a word- /// and are asserted separately.+ /// #239 rewrote the splitter for cost, not for behaviour: every input must+ /// still produce byte-identical sections. Multi-byte cases are the ones a+ /// hand-rolled boundary walk gets wrong, so they are all in here.#[test]fn matches_the_pre_fix_implementation_byte_for_byte() {let mut cases: Vec<String> = vec![@@ -1152,9 +999,7 @@ mod section_tests {"z".repeat(10 * SECTION_CHARS),doc(200, 400),doc(60, 300),- // doc(3, 9000) hard-splits inside a spaced paragraph. Word snapping- // intentionally moves that cut; it is covered by- // `hard_split_of_spaced_prose_cuts_on_a_word_boundary` instead.+ doc(3, 9000),// A short paragraph before ones too big to join it, at the// overlap boundary and one byte either side.format!("{}\n\n{}", "h".repeat(32), doc(4, 2100)),diff --git a/engine/crates/xerj-autoindex/src/extract/txt.rs b/engine/crates/xerj-autoindex/src/extract/txt.rsindex 8e6b2588f..848693921 100644--- a/engine/crates/xerj-autoindex/src/extract/txt.rs+++ b/engine/crates/xerj-autoindex/src/extract/txt.rs@@ -1,7 +1,6 @@//! Plain text: prose files become one document record (title = first line,-//! or the first ATX heading / file stem when that line is markup; body split-//! into ~2KB sections); line-oriented files become overlapping line-window-//! chunks.+//! body split into ~32KB sections); line-oriented files become overlapping+//! line-window chunks.//!//! ## Why line-oriented text is chunked, not emitted per line//!@@ -23,7 +22,7 @@//! caller can jump straight to the source — the old per-line record only//! carried an opaque byte locator.-use super::{emit_document_from, ExtractStats, FieldOrigin, RawRecord, Sink, MAX_RECORDS_PER_FILE};+use super::{emit_document, ExtractStats, FieldOrigin, RawRecord, Sink, MAX_RECORDS_PER_FILE};use anyhow::Result;use serde_json::{Map, Value};use std::path::Path;@@ -47,11 +46,23 @@ pub fn extract_prose(path: &Path, gzip: bool, sink: Sink) -> Result<ExtractStatsreturn Ok(stats);};let (text, _) = crate::sniff::decode_text(&bytes);- let title = prose_title(&text, path);- let skipped = text.len() - text.trim_start().len();- let skipped_lines = text[..skipped].bytes().filter(|b| *b == b'\n').count() as u64;- emit_document_from(- skipped_lines,+ let title = text+ .lines()+ .find(|l| !l.trim().is_empty())+ .map(|l| {+ let t = l.trim();+ let mut s: String = t.chars().take(200).collect();+ if s.len() < t.len() {+ s.push('…');+ }+ s+ })+ .unwrap_or_else(|| {+ path.file_stem()+ .map(|s| s.to_string_lossy().to_string())+ .unwrap_or_else(|| "untitled".into())+ });+ emit_document(&title,&[],text.trim(),@@ -62,47 +73,6 @@ pub fn extract_prose(path: &Path, gzip: bool, sink: Sink) -> Result<ExtractStatsOk(stats)}-/// First non-empty line, clipped to 200 chars. A line that is markup (`<…`)-/// is a useless title — an HTML fragment saved as text, a front-matter-less-/// export — so the first ATX heading wins, then the file stem.-fn prose_title(text: &str, path: &Path) -> String {- let stem = path- .file_stem()- .map(|s| s.to_string_lossy().into_owned())- .unwrap_or_else(|| "untitled".into());- let Some(first) = text.lines().find(|l| !l.trim().is_empty()).map(str::trim) else {- return stem;- };- if first.starts_with('<') {- return atx_heading(text).map(clip_title).unwrap_or(stem);- }- clip_title(first)-}--fn atx_heading(text: &str) -> Option<&str> {- text.lines().find_map(|line| {- let t = line.trim();- let hashes = t.bytes().take_while(|b| *b == b'#').count();- if !(1..=6).contains(&hashes) || t.as_bytes().get(hashes) != Some(&b' ') {- return None;- }- let title = t[hashes + 1..].trim();- if title.is_empty() {- None- } else {- Some(title)- }- })-}--fn clip_title(t: &str) -> String {- let mut s: String = t.chars().take(200).collect();- if s.len() < t.len() {- s.push('…');- }- s-}-pub fn extract_lines(path: &Path,gzip: bool,@@ -320,54 +290,3 @@ mod chunk_tests {);}}--#[cfg(test)]-mod prose_tests {- use super::*;-- fn prose(name: &str, body: &str) -> Vec<RawRecord> {- let dir = std::env::temp_dir().join(format!("xerj-prose-{name}"));- std::fs::create_dir_all(&dir).unwrap();- let p = dir.join(name);- std::fs::write(&p, body).unwrap();- let mut out = Vec::new();- extract_prose(&p, false, &mut |r| {- out.push(r);- true- })- .unwrap();- let _ = std::fs::remove_file(&p);- out- }-- #[test]- fn a_markup_first_line_uses_an_atx_heading_or_the_file_stem() {- let recs = prose(- "notes.md",- "<div class=\"x\">\n\n# Actual Heading\n\nbody text\n",- );- assert_eq!(recs[0].fields["title"], serde_json::json!("Actual Heading"));-- let recs = prose("page.md", "<html><body>no heading, just words that go on\n");- assert_eq!(recs[0].fields["title"], serde_json::json!("page"));- }-- #[test]- fn an_ordinary_first_line_stays_the_title() {- let recs = prose("plain.md", "# Architecture\n\nThe design.\n");- assert_eq!(recs[0].fields["title"], serde_json::json!("# Architecture"));- }-- /// `extract_prose` trims leading whitespace before sectioning. The lines- /// that trim removed still count, so a body that starts on line 4 is not- /// reported as line 1.- #[test]- fn leading_blank_lines_shift_start_line() {- let recs = prose("pad.md", "\n\n\nHello from line four.\n");- assert_eq!(recs[0].fields["start_line"], serde_json::json!(4));- assert_eq!(- recs[0].fields["body"],- serde_json::json!("Hello from line four.")- );- }-}diff --git a/engine/crates/xerj-autoindex/src/progress.rs b/engine/crates/xerj-autoindex/src/progress.rsindex 9423281e0..ae77fb6ce 100644--- a/engine/crates/xerj-autoindex/src/progress.rs+++ b/engine/crates/xerj-autoindex/src/progress.rs@@ -654,12 +654,6 @@ impl Progress {for (key, value) in flags {doc.insert((*key).to_string(), (*value).into());}- if ok && exit == 3 {- doc.insert(- "exit_meaning".into(),- "success: junk was recorded, the run did not fail".into(),- );- }let line = serde_json::Value::Object(doc);self.sink.write(format!("{line}\n").as_bytes());}@@ -680,11 +674,6 @@ impl Progress {line.push_str(&format!(" {key}={value}"));}line.push('\n');- if ok && exit == 3 {- line.push_str(- "xerj-note exit=3 means success: junk was recorded, the run did not fail\n",- );- }self.sink.write(line.as_bytes());}}@@ -2034,10 +2023,6 @@ mod tests {assert_eq!(last["exit"], 3);assert_eq!(last["reason"], "completed-with-junk");assert_eq!(last["files"], 1922);- assert_eq!(- last["exit_meaning"],- "success: junk was recorded, the run did not fail"- );}/// #279. `ignored_files_in_pruned_dirs` is budget-capped, so on its own it@@ -2185,12 +2170,6 @@ mod tests {"{text}");assert!(text.contains("files=12 records=7"), "{text}");- assert!(- text.contains(- "xerj-note exit=3 means success: junk was recorded, the run did not fail"- ),- "{text}"- );}#[test]