Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

Revert #1623 + #1643 (thrash-era regressions) to unblock a red main (#1730)

xerj-org46f6b0c2aede
diff --git a/engine/crates/xerj-autoindex/src/cli.rs b/engine/crates/xerj-autoindex/src/cli.rs
index 1beb0a4df..ac9e91bb3 100644
--- a/engine/crates/xerj-autoindex/src/cli.rs
+++ b/engine/crates/xerj-autoindex/src/cli.rs
@@ -629,9 +629,7 @@ pub fn help_text_with(feedback: bool) -> String {
\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20journal committed; only the catalog's duplicate-alias cleanup\n\
\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20could not run. Read `reason` on the xerj-done line to tell the\n\
\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20two apart, and rerun the same command once the reported server\n\
- \x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20condition clears. ok=true with exit=3 is success: the next\n\
- \x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20line is a xerj-note that says so (--progress json sets\n\
- \x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20exit_meaning on the done object);\n\
+ \x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20condition clears;\n\
\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x204 NEEDS A DECISION — the estimate exceeded --max-minutes, or a\n\
\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20--watch poll cost more than --max-monthly-ops; nothing was\n\
\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20indexed and a JSON decision request is on stdout;\n\
diff --git a/engine/crates/xerj-autoindex/src/extract/mod.rs b/engine/crates/xerj-autoindex/src/extract/mod.rs
index 4ad292db8..7e3162691 100644
--- a/engine/crates/xerj-autoindex/src/extract/mod.rs
+++ b/engine/crates/xerj-autoindex/src/extract/mod.rs
@@ -117,12 +117,6 @@ pub const MAX_RECORDS_PER_FILE: usize = 4 << 10; // 4096
/// answer spanning a boundary stays retrievable from both sides.
pub const SECTION_OVERLAP: usize = 200;
-/// How far a hard split may move back from `SECTION_CHARS` to land on
-/// whitespace. A run with no whitespace inside the window keeps the char
-/// boundary, so multi-byte scalars and unbroken tokens are never severed
-/// just to chase a word that is not there.
-const WORD_SNAP_BYTES: usize = 64;
-
/// Open a (possibly gzipped) file as a buffered reader of DECODED-transparent
/// bytes, optionally capped at `limit` decoded bytes (sampling).
pub fn open_reader(path: &Path, gzip: bool, limit: Option<u64>) -> Result<Box<dyn BufRead + Send>> {
@@ -189,10 +183,7 @@ pub fn extract_as_document_with_name(
.file_stem()
.map(|s| s.to_string_lossy().to_string())
.unwrap_or_else(|| "untitled".into());
- let skipped = text.len() - text.trim_start().len();
- let skipped_lines = text[..skipped].bytes().filter(|b| *b == b'\n').count() as u64;
- emit_document_from(
- skipped_lines,
+ emit_document(
&title,
&[],
text.trim(),
@@ -594,34 +585,6 @@ fn floor_boundary(s: &str, i: usize) -> usize {
i
}
-/// Cut `cur` near `target` bytes. Prefer the nearest whitespace at or before
-/// `target` within [`WORD_SNAP_BYTES`], and cut *after* that whitespace so
-/// the next section starts on a word. No whitespace in the window → the char
-/// boundary at `target`, which never splits a scalar.
-fn cut_on_word_or_char(cur: &str, target: usize) -> usize {
- let boundary = ceil_boundary(cur, target.min(cur.len()));
- if boundary == 0 || boundary >= cur.len() {
- return boundary;
- }
- let look = floor_boundary(cur, boundary.saturating_sub(WORD_SNAP_BYTES));
- let window = &cur[look..boundary];
- if let Some(rel) = window.rfind(char::is_whitespace) {
- let at = look + rel;
- let ws_len = cur[at..].chars().next().map(|c| c.len_utf8()).unwrap_or(1);
- let after = at + ws_len;
- if after > 0 && after < cur.len() {
- return after;
- }
- }
- boundary
-}
-
-/// 1-based line number of `text` at byte offset `byte`.
-fn line_number_at(text: &str, byte: usize) -> u64 {
- let byte = byte.min(text.len());
- 1 + text[..byte].bytes().filter(|b| *b == b'\n').count() as u64
-}
-
/// Top up `cur` from the front of `rest` until it holds `want` bytes, moving
/// whole chars only. Guarantees progress: if `want - cur.len()` lands inside
/// the next char, that whole char moves anyway.
@@ -655,23 +618,13 @@ fn fill_window(cur: &mut String, rest: &mut &str, want: usize) {
/// that never exceeds `2 * SECTION_CHARS + 4` bytes, and the paragraph itself
/// stays borrowed from `text`.
pub fn for_each_section(text: &str, emit: &mut dyn FnMut(String) -> bool) -> bool {
- for_each_section_spans(text, &mut |_at, section| emit(section))
-}
-
-/// [`for_each_section`] that also reports the byte offset in `text` where each
-/// section begins. Paragraph overlap is included: a section that repeats the
-/// previous tail starts at that tail.
-fn for_each_section_spans(text: &str, emit: &mut dyn FnMut(usize, String) -> bool) -> bool {
if text.len() <= SECTION_CHARS {
- return emit(0, text.to_string());
+ return emit(text.to_string());
}
// One byte past the largest section the loop below will ever hold, so a
// full window always trips the hard-split condition.
let window = 2 * SECTION_CHARS + 1;
let mut cur = String::new();
- // Byte offset in `text` of `cur`'s first byte once `cur` is non-empty.
- let mut origin = 0usize;
- let mut pos = 0usize;
for para in text.split("\n\n") {
// `cur.len() > SECTION_OVERLAP`: a shorter section would be carried
@@ -681,20 +634,16 @@ fn for_each_section_spans(text: &str, emit: &mut dyn FnMut(usize, String) -> boo
if !cur.is_empty() && cur.len() + para.len() > SECTION_CHARS && cur.len() > SECTION_OVERLAP
{
let done = std::mem::take(&mut cur);
- let start = origin;
let carry = tail(&done, SECTION_OVERLAP);
- origin = start + done.len() - carry.len();
+ if !emit(done) {
+ return false;
+ }
if !carry.is_empty() {
cur.push_str(&carry);
}
- if !emit(start, done) {
- return false;
- }
}
if !cur.is_empty() {
cur.push_str("\n\n");
- } else {
- origin = pos;
}
// `rest` is the not-yet-buffered remainder of this paragraph, borrowed
// from `text` — it is never copied wholesale into `cur`.
@@ -706,23 +655,16 @@ fn for_each_section_spans(text: &str, emit: &mut dyn FnMut(usize, String) -> boo
if cur.len() + rest.len() <= 2 * SECTION_CHARS {
break;
}
- let cut = cut_on_word_or_char(&cur, SECTION_CHARS);
+ let cut = ceil_boundary(&cur, SECTION_CHARS);
let section = cur[..cut].to_string();
- let start = origin;
cur.drain(..cut);
- origin += cut;
- if !emit(start, section) {
+ if !emit(section) {
return false;
}
}
- pos += para.len();
- if pos < text.len() {
- // The `\n\n` separator `split` consumed.
- pos += 2;
- }
}
if !cur.trim().is_empty() {
- return emit(origin, cur);
+ return emit(cur);
}
true
}
@@ -744,7 +686,6 @@ fn section_record(
i: usize,
multi: bool,
section: String,
- start_line: u64,
) -> RawRecord {
let mut fields = Map::new();
fields.insert("title".into(), Value::String(title.to_string()));
@@ -758,13 +699,12 @@ fn section_record(
fields.insert("section".into(), Value::Number((i as u64).into()));
}
fields.insert("body".into(), Value::String(section));
- fields.insert("start_line".into(), Value::Number(start_line.into()));
RawRecord {
fields,
locator: format!("s{i}"),
group: None,
- // title/headings/section/body/start_line are this function's vocabulary,
- // and `headings`/`section` come and go — never a clustering key.
+ // title/headings/section/body are this function's vocabulary, and
+ // `headings`/`section` come and go — never a clustering key.
origin: FieldOrigin::Extractor,
}
}
@@ -780,30 +720,14 @@ pub fn emit_document(
max_records: usize,
sink: Sink,
stats: &mut ExtractStats,
-) -> bool {
- emit_document_from(0, title, headings, body, max_records, sink, stats)
-}
-
-/// [`emit_document`] whose `start_line` values are shifted by `skipped_lines`.
-///
-/// Callers that strip leading whitespace before sectioning (prose) pass the
-/// number of newlines they removed, so `start_line` stays a file line.
-pub(crate) fn emit_document_from(
- skipped_lines: u64,
- title: &str,
- headings: &[String],
- body: &str,
- max_records: usize,
- sink: Sink,
- stats: &mut ExtractStats,
) -> bool {
// One section of lookahead: the `section` field is only present when the
// document actually splits, which is not known until a second one arrives.
- let mut pending: Option<(usize, String)> = None;
+ let mut pending: Option<String> = None;
let mut next: usize = 0;
let mut alive = true;
- for_each_section_spans(body, &mut |at, sec| {
- let Some((prev_at, prev)) = pending.replace((at, sec)) else {
+ for_each_section(body, &mut |sec| {
+ let Some(prev) = pending.replace(sec) else {
return true;
};
// #381: bound noise from a misclassified payload. Stopping the split
@@ -815,8 +739,7 @@ pub(crate) fn emit_document_from(
let i = next;
next += 1;
stats.records += 1;
- let line = skipped_lines + line_number_at(body, prev_at);
- alive = sink(section_record(title, headings, i, true, prev, line));
+ alive = sink(section_record(title, headings, i, true, prev));
alive
});
// A sink that stopped early (sampling) sets `alive = false`; the cap leaves
@@ -829,14 +752,13 @@ pub(crate) fn emit_document_from(
return true;
}
match pending {
- Some((at, last)) => {
+ Some(last) => {
if next >= max_records {
stats.truncated = true;
return true;
}
stats.records += 1;
- let line = skipped_lines + line_number_at(body, at);
- sink(section_record(title, headings, next, next > 0, last, line))
+ sink(section_record(title, headings, next, next > 0, last))
}
None => true,
}
@@ -859,9 +781,9 @@ pub(crate) fn emit_document_with_fields(
// Collect sections first so we know whether to stamp a `section` field.
// One past the cap is collected so that "exactly at the cap" and "over the
// cap" are distinguishable: only the latter dropped anything (#381).
- let mut secs: Vec<(usize, String)> = Vec::new();
- for_each_section_spans(body, &mut |at, s| {
- secs.push((at, s));
+ let mut secs: Vec<String> = Vec::new();
+ for_each_section(body, &mut |s| {
+ secs.push(s);
secs.len() <= MAX_RECORDS_PER_FILE
});
if secs.len() > MAX_RECORDS_PER_FILE {
@@ -869,17 +791,13 @@ pub(crate) fn emit_document_with_fields(
stats.truncated = true;
}
if secs.is_empty() {
- secs.push((0, String::new()));
+ secs.push(String::new());
}
let multi = secs.len() > 1;
- for (i, (at, sec)) in secs.into_iter().enumerate() {
+ for (i, sec) in secs.into_iter().enumerate() {
let mut fields = base_fields.clone();
fields.insert("title".into(), Value::String(title.to_string()));
fields.insert("body".into(), Value::String(sec));
- fields.insert(
- "start_line".into(),
- Value::Number(line_number_at(body, at).into()),
- );
if multi {
fields.insert("section".into(), Value::Number((i as u64).into()));
}
@@ -1008,75 +926,6 @@ mod section_tests {
);
}
- /// A single spaced paragraph is cut after a word, and the pieces still
- /// partition the input: nothing dropped, nothing repeated. A hard split
- /// does not overlap (overlap is the paragraph-boundary path).
- #[test]
- fn hard_split_of_spaced_prose_cuts_on_a_word_boundary() {
- let text = "alpha ".repeat(SECTION_CHARS);
- assert!(!text.contains('\n'));
- let secs = split_sections(&text);
- assert!(secs.len() > 2, "expected a hard split, got {}", secs.len());
- let mut pos = 0usize;
- for (i, section) in secs.iter().enumerate() {
- assert_eq!(
- &text[pos..pos + section.len()],
- section.as_str(),
- "section {i} dropped or duplicated content"
- );
- if i + 1 < secs.len() {
- assert!(
- section.ends_with(' '),
- "section {i} ends mid-word: {:?}",
- &section[section.len().saturating_sub(24)..]
- );
- assert!(
- secs[i + 1].starts_with("alpha"),
- "section {} starts mid-word: {:?}",
- i + 1,
- &secs[i + 1][..secs[i + 1].len().min(24)]
- );
- }
- pos += section.len();
- }
- assert_eq!(pos, text.len(), "sections did not cover the input");
- }
-
- /// `start_line` is the file line of the section's first byte, not the
- /// passage ordinal and not always 1.
- #[test]
- fn section_records_carry_the_source_line() {
- let body = "word\n".repeat(SECTION_CHARS);
- assert!(!body.contains("\n\n"));
- let mut lines = Vec::new();
- let mut stats = ExtractStats::default();
- let mut sink = |r: RawRecord| {
- lines.push(r.fields["start_line"].as_u64().unwrap());
- true
- };
- assert!(emit_document(
- "t",
- &[],
- &body,
- MAX_RECORDS_PER_FILE,
- &mut sink,
- &mut stats
- ));
- let secs = split_sections(&body);
- assert_eq!(lines.len(), secs.len());
- assert!(secs.len() > 1);
- let mut at = 0usize;
- for (i, section) in secs.iter().enumerate() {
- let expect = 1 + body[..at].bytes().filter(|b| *b == b'\n').count() as u64;
- assert_eq!(lines[i], expect, "section {i}");
- if i > 0 {
- assert!(lines[i] > lines[i - 1]);
- }
- at += section.len();
- }
- assert_eq!(lines[0], 1);
- }
-
/// A single paragraph larger than two sections must still be bounded.
#[test]
fn pathological_single_paragraph_is_hard_split() {
@@ -1133,11 +982,9 @@ mod section_tests {
out
}
- /// #239 rewrote the splitter for cost, not for behaviour: every input that
- /// does not hard-split on whitespace must still produce byte-identical
- /// sections. Multi-byte cases are the ones a hand-rolled boundary walk
- /// gets wrong, so they are all in here. Spaced hard splits snap to a word
- /// and are asserted separately.
+ /// #239 rewrote the splitter for cost, not for behaviour: every input must
+ /// still produce byte-identical sections. Multi-byte cases are the ones a
+ /// hand-rolled boundary walk gets wrong, so they are all in here.
#[test]
fn matches_the_pre_fix_implementation_byte_for_byte() {
let mut cases: Vec<String> = vec![
@@ -1152,9 +999,7 @@ mod section_tests {
"z".repeat(10 * SECTION_CHARS),
doc(200, 400),
doc(60, 300),
- // doc(3, 9000) hard-splits inside a spaced paragraph. Word snapping
- // intentionally moves that cut; it is covered by
- // `hard_split_of_spaced_prose_cuts_on_a_word_boundary` instead.
+ doc(3, 9000),
// A short paragraph before ones too big to join it, at the
// overlap boundary and one byte either side.
format!("{}\n\n{}", "h".repeat(32), doc(4, 2100)),
diff --git a/engine/crates/xerj-autoindex/src/extract/txt.rs b/engine/crates/xerj-autoindex/src/extract/txt.rs
index 8e6b2588f..848693921 100644
--- a/engine/crates/xerj-autoindex/src/extract/txt.rs
+++ b/engine/crates/xerj-autoindex/src/extract/txt.rs
@@ -1,7 +1,6 @@
//! Plain text: prose files become one document record (title = first line,
-//! or the first ATX heading / file stem when that line is markup; body split
-//! into ~2KB sections); line-oriented files become overlapping line-window
-//! chunks.
+//! body split into ~32KB sections); line-oriented files become overlapping
+//! line-window chunks.
//!
//! ## Why line-oriented text is chunked, not emitted per line
//!
@@ -23,7 +22,7 @@
//! caller can jump straight to the source — the old per-line record only
//! carried an opaque byte locator.
-use super::{emit_document_from, ExtractStats, FieldOrigin, RawRecord, Sink, MAX_RECORDS_PER_FILE};
+use super::{emit_document, ExtractStats, FieldOrigin, RawRecord, Sink, MAX_RECORDS_PER_FILE};
use anyhow::Result;
use serde_json::{Map, Value};
use std::path::Path;
@@ -47,11 +46,23 @@ pub fn extract_prose(path: &Path, gzip: bool, sink: Sink) -> Result<ExtractStats
return Ok(stats);
};
let (text, _) = crate::sniff::decode_text(&bytes);
- let title = prose_title(&text, path);
- let skipped = text.len() - text.trim_start().len();
- let skipped_lines = text[..skipped].bytes().filter(|b| *b == b'\n').count() as u64;
- emit_document_from(
- skipped_lines,
+ let title = text
+ .lines()
+ .find(|l| !l.trim().is_empty())
+ .map(|l| {
+ let t = l.trim();
+ let mut s: String = t.chars().take(200).collect();
+ if s.len() < t.len() {
+ s.push('…');
+ }
+ s
+ })
+ .unwrap_or_else(|| {
+ path.file_stem()
+ .map(|s| s.to_string_lossy().to_string())
+ .unwrap_or_else(|| "untitled".into())
+ });
+ emit_document(
&title,
&[],
text.trim(),
@@ -62,47 +73,6 @@ pub fn extract_prose(path: &Path, gzip: bool, sink: Sink) -> Result<ExtractStats
Ok(stats)
}
-/// First non-empty line, clipped to 200 chars. A line that is markup (`<…`)
-/// is a useless title — an HTML fragment saved as text, a front-matter-less
-/// export — so the first ATX heading wins, then the file stem.
-fn prose_title(text: &str, path: &Path) -> String {
- let stem = path
- .file_stem()
- .map(|s| s.to_string_lossy().into_owned())
- .unwrap_or_else(|| "untitled".into());
- let Some(first) = text.lines().find(|l| !l.trim().is_empty()).map(str::trim) else {
- return stem;
- };
- if first.starts_with('<') {
- return atx_heading(text).map(clip_title).unwrap_or(stem);
- }
- clip_title(first)
-}
-
-fn atx_heading(text: &str) -> Option<&str> {
- text.lines().find_map(|line| {
- let t = line.trim();
- let hashes = t.bytes().take_while(|b| *b == b'#').count();
- if !(1..=6).contains(&hashes) || t.as_bytes().get(hashes) != Some(&b' ') {
- return None;
- }
- let title = t[hashes + 1..].trim();
- if title.is_empty() {
- None
- } else {
- Some(title)
- }
- })
-}
-
-fn clip_title(t: &str) -> String {
- let mut s: String = t.chars().take(200).collect();
- if s.len() < t.len() {
- s.push('…');
- }
- s
-}
-
pub fn extract_lines(
path: &Path,
gzip: bool,
@@ -320,54 +290,3 @@ mod chunk_tests {
);
}
}
-
-#[cfg(test)]
-mod prose_tests {
- use super::*;
-
- fn prose(name: &str, body: &str) -> Vec<RawRecord> {
- let dir = std::env::temp_dir().join(format!("xerj-prose-{name}"));
- std::fs::create_dir_all(&dir).unwrap();
- let p = dir.join(name);
- std::fs::write(&p, body).unwrap();
- let mut out = Vec::new();
- extract_prose(&p, false, &mut |r| {
- out.push(r);
- true
- })
- .unwrap();
- let _ = std::fs::remove_file(&p);
- out
- }
-
- #[test]
- fn a_markup_first_line_uses_an_atx_heading_or_the_file_stem() {
- let recs = prose(
- "notes.md",
- "<div class=\"x\">\n\n# Actual Heading\n\nbody text\n",
- );
- assert_eq!(recs[0].fields["title"], serde_json::json!("Actual Heading"));
-
- let recs = prose("page.md", "<html><body>no heading, just words that go on\n");
- assert_eq!(recs[0].fields["title"], serde_json::json!("page"));
- }
-
- #[test]
- fn an_ordinary_first_line_stays_the_title() {
- let recs = prose("plain.md", "# Architecture\n\nThe design.\n");
- assert_eq!(recs[0].fields["title"], serde_json::json!("# Architecture"));
- }
-
- /// `extract_prose` trims leading whitespace before sectioning. The lines
- /// that trim removed still count, so a body that starts on line 4 is not
- /// reported as line 1.
- #[test]
- fn leading_blank_lines_shift_start_line() {
- let recs = prose("pad.md", "\n\n\nHello from line four.\n");
- assert_eq!(recs[0].fields["start_line"], serde_json::json!(4));
- assert_eq!(
- recs[0].fields["body"],
- serde_json::json!("Hello from line four.")
- );
- }
-}
diff --git a/engine/crates/xerj-autoindex/src/progress.rs b/engine/crates/xerj-autoindex/src/progress.rs
index 9423281e0..ae77fb6ce 100644
--- a/engine/crates/xerj-autoindex/src/progress.rs
+++ b/engine/crates/xerj-autoindex/src/progress.rs
@@ -654,12 +654,6 @@ impl Progress {
for (key, value) in flags {
doc.insert((*key).to_string(), (*value).into());
}
- if ok && exit == 3 {
- doc.insert(
- "exit_meaning".into(),
- "success: junk was recorded, the run did not fail".into(),
- );
- }
let line = serde_json::Value::Object(doc);
self.sink.write(format!("{line}\n").as_bytes());
}
@@ -680,11 +674,6 @@ impl Progress {
line.push_str(&format!(" {key}={value}"));
}
line.push('\n');
- if ok && exit == 3 {
- line.push_str(
- "xerj-note exit=3 means success: junk was recorded, the run did not fail\n",
- );
- }
self.sink.write(line.as_bytes());
}
}
@@ -2034,10 +2023,6 @@ mod tests {
assert_eq!(last["exit"], 3);
assert_eq!(last["reason"], "completed-with-junk");
assert_eq!(last["files"], 1922);
- assert_eq!(
- last["exit_meaning"],
- "success: junk was recorded, the run did not fail"
- );
}
/// #279. `ignored_files_in_pruned_dirs` is budget-capped, so on its own it
@@ -2185,12 +2170,6 @@ mod tests {
"{text}"
);
assert!(text.contains("files=12 records=7"), "{text}");
- assert!(
- text.contains(
- "xerj-note exit=3 means success: junk was recorded, the run did not fail"
- ),
- "{text}"
- );
}
#[test]