Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

fix(ai): semantic chunker no longer drops up to 64 chars between chunks

xerj-orga36ab8e39312
diff --git a/engine/crates/xerj-ai/src/chunker.rs b/engine/crates/xerj-ai/src/chunker.rs
index acb67ad13..33193546d 100644
--- a/engine/crates/xerj-ai/src/chunker.rs
+++ b/engine/crates/xerj-ai/src/chunker.rs
@@ -90,7 +90,6 @@ impl TextChunker {
let mut chunks = Vec::new();
let mut chunk_start_char = 0usize; // char index
- let step = self.chunk_size - self.overlap;
while chunk_start_char < char_count {
let chunk_end_char = (chunk_start_char + self.chunk_size).min(char_count);
@@ -126,14 +125,40 @@ impl TextChunker {
break;
}
- // Advance by step, but account for the overlap
- let next_start = chunk_start_char + step;
- if next_start >= char_count {
- break;
- }
- chunk_start_char = next_start;
+ // Advance from the ACTUAL break, not by a fixed
+ // `chunk_size - overlap` step from the chunk start (RC4 W2
+ // item 18): when the break landed before `start + chunk_size`
+ // (early sentence/word boundary), the fixed step skipped the
+ // chars in `break_char..start + step` — up to `chunk_size/4`
+ // minus overlap chars (64 with the auto-embed 512/64 defaults)
+ // silently absent from every chunk and every passage vector.
+ // Starting the next chunk `overlap` chars before the previous
+ // chunk's real end guarantees contiguous coverage and exactly
+ // `overlap` shared chars. The `max(start + 1)` clamp guarantees
+ // forward progress for degenerate configs whose overlap reaches
+ // back past the early break.
+ chunk_start_char = break_char
+ .saturating_sub(self.overlap)
+ .max(chunk_start_char + 1);
}
+ // Coverage assertion (debug builds): every non-whitespace char of
+ // the input lies inside at least one chunk. Only whitespace may be
+ // skipped (whitespace-only chunks are intentionally dropped above).
+ debug_assert!(
+ {
+ let mut covered = vec![false; text.len()];
+ for c in &chunks {
+ for b in &mut covered[c.start_offset..c.end_offset] {
+ *b = true;
+ }
+ }
+ text.char_indices()
+ .all(|(i, ch)| covered[i] || ch.is_whitespace())
+ },
+ "TextChunker dropped non-whitespace input between chunks"
+ );
+
chunks
}
@@ -235,6 +260,84 @@ mod tests {
assert!(chunker.chunk("", None).is_empty());
}
+ /// Every non-whitespace byte of `text` must be covered by at least one
+ /// chunk's `[start_offset, end_offset)` range. (Whitespace-only regions
+ /// may legitimately be skipped: whitespace-only chunks are dropped.)
+ fn assert_full_coverage(text: &str, chunks: &[Chunk], ctx: &str) {
+ let mut covered = vec![false; text.len()];
+ for c in chunks {
+ for b in &mut covered[c.start_offset..c.end_offset] {
+ *b = true;
+ }
+ }
+ let bytes = text.as_bytes();
+ let dropped: Vec<usize> = covered
+ .iter()
+ .enumerate()
+ .filter(|(i, cov)| !**cov && !bytes[*i].is_ascii_whitespace())
+ .map(|(i, _)| i)
+ .collect();
+ assert!(
+ dropped.is_empty(),
+ "{ctx}: chunker dropped {} non-whitespace byte(s); first gap at byte {} ({:?}...)",
+ dropped.len(),
+ dropped[0],
+ &text[dropped[0]..(dropped[0] + 24).min(text.len())]
+ );
+ }
+
+ /// Regression (RC4 W2 item 18): the chunker advanced by a fixed
+ /// `chunk_size - overlap` step from the chunk START even when the chunk
+ /// actually ENDED earlier at a sentence/word break, silently omitting
+ /// the bytes between the actual break and the fixed step — up to
+ /// `chunk_size/4 - overlap` chars per boundary (64 with the auto-embed
+ /// 512/64 defaults). Those bytes appeared in no chunk and therefore in
+ /// no passage vector.
+ #[test]
+ fn no_text_dropped_after_early_sentence_break() {
+ // Layout (single-byte chars so byte == char offsets):
+ // [0..390) words, ending with '.' at 390 and ' ' at 391
+ // [392..600) one unbroken run of 'z' (no space, no period)
+ // With chunk_size=512/overlap=64 the sentence-break search window
+ // for the first chunk is [384..512): the only break is at 392, so
+ // chunk 0 = [0..392). The fixed-step advance then started chunk 1
+ // at 448, dropping the 56 bytes [392..448) forever.
+ let mut text = String::new();
+ while text.len() < 385 {
+ text.push_str("lorem ipsum dolor sit amet consectetur ");
+ }
+ text.truncate(390);
+ text.push('.');
+ text.push(' ');
+ while text.len() < 600 {
+ text.push('z');
+ }
+ let chunker = TextChunker::new(512, 64);
+ let chunks = chunker.chunk(&text, None);
+ assert_full_coverage(&text, &chunks, "early-sentence-break");
+ }
+
+ /// Coverage must hold across assorted chunk_size/overlap combinations
+ /// and break-density profiles (sentence-heavy, word-only, break-free).
+ #[test]
+ fn full_coverage_across_configs() {
+ let sentence_heavy = "The quick brown fox jumps over the lazy dog. ".repeat(60);
+ let word_only = "wordy ".repeat(400);
+ let break_free = "x".repeat(2000);
+ let mixed = format!(
+ "{}{}{}",
+ "Intro sentence here. ",
+ "y".repeat(700),
+ " Tail words follow the blob. ".repeat(30)
+ );
+ for text in [&sentence_heavy, &word_only, &break_free, &mixed] {
+ for (cs, ov) in [(512, 64), (100, 20), (80, 15), (64, 0), (50, 10), (256, 128)] {
+ let chunks = TextChunker::new(cs, ov).chunk(text, None);
+ assert_full_coverage(text, &chunks, &format!("cs={cs} ov={ov}"));
+ }
+ }
+ }
+
#[test]
fn unicode_text_handled() {
let text = "日本語のテキストは正しく分割される必要があります。".repeat(5);