AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
fix(autoindex): cap section records per file + flag truncation (#381)
xerj-orge81faa787f71
diff --git a/engine/crates/xerj-autoindex/src/extract/docx.rs b/engine/crates/xerj-autoindex/src/extract/docx.rsindex a461ca5a7..90baa261f 100644--- a/engine/crates/xerj-autoindex/src/extract/docx.rs+++ b/engine/crates/xerj-autoindex/src/extract/docx.rs@@ -2,7 +2,7 @@//! Collects w:t runs, breaks paragraphs on w:p, records Heading-styled//! paragraphs as headings. One document record (sectioned at 32KB).-use super::{emit_document, ExtractStats, Sink};+use super::{emit_document, ExtractStats, Sink, MAX_RECORDS_PER_FILE};use anyhow::{Context, Result};use quick_xml::events::Event;use quick_xml::Reader;@@ -138,7 +138,14 @@ fn extract_bounded(.unwrap_or_else(|| "untitled".into())})});- emit_document(&title, &headings, body, sink, &mut stats);+ emit_document(+ &title,+ &headings,+ body,+ MAX_RECORDS_PER_FILE,+ sink,+ &mut stats,+ );Ok(stats)}diff --git a/engine/crates/xerj-autoindex/src/extract/html.rs b/engine/crates/xerj-autoindex/src/extract/html.rsindex 789e39c28..db17e0ff0 100644--- a/engine/crates/xerj-autoindex/src/extract/html.rs+++ b/engine/crates/xerj-autoindex/src/extract/html.rs@@ -5,7 +5,8 @@//! document record {title, headings, body}.use super::{- emit_document, sanitize_field_name, ExtractStats, FieldOrigin, RawRecord, Sink, MAX_WHOLE_FILE,+ emit_document, sanitize_field_name, ExtractStats, FieldOrigin, RawRecord, Sink,+ MAX_RECORDS_PER_FILE, MAX_WHOLE_FILE,};use anyhow::Result;use serde_json::{Map, Value};@@ -92,7 +93,14 @@ pub fn extract(path: &Path, gzip: bool, sink: Sink) -> Result<ExtractStats> {} else {doc.title.trim().to_string()};- emit_document(&title, &doc.headings, doc.body.trim(), sink, &mut stats);+ emit_document(+ &title,+ &doc.headings,+ doc.body.trim(),+ MAX_RECORDS_PER_FILE,+ sink,+ &mut stats,+ );Ok(stats)}diff --git a/engine/crates/xerj-autoindex/src/extract/mod.rs b/engine/crates/xerj-autoindex/src/extract/mod.rsindex 0cf80b0d5..f2e5892c4 100644--- a/engine/crates/xerj-autoindex/src/extract/mod.rs+++ b/engine/crates/xerj-autoindex/src/extract/mod.rs@@ -67,6 +67,10 @@ pub struct RawRecord {pub struct ExtractStats {pub records: u64,pub junk: u64,+ /// Set when a per-file record cap (`MAX_RECORDS_PER_FILE`) stopped the+ /// split before the whole body was emitted (#381). Carried out so the+ /// truncation is reported, never silent.+ pub truncated: bool,}/// Sink returns false to stop extraction early (sampling).@@ -87,6 +91,18 @@ pub const MAX_WHOLE_FILE: u64 = 64 << 20; // whole-file parse cap (json/html/yam/// a section maps to one vector without truncation.pub const SECTION_CHARS: usize = 2 << 10;+/// Hard cap on section records one file may emit (#381).+///+/// A magic-less printable byte payload (a raw texture, a packed `.bytes` asset)+/// decodes under the text fallback, classifies as prose and is sectioned in+/// full — one record per `SECTION_CHARS`, so the 16 MiB txt read cap becomes+/// ~8k noise records with no ceiling of its own. This bounds that; the drop is+/// surfaced via `ExtractStats.truncated`, so a genuinely large *legitimate*+/// document is reported and never silently cut. A resource guard keyed off+/// record count only — never off byte statistics, which is what deleted CJK+/// text worldwide and was removed in #371.+pub const MAX_RECORDS_PER_FILE: usize = 4 << 10; // 4096+/// Characters of the previous section repeated at the start of the next, so an/// answer spanning a boundary stays retrievable from both sides.pub const SECTION_OVERLAP: usize = 200;@@ -155,7 +171,14 @@ pub fn extract_as_document_with_name(.file_stem().map(|s| s.to_string_lossy().to_string()).unwrap_or_else(|| "untitled".into());- emit_document(&title, &[], text.trim(), sink, &mut stats);+ emit_document(+ &title,+ &[],+ text.trim(),+ MAX_RECORDS_PER_FILE,+ sink,+ &mut stats,+ );Ok(stats)}@@ -506,6 +529,7 @@ pub fn emit_document(title: &str,headings: &[String],body: &str,+ max_records: usize,sink: Sink,stats: &mut ExtractStats,) -> bool {@@ -518,17 +542,33 @@ pub fn emit_document(let Some(prev) = pending.replace(sec) else {return true;};+ // #381: bound noise from a misclassified payload. Stopping the split+ // (not just the sink) also stops paying to section the rest of the body.+ if next >= max_records {+ stats.truncated = true;+ return false;+ }let i = next;next += 1;stats.records += 1;alive = sink(section_record(title, headings, i, true, prev));alive});+ // A sink that stopped early (sampling) sets `alive = false`; the cap leaves+ // it true and sets `truncated` instead. Distinguish so a capped run still+ // reports success and does NOT emit the over-cap `pending` tail section.if !alive {return false;}+ if stats.truncated {+ return true;+ }match pending {Some(last) => {+ if next >= max_records {+ stats.truncated = true;+ return true;+ }stats.records += 1;sink(section_record(title, headings, next, next > 0, last))}@@ -771,7 +811,14 @@ mod section_tests {got.push((r.locator.clone(), r.fields.contains_key("section")));true};- assert!(emit_document("t", &[], body, &mut sink, &mut stats));+ assert!(emit_document(+ "t",+ &[],+ body,+ MAX_RECORDS_PER_FILE,+ &mut sink,+ &mut stats+ ));assert_eq!(stats.records as usize, got.len());got}@@ -801,10 +848,61 @@ mod section_tests {"t",&[],&doc(60, 300),+ MAX_RECORDS_PER_FILE,&mut sink,&mut stats));assert_eq!(got, vec!["s0".to_string(), "s1".to_string()]);assert_eq!(stats.records, 2);}++ /// #381: a body that sections into far more chunks than the cap yields+ /// exactly the cap and REPORTS the truncation — it is never silent.+ #[test]+ fn emit_document_caps_records_and_reports_truncation() {+ let mut got = Vec::new();+ let mut stats = ExtractStats::default();+ let mut sink = |r: RawRecord| {+ got.push(r.locator.clone());+ true+ };+ // doc(60, 300) sections into well more than 3 chunks.+ assert!(emit_document(+ "t",+ &[],+ &doc(60, 300),+ 3,+ &mut sink,+ &mut stats+ ));+ assert_eq!(got.len(), 3, "record count must be bounded by the cap");+ assert_eq!(stats.records, 3);+ assert!(stats.truncated, "truncation must be reported, not silent");+ }++ /// A document within the cap indexes in full and is not flagged truncated.+ #[test]+ fn emit_document_under_the_cap_is_not_truncated() {+ let mut n = 0usize;+ let mut stats = ExtractStats::default();+ let mut sink = |_r: RawRecord| {+ n += 1;+ true+ };+ let body = doc(60, 300); // ~9 sections, well under the cap+ assert!(emit_document(+ "t",+ &[],+ &body,+ MAX_RECORDS_PER_FILE,+ &mut sink,+ &mut stats+ ));+ assert!(!stats.truncated);+ assert_eq!(stats.records as usize, n);+ assert!(+ n > 1 && n < MAX_RECORDS_PER_FILE,+ "sanity: multi-section but under the cap, got {n}"+ );+ }}diff --git a/engine/crates/xerj-autoindex/src/extract/txt.rs b/engine/crates/xerj-autoindex/src/extract/txt.rsindex e2a18be7d..848693921 100644--- a/engine/crates/xerj-autoindex/src/extract/txt.rs+++ b/engine/crates/xerj-autoindex/src/extract/txt.rs@@ -22,7 +22,7 @@//! caller can jump straight to the source — the old per-line record only//! carried an opaque byte locator.-use super::{emit_document, ExtractStats, FieldOrigin, RawRecord, Sink};+use super::{emit_document, ExtractStats, FieldOrigin, RawRecord, Sink, MAX_RECORDS_PER_FILE};use anyhow::Result;use serde_json::{Map, Value};use std::path::Path;@@ -62,7 +62,14 @@ pub fn extract_prose(path: &Path, gzip: bool, sink: Sink) -> Result<ExtractStats.map(|s| s.to_string_lossy().to_string()).unwrap_or_else(|| "untitled".into())});- emit_document(&title, &[], text.trim(), sink, &mut stats);+ emit_document(+ &title,+ &[],+ text.trim(),+ MAX_RECORDS_PER_FILE,+ sink,+ &mut stats,+ );Ok(stats)}