Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

fix(autoindex): surface the #381 truncation on the graph index path

xerj-org0b08bb3ede1b
diff --git a/engine/crates/xerj-autoindex/src/lib.rs b/engine/crates/xerj-autoindex/src/lib.rs
index 336bcb259..21efcd7d3 100644
--- a/engine/crates/xerj-autoindex/src/lib.rs
+++ b/engine/crates/xerj-autoindex/src/lib.rs
@@ -4749,6 +4749,9 @@ pub fn run_index_report(cfg: IndexCfg) -> Result<(i32, Option<Value>)> {
// Records the backend refused in a bulk response (invocation-local: no
// journal record exists for a document that was never accepted).
let rejected_records = AtomicU64::new(0);
+ // Files whose section stream hit the per-file record cap (#381): the tail
+ // was dropped, so it is surfaced after the run rather than silently lost.
+ let truncated_files = AtomicU64::new(0);
let bulk_errors = Mutex::new(Vec::<String>::new());
let graph: Option<GraphRt> = if cfg.no_graph {
None
@@ -5255,6 +5258,9 @@ pub fn run_index_report(cfg: IndexCfg) -> Result<(i32, Option<Value>)> {
match res {
Ok(stats) => {
file_junk += stats.junk;
+ if stats.truncated {
+ truncated_files.fetch_add(1, Ordering::Relaxed);
+ }
}
Err(e) => {
send_err = Some(format!("extract {}: {e}", f.rel));
@@ -5832,6 +5838,17 @@ pub fn run_index_report(cfg: IndexCfg) -> Result<(i32, Option<Value>)> {
cat_buf.push(b'\n');
}
+ // #381: the per-file record cap dropped a file's tail. Report it (never
+ // silent) so a genuinely large legitimate document is visible, not guessed.
+ let truncated = truncated_files.load(Ordering::Relaxed);
+ if truncated > 0 {
+ pr.note(&format!(
+ "{truncated} file(s) hit the per-file record cap ({}) and were truncated; \
+ split the input or raise the cap if the dropped tail matters (#381)",
+ extract::MAX_RECORDS_PER_FILE
+ ));
+ }
+
// dataset docs
let mut junk_records_by_run: u64 = junk_records.load(Ordering::Relaxed);
let (dataset_stats, durable_junk_records) = {