Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

fix(autoindex): surface the #381 truncation on the --no-graph generated path

xerj-orgdbd6ee514323
diff --git a/engine/crates/xerj-autoindex/src/lib.rs b/engine/crates/xerj-autoindex/src/lib.rs
index 21efcd7d3..39a82b368 100644
--- a/engine/crates/xerj-autoindex/src/lib.rs
+++ b/engine/crates/xerj-autoindex/src/lib.rs
@@ -595,6 +595,7 @@ fn finish_generated_progress(pr: &Progress, code: i32, summary: &Value) {
);
}
+#[allow(clippy::too_many_arguments)] // cutover threads the whole run context
fn begin_non_graph_generation(
es: &Es,
journal: &mut state::Journal,
@@ -602,6 +603,7 @@ fn begin_non_graph_generation(
cfg: &IndexCfg,
root_identity: &str,
inventory: &content::Inventory,
+ pr: &Progress,
plan: Plan,
) -> Result<()> {
anyhow::ensure!(
@@ -630,6 +632,22 @@ fn begin_non_graph_generation(
&preparation_contract,
cfg.snapshot_max_bytes,
)?;
+ // #381: the per-file record cap dropped a file's tail during preparation.
+ // The generated path seals before the graph worker loop runs, so report it
+ // here rather than at the shared post-run summary.
+ let truncated = snapshot
+ .files
+ .iter()
+ .filter_map(|f| f.prepared.as_ref())
+ .filter(|p| p.truncated)
+ .count();
+ if truncated > 0 {
+ pr.note(&format!(
+ "{truncated} file(s) hit the per-file record cap ({}) and were truncated; \
+ split the input or raise the cap if the dropped tail matters (#381)",
+ extract::MAX_RECORDS_PER_FILE
+ ));
+ }
let chunker_identity = prepared_records_identity(cfg)?;
let semantic = plan
.datasets
@@ -3847,6 +3865,7 @@ pub fn run_index_report(cfg: IndexCfg) -> Result<(i32, Option<Value>)> {
&cfg,
&root_str,
&inventory,
+ &pr,
plan,
)?;
let mut backend = sync_executor::EsSyncBackend::new(&es, &state_dir, cfg.bulk_mb << 20);
@@ -4512,6 +4531,7 @@ pub fn run_index_report(cfg: IndexCfg) -> Result<(i32, Option<Value>)> {
&cfg,
&root_str,
&inventory,
+ &pr,
plan,
)?;
let mut backend = sync_executor::EsSyncBackend::new(&es, &state_dir, cfg.bulk_mb << 20);
diff --git a/engine/crates/xerj-autoindex/src/sync_executor.rs b/engine/crates/xerj-autoindex/src/sync_executor.rs
index 0ca1a3b47..d8938b90c 100644
--- a/engine/crates/xerj-autoindex/src/sync_executor.rs
+++ b/engine/crates/xerj-autoindex/src/sync_executor.rs
@@ -59,6 +59,12 @@ pub struct PreparedArtifact {
/// never published, so no read-back count sees it.
#[serde(default)]
pub junk: u64,
+ /// The per-file record cap (#381) stopped this file's section stream before
+ /// the whole body was emitted. Skip-serialized when false so a non-truncated
+ /// artifact hashes exactly as before (a truncating file already differs by
+ /// design — it seals fewer records).
+ #[serde(default, skip_serializing_if = "std::ops::Not::not")]
+ pub truncated: bool,
/// Per-dataset breakdown of `records`, keyed by the dataset each record was
/// written under. One file can feed several datasets — a SQL dump is one
/// file and N tables — so the flat total is not comparable to any single
@@ -1645,6 +1651,7 @@ fn prepare_artifact(
passages,
vectors,
junk: stats.junk,
+ truncated: stats.truncated,
records_by_dataset,
bytes,
digest: stream_digest(&path, "axp1")?,