Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

fix(autoindex): keep legacy catalogs and durable quality metadata

xerj-orgfffcde3373e5
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 9160e83f7..8765fc7ce 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -211,6 +211,37 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
document. This does **not** implement add/change/delete reconciliation for
*indexed* files, which remains open.
+- **Repeated `autoindex` scans keep agent-facing map metadata durable.**
+ Dataset source bytes, parser-junk counts and coercion-drop notes now derive
+ from committed per-file journal records instead of invocation-local worker
+ counters, so an unchanged resume does not overwrite them with zero. Run
+ timestamps now distinguish invocation start (`started`) from summary
+ generation (`summary_generated_at`), and `junk_records_total` on the run
+ document is the same durable sum the per-dataset `junk_records` reports
+ rather than an invocation-local counter that read `0` after a no-op resume.
+ Existing catalogs whose historical `started` field was dynamically mapped
+ as text remain usable: the additive mapping upgrade no longer attempts an
+ incompatible text-to-date type change.
+
+ Two consequences worth knowing before you upgrade:
+
+ - **`bytes` is redefined.** It used to be the bytes the latest invocation
+ processed, credited to the first dataset a file was assigned to. It is now
+ the complete canonical source size, counted once in **every** distinct
+ dataset that source is assigned to. Per-dataset values therefore get
+ larger, and summing `bytes` across datasets can exceed the corpus size
+ when one source feeds several datasets — exactly as one source already
+ contributed to several dataset-local file counts. It is a dataset-local
+ source footprint, not a partition of physical storage.
+ - **Journals written before this release carry no per-dataset coercion
+ record**, because the new `dropped_by_dataset` field defaults to empty
+ when an older journal is replayed. Byte and junk counts reconstruct fine
+ (their journal fields already existed), but a dataset whose files are all
+ unchanged loses its `N field values dropped by coercion` note on the first
+ resume after upgrading, and only regains it once one of its files is
+ reprocessed. Re-run with `--fresh` if the note matters more than the
+ rescan.
+
- **The installer is now fail-closed on checksum *verification*, not just on
checksum *download*.** `landing/get` printed
`warning: sha256sum/shasum not found — skipping checksum verification` and
diff --git a/engine/crates/xerj-autoindex/src/catalog.rs b/engine/crates/xerj-autoindex/src/catalog.rs
index 2bae791b7..90349e208 100644
--- a/engine/crates/xerj-autoindex/src/catalog.rs
+++ b/engine/crates/xerj-autoindex/src/catalog.rs
@@ -56,11 +56,11 @@ pub fn catalog_mapping() -> Value {
.expect("catalog mapping properties");
properties.insert(
"started".into(),
- json!({"type": "date", "format": "strict_date_optional_time"}),
+ json!({"type": "date", "format": "strict_date_optional_time||epoch_millis"}),
);
properties.insert(
"summary_generated_at".into(),
- json!({"type": "date", "format": "strict_date_optional_time"}),
+ json!({"type": "date", "format": "strict_date_optional_time||epoch_millis"}),
);
properties.insert(
"invocation_telemetry_scope".into(),
diff --git a/engine/crates/xerj-autoindex/src/failure_resume_http_tests.rs b/engine/crates/xerj-autoindex/src/failure_resume_http_tests.rs
index 3e7c4a40d..2edc173fd 100644
--- a/engine/crates/xerj-autoindex/src/failure_resume_http_tests.rs
+++ b/engine/crates/xerj-autoindex/src/failure_resume_http_tests.rs
@@ -155,19 +155,27 @@ fn handle(mut stream: TcpStream, state: &Arc<Mutex<MockState>>) {
}
} else if method == "PUT" && path == "/autoindex-catalog/_mapping" {
let mapping: Value = serde_json::from_slice(&body).unwrap();
- let required = [
- "started",
- "summary_generated_at",
- "invocation_telemetry_scope",
- ];
+ let started_requested = mapping.pointer("/properties/started").is_some();
+ let required = ["summary_generated_at", "invocation_telemetry_scope"];
let upgraded = required.iter().all(|field| {
mapping
.pointer(&format!("/properties/{field}/type"))
.and_then(Value::as_str)
.is_some()
});
- state.lock().unwrap().catalog_mapping_upgraded = upgraded;
- (200, json!({"acknowledged": true}))
+ let mut locked = state.lock().unwrap();
+ if locked.catalog_preexists && started_requested {
+ (
+ 400,
+ json!({"error": {
+ "type": "mapper_parsing_exception",
+ "reason": "field [started] already exists as [text], cannot add [date]"
+ }}),
+ )
+ } else {
+ locked.catalog_mapping_upgraded = upgraded;
+ (200, json!({"acknowledged": true}))
+ }
} else if method == "POST" && path == "/_bulk" {
(200, bulk_response(&body, state))
} else if method == "POST" && path.contains("/_delete_by_query") {
@@ -658,6 +666,46 @@ fn existing_catalog_is_upgraded_before_new_run_metadata_is_written() {
);
}
+#[test]
+fn unchanged_resume_keeps_durable_dataset_junk_and_coercion_notes() {
+ let _guard = FAILPOINT_TEST_LOCK.lock().unwrap();
+ let _io_guard = state::FILE_DONE_IO_FAILPOINT_TEST_LOCK.lock().unwrap();
+ let corpus = tempfile::tempdir().unwrap();
+ let state_dir = tempfile::tempdir().unwrap();
+ fs::write(corpus.path().join("records.csv"), "id,value\n0,one\n").unwrap();
+ let endpoint = MockEndpoint::start(usize::MAX);
+ let config = cfg(corpus.path(), state_dir.path(), &endpoint.url);
+
+ assert_eq!(run_index(config.clone()).unwrap(), 0);
+ let root = config.root.canonicalize().unwrap();
+ let mut journal = state::Journal::open(
+ state_dir.path(),
+ &root.to_string_lossy(),
+ &config.url,
+ &config.prefix,
+ config.bulk_timeout_secs,
+ false,
+ )
+ .unwrap();
+ let plan = journal.plan.clone().unwrap();
+ let slug = plan.datasets[0].slug.clone();
+ let mut completion = journal.done.values().next().unwrap().clone();
+ completion.junk = 7;
+ completion.dropped_by_dataset.insert(slug.clone(), 3);
+ journal.file_done(&completion).unwrap();
+ drop(journal);
+
+ assert_eq!(run_index(config).unwrap(), 0);
+ let dataset = dataset_catalog_docs(&endpoint)
+ .into_iter()
+ .find(|doc| doc["slug"] == slug)
+ .unwrap();
+ assert_eq!(dataset["junk_records"], 7);
+ assert!(dataset["notes"].as_array().unwrap().iter().any(|note| note
+ .as_str()
+ .is_some_and(|text| { text.contains("3 field values could not be coerced") })));
+}
+
#[test]
fn legacy_plan_without_completion_cleans_possible_partial_visibility() {
let _guard = FAILPOINT_TEST_LOCK.lock().unwrap();
diff --git a/engine/crates/xerj-autoindex/src/lib.rs b/engine/crates/xerj-autoindex/src/lib.rs
index b1d89cd52..f58218f68 100644
--- a/engine/crates/xerj-autoindex/src/lib.rs
+++ b/engine/crates/xerj-autoindex/src/lib.rs
@@ -920,15 +920,28 @@ pub fn derive_brain_name(root: &Path) -> String {
}
}
-/// Source bytes durably represented by each dataset.
+#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)]
+struct DurableDatasetStats {
+ bytes: u64,
+ junk: u64,
+ dropped: u64,
+}
+
+/// Source metadata durably represented by each dataset.
///
/// `FileDone` is the commit record for a successfully published canonical
/// source. A source can feed more than one inferred dataset (for example, a
/// workbook or database with multiple tables), so each distinct assigned
/// dataset reports the complete source bytes it depends on. Repeated groups
-/// within the same dataset count the source only once.
-fn durable_dataset_bytes(plan: &Plan, done: &HashMap<String, FileDone>) -> HashMap<String, u64> {
- let mut bytes_by_dataset = HashMap::new();
+/// within the same dataset count the source only once. Parser-level junk has
+/// no group after extraction rejects it, so it retains the historical
+/// attribution to the file's first assigned dataset. Coercion drops retain
+/// their exact dataset captured in the completion record.
+fn durable_dataset_stats(
+ plan: &Plan,
+ done: &HashMap<String, FileDone>,
+) -> HashMap<String, DurableDatasetStats> {
+ let mut stats_by_dataset: HashMap<String, DurableDatasetStats> = HashMap::new();
for (file_key, completed) in done {
let Some(assignment) = plan.files.get(file_key) else {
continue;
@@ -939,11 +952,25 @@ fn durable_dataset_bytes(plan: &Plan, done: &HashMap<String, FileDone>) -> HashM
.map(|(_, slug)| slug.as_str())
.collect();
for slug in assigned_datasets {
- let bytes = bytes_by_dataset.entry(slug.to_string()).or_insert(0u64);
- *bytes = bytes.saturating_add(completed.bytes);
+ let stats = stats_by_dataset.entry(slug.to_string()).or_default();
+ stats.bytes = stats.bytes.saturating_add(completed.bytes);
+ }
+ if let Some((_, slug)) = assignment.assignments.first() {
+ let stats = stats_by_dataset.entry(slug.clone()).or_default();
+ stats.junk = stats.junk.saturating_add(completed.junk);
+ }
+ for (slug, dropped) in &completed.dropped_by_dataset {
+ if assignment
+ .assignments
+ .iter()
+ .any(|(_, assigned)| assigned == slug)
+ {
+ let stats = stats_by_dataset.entry(slug.clone()).or_default();
+ stats.dropped = stats.dropped.saturating_add(*dropped);
+ }
}
}
- bytes_by_dataset
+ stats_by_dataset
}
fn invocation_report_timestamps(
@@ -1319,8 +1346,10 @@ pub fn run_index_report(cfg: IndexCfg) -> Result<(i32, Option<Value>)> {
catalog::CATALOG_INDEX,
&json!({"properties": {
"duplicate_of": {"type": "keyword"},
- "started": {"type": "date", "format": "strict_date_optional_time"},
- "summary_generated_at": {"type": "date", "format": "strict_date_optional_time"},
+ // `started` intentionally stays out of this additive upgrade.
+ // Older catalogs dynamically mapped it as text, and asking the
+ // engine to change that existing field to date aborts every run.
+ "summary_generated_at": {"type": "date", "format": "strict_date_optional_time||epoch_millis"},
"invocation_telemetry_scope": {"type": "keyword"}
}}),
)
@@ -1376,8 +1405,6 @@ pub fn run_index_report(cfg: IndexCfg) -> Result<(i32, Option<Value>)> {
index: String,
plan: HashMap<String, coerce::Coerce>,
records: AtomicU64,
- junk: AtomicU64,
- dropped: AtomicU64,
}
let mut ds_rt: HashMap<String, DsRt> = HashMap::new();
for d in &plan.datasets {
@@ -1387,8 +1414,6 @@ pub fn run_index_report(cfg: IndexCfg) -> Result<(i32, Option<Value>)> {
index: d.index.clone(),
plan: coerce::plan_from_specs(&d.specs),
records: AtomicU64::new(0),
- junk: AtomicU64::new(0),
- dropped: AtomicU64::new(0),
},
);
}
@@ -1659,6 +1684,7 @@ pub fn run_index_report(cfg: IndexCfg) -> Result<(i32, Option<Value>)> {
};
let mut file_records = 0u64;
let mut file_junk = 0u64;
+ let mut file_dropped_by_dataset: HashMap<String, u64> = HashMap::new();
let mut send_err: Option<String> = None;
// Edges this file teaches — buffered apart from the node
// staging file (different target index) and sent only
@@ -1761,7 +1787,9 @@ pub fn run_index_report(cfg: IndexCfg) -> Result<(i32, Option<Value>)> {
let mut fields = rec.fields;
let dropped = coerce::coerce_record(&mut fields, &rt.plan);
if dropped > 0 {
- rt.dropped.fetch_add(dropped as u64, Ordering::Relaxed);
+ let durable =
+ file_dropped_by_dataset.entry(slug.clone()).or_default();
+ *durable = durable.saturating_add(dropped as u64);
}
fields.insert("ax_path".into(), Value::String(f.rel.clone()));
fields.insert(
@@ -2037,11 +2065,6 @@ pub fn run_index_report(cfg: IndexCfg) -> Result<(i32, Option<Value>)> {
}
records_total.fetch_add(file_records, Ordering::Relaxed);
junk_records.fetch_add(file_junk, Ordering::Relaxed);
- if let Some(rt) = fa.assignments.first().and_then(|(_, slug)| ds_rt.get(slug)) {
- if file_junk > 0 {
- rt.junk.fetch_add(file_junk, Ordering::Relaxed);
- }
- }
let (commit_result, journal_path) = {
let mut journal = journal_mx.lock().unwrap();
let path = journal.path().display().to_string();
@@ -2051,6 +2074,7 @@ pub fn run_index_report(cfg: IndexCfg) -> Result<(i32, Option<Value>)> {
records: file_records,
junk: file_junk,
bytes: f.size,
+ dropped_by_dataset: file_dropped_by_dataset,
generation: Some(expected_digest.clone()),
});
(result, path)
@@ -2336,15 +2360,15 @@ pub fn run_index_report(cfg: IndexCfg) -> Result<(i32, Option<Value>)> {
// dataset docs
let mut junk_records_by_run: u64 = junk_records.load(Ordering::Relaxed);
- let dataset_bytes = {
+ let dataset_stats = {
let journal = journal_mx.lock().unwrap();
- durable_dataset_bytes(&plan, &journal.done)
+ durable_dataset_stats(&plan, &journal.done)
};
for d in &plan.datasets {
- let rt = &ds_rt[&d.slug];
let sample_queries = catalog::build_sample_queries(d, &key_corrs);
let mut notes = Vec::new();
- let dropped = rt.dropped.load(Ordering::Relaxed);
+ let durable = dataset_stats.get(&d.slug).copied().unwrap_or_default();
+ let dropped = durable.dropped;
if dropped > 0 {
notes.push(format!(
"{dropped} field values could not be coerced to the inferred types and were dropped (records still indexed)"
@@ -2377,8 +2401,8 @@ pub fn run_index_report(cfg: IndexCfg) -> Result<(i32, Option<Value>)> {
let (id, doc) = catalog::dataset_doc(&catalog::DatasetDocInput {
pd: d,
record_count: *ds_counts.get(&d.slug).unwrap_or(&0),
- junk_records: rt.junk.load(Ordering::Relaxed),
- bytes: dataset_bytes.get(&d.slug).copied().unwrap_or(0),
+ junk_records: durable.junk,
+ bytes: durable.bytes,
file_count: d.file_count,
formats,
time_min: tmin,
@@ -3129,6 +3153,7 @@ mod duplicate_integration_tests {
records,
junk: 0,
bytes: inventory.files[0].size,
+ dropped_by_dataset: HashMap::new(),
generation: Some(inventory.digests[0].clone()),
})
.unwrap();
@@ -3258,8 +3283,9 @@ mod map_metadata_tests {
file_key: "shared".into(),
path: "report.dat".into(),
records: 10,
- junk: 0,
+ junk: 5,
bytes: 100,
+ dropped_by_dataset: HashMap::from([("quarterly".into(), 7), ("annual".into(), 3)]),
generation: Some("digest".into()),
})
.unwrap();
@@ -3268,12 +3294,13 @@ mod map_metadata_tests {
file_key: "quarterly-only".into(),
path: "quarterly.json".into(),
records: 2,
- junk: 0,
+ junk: 2,
bytes: 23,
+ dropped_by_dataset: HashMap::from([("quarterly".into(), 11)]),
generation: Some("digest-2".into()),
})
.unwrap();
- let before_resume = durable_dataset_bytes(&plan, &initial.done);
+ let before_resume = durable_dataset_stats(&plan, &initial.done);
drop(initial);
// Opening the same durable run appends only an invocation-level
@@ -3288,11 +3315,15 @@ mod map_metadata_tests {
)
.unwrap();
let after_unchanged_resume =
- durable_dataset_bytes(resumed.plan.as_ref().unwrap(), &resumed.done);
+ durable_dataset_stats(resumed.plan.as_ref().unwrap(), &resumed.done);
assert_eq!(before_resume, after_unchanged_resume);
- assert_eq!(after_unchanged_resume["quarterly"], 123);
- assert_eq!(after_unchanged_resume["annual"], 100);
+ assert_eq!(after_unchanged_resume["quarterly"].bytes, 123);
+ assert_eq!(after_unchanged_resume["annual"].bytes, 100);
+ assert_eq!(after_unchanged_resume["quarterly"].junk, 7);
+ assert_eq!(after_unchanged_resume["annual"].junk, 0);
+ assert_eq!(after_unchanged_resume["quarterly"].dropped, 18);
+ assert_eq!(after_unchanged_resume["annual"].dropped, 3);
}
#[test]
@@ -3331,6 +3362,14 @@ mod map_metadata_tests {
mapping.pointer("/mappings/properties/invocation_telemetry_scope/type"),
Some(&json!("keyword"))
);
+ assert_eq!(
+ mapping.pointer("/mappings/properties/started/format"),
+ Some(&json!("strict_date_optional_time||epoch_millis"))
+ );
+ assert_eq!(
+ mapping.pointer("/mappings/properties/summary_generated_at/format"),
+ Some(&json!("strict_date_optional_time||epoch_millis"))
+ );
}
}
diff --git a/engine/crates/xerj-autoindex/src/state.rs b/engine/crates/xerj-autoindex/src/state.rs
index 50487c15f..c72621c05 100644
--- a/engine/crates/xerj-autoindex/src/state.rs
+++ b/engine/crates/xerj-autoindex/src/state.rs
@@ -246,6 +246,10 @@ pub struct FileDone {
pub records: u64,
pub junk: u64,
pub bytes: u64,
+ /// Field values dropped by coercion, grouped by the dataset whose
+ /// inferred schema rejected them. Older journals predate this accounting.
+ #[serde(default, skip_serializing_if = "HashMap::is_empty")]
+ pub dropped_by_dataset: HashMap<String, u64>,
/// Content generation committed by this record. Legacy completions have
/// no generation and cannot commit a newer pending replacement.
#[serde(default, skip_serializing_if = "Option::is_none")]
@@ -764,6 +768,7 @@ mod compatibility_tests {
records: 6_001,
junk: 0,
bytes: 100,
+ dropped_by_dataset: HashMap::new(),
generation: Some("old-generation".into()),
};
journal.file_done(&old).unwrap();
@@ -836,6 +841,7 @@ mod compatibility_tests {
records: 2,
junk: 0,
bytes: 20,
+ dropped_by_dataset: HashMap::new(),
generation: Some("generation-2".into()),
})
.unwrap();
@@ -916,6 +922,7 @@ mod compatibility_tests {
records: 10,
junk: 0,
bytes: 10,
+ dropped_by_dataset: HashMap::new(),
generation: Some("old".into()),
})
.unwrap_err();
@@ -940,6 +947,7 @@ mod compatibility_tests {
records: 2,
junk: 0,
bytes: 20,
+ dropped_by_dataset: HashMap::new(),
generation: Some("new".into()),
})
.unwrap_err();
@@ -977,6 +985,7 @@ mod compatibility_tests {
records: 2,
junk: 0,
bytes: 20,
+ dropped_by_dataset: HashMap::new(),
generation: Some("generation-c".into()),
})
.unwrap_err();
@@ -991,6 +1000,7 @@ mod compatibility_tests {
records: 3,
junk: 0,
bytes: 30,
+ dropped_by_dataset: HashMap::new(),
generation: Some("generation-d".into()),
})
.unwrap();
@@ -1008,6 +1018,7 @@ mod compatibility_tests {
records: 2,
junk: 0,
bytes: 20,
+ dropped_by_dataset: HashMap::new(),
generation: Some("generation-c".into()),
})
.unwrap();