AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
fix(autoindex): keep legacy catalogs and durable quality metadata
xerj-orgfffcde3373e5
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 9160e83f7..8765fc7ce 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -211,6 +211,37 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0document. This does **not** implement add/change/delete reconciliation for*indexed* files, which remains open.+- **Repeated `autoindex` scans keep agent-facing map metadata durable.**+ Dataset source bytes, parser-junk counts and coercion-drop notes now derive+ from committed per-file journal records instead of invocation-local worker+ counters, so an unchanged resume does not overwrite them with zero. Run+ timestamps now distinguish invocation start (`started`) from summary+ generation (`summary_generated_at`), and `junk_records_total` on the run+ document is the same durable sum the per-dataset `junk_records` reports+ rather than an invocation-local counter that read `0` after a no-op resume.+ Existing catalogs whose historical `started` field was dynamically mapped+ as text remain usable: the additive mapping upgrade no longer attempts an+ incompatible text-to-date type change.++ Two consequences worth knowing before you upgrade:++ - **`bytes` is redefined.** It used to be the bytes the latest invocation+ processed, credited to the first dataset a file was assigned to. It is now+ the complete canonical source size, counted once in **every** distinct+ dataset that source is assigned to. Per-dataset values therefore get+ larger, and summing `bytes` across datasets can exceed the corpus size+ when one source feeds several datasets — exactly as one source already+ contributed to several dataset-local file counts. It is a dataset-local+ source footprint, not a partition of physical storage.+ - **Journals written before this release carry no per-dataset coercion+ record**, because the new `dropped_by_dataset` field defaults to empty+ when an older journal is replayed. Byte and junk counts reconstruct fine+ (their journal fields already existed), but a dataset whose files are all+ unchanged loses its `N field values dropped by coercion` note on the first+ resume after upgrading, and only regains it once one of its files is+ reprocessed. Re-run with `--fresh` if the note matters more than the+ rescan.+- **The installer is now fail-closed on checksum *verification*, not just onchecksum *download*.** `landing/get` printed`warning: sha256sum/shasum not found — skipping checksum verification` anddiff --git a/engine/crates/xerj-autoindex/src/catalog.rs b/engine/crates/xerj-autoindex/src/catalog.rsindex 2bae791b7..90349e208 100644--- a/engine/crates/xerj-autoindex/src/catalog.rs+++ b/engine/crates/xerj-autoindex/src/catalog.rs@@ -56,11 +56,11 @@ pub fn catalog_mapping() -> Value {.expect("catalog mapping properties");properties.insert("started".into(),- json!({"type": "date", "format": "strict_date_optional_time"}),+ json!({"type": "date", "format": "strict_date_optional_time||epoch_millis"}),);properties.insert("summary_generated_at".into(),- json!({"type": "date", "format": "strict_date_optional_time"}),+ json!({"type": "date", "format": "strict_date_optional_time||epoch_millis"}),);properties.insert("invocation_telemetry_scope".into(),diff --git a/engine/crates/xerj-autoindex/src/failure_resume_http_tests.rs b/engine/crates/xerj-autoindex/src/failure_resume_http_tests.rsindex 3e7c4a40d..2edc173fd 100644--- a/engine/crates/xerj-autoindex/src/failure_resume_http_tests.rs+++ b/engine/crates/xerj-autoindex/src/failure_resume_http_tests.rs@@ -155,19 +155,27 @@ fn handle(mut stream: TcpStream, state: &Arc<Mutex<MockState>>) {}} else if method == "PUT" && path == "/autoindex-catalog/_mapping" {let mapping: Value = serde_json::from_slice(&body).unwrap();- let required = [- "started",- "summary_generated_at",- "invocation_telemetry_scope",- ];+ let started_requested = mapping.pointer("/properties/started").is_some();+ let required = ["summary_generated_at", "invocation_telemetry_scope"];let upgraded = required.iter().all(|field| {mapping.pointer(&format!("/properties/{field}/type")).and_then(Value::as_str).is_some()});- state.lock().unwrap().catalog_mapping_upgraded = upgraded;- (200, json!({"acknowledged": true}))+ let mut locked = state.lock().unwrap();+ if locked.catalog_preexists && started_requested {+ (+ 400,+ json!({"error": {+ "type": "mapper_parsing_exception",+ "reason": "field [started] already exists as [text], cannot add [date]"+ }}),+ )+ } else {+ locked.catalog_mapping_upgraded = upgraded;+ (200, json!({"acknowledged": true}))+ }} else if method == "POST" && path == "/_bulk" {(200, bulk_response(&body, state))} else if method == "POST" && path.contains("/_delete_by_query") {@@ -658,6 +666,46 @@ fn existing_catalog_is_upgraded_before_new_run_metadata_is_written() {);}+#[test]+fn unchanged_resume_keeps_durable_dataset_junk_and_coercion_notes() {+ let _guard = FAILPOINT_TEST_LOCK.lock().unwrap();+ let _io_guard = state::FILE_DONE_IO_FAILPOINT_TEST_LOCK.lock().unwrap();+ let corpus = tempfile::tempdir().unwrap();+ let state_dir = tempfile::tempdir().unwrap();+ fs::write(corpus.path().join("records.csv"), "id,value\n0,one\n").unwrap();+ let endpoint = MockEndpoint::start(usize::MAX);+ let config = cfg(corpus.path(), state_dir.path(), &endpoint.url);++ assert_eq!(run_index(config.clone()).unwrap(), 0);+ let root = config.root.canonicalize().unwrap();+ let mut journal = state::Journal::open(+ state_dir.path(),+ &root.to_string_lossy(),+ &config.url,+ &config.prefix,+ config.bulk_timeout_secs,+ false,+ )+ .unwrap();+ let plan = journal.plan.clone().unwrap();+ let slug = plan.datasets[0].slug.clone();+ let mut completion = journal.done.values().next().unwrap().clone();+ completion.junk = 7;+ completion.dropped_by_dataset.insert(slug.clone(), 3);+ journal.file_done(&completion).unwrap();+ drop(journal);++ assert_eq!(run_index(config).unwrap(), 0);+ let dataset = dataset_catalog_docs(&endpoint)+ .into_iter()+ .find(|doc| doc["slug"] == slug)+ .unwrap();+ assert_eq!(dataset["junk_records"], 7);+ assert!(dataset["notes"].as_array().unwrap().iter().any(|note| note+ .as_str()+ .is_some_and(|text| { text.contains("3 field values could not be coerced") })));+}+#[test]fn legacy_plan_without_completion_cleans_possible_partial_visibility() {let _guard = FAILPOINT_TEST_LOCK.lock().unwrap();diff --git a/engine/crates/xerj-autoindex/src/lib.rs b/engine/crates/xerj-autoindex/src/lib.rsindex b1d89cd52..f58218f68 100644--- a/engine/crates/xerj-autoindex/src/lib.rs+++ b/engine/crates/xerj-autoindex/src/lib.rs@@ -920,15 +920,28 @@ pub fn derive_brain_name(root: &Path) -> String {}}-/// Source bytes durably represented by each dataset.+#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)]+struct DurableDatasetStats {+ bytes: u64,+ junk: u64,+ dropped: u64,+}++/// Source metadata durably represented by each dataset.////// `FileDone` is the commit record for a successfully published canonical/// source. A source can feed more than one inferred dataset (for example, a/// workbook or database with multiple tables), so each distinct assigned/// dataset reports the complete source bytes it depends on. Repeated groups-/// within the same dataset count the source only once.-fn durable_dataset_bytes(plan: &Plan, done: &HashMap<String, FileDone>) -> HashMap<String, u64> {- let mut bytes_by_dataset = HashMap::new();+/// within the same dataset count the source only once. Parser-level junk has+/// no group after extraction rejects it, so it retains the historical+/// attribution to the file's first assigned dataset. Coercion drops retain+/// their exact dataset captured in the completion record.+fn durable_dataset_stats(+ plan: &Plan,+ done: &HashMap<String, FileDone>,+) -> HashMap<String, DurableDatasetStats> {+ let mut stats_by_dataset: HashMap<String, DurableDatasetStats> = HashMap::new();for (file_key, completed) in done {let Some(assignment) = plan.files.get(file_key) else {continue;@@ -939,11 +952,25 @@ fn durable_dataset_bytes(plan: &Plan, done: &HashMap<String, FileDone>) -> HashM.map(|(_, slug)| slug.as_str()).collect();for slug in assigned_datasets {- let bytes = bytes_by_dataset.entry(slug.to_string()).or_insert(0u64);- *bytes = bytes.saturating_add(completed.bytes);+ let stats = stats_by_dataset.entry(slug.to_string()).or_default();+ stats.bytes = stats.bytes.saturating_add(completed.bytes);+ }+ if let Some((_, slug)) = assignment.assignments.first() {+ let stats = stats_by_dataset.entry(slug.clone()).or_default();+ stats.junk = stats.junk.saturating_add(completed.junk);+ }+ for (slug, dropped) in &completed.dropped_by_dataset {+ if assignment+ .assignments+ .iter()+ .any(|(_, assigned)| assigned == slug)+ {+ let stats = stats_by_dataset.entry(slug.clone()).or_default();+ stats.dropped = stats.dropped.saturating_add(*dropped);+ }}}- bytes_by_dataset+ stats_by_dataset}fn invocation_report_timestamps(@@ -1319,8 +1346,10 @@ pub fn run_index_report(cfg: IndexCfg) -> Result<(i32, Option<Value>)> {catalog::CATALOG_INDEX,&json!({"properties": {"duplicate_of": {"type": "keyword"},- "started": {"type": "date", "format": "strict_date_optional_time"},- "summary_generated_at": {"type": "date", "format": "strict_date_optional_time"},+ // `started` intentionally stays out of this additive upgrade.+ // Older catalogs dynamically mapped it as text, and asking the+ // engine to change that existing field to date aborts every run.+ "summary_generated_at": {"type": "date", "format": "strict_date_optional_time||epoch_millis"},"invocation_telemetry_scope": {"type": "keyword"}}}),)@@ -1376,8 +1405,6 @@ pub fn run_index_report(cfg: IndexCfg) -> Result<(i32, Option<Value>)> {index: String,plan: HashMap<String, coerce::Coerce>,records: AtomicU64,- junk: AtomicU64,- dropped: AtomicU64,}let mut ds_rt: HashMap<String, DsRt> = HashMap::new();for d in &plan.datasets {@@ -1387,8 +1414,6 @@ pub fn run_index_report(cfg: IndexCfg) -> Result<(i32, Option<Value>)> {index: d.index.clone(),plan: coerce::plan_from_specs(&d.specs),records: AtomicU64::new(0),- junk: AtomicU64::new(0),- dropped: AtomicU64::new(0),},);}@@ -1659,6 +1684,7 @@ pub fn run_index_report(cfg: IndexCfg) -> Result<(i32, Option<Value>)> {};let mut file_records = 0u64;let mut file_junk = 0u64;+ let mut file_dropped_by_dataset: HashMap<String, u64> = HashMap::new();let mut send_err: Option<String> = None;// Edges this file teaches — buffered apart from the node// staging file (different target index) and sent only@@ -1761,7 +1787,9 @@ pub fn run_index_report(cfg: IndexCfg) -> Result<(i32, Option<Value>)> {let mut fields = rec.fields;let dropped = coerce::coerce_record(&mut fields, &rt.plan);if dropped > 0 {- rt.dropped.fetch_add(dropped as u64, Ordering::Relaxed);+ let durable =+ file_dropped_by_dataset.entry(slug.clone()).or_default();+ *durable = durable.saturating_add(dropped as u64);}fields.insert("ax_path".into(), Value::String(f.rel.clone()));fields.insert(@@ -2037,11 +2065,6 @@ pub fn run_index_report(cfg: IndexCfg) -> Result<(i32, Option<Value>)> {}records_total.fetch_add(file_records, Ordering::Relaxed);junk_records.fetch_add(file_junk, Ordering::Relaxed);- if let Some(rt) = fa.assignments.first().and_then(|(_, slug)| ds_rt.get(slug)) {- if file_junk > 0 {- rt.junk.fetch_add(file_junk, Ordering::Relaxed);- }- }let (commit_result, journal_path) = {let mut journal = journal_mx.lock().unwrap();let path = journal.path().display().to_string();@@ -2051,6 +2074,7 @@ pub fn run_index_report(cfg: IndexCfg) -> Result<(i32, Option<Value>)> {records: file_records,junk: file_junk,bytes: f.size,+ dropped_by_dataset: file_dropped_by_dataset,generation: Some(expected_digest.clone()),});(result, path)@@ -2336,15 +2360,15 @@ pub fn run_index_report(cfg: IndexCfg) -> Result<(i32, Option<Value>)> {// dataset docslet mut junk_records_by_run: u64 = junk_records.load(Ordering::Relaxed);- let dataset_bytes = {+ let dataset_stats = {let journal = journal_mx.lock().unwrap();- durable_dataset_bytes(&plan, &journal.done)+ durable_dataset_stats(&plan, &journal.done)};for d in &plan.datasets {- let rt = &ds_rt[&d.slug];let sample_queries = catalog::build_sample_queries(d, &key_corrs);let mut notes = Vec::new();- let dropped = rt.dropped.load(Ordering::Relaxed);+ let durable = dataset_stats.get(&d.slug).copied().unwrap_or_default();+ let dropped = durable.dropped;if dropped > 0 {notes.push(format!("{dropped} field values could not be coerced to the inferred types and were dropped (records still indexed)"@@ -2377,8 +2401,8 @@ pub fn run_index_report(cfg: IndexCfg) -> Result<(i32, Option<Value>)> {let (id, doc) = catalog::dataset_doc(&catalog::DatasetDocInput {pd: d,record_count: *ds_counts.get(&d.slug).unwrap_or(&0),- junk_records: rt.junk.load(Ordering::Relaxed),- bytes: dataset_bytes.get(&d.slug).copied().unwrap_or(0),+ junk_records: durable.junk,+ bytes: durable.bytes,file_count: d.file_count,formats,time_min: tmin,@@ -3129,6 +3153,7 @@ mod duplicate_integration_tests {records,junk: 0,bytes: inventory.files[0].size,+ dropped_by_dataset: HashMap::new(),generation: Some(inventory.digests[0].clone()),}).unwrap();@@ -3258,8 +3283,9 @@ mod map_metadata_tests {file_key: "shared".into(),path: "report.dat".into(),records: 10,- junk: 0,+ junk: 5,bytes: 100,+ dropped_by_dataset: HashMap::from([("quarterly".into(), 7), ("annual".into(), 3)]),generation: Some("digest".into()),}).unwrap();@@ -3268,12 +3294,13 @@ mod map_metadata_tests {file_key: "quarterly-only".into(),path: "quarterly.json".into(),records: 2,- junk: 0,+ junk: 2,bytes: 23,+ dropped_by_dataset: HashMap::from([("quarterly".into(), 11)]),generation: Some("digest-2".into()),}).unwrap();- let before_resume = durable_dataset_bytes(&plan, &initial.done);+ let before_resume = durable_dataset_stats(&plan, &initial.done);drop(initial);// Opening the same durable run appends only an invocation-level@@ -3288,11 +3315,15 @@ mod map_metadata_tests {).unwrap();let after_unchanged_resume =- durable_dataset_bytes(resumed.plan.as_ref().unwrap(), &resumed.done);+ durable_dataset_stats(resumed.plan.as_ref().unwrap(), &resumed.done);assert_eq!(before_resume, after_unchanged_resume);- assert_eq!(after_unchanged_resume["quarterly"], 123);- assert_eq!(after_unchanged_resume["annual"], 100);+ assert_eq!(after_unchanged_resume["quarterly"].bytes, 123);+ assert_eq!(after_unchanged_resume["annual"].bytes, 100);+ assert_eq!(after_unchanged_resume["quarterly"].junk, 7);+ assert_eq!(after_unchanged_resume["annual"].junk, 0);+ assert_eq!(after_unchanged_resume["quarterly"].dropped, 18);+ assert_eq!(after_unchanged_resume["annual"].dropped, 3);}#[test]@@ -3331,6 +3362,14 @@ mod map_metadata_tests {mapping.pointer("/mappings/properties/invocation_telemetry_scope/type"),Some(&json!("keyword")));+ assert_eq!(+ mapping.pointer("/mappings/properties/started/format"),+ Some(&json!("strict_date_optional_time||epoch_millis"))+ );+ assert_eq!(+ mapping.pointer("/mappings/properties/summary_generated_at/format"),+ Some(&json!("strict_date_optional_time||epoch_millis"))+ );}}diff --git a/engine/crates/xerj-autoindex/src/state.rs b/engine/crates/xerj-autoindex/src/state.rsindex 50487c15f..c72621c05 100644--- a/engine/crates/xerj-autoindex/src/state.rs+++ b/engine/crates/xerj-autoindex/src/state.rs@@ -246,6 +246,10 @@ pub struct FileDone {pub records: u64,pub junk: u64,pub bytes: u64,+ /// Field values dropped by coercion, grouped by the dataset whose+ /// inferred schema rejected them. Older journals predate this accounting.+ #[serde(default, skip_serializing_if = "HashMap::is_empty")]+ pub dropped_by_dataset: HashMap<String, u64>,/// Content generation committed by this record. Legacy completions have/// no generation and cannot commit a newer pending replacement.#[serde(default, skip_serializing_if = "Option::is_none")]@@ -764,6 +768,7 @@ mod compatibility_tests {records: 6_001,junk: 0,bytes: 100,+ dropped_by_dataset: HashMap::new(),generation: Some("old-generation".into()),};journal.file_done(&old).unwrap();@@ -836,6 +841,7 @@ mod compatibility_tests {records: 2,junk: 0,bytes: 20,+ dropped_by_dataset: HashMap::new(),generation: Some("generation-2".into()),}).unwrap();@@ -916,6 +922,7 @@ mod compatibility_tests {records: 10,junk: 0,bytes: 10,+ dropped_by_dataset: HashMap::new(),generation: Some("old".into()),}).unwrap_err();@@ -940,6 +947,7 @@ mod compatibility_tests {records: 2,junk: 0,bytes: 20,+ dropped_by_dataset: HashMap::new(),generation: Some("new".into()),}).unwrap_err();@@ -977,6 +985,7 @@ mod compatibility_tests {records: 2,junk: 0,bytes: 20,+ dropped_by_dataset: HashMap::new(),generation: Some("generation-c".into()),}).unwrap_err();@@ -991,6 +1000,7 @@ mod compatibility_tests {records: 3,junk: 0,bytes: 30,+ dropped_by_dataset: HashMap::new(),generation: Some("generation-d".into()),}).unwrap();@@ -1008,6 +1018,7 @@ mod compatibility_tests {records: 2,junk: 0,bytes: 20,+ dropped_by_dataset: HashMap::new(),generation: Some("generation-c".into()),}).unwrap();