AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
fix(autoindex): keep --fresh reachable, bound the refusal, and let brain see a deleted index
xerj-org4accc90091b1
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 26338783f..6c455cc26 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -555,7 +555,23 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0URL, prefix, and brain identity plus recovery instructions, including theexplicit `--fresh` rerun for a genuinely wiped data directory. Probetransport and malformed-response failures remain errors instead of being- reported as an empty destination.+ reported as an empty destination. A nodes index that was deleted out from+ under a surviving brain meta doc reads as absence and reaches that recovery+ text, rather than surfacing as a raw HTTP 404.+- **`--fresh` still recovers a state directory whose journal cannot be+ parsed.** The rerun gate reads the durable plan before the run starts, and a+ plan that is malformed, or recorded for a different root/URL/prefix, is fatal+ to a resume — but not to `--fresh`, which deletes that journal unread. The+ preflight is now no more fatal than the open it precedes: under `--fresh` the+ unreadable plan is reported on stderr and rebuilt from the current folder.+ Without `--fresh` the refusal is unchanged. Because the removal gate has no+ comparison basis in that case, the warning says so: documents already+ published for files that are now gone cannot be identified from an unreadable+ plan and are not deleted.+- **Refusal and skip listings are capped at ten entries plus an "and N more"+ tail.** Unmounting a bind mount under an indexed root vanishes every content+ group at once, so the uncapped listing was one rendered entry per journalled+ file. `--json` still carries every entry.## [1.0.0-rc.10] - 2026-08-03diff --git a/engine/crates/xerj-autoindex/src/esclient.rs b/engine/crates/xerj-autoindex/src/esclient.rsindex 3a80a22c8..d5f9875fd 100644--- a/engine/crates/xerj-autoindex/src/esclient.rs+++ b/engine/crates/xerj-autoindex/src/esclient.rs@@ -665,7 +665,7 @@ impl Es {)}- pub fn search(&self, index: &str, body: &Value) -> Result<Value> {+ fn search_raw(&self, index: &str, body: &Value) -> Result<(reqwest::StatusCode, Value)> {self.with_retry("search",|| {@@ -677,14 +677,38 @@ impl Es {|resp| {let status = resp.status();let v: Value = resp.json().unwrap_or(Value::Null);- if !status.is_success() {- return Err(anyhow!("search /{index} HTTP {status}: {v}"));- }- Ok(v)+ Ok((status, v))},)}+ pub fn search(&self, index: &str, body: &Value) -> Result<Value> {+ let (status, v) = self.search_raw(index, body)?;+ if !status.is_success() {+ return Err(anyhow!("search /{index} HTTP {status}: {v}"));+ }+ Ok(v)+ }++ /// `search`, but a missing index is an answer rather than a failure.+ ///+ /// A probe whose whole job is to decide "does the server still hold what+ /// the journal says it holds?" must be able to tell *the index is gone*+ /// apart from *the server refused* — only the first has a recovery worth+ /// printing, and propagating its 404 replaces that recovery text with a raw+ /// HTTP error. Every other status keeps `search`'s behaviour, so this never+ /// launders a real failure into an absence.+ pub fn search_present(&self, index: &str, body: &Value) -> Result<Option<Value>> {+ let (status, v) = self.search_raw(index, body)?;+ if status == reqwest::StatusCode::NOT_FOUND {+ return Ok(None);+ }+ if !status.is_success() {+ return Err(anyhow!("search /{index} HTTP {status}: {v}"));+ }+ Ok(Some(v))+ }+pub fn count(&self, index: &str) -> Result<u64> {// _count may not exist on all builds — use size:0 search with totals.let v = self.search(@@ -822,15 +846,72 @@ mod tests {}fn respond_json(stream: &mut std::net::TcpStream, body: &[u8]) {+ respond_status(stream, "200 OK", body);+ }++ fn respond_status(stream: &mut std::net::TcpStream, status: &str, body: &[u8]) {write!(stream,- "HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n",+ "HTTP/1.1 {status}\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n",body.len()).unwrap();stream.write_all(body).unwrap();}+ /// A deleted index is an answer for the probes that ask whether the server+ /// still holds what a journal claims it holds — `search` propagating that+ /// 404 replaces `xerj brain`'s recovery text with a raw HTTP error. Every+ /// other refusal must still propagate, so absence is never laundered out of+ /// a real failure.+ #[test]+ fn search_present_reports_a_missing_index_as_absence_and_nothing_else() {+ for (status, body, expect_absent) in [+ (+ "404 Not Found",+ &br#"{"error":{"type":"index_not_found_exception"},"status":404}"#[..],+ true,+ ),+ (+ "403 Forbidden",+ &br#"{"error":{"type":"security_exception"},"status":403}"#[..],+ false,+ ),+ (+ "200 OK",+ &br#"{"hits":{"total":{"value":7,"relation":"eq"}}}"#[..],+ false,+ ),+ ] {+ let listener = TcpListener::bind("127.0.0.1:0").unwrap();+ let address = listener.local_addr().unwrap();+ let owned = body.to_vec();+ let status_line = status.to_owned();+ let server = std::thread::spawn(move || {+ let (mut stream, _) = listener.accept().unwrap();+ let request = read_request(&mut stream);+ assert!(+ String::from_utf8_lossy(&request).starts_with("POST /nodes/_search HTTP/1.1"),+ "{}",+ String::from_utf8_lossy(&request)+ );+ respond_status(&mut stream, &status_line, &owned);+ });+ let es = Es::new(&format!("http://{address}"), None).unwrap();+ let result = es.search_present("nodes", &serde_json::json!({"size": 0}));+ if expect_absent {+ assert!(result.unwrap().is_none(), "{status}");+ } else if status.starts_with("200") {+ let value = result.unwrap().expect("a served index is present");+ assert_eq!(value.pointer("/hits/total/value").unwrap(), 7);+ } else {+ let error = result.unwrap_err();+ assert!(format!("{error:#}").contains("403"), "{error:#}");+ }+ server.join().unwrap();+ }+ }+#[test]fn embedding_identity_uses_native_endpoint_and_parses_sanitized_data() {let listener = TcpListener::bind("127.0.0.1:0").unwrap();diff --git a/engine/crates/xerj-autoindex/src/failure_resume_http_tests.rs b/engine/crates/xerj-autoindex/src/failure_resume_http_tests.rsindex bf700acce..98c34becd 100644--- a/engine/crates/xerj-autoindex/src/failure_resume_http_tests.rs+++ b/engine/crates/xerj-autoindex/src/failure_resume_http_tests.rs@@ -583,7 +583,13 @@ fn a_rerun_after_an_added_file_succeeds_and_fresh_absorbs_it() {assert_eq!(code, 3);let report = report.unwrap();assert_eq!(report["files_junk"], 1, "{report}");- assert_eq!(report["records_total"], 0, "{report}");+ // `records_total` is the live server-side count (#195), so it still reports+ // the record the first run published. What must be zero is the run-local+ // counter: this rerun indexed nothing, because the frozen plan cannot+ // absorb a file that appeared after it was written.+ assert_eq!(report["records_submitted_this_run"], 0, "{report}");+ assert_eq!(report["files_submitted_this_run"], 0, "{report}");+ assert_eq!(report["records_total"], 1, "{report}");{let locked = endpoint.state.lock().unwrap();let live: Vec<&str> = lockeddiff --git a/engine/crates/xerj-autoindex/src/lib.rs b/engine/crates/xerj-autoindex/src/lib.rsindex 688d3de31..923f84b95 100644--- a/engine/crates/xerj-autoindex/src/lib.rs+++ b/engine/crates/xerj-autoindex/src/lib.rs@@ -102,6 +102,12 @@ pub fn run_cli() -> i32 {}const GB: u64 = 1 << 30;+/// How many entries a human-facing listing prints before it summarises the+/// rest. These lists are bounded by the corpus, not by the fault: unmounting a+/// bind mount under an indexed root makes every content group vanish at once,+/// so an uncapped listing is one rendered entry per journalled file — megabytes+/// of stderr, in the code paths whose entire job is to be read by a person.+const REFUSAL_LIST_CAP: usize = 10;const SAMPLE_LIMIT_BYTES: u64 = 4 << 20;const SQLDUMP_SAMPLE_LIMIT: u64 = 64 << 20;@@ -1211,12 +1217,21 @@ impl UnsupportedInventoryDeltaError {impl std::fmt::Display for UnsupportedInventoryDeltaError {fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {+ // Capped like the duplicate and unplanned-file listings above; see+ // REFUSAL_LIST_CAP. The machine-readable `--json` rendering keeps the+ // full lists, so nothing is lost — only the prose is bounded.let render = |entries: &[InventoryDeltaEntry]| {- entries+ let mut rendered = entries.iter()+ .take(REFUSAL_LIST_CAP).map(|entry| format!("{} ({})", entry.path, entry.file_key)).collect::<Vec<_>>()- .join(", ")+ .join(", ");+ let remaining = entries.len().saturating_sub(REFUSAL_LIST_CAP);+ if remaining > 0 {+ rendered.push_str(&format!(", … and {remaining} more"));+ }+ rendered};write!(formatter,@@ -1515,7 +1530,20 @@ pub fn run_index_report(cfg: IndexCfg) -> Result<(i32, Option<Value>)> {// Acquire state authority before discovery as well as hashing. A waiter// must never classify a path snapshot taken while another owner was// publishing or replacing the durable plan.- let preflight = state::Journal::preflight(&state_dir, &root_str, &cfg.url, &cfg.prefix)?;+ let preflight =+ state::Journal::preflight(&state_dir, &root_str, &cfg.url, &cfg.prefix, cfg.fresh)?;+ if let Some(reason) = preflight.unreadable_plan.as_deref() {+ // Only reachable under --fresh; without it the preflight would have+ // returned this as an error. It goes through the progress surface, not+ // a raw eprintln!, so `--progress none` (which `--quiet` selects) stays+ // silent and `--progress json` stays one parseable stream (#241).+ pr.note(&format!(+ "autoindex: --fresh: the durable resume plan in {} could not be read ({reason}); \+ rebuilding it from the current folder. Documents published for files that are no \+ longer present cannot be identified from an unreadable plan and are not deleted.",+ state_dir.join("journal.ndjson").display()+ ));+ }// Totals are unknown until the walk returns, so this phase honestly// reports `pct=unknown` and proves liveness with the clock alone.pr.phase("walk", 0, 0);@@ -1722,14 +1750,15 @@ pub fn run_index_report(cfg: IndexCfg) -> Result<(i32, Option<Value>)> {"autoindex: {} byte-identical duplicate path(s) will reuse canonical content",duplicate_files.len()));- for duplicate in duplicate_files.iter().take(10) {+ for duplicate in duplicate_files.iter().take(REFUSAL_LIST_CAP) {pr.note(&format!(" duplicate: {} → {}",duplicate.rel, duplicate.duplicate_of));}- if duplicate_files.len() > 10 {- pr.note(&format!(" … and {} more", duplicate_files.len() - 10));+ let remaining = duplicate_files.len().saturating_sub(REFUSAL_LIST_CAP);+ if remaining > 0 {+ pr.note(&format!(" … and {remaining} more"));}}@@ -1995,11 +2024,12 @@ pub fn run_index_report(cfg: IndexCfg) -> Result<(i32, Option<Value>)> {indexed:",new_unplanned.len());- for jf in new_unplanned.iter().take(10) {+ for jf in new_unplanned.iter().take(REFUSAL_LIST_CAP) {eprintln!(" not in plan, skipped: {}", jf.rel);}- if new_unplanned.len() > 10 {- eprintln!(" … and {} more", new_unplanned.len() - 10);+ let remaining = new_unplanned.len().saturating_sub(REFUSAL_LIST_CAP);+ if remaining > 0 {+ eprintln!(" … and {remaining} more");}eprintln!("autoindex: re-run with --fresh to rebuild the plan in place and index them (ids \@@ -3802,6 +3832,55 @@ mod inventory_delta_tests {assert!(message.contains("`--fresh`"), "{message}");}+ /// The prose refusal is bounded by REFUSAL_LIST_CAP, not by the corpus: an+ /// unmounted bind mount under an indexed root vanishes every group at once,+ /// and rendering 82k entries into one String is megabytes of stderr in the+ /// one path whose job is to be read. `--json` still carries all of them.+ #[test]+ fn refusal_prose_caps_its_listings_while_json_keeps_every_entry() {+ let mut plan = Plan::default();+ let total = REFUSAL_LIST_CAP * 3;+ for i in 0..total {+ plan.files.insert(+ format!("gone{i:03}"),+ assignment(&format!("gone{i:03}.csv")),+ );+ }+ let error = UnsupportedInventoryDelta::between(&[], &[], &plan).into_error(targets());+ let message = format!("{error:#}");+ assert!(message.contains("gone000.csv"), "{message}");+ assert!(+ message.contains(&format!(", … and {} more", total - REFUSAL_LIST_CAP)),+ "{message}"+ );+ // The tail past the cap must not be rendered at all, not merely elided.+ let last = format!("gone{:03}.csv", total - 1);+ assert!(!message.contains(&last), "{message}");+ assert_eq!(+ message.matches(".csv (").count(),+ REFUSAL_LIST_CAP,+ "{message}"+ );++ let stdout = route_cli_error(&error, true)+ .stdout+ .expect("the typed refusal renders as JSON under --json");+ let value: Value = serde_json::from_str(&stdout).unwrap();+ assert_eq!(+ value["vanished_content_groups"].as_array().unwrap().len(),+ total+ );+ }++ #[test]+ fn refusal_prose_below_the_cap_has_no_more_tail() {+ let mut plan = Plan::default();+ plan.files.insert("gone".into(), assignment("gone.csv"));+ let error = UnsupportedInventoryDelta::between(&[], &[], &plan).into_error(targets());+ let message = format!("{error:#}");+ assert!(!message.contains("… and "), "{message}");+ }+#[test]fn cli_error_routing_separates_typed_json_from_unrelated_human_errors() {let typed = UnsupportedInventoryDelta {diff --git a/engine/crates/xerj-autoindex/src/state.rs b/engine/crates/xerj-autoindex/src/state.rsindex 1a63c0487..992d79530 100644--- a/engine/crates/xerj-autoindex/src/state.rs+++ b/engine/crates/xerj-autoindex/src/state.rs@@ -179,10 +179,76 @@ mod tests {}}+ /// The preflight must never be more fatal than the open it precedes.+ /// `open_after_preflight` deletes the journal unread when `--fresh` is set,+ /// so a preflight that hard-errors while parsing that journal would make+ /// `--fresh` unreachable in exactly the states whose error text recommends+ /// it — a state directory no supported flag can recover.+ #[test]+ fn fresh_recovers_journals_that_the_preflight_refuses_to_resume() {+ for (label, journal, refusal_marker) in [+ (+ "malformed record",+ concat!(+ "{\"v\":1,\"kind\":\"run\",\"root\":\"root\",\"url\":\"url\",",+ "\"prefix\":\"prefix\",\"run_id\":\"legacy\"}\n",+ "{not json at all\n"+ ),+ "--fresh",+ ),+ (+ "root/url/prefix mismatch",+ concat!(+ "{\"v\":1,\"kind\":\"run\",\"root\":\"elsewhere\",\"url\":\"url\",",+ "\"prefix\":\"prefix\",\"run_id\":\"legacy\"}\n"+ ),+ "was created for root=",+ ),+ ] {+ let dir = tempfile::tempdir().unwrap();+ let path = dir.path().join("journal.ndjson");+ std::fs::write(&path, journal).unwrap();++ let refused = Journal::preflight(dir.path(), "root", "url", "prefix", false)+ .err()+ .unwrap_or_else(|| panic!("{label}: a resume must still refuse this journal"));+ let refused = format!("{refused:#}");+ assert!(refused.contains(refusal_marker), "{label}: {refused}");++ let preflight = Journal::preflight(dir.path(), "root", "url", "prefix", true)+ .unwrap_or_else(|e| panic!("{label}: --fresh must pass the preflight: {e:#}"));+ assert!(preflight.plan.is_none(), "{label}");+ let reason = preflight+ .unreadable_plan+ .clone()+ .unwrap_or_else(|| panic!("{label}: the discarded plan must be reported"));+ assert!(!reason.is_empty(), "{label}");++ let journal =+ Journal::open_after_preflight(preflight, "root", "url", "prefix", 300, true)+ .unwrap_or_else(|e| panic!("{label}: --fresh must open: {e:#}"));+ drop(journal);+ // And the rebuilt state directory is resumable again without --fresh.+ Journal::open(dir.path(), "root", "url", "prefix", 300, false)+ .unwrap_or_else(|e| panic!("{label}: the rebuilt journal must resume: {e:#}"));+ }+ }++ /// A readable journal must not be reported as unreadable just because the+ /// run asked for `--fresh`; the note is a discarded-state warning, not a+ /// `--fresh` banner.+ #[test]+ fn fresh_over_a_readable_journal_reports_nothing_unreadable() {+ let dir = tempfile::tempdir().unwrap();+ drop(Journal::open(dir.path(), "root", "url", "prefix", 300, true).unwrap());+ let preflight = Journal::preflight(dir.path(), "root", "url", "prefix", true).unwrap();+ assert!(preflight.unreadable_plan.is_none());+ }+#[test]fn preflight_holds_the_same_exclusive_lock_through_authoritative_open() {let dir = tempfile::tempdir().unwrap();- let preflight = Journal::preflight(dir.path(), "root", "url", "prefix").unwrap();+ let preflight = Journal::preflight(dir.path(), "root", "url", "prefix", false).unwrap();let error = Journal::open(dir.path(), "root", "url", "prefix", 300, false).err().expect("a second opener must not pass the held preflight lock");@@ -376,14 +442,25 @@ pub struct JournalPreflight {state_lock: std::fs::File,pub plan: Option<Plan>,pub journal_exists: bool,+ /// Set only under `--fresh`, when the durable plan could not be read and+ /// the run is about to discard the journal anyway. The caller owns the+ /// decision to print it (autoindex honours `--quiet`).+ pub unreadable_plan: Option<String>,}impl Journal {+ /// `fresh` is not a formality here. `open_after_preflight` removes the+ /// journal before replaying it when `--fresh` is set, so nothing inside a+ /// journal can be fatal to that path — and a preflight that hard-errors on+ /// a corrupt record or a root/url/prefix mismatch would make `--fresh`+ /// unreachable in exactly the cases whose error text recommends it. The+ /// preflight must therefore never be more fatal than the open it precedes.pub fn preflight(state_dir: &Path,root: &str,url: &str,prefix: &str,+ fresh: bool,) -> Result<JournalPreflight> {std::fs::create_dir_all(state_dir).with_context(|| format!("create state dir {}", state_dir.display()))?;@@ -403,12 +480,22 @@ impl Journal {})?;let journal_path = state_dir.join("journal.ndjson");let journal_exists = journal_path.exists();- let plan = read_plan_for_preflight(&journal_path, root, url, prefix)?;+ let (plan, unreadable_plan) =+ match read_plan_for_preflight(&journal_path, root, url, prefix) {+ Ok(plan) => (plan, None),+ // The journal is about to be deleted unread. Losing the plan here+ // costs the removed-file gate its comparison basis, which is+ // exactly what a full in-place rebuild accepts; the alternative is+ // a state directory that no supported flag can recover.+ Err(error) if fresh => (None, Some(format!("{error:#}"))),+ Err(error) => return Err(error),+ };Ok(JournalPreflight {state_dir: state_dir.to_owned(),state_lock,plan,journal_exists,+ unreadable_plan,})}@@ -420,7 +507,7 @@ impl Journal {bulk_timeout_secs: u64,fresh: bool,) -> Result<Journal> {- let preflight = Self::preflight(state_dir, root, url, prefix)?;+ let preflight = Self::preflight(state_dir, root, url, prefix, fresh)?;Self::open_after_preflight(preflight, root, url, prefix, bulk_timeout_secs, fresh)}diff --git a/engine/crates/xerj-server/src/brain.rs b/engine/crates/xerj-server/src/brain.rsindex 8e8e62e48..594870e32 100644--- a/engine/crates/xerj-server/src/brain.rs+++ b/engine/crates/xerj-server/src/brain.rs@@ -398,10 +398,19 @@ fn live_node_docs(es: &Es, brain: &str) -> Result<Option<u64>> {else {return Ok(None);};- let response = es.search(+ // A meta doc that outlived its nodes index is precisely the wiped-data-dir+ // case this probe exists to catch, so the index being gone must reach+ // `journal_server_disagreement()` as absence. Propagating the 404 would+ // short-circuit the caller and print a raw HTTP error instead of the+ // recovery text. Any other failure still propagates: a probe failure is+ // never converted into reset authorization.+ let Some(response) = es.search_present(nodes_index,&json!({"size": 0, "track_total_hits": true, "query": {"match_all": {}}}),- )?;+ )?+ else {+ return Ok(None);+ };response.pointer("/hits/total/value").and_then(Value::as_u64)