Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

fix(autoindex): keep --fresh reachable, bound the refusal, and let brain see a deleted index

xerj-org4accc90091b1
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 26338783f..6c455cc26 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -555,7 +555,23 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
URL, prefix, and brain identity plus recovery instructions, including the
explicit `--fresh` rerun for a genuinely wiped data directory. Probe
transport and malformed-response failures remain errors instead of being
- reported as an empty destination.
+ reported as an empty destination. A nodes index that was deleted out from
+ under a surviving brain meta doc reads as absence and reaches that recovery
+ text, rather than surfacing as a raw HTTP 404.
+- **`--fresh` still recovers a state directory whose journal cannot be
+ parsed.** The rerun gate reads the durable plan before the run starts, and a
+ plan that is malformed, or recorded for a different root/URL/prefix, is fatal
+ to a resume — but not to `--fresh`, which deletes that journal unread. The
+ preflight is now no more fatal than the open it precedes: under `--fresh` the
+ unreadable plan is reported on stderr and rebuilt from the current folder.
+ Without `--fresh` the refusal is unchanged. Because the removal gate has no
+ comparison basis in that case, the warning says so: documents already
+ published for files that are now gone cannot be identified from an unreadable
+ plan and are not deleted.
+- **Refusal and skip listings are capped at ten entries plus an "and N more"
+ tail.** Unmounting a bind mount under an indexed root vanishes every content
+ group at once, so the uncapped listing was one rendered entry per journalled
+ file. `--json` still carries every entry.
## [1.0.0-rc.10] - 2026-08-03
diff --git a/engine/crates/xerj-autoindex/src/esclient.rs b/engine/crates/xerj-autoindex/src/esclient.rs
index 3a80a22c8..d5f9875fd 100644
--- a/engine/crates/xerj-autoindex/src/esclient.rs
+++ b/engine/crates/xerj-autoindex/src/esclient.rs
@@ -665,7 +665,7 @@ impl Es {
)
}
- pub fn search(&self, index: &str, body: &Value) -> Result<Value> {
+ fn search_raw(&self, index: &str, body: &Value) -> Result<(reqwest::StatusCode, Value)> {
self.with_retry(
"search",
|| {
@@ -677,14 +677,38 @@ impl Es {
|resp| {
let status = resp.status();
let v: Value = resp.json().unwrap_or(Value::Null);
- if !status.is_success() {
- return Err(anyhow!("search /{index} HTTP {status}: {v}"));
- }
- Ok(v)
+ Ok((status, v))
},
)
}
+ pub fn search(&self, index: &str, body: &Value) -> Result<Value> {
+ let (status, v) = self.search_raw(index, body)?;
+ if !status.is_success() {
+ return Err(anyhow!("search /{index} HTTP {status}: {v}"));
+ }
+ Ok(v)
+ }
+
+ /// `search`, but a missing index is an answer rather than a failure.
+ ///
+ /// A probe whose whole job is to decide "does the server still hold what
+ /// the journal says it holds?" must be able to tell *the index is gone*
+ /// apart from *the server refused* — only the first has a recovery worth
+ /// printing, and propagating its 404 replaces that recovery text with a raw
+ /// HTTP error. Every other status keeps `search`'s behaviour, so this never
+ /// launders a real failure into an absence.
+ pub fn search_present(&self, index: &str, body: &Value) -> Result<Option<Value>> {
+ let (status, v) = self.search_raw(index, body)?;
+ if status == reqwest::StatusCode::NOT_FOUND {
+ return Ok(None);
+ }
+ if !status.is_success() {
+ return Err(anyhow!("search /{index} HTTP {status}: {v}"));
+ }
+ Ok(Some(v))
+ }
+
pub fn count(&self, index: &str) -> Result<u64> {
// _count may not exist on all builds — use size:0 search with totals.
let v = self.search(
@@ -822,15 +846,72 @@ mod tests {
}
fn respond_json(stream: &mut std::net::TcpStream, body: &[u8]) {
+ respond_status(stream, "200 OK", body);
+ }
+
+ fn respond_status(stream: &mut std::net::TcpStream, status: &str, body: &[u8]) {
write!(
stream,
- "HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n",
+ "HTTP/1.1 {status}\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n",
body.len()
)
.unwrap();
stream.write_all(body).unwrap();
}
+ /// A deleted index is an answer for the probes that ask whether the server
+ /// still holds what a journal claims it holds — `search` propagating that
+ /// 404 replaces `xerj brain`'s recovery text with a raw HTTP error. Every
+ /// other refusal must still propagate, so absence is never laundered out of
+ /// a real failure.
+ #[test]
+ fn search_present_reports_a_missing_index_as_absence_and_nothing_else() {
+ for (status, body, expect_absent) in [
+ (
+ "404 Not Found",
+ &br#"{"error":{"type":"index_not_found_exception"},"status":404}"#[..],
+ true,
+ ),
+ (
+ "403 Forbidden",
+ &br#"{"error":{"type":"security_exception"},"status":403}"#[..],
+ false,
+ ),
+ (
+ "200 OK",
+ &br#"{"hits":{"total":{"value":7,"relation":"eq"}}}"#[..],
+ false,
+ ),
+ ] {
+ let listener = TcpListener::bind("127.0.0.1:0").unwrap();
+ let address = listener.local_addr().unwrap();
+ let owned = body.to_vec();
+ let status_line = status.to_owned();
+ let server = std::thread::spawn(move || {
+ let (mut stream, _) = listener.accept().unwrap();
+ let request = read_request(&mut stream);
+ assert!(
+ String::from_utf8_lossy(&request).starts_with("POST /nodes/_search HTTP/1.1"),
+ "{}",
+ String::from_utf8_lossy(&request)
+ );
+ respond_status(&mut stream, &status_line, &owned);
+ });
+ let es = Es::new(&format!("http://{address}"), None).unwrap();
+ let result = es.search_present("nodes", &serde_json::json!({"size": 0}));
+ if expect_absent {
+ assert!(result.unwrap().is_none(), "{status}");
+ } else if status.starts_with("200") {
+ let value = result.unwrap().expect("a served index is present");
+ assert_eq!(value.pointer("/hits/total/value").unwrap(), 7);
+ } else {
+ let error = result.unwrap_err();
+ assert!(format!("{error:#}").contains("403"), "{error:#}");
+ }
+ server.join().unwrap();
+ }
+ }
+
#[test]
fn embedding_identity_uses_native_endpoint_and_parses_sanitized_data() {
let listener = TcpListener::bind("127.0.0.1:0").unwrap();
diff --git a/engine/crates/xerj-autoindex/src/failure_resume_http_tests.rs b/engine/crates/xerj-autoindex/src/failure_resume_http_tests.rs
index bf700acce..98c34becd 100644
--- a/engine/crates/xerj-autoindex/src/failure_resume_http_tests.rs
+++ b/engine/crates/xerj-autoindex/src/failure_resume_http_tests.rs
@@ -583,7 +583,13 @@ fn a_rerun_after_an_added_file_succeeds_and_fresh_absorbs_it() {
assert_eq!(code, 3);
let report = report.unwrap();
assert_eq!(report["files_junk"], 1, "{report}");
- assert_eq!(report["records_total"], 0, "{report}");
+ // `records_total` is the live server-side count (#195), so it still reports
+ // the record the first run published. What must be zero is the run-local
+ // counter: this rerun indexed nothing, because the frozen plan cannot
+ // absorb a file that appeared after it was written.
+ assert_eq!(report["records_submitted_this_run"], 0, "{report}");
+ assert_eq!(report["files_submitted_this_run"], 0, "{report}");
+ assert_eq!(report["records_total"], 1, "{report}");
{
let locked = endpoint.state.lock().unwrap();
let live: Vec<&str> = locked
diff --git a/engine/crates/xerj-autoindex/src/lib.rs b/engine/crates/xerj-autoindex/src/lib.rs
index 688d3de31..923f84b95 100644
--- a/engine/crates/xerj-autoindex/src/lib.rs
+++ b/engine/crates/xerj-autoindex/src/lib.rs
@@ -102,6 +102,12 @@ pub fn run_cli() -> i32 {
}
const GB: u64 = 1 << 30;
+/// How many entries a human-facing listing prints before it summarises the
+/// rest. These lists are bounded by the corpus, not by the fault: unmounting a
+/// bind mount under an indexed root makes every content group vanish at once,
+/// so an uncapped listing is one rendered entry per journalled file — megabytes
+/// of stderr, in the code paths whose entire job is to be read by a person.
+const REFUSAL_LIST_CAP: usize = 10;
const SAMPLE_LIMIT_BYTES: u64 = 4 << 20;
const SQLDUMP_SAMPLE_LIMIT: u64 = 64 << 20;
@@ -1211,12 +1217,21 @@ impl UnsupportedInventoryDeltaError {
impl std::fmt::Display for UnsupportedInventoryDeltaError {
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+ // Capped like the duplicate and unplanned-file listings above; see
+ // REFUSAL_LIST_CAP. The machine-readable `--json` rendering keeps the
+ // full lists, so nothing is lost — only the prose is bounded.
let render = |entries: &[InventoryDeltaEntry]| {
- entries
+ let mut rendered = entries
.iter()
+ .take(REFUSAL_LIST_CAP)
.map(|entry| format!("{} ({})", entry.path, entry.file_key))
.collect::<Vec<_>>()
- .join(", ")
+ .join(", ");
+ let remaining = entries.len().saturating_sub(REFUSAL_LIST_CAP);
+ if remaining > 0 {
+ rendered.push_str(&format!(", … and {remaining} more"));
+ }
+ rendered
};
write!(
formatter,
@@ -1515,7 +1530,20 @@ pub fn run_index_report(cfg: IndexCfg) -> Result<(i32, Option<Value>)> {
// Acquire state authority before discovery as well as hashing. A waiter
// must never classify a path snapshot taken while another owner was
// publishing or replacing the durable plan.
- let preflight = state::Journal::preflight(&state_dir, &root_str, &cfg.url, &cfg.prefix)?;
+ let preflight =
+ state::Journal::preflight(&state_dir, &root_str, &cfg.url, &cfg.prefix, cfg.fresh)?;
+ if let Some(reason) = preflight.unreadable_plan.as_deref() {
+ // Only reachable under --fresh; without it the preflight would have
+ // returned this as an error. It goes through the progress surface, not
+ // a raw eprintln!, so `--progress none` (which `--quiet` selects) stays
+ // silent and `--progress json` stays one parseable stream (#241).
+ pr.note(&format!(
+ "autoindex: --fresh: the durable resume plan in {} could not be read ({reason}); \
+ rebuilding it from the current folder. Documents published for files that are no \
+ longer present cannot be identified from an unreadable plan and are not deleted.",
+ state_dir.join("journal.ndjson").display()
+ ));
+ }
// Totals are unknown until the walk returns, so this phase honestly
// reports `pct=unknown` and proves liveness with the clock alone.
pr.phase("walk", 0, 0);
@@ -1722,14 +1750,15 @@ pub fn run_index_report(cfg: IndexCfg) -> Result<(i32, Option<Value>)> {
"autoindex: {} byte-identical duplicate path(s) will reuse canonical content",
duplicate_files.len()
));
- for duplicate in duplicate_files.iter().take(10) {
+ for duplicate in duplicate_files.iter().take(REFUSAL_LIST_CAP) {
pr.note(&format!(
" duplicate: {} → {}",
duplicate.rel, duplicate.duplicate_of
));
}
- if duplicate_files.len() > 10 {
- pr.note(&format!(" … and {} more", duplicate_files.len() - 10));
+ let remaining = duplicate_files.len().saturating_sub(REFUSAL_LIST_CAP);
+ if remaining > 0 {
+ pr.note(&format!(" … and {remaining} more"));
}
}
@@ -1995,11 +2024,12 @@ pub fn run_index_report(cfg: IndexCfg) -> Result<(i32, Option<Value>)> {
indexed:",
new_unplanned.len()
);
- for jf in new_unplanned.iter().take(10) {
+ for jf in new_unplanned.iter().take(REFUSAL_LIST_CAP) {
eprintln!(" not in plan, skipped: {}", jf.rel);
}
- if new_unplanned.len() > 10 {
- eprintln!(" … and {} more", new_unplanned.len() - 10);
+ let remaining = new_unplanned.len().saturating_sub(REFUSAL_LIST_CAP);
+ if remaining > 0 {
+ eprintln!(" … and {remaining} more");
}
eprintln!(
"autoindex: re-run with --fresh to rebuild the plan in place and index them (ids \
@@ -3802,6 +3832,55 @@ mod inventory_delta_tests {
assert!(message.contains("`--fresh`"), "{message}");
}
+ /// The prose refusal is bounded by REFUSAL_LIST_CAP, not by the corpus: an
+ /// unmounted bind mount under an indexed root vanishes every group at once,
+ /// and rendering 82k entries into one String is megabytes of stderr in the
+ /// one path whose job is to be read. `--json` still carries all of them.
+ #[test]
+ fn refusal_prose_caps_its_listings_while_json_keeps_every_entry() {
+ let mut plan = Plan::default();
+ let total = REFUSAL_LIST_CAP * 3;
+ for i in 0..total {
+ plan.files.insert(
+ format!("gone{i:03}"),
+ assignment(&format!("gone{i:03}.csv")),
+ );
+ }
+ let error = UnsupportedInventoryDelta::between(&[], &[], &plan).into_error(targets());
+ let message = format!("{error:#}");
+ assert!(message.contains("gone000.csv"), "{message}");
+ assert!(
+ message.contains(&format!(", … and {} more", total - REFUSAL_LIST_CAP)),
+ "{message}"
+ );
+ // The tail past the cap must not be rendered at all, not merely elided.
+ let last = format!("gone{:03}.csv", total - 1);
+ assert!(!message.contains(&last), "{message}");
+ assert_eq!(
+ message.matches(".csv (").count(),
+ REFUSAL_LIST_CAP,
+ "{message}"
+ );
+
+ let stdout = route_cli_error(&error, true)
+ .stdout
+ .expect("the typed refusal renders as JSON under --json");
+ let value: Value = serde_json::from_str(&stdout).unwrap();
+ assert_eq!(
+ value["vanished_content_groups"].as_array().unwrap().len(),
+ total
+ );
+ }
+
+ #[test]
+ fn refusal_prose_below_the_cap_has_no_more_tail() {
+ let mut plan = Plan::default();
+ plan.files.insert("gone".into(), assignment("gone.csv"));
+ let error = UnsupportedInventoryDelta::between(&[], &[], &plan).into_error(targets());
+ let message = format!("{error:#}");
+ assert!(!message.contains("… and "), "{message}");
+ }
+
#[test]
fn cli_error_routing_separates_typed_json_from_unrelated_human_errors() {
let typed = UnsupportedInventoryDelta {
diff --git a/engine/crates/xerj-autoindex/src/state.rs b/engine/crates/xerj-autoindex/src/state.rs
index 1a63c0487..992d79530 100644
--- a/engine/crates/xerj-autoindex/src/state.rs
+++ b/engine/crates/xerj-autoindex/src/state.rs
@@ -179,10 +179,76 @@ mod tests {
}
}
+ /// The preflight must never be more fatal than the open it precedes.
+ /// `open_after_preflight` deletes the journal unread when `--fresh` is set,
+ /// so a preflight that hard-errors while parsing that journal would make
+ /// `--fresh` unreachable in exactly the states whose error text recommends
+ /// it — a state directory no supported flag can recover.
+ #[test]
+ fn fresh_recovers_journals_that_the_preflight_refuses_to_resume() {
+ for (label, journal, refusal_marker) in [
+ (
+ "malformed record",
+ concat!(
+ "{\"v\":1,\"kind\":\"run\",\"root\":\"root\",\"url\":\"url\",",
+ "\"prefix\":\"prefix\",\"run_id\":\"legacy\"}\n",
+ "{not json at all\n"
+ ),
+ "--fresh",
+ ),
+ (
+ "root/url/prefix mismatch",
+ concat!(
+ "{\"v\":1,\"kind\":\"run\",\"root\":\"elsewhere\",\"url\":\"url\",",
+ "\"prefix\":\"prefix\",\"run_id\":\"legacy\"}\n"
+ ),
+ "was created for root=",
+ ),
+ ] {
+ let dir = tempfile::tempdir().unwrap();
+ let path = dir.path().join("journal.ndjson");
+ std::fs::write(&path, journal).unwrap();
+
+ let refused = Journal::preflight(dir.path(), "root", "url", "prefix", false)
+ .err()
+ .unwrap_or_else(|| panic!("{label}: a resume must still refuse this journal"));
+ let refused = format!("{refused:#}");
+ assert!(refused.contains(refusal_marker), "{label}: {refused}");
+
+ let preflight = Journal::preflight(dir.path(), "root", "url", "prefix", true)
+ .unwrap_or_else(|e| panic!("{label}: --fresh must pass the preflight: {e:#}"));
+ assert!(preflight.plan.is_none(), "{label}");
+ let reason = preflight
+ .unreadable_plan
+ .clone()
+ .unwrap_or_else(|| panic!("{label}: the discarded plan must be reported"));
+ assert!(!reason.is_empty(), "{label}");
+
+ let journal =
+ Journal::open_after_preflight(preflight, "root", "url", "prefix", 300, true)
+ .unwrap_or_else(|e| panic!("{label}: --fresh must open: {e:#}"));
+ drop(journal);
+ // And the rebuilt state directory is resumable again without --fresh.
+ Journal::open(dir.path(), "root", "url", "prefix", 300, false)
+ .unwrap_or_else(|e| panic!("{label}: the rebuilt journal must resume: {e:#}"));
+ }
+ }
+
+ /// A readable journal must not be reported as unreadable just because the
+ /// run asked for `--fresh`; the note is a discarded-state warning, not a
+ /// `--fresh` banner.
+ #[test]
+ fn fresh_over_a_readable_journal_reports_nothing_unreadable() {
+ let dir = tempfile::tempdir().unwrap();
+ drop(Journal::open(dir.path(), "root", "url", "prefix", 300, true).unwrap());
+ let preflight = Journal::preflight(dir.path(), "root", "url", "prefix", true).unwrap();
+ assert!(preflight.unreadable_plan.is_none());
+ }
+
#[test]
fn preflight_holds_the_same_exclusive_lock_through_authoritative_open() {
let dir = tempfile::tempdir().unwrap();
- let preflight = Journal::preflight(dir.path(), "root", "url", "prefix").unwrap();
+ let preflight = Journal::preflight(dir.path(), "root", "url", "prefix", false).unwrap();
let error = Journal::open(dir.path(), "root", "url", "prefix", 300, false)
.err()
.expect("a second opener must not pass the held preflight lock");
@@ -376,14 +442,25 @@ pub struct JournalPreflight {
state_lock: std::fs::File,
pub plan: Option<Plan>,
pub journal_exists: bool,
+ /// Set only under `--fresh`, when the durable plan could not be read and
+ /// the run is about to discard the journal anyway. The caller owns the
+ /// decision to print it (autoindex honours `--quiet`).
+ pub unreadable_plan: Option<String>,
}
impl Journal {
+ /// `fresh` is not a formality here. `open_after_preflight` removes the
+ /// journal before replaying it when `--fresh` is set, so nothing inside a
+ /// journal can be fatal to that path — and a preflight that hard-errors on
+ /// a corrupt record or a root/url/prefix mismatch would make `--fresh`
+ /// unreachable in exactly the cases whose error text recommends it. The
+ /// preflight must therefore never be more fatal than the open it precedes.
pub fn preflight(
state_dir: &Path,
root: &str,
url: &str,
prefix: &str,
+ fresh: bool,
) -> Result<JournalPreflight> {
std::fs::create_dir_all(state_dir)
.with_context(|| format!("create state dir {}", state_dir.display()))?;
@@ -403,12 +480,22 @@ impl Journal {
})?;
let journal_path = state_dir.join("journal.ndjson");
let journal_exists = journal_path.exists();
- let plan = read_plan_for_preflight(&journal_path, root, url, prefix)?;
+ let (plan, unreadable_plan) =
+ match read_plan_for_preflight(&journal_path, root, url, prefix) {
+ Ok(plan) => (plan, None),
+ // The journal is about to be deleted unread. Losing the plan here
+ // costs the removed-file gate its comparison basis, which is
+ // exactly what a full in-place rebuild accepts; the alternative is
+ // a state directory that no supported flag can recover.
+ Err(error) if fresh => (None, Some(format!("{error:#}"))),
+ Err(error) => return Err(error),
+ };
Ok(JournalPreflight {
state_dir: state_dir.to_owned(),
state_lock,
plan,
journal_exists,
+ unreadable_plan,
})
}
@@ -420,7 +507,7 @@ impl Journal {
bulk_timeout_secs: u64,
fresh: bool,
) -> Result<Journal> {
- let preflight = Self::preflight(state_dir, root, url, prefix)?;
+ let preflight = Self::preflight(state_dir, root, url, prefix, fresh)?;
Self::open_after_preflight(preflight, root, url, prefix, bulk_timeout_secs, fresh)
}
diff --git a/engine/crates/xerj-server/src/brain.rs b/engine/crates/xerj-server/src/brain.rs
index 8e8e62e48..594870e32 100644
--- a/engine/crates/xerj-server/src/brain.rs
+++ b/engine/crates/xerj-server/src/brain.rs
@@ -398,10 +398,19 @@ fn live_node_docs(es: &Es, brain: &str) -> Result<Option<u64>> {
else {
return Ok(None);
};
- let response = es.search(
+ // A meta doc that outlived its nodes index is precisely the wiped-data-dir
+ // case this probe exists to catch, so the index being gone must reach
+ // `journal_server_disagreement()` as absence. Propagating the 404 would
+ // short-circuit the caller and print a raw HTTP error instead of the
+ // recovery text. Any other failure still propagates: a probe failure is
+ // never converted into reset authorization.
+ let Some(response) = es.search_present(
nodes_index,
&json!({"size": 0, "track_total_hits": true, "query": {"match_all": {}}}),
- )?;
+ )?
+ else {
+ return Ok(None);
+ };
response
.pointer("/hits/total/value")
.and_then(Value::as_u64)