Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

fix(engine): refuse cluster-metadata writes when cluster_state.json did not load

xerj-orgf64c1e868084
diff --git a/engine/crates/xerj-api/tests/cluster_metadata_survives_restart.rs b/engine/crates/xerj-api/tests/cluster_metadata_survives_restart.rs
index 19a5f83f0..590aad19f 100644
--- a/engine/crates/xerj-api/tests/cluster_metadata_survives_restart.rs
+++ b/engine/crates/xerj-api/tests/cluster_metadata_survives_restart.rs
@@ -338,10 +338,12 @@ async fn a_rollover_interrupted_before_its_flush_does_not_wedge_the_stream() {
/// A corrupt `cluster_state.json` must not be destroyed by the next write.
///
-/// The node boots with empty maps and logs the failure, so the first
-/// `PUT /_index_template/...` rewrites the file — taking whatever was still
-/// legible in it along. Hand-recovery out of a damaged document is the
-/// operator's last option; it should not be silently removed.
+/// The node boots with empty maps and logs the failure, so without this the
+/// first `PUT /_index_template/...` rewrote the file — taking whatever was
+/// still legible in it along. Hand-recovery out of a damaged document is the
+/// operator's last option; it should not be silently removed. The write is
+/// refused (the original stays where it is) *and* a copy is kept, because
+/// un-wedging the node means moving the original aside by hand.
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn a_corrupt_cluster_state_is_preserved_before_it_is_overwritten() {
let dir = tempfile::tempdir().expect("tempdir");
@@ -367,13 +369,18 @@ async fn a_corrupt_cluster_state_is_preserved_before_it_is_overwritten() {
);
// The write that would have destroyed the evidence.
- let (st, _) = put_json(
+ let (st, body) = put_json(
&app,
"/_index_template/fresh",
json!({ "index_patterns": ["fresh-*"] }),
)
.await;
- assert_eq!(st, StatusCode::OK, "PUT after corrupt load");
+ assert_eq!(
+ st,
+ StatusCode::INTERNAL_SERVER_ERROR,
+ "a management write on top of a state that did not load must be \
+ refused, not acknowledged: {body}"
+ );
let salvage = dir.path().join("cluster_state.corrupt.json");
assert!(
@@ -385,11 +392,153 @@ async fn a_corrupt_cluster_state_is_preserved_before_it_is_overwritten() {
truncated,
"the preserved copy must be the bytes that were found, unmodified"
);
+ assert_eq!(
+ std::fs::read_to_string(&state_path).expect("read state"),
+ truncated,
+ "the live document must be left exactly as it was found — the copy is \
+ a convenience, not a licence to overwrite the original"
+ );
+
+ // Moving the damaged file aside is the documented recovery, and it must
+ // actually work: the next boot loads cleanly and writes are accepted again.
+ drop(app);
+ std::fs::remove_file(&state_path).expect("operator moves the file aside");
+ let app = app_over(dir.path());
+ let (st, body) = put_json(
+ &app,
+ "/_index_template/fresh",
+ json!({ "index_patterns": ["fresh-*"] }),
+ )
+ .await;
+ assert_eq!(
+ st,
+ StatusCode::OK,
+ "PUT after the operator recovered: {body}"
+ );
assert!(
std::fs::read_to_string(&state_path)
.expect("read state")
.contains("fresh-*"),
- "the live document must have moved on"
+ "the live document must have moved on once the node loaded cleanly"
+ );
+}
+
+/// An **intact but unreadable** `cluster_state.json` must survive untouched,
+/// and the node must refuse management writes until a boot can load it.
+///
+/// This is the worse half of the corrupt case and it was missed on the first
+/// pass: the bytes are perfectly good, only the `read(2)` failed — a uid
+/// change on a container volume, a backup tool's chmod, `EIO`, `EMFILE` at
+/// boot. The maps come up empty, and `write_file_atomic` renames
+/// `cluster_state.tmp` over the target; `rename(2)` needs write permission on
+/// the *directory*, not on the file, so a `PUT` answering
+/// `{"acknowledged": true}` unlinked a fully recoverable document. Measured
+/// on this test before the fix, with `chmod 000` for the boot only:
+///
+/// ```text
+/// PUT /_index_template/fresh -> 200 {"acknowledged":true}
+/// cluster_state.corrupt.json exists: false
+/// live file still contains "logs-*": false <-- the operator's templates
+/// ```
+#[cfg(unix)]
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn an_unreadable_cluster_state_is_never_overwritten() {
+ use std::os::unix::fs::PermissionsExt;
+
+ let dir = tempfile::tempdir().expect("tempdir");
+ {
+ let app = app_over(dir.path());
+ write_all_the_config(&app).await;
+ }
+
+ let state_path = dir.path().join("cluster_state.json");
+ let original = std::fs::read_to_string(&state_path).expect("read state");
+ assert!(original.contains("logs-*"), "fixture must be on disk");
+
+ // Unreadable for the boot only; restored immediately afterwards so the
+ // assertions below read the file the node was left holding, and so a
+ // failure cannot be blamed on the test's own permissions.
+ std::fs::set_permissions(&state_path, std::fs::Permissions::from_mode(0o000))
+ .expect("chmod 000");
+ let app = app_over(dir.path());
+ std::fs::set_permissions(&state_path, std::fs::Permissions::from_mode(0o644))
+ .expect("chmod 644");
+
+ let (st, _) = get(&app, "/_index_template/logs").await;
+ assert_eq!(
+ st,
+ StatusCode::NOT_FOUND,
+ "a document that could not be read cannot be restored — that part is \
+ honest already"
+ );
+
+ // The write that used to destroy it.
+ let (st, body) = put_json(
+ &app,
+ "/_index_template/fresh",
+ json!({ "index_patterns": ["fresh-*"] }),
+ )
+ .await;
+ assert_eq!(
+ st,
+ StatusCode::INTERNAL_SERVER_ERROR,
+ "a management write must be refused while the persisted state is \
+ unloaded, not acknowledged: {body}"
+ );
+ assert!(
+ body["error"]["reason"]
+ .as_str()
+ .unwrap_or_default()
+ .contains("could not be loaded at boot"),
+ "the 500 must say why, so the operator can act on it: {body}"
+ );
+
+ assert_eq!(
+ std::fs::read_to_string(&state_path).expect("read state"),
+ original,
+ "the intact document must still be on disk, byte for byte"
+ );
+ assert!(
+ !dir.path().join("cluster_state.tmp").exists(),
+ "a refused write must not even stage a temp file"
+ );
+
+ // And the refusal is not a one-off: the in-memory map must not be left
+ // holding a change that was reported as failed.
+ let (st, _) = get(&app, "/_index_template/fresh").await;
+ assert_eq!(
+ st,
+ StatusCode::NOT_FOUND,
+ "a refused PUT must roll its in-memory change back"
+ );
+ let (st, _) = delete(&app, "/_ilm/policy/hot-warm").await;
+ assert_eq!(
+ st,
+ StatusCode::INTERNAL_SERVER_ERROR,
+ "deletes are writes too — they must be refused as well"
+ );
+
+ // Recovery path: the file was readable all along, so a plain restart is
+ // the whole fix.
+ drop(app);
+ let app = app_over(dir.path());
+ let (st, body) = get(&app, "/_index_template/logs").await;
+ assert_eq!(
+ st,
+ StatusCode::OK,
+ "a restart once the file is readable must restore everything: {body}"
+ );
+ let (st, _) = put_json(
+ &app,
+ "/_index_template/fresh",
+ json!({ "index_patterns": ["fresh-*"] }),
+ )
+ .await;
+ assert_eq!(
+ st,
+ StatusCode::OK,
+ "and writes must work again — the refusal must not be sticky across a \
+ clean boot"
);
}
diff --git a/engine/crates/xerj-engine/src/engine.rs b/engine/crates/xerj-engine/src/engine.rs
index aa51406d6..4c575e817 100644
--- a/engine/crates/xerj-engine/src/engine.rs
+++ b/engine/crates/xerj-engine/src/engine.rs
@@ -627,6 +627,37 @@ pub struct Engine {
///
/// Never taken while holding a `DashMap` guard.
cluster_state_write: Arc<parking_lot::Mutex<()>>,
+
+ /// False once `<data_dir>/cluster_state.json` was found on disk and could
+ /// not be loaded — unreadable (`EACCES` after a uid change on a container
+ /// volume, `EIO`, `EMFILE` at boot) or unparseable. While false,
+ /// `flush_cluster_state` refuses to write and every management PUT
+ /// answers 500.
+ ///
+ /// Without this the failure is silent *and* destructive. The maps boot
+ /// empty, so the first `PUT /_index_template/...` snapshots six empty
+ /// maps and `write_file_atomic` renames `cluster_state.tmp` over the
+ /// target — and `rename(2)` needs only write permission on the
+ /// *directory*, so an unreadable-but-perfectly-intact document is
+ /// unlinked by a write that answers `{"acknowledged": true}`. Every
+ /// trigger listed above is transient or trivially repairable *until*
+ /// xerj overwrites the file, which is the one outcome that is not.
+ ///
+ /// Refusing is also the honest answer rather than merely the safe one:
+ /// the node is running without the templates that shape new indices and
+ /// the pipelines `?pipeline=x` names, so accepting more configuration on
+ /// top of a config that is silently a subset is the same
+ /// accepted-and-ignored shape issue #204 tracks.
+ ///
+ /// Cleared only by a boot that loads cleanly — fix or move the file
+ /// aside and restart. redb takes the same position after a failed
+ /// integrity check or a failed I/O: an `AtomicBool` on the backend makes
+ /// every later write return `StorageError::PreviousIo`, whose message is
+ /// "Please close and re-open the database"
+ /// (`redb/src/tree_store/page_store/cached_file.rs:125-145`,
+ /// `redb/src/error.rs:65`; Apache-2.0/MIT, shape adapted, no code
+ /// copied).
+ cluster_state_loaded: Arc<std::sync::atomic::AtomicBool>,
}
impl Engine {
@@ -739,6 +770,7 @@ impl Engine {
)),
_node_lock: node_lock,
cluster_state_write: Arc::new(parking_lot::Mutex::new(())),
+ cluster_state_loaded: Arc::new(std::sync::atomic::AtomicBool::new(true)),
};
// Scan data_dir for existing index directories.
@@ -1753,6 +1785,40 @@ impl Engine {
self.data_dir.join("cluster_state.json")
}
+ /// `Err` when boot found `cluster_state.json` and could not load it — see
+ /// `cluster_state_loaded` for why that has to latch.
+ ///
+ /// Every management mutation calls this, not only the ones that reach
+ /// `flush_cluster_state` with something to write. A `DELETE` consults the
+ /// in-memory map first, and while the load has failed that map is empty,
+ /// so an unguarded delete answers `404 not found` about an object that is
+ /// sitting in the document on disk — the same lie in a different shape.
+ /// A 500 that names the file is the only honest answer while the node
+ /// cannot see the operator's configuration.
+ fn ensure_cluster_state_writable(&self) -> Result<()> {
+ if self
+ .cluster_state_loaded
+ .load(std::sync::atomic::Ordering::Acquire)
+ {
+ return Ok(());
+ }
+ let path = self.cluster_state_path();
+ error!(
+ path = %path.display(),
+ "refusing a cluster-metadata write: this node could not load \
+ cluster_state.json at boot, so its management state is not the \
+ operator's. Recover or move the file aside, then restart."
+ );
+ Err(EngineError::Common(xerj_common::XerjError::storage(
+ format!(
+ "refusing to write {} — it could not be loaded at boot, so \
+ writing now would destroy cluster configuration that is still \
+ on disk; recover or move it aside and restart xerj",
+ path.display()
+ ),
+ )))
+ }
+
/// Snapshot every persisted management map into one document.
fn cluster_state_snapshot(&self) -> PersistedClusterState {
fn dump<V: Clone>(map: &DashMap<String, V>) -> std::collections::BTreeMap<String, V> {
@@ -1778,7 +1844,14 @@ impl Engine {
/// `{"acknowledged": true}` for a change it cannot keep. Callers roll the
/// in-memory change back and surface a 500, so the operator finds out at
/// the moment of the write instead of at the next restart.
+ ///
+ /// Refuses outright when boot could not load the existing document — see
+ /// `cluster_state_loaded`. The snapshot below is taken from the live maps,
+ /// which in that state hold only what this process was told after boot, so
+ /// writing it would rename an empty-ish document over configuration that is
+ /// still intact on disk.
fn flush_cluster_state(&self) -> Result<()> {
+ self.ensure_cluster_state_writable()?;
// One writer at a time — see `cluster_state_write`. Snapshotting
// inside the lock also means the bytes that land are never older
// than a rewrite that has already returned.
@@ -1812,9 +1885,12 @@ impl Engine {
/// Load `<data_dir>/cluster_state.json` back into the management maps.
///
/// A missing file is normal (fresh node, or a data dir written by a build
- /// that predates this). A corrupt file is loud: it means the operator's
- /// templates, pipelines and data streams are *not* coming back, which is
- /// exactly the silence this whole mechanism exists to remove.
+ /// that predates this). A file that is present but does not load is loud
+ /// *and* latching: it means the operator's templates, pipelines and data
+ /// streams are not coming back, which is exactly the silence this whole
+ /// mechanism exists to remove, so `cluster_state_loaded` goes false and
+ /// every subsequent management write is refused rather than allowed to
+ /// rename a snapshot of empty maps over the document on disk.
fn load_cluster_state(&self) {
// An interrupted rewrite can leave a partial `cluster_state.tmp`
// behind. It is never read (the rename is the commit point), but
@@ -1835,10 +1911,20 @@ impl Engine {
Ok(b) => b,
Err(e) if e.kind() == std::io::ErrorKind::NotFound => return,
Err(e) => {
+ // The file is *there* and its bytes are almost certainly fine
+ // — `EACCES` after a uid change on a container volume, a
+ // backup tool's chmod, `EIO`, `EMFILE` at boot. Do not copy it
+ // aside (we cannot read it) and do not touch it: mark the load
+ // failed so no later write can rename a snapshot of empty maps
+ // over configuration that is still perfectly recoverable.
+ self.cluster_state_loaded
+ .store(false, std::sync::atomic::Ordering::Release);
error!(
path = %path.display(), error = %e,
- "could not read cluster_state.json — index templates, ingest \
- pipelines, data streams and ILM policies are NOT restored"
+ "could not READ cluster_state.json — index templates, ingest \
+ pipelines, data streams and ILM policies are NOT restored, \
+ and management writes will be refused until a boot loads it \
+ (the file itself is left untouched)"
);
return;
}
@@ -1846,20 +1932,30 @@ impl Engine {
let state: PersistedClusterState = match serde_json::from_slice(&bytes) {
Ok(s) => s,
Err(e) => {
+ // Same position as the read error above: the load failed, so
+ // the live maps are not the operator's configuration and
+ // nothing may be written over the document on disk. Refusing
+ // rather than salvaging-then-overwriting also closes the
+ // second-corruption hole in the `!salvage.exists()` guard
+ // below — with an overwrite still allowed, a second damaged
+ // document would be destroyed *and* not copied, because the
+ // copy from the first one is already there.
+ self.cluster_state_loaded
+ .store(false, std::sync::atomic::Ordering::Release);
error!(
path = %path.display(), error = %e,
"cluster_state.json is corrupt — index templates, ingest \
- pipelines, data streams and ILM policies are NOT restored"
+ pipelines, data streams and ILM policies are NOT restored, \
+ and management writes will be refused until a boot loads it"
);
- // Keep a copy before the node runs on. The maps are empty
- // now, so the very next `PUT /_index_template/...` rewrites
- // this file and takes whatever was still legible in it with
- // it — and hand-recovering a template out of a damaged
- // document is the operator's last option, not one to destroy
- // silently. The original is deliberately left in place so
- // every boot keeps logging the error above until someone
- // deals with it, and an existing copy is never clobbered:
- // the first corruption is the informative one.
+ // Keep a copy anyway. The original is now safe from xerj, but
+ // un-wedging the node means moving it aside by hand, and
+ // hand-recovering a template out of a damaged document is the
+ // operator's last option — leave them something to recover
+ // from that survives that step. The original is deliberately
+ // left in place so every boot keeps logging the error above
+ // until someone deals with it, and an existing copy is never
+ // clobbered: the first corruption is the informative one.
let salvage = path.with_extension("corrupt.json");
if !salvage.exists() {
match std::fs::write(&salvage, &bytes) {
@@ -2008,6 +2104,7 @@ impl Engine {
name: String,
value: V,
) -> Result<()> {
+ self.ensure_cluster_state_writable()?;
let previous = map.insert(name.clone(), value);
if let Err(e) = self.flush_cluster_state() {
match previous {
@@ -2023,6 +2120,10 @@ impl Engine {
/// `false` when there was nothing to remove (the caller answers 404);
/// an `Err` means the removal did not reach disk and was undone.
fn persisted_remove<V: Clone>(&self, map: &DashMap<String, V>, name: &str) -> Result<bool> {
+ // Before the map lookup: an empty map after a failed load would
+ // otherwise turn "I cannot see your configuration" into "it does not
+ // exist" — see `ensure_cluster_state_writable`.
+ self.ensure_cluster_state_writable()?;
let Some((key, removed)) = map.remove(name) else {
return Ok(false);
};
@@ -2107,6 +2208,9 @@ impl Engine {
config: Value,
keep_if_uncompilable: Option<Value>,
) -> Result<Option<xerj_wasm::WasmError>> {
+ // Up front, so a refused write never disturbs the compiled form of a
+ // pipeline that is still running.
+ self.ensure_cluster_state_writable()?;
let previous_doc = self.pipelines.get(id).map(|v| v.value().clone());
// Taken out up front so a failed compile cannot leave a stale
// executable behind; put back verbatim on every abort path.
@@ -2668,6 +2772,10 @@ impl Engine {
/// Create a new data stream with its first backing index.
pub fn create_data_stream(&self, name: &str) -> Result<()> {
+ // Before `create_index` below, not just before the flush: refusing
+ // after the backing index exists would leave a `.ds-*` directory with
+ // no stream in front of it.
+ self.ensure_cluster_state_writable()?;
if self.data_streams.contains_key(name) {
return Err(EngineError::Common(
xerj_common::XerjError::index_already_exists(name),
@@ -2694,6 +2802,10 @@ impl Engine {
/// Roll over a data stream: create the next backing index and update the alias.
pub fn rollover_data_stream(&self, name: &str) -> Result<String> {
+ // Same reason as `create_data_stream`, plus: the generation counter is
+ // the one value that must never be handed out twice, so a rollover
+ // that cannot record it must not create its backing index at all.
+ self.ensure_cluster_state_writable()?;
let mut ds = self
.data_streams
.get_mut(name)
@@ -2734,6 +2846,10 @@ impl Engine {
/// Delete a data stream and all its backing indices.
pub async fn delete_data_stream(&self, name: &str) -> Result<()> {
+ // Before the map lookup, so an unloaded state cannot answer
+ // `index_not_found` for a stream that is recorded on disk — and, more
+ // importantly, cannot go on to destroy its backing indices.
+ self.ensure_cluster_state_writable()?;
let ds = self
.data_streams
.remove(name)