Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

fix(engine): refuse to open an index whose metadata is unparseable (#202)

xerj-orgc875a8a3de11
diff --git a/engine/crates/xerj-engine/src/engine.rs b/engine/crates/xerj-engine/src/engine.rs
index 136c30deb..e0ccf13a9 100644
--- a/engine/crates/xerj-engine/src/engine.rs
+++ b/engine/crates/xerj-engine/src/engine.rs
@@ -505,8 +505,14 @@ impl Engine {
// Restore the raw ES mapping blob (analyzers, formats,
// dims — full fidelity) BEFORE any ingest/query can run,
// so GET /_mapping and mapping-dependent code paths see
- // the same mapping as pre-restart.
- engine.load_persisted_es_mapping(&name_str);
+ // the same mapping as pre-restart. A corrupt blob fails
+ // the index (#202) rather than serving it with a
+ // silently reduced mapping.
+ if let Err(e) = engine.load_persisted_es_mapping(&name_str) {
+ warn!(name = name_str.as_str(), error = %e, "failed to open index");
+ engine.failed_indices.insert(name_str, e.to_string());
+ continue;
+ }
// The index isn't registered yet, so the propagation
// inside load can't find it — set the toggles on the
// local handle instead.
@@ -831,6 +837,24 @@ impl Engine {
));
}
+ // An index that failed to open is absent from `indices` but its data
+ // directory is still on disk, so without this check both `PUT /{index}`
+ // and bulk auto-create would happily "create" it — running
+ // `Index::create` over the existing store and overwriting `schema.json`
+ // with an empty mapping. That is the very mapping loss #202 is about,
+ // reached through the other door, and it would also destroy the
+ // evidence. Refuse, and say what is wrong and how to get out of it.
+ if let Some(reason) = self.failed_indices.get(name) {
+ return Err(EngineError::Common(xerj_common::XerjError::storage(
+ format!(
+ "index [{name}] exists on disk but failed to open ({}); refusing to \
+ re-create it over the existing data — repair the file, or DELETE the \
+ index to discard it",
+ reason.value()
+ ),
+ )));
+ }
+
// Apply matching template (highest priority wins) on every create path.
let effective_schema = self.apply_index_template(name, schema);
let idx = Index::create_with_settings(
@@ -941,19 +965,36 @@ impl Engine {
/// (re)opened from disk — boot scan and snapshot restore. A missing
/// file is fine (pre-fix indices, dynamic-only indices): readers fall
/// back to schema-derived properties from `schema.json`.
- fn load_persisted_es_mapping(&self, name: &str) {
+ ///
+ /// A file that is present but unreadable or unparseable is **not** fine and
+ /// is no longer logged-and-ignored (#202). This blob is the full-fidelity
+ /// mapping — analyzers, date formats, `dense_vector` dims — and is what
+ /// `GET /{index}/_mapping` answers with; dropping it leaves the index
+ /// serving a quietly emptier mapping than the one its own data was written
+ /// under. The caller fails the index instead, which turns cluster health
+ /// red and keeps the corrupt index out of the served set.
+ fn load_persisted_es_mapping(&self, name: &str) -> Result<()> {
let path = self.data_dir.join(name).join("es_mapping.json");
- let Ok(bytes) = std::fs::read(&path) else {
- return;
+ let bytes = match std::fs::read(&path) {
+ Ok(bytes) => bytes,
+ Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(()),
+ Err(e) => {
+ return Err(EngineError::CorruptIndexMetadata {
+ file: path.display().to_string(),
+ reason: e.to_string(),
+ })
+ }
};
match serde_json::from_slice::<Value>(&bytes) {
Ok(mapping) => {
self.propagate_date_detection(name, &mapping);
self.index_mappings.insert(name.to_string(), mapping);
+ Ok(())
}
- Err(e) => {
- warn!(index = name, error = %e, "ignoring corrupt es_mapping.json");
- }
+ Err(e) => Err(EngineError::CorruptIndexMetadata {
+ file: path.display().to_string(),
+ reason: format!("{e} ({} bytes on disk)", bytes.len()),
+ }),
}
}
@@ -1186,12 +1227,52 @@ impl Engine {
xerj_common::XerjError::index_not_found(name),
));
}
- let idx =
- self.indices.remove(name).map(|(_, v)| v).ok_or_else(|| {
- EngineError::Common(xerj_common::XerjError::index_not_found(name))
- })?;
-
- idx.delete_all_data().await?;
+ match self.indices.remove(name).map(|(_, v)| v) {
+ Some(idx) => idx.delete_all_data().await?,
+ // An index that failed to open (corrupt metadata, embedding
+ // identity mismatch, …) is not in `indices`, but its directory is
+ // still on disk and `create` now refuses to write over it. Without
+ // this branch that name would be permanently stuck: unopenable,
+ // uncreatable, undeletable — the operator's only recovery would be
+ // stopping the node and removing the directory by hand. Deleting a
+ // failed index is exactly the "restore from backup" escape hatch,
+ // so it must work through the API.
+ None => {
+ let Some((_, reason)) = self.failed_indices.remove(name) else {
+ return Err(EngineError::Common(
+ xerj_common::XerjError::index_not_found(name),
+ ));
+ };
+ // `IndexName` rejects '.', '..', separators and '..'
+ // substrings, so the join cannot escape data_dir; the
+ // canonical re-check is defence in depth against a symlinked
+ // index directory.
+ let index_name = IndexName::new(name).map_err(EngineError::Common)?;
+ let dir = self.data_dir.join(index_name.as_str());
+ if let (Ok(dir_canon), Ok(root_canon)) =
+ (dir.canonicalize(), self.data_dir.canonicalize())
+ {
+ if !dir_canon.starts_with(&root_canon) {
+ self.failed_indices.insert(name.to_string(), reason);
+ return Err(EngineError::Common(
+ xerj_common::XerjError::invalid_mapping(format!(
+ "refusing to delete index [{name}] outside data_dir (canonical)"
+ )),
+ ));
+ }
+ }
+ if dir.exists() {
+ if let Err(e) = std::fs::remove_dir_all(&dir) {
+ // Put the failure back so health stays red and the
+ // operator can retry — a half-deleted corrupt index
+ // must not look like a clean slate.
+ self.failed_indices.insert(name.to_string(), reason);
+ return Err(EngineError::Io(e));
+ }
+ }
+ warn!(name, "deleted an index that had failed to open");
+ }
+ }
// Remove this index from every alias that references it; drop the
// alias entirely when its backing list becomes empty.
@@ -1965,11 +2046,22 @@ impl Engine {
Ok(idx) => {
// Snapshot dirs carry es_mapping.json — reload it so the
// restored index serves the same mapping it was saved with.
- self.load_persisted_es_mapping(idx_name);
+ // A corrupt blob in the snapshot fails the restore of that
+ // index instead of restoring it with a reduced mapping.
+ if let Err(e) = self.load_persisted_es_mapping(idx_name) {
+ warn!(index = idx_name, error = %e, "failed to reopen restored index");
+ self.failed_indices.insert(idx_name.clone(), e.to_string());
+ continue;
+ }
if let Some(m) = self.index_mappings.get(idx_name) {
Engine::apply_date_mapping_flags(&idx, m.value());
}
self.indices.insert(idx_name.clone(), idx);
+ // Restoring from a snapshot is *the* repair for an index
+ // that failed to open, so clear the recorded failure —
+ // otherwise cluster health stays red after the repair
+ // actually worked.
+ self.failed_indices.remove(idx_name);
info!(index = idx_name, "index restored from snapshot");
}
Err(e) => {
diff --git a/engine/crates/xerj-engine/src/index.rs b/engine/crates/xerj-engine/src/index.rs
index 66a2a0ccf..3b6732f8c 100644
--- a/engine/crates/xerj-engine/src/index.rs
+++ b/engine/crates/xerj-engine/src/index.rs
@@ -5564,8 +5564,11 @@ impl Index {
// Load persisted settings BEFORE opening the store so the WAL shard
// count (index.xerj_ingest_shards) matches what create used — the WAL
- // write layout (root vs s{N}/) and doc routing depend on it.
- let settings = load_settings(&index_dir).unwrap_or(Value::Null);
+ // write layout (root vs s{N}/) and doc routing depend on it. An
+ // unparseable settings.json is refused rather than defaulted to null:
+ // the shard count, the custom analyzers and the index blocks all live
+ // in here, so "null" is a different index, not a safe fallback.
+ let settings = load_settings(&index_dir)?.unwrap_or(Value::Null);
let store_config = store_config_from(config, wal_shards_override_from_settings(&settings));
let store = IndexStore::open(&index_dir, store_config)?;
@@ -5574,8 +5577,11 @@ impl Index {
let segment_doc_count: u64 = snap.segments.iter().map(|s| s.doc_count).sum();
drop(snap);
- // Load schema from disk if it exists.
- let schema = load_schema(&index_dir).unwrap_or_else(|_| ManagedSchema::dynamic());
+ // Load schema from disk if it exists. Absent → dynamic mapping (a
+ // pre-persistence index legitimately has no schema.json). Present but
+ // unparseable → refuse: the explicit mapping is gone and every field
+ // type would be silently re-inferred from the next writes (#202).
+ let schema = load_schema(&index_dir)?.unwrap_or_else(ManagedSchema::dynamic);
validate_embedding_identity(
&index_dir,
&schema.schema,
@@ -17039,9 +17045,13 @@ impl Index {
);
}
}
+ // Atomic, like every other settings.json writer (create and the index
+ // block path). This was the one plain `fs::write` left, i.e. the one
+ // way to actually produce the torn settings.json that `Index::open`
+ // now refuses (#202) — a kill -9 mid-update left a half-written file.
let path = self.data_dir.join("settings.json");
let bytes = serde_json::to_vec_pretty(&new_settings)?;
- std::fs::write(&path, bytes).map_err(EngineError::Io)?;
+ write_file_atomic(&path, &bytes).map_err(EngineError::Io)?;
*self.settings.write().await = new_settings;
Ok(())
}
@@ -25478,16 +25488,57 @@ pub fn write_file_atomic(path: &Path, bytes: &[u8]) -> std::io::Result<()> {
Ok(())
}
-fn load_schema(index_dir: &Path) -> std::result::Result<ManagedSchema, ()> {
- let path = index_dir.join("schema.json");
- let bytes = std::fs::read(path).map_err(|_| ())?;
- serde_json::from_slice(&bytes).map_err(|_| ())
+/// Read one of the index's JSON sidecar files, keeping ABSENT and UNPARSEABLE
+/// apart (issue #202).
+///
+/// * `Ok(None)` — the file is genuinely not there. That is a legitimate state
+/// (`schema.json` was only written by `put_mapping` before create-time
+/// persistence landed; `settings.json` is only written when settings are
+/// non-null), and the caller's default is the correct answer.
+/// * `Err(..)` — the file exists but cannot be read or does not deserialize.
+/// The recorded mapping/settings are *lost*: substituting a default here does
+/// not preserve behaviour, it silently re-derives every field type from the
+/// next documents to arrive (verified: a `keyword` field came back as `long`
+/// after one post-corruption write). The caller must refuse to open, exactly
+/// as a segment with an unexpected header is refused rather than guessed at
+/// (`xerj-storage/src/segment.rs:243-246`).
+///
+/// Prior art for failing loudly on unparseable index metadata rather than
+/// defaulting: tantivy's `load_metas` turns a `meta.json` that does not
+/// deserialize into `DataCorruption` naming the file
+/// (tantivy `src/index/index.rs:29-49`), while absence is a *separate*
+/// pre-check (`Index::exists`, `src/index/index.rs:324-326`) that routes to
+/// create; sled likewise raises `InvalidData` on any unreadable metadata frame
+/// instead of skipping it (sled `src/metadata_store.rs:518,546,582`).
+fn load_sidecar<T: serde::de::DeserializeOwned>(index_dir: &Path, file: &str) -> Result<Option<T>> {
+ let path = index_dir.join(file);
+ let bytes = match std::fs::read(&path) {
+ Ok(bytes) => bytes,
+ Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(None),
+ // A file we cannot read is not a file that is absent: an EACCES or an
+ // EIO here used to be indistinguishable from "never written".
+ Err(e) => {
+ return Err(EngineError::CorruptIndexMetadata {
+ file: path.display().to_string(),
+ reason: e.to_string(),
+ })
+ }
+ };
+ match serde_json::from_slice(&bytes) {
+ Ok(value) => Ok(Some(value)),
+ Err(e) => Err(EngineError::CorruptIndexMetadata {
+ file: path.display().to_string(),
+ reason: format!("{e} ({} bytes on disk)", bytes.len()),
+ }),
+ }
+}
+
+fn load_schema(index_dir: &Path) -> Result<Option<ManagedSchema>> {
+ load_sidecar(index_dir, "schema.json")
}
-fn load_settings(index_dir: &Path) -> std::result::Result<Value, ()> {
- let path = index_dir.join("settings.json");
- let bytes = std::fs::read(path).map_err(|_| ())?;
- serde_json::from_slice(&bytes).map_err(|_| ())
+fn load_settings(index_dir: &Path) -> Result<Option<Value>> {
+ load_sidecar(index_dir, "settings.json")
}
// ── Date math in index names ──────────────────────────────────────────────────
diff --git a/engine/crates/xerj-engine/src/lib.rs b/engine/crates/xerj-engine/src/lib.rs
index 1e07ea989..e4bf537c8 100644
--- a/engine/crates/xerj-engine/src/lib.rs
+++ b/engine/crates/xerj-engine/src/lib.rs
@@ -331,6 +331,20 @@ pub enum EngineError {
#[error("serialization error: {0}")]
Serde(#[from] serde_json::Error),
+
+ /// An index metadata sidecar (`schema.json`, `settings.json`,
+ /// `es_mapping.json`) is present on disk but could not be read or parsed.
+ ///
+ /// This is deliberately distinct from the file being **absent**, which is a
+ /// legitimate state (indices created before create-time schema persistence
+ /// have no `schema.json`; an index created without settings has no
+ /// `settings.json`) and still opens with a dynamic mapping. Unparseable
+ /// means the recorded mapping is *lost*, and defaulting there silently
+ /// re-infers every field type from whatever documents arrive next — so the
+ /// open is refused instead, the same way `xerj-storage` refuses a segment
+ /// whose header does not match (`StorageError::UnsupportedVersion`).
+ #[error("corrupt index metadata: {file} exists but could not be loaded ({reason}); refusing to open the index rather than fall back to a default mapping — restore the file or restore the index from a snapshot")]
+ CorruptIndexMetadata { file: String, reason: String },
}
/// Engine-level result alias.
@@ -345,6 +359,12 @@ impl From<EngineError> for xerj_common::XerjError {
EngineError::Fts(f) => xerj_common::XerjError::internal(f.to_string()),
EngineError::Io(io) => xerj_common::XerjError::storage_io("IO error", io),
EngineError::Serde(s) => xerj_common::XerjError::serialization(s.to_string()),
+ // `store_exception` (500) is the ES type for "this shard's on-disk
+ // state is unusable" — the closest honest mapping for a corrupt
+ // sidecar, and it keeps the reason string intact for the client.
+ corrupt @ EngineError::CorruptIndexMetadata { .. } => {
+ xerj_common::XerjError::storage(corrupt.to_string())
+ }
}
}
}
diff --git a/engine/crates/xerj-engine/tests/corrupt_index_metadata.rs b/engine/crates/xerj-engine/tests/corrupt_index_metadata.rs
new file mode 100644
index 000000000..55bbaa617
--- /dev/null
+++ b/engine/crates/xerj-engine/tests/corrupt_index_metadata.rs
@@ -0,0 +1,329 @@
+//! Issue #202 — an unparseable index sidecar must never be silently replaced
+//! by a default.
+//!
+//! `schema.json` and `settings.json` are the only on-disk record of an index's
+//! explicit mapping and analysis chain. Before this fix `Index::open` did
+//!
+//! ```ignore
+//! let schema = load_schema(&index_dir).unwrap_or_else(|_| ManagedSchema::dynamic());
+//! ```
+//!
+//! so a torn write, a bad disk or a truncated restore opened the index green
+//! with every explicit field gone — and the next documents re-inferred the
+//! field types from their own values. A `keyword` that was deliberately not
+//! analyzed silently became whatever the first value looked like.
+//!
+//! ABSENT and UNPARSEABLE are different conditions. Absent is legitimate
+//! (pre-0.6 indices never wrote `schema.json` at create time, and an index
+//! created without settings has no `settings.json`) and must still open with a
+//! dynamic mapping. Unparseable means the mapping is lost, and the index must
+//! refuse to open so the operator sees red instead of quiet data corruption.
+
+use serde_json::json;
+use tempfile::TempDir;
+use xerj_common::config::Config;
+use xerj_common::types::{FieldConfig, FieldType, IndexName, Schema};
+use xerj_engine::index::Index;
+use xerj_engine::Engine;
+
+fn config_for(dir: &TempDir) -> Config {
+ let mut config = Config::default();
+ config.server.data_dir = dir.path().to_str().unwrap().to_string();
+ // `Index::create`/`Index::open` outside `Engine::new` still need the
+ // process-wide governor; `init` is idempotent (first call wins).
+ xerj_engine::governor::init(&config);
+ config
+}
+
+/// An index whose mapping pins `code` as a `keyword` — the field whose meaning
+/// silently changed when the schema was dropped on the floor.
+fn mapped_schema() -> Schema {
+ let mut schema = Schema::empty();
+ schema
+ .add_field(FieldConfig::new("code", FieldType::Keyword))
+ .unwrap();
+ schema
+ .add_field(FieldConfig::new("body", FieldType::Text))
+ .unwrap();
+ schema
+}
+
+/// Create `name` with an explicit mapping, then drop it (closing the store).
+async fn create_mapped_index(dir: &TempDir, name: &str) {
+ let config = config_for(dir);
+ let index = Index::create(
+ IndexName::new(name).unwrap(),
+ mapped_schema(),
+ &config,
+ dir.path(),
+ )
+ .unwrap();
+ index
+ .index_document(Some("d1".to_string()), json!({"code": "A-1", "body": "hi"}))
+ .await
+ .unwrap();
+ index.flush().await.unwrap();
+ drop(index);
+}
+
+// ── schema.json ───────────────────────────────────────────────────────────────
+
+/// The core defect. Pre-fix this opened `Ok` with `field_count() == 0`.
+#[tokio::test]
+async fn unparseable_schema_json_refuses_to_open() {
+ let dir = TempDir::new().unwrap();
+ create_mapped_index(&dir, "torn_schema").await;
+
+ // A torn write: the file exists, is non-empty, and is not valid JSON.
+ let path = dir.path().join("torn_schema").join("schema.json");
+ let good = std::fs::read(&path).unwrap();
+ std::fs::write(&path, &good[..good.len() / 2]).unwrap();
+
+ let config = config_for(&dir);
+ let err = Index::open(IndexName::new("torn_schema").unwrap(), &config, dir.path())
+ .err()
+ .map(|e| e.to_string())
+ .unwrap_or_else(|| {
+ panic!(
+ "opened an index whose schema.json is unparseable — the explicit mapping is gone"
+ )
+ });
+ assert!(
+ err.contains("schema.json"),
+ "the error must name the file the operator has to restore, got: {err}"
+ );
+}
+
+/// Semantically valid JSON that is not a `ManagedSchema` is just as lost as a
+/// torn file — `serde_json::from_slice` fails either way and the old code
+/// swallowed both.
+#[tokio::test]
+async fn wrong_shape_schema_json_refuses_to_open() {
+ let dir = TempDir::new().unwrap();
+ create_mapped_index(&dir, "wrong_shape").await;
+
+ let path = dir.path().join("wrong_shape").join("schema.json");
+ std::fs::write(&path, br#"{"schema": "not an object"}"#).unwrap();
+
+ let config = config_for(&dir);
+ assert!(
+ Index::open(IndexName::new("wrong_shape").unwrap(), &config, dir.path()).is_err(),
+ "a schema.json that does not deserialize must not fall back to a dynamic mapping"
+ );
+}
+
+/// The other half of the contract: absent is not unparseable. Indices created
+/// before create-time schema persistence have no `schema.json` at all and must
+/// keep opening.
+#[tokio::test]
+async fn absent_schema_json_still_opens_dynamic() {
+ let dir = TempDir::new().unwrap();
+ create_mapped_index(&dir, "no_schema").await;
+ std::fs::remove_file(dir.path().join("no_schema").join("schema.json")).unwrap();
+
+ let config = config_for(&dir);
+ let index = Index::open(IndexName::new("no_schema").unwrap(), &config, dir.path())
+ .expect("an index without schema.json must still open with a dynamic mapping");
+ assert_eq!(index.schema().await.field_count(), 0);
+}
+
+/// A mapping that parses must still round-trip — the fix must not turn a
+/// healthy reopen into a failure.
+#[tokio::test]
+async fn intact_schema_json_still_opens_with_its_mapping() {
+ let dir = TempDir::new().unwrap();
+ create_mapped_index(&dir, "intact").await;
+
+ let config = config_for(&dir);
+ let index = Index::open(IndexName::new("intact").unwrap(), &config, dir.path())
+ .expect("healthy reopen");
+ let schema = index.schema().await;
+ assert_eq!(
+ schema.field("code").map(|f| f.field_type),
+ Some(FieldType::Keyword),
+ "the explicit keyword mapping must survive a restart"
+ );
+}
+
+// ── settings.json ─────────────────────────────────────────────────────────────
+
+/// `settings.json` carries the analysis chain and the WAL shard count. A
+/// truncated one used to become `Value::Null`, which silently swaps every
+/// custom analyzer for the default one — and changes the WAL layout the store
+/// is opened with.
+#[tokio::test]
+async fn unparseable_settings_json_refuses_to_open() {
+ let dir = TempDir::new().unwrap();
+ {
+ let config = config_for(&dir);
+ let index = Index::create_with_settings(
+ IndexName::new("torn_settings").unwrap(),
+ mapped_schema(),
+ json!({"index": {"number_of_replicas": 1, "refresh_interval": "5s"}}),
+ &config,
+ dir.path(),
+ )
+ .unwrap();
+ drop(index);
+ }
+
+ let path = dir.path().join("torn_settings").join("settings.json");
+ let good = std::fs::read(&path).unwrap();
+ std::fs::write(&path, &good[..good.len() / 2]).unwrap();
+
+ let config = config_for(&dir);
+ let err = Index::open(
+ IndexName::new("torn_settings").unwrap(),
+ &config,
+ dir.path(),
+ )
+ .err()
+ .map(|e| e.to_string())
+ .unwrap_or_else(|| panic!("opened an index whose settings.json is unparseable"));
+ assert!(
+ err.contains("settings.json"),
+ "the error must name the file, got: {err}"
+ );
+}
+
+/// An index created without settings has no `settings.json`; that must keep
+/// opening.
+#[tokio::test]
+async fn absent_settings_json_still_opens() {
+ let dir = TempDir::new().unwrap();
+ create_mapped_index(&dir, "no_settings").await;
+ assert!(
+ !dir.path()
+ .join("no_settings")
+ .join("settings.json")
+ .exists(),
+ "Index::create with null settings writes no settings.json"
+ );
+
+ let config = config_for(&dir);
+ Index::open(IndexName::new("no_settings").unwrap(), &config, dir.path())
+ .expect("an index without settings.json must still open");
+}
+
+// ── es_mapping.json ───────────────────────────────────────────────────────────
+
+/// The raw ES mapping blob (analyzers, date formats, `dense_vector` dims) is
+/// the mapping users actually see through `GET /{index}/_mapping`. A corrupt
+/// one used to be logged and ignored, leaving the index serving with a
+/// silently emptier mapping than the one on disk.
+#[tokio::test]
+async fn unparseable_es_mapping_json_fails_the_index_on_startup() {
+ let dir = TempDir::new().unwrap();
+ create_mapped_index(&dir, "torn_es_mapping").await;
+ std::fs::write(
+ dir.path().join("torn_es_mapping").join("es_mapping.json"),
+ b"{\"properties\": ",
+ )
+ .unwrap();
+
+ let engine = Engine::new(config_for(&dir)).expect("engine::new");
+ assert!(
+ engine.get_index("torn_es_mapping").is_err(),
+ "an index with a corrupt es_mapping.json must not be served"
+ );
+ assert!(
+ engine.failed_indices.contains_key("torn_es_mapping"),
+ "the failure must be recorded so cluster health goes red"
+ );
+ assert_eq!(engine.health().await.status, "red");
+}
+
+// ── the user-visible signal ───────────────────────────────────────────────────
+
+/// End to end: a node booting over a corrupt `schema.json` must come up red
+/// with the index unserved, not green with an empty mapping.
+#[tokio::test]
+async fn engine_startup_reports_red_for_a_corrupt_schema() {
+ let dir = TempDir::new().unwrap();
+ create_mapped_index(&dir, "torn_boot").await;
+ let path = dir.path().join("torn_boot").join("schema.json");
+ let good = std::fs::read(&path).unwrap();
+ std::fs::write(&path, &good[..good.len() / 2]).unwrap();
+
+ let engine = Engine::new(config_for(&dir)).expect("engine::new");
+ assert!(
+ engine.get_index("torn_boot").is_err(),
+ "a corrupt index must not be served with a re-inferred mapping"
+ );
+ let reason = engine
+ .failed_indices
+ .get("torn_boot")
+ .map(|r| r.value().clone())
+ .expect("the failure must be recorded in failed_indices");
+ assert!(
+ reason.contains("schema.json"),
+ "the recorded reason must name the file, got: {reason}"
+ );
+ assert_eq!(engine.health().await.status, "red");
+}
+
+/// Refusing the open is only half the fix: a failed index is absent from
+/// `indices`, so `PUT /{index}` and bulk auto-create used to sail straight past
+/// the "already exists" check and run `Index::create` over the corrupt
+/// directory — overwriting `schema.json` with an empty mapping and destroying
+/// the evidence. That door has to be shut too.
+#[tokio::test]
+async fn create_over_a_failed_index_is_refused() {
+ let dir = TempDir::new().unwrap();
+ create_mapped_index(&dir, "no_recreate").await;
+ let path = dir.path().join("no_recreate").join("schema.json");
+ let good = std::fs::read(&path).unwrap();
+ std::fs::write(&path, &good[..good.len() / 2]).unwrap();
+
+ let engine = Engine::new(config_for(&dir)).expect("engine::new");
+ let err = engine
+ .create_index("no_recreate", Schema::empty())
+ .expect_err("re-creating a failed index must not silently succeed")
+ .to_string();
+ assert!(
+ err.contains("failed to open") && err.contains("DELETE"),
+ "the error must explain the state and the way out, got: {err}"
+ );
+ // Auto-create (the bulk/index-a-document door) goes through the same gate.
+ assert!(engine.get_or_create_index("no_recreate").is_err());
+ // And the corrupt file is still there, untouched, for the operator.
+ assert_eq!(
+ std::fs::read(&path).unwrap().len(),
+ good.len() / 2,
+ "a refused create must not have rewritten schema.json"
+ );
+}
+
+/// …and refusing to create must not brick the name. Delete is the operator's
+/// recovery path (restore from backup afterwards), so it has to work on an
+/// index that never opened.
+#[tokio::test]
+async fn a_failed_index_can_still_be_deleted() {
+ let dir = TempDir::new().unwrap();
+ create_mapped_index(&dir, "removable").await;
+ let path = dir.path().join("removable").join("schema.json");
+ let good = std::fs::read(&path).unwrap();
+ std::fs::write(&path, &good[..good.len() / 2]).unwrap();
+
+ let engine = Engine::new(config_for(&dir)).expect("engine::new");
+ assert_eq!(engine.health().await.status, "red");
+
+ engine
+ .delete_index("removable")
+ .await
+ .expect("a failed index must be deletable — otherwise the name is bricked");
+ assert!(
+ !dir.path().join("removable").exists(),
+ "the data directory must be gone"
+ );
+ assert!(!engine.failed_indices.contains_key("removable"));
+ assert_eq!(
+ engine.health().await.status,
+ "green",
+ "health must recover once the corrupt index is gone"
+ );
+
+ // The name is usable again.
+ engine.create_index("removable", Schema::empty()).unwrap();
+ assert!(engine.get_index("removable").is_ok());
+}