Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

feat(cluster): replicate index metadata through the Raft log (#2257)

xerj-org4c52a68889a3
diff --git a/engine/Cargo.lock b/engine/Cargo.lock
index fd5cd2af0..2aa6c91d6 100644
--- a/engine/Cargo.lock
+++ b/engine/Cargo.lock
@@ -7376,6 +7376,7 @@ dependencies = [
"uuid",
"xerj-ai",
"xerj-autoindex",
+ "xerj-cluster",
"xerj-common",
"xerj-engine",
"xerj-hub",
@@ -7564,6 +7565,7 @@ version = "1.0.0-rc.98"
dependencies = [
"anyhow",
"arc-swap",
+ "async-trait",
"bytes",
"chrono",
"chrono-tz",
diff --git a/engine/crates/xerj-api/Cargo.toml b/engine/crates/xerj-api/Cargo.toml
index ce7e0800e..12662dfd9 100644
--- a/engine/crates/xerj-api/Cargo.toml
+++ b/engine/crates/xerj-api/Cargo.toml
@@ -71,5 +71,6 @@ sha2 = "0.10"
[dev-dependencies]
tempfile = { workspace = true }
+xerj-cluster = { path = "../xerj-cluster" }
tower = { workspace = true, features = ["util"] }
http-body-util = "0.1"
diff --git a/engine/crates/xerj-api/src/error.rs b/engine/crates/xerj-api/src/error.rs
index a0d95c28b..7afc43a9c 100644
--- a/engine/crates/xerj-api/src/error.rs
+++ b/engine/crates/xerj-api/src/error.rs
@@ -310,6 +310,7 @@ impl IntoResponse for ApiError {
/// | `VersionConflict` | `version_conflict_engine_exception` | 409 |
/// | `ResultWindowTooLarge` | `illegal_argument_exception` | 400 |
/// | `IndexBlocked` | `cluster_block_exception` | 403 |
+/// | `ClusterUnavailable` | `master_not_discovered_exception` or `cluster_unavailable_exception` | 503 |
/// | `Internal` | `internal_server_error_exception` | 500 |
fn xerj_error_type(e: &XerjError) -> String {
match e {
@@ -351,6 +352,16 @@ fn xerj_error_type(e: &XerjError) -> String {
XerjError::ResultWindowTooLarge { .. } => "illegal_argument_exception",
// ── Index blocks ──────────────────────────────────────────────────
XerjError::IndexBlocked { .. } => "cluster_block_exception",
+ // No leader is the ES "master not discovered" case. Any other reason
+ // (timeout, leadership lost, shutting down) is a generic 503: the
+ // write was not acknowledged and was not applied on this node.
+ XerjError::ClusterUnavailable { reason } => {
+ if reason.contains("no elected raft leader") {
+ "master_not_discovered_exception"
+ } else {
+ "cluster_unavailable_exception"
+ }
+ }
// ── Catch-all ─────────────────────────────────────────────────────
XerjError::Internal { .. } => "internal_server_error_exception",
}
diff --git a/engine/crates/xerj-api/src/es_compat.rs b/engine/crates/xerj-api/src/es_compat.rs
index d75b4b682..a9f3f2d97 100644
--- a/engine/crates/xerj-api/src/es_compat.rs
+++ b/engine/crates/xerj-api/src/es_compat.rs
@@ -1415,6 +1415,62 @@ pub struct EsCreateIndexMappings {
pub properties: Option<Value>,
}
+/// Display-side settings blob stored for `GET /{index}/_settings` at create.
+///
+/// `None` when the request carried neither settings nor an index-sort
+/// decision — the single-node path then leaves `index_settings` untouched.
+fn create_time_index_settings(
+ body_settings: Option<&Value>,
+ explicit_sort_field: &Option<String>,
+ explicit_sort_order: &str,
+ declared_timestamp_date: bool,
+) -> Option<Value> {
+ if let Some(settings) = body_settings {
+ let mut s = settings.clone();
+ if let Some(f) = explicit_sort_field.as_ref() {
+ if let Some(o) = s.as_object_mut() {
+ o.insert("__xy_index_sort_field".to_string(), json!(f));
+ o.insert(
+ "__xy_index_sort_order".to_string(),
+ json!(explicit_sort_order),
+ );
+ o.insert("__xy_index_sort_explicit".to_string(), json!(true));
+ }
+ } else if declared_timestamp_date {
+ if let Some(o) = s.as_object_mut() {
+ o.insert("__xy_index_sort_field".to_string(), json!("@timestamp"));
+ o.insert("__xy_index_sort_order".to_string(), json!("desc"));
+ }
+ }
+ return Some(s);
+ }
+ if let Some(f) = explicit_sort_field.as_ref() {
+ return Some(json!({
+ "__xy_index_sort_field": f,
+ "__xy_index_sort_order": explicit_sort_order,
+ "__xy_index_sort_explicit": true,
+ }));
+ }
+ if declared_timestamp_date {
+ return Some(json!({
+ "__xy_index_sort_field": "@timestamp",
+ "__xy_index_sort_order": "desc",
+ }));
+ }
+ None
+}
+
+/// Alias map with date-math names resolved. `None` when the body has no
+/// `aliases` object.
+fn resolved_create_aliases(body: &Value) -> Option<serde_json::Map<String, Value>> {
+ let aliases = body.get("aliases")?.as_object()?;
+ let mut resolved = serde_json::Map::new();
+ for (alias, opts) in aliases {
+ resolved.insert(resolve_date_math_index(alias), opts.clone());
+ }
+ Some(resolved)
+}
+
pub async fn create_index(
State(state): State<AppState>,
Path(index): Path<String>,
@@ -1535,6 +1591,41 @@ pub async fn create_index(
// indices at a single WAL file descriptor). create_index_with_settings
// applies index templates identically to create_index.
let create_settings = body.get("settings").cloned().unwrap_or(Value::Null);
+ let display_settings = create_time_index_settings(
+ body.get("settings"),
+ &explicit_sort_field,
+ &explicit_sort_order,
+ declared_timestamp_date,
+ );
+ let resolved_aliases = resolved_create_aliases(&body);
+
+ // Cluster members replicate the create. The accepting node evaluates
+ // templates and date-math aliases; peers apply that result. ILM attach
+ // stays on this node either way — it is not in the log.
+ if state.engine.has_cluster() {
+ let spec = xerj_engine::ReplicatedIndexSpec {
+ name: index.clone(),
+ schema,
+ settings: create_settings,
+ mappings: mappings_val,
+ index_settings: display_settings.unwrap_or(Value::Null),
+ aliases: resolved_aliases
+ .clone()
+ .map(Value::Object)
+ .unwrap_or(Value::Null),
+ };
+ return match state.engine.replicate_index_create(spec).await {
+ Ok(()) => {
+ if let Some(Some(policy_name)) = ilm_lifecycle_directive_from_settings(&body) {
+ let _ = crate::ism_api::attach_policy(&state, &index, &policy_name).await;
+ }
+ let resp = EsIndexResponse::ok(&index);
+ (StatusCode::OK, Json(resp)).into_response()
+ }
+ Err(e) => ApiError::new(xerj_common::XerjError::from(e)).into_response(),
+ };
+ }
+
match state
.engine
.create_index_with_settings(&index, schema, create_settings)
@@ -1543,44 +1634,8 @@ pub async fn create_index(
// Store the raw settings / mappings / aliases blob as written.
// These are used by `GET /{index}/_settings`, `GET /{index}/_mapping`,
// `GET /{index}`, and cluster health replica checks.
- if let Some(settings) = body.get("settings") {
- let mut s = settings.clone();
- if let Some(f) = explicit_sort_field.as_ref() {
- if let Some(o) = s.as_object_mut() {
- o.insert("__xy_index_sort_field".to_string(), json!(f));
- o.insert(
- "__xy_index_sort_order".to_string(),
- json!(explicit_sort_order),
- );
- // Mark this as an EXPLICIT index sort (vs the
- // @timestamp auto-heuristic) so the search path can
- // map a lone `_doc` sort onto the index-sort field.
- o.insert("__xy_index_sort_explicit".to_string(), json!(true));
- }
- } else if declared_timestamp_date {
- if let Some(o) = s.as_object_mut() {
- o.insert("__xy_index_sort_field".to_string(), json!("@timestamp"));
- o.insert("__xy_index_sort_order".to_string(), json!("desc"));
- }
- }
+ if let Some(s) = display_settings {
state.engine.index_settings.insert(index.clone(), s);
- } else if let Some(f) = explicit_sort_field.as_ref() {
- state.engine.index_settings.insert(
- index.clone(),
- json!({
- "__xy_index_sort_field": f,
- "__xy_index_sort_order": explicit_sort_order,
- "__xy_index_sort_explicit": true,
- }),
- );
- } else if declared_timestamp_date {
- state.engine.index_settings.insert(
- index.clone(),
- json!({
- "__xy_index_sort_field": "@timestamp",
- "__xy_index_sort_order": "desc",
- }),
- );
}
if !mappings_val.is_null() {
// Persists es_mapping.json into the index dir (atomic) so
@@ -1589,16 +1644,13 @@ pub async fn create_index(
return ApiError::new(xerj_common::XerjError::from(error)).into_response();
}
}
- if let Some(aliases) = body.get("aliases").and_then(Value::as_object) {
+ if let Some(resolved_aliases) = resolved_aliases {
// Alias keys can also contain date math
// (`<logs_{now/d}>` → `logs_2026-04-19`). Resolve each
// before storing / registering. The meta map keys use the
// resolved name so GET /_alias round-trips correctly.
- let mut resolved_aliases = serde_json::Map::new();
- for (alias, opts) in aliases {
- let resolved = resolve_date_math_index(alias);
- resolved_aliases.insert(resolved.clone(), opts.clone());
- if let Err(error) = state.engine.add_alias(&resolved, &index) {
+ for alias in resolved_aliases.keys() {
+ if let Err(error) = state.engine.add_alias(alias, &index) {
return ApiError::new(xerj_common::XerjError::from(error)).into_response();
}
}
@@ -3223,6 +3275,19 @@ pub async fn put_mapping(
plans.push((idx_name.clone(), idx, existing, additions));
}
+ if state.engine.has_cluster() {
+ for (idx_name, _, existing, additions) in plans {
+ if let Err(error) = state
+ .engine
+ .replicate_mapping(&idx_name, &existing, &additions)
+ .await
+ {
+ return ApiError::new(xerj_common::XerjError::from(error)).into_response();
+ }
+ }
+ return Json(json!({ "acknowledged": true })).into_response();
+ }
+
// Publication begins only after every deterministic candidate has passed.
// `add_fields` repeats engine-side validation defensively. Concurrent
// schema evolution and filesystem failures still require a broader
@@ -32105,46 +32170,54 @@ pub async fn put_settings(
return ApiError::new(e).into_response();
}
- for idx in &targets {
- let mut existing = state
- .engine
- .index_settings
- .get(idx)
- .map(|v| v.clone())
- .unwrap_or(json!({ "index": {} }));
- let inner_slot = existing
- .as_object_mut()
- .and_then(|m| m.get_mut("index").and_then(Value::as_object_mut));
- match (inner_slot, inner.as_object()) {
- (Some(existing_inner), Some(new_inner)) => {
- for (k, v) in new_inner {
- existing_inner.insert(k.clone(), v.clone());
- }
+ if state.engine.has_cluster() {
+ for idx in &targets {
+ if let Err(e) = state.engine.replicate_index_settings(idx, &body).await {
+ return ApiError::new(xerj_common::XerjError::from(e)).into_response();
}
- _ => {
- // Either no existing index block or body wasn't an object —
- // replace wholesale.
- existing = json!({ "index": inner });
- state.engine.index_settings.insert(idx.clone(), existing);
- continue;
+ }
+ } else {
+ for idx in &targets {
+ let mut existing = state
+ .engine
+ .index_settings
+ .get(idx)
+ .map(|v| v.clone())
+ .unwrap_or(json!({ "index": {} }));
+ let inner_slot = existing
+ .as_object_mut()
+ .and_then(|m| m.get_mut("index").and_then(Value::as_object_mut));
+ match (inner_slot, inner.as_object()) {
+ (Some(existing_inner), Some(new_inner)) => {
+ for (k, v) in new_inner {
+ existing_inner.insert(k.clone(), v.clone());
+ }
+ }
+ _ => {
+ // Either no existing index block or body wasn't an object —
+ // replace wholesale.
+ existing = json!({ "index": inner });
+ state.engine.index_settings.insert(idx.clone(), existing);
+ continue;
+ }
}
+ state.engine.index_settings.insert(idx.clone(), existing);
}
- state.engine.index_settings.insert(idx.clone(), existing);
- }
- // `index.blocks.*` is not a display-only setting: it gates writes and reads
- // in the engine, whose `Index::settings` is a different store from the
- // `engine.index_settings` map updated above. Forward the block keys so a
- // settings PUT can both set *and clear* a block — previously it updated only
- // the display copy, leaving every block permanent once set.
- for name in &targets {
- let Ok(idx) = state.engine.get_index(name) else {
- continue;
- };
- if let Err(e) = idx.apply_block_settings(&body).await {
- return ApiError::new(xerj_common::XerjError::from(e)).into_response();
+ // `index.blocks.*` is not a display-only setting: it gates writes and reads
+ // in the engine, whose `Index::settings` is a different store from the
+ // `engine.index_settings` map updated above. Forward the block keys so a
+ // settings PUT can both set *and clear* a block — previously it updated only
+ // the display copy, leaving every block permanent once set.
+ for name in &targets {
+ let Ok(idx) = state.engine.get_index(name) else {
+ continue;
+ };
+ if let Err(e) = idx.apply_block_settings(&body).await {
+ return ApiError::new(xerj_common::XerjError::from(e)).into_response();
+ }
+ sync_display_blocks(&state, name).await;
}
- sync_display_blocks(&state, name).await;
}
// `PUT {index}/_settings` is the other real ILM entry point — for
diff --git a/engine/crates/xerj-api/src/native.rs b/engine/crates/xerj-api/src/native.rs
index 6406ca689..da09e1175 100644
--- a/engine/crates/xerj-api/src/native.rs
+++ b/engine/crates/xerj-api/src/native.rs
@@ -184,7 +184,22 @@ pub async fn create_index(
}
}
- if let Err(e) = state.engine.create_index(&name_str, schema) {
+ let create_result = if state.engine.has_cluster() {
+ state
+ .engine
+ .replicate_index_create(xerj_engine::ReplicatedIndexSpec {
+ name: name_str.clone(),
+ schema,
+ settings: Value::Null,
+ mappings: Value::Null,
+ index_settings: Value::Null,
+ aliases: Value::Null,
+ })
+ .await
+ } else {
+ state.engine.create_index(&name_str, schema)
+ };
+ if let Err(e) = create_result {
let xerj_err: xerj_common::XerjError = e.into();
return native_error(
xerj_err,
diff --git a/engine/crates/xerj-api/tests/cluster_metadata_replicates.rs b/engine/crates/xerj-api/tests/cluster_metadata_replicates.rs
new file mode 100644
index 000000000..b08f64760
--- /dev/null
+++ b/engine/crates/xerj-api/tests/cluster_metadata_replicates.rs
@@ -0,0 +1,278 @@
+//! Index create, mapping, settings, and delete on one member of a 3-node
+//! ring show up on the others. Documents do not. A ring that is not running
+//! refuses the create instead of applying it locally.
+
+use std::time::{Duration, Instant};
+
+use axum::body::Body;
+use axum::http::{Request, StatusCode};
+use axum::Router;
+use serde_json::{json, Value};
+use tokio::sync::watch;
+use tower::ServiceExt;
+use xerj_cluster::node::in_memory::{InMemoryBus, InMemoryTransport};
+use xerj_cluster::runner::ClusterRunner;
+use xerj_cluster::{ClusterHandle, ClusterNode};
+use xerj_engine::EngineMetadataApplier;
+
+struct Member {
+ app: Router,
+ shutdown: watch::Sender<bool>,
+ handle: ClusterHandle,
+ /// Held when the runner is not started, so the proposal channel stays open.
+ _node: Option<ClusterNode>,
+ _dir: tempfile::TempDir,
+}
+
+async fn ring(start_runners: bool) -> Vec<Member> {
+ let bus = InMemoryBus::new();
+ let ids = ["n1", "n2", "n3"];
+ let mut members = Vec::new();
+ for id in ids {
+ let dir = tempfile::tempdir().expect("tempdir");
+ let mut config = xerj_common::config::Config::default();
+ config.server.data_dir = dir.path().to_string_lossy().into_owned();
+ let metrics = xerj_common::metrics::Metrics::new().expect("metrics");
+ let engine = xerj_engine::Engine::new(config.clone()).expect("engine");
+ let state = xerj_api::state::AppState::new(config, engine, metrics);
+
+ let peers: Vec<String> = ids
+ .iter()
+ .filter(|peer| **peer != id)
+ .map(|peer| (*peer).to_string())
+ .collect();
+ let transport = InMemoryTransport::new(id.to_string(), bus.clone()).await;
+ let mut node = ClusterNode::new(id.to_string(), peers, Box::new(transport));
+ node.set_applier(std::sync::Arc::new(EngineMetadataApplier {
+ engine: std::sync::Arc::clone(&state.engine),
+ }));
+ let handle = node.handle();
+ state.engine.attach_cluster(handle.clone());
+ let (shutdown, shutdown_rx) = watch::channel(false);
+ let kept = if start_runners {
+ let mut runner = ClusterRunner::new(node, Duration::from_millis(5), shutdown_rx);
+ tokio::spawn(async move {
+ runner.run().await;
+ });
+ None
+ } else {
+ handle.set_timeout(Duration::from_millis(400));
+ let _ = shutdown_rx;
+ Some(node)
+ };
+ let app = xerj_api::router::build_es_compat_router(state);
+ members.push(Member {
+ app,
+ shutdown,
+ handle,
+ _node: kept,
+ _dir: dir,
+ });
+ }
+ members
+}
+
+async fn send(app: &Router, req: Request<Body>) -> (StatusCode, Value) {
+ let response = app.clone().oneshot(req).await.expect("response");
+ let status = response.status();
+ let bytes = axum::body::to_bytes(response.into_body(), usize::MAX)
+ .await
+ .expect("body");
+ let body = serde_json::from_slice(&bytes).unwrap_or(Value::Null);
+ (status, body)
+}
+
+async fn put_json(app: &Router, path: &str, body: Value) -> (StatusCode, Value) {
+ send(
+ app,
+ Request::put(path)
+ .header("content-type", "application/json")
+ .body(Body::from(body.to_string()))
+ .expect("request"),
+ )
+ .await
+}
+
+async fn get(app: &Router, path: &str) -> (StatusCode, Value) {
+ send(
+ app,
+ Request::get(path).body(Body::empty()).expect("request"),
+ )
+ .await
+}
+
+async fn delete(app: &Router, path: &str) -> (StatusCode, Value) {
+ send(
+ app,
+ Request::delete(path).body(Body::empty()).expect("request"),
+ )
+ .await
+}
+
+async fn wait_leader(members: &[Member], dead: &[bool]) -> usize {
+ let deadline = Instant::now() + Duration::from_secs(10);
+ loop {
+ let leaders: Vec<usize> = members
+ .iter()
+ .enumerate()
+ .filter(|(i, member)| !dead[*i] && member.handle.is_leader())
+ .map(|(i, _)| i)
+ .collect();
+ if leaders.len() == 1 {
+ return leaders[0];
+ }
+ if Instant::now() > deadline {
+ panic!("timed out waiting for one raft leader");
+ }
+ tokio::time::sleep(Duration::from_millis(20)).await;
+ }
+}
+
+async fn wait_status(app: &Router, path: &str, want: StatusCode) -> Value {
+ let deadline = Instant::now() + Duration::from_secs(5);
+ loop {
+ let (status, body) = get(app, path).await;
+ if status == want {
+ return body;
+ }
+ if Instant::now() > deadline {
+ panic!("GET {path} stayed {status} body={body} (wanted {want})");
+ }
+ tokio::time::sleep(Duration::from_millis(20)).await;
+ }
+}
+
+async fn wait_body_contains(app: &Router, path: &str, needle: &str) -> Value {
+ let deadline = Instant::now() + Duration::from_secs(5);
+ loop {
+ let (status, body) = get(app, path).await;
+ if status.is_success() && body.to_string().contains(needle) {
+ return body;
+ }
+ if Instant::now() > deadline {
+ panic!("GET {path} never contained {needle}: {status} {body}");
+ }
+ tokio::time::sleep(Duration::from_millis(20)).await;
+ }
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
+async fn index_metadata_propagates_documents_do_not_and_a_new_leader_still_commits() {
+ let members = ring(true).await;
+ let mut dead = [false, false, false];
+ let leader = wait_leader(&members, &dead).await;
+ let follower = (0..3).find(|i| *i != leader).expect("a follower");
+
+ let (status, body) = put_json(
+ &members[follower].app,
+ "/orders",
+ json!({
+ "mappings": { "properties": { "title": { "type": "text" } } }
+ }),
+ )
+ .await;
+ assert!(status.is_success(), "create from follower: {status} {body}");
+
+ for member in &members {
+ wait_status(&member.app, "/orders", StatusCode::OK).await;
+ let mapping = wait_status(&member.app, "/orders/_mapping", StatusCode::OK).await;
+ assert!(
+ mapping.to_string().contains("title"),
+ "mapping did not propagate: {mapping}"
+ );
+ }
+
+ let (status, body) = put_json(
+ &members[follower].app,
+ "/orders/_settings",
+ json!({ "index": { "number_of_replicas": "2" } }),
+ )
+ .await;
+ assert!(status.is_success(), "settings: {status} {body}");
+ for member in &members {
+ wait_body_contains(&member.app, "/orders/_settings", "number_of_replicas").await;
+ }
+
+ let (status, body) = put_json(
+ &members[follower].app,
+ "/orders/_doc/1",
+ json!({ "title": "local only" }),
+ )
+ .await;
+ assert!(
+ status.is_success(),
+ "document write is still local: {status} {body}"
+ );
+ let (status, _) = get(&members[follower].app, "/orders/_doc/1").await;
+ assert!(status.is_success(), "writer should read its own document");
+ let other = (0..3).find(|i| *i != follower).expect("peer");
+ let (status, body) = get(&members[other].app, "/orders/_doc/1").await;
+ assert_eq!(
+ status,
+ StatusCode::NOT_FOUND,
+ "documents are not replicated yet: {body}"
+ );
+ let (status, _) = get(&members[other].app, "/orders").await;
+ assert_eq!(status, StatusCode::OK, "the index itself is on the peer");
+
+ let (status, body) = delete(&members[follower].app, "/orders").await;
+ assert!(status.is_success(), "delete: {status} {body}");
+ for member in &members {
+ wait_status(&member.app, "/orders", StatusCode::NOT_FOUND).await;
+ }
+
+ let _ = members[leader].shutdown.send(true);
+ dead[leader] = true;
+ let next = wait_leader(&members, &dead).await;
+ assert_ne!(next, leader);
+ let proposer = (0..3).find(|i| !dead[*i] && *i != next).unwrap_or(next);
+ let (status, body) = put_json(&members[proposer].app, "/tickets", json!({})).await;
+ assert!(
+ status.is_success(),
+ "create after leader change: {status} {body}"
+ );
+ for (i, member) in members.iter().enumerate() {
+ if dead[i] {
+ continue;
+ }
+ wait_status(&member.app, "/tickets", StatusCode::OK).await;
+ }
+
+ for member in &members {
+ let _ = member.shutdown.send(true);
+ }
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn a_ring_with_no_leader_does_not_create_the_index() {
+ let members = ring(false).await;
+ let (status, body) = put_json(&members[0].app, "/orders", json!({})).await;
+ assert_eq!(status, StatusCode::SERVICE_UNAVAILABLE, "{body}");
+ let (status, _) = get(&members[0].app, "/orders").await;
+ assert_eq!(status, StatusCode::NOT_FOUND);
+ let (status, _) = get(&members[1].app, "/orders").await;
+ assert_eq!(status, StatusCode::NOT_FOUND);
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn cluster_disabled_create_stays_on_the_node_that_received_it() {
+ async fn lone() -> (Router, tempfile::TempDir) {
+ let dir = tempfile::tempdir().expect("tempdir");
+ let mut config = xerj_common::config::Config::default();
+ config.server.data_dir = dir.path().to_string_lossy().into_owned();
+ let metrics = xerj_common::metrics::Metrics::new().expect("metrics");
+ let engine = xerj_engine::Engine::new(config.clone()).expect("engine");
+ assert!(!engine.has_cluster());
+ let state = xerj_api::state::AppState::new(config, engine, metrics);
+ (xerj_api::router::build_es_compat_router(state), dir)
+ }
+
+ let (a, _dir_a) = lone().await;
+ let (b, _dir_b) = lone().await;
+ let (status, body) = put_json(&a, "/local", json!({})).await;
+ assert!(status.is_success(), "{status} {body}");
+ let (status, _) = get(&a, "/local").await;
+ assert_eq!(status, StatusCode::OK);
+ let (status, _) = get(&b, "/local").await;
+ assert_eq!(status, StatusCode::NOT_FOUND);
+}
diff --git a/engine/crates/xerj-cluster/Cargo.toml b/engine/crates/xerj-cluster/Cargo.toml
index 0842a0069..b63d8b6a8 100644
--- a/engine/crates/xerj-cluster/Cargo.toml
+++ b/engine/crates/xerj-cluster/Cargo.toml
@@ -31,3 +31,4 @@ subtle = "2"
[dev-dependencies]
tokio = { version = "1", features = ["full", "test-util"] }
tempfile.workspace = true
+async-trait = "0.1"
diff --git a/engine/crates/xerj-cluster/src/lib.rs b/engine/crates/xerj-cluster/src/lib.rs
index e106d4b58..a44322f07 100644
--- a/engine/crates/xerj-cluster/src/lib.rs
+++ b/engine/crates/xerj-cluster/src/lib.rs
@@ -50,6 +50,7 @@ pub mod auth;
pub mod coordinator;
pub mod metadata;
pub mod node;
+pub mod propose;
pub mod raft;
pub mod raft_log;
pub mod regions;
@@ -67,6 +68,7 @@ pub use coordinator::{
};
pub use metadata::{ClusterMetadata, IndexMetadata, NodeInfo, NodeState};
pub use node::{ClusterNode, ClusterTransport};
+pub use propose::{ApplyEffect, ClusterHandle, MetadataApplier, ProposeError};
pub use raft::{ClusterCommand, LogEntry, RaftMessage, RaftNode, RaftState};
pub use regions::{Region, RegionManager, RegionMove};
pub use replication::{
diff --git a/engine/crates/xerj-cluster/src/metadata.rs b/engine/crates/xerj-cluster/src/metadata.rs
index c095a8aea..04636a5a5 100644
--- a/engine/crates/xerj-cluster/src/metadata.rs
+++ b/engine/crates/xerj-cluster/src/metadata.rs
@@ -111,6 +111,7 @@ impl ClusterMetadata {
return;
}
+ let command = command.as_metadata();
match command {
ClusterCommand::CreateIndex { name, schema_json } => {
self.apply_create_index(name, schema_json);
@@ -140,6 +141,20 @@ impl ClusterMetadata {
ClusterCommand::UpdateConfig { key, value } => {
self.apply_update_config(key, value);
}
+ ClusterCommand::UpdateIndexSettings {
+ index,
+ settings_json,
+ } => {
+ // The engine applier merges this into the live index. The
+ // metadata store keeps the latest body so a reader of the
+ // replicated state can see what was last committed.
+ info!(index = %index, "Applying UpdateIndexSettings");
+ self.config
+ .insert(format!("index_settings:{index}"), settings_json.clone());
+ }
+ ClusterCommand::ClientMetadata { .. } => {
+ warn!("nested ClientMetadata wrapper; not applied");
+ }
}
self.last_applied_index = log_index;
diff --git a/engine/crates/xerj-cluster/src/node.rs b/engine/crates/xerj-cluster/src/node.rs
index f3d3b76fe..e75229a22 100644
--- a/engine/crates/xerj-cluster/src/node.rs
+++ b/engine/crates/xerj-cluster/src/node.rs
@@ -8,13 +8,19 @@
//! In production, the transport will be a gRPC/QUIC channel. In tests, an
//! in-memory channel is used (see [`transport`] module).
-use std::time::Duration;
+use std::collections::HashMap;
+use std::sync::{Arc, RwLock};
+use std::time::{Duration, Instant};
use anyhow::Result;
use async_trait::async_trait;
+use tokio::sync::mpsc;
use tracing::{debug, info, warn};
use crate::metadata::ClusterMetadata;
+use crate::propose::{
+ ApplyEffect, ClusterHandle, ClusterView, MetadataApplier, ProposalRequest, ProposeError,
+};
use crate::raft::{ClusterCommand, OutboundMessage, RaftMessage, RaftNode, RaftState};
// ── Transport abstraction ─────────────────────────────────────────────────────
@@ -38,29 +44,85 @@ pub trait ClusterTransport: Send + Sync {
///
/// Drives the Raft state machine forward via periodic ticks and delivers
/// committed log entries to the local [`ClusterMetadata`] state machine.
+enum DrivePlan {
+ Proposed { index: u64, term: u64 },
+ Forward(Option<String>),
+ Wait,
+}
+
+enum WaiterState {
+ /// Not yet proposed or forwarded.
+ Pending,
+ /// Sent to `to`; do not send again until the leader changes or they reject.
+ Forwarded { to: String },
+ /// `to` rejected. `hint` is who they believe the leader is.
+ Rejected { hint: Option<String> },
+ /// Appended locally at `index`/`term`. Completed when that entry is applied.
+ Proposed { index: u64, term: u64 },
+}
+
+struct Waiter {
+ command: ClusterCommand,
+ deadline: Instant,
+ state: WaiterState,
+ ack: tokio::sync::oneshot::Sender<Result<ApplyEffect, ProposeError>>,
+}
+
pub struct ClusterNode {
pub raft: RaftNode,
pub metadata: ClusterMetadata,
transport: Box<dyn ClusterTransport>,
+ /// Cloneable proposer the engine/HTTP layer holds.
+ handle: ClusterHandle,
+ proposal_rx: mpsc::UnboundedReceiver<ProposalRequest>,
+ waiters: HashMap<u64, Waiter>,
+ applier: Option<Arc<dyn MetadataApplier>>,
+ view: Arc<RwLock<ClusterView>>,
}
impl ClusterNode {
/// Create a new cluster node with the given identity, peers, and transport.
pub fn new(id: String, peers: Vec<String>, transport: Box<dyn ClusterTransport>) -> Self {
+ let (tx, rx) = mpsc::unbounded_channel();
+ let view = Arc::new(RwLock::new(ClusterView {
+ node_id: id.clone(),
+ state: RaftState::Follower,
+ leader_id: None,
+ }));
+ let handle = ClusterHandle::new(id.clone(), tx, Arc::clone(&view));
ClusterNode {
raft: RaftNode::new(id, peers),
metadata: ClusterMetadata::new(),
transport,
+ handle,
+ proposal_rx: rx,
+ waiters: HashMap::new(),
+ applier: None,
+ view,
}
}
+ /// Handle the engine layer uses to propose metadata and read leadership.
+ pub fn handle(&self) -> ClusterHandle {
+ self.handle.clone()
+ }
+
+ /// Install the engine applier. Committed entries received after this call
+ /// are applied to local storage as well as [`ClusterMetadata`].
+ ///
+ /// Must be set before the first proposal. Entries applied with no applier
+ /// update only the in-memory metadata store.
+ pub fn set_applier(&mut self, applier: Arc<dyn MetadataApplier>) {
+ self.applier = Some(applier);
+ }
+
/// Tick the Raft state machine and dispatch any outbound messages.
///
/// Call this periodically (e.g. every 10 ms) from the cluster run loop.
pub async fn tick(&mut self) -> Result<()> {
let msgs = self.raft.tick();
self.dispatch(msgs).await?;
- self.apply_ready();
+ self.apply_ready().await;
Ok(())
}
@@ -68,9 +130,17 @@ impl ClusterNode {
pub async fn handle_message(&mut self, from: &str, msg: RaftMessage) -> Result<()> {
debug!(node = %self.raft.id, %from, "Handling inbound Raft message");
let _ = from; // currently unused directly — sender is embedded in message
+ if let RaftMessage::ForwardRejected {
+ request_id,
+ leader_id,
+ ..
+ } = &msg
+ {
+ self.note_rejection(*request_id, leader_id.clone());
+ }
let replies = self.raft.handle_message(msg);
self.dispatch(replies).await?;
- self.apply_ready();
+ self.apply_ready().await;
Ok(())
}
@@ -93,22 +163,32 @@ impl ClusterNode {
/// before it resolves only abandons an in-flight `recv`/`send`, which Raft
/// tolerates via retransmission.
pub async fn step(&mut self, tick_interval: Duration) -> Result<()> {
- // Try to receive a message within the tick interval.
- let recv_result = tokio::time::timeout(tick_interval, self.transport.recv()).await;
-
- match recv_result {
- Ok(Ok((from, msg))) => {
- self.handle_message(&from, msg).await?;
- }
- Ok(Err(e)) => {
- warn!(node = %self.raft.id, error = %e, "Transport recv error");
+ tokio::select! {
+ biased;
+ req = self.proposal_rx.recv() => {
+ if let Some(req) = req {
+ self.accept_proposal(req).await;
+ }
}
- Err(_timeout) => {
- // Timeout is normal — just tick.
+ recv_result = self.transport.recv() => {
+ match recv_result {
+ Ok((from, msg)) => {
+ if let Err(e) = self.handle_message(&from, msg).await {
+ warn!(node = %self.raft.id, error = %e, "Raft message handling error");
+ }
+ }
+ Err(e) => {
+ warn!(node = %self.raft.id, error = %e, "Transport recv error");
+ }
+ }
}
+ _ = tokio::time::sleep(tick_interval) => {}
}
self.tick().await?;
+ self.expire_waiters();
+ self.pump_waiters().await;
+ self.refresh_view();
Ok(())
}
@@ -160,7 +240,11 @@ impl ClusterNode {
}
/// Drain and apply all newly committed log entries.
- fn apply_ready(&mut self) {
+ ///
+ /// The engine applier runs before the waiting caller is released, so a
+ /// successful return means this node has the metadata locally and a
+ /// majority has the log entry.
+ async fn apply_ready(&mut self) {
let entries = self.raft.ready();
for entry in entries {
debug!(
@@ -170,6 +254,208 @@ impl ClusterNode {
"Applying committed entry"
);
self.metadata.apply(entry.index, &entry.command);
+ let effect = if let Some(applier) = &self.applier {
+ applier.apply_metadata(&entry.command).await
+ } else {
+ ApplyEffect::Applied
+ };
+ // Request ids are per accepting node. A leader applying a
+ // forwarded entry must not complete its own waiter that happens
+ // to share the number.
+ if entry.command.client_origin() == Some(self.raft.id.as_str()) {
+ if let Some(request_id) = entry.command.client_request_id() {
+ if let Some(waiter) = self.waiters.remove(&request_id) {
+ let _ = waiter.ack.send(Ok(effect));
+ }
+ }
+ }
+ }
+ }
+
+ fn refresh_view(&self) {
+ let mut view = self.view.write().unwrap_or_else(|e| e.into_inner());
+ view.state = self.raft.state().clone();
+ view.leader_id = self.raft.leader_id().map(|s| s.to_string());
+ }
+
+ async fn accept_proposal(&mut self, req: ProposalRequest) {
+ if Instant::now() >= req.deadline {
+ let _ = req.ack.send(Err(self.deadline_error()));
+ return;
+ }
+ self.waiters.insert(
+ req.request_id,
+ Waiter {
+ command: req.command,
+ deadline: req.deadline,
+ state: WaiterState::Pending,
+ ack: req.ack,
+ },
+ );
+ self.drive_waiter(req.request_id).await;
+ }
+
+ fn deadline_error(&self) -> ProposeError {
+ if self.raft.is_leader() || self.raft.leader_id().is_some() {
+ ProposeError::Timeout
+ } else {
+ ProposeError::NoLeader
+ }
+ }
+
+ fn note_rejection(&mut self, request_id: u64, hint: Option<String>) {
+ let Some(waiter) = self.waiters.get_mut(&request_id) else {
+ return;
+ };
+ // A rejection from the node we forwarded to. Do not forward there again.
+ if let WaiterState::Forwarded { to } = &waiter.state {
+ if hint.as_deref() == Some(to.as_str()) {
+ waiter.state = WaiterState::Rejected { hint: None };
+ return;
+ }
+ }
+ waiter.state = WaiterState::Rejected { hint };
+ }
+
+ fn expire_waiters(&mut self) {
+ let now = Instant::now();
+ let expired: Vec<u64> = self
+ .waiters
+ .iter()
+ .filter(|(_, w)| now >= w.deadline)
+ .map(|(id, _)| *id)
+ .collect();
+ for id in expired {
+ let Some(waiter) = self.waiters.remove(&id) else {
+ continue;
+ };
+ let err = match waiter.state {
+ WaiterState::Proposed { index, term }
+ if self.raft.entry_term(index) != Some(term) =>
+ {
+ ProposeError::LeadershipLost
+ }
+ _ => self.deadline_error(),
+ };
+ let _ = waiter.ack.send(Err(err));
+ }
+ }
+
+ async fn pump_waiters(&mut self) {
+ let ids: Vec<u64> = self.waiters.keys().copied().collect();
+ for id in ids {
+ self.drive_waiter(id).await;
+ }
+ }
+
+ async fn drive_waiter(&mut self, id: u64) {
+ let snapshot = {
+ let Some(waiter) = self.waiters.get(&id) else {
+ return;
+ };
+ if Instant::now() >= waiter.deadline {
+ return;
+ }
+ match &waiter.state {
+ WaiterState::Proposed { index, term } => DrivePlan::Proposed {
+ index: *index,
+ term: *term,
+ },
+ WaiterState::Forwarded { .. } => DrivePlan::Wait,
+ WaiterState::Rejected { hint } => DrivePlan::Forward(hint.clone()),
+ WaiterState::Pending => {
+ DrivePlan::Forward(self.raft.leader_id().map(|s| s.to_string()))
+ }
+ }
+ };
+ if let DrivePlan::Proposed { index, term } = snapshot {
+ // The entry was truncated (new leader overwrote it) before it
+ // committed. Fail the caller; do not append a second copy here.
+ if self.raft.entry_term(index) != Some(term) && self.raft.commit_index() < index {
+ if let Some(waiter) = self.waiters.remove(&id) {
+ let _ = waiter.ack.send(Err(ProposeError::LeadershipLost));
+ }
+ }
+ return;
+ }
+ if self.raft.is_leader() {
+ self.propose_waiter(id).await;
+ return;
+ }
+ let DrivePlan::Forward(target) = snapshot else {
+ return;
+ };
+ let Some(leader) = target.or_else(|| self.raft.leader_id().map(|s| s.to_string())) else {
+ return;
+ };
+ if leader == self.raft.id {
+ return;
+ }
+ self.forward_waiter(id, leader).await;
+ }
+
+ async fn propose_waiter(&mut self, id: u64) {
+ let Some(waiter) = self.waiters.get(&id) else {
+ return;
+ };
+ if matches!(waiter.state, WaiterState::Proposed { .. }) {
+ return;
+ }
+ let command = waiter.command.clone();
+ match self.raft.propose(command) {
+ Ok(index) => {
+ let term = self.raft.current_term();
+ if let Some(waiter) = self.waiters.get_mut(&id) {
+ waiter.state = WaiterState::Proposed { index, term };
+ }
+ let msgs = self.raft.force_heartbeat();
+ let _ = self.dispatch(msgs).await;
+ // Single-node clusters commit inside propose(); apply now so
+ // the caller is not waiting for the next tick.
+ self.apply_ready().await;
+ }
+ Err(e) => {
+ warn!(node = %self.raft.id, error = %e, "propose failed; will forward");
+ if let Some(waiter) = self.waiters.get_mut(&id) {
+ waiter.state = WaiterState::Pending;
+ }
+ }
+ }
+ }
+
+ async fn forward_waiter(&mut self, id: u64, leader: String) {
+ let Some(waiter) = self.waiters.get(&id) else {
+ return;
+ };
+ if matches!(&waiter.state, WaiterState::Forwarded { to } if to == &leader) {
+ return;
+ }
+ if matches!(waiter.state, WaiterState::Proposed { .. }) {
+ return;
+ }
+ let command = waiter.command.clone();
+ let term = self.raft.current_term();
+ let from = self.raft.id.clone();
+ match self
+ .transport
+ .send(
+ &leader,
+ RaftMessage::ForwardProposal {
+ term,
+ from,
+ command,
+ },
+ )
+ .await
+ {
+ Ok(()) => {
+ if let Some(waiter) = self.waiters.get_mut(&id) {
+ waiter.state = WaiterState::Forwarded { to: leader };
+ }
+ }
+ Err(e) => {
+ warn!(node = %self.raft.id, to = %leader, error = %e, "failed to forward metadata proposal");
+ }
}
}
}
diff --git a/engine/crates/xerj-cluster/src/propose.rs b/engine/crates/xerj-cluster/src/propose.rs
new file mode 100644
index 000000000..646eea79f
--- /dev/null
+++ b/engine/crates/xerj-cluster/src/propose.rs
@@ -0,0 +1,172 @@
+//! Caller-facing handle for proposing a metadata command and waiting until
+//! this node has applied the committed entry.
+//!
+//! The Raft state machine is owned by the cluster run loop. HTTP handlers
+//! do not touch it. They send a [`ProposalRequest`] through [`ClusterHandle`]
+//! and await the oneshot that the run loop completes from `apply_ready` —
+//! after the entry is committed, never before.
+
+use std::sync::atomic::{AtomicU64, Ordering};
+use std::sync::{Arc, RwLock};
+use std::time::{Duration, Instant};
+
+use tokio::sync::{mpsc, oneshot};
+
+use crate::raft::{ClusterCommand, RaftState};
+
+/// How a committed metadata command changed the local engine.
+///
+/// `Applied` is the success the caller acknowledges. The other variants are
+/// committed no-ops or local failures: the log entry exists either way, and
+/// the caller must not treat them as a successful mutation.
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub enum ApplyEffect {
+ Applied,
+ AlreadyExists,
+ NotFound,
+ Failed(String),
+}
+
+/// Why a metadata proposal was not acknowledged.
+///
+/// Every variant means the caller did not observe a commit of this request.
+/// The write must not be reported as success.
+#[derive(Debug, thiserror::Error)]
+pub enum ProposeError {
+ #[error("no elected raft leader; metadata writes are not accepted until a leader is elected")]
+ NoLeader,
+ #[error(
+ "timed out waiting for the metadata change to commit on a majority; the write was not acknowledged"
+ )]
+ Timeout,
+ #[error("cluster node is shutting down; the metadata write was not acknowledged")]
+ ShuttingDown,
+ #[error(
+ "raft leadership changed before the metadata write committed; the write was not acknowledged"
+ )]
+ LeadershipLost,
+}
+
+/// What the run loop publishes for [`ClusterHandle::is_leader`] / [`ClusterHandle::leader_id`].
+#[derive(Debug, Clone)]
+pub struct ClusterView {
+ pub node_id: String,
+ pub state: RaftState,
+ pub leader_id: Option<String>,
+}
+
+/// A metadata command waiting to be proposed or forwarded by the run loop.
+pub struct ProposalRequest {
+ pub request_id: u64,
+ pub command: ClusterCommand,
+ pub deadline: Instant,
+ pub ack: oneshot::Sender<Result<ApplyEffect, ProposeError>>,
+}
+
+/// Cloneable handle shared by the HTTP/engine layer.
+///
+/// Proposing does not apply the command locally. The run loop proposes (or
+/// forwards to the leader) and completes the future when this node applies
+/// the committed entry, or with an error if that does not happen in time.
+#[derive(Clone)]
+pub struct ClusterHandle {
+ inner: Arc<ClusterHandleInner>,
+}
+
+struct ClusterHandleInner {
+ node_id: String,
+ tx: mpsc::UnboundedSender<ProposalRequest>,
+ next: AtomicU64,
+ timeout_ms: AtomicU64,
+ view: Arc<RwLock<ClusterView>>,
+}
+
+impl ClusterHandle {
+ pub(crate) fn new(
+ node_id: String,
+ tx: mpsc::UnboundedSender<ProposalRequest>,
+ view: Arc<RwLock<ClusterView>>,
+ ) -> Self {
+ ClusterHandle {
+ inner: Arc::new(ClusterHandleInner {
+ node_id,
+ tx,
+ next: AtomicU64::new(1),
+ timeout_ms: AtomicU64::new(5_000),
+ view,
+ }),
+ }
+ }
+
+ pub fn node_id(&self) -> &str {
+ &self.inner.node_id
+ }
+
+ /// How long [`ClusterHandle::propose`] waits for a commit.
+ ///
+ /// The run loop uses the same budget: past the deadline it fails the
+ /// waiter and does not append the command.
+ pub fn set_timeout(&self, timeout: Duration) {
+ self.inner
+ .timeout_ms
+ .store(timeout.as_millis() as u64, Ordering::Relaxed);
+ }
+
+ pub fn is_leader(&self) -> bool {
+ self.view().state == RaftState::Leader
+ }
+
+ pub fn leader_id(&self) -> Option<String> {
+ self.view().leader_id
+ }
+
+ fn view(&self) -> ClusterView {
+ self.inner
+ .view
+ .read()
+ .unwrap_or_else(|e| e.into_inner())
+ .clone()
+ }
+
+ /// Propose `command` and wait until this node applies the committed entry.
+ ///
+ /// The command is wrapped in [`ClusterCommand::ClientMetadata`] so the
+ /// run loop can match the applied entry to this caller.
+ pub async fn propose(&self, command: ClusterCommand) -> Result<ApplyEffect, ProposeError> {
+ let request_id = self.inner.next.fetch_add(1, Ordering::Relaxed);
+ let wrapped = ClusterCommand::ClientMetadata {
+ origin: self.inner.node_id.clone(),
+ request_id,
+ inner: Box::new(command),
+ };
+ let timeout = Duration::from_millis(self.inner.timeout_ms.load(Ordering::Relaxed));
+ let (ack, rx) = oneshot::channel();
+ self.inner
+ .tx
+ .send(ProposalRequest {
+ request_id,
+ command: wrapped,
+ deadline: Instant::now() + timeout,
+ ack,
+ })
+ .map_err(|_| ProposeError::ShuttingDown)?;
+ // A little longer than the run loop's deadline so a `NoLeader` /
+ // `Timeout` it sends wins over this timer.
+ match tokio::time::timeout(timeout + Duration::from_millis(250), rx).await {
+ Ok(Ok(result)) => result,
+ Ok(Err(_)) => Err(ProposeError::ShuttingDown),
+ Err(_) => Err(ProposeError::Timeout),
+ }
+ }
+}
+
+/// Applies a committed metadata command to the local engine.
+///
+/// Called on the cluster run loop for every committed entry, on every member.
+/// Implementations must be deterministic: the same command against the same
+/// prior state does the same thing on every node. They must not propose a
+/// new command.
+#[async_trait::async_trait]
+pub trait MetadataApplier: Send + Sync {
+ async fn apply_metadata(&self, command: &ClusterCommand) -> ApplyEffect;
+}
diff --git a/engine/crates/xerj-cluster/src/raft.rs b/engine/crates/xerj-cluster/src/raft.rs
index f6fc8c321..fbbcb4220 100644
--- a/engine/crates/xerj-cluster/src/raft.rs
+++ b/engine/crates/xerj-cluster/src/raft.rs
@@ -72,6 +72,54 @@ pub enum ClusterCommand {
key: String,
value: String,
},
+ /// `PUT /{index}/_settings` body, applied by the engine on every member.
+ ///
+ /// `settings_json` is the request body as the accepting node validated it.
+ /// [`crate::metadata::ClusterMetadata`] records it; the engine applier is
+ /// what merges it into the live index.
+ UpdateIndexSettings {
+ index: String,
+ settings_json: String,
+ },
+ /// A client-originated metadata command.
+ ///
+ /// `request_id` is unique on the node that accepted the caller. That node
+ /// completes the waiting request when this entry is applied. Other nodes
+ /// apply `inner` and ignore the id. The wrapper is persisted with the
+ /// log entry so a restarted node (once the server opens the file log)
+ /// can still tell client entries from internal ones.
+ ClientMetadata {
+ /// Node that accepted the caller. Only that node completes a waiter.
+ origin: String,
+ request_id: u64,
+ inner: Box<ClusterCommand>,
+ },
+}
+
+impl ClusterCommand {
+ /// The originator's request id, if this entry was proposed for a waiting caller.
+ pub fn client_request_id(&self) -> Option<u64> {
+ match self {
+ ClusterCommand::ClientMetadata { request_id, .. } => Some(*request_id),
+ _ => None,
+ }
+ }
+
+ /// Node that accepted the waiting caller, if this entry is a client proposal.
+ pub fn client_origin(&self) -> Option<&str> {
+ match self {
+ ClusterCommand::ClientMetadata { origin, .. } => Some(origin.as_str()),
+ _ => None,
+ }
+ }
+
+ /// The metadata operation, unwrapping one [`ClusterCommand::ClientMetadata`] layer.
+ pub fn as_metadata(&self) -> &ClusterCommand {
+ match self {
+ ClusterCommand::ClientMetadata { inner, .. } => inner,
+ other => other,
+ }
+ }
}
/// A single entry in the Raft replicated log.
@@ -120,6 +168,38 @@ pub enum RaftMessage {
term: u64,
vote_granted: bool,
},
+ /// Follower → leader: please propose this metadata command.
+ ///
+ /// Not a Raft RPC. A follower that accepted an HTTP metadata write forwards
+ /// it here instead of appending to its own log. The leader proposes it
+ /// through the normal path; the follower completes its caller when the
+ /// committed entry is applied locally.
+ ForwardProposal {
+ term: u64,
+ from: String,
+ command: ClusterCommand,
+ },
+ /// Leader (or whoever received [`RaftMessage::ForwardProposal`]) → originator:
+ /// this node did not propose the command.
+ ForwardRejected {
+ term: u64,
+ from: String,
+ request_id: u64,
+ /// Who the rejector believes the leader is, if anyone.
+ leader_id: Option<String>,
+ reason: String,
+ },
+}
+
+fn message_term(msg: &RaftMessage) -> u64 {
+ match msg {
+ RaftMessage::AppendEntries { term, .. }
+ | RaftMessage::AppendEntriesResponse { term, .. }
+ | RaftMessage::RequestVote { term, .. }
+ | RaftMessage::RequestVoteResponse { term, .. }
+ | RaftMessage::ForwardProposal { term, .. }
+ | RaftMessage::ForwardRejected { term, .. } => *term,
+ }
}
/// An outbound message — the node ID of the recipient plus the payload.
@@ -286,12 +366,7 @@ impl RaftNode {
// All servers: If RPC request or response contains term T > currentTerm:
// set currentTerm = T, convert to follower (§5.1).
- let msg_term = match &msg {
- RaftMessage::AppendEntries { term, .. } => *term,
- RaftMessage::AppendEntriesResponse { term, .. } => *term,
- RaftMessage::RequestVote { term, .. } => *term,
- RaftMessage::RequestVoteResponse { term, .. } => *term,
- };
+ let msg_term = message_term(&msg);
if msg_term > self.current_term {
debug!(
node = %self.id,
@@ -350,6 +425,39 @@ impl RaftNode {
} => {
out.extend(self.handle_request_vote_response(from, term, vote_granted));
}
+ RaftMessage::ForwardProposal {
+ term,
+ from,
+ command,
+ } => {
+ if self.state == RaftState::Leader && term <= self.current_term {
+ match self.propose(command) {
+ Ok(_) => out.extend(self.force_heartbeat()),
+ Err(e) => {
+ warn!(node = %self.id, error = %e, "leader failed to propose forwarded command");
+ }
+ }
+ } else {
+ out.push(OutboundMessage {
+ to: from,
+ msg: RaftMessage::ForwardRejected {
+ term: self.current_term,
+ from: self.id.clone(),
+ request_id: command.client_request_id().unwrap_or(0),
+ leader_id: self.leader_id.clone(),
+ reason: format!(
+ "node {} is not the raft leader (state={}, leader={:?})",
+ self.id, self.state, self.leader_id
+ ),
+ },
+ });
+ }
+ }
+ RaftMessage::ForwardRejected { .. } => {
+ // Term bookkeeping already ran above. The cluster node matches
+ // the request id to a waiting caller; the state machine has
+ // nothing else to record.
+ }
}
out
}
@@ -471,8 +579,24 @@ impl RaftNode {
self.commit_index
}
+ /// Term of the log entry at `index`, if that entry is still present.
+ ///
+ /// A waiting proposer uses this to notice that its uncommitted entry was
+ /// truncated after a leadership change. `None` means the index is gone.
+ pub fn entry_term(&self, index: u64) -> Option<u64> {
+ self.log_entry(index).map(|e| e.term)
+ }
+
// ── Private helpers ──────────────────────────────────────────────────────
+ fn persist_commit_index(&mut self) {
+ if let Some(flog) = self.persistent.as_mut() {
+ if let Err(e) = flog.set_commit_index(self.commit_index) {
+ warn!(node = %self.id, error = %e, "failed to persist raft commit index");
+ }
+ }
+ }
+
fn cluster_size(&self) -> usize {
self.peers.len() + 1 // peers + self
}
@@ -703,6 +827,7 @@ impl RaftNode {
// min(leaderCommit, index of last new entry) (§5.3)
if leader_commit > self.commit_index {
self.commit_index = leader_commit.min(self.last_log_index());
+ self.persist_commit_index();
}
let match_index = self.last_log_index();
@@ -749,6 +874,7 @@ impl RaftNode {
// Find the highest N such that a majority of match_index[i] ≥ N
// and log[N].term == currentTerm (§5.4)
let last_idx = self.last_log_index();
+ let prev_commit = self.commit_index;
for n in (self.commit_index + 1..=last_idx).rev() {
if self.log_term_at(n) != self.current_term {
continue;
@@ -766,6 +892,14 @@ impl RaftNode {
break;
}
}
+ // Tell followers the new commit index now. Waiting for the next
+ // heartbeat tick leaves a committed metadata write invisible on
+ // the other members for a full tick after the caller was allowed
+ // to proceed.
+ if self.commit_index > prev_commit {
+ self.persist_commit_index();
+ return self.send_heartbeats();
+ }
} else {
// Decrement next_index and retry
let ni = self.next_index.entry(from.clone()).or_insert(1);
diff --git a/engine/crates/xerj-cluster/tests/metadata_proposal.rs b/engine/crates/xerj-cluster/tests/metadata_proposal.rs
new file mode 100644
index 000000000..8986489ce
--- /dev/null
+++ b/engine/crates/xerj-cluster/tests/metadata_proposal.rs
@@ -0,0 +1,181 @@
+//! Metadata commands proposed through a 3-node in-process ring commit on a
+//! majority and apply on every member, including after the leader is stopped.
+
+use std::sync::{Arc, Mutex};
+use std::time::Duration;
+
+use async_trait::async_trait;
+use tokio::sync::watch;
+use xerj_cluster::node::in_memory::{InMemoryBus, InMemoryTransport};
+use xerj_cluster::runner::ClusterRunner;
+use xerj_cluster::{
+ ApplyEffect, ClusterCommand, ClusterHandle, ClusterNode, MetadataApplier, ProposeError,
+};
+
+struct RecordingApplier {
+ events: Arc<Mutex<Vec<String>>>,
+}
+
+#[async_trait]
+impl MetadataApplier for RecordingApplier {
+ async fn apply_metadata(&self, command: &ClusterCommand) -> ApplyEffect {
+ let label = match command.as_metadata() {
+ ClusterCommand::CreateIndex { name, .. } => format!("create:{name}"),
+ ClusterCommand::DeleteIndex { name } => format!("delete:{name}"),
+ other => format!("other:{other:?}"),
+ };
+ self.events.lock().expect("events").push(label);
+ ApplyEffect::Applied
+ }
+}
+
+struct Node {
+ shutdown: watch::Sender<bool>,
+ handle: ClusterHandle,
+ events: Arc<Mutex<Vec<String>>>,
+}
+
+async fn boot() -> Vec<Node> {
+ let bus = InMemoryBus::new();
+ let ids = ["n1", "n2", "n3"];
+ let mut nodes = Vec::new();
+ for id in ids {
+ let peers: Vec<String> = ids
+ .iter()
+ .filter(|peer| **peer != id)
+ .map(|peer| (*peer).to_string())
+ .collect();
+ let transport = InMemoryTransport::new(id.to_string(), bus.clone()).await;
+ let mut node = ClusterNode::new(id.to_string(), peers, Box::new(transport));
+ let events = Arc::new(Mutex::new(Vec::new()));
+ node.set_applier(Arc::new(RecordingApplier {
+ events: Arc::clone(&events),
+ }));
+ let handle = node.handle();
+ let (shutdown, shutdown_rx) = watch::channel(false);
+ let mut runner = ClusterRunner::new(node, Duration::from_millis(5), shutdown_rx);
+ tokio::spawn(async move {
+ runner.run().await;
+ });
+ nodes.push(Node {
+ shutdown,
+ handle,
+ events,
+ });
+ }
+ nodes
+}
+
+async fn wait_leader(nodes: &[Node], dead: &[bool]) -> usize {
+ let deadline = std::time::Instant::now() + Duration::from_secs(10);
+ loop {
+ let leaders: Vec<usize> = nodes
+ .iter()
+ .enumerate()
+ .filter(|(i, node)| !dead[*i] && node.handle.is_leader())
+ .map(|(i, _)| i)
+ .collect();
+ if leaders.len() == 1 {
+ return leaders[0];
+ }
+ if std::time::Instant::now() > deadline {
+ panic!("timed out waiting for a single leader among the live nodes");
+ }
+ tokio::time::sleep(Duration::from_millis(20)).await;
+ }
+}
+
+async fn wait_event(node: &Node, label: &str) {
+ let deadline = std::time::Instant::now() + Duration::from_secs(5);
+ loop {
+ if node
+ .events
+ .lock()
+ .expect("events")
+ .iter()
+ .any(|event| event == label)
+ {
+ return;
+ }
+ if std::time::Instant::now() > deadline {
+ panic!(
+ "timed out waiting for {label}; saw {:?}",
+ node.events.lock().expect("events")
+ );
+ }
+ tokio::time::sleep(Duration::from_millis(20)).await;
+ }
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
+async fn metadata_replicates_and_survives_leader_loss() {
+ let nodes = boot().await;
+ let mut dead = [false, false, false];
+ let leader = wait_leader(&nodes, &dead).await;
+ let follower = (0..3).find(|i| *i != leader).expect("follower");
+
+ nodes[follower]
+ .handle
+ .propose(ClusterCommand::CreateIndex {
+ name: "orders".into(),
+ schema_json: r#"{"v":1,"kind":"index_create"}"#.into(),
+ })
+ .await
+ .expect("create commits");
+
+ for node in &nodes {
+ wait_event(node, "create:orders").await;
+ }
+
+ let _ = nodes[leader].shutdown.send(true);
+ dead[leader] = true;
+ let next = wait_leader(&nodes, &dead).await;
+ assert_ne!(
+ next, leader,
+ "the stopped node must not stay the only leader"
+ );
+
+ let proposer = (0..3).find(|i| !dead[*i] && *i != next).unwrap_or(next);
+ nodes[proposer]
+ .handle
+ .propose(ClusterCommand::DeleteIndex {
+ name: "orders".into(),
+ })
+ .await
+ .expect("delete commits after the leader change");
+
+ for (i, node) in nodes.iter().enumerate() {
+ if dead[i] {
+ continue;
+ }
+ wait_event(node, "delete:orders").await;
+ }
+
+ for node in &nodes {
+ let _ = node.shutdown.send(true);
+ }
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn propose_without_a_running_node_is_not_acknowledged() {
+ let bus = InMemoryBus::new();
+ let transport = InMemoryTransport::new("solo".into(), bus).await;
+ let node = ClusterNode::new("solo".into(), vec!["other".into()], Box::new(transport));
+ let handle = node.handle();
+ handle.set_timeout(Duration::from_millis(200));
+ // Keep the node so the proposal channel stays open. Nothing steps it,
+ // so the command is never appended.
+ let _keep = node;
+
+ let err = handle
+ .propose(ClusterCommand::CreateIndex {
+ name: "orders".into(),
+ schema_json: "{}".into(),
+ })
+ .await
+ .expect_err("a write with no running raft loop must not succeed");
+ assert!(
+ matches!(err, ProposeError::Timeout | ProposeError::NoLeader),
+ "{err:?}"
+ );
+}
diff --git a/engine/crates/xerj-common/src/error.rs b/engine/crates/xerj-common/src/error.rs
index 6f6931653..05c997994 100644
--- a/engine/crates/xerj-common/src/error.rs
+++ b/engine/crates/xerj-common/src/error.rs
@@ -150,6 +150,14 @@ pub enum XerjError {
#[error("index [{index}] is blocked for {block_type} operations")]
IndexBlocked { index: String, block_type: String },
+ // ── Cluster metadata ──────────────────────────────────────────────────
+ /// A cluster metadata write was not acknowledged.
+ ///
+ /// The change was not applied on this node. The reason is the proposal
+ /// error (no elected leader, timeout, shutdown, leadership lost).
+ #[error("{reason}")]
+ ClusterUnavailable { reason: String },
+
// ── Catch-all ─────────────────────────────────────────────────────────
/// An unexpected internal error. Indicates a bug.
#[error("internal error: {reason}")]
@@ -278,6 +286,12 @@ impl XerjError {
}
}
+ pub fn cluster_unavailable(reason: impl Into<String>) -> Self {
+ Self::ClusterUnavailable {
+ reason: reason.into(),
+ }
+ }
+
pub fn version_conflict(id: impl Into<String>, expected: u64, actual: u64) -> Self {
Self::VersionConflict {
id: id.into(),
@@ -317,7 +331,9 @@ impl XerjError {
Self::IndexBlocked { .. } => 403,
// The index exists but has no serving copy — ES answers the same
// shape (no_shard_available_action_exception) with 503.
- Self::IndexUnavailable { .. } | Self::ClusterStateUnavailable => 503,
+ Self::IndexUnavailable { .. }
+ | Self::ClusterStateUnavailable
+ | Self::ClusterUnavailable { .. } => 503,
Self::AuthError { .. } => 401,
Self::ResourceExhausted { .. } | Self::CircuitBreaking { .. } => 429,
Self::StorageError { .. }
@@ -338,6 +354,7 @@ impl XerjError {
| Self::StorageError { .. }
| Self::WalError { .. }
| Self::EmbeddingError { .. }
+ | Self::ClusterUnavailable { .. }
)
}
diff --git a/engine/crates/xerj-engine/Cargo.toml b/engine/crates/xerj-engine/Cargo.toml
index 93261f4e8..0bb9532a8 100644
--- a/engine/crates/xerj-engine/Cargo.toml
+++ b/engine/crates/xerj-engine/Cargo.toml
@@ -19,6 +19,8 @@ xerj-wasm = { path = "../xerj-wasm" }
xerj-cluster = { path = "../xerj-cluster" }
tokio = { workspace = true }
+# `MetadataApplier` is an async-trait. Already in the tree via xerj-cluster.
+async-trait = "0.1"
# `join_all` for the hybrid legs (#939). Already in the lockfile via the
# workspace's `futures`; no new crate is pulled in.
futures-util = "0.3"
diff --git a/engine/crates/xerj-engine/src/cluster_apply.rs b/engine/crates/xerj-engine/src/cluster_apply.rs
new file mode 100644
index 000000000..1a5875c28
--- /dev/null
+++ b/engine/crates/xerj-engine/src/cluster_apply.rs
@@ -0,0 +1,393 @@
+//! Apply committed cluster-metadata commands to the local engine.
+//!
+//! The HTTP layer proposes. This module runs on the Raft task, after the
+//! entry has committed, and materialises the same index create, delete,
+//! mapping, or settings change on every member. It must not propose.
+
+use std::sync::Arc;
+
+use serde::Deserialize;
+use serde_json::Value;
+use xerj_cluster::{ApplyEffect, ClusterCommand, MetadataApplier, ProposeError};
+use xerj_common::types::{FieldConfig, Schema};
+
+use crate::{Engine, EngineError, Result};
+
+/// What the accepting node resolved for `PUT /{index}` before proposing.
+///
+/// Templates are evaluated there. Peers create this schema and do not run
+/// their own templates, so two nodes with different local templates still
+/// converge.
+#[derive(Debug, Clone)]
+pub struct ReplicatedIndexSpec {
+ pub name: String,
+ pub schema: Schema,
+ pub settings: Value,
+ pub mappings: Value,
+ pub index_settings: Value,
+ pub aliases: Value,
+}
+
+/// Engine applier installed on the cluster node before the runner starts.
+pub struct EngineMetadataApplier {
+ pub engine: Arc<Engine>,
+}
+
+#[async_trait::async_trait]
+impl MetadataApplier for EngineMetadataApplier {
+ async fn apply_metadata(&self, command: &ClusterCommand) -> ApplyEffect {
+ apply_committed(&self.engine, command.as_metadata()).await
+ }
+}
+
+pub(crate) fn propose_error(err: ProposeError) -> EngineError {
+ EngineError::Common(xerj_common::XerjError::cluster_unavailable(err.to_string()))
+}
+
+pub(crate) fn effect_result(index: &str, effect: ApplyEffect) -> Result<()> {
+ match effect {
+ ApplyEffect::Applied => Ok(()),
+ ApplyEffect::AlreadyExists => Err(EngineError::Common(
+ xerj_common::XerjError::index_already_exists(index),
+ )),
+ ApplyEffect::NotFound => Err(EngineError::Common(
+ xerj_common::XerjError::index_not_found(index),
+ )),
+ ApplyEffect::Failed(reason) => Err(EngineError::Common(xerj_common::XerjError::internal(
+ reason,
+ ))),
+ }
+}
+
+fn is_already_exists(err: &EngineError) -> bool {
+ matches!(
+ err,
+ EngineError::Common(xerj_common::XerjError::IndexAlreadyExists { .. })
+ )
+}
+
+fn is_not_found(err: &EngineError) -> bool {
+ matches!(
+ err,
+ EngineError::Common(xerj_common::XerjError::IndexNotFound { .. })
+ )
+}
+
+impl Engine {
+ /// Propose an index create and wait until this node has applied it.
+ ///
+ /// The caller's template is applied once, here. Success means the index
+ /// exists on this node and a majority has the log entry. No leader or a
+ /// timeout leaves the index uncreated.
+ pub async fn replicate_index_create(&self, mut spec: ReplicatedIndexSpec) -> Result<()> {
+ self.ensure_cluster_state_writable()?;
+ if !crate::index_guard::visible(&spec.name) {
+ return Err(EngineError::Common(
+ xerj_common::XerjError::index_not_found(&spec.name),
+ ));
+ }
+ let Some(handle) = self.cluster_handle() else {
+ return Err(EngineError::Common(xerj_common::XerjError::internal(
+ "replicate_index_create called with no cluster handle",
+ )));
+ };
+ spec.schema = self.apply_index_template(&spec.name, spec.schema);
+ let envelope = serde_json::json!({
+ "v": 1,
+ "kind": "index_create",
+ "schema": spec.schema,
+ "settings": spec.settings,
+ "mappings": spec.mappings,
+ "index_settings": spec.index_settings,
+ "aliases": spec.aliases,
+ });
+ let schema_json = serde_json::to_string(&envelope)?;
+ let effect = handle
+ .propose(ClusterCommand::CreateIndex {
+ name: spec.name.clone(),
+ schema_json,
+ })
+ .await
+ .map_err(propose_error)?;
+ effect_result(&spec.name, effect)
+ }
+
+ /// Propose one index's merged mapping. Callers validate before this.
+ pub async fn replicate_mapping(
+ &self,
+ index: &str,
+ mapping: &Value,
+ additions: &[FieldConfig],
+ ) -> Result<()> {
+ self.ensure_cluster_state_writable()?;
+ if !crate::index_guard::visible(index) {
+ return Err(EngineError::Common(
+ xerj_common::XerjError::index_not_found(index),
+ ));
+ }
+ let Some(handle) = self.cluster_handle() else {
+ return Err(EngineError::Common(xerj_common::XerjError::internal(
+ "replicate_mapping called with no cluster handle",
+ )));
+ };
+ let envelope = serde_json::json!({
+ "v": 1,
+ "mapping": mapping,
+ "add_fields": additions,
+ });
+ let effect = handle
+ .propose(ClusterCommand::UpdateMapping {
+ index: index.to_string(),
+ mapping_json: serde_json::to_string(&envelope)?,
+ })
+ .await
+ .map_err(propose_error)?;
+ effect_result(index, effect)
+ }
+
+ /// Propose a `PUT /{index}/_settings` body. Analysis updates are refused
+ /// by the handler before this is called. Lifecycle attach stays on the
+ /// accepting node; it is not part of the command.
+ pub async fn replicate_index_settings(&self, index: &str, body: &Value) -> Result<()> {
+ self.ensure_cluster_state_writable()?;
+ if !crate::index_guard::visible(index) {
+ return Err(EngineError::Common(
+ xerj_common::XerjError::index_not_found(index),
+ ));
+ }
+ let Some(handle) = self.cluster_handle() else {
+ return Err(EngineError::Common(xerj_common::XerjError::internal(
+ "replicate_index_settings called with no cluster handle",
+ )));
+ };
+ let effect = handle
+ .propose(ClusterCommand::UpdateIndexSettings {
+ index: index.to_string(),
+ settings_json: serde_json::to_string(body)?,
+ })
+ .await
+ .map_err(propose_error)?;
+ effect_result(index, effect)
+ }
+
+ /// Merge a settings body into the display map, then apply block keys.
+ ///
+ /// Same order as `PUT /{index}/_settings` on a single node: the display
+ /// copy updates, `Index::apply_block_settings` updates enforcement, and
+ /// the block map is copied back so `GET /_settings` matches.
+ pub(crate) async fn apply_user_index_settings(&self, name: &str, body: &Value) -> Result<()> {
+ if self.get_index(name).is_err() {
+ return Err(EngineError::Common(
+ xerj_common::XerjError::index_not_found(name),
+ ));
+ }
+ let inner = body.get("index").cloned().unwrap_or_else(|| body.clone());
+ let mut existing = self
+ .index_settings
+ .get(name)
+ .map(|v| v.clone())
+ .unwrap_or_else(|| serde_json::json!({ "index": {} }));
+ let inner_slot = existing
+ .as_object_mut()
+ .and_then(|m| m.get_mut("index").and_then(Value::as_object_mut));
+ match (inner_slot, inner.as_object()) {
+ (Some(existing_inner), Some(new_inner)) => {
+ for (k, v) in new_inner {
+ existing_inner.insert(k.clone(), v.clone());
+ }
+ self.index_settings.insert(name.to_string(), existing);
+ }
+ _ => {
+ self.index_settings
+ .insert(name.to_string(), serde_json::json!({ "index": inner }));
+ }
+ }
+ let idx = self.get_index(name)?;
+ idx.apply_block_settings(body).await?;
+ self.sync_display_blocks(name).await;
+ Ok(())
+ }
+
+ /// Copy `index.blocks` from the index's settings file into the display map.
+ pub(crate) async fn sync_display_blocks(&self, name: &str) {
+ let Ok(idx) = self.get_index(name) else {
+ return;
+ };
+ let blocks = idx
+ .get_settings()
+ .await
+ .pointer("/index/blocks")
+ .cloned()
+ .filter(|b| b.as_object().is_some_and(|m| !m.is_empty()));
+ let mut display = self
+ .index_settings
+ .get(name)
+ .map(|v| v.clone())
+ .unwrap_or_else(|| serde_json::json!({ "index": {} }));
+ if !display.is_object() {
+ display = serde_json::json!({ "index": {} });
+ }
+ let Some(root) = display.as_object_mut() else {
+ return;
+ };
+ if !root.get("index").is_some_and(Value::is_object) {
+ root.insert("index".to_string(), serde_json::json!({}));
+ }
+ let Some(inner) = root.get_mut("index").and_then(Value::as_object_mut) else {
+ return;
+ };
+ inner.retain(|k, _| !k.starts_with("blocks."));
+ match blocks {
+ Some(b) => {
+ inner.insert("blocks".to_string(), b);
+ }
+ None => {
+ inner.remove("blocks");
+ }
+ }
+ self.index_settings.insert(name.to_string(), display);
+ }
+}
+
+#[derive(Deserialize)]
+struct CreateEnvelope {
+ v: u32,
+ kind: String,
+ schema: Schema,
+ settings: Value,
+ mappings: Value,
+ index_settings: Value,
+ aliases: Value,
+}
+
+#[derive(Deserialize)]
+struct MappingEnvelope {
+ v: u32,
+ mapping: Value,
+ add_fields: Vec<FieldConfig>,
+}
+
+async fn apply_committed(engine: &Engine, command: &ClusterCommand) -> ApplyEffect {
+ match command {
+ ClusterCommand::CreateIndex { name, schema_json } => {
+ apply_create(engine, name, schema_json).await
+ }
+ ClusterCommand::DeleteIndex { name } => apply_delete(engine, name).await,
+ ClusterCommand::UpdateMapping {
+ index,
+ mapping_json,
+ } => apply_mapping(engine, index, mapping_json).await,
+ ClusterCommand::UpdateIndexSettings {
+ index,
+ settings_json,
+ } => apply_settings(engine, index, settings_json).await,
+ // Shard assignment, membership, and generic config are stored by
+ // ClusterMetadata. They do not change this node's indices.
+ _ => ApplyEffect::Applied,
+ }
+}
+
+async fn apply_create(engine: &Engine, name: &str, schema_json: &str) -> ApplyEffect {
+ let envelope: CreateEnvelope = match serde_json::from_str(schema_json) {
+ Ok(envelope) => envelope,
+ Err(e) => {
+ return ApplyEffect::Failed(format!(
+ "index create for [{name}] is not a replicated spec: {e}"
+ ))
+ }
+ };
+ if envelope.v != 1 || envelope.kind != "index_create" {
+ return ApplyEffect::Failed(format!(
+ "index create for [{name}] has unsupported envelope v={} kind={}",
+ envelope.v, envelope.kind
+ ));
+ }
+ if engine.get_index(name).is_ok() {
+ return ApplyEffect::AlreadyExists;
+ }
+ if let Err(e) = engine.create_index_exact(name, envelope.schema, envelope.settings) {
+ return if is_already_exists(&e) {
+ ApplyEffect::AlreadyExists
+ } else {
+ ApplyEffect::Failed(e.to_string())
+ };
+ }
+ if !envelope.mappings.is_null() {
+ if let Err(e) = engine.put_index_mapping(name, envelope.mappings) {
+ return ApplyEffect::Failed(e.to_string());
+ }
+ }
+ if !envelope.index_settings.is_null() {
+ engine
+ .index_settings
+ .insert(name.to_string(), envelope.index_settings);
+ }
+ if let Some(aliases) = envelope.aliases.as_object() {
+ for alias in aliases.keys() {
+ if let Err(e) = engine.add_alias(alias, name) {
+ return ApplyEffect::Failed(e.to_string());
+ }
+ }
+ engine
+ .index_alias_metadata
+ .insert(name.to_string(), envelope.aliases);
+ engine.flush_alias_metadata();
+ }
+ ApplyEffect::Applied
+}
+
+async fn apply_delete(engine: &Engine, name: &str) -> ApplyEffect {
+ match engine.delete_index_local(name).await {
+ Ok(()) => ApplyEffect::Applied,
+ Err(e) if is_not_found(&e) => ApplyEffect::NotFound,
+ Err(e) => ApplyEffect::Failed(e.to_string()),
+ }
+}
+
+async fn apply_mapping(engine: &Engine, index: &str, mapping_json: &str) -> ApplyEffect {
+ let envelope: MappingEnvelope = match serde_json::from_str(mapping_json) {
+ Ok(envelope) => envelope,
+ Err(e) => {
+ return ApplyEffect::Failed(format!(
+ "mapping update for [{index}] is not a replicated spec: {e}"
+ ))
+ }
+ };
+ if envelope.v != 1 {
+ return ApplyEffect::Failed(format!(
+ "mapping update for [{index}] has unsupported envelope v={}",
+ envelope.v
+ ));
+ }
+ let idx = match engine.get_index(index) {
+ Ok(idx) => idx,
+ Err(_) => return ApplyEffect::NotFound,
+ };
+ let current = idx.schema().await;
+ let additions: Vec<FieldConfig> = envelope
+ .add_fields
+ .into_iter()
+ .filter(|field| current.field(&field.name).is_none())
+ .collect();
+ if let Err(e) = idx.add_fields(additions).await {
+ return ApplyEffect::Failed(e.to_string());
+ }
+ match engine.put_index_mapping(index, envelope.mapping) {
+ Ok(()) => ApplyEffect::Applied,
+ Err(e) => ApplyEffect::Failed(e.to_string()),
+ }
+}
+
+async fn apply_settings(engine: &Engine, index: &str, settings_json: &str) -> ApplyEffect {
+ let body: Value = match serde_json::from_str(settings_json) {
+ Ok(body) => body,
+ Err(e) => {
+ return ApplyEffect::Failed(format!("settings update for [{index}] is not JSON: {e}"))
+ }
+ };
+ match engine.apply_user_index_settings(index, &body).await {
+ Ok(()) => ApplyEffect::Applied,
+ Err(e) if is_not_found(&e) => ApplyEffect::NotFound,
+ Err(e) => ApplyEffect::Failed(e.to_string()),
+ }
+}
diff --git a/engine/crates/xerj-engine/src/engine.rs b/engine/crates/xerj-engine/src/engine.rs
index 27883c2bd..80b4bd5c5 100644
--- a/engine/crates/xerj-engine/src/engine.rs
+++ b/engine/crates/xerj-engine/src/engine.rs
@@ -864,6 +864,11 @@ pub struct Engine {
/// `redb/src/error.rs:65`; Apache-2.0/MIT, shape adapted, no code
/// copied).
cluster_state_boot: Arc<ClusterStateBootStatus>,
+ /// Raft proposer for cluster metadata. `None` unless
+ /// [`Engine::attach_cluster`] ran, which the server does only after the
+ /// cluster transport is up. Single-node boots leave this empty and every
+ /// index create/delete/mapping/settings path stays local.
+ cluster: Arc<parking_lot::Mutex<Option<xerj_cluster::ClusterHandle>>>,
}
impl Engine {
@@ -1016,6 +1021,7 @@ impl Engine {
_node_lock: node_lock,
cluster_state_write: Arc::new(parking_lot::Mutex::new(())),
cluster_state_boot: Arc::new(cluster_state_status),
+ cluster: Arc::new(parking_lot::Mutex::new(None)),
};
// A rejected cluster-state document means this binary cannot know the
@@ -1396,7 +1402,11 @@ impl Engine {
/// into `schema` (adding any the schema does not already define). Applied
/// on every create path so template behavior is independent of whether the
/// caller supplied settings.
- fn apply_index_template(&self, name: &str, schema: Schema) -> Schema {
+ ///
+ /// The cluster proposer calls this once and replicates the effective
+ /// schema. The applier must not call it again: nodes can have different
+ /// local templates, and a second pass would diverge.
+ pub(crate) fn apply_index_template(&self, name: &str, schema: Schema) -> Schema {
let mut effective_schema = schema;
if let Some(tmpl) = self.best_matching_template(name) {
if let Some(props) = tmpl.mappings.get("properties") {
@@ -1449,7 +1459,9 @@ impl Engine {
// must reject here so HTTP, gRPC, direct CLI, and Console callers share
// the same fail-closed boundary.
self.ensure_cluster_state_writable()?;
- let index_name = IndexName::new(name).map_err(EngineError::Common)?;
+ // Same rejection order as before the exact-create split: a bad name
+ // fails before the visibility and already-exists checks.
+ let _ = IndexName::new(name).map_err(EngineError::Common)?;
// Per-index authorization backstop (issue #79): a caller that may not
// see this name must not be able to bring it into existence either —
@@ -1482,9 +1494,39 @@ impl Engine {
// Apply matching template (highest priority wins) on every create path.
let effective_schema = self.apply_index_template(name, schema);
+ self.create_index_exact(name, effective_schema, settings)
+ }
+
+ /// Create `name` with `schema` exactly as given.
+ ///
+ /// Does not apply index templates and does not consult the per-request
+ /// visibility guard. The single-node create path applies the template and
+ /// checks visibility first. The cluster applier calls this so every member
+ /// materialises the schema the accepting node already resolved.
+ pub(crate) fn create_index_exact(
+ &self,
+ name: &str,
+ schema: Schema,
+ settings: serde_json::Value,
+ ) -> Result<()> {
+ self.ensure_cluster_state_writable()?;
+ let index_name = IndexName::new(name).map_err(EngineError::Common)?;
+
+ if self.indices.contains_key(name) {
+ return Err(EngineError::Common(
+ xerj_common::XerjError::index_already_exists(name),
+ ));
+ }
+
+ if let Some(f) = self.failed_indices.get(name) {
+ return Err(EngineError::Common(
+ xerj_common::XerjError::index_unavailable(name, f.reason.clone()),
+ ));
+ }
+
let idx = Index::create_with_settings(
index_name,
- effective_schema,
+ schema,
settings,
&self.config,
&self.data_dir,
@@ -2440,6 +2482,24 @@ impl Engine {
/// sitting in the document on disk — the same lie in a different shape.
/// A typed 503 is the only honest answer while the node cannot see the
/// operator's configuration.
+ /// Install the Raft handle the server built for this process.
+ ///
+ /// Called once, after console bootstrap, so system indices created during
+ /// boot stay node-local. Later calls replace the handle.
+ pub fn attach_cluster(&self, handle: xerj_cluster::ClusterHandle) {
+ *self.cluster.lock() = Some(handle);
+ }
+
+ /// `true` when metadata writes must go through the Raft log.
+ pub fn has_cluster(&self) -> bool {
+ self.cluster.lock().is_some()
+ }
+
+ /// Clone of the proposer. The lock is not held across the subsequent await.
+ pub fn cluster_handle(&self) -> Option<xerj_cluster::ClusterHandle> {
+ self.cluster.lock().clone()
+ }
+
pub fn ensure_cluster_state_writable(&self) -> Result<()> {
if self.cluster_state_boot.is_writable() {
return Ok(());
@@ -3094,6 +3154,33 @@ impl Engine {
xerj_common::XerjError::index_not_found(name),
));
}
+ if let Some(handle) = self.cluster_handle() {
+ // A name this node does not have is a local 404. Proposing it
+ // would delete the index on peers and, on the ES surface, come
+ // back as acknowledged once the not-found is swallowed.
+ if !self.indices.contains_key(name) && !self.failed_indices.contains_key(name) {
+ return Err(EngineError::Common(
+ xerj_common::XerjError::index_not_found(name),
+ ));
+ }
+ let effect = handle
+ .propose(xerj_cluster::ClusterCommand::DeleteIndex {
+ name: name.to_string(),
+ })
+ .await
+ .map_err(crate::cluster_apply::propose_error)?;
+ return crate::cluster_apply::effect_result(name, effect);
+ }
+ self.delete_index_local(name).await
+ }
+
+ /// Remove `name` from this node's storage.
+ ///
+ /// Does not propose. The cluster applier calls this after the delete has
+ /// committed. The single-node [`Engine::delete_index`] falls through here
+ /// after the visibility check.
+ pub(crate) async fn delete_index_local(&self, name: &str) -> Result<()> {
+ self.ensure_cluster_state_writable()?;
match self.indices.remove(name).map(|(_, v)| v) {
Some(idx) => {
// The handle is pulled out of the map first so no write can
diff --git a/engine/crates/xerj-engine/src/lib.rs b/engine/crates/xerj-engine/src/lib.rs
index 1115db6e4..14507a36d 100644
--- a/engine/crates/xerj-engine/src/lib.rs
+++ b/engine/crates/xerj-engine/src/lib.rs
@@ -19,6 +19,7 @@
pub mod aggs;
pub mod audit;
pub mod bulk;
+pub mod cluster_apply;
mod cluster_state;
mod code_blocks;
mod collection_publication;
@@ -47,6 +48,7 @@ mod bounded_ingest_resources_tests;
// ── Re-exports ────────────────────────────────────────────────────────────────
+pub use cluster_apply::{EngineMetadataApplier, ReplicatedIndexSpec};
pub use cluster_state::{ClusterStateBlockKind, ClusterStateBootStatus};
pub use engine::{EmbeddingExecutionIdentity, Engine, HealthStatus, IndexInfo};
pub use index::{
diff --git a/engine/crates/xerj-server/src/main.rs b/engine/crates/xerj-server/src/main.rs
index f947aabcf..2bb8078cd 100644
--- a/engine/crates/xerj-server/src/main.rs
+++ b/engine/crates/xerj-server/src/main.rs
@@ -3870,6 +3870,14 @@ async fn async_main() -> Result<()> {
} else {
xerj_api::state::ClusterTransportStatus::Disabled
};
+ // The runner starts only after AppState exists, so the engine applier
+ // and the propose handle are installed first. Console bootstrap (below)
+ // still runs against the unattached engine: system indices stay local.
+ let mut pending_cluster: Option<(
+ ClusterNode,
+ std::time::Duration,
+ tokio::sync::watch::Receiver<bool>,
+ )> = None;
let _cluster_shutdown = if cfg.cluster.enabled && storage_available {
// Fail closed (issue #75). The cluster port carries Raft control
// messages; without a shared secret every frame on it is
@@ -3950,11 +3958,8 @@ async fn async_main() -> Result<()> {
.collect();
let node = ClusterNode::new(node_id.clone(), peer_ids, Box::new(transport));
- let mut runner = ClusterRunner::new(node, tick, shutdown_rx);
-
- info!(node = %node_id, "Starting cluster runner (Raft mode)");
cluster_transport_outcome = xerj_api::state::ClusterTransportStatus::Running;
- tokio::spawn(async move { runner.run().await });
+ pending_cluster = Some((node, tick, shutdown_rx));
}
Err(e) => {
error!("Failed to start cluster TCP transport: {e:#}");
@@ -4160,6 +4165,20 @@ async fn async_main() -> Result<()> {
// what lets /_cluster/health say yellow instead of green forever.
state.cluster_transport = cluster_transport_outcome;
+ // Metadata writes propose through this handle. The runner applies
+ // committed entries onto `state.engine` before any router is built, so
+ // the first request already sees a node that can elect and replicate.
+ if let Some((mut node, tick, shutdown_rx)) = pending_cluster {
+ node.set_applier(std::sync::Arc::new(xerj_engine::EngineMetadataApplier {
+ engine: std::sync::Arc::clone(&state.engine),
+ }));
+ let node_id = node.raft.id.clone();
+ state.engine.attach_cluster(node.handle());
+ let mut runner = ClusterRunner::new(node, tick, shutdown_rx);
+ info!(node = %node_id, "Starting cluster runner (Raft mode)");
+ tokio::spawn(async move { runner.run().await });
+ }
+
// 9b-iia. Decide-ladder override (#1057): `--decide-mode` /
// `--decide-model-dir` flags or XERJ_DECIDE_MODE / XERJ_DECIDE_MODEL_DIR
// env (flags win; the flag can fill either half, so `--decide-mode local`
diff --git a/landing/docs/clustering.html b/landing/docs/clustering.html
index 36424c780..d1ff0cb6d 100644
--- a/landing/docs/clustering.html
+++ b/landing/docs/clustering.html
@@ -234,23 +234,24 @@
<h1>Clustering</h1>
- <p>XERJ ships an <strong>embedded Raft implementation</strong> — no etcd, no ZooKeeper, no Consul. One binary is still one binary; cluster mode is a config switch, not a separate process. <strong>Where the ring stands today, plainly:</strong> leader election, heartbeats, and the authenticated inter-node transport are real and tested; nothing in the production write path rides on the log yet. Indices, mappings, and documents are node-local — a document indexed on one node is not visible on another until the replication work lands (public tracking: <a href="https://github.com/xerj-org/xerj/issues/1170">xerj-org/xerj#1170</a>). If you need a second live copy of your data today, use the <a href="/docs/config#wal_tap">WAL tap</a> or per-node backups.</p>
+ <p>XERJ ships an <strong>embedded Raft implementation</strong> — no etcd, no ZooKeeper, no Consul. One binary is still one binary; cluster mode is a config switch, not a separate process. <strong>Where the ring stands today, plainly:</strong> leader election, heartbeats, and the authenticated inter-node transport are real, and <strong>index metadata</strong> (create, delete, mapping, and settings) is proposed through the Raft log and applied on every member. Documents are still node-local. A document indexed on one node is not visible on another (public tracking for the rest of the wiring: <a href="https://github.com/xerj-org/xerj/issues/1170">xerj-org/xerj#1170</a>). If you need a second live copy of your data today, use the <a href="/docs/config#wal_tap">WAL tap</a> or per-node backups.</p>
<h2>What cluster mode does not give you yet</h2>
<div class="enum-list">
- <div class="item">No metadata replication<span class="dsc">An index created on node A does not exist on node B — no production path proposes a ClusterCommand yet.</span></div>
- <div class="item">No document replication<span class="dsc">PUT a document on node A, GET it from node B: <code class="inline">404 index_not_found</code>. Measured on a 3-node ring.</span></div>
- <div class="item">No shard routing<span class="dsc">No forwarding to a primary, no replica reads. Each node's API serves that node's storage only.</span></div>
+ <div class="item">No document replication<span class="dsc">The index exists on every member after create. The documents do not. <code class="inline">PUT n1/idx/_doc/1</code> then <code class="inline">GET n2/idx/_doc/1</code> is <code class="inline">404</code>. <code class="inline">GET n2/idx</code> is <code class="inline">200</code>.</span></div>
+ <div class="item">No shard routing<span class="dsc">No forwarding to a primary, no replica reads. Each node's API serves that node's storage only. <code class="inline">Engine::route_write</code> is unchanged.</span></div>
+ <div class="item">No snapshot catch-up<span class="dsc">A node that missed a metadata entry does not install it later. The server's Raft log is in memory; restart keeps indices already applied in that node's data directory and does not replay entries it never saw.</span></div>
<div class="item">No node-lifecycle APIs<span class="dsc">There is no drain / activate / remove endpoint and no region split-merge. Earlier revisions of this page documented them; the binary has never shipped them.</span></div>
</div>
- <p>Treat a clustered node's data durability exactly like a single node's.</p>
+ <p>Treat a clustered node's <em>document</em> durability exactly like a single node's. Index metadata is the exception, and only while a leader can commit.</p>
<h2>What you get</h2>
<ul>
<li><strong>Leader election</strong> — exactly one leader per term, randomized 150–300 ms election timeouts, automatic failover. Measured on a 3-node loopback ring (tick 50 ms): a new leader within ~250 ms of killing the leader, then election-stable for minutes. Leader death used to cause perpetual election churn — fixed by <a href="https://github.com/xerj-org/xerj/issues/1168">#1168</a>.</li>
<li><strong>Authenticated transport</strong> — every inter-node connection is HMAC-SHA256 over a per-connection challenge using the cluster secret. There is no unauthenticated mode; the node refuses to start without a secret.</li>
<li><strong>Dead-peer isolation</strong> — a hung or dead peer costs one <code class="inline">WARN</code> on the transition and a retry ladder backing off 100 ms → 6.4 s; it never delays messages to live peers.</li>
- <li><strong>Durable local Raft log</strong> — each node persists its term, votes, and committed entries, and replays them on restart.</li>
+ <li><strong>Index metadata replication</strong> — <code class="inline">PUT /{index}</code>, <code class="inline">DELETE /{index}</code>, <code class="inline">PUT /{index}/_mapping</code>, and <code class="inline">PUT /{index}/_settings</code> are proposed on the member that received them. A follower forwards to the leader. The HTTP call succeeds only after the entry commits on a majority and this node has applied it. No elected leader, or a commit that does not land in time, is <code class="inline">503</code> and the change is not applied on the caller. With <code class="inline">[cluster] enabled = false</code> these calls stay local, as before.</li>
+ <li><strong>In-memory Raft log in the server</strong> — the on-disk log type exists and is tested, and the server does not open it. Applied indices live in each node's data directory, so a restart still has the indices it already applied. It does not catch up on a metadata change it missed.</li>
</ul>
<h2>Bootstrapping a 3-node cluster</h2>
@@ -305,7 +306,7 @@ tick_ms = 50</pre>
$ xerj --config /etc/xerj/node-b.toml # on 10.0.0.12
$ xerj --config /etc/xerj/node-c.toml # on 10.0.0.13</pre>
- <p>Within a second the ring elects a leader. The honest place to watch it today is the log — the health endpoints report the node answering the request, not the ring's membership (that plumbing is part of the <a href="https://github.com/xerj-org/xerj/issues/1170">wiring work</a>):</p>
+ <p>Within a second the ring elects a leader. The log is still the place to see who won. <code class="inline">GET /_cluster/health</code> reports the <em>configured</em> membership (this node's address plus distinct peers), not a liveness probe and not a count of nodes holding your documents:</p>
<pre class="code">$ journalctl -u xerj -f | grep xerj_cluster
INFO xerj_cluster::transport: TCP transport listening (authenticated) node=10.0.0.11:9300
INFO xerj_cluster::raft: Starting election node=10.0.0.12:9300 term=1
@@ -313,22 +314,29 @@ INFO xerj_cluster::raft: Became leader node=10.0.0.12:9300 term=1</pre>
<pre class="code">$ curl -sH "Authorization: ApiKey $XERJ_API_KEY" \
http://10.0.0.11:9200/_cluster/health | jq '.status, .number_of_nodes'
"green"
-1</pre>
+3</pre>
- <h2>Indexes and data are node-local today</h2>
- <p>Index creation, mappings, and documents do not cross the wire — every node's API serves that node's <a href="/docs/storage">storage</a> only. There is no shard assignment to other members, no replica set, and no drain/region API; the sections that used to document them were removed because the binary has never shipped them.</p>
- <p>For a second live copy of your data, point the <a href="/docs/config#wal_tap">WAL tap</a> at another XERJ or any ES-compatible target: it ships indexed writes near-real-time from the WAL, one-directional, without touching the write path.</p>
+ <h2>What replicates, and what stays on the node that wrote it</h2>
+ <p><code class="inline">PUT /orders</code> on any member — leader or follower — creates <code class="inline">orders</code> on every member, including its mapping and the settings and aliases in that request. <code class="inline">GET /orders</code> and <code class="inline">GET /orders/_mapping</code> on another member succeed after the call returns. <code class="inline">DELETE /orders</code> removes it on every member. A settings or mapping change is one log entry per concrete index. A follower that cannot reach a leader returns <code class="inline">503</code>; it does not create the index locally and report success.</p>
+ <p>Several things around that create stay on the accepting node, on purpose for this step:</p>
+ <ul>
+ <li><strong>Documents</strong>, including an index that a document write auto-creates, are local. The replicated create is the explicit index API.</li>
+ <li><strong>Index templates</strong> are not replicated. The accepting node applies its templates once and the effective schema is what the log carries.</li>
+ <li><strong>ILM/ISM attach</strong> from <code class="inline">index.lifecycle.name</code> runs on the accepting node only.</li>
+ <li><strong>Console system indices</strong> created at boot, before the ring is attached, stay local.</li>
+ </ul>
+ <p>There is still no shard assignment to other members and no replica set. For a second live copy of your documents, point the <a href="/docs/config#wal_tap">WAL tap</a> at another XERJ or any ES-compatible target: it ships indexed writes near-real-time from the WAL, one-directional, without touching the write path.</p>
<h2>Failure modes</h2>
<div class="enum-list">
- <div class="item">Leader crash<span class="dsc">Followers time out and elect a new leader — measured ~250 ms on a 3-node loopback ring, including one split-vote retry. The ring stays stable with the dead member absent; its Raft log survives on disk for the rejoin.</span></div>
+ <div class="item">Leader crash<span class="dsc">Followers time out and elect a new leader — measured ~250 ms on a 3-node loopback ring, including one split-vote retry. The ring stays stable with the dead member absent. Metadata already applied on the survivors stays; the dead member's in-memory log does not.</span></div>
<div class="item">Hung or dead peer<span class="dsc">One WARN on the down transition, then quiet: retries back off 100 ms → 6.4 s, and messages to live peers are never delayed behind a dead one. A peer that answers again logs "peer reachable again" at INFO.</span></div>
<div class="item">Minority partition<span class="dsc">The minority side cannot reach quorum and its elections never conclude; the majority side keeps its leader. Raft safety holds — at most one leader per term.</span></div>
- <div class="item">Node rejoin<span class="dsc">Restart with the same config: the node replays its local Raft log and re-enters the election. Catch-up from the leader's log (snapshot install) is part of the wiring roadmap.</span></div>
+ <div class="item">Node rejoin<span class="dsc">Restart with the same config: the node re-enters the election with an empty in-memory Raft log. Indices it already applied are still in its data directory. A metadata change committed while it was down is not installed. Snapshot catch-up is still open (<a href="https://github.com/xerj-org/xerj/issues/1170">#1170</a>).</span></div>
</div>
<h2>When not to cluster</h2>
- <p>Today, almost always. Until the replication wiring lands, cluster mode gives you a leader-election ring — not copies of your data. One node avoids network round-trips, needs no quorum, and is trivial to back up; add a WAL tap to a second node if you want a live spare. The default config ships with <code class="inline">[cluster] enabled = false</code>.</p>
+ <p>When you need copies of documents. Cluster mode now keeps index metadata in step and still does not copy documents, route shards, or catch a rejoining node up. One node avoids network round-trips, needs no quorum, and is trivial to back up; add a WAL tap to a second node if you want a live spare of the data. The default config ships with <code class="inline">[cluster] enabled = false</code>.</p>
<p class="source-note">Source · engine/crates/xerj-cluster/src/raft.rs · engine/crates/xerj-cluster/src/transport.rs · engine/crates/xerj-cluster/src/runner.rs</p>