AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
feat(cluster): route documents to the primary named in the create log (#2271)
xerj-orgfff5756aafbb
diff --git a/engine/crates/xerj-api/Cargo.toml b/engine/crates/xerj-api/Cargo.tomlindex 12662dfd9..5c680e4dd 100644--- a/engine/crates/xerj-api/Cargo.toml+++ b/engine/crates/xerj-api/Cargo.toml@@ -32,6 +32,10 @@ xerj-rerank = { path = "../xerj-rerank" }xerj-hub = { path = "../xerj-hub" }xerj-autoindex = { path = "../xerj-autoindex" }xerj-wasm = { path = "../xerj-wasm" }+# Shard routing: the ES document and search handlers forward to the+# owner named by the Raft shard plan. The cluster crate does not depend+# on this one.+xerj-cluster = { path = "../xerj-cluster" }# Local decision head (feature `decide-local`); no other purpose, so the# dependency itself is optional.xerj-ai = { path = "../xerj-ai", optional = true }@@ -71,6 +75,5 @@ sha2 = "0.10"[dev-dependencies]tempfile = { workspace = true }-xerj-cluster = { path = "../xerj-cluster" }tower = { workspace = true, features = ["util"] }http-body-util = "0.1"diff --git a/engine/crates/xerj-api/src/cluster_shard.rs b/engine/crates/xerj-api/src/cluster_shard.rsnew file mode 100644index 000000000..a83e76d08--- /dev/null+++ b/engine/crates/xerj-api/src/cluster_shard.rs@@ -0,0 +1,684 @@+//! Shard routing for the ES-compat document and search handlers.+//!+//! With a cluster handle attached, a replicated index carries a primary plan+//! in the same Raft entry as the create. [`dispatch_index`] forwards a+//! document to the owner [`Engine::route_write`] names. [`fanout_search`]+//! asks every owner and merges hits. Single-node mode (`has_cluster` false)+//! never enters these paths.+//!+//! The owner's handler writes local storage only. It does not route again.++use std::time::{Duration, Instant};++use axum::http::StatusCode;+use axum::response::{IntoResponse, Response};+use axum::Json;+use serde_json::{json, Value};+use xerj_cluster::{ShardOp, ShardResult};++use crate::es_compat::{+ apply_get_doc_source_filter, search_shard_local, EsSearchBody, EsSearchJson,+ EsSearchQueryParams, GetDocParams,+};+use crate::responses::{EsDeleteDocResponse, EsDocResponse, EsGetResponse};+use crate::state::AppState;+use crate::ApiError;++/// Per-shard `from + size` cap when a search is merged across owners.+const MAX_FANOUT_WINDOW: usize = 10_000;++/// How long a forward waits for the owner to apply the index create.+const OWNER_READY_WAIT: Duration = Duration::from_secs(2);++pub(crate) enum Dispatch {+ /// This node holds the shard (or the index has no plan). The caller+ /// uses its existing local path.+ Local,+ /// The response is complete: forwarded, or refused.+ Done(Response),+}++pub(crate) fn shard_not_ready_body(index: &str) -> String {+ json!({+ "error": {+ "type": "shard_not_ready",+ "reason": format!("index [{index}] is not on this shard yet")+ },+ "status": 503+ })+ .to_string()+}++pub(crate) fn shard_not_ready_response(index: &str) -> Response {+ (+ StatusCode::SERVICE_UNAVAILABLE,+ Json(json!({+ "error": {+ "type": "shard_not_ready",+ "reason": format!("index [{index}] is not on this shard yet")+ },+ "status": 503+ })),+ )+ .into_response()+}++fn http_result(result: ShardResult) -> Response {+ let status = StatusCode::from_u16(result.status).unwrap_or(StatusCode::BAD_GATEWAY);+ let value: Value = serde_json::from_str(&result.body).unwrap_or(Value::Null);+ (status, Json(value)).into_response()+}++fn is_shard_not_ready(result: &ShardResult) -> bool {+ result.status == 503 && result.body.contains("shard_not_ready")+}++fn engine_error_response(err: xerj_engine::EngineError) -> Response {+ ApiError::new(xerj_common::XerjError::from(err)).into_response()+}++async fn forward_retrying(state: &AppState, owner: &str, op: ShardOp) -> Response {+ let deadline = Instant::now() + OWNER_READY_WAIT;+ loop {+ match state.engine.forward_shard(owner, op.clone()).await {+ Ok(result) if is_shard_not_ready(&result) && Instant::now() < deadline => {+ tokio::time::sleep(Duration::from_millis(20)).await;+ }+ Ok(result) => return http_result(result),+ Err(err) => return engine_error_response(err),+ }+ }+}++/// Route a prepared document. `Ok`-shaped [`Dispatch::Local`] means index here.+///+/// `op` is `"index"` or `"create"`. Sequence numbers are the engine's+/// internal (1-based) values.+#[allow(clippy::too_many_arguments)]+pub(crate) async fn dispatch_index(+ state: &AppState,+ index: &str,+ id: &str,+ doc: &Value,+ op: &str,+ if_seq_no: Option<u64>,+ if_primary_term: Option<u64>,+ version: Option<u64>,+ version_type: Option<String>,+) -> Dispatch {+ if !state.engine.has_cluster() {+ return Dispatch::Local;+ }+ if state.engine.get_index(index).is_err() {+ if let Err(err) = state.engine.ensure_replicated_index(index).await {+ return Dispatch::Done(engine_error_response(err));+ }+ }+ let Some(owner) = state.engine.route_write(index, id) else {+ return Dispatch::Local;+ };+ let shard_op = ShardOp::IndexDoc {+ index: index.to_string(),+ id: id.to_string(),+ source_json: doc.to_string(),+ op: op.to_string(),+ if_seq_no,+ if_primary_term,+ version,+ version_type,+ };+ Dispatch::Done(forward_retrying(state, &owner, shard_op).await)+}++pub(crate) async fn dispatch_get(+ state: &AppState,+ index: &str,+ id: &str,+ params: &GetDocParams,+) -> Dispatch {+ if !state.engine.has_cluster() {+ return Dispatch::Local;+ }+ let Some(owner) = state.engine.route_write(index, id) else {+ return Dispatch::Local;+ };+ let op = ShardOp::GetDoc {+ index: index.to_string(),+ id: id.to_string(),+ source: params.source.clone(),+ source_includes: params.source_includes.clone(),+ source_excludes: params.source_excludes.clone(),+ };+ Dispatch::Done(forward_retrying(state, &owner, op).await)+}++pub(crate) async fn dispatch_delete(+ state: &AppState,+ index: &str,+ id: &str,+ if_seq_no: Option<u64>,+ if_primary_term: Option<u64>,+) -> Dispatch {+ if !state.engine.has_cluster() {+ return Dispatch::Local;+ }+ let Some(owner) = state.engine.route_write(index, id) else {+ return Dispatch::Local;+ };+ let op = ShardOp::DeleteDoc {+ index: index.to_string(),+ id: id.to_string(),+ if_seq_no,+ if_primary_term,+ };+ Dispatch::Done(forward_retrying(state, &owner, op).await)+}++fn cluster_search_unsupported(feature: &str) -> Response {+ let reason = format!(+ "cluster search does not merge {feature} across shards yet; the request was not executed"+ );+ (+ StatusCode::BAD_REQUEST,+ Json(json!({+ "error": {+ "type": "illegal_argument_exception",+ "reason": reason+ },+ "status": 400+ })),+ )+ .into_response()+}++/// `Some` when this search was answered by fan-out (including a refusal).+/// `None` when the caller should run the ordinary local search.+pub(crate) async fn fanout_search(+ state: &AppState,+ index: &str,+ params: &EsSearchQueryParams,+ principal: &crate::auth::Principal,+ body: &EsSearchJson,+) -> Option<Response> {+ if !state.engine.has_cluster() {+ return None;+ }+ if index.contains(',') || index.contains('*') || index == "_all" {+ return None;+ }+ let targets = state.engine.shard_search_targets(index);+ let me = state.engine.routing_node_id();+ if targets.is_empty() || (targets.len() == 1 && targets[0] == me) {+ return None;+ }+ let parsed = body.0.clone().unwrap_or_default();+ if parsed.aggs.is_some() || parsed.aggregations.is_some() {+ return Some(cluster_search_unsupported("aggregations"));+ }+ if parsed.suggest.is_some() {+ return Some(cluster_search_unsupported("suggest"));+ }+ if parsed.collapse.is_some() {+ return Some(cluster_search_unsupported("collapse"));+ }+ if parsed.rescore.is_some() {+ return Some(cluster_search_unsupported("rescore"));+ }+ if parsed.profile {+ return Some(cluster_search_unsupported("profile"));+ }+ if parsed.search_after.is_some() {+ return Some(cluster_search_unsupported("search_after"));+ }+ if params.scroll.is_some() {+ return Some(cluster_search_unsupported("scroll"));+ }++ let from = parsed.from;+ let size = parsed.size;+ let window = from.saturating_add(size);+ if window > MAX_FANOUT_WINDOW {+ return Some(cluster_search_unsupported("a from+size window above 10000"));+ }+ let mut shard_body = parsed;+ shard_body.from = 0;+ shard_body.size = window;+ let mut shard_params = params.clone();+ shard_params.from = Some("0".to_string());+ shard_params.size = Some(window.to_string());++ let started = Instant::now();+ let mut parts: Vec<Value> = Vec::with_capacity(targets.len());+ for target in &targets {+ let (status, value) = if target == &me {+ let response = Box::pin(search_shard_local(+ state.clone(),+ index.to_string(),+ shard_params.clone(),+ principal.clone(),+ shard_body.clone(),+ ))+ .await;+ match response_value(response).await {+ Ok(pair) => pair,+ Err(err) => {+ return Some(shard_failure(target, &err));+ }+ }+ } else {+ let op = match (+ serde_json::to_string(&shard_body),+ serde_json::to_string(&shard_params),+ ) {+ (Ok(body_json), Ok(params_json)) => ShardOp::Search {+ index: index.to_string(),+ body_json,+ params_json,+ },+ (Err(err), _) | (_, Err(err)) => {+ return Some(shard_failure(target, &err.to_string()));+ }+ };+ match state.engine.forward_shard(target, op).await {+ Ok(result) => {+ let value = serde_json::from_str(&result.body).unwrap_or(Value::Null);+ (result.status, value)+ }+ Err(err) => return Some(engine_error_response(err)),+ }+ };+ if !(200..300).contains(&status) {+ return Some(+ (+ StatusCode::from_u16(status).unwrap_or(StatusCode::BAD_GATEWAY),+ Json(value),+ )+ .into_response(),+ );+ }+ parts.push(value);+ }++ let took_ms = started.elapsed().as_millis() as u64;+ let as_int = matches!(+ params.rest_total_hits_as_int.as_deref(),+ Some("true") | Some("1")+ );+ let merged = merge_hits(&parts, from, size, as_int, took_ms);+ Some(Json(merged).into_response())+}++fn shard_failure(target: &str, err: &str) -> Response {+ (+ StatusCode::SERVICE_UNAVAILABLE,+ Json(json!({+ "error": {+ "type": "shard_not_ready",+ "reason": format!("search on shard owner [{target}] failed: {err}")+ },+ "status": 503+ })),+ )+ .into_response()+}++async fn response_value(response: Response) -> Result<(u16, Value), String> {+ let status = response.status().as_u16();+ let bytes = axum::body::to_bytes(response.into_body(), usize::MAX)+ .await+ .map_err(|err| err.to_string())?;+ let value = serde_json::from_slice(&bytes).unwrap_or(Value::Null);+ Ok((status, value))+}++fn merge_hits(+ parts: &[Value],+ from: usize,+ size: usize,+ total_as_int: bool,+ took_ms: u64,+) -> Value {+ let mut hits: Vec<Value> = Vec::new();+ let mut total: u64 = 0;+ let mut relation = "eq";+ let mut max_score: Option<f64> = None;+ let mut timed_out = false;+ for part in parts {+ if part.get("timed_out").and_then(Value::as_bool) == Some(true) {+ timed_out = true;+ }+ match part.pointer("/hits/total") {+ Some(Value::Number(n)) => total = total.saturating_add(n.as_u64().unwrap_or(0)),+ Some(Value::Object(_)) => {+ total = total.saturating_add(+ part.pointer("/hits/total/value")+ .and_then(Value::as_u64)+ .unwrap_or(0),+ );+ if part.pointer("/hits/total/relation").and_then(Value::as_str) == Some("gte") {+ relation = "gte";+ }+ }+ _ => {}+ }+ if let Some(score) = part.pointer("/hits/max_score").and_then(Value::as_f64) {+ max_score = Some(max_score.map(|current| current.max(score)).unwrap_or(score));+ }+ if let Some(arr) = part.pointer("/hits/hits").and_then(Value::as_array) {+ hits.extend(arr.iter().cloned());+ }+ }+ hits.sort_by(|left, right| {+ let left_score = left.get("_score").and_then(Value::as_f64).unwrap_or(0.0);+ let right_score = right.get("_score").and_then(Value::as_f64).unwrap_or(0.0);+ right_score+ .partial_cmp(&left_score)+ .unwrap_or(std::cmp::Ordering::Equal)+ .then_with(|| {+ left.get("_id")+ .and_then(Value::as_str)+ .unwrap_or("")+ .cmp(right.get("_id").and_then(Value::as_str).unwrap_or(""))+ })+ });+ if size == 0 {+ hits.clear();+ } else {+ hits = hits.into_iter().skip(from).take(size).collect();+ }+ let total_value = if total_as_int {+ json!(total)+ } else {+ json!({ "value": total, "relation": relation })+ };+ json!({+ "took": took_ms,+ "timed_out": timed_out,+ "_shards": {+ "total": parts.len(),+ "successful": parts.len(),+ "skipped": 0,+ "failed": 0+ },+ "hits": {+ "total": total_value,+ "max_score": max_score,+ "hits": hits+ }+ })+}++/// Handler installed on the cluster handle. Runs the operation locally.+pub struct ApiShardHandler {+ state: AppState,+}++impl ApiShardHandler {+ pub fn new(state: AppState) -> Self {+ ApiShardHandler { state }+ }+}++#[async_trait::async_trait]+impl xerj_cluster::ShardHandler for ApiShardHandler {+ async fn handle(&self, op: ShardOp) -> ShardResult {+ match op {+ ShardOp::IndexDoc {+ index,+ id,+ source_json,+ op,+ if_seq_no,+ if_primary_term,+ version,+ version_type,+ } => {+ local_index(+ &self.state,+ &index,+ &id,+ &source_json,+ &op,+ if_seq_no,+ if_primary_term,+ version,+ version_type.as_deref(),+ )+ .await+ }+ ShardOp::GetDoc {+ index,+ id,+ source,+ source_includes,+ source_excludes,+ } => {+ local_get(+ &self.state,+ &index,+ &id,+ source,+ source_includes,+ source_excludes,+ )+ .await+ }+ ShardOp::DeleteDoc {+ index,+ id,+ if_seq_no,+ if_primary_term,+ } => local_delete(&self.state, &index, &id, if_seq_no, if_primary_term).await,+ ShardOp::Search {+ index,+ body_json,+ params_json,+ } => local_search(&self.state, &index, &body_json, ¶ms_json).await,+ }+ }+}++async fn response_to_result(response: Response) -> ShardResult {+ let status = response.status().as_u16();+ let bytes = axum::body::to_bytes(response.into_body(), usize::MAX)+ .await+ .unwrap_or_default();+ let body = String::from_utf8(bytes.to_vec()).unwrap_or_else(|_| "{}".to_string());+ ShardResult::http(status, body)+}++#[allow(clippy::too_many_arguments)]+async fn local_index(+ state: &AppState,+ index: &str,+ id: &str,+ source_json: &str,+ op: &str,+ if_seq_no: Option<u64>,+ if_primary_term: Option<u64>,+ version: Option<u64>,+ version_type: Option<&str>,+) -> ShardResult {+ if state.engine.get_index(index).is_err() {+ return ShardResult::http(503, shard_not_ready_body(index));+ }+ let doc: Value = match serde_json::from_str(source_json) {+ Ok(doc) => doc,+ Err(err) => {+ return ShardResult::http(+ 400,+ json!({+ "error": {+ "type": "document_parsing_exception",+ "reason": format!("forwarded document is not JSON: {err}")+ },+ "status": 400+ })+ .to_string(),+ );+ }+ };+ let idx = match state.engine.get_index(index) {+ Ok(idx) => idx,+ Err(_) => return ShardResult::http(503, shard_not_ready_body(index)),+ };+ let result = if op == "create" {+ idx.create_document(id.to_string(), doc).await+ } else if let (Some(version), Some(version_type)) = (version, version_type) {+ if version_type == "external" || version_type == "external_gte" {+ idx.index_document_external(Some(id.to_string()), doc, version, version_type)+ .await+ } else {+ idx.index_document_with_version(Some(id.to_string()), doc, if_seq_no, if_primary_term)+ .await+ }+ } else {+ idx.index_document_with_version(Some(id.to_string()), doc, if_seq_no, if_primary_term)+ .await+ };+ match result {+ Ok(resp) => {+ state.metrics.record_doc_indexed(index);+ let wire_seq = resp.seq_no.saturating_sub(1);+ let (body, status) = if resp.result == "updated" {+ (+ EsDocResponse::updated(index, &resp.id, resp.version, wire_seq),+ 200,+ )+ } else {+ (+ EsDocResponse::created(index, &resp.id, resp.version, wire_seq),+ 201,+ )+ };+ ShardResult::http(+ status,+ serde_json::to_string(&body).unwrap_or_else(|_| "{}".into()),+ )+ }+ Err(err) => response_to_result(engine_error_response(err)).await,+ }+}++async fn local_get(+ state: &AppState,+ index: &str,+ id: &str,+ source: Option<String>,+ source_includes: Option<String>,+ source_excludes: Option<String>,+) -> ShardResult {+ if state.engine.closed_indices.contains_key(index) {+ return response_to_result(crate::es_compat::closed_index_error(index)).await;+ }+ let idx = match state.engine.get_index(index) {+ Ok(idx) => idx,+ Err(_) => return ShardResult::http(503, shard_not_ready_body(index)),+ };+ match idx.get_document(id).await {+ Ok(Some(mut doc)) => {+ xerj_query::executor::strip_internal_passage_metadata(&mut doc);+ let params = GetDocParams {+ source,+ source_includes,+ source_excludes,+ };+ let filtered = apply_get_doc_source_filter(doc, ¶ms);+ let version = idx.lookup_version(id).unwrap_or(1);+ let seq_no = idx.lookup_seq_no(id).unwrap_or(0);+ let resp = EsGetResponse::found(index, id, version, seq_no, filtered);+ ShardResult::http(+ 200,+ serde_json::to_string(&resp).unwrap_or_else(|_| "{}".into()),+ )+ }+ Ok(None) => {+ let resp = EsGetResponse::not_found(index, id);+ ShardResult::http(+ 404,+ serde_json::to_string(&resp).unwrap_or_else(|_| "{}".into()),+ )+ }+ Err(err) => response_to_result(engine_error_response(err)).await,+ }+}++async fn local_delete(+ state: &AppState,+ index: &str,+ id: &str,+ if_seq_no: Option<u64>,+ if_primary_term: Option<u64>,+) -> ShardResult {+ let idx = match state.engine.get_index(index) {+ Ok(idx) => idx,+ Err(_) => return ShardResult::http(503, shard_not_ready_body(index)),+ };+ match idx+ .delete_document_versioned(id, if_seq_no, if_primary_term)+ .await+ {+ Ok(outcome) if outcome.found => {+ let resp = EsDeleteDocResponse::deleted(+ index,+ id,+ outcome.version,+ outcome.seq_no.saturating_sub(1),+ );+ ShardResult::http(+ 200,+ serde_json::to_string(&resp).unwrap_or_else(|_| "{}".into()),+ )+ }+ Ok(outcome) => {+ let resp = EsDeleteDocResponse::not_found(+ index,+ id,+ outcome.version,+ outcome.seq_no.saturating_sub(1),+ );+ ShardResult::http(+ 404,+ serde_json::to_string(&resp).unwrap_or_else(|_| "{}".into()),+ )+ }+ Err(err) => response_to_result(engine_error_response(err)).await,+ }+}++async fn local_search(+ state: &AppState,+ index: &str,+ body_json: &str,+ params_json: &str,+) -> ShardResult {+ let params: EsSearchQueryParams = serde_json::from_str(params_json).unwrap_or_default();+ let body: EsSearchBody = match serde_json::from_str(body_json) {+ Ok(body) => body,+ Err(err) => {+ return ShardResult::http(+ 400,+ json!({+ "error": {+ "type": "parsing_exception",+ "reason": format!("forwarded search body is not JSON: {err}")+ },+ "status": 400+ })+ .to_string(),+ );+ }+ };+ // The accepting node already passed its auth middleware. This RPC is the+ // cluster transport, not a second client. The local shard of the+ // coordinator still runs under the caller's principal.+ let response = Box::pin(search_shard_local(+ state.clone(),+ index.to_string(),+ params,+ crate::auth::Principal::Superuser,+ body,+ ))+ .await;+ response_to_result(response).await+}diff --git a/engine/crates/xerj-api/src/es_compat.rs b/engine/crates/xerj-api/src/es_compat.rsindex 1fab49078..5fd14462a 100644--- a/engine/crates/xerj-api/src/es_compat.rs+++ b/engine/crates/xerj-api/src/es_compat.rs@@ -11,7 +11,7 @@ use axum::{Json,};use chrono::{Datelike, TimeZone, Timelike, Utc};-use serde::Deserialize;+use serde::{Deserialize, Serialize};use serde_json::{json, Value};use std::collections::{HashMap, HashSet};use uuid::Uuid;@@ -3911,12 +3911,36 @@ pub async fn index_doc_auto(Err(resp) => return resp,};- let idx = match state.engine.get_or_create_index(&index) {- Ok(i) => i,- Err(e) => return ApiError::new(xerj_common::XerjError::from(e)).into_response(),+ // Cluster mode picks the id before routing so the shard owner is a+ // function of the id that will be stored. Single-node keeps the engine's+ // auto-id.+ let preset_id = if state.engine.has_cluster() {+ let id = uuid::Uuid::new_v4().to_string();+ match crate::cluster_shard::dispatch_index(+ &state, &index, &id, &doc, "index", None, None, None, None,+ )+ .await+ {+ crate::cluster_shard::Dispatch::Done(response) => return response,+ crate::cluster_shard::Dispatch::Local => Some(id),+ }+ } else {+ None+ };++ let idx = if preset_id.is_some() {+ match state.engine.get_index(&index) {+ Ok(i) => i,+ Err(_) => return crate::cluster_shard::shard_not_ready_response(&index),+ }+ } else {+ match state.engine.get_or_create_index(&index) {+ Ok(i) => i,+ Err(e) => return ApiError::new(xerj_common::XerjError::from(e)).into_response(),+ }};- match idx.index_document(None, doc).await {+ match idx.index_document(preset_id, doc).await {Ok(resp) => {state.metrics.record_doc_indexed(&index);let er = EsDocResponse::created(@@ -4046,11 +4070,6 @@ pub async fn index_doc(Err(resp) => return resp,};- let idx = match state.engine.get_or_create_index(&index) {- Ok(i) => i,- Err(e) => return ApiError::new(xerj_common::XerjError::from(e)).into_response(),- };-// op_type=create means fail if document already exists.let is_create_op = params.op_type.as_deref() == Some("create");@@ -4060,6 +4079,37 @@ pub async fn index_doc(// or a previous write response round-trips correctly.let if_seq_no_internal = params.if_seq_no.map(|s| s.saturating_add(1));+ if state.engine.has_cluster() {+ match crate::cluster_shard::dispatch_index(+ &state,+ &index,+ &id,+ &doc,+ if is_create_op { "create" } else { "index" },+ if_seq_no_internal,+ params.if_primary_term,+ params.version,+ params.version_type.clone(),+ )+ .await+ {+ crate::cluster_shard::Dispatch::Done(response) => return response,+ crate::cluster_shard::Dispatch::Local => {}+ }+ }++ let idx = if state.engine.has_cluster() {+ match state.engine.get_index(&index) {+ Ok(i) => i,+ Err(_) => return crate::cluster_shard::shard_not_ready_response(&index),+ }+ } else {+ match state.engine.get_or_create_index(&index) {+ Ok(i) => i,+ Err(e) => return ApiError::new(xerj_common::XerjError::from(e)).into_response(),+ }+ };+if is_create_op {match idx.create_document(id.clone(), doc).await {Ok(resp) => {@@ -4179,9 +4229,27 @@ pub async fn create_doc(Err(resp) => return resp,};- let idx = match state.engine.get_or_create_index(&index) {- Ok(i) => i,- Err(e) => return ApiError::new(xerj_common::XerjError::from(e)).into_response(),+ if state.engine.has_cluster() {+ match crate::cluster_shard::dispatch_index(+ &state, &index, &id, &doc, "create", None, None, None, None,+ )+ .await+ {+ crate::cluster_shard::Dispatch::Done(response) => return response,+ crate::cluster_shard::Dispatch::Local => {}+ }+ }++ let idx = if state.engine.has_cluster() {+ match state.engine.get_index(&index) {+ Ok(i) => i,+ Err(_) => return crate::cluster_shard::shard_not_ready_response(&index),+ }+ } else {+ match state.engine.get_or_create_index(&index) {+ Ok(i) => i,+ Err(e) => return ApiError::new(xerj_common::XerjError::from(e)).into_response(),+ }};match idx.create_document(id.clone(), doc).await {@@ -4227,6 +4295,12 @@ pub async fn get_doc(return closed_index_error(&index);}+ if let crate::cluster_shard::Dispatch::Done(response) =+ crate::cluster_shard::dispatch_get(&state, &index, &id, ¶ms).await+ {+ return response;+ }+let idx = match state.engine.get_index(&index) {Ok(i) => i,Err(e) => return ApiError::new(xerj_common::XerjError::from(e)).into_response(),@@ -4277,7 +4351,7 @@ pub async fn get_doc(}/// Filter the _source of a GET _doc response based on query params.-fn apply_get_doc_source_filter(source: Value, params: &GetDocParams) -> Value {+pub(crate) fn apply_get_doc_source_filter(source: Value, params: &GetDocParams) -> Value {// _source=false → suppress everything.if params.source.as_deref() == Some("false") {return Value::Null;@@ -4374,14 +4448,26 @@ pub async fn delete_doc(Path((index, id)): Path<(String, String)>,Query(params): Query<DeleteDocParams>,) -> impl IntoResponse {+ // Wire → internal seq translation, same convention as the index path.+ let if_seq_no_internal = params.if_seq_no.map(|s| s.saturating_add(1));++ if let crate::cluster_shard::Dispatch::Done(response) = crate::cluster_shard::dispatch_delete(+ &state,+ &index,+ &id,+ if_seq_no_internal,+ params.if_primary_term,+ )+ .await+ {+ return response;+ }+let idx = match state.engine.get_index(&index) {Ok(i) => i,Err(e) => return ApiError::new(xerj_common::XerjError::from(e)).into_response(),};- // Wire → internal seq translation, same convention as the index path.- let if_seq_no_internal = params.if_seq_no.map(|s| s.saturating_add(1));-match idx.delete_document_versioned(&id, if_seq_no_internal, params.if_primary_term).await@@ -4421,7 +4507,7 @@ pub async fn delete_doc(// Supports comma-separated multi-index: /index1,index2/_search// ─────────────────────────────────────────────────────────────────────────────-#[derive(Debug, Clone, Deserialize)]+#[derive(Debug, Clone, Deserialize, Serialize)]pub struct EsSearchBody {#[serde(default)]pub query: Option<Value>,@@ -8963,7 +9049,7 @@ fn parse_highlight(hl_val: &Value) -> Option<xerj_query::ast::HighlightRequest>////// These are common params used by simple ES clients and curl one-liners./// Most are accepted and ignored (preference, routing, etc.) to avoid 400 errors.-#[derive(Debug, Default, Deserialize)]+#[derive(Debug, Clone, Default, Deserialize, Serialize)]pub struct EsSearchQueryParams {/// Simple query string — `?q=title:rust`.pub q: Option<String>,@@ -11972,7 +12058,7 @@ pub async fn search(// `memory_api`'s recall test, which calls this handler directly, overflows// its stack on the extra copy. Boxing keeps the wrapper pointer-sized.let (response, fault) = xerj_engine::painless::with_script_fault_capture(Box::pin(- search_impl(state, index, params, principal, body),+ search_dispatch(state, index, params, principal, body),)).await;match fault {@@ -12482,6 +12568,32 @@ impl SkippedIndices {}}+async fn search_dispatch(+ state: AppState,+ index: String,+ params: EsSearchQueryParams,+ principal: crate::auth::Principal,+ body: EsSearchJson,+) -> Response {+ if let Some(response) =+ crate::cluster_shard::fanout_search(&state, &index, ¶ms, &principal, &body).await+ {+ return response;+ }+ search_impl(state, index, params, principal, body).await+}++/// One shard's search, used by the cluster fan-out. Does not fan out again.+pub(crate) async fn search_shard_local(+ state: AppState,+ index: String,+ params: EsSearchQueryParams,+ principal: crate::auth::Principal,+ body: EsSearchBody,+) -> Response {+ search_impl(state, index, params, principal, EsSearchJson(Some(body))).await+}+async fn search_impl(state: AppState,index: String,@@ -47394,7 +47506,7 @@ fn count_dense_vector_fields(node: &Value) -> u64 {/// targets an index that has been closed via `POST /:index/_close`./// Mirrors Elasticsearch's body shape so wire-compat clients see the/// exact `type`/`reason`/`index` triplet plus the duplicated root_cause.-fn closed_index_error(index: &str) -> axum::response::Response {+pub(crate) fn closed_index_error(index: &str) -> axum::response::Response {(StatusCode::BAD_REQUEST,Json(json!({diff --git a/engine/crates/xerj-api/src/lib.rs b/engine/crates/xerj-api/src/lib.rsindex 1141762dd..cff45e259 100644--- a/engine/crates/xerj-api/src/lib.rs+++ b/engine/crates/xerj-api/src/lib.rs@@ -69,6 +69,7 @@ pub mod auth;pub mod authz;pub mod binary_protocol;pub mod cat;+pub mod cluster_shard;pub mod error;pub mod es_compat;pub mod extract;diff --git a/engine/crates/xerj-api/tests/cluster_metadata_replicates.rs b/engine/crates/xerj-api/tests/cluster_metadata_replicates.rsindex b08f64760..225258a98 100644--- a/engine/crates/xerj-api/tests/cluster_metadata_replicates.rs+++ b/engine/crates/xerj-api/tests/cluster_metadata_replicates.rs@@ -1,6 +1,7 @@//! Index create, mapping, settings, and delete on one member of a 3-node-//! ring show up on the others. Documents do not. A ring that is not running-//! refuses the create instead of applying it locally.+//! ring show up on the others. A document written on one member is readable+//! from the others because it is stored on the shard owner. A ring that is+//! not running refuses the create instead of applying it locally.use std::time::{Duration, Instant};@@ -10,6 +11,7 @@ use axum::Router;use serde_json::{json, Value};use tokio::sync::watch;use tower::ServiceExt;+use xerj_api::cluster_shard::ApiShardHandler;use xerj_cluster::node::in_memory::{InMemoryBus, InMemoryTransport};use xerj_cluster::runner::ClusterRunner;use xerj_cluster::{ClusterHandle, ClusterNode};@@ -17,6 +19,7 @@ use xerj_engine::EngineMetadataApplier;struct Member {app: Router,+ engine: std::sync::Arc<xerj_engine::Engine>,shutdown: watch::Sender<bool>,handle: ClusterHandle,/// Held when the runner is not started, so the proposal channel stays open.@@ -48,6 +51,8 @@ async fn ring(start_runners: bool) -> Vec<Member> {}));let handle = node.handle();state.engine.attach_cluster(handle.clone());+ handle.set_shard_handler(std::sync::Arc::new(ApiShardHandler::new(state.clone())));+ let engine = std::sync::Arc::clone(&state.engine);let (shutdown, shutdown_rx) = watch::channel(false);let kept = if start_runners {let mut runner = ClusterRunner::new(node, Duration::from_millis(5), shutdown_rx);@@ -63,6 +68,7 @@ async fn ring(start_runners: bool) -> Vec<Member> {let app = xerj_api::router::build_es_compat_router(state);members.push(Member {app,+ engine,shutdown,handle,_node: kept,@@ -93,6 +99,17 @@ async fn put_json(app: &Router, path: &str, body: Value) -> (StatusCode, Value).await}+async fn post_json(app: &Router, path: &str, body: Value) -> (StatusCode, Value) {+ send(+ app,+ Request::post(path)+ .header("content-type", "application/json")+ .body(Body::from(body.to_string()))+ .expect("request"),+ )+ .await+}+async fn get(app: &Router, path: &str) -> (StatusCode, Value) {send(app,@@ -157,7 +174,7 @@ async fn wait_body_contains(app: &Router, path: &str, needle: &str) -> Value {}#[tokio::test(flavor = "multi_thread", worker_threads = 4)]-async fn index_metadata_propagates_documents_do_not_and_a_new_leader_still_commits() {+async fn index_metadata_propagates_and_a_new_leader_still_commits() {let members = ring(true).await;let mut dead = [false, false, false];let leader = wait_leader(&members, &dead).await;@@ -196,24 +213,22 @@ async fn index_metadata_propagates_documents_do_not_and_a_new_leader_still_commilet (status, body) = put_json(&members[follower].app,"/orders/_doc/1",- json!({ "title": "local only" }),+ json!({ "title": "routed" }),).await;- assert!(- status.is_success(),- "document write is still local: {status} {body}"- );- let (status, _) = get(&members[follower].app, "/orders/_doc/1").await;- assert!(status.is_success(), "writer should read its own document");- let other = (0..3).find(|i| *i != follower).expect("peer");- let (status, body) = get(&members[other].app, "/orders/_doc/1").await;- assert_eq!(- status,- StatusCode::NOT_FOUND,- "documents are not replicated yet: {body}"- );- let (status, _) = get(&members[other].app, "/orders").await;- assert_eq!(status, StatusCode::OK, "the index itself is on the peer");+ assert!(status.is_success(), "document write: {status} {body}");+ for member in &members {+ let (status, body) = get(&member.app, "/orders/_doc/1").await;+ assert!(+ status.is_success(),+ "document is readable on every member: {status} {body}"+ );+ }+ let owners = members+ .iter()+ .filter(|member| member.engine.owns_shard("orders", "1"))+ .count();+ assert_eq!(owners, 1, "the document has one primary owner");let (status, body) = delete(&members[follower].app, "/orders").await;assert!(status.is_success(), "delete: {status} {body}");@@ -250,6 +265,15 @@ async fn a_ring_with_no_leader_does_not_create_the_index() {assert_eq!(status, StatusCode::SERVICE_UNAVAILABLE, "{body}");let (status, _) = get(&members[0].app, "/orders").await;assert_eq!(status, StatusCode::NOT_FOUND);+ let (status, body) = put_json(+ &members[0].app,+ "/orders/_doc/1",+ json!({ "title": "no leader" }),+ )+ .await;+ assert_eq!(status, StatusCode::SERVICE_UNAVAILABLE, "{body}");+ let (status, _) = get(&members[0].app, "/orders/_doc/1").await;+ assert_eq!(status, StatusCode::NOT_FOUND);let (status, _) = get(&members[1].app, "/orders").await;assert_eq!(status, StatusCode::NOT_FOUND);}@@ -275,4 +299,153 @@ async fn cluster_disabled_create_stays_on_the_node_that_received_it() {assert_eq!(status, StatusCode::OK);let (status, _) = get(&b, "/local").await;assert_eq!(status, StatusCode::NOT_FOUND);+ let (status, body) = put_json(&a, "/local/_doc/1", json!({ "title": "here" })).await;+ assert!(+ status.is_success(),+ "single-node document write: {status} {body}"+ );+ let (status, _) = get(&a, "/local/_doc/1").await;+ assert_eq!(status, StatusCode::OK);+ let (status, _) = get(&b, "/local/_doc/1").await;+ assert_eq!(status, StatusCode::NOT_FOUND);+ let (status, body) = post_json(+ &a,+ "/local/_search",+ json!({ "query": { "match_all": {} }, "size": 10 }),+ )+ .await;+ assert!(status.is_success(), "single-node search: {status} {body}");+ assert_eq!(+ body.pointer("/hits/total/value").and_then(|v| v.as_u64()),+ Some(1),+ "single-node search total: {body}"+ );+}++#[tokio::test(flavor = "multi_thread", worker_threads = 4)]+async fn a_document_is_stored_on_its_shard_owner_and_search_sees_it_from_any_member() {+ let members = ring(true).await;+ let dead = [false, false, false];+ let _leader = wait_leader(&members, &dead).await;++ let (status, body) = put_json(&members[0].app, "/orders", json!({})).await;+ assert!(status.is_success(), "create: {status} {body}");+ for member in &members {+ wait_status(&member.app, "/orders", StatusCode::OK).await;+ assert_eq!(+ member.engine.shard_search_targets("orders").len(),+ 3,+ "create installs one primary on each member"+ );+ }++ let mut owner_of: Vec<(String, usize)> = Vec::new();+ for i in 0..48 {+ let id = format!("d{i}");+ if let Some(owner) = members+ .iter()+ .position(|member| member.engine.owns_shard("orders", &id))+ {+ if !owner_of.iter().any(|(_, seen)| *seen == owner) {+ owner_of.push((id, owner));+ }+ }+ if owner_of.len() == 3 {+ break;+ }+ }+ assert_eq!(+ owner_of.len(),+ 3,+ "expected the plan to place some id on each member"+ );++ for (id, _) in &owner_of {+ let writer = (owner_of[0].1 + 1) % 3;+ let (status, body) = put_json(+ &members[writer].app,+ &format!("/orders/_doc/{id}"),+ json!({ "title": id }),+ )+ .await;+ assert!(status.is_success(), "write {id}: {status} {body}");+ }++ for (id, owner) in &owner_of {+ for (i, member) in members.iter().enumerate() {+ let (status, body) = get(&member.app, &format!("/orders/_doc/{id}")).await;+ assert!(+ status.is_success(),+ "GET {id} on member {i}: {status} {body}"+ );+ let local = member+ .engine+ .get_index("orders")+ .expect("index")+ .get_document(id)+ .await+ .expect("local get");+ if i == *owner {+ assert!(local.is_some(), "owner stores {id}");+ } else {+ assert!(local.is_none(), "non-owner does not store {id}");+ }+ }+ }++ let reader = (owner_of[0].1 + 2) % 3;+ let (status, body) = post_json(+ &members[reader].app,+ "/orders/_search",+ json!({ "query": { "match_all": {} }, "size": 20 }),+ )+ .await;+ assert!(status.is_success(), "search: {status} {body}");+ let hits = body+ .pointer("/hits/hits")+ .and_then(|v| v.as_array())+ .cloned()+ .unwrap_or_default();+ let ids: Vec<&str> = hits+ .iter()+ .filter_map(|hit| hit.get("_id").and_then(|v| v.as_str()))+ .collect();+ for (id, _) in &owner_of {+ assert!(ids.contains(&id.as_str()), "search missed {id} in {ids:?}");+ }+ let total = body.pointer("/hits/total/value").and_then(|v| v.as_u64());+ assert_eq!(total, Some(owner_of.len() as u64), "merged total: {body}");++ let (status, body) = post_json(+ &members[reader].app,+ "/orders/_search",+ json!({ "size": 0, "aggs": { "titles": { "terms": { "field": "title" } } } }),+ )+ .await;+ assert_eq!(+ status,+ StatusCode::BAD_REQUEST,+ "aggregations are not merged: {body}"+ );++ let (id, owner) = &owner_of[0];+ let deleter = (*owner + 1) % 3;+ let (status, body) = delete(&members[deleter].app, &format!("/orders/_doc/{id}")).await;+ assert!(status.is_success(), "delete: {status} {body}");+ for member in &members {+ let (status, _) = get(&member.app, &format!("/orders/_doc/{id}")).await;+ assert_eq!(status, StatusCode::NOT_FOUND);+ }+ let still = members[*owner]+ .engine+ .get_index("orders")+ .expect("index")+ .get_document(id)+ .await+ .expect("local get");+ assert!(still.is_none(), "delete reached the owner");++ for member in &members {+ let _ = member.shutdown.send(true);+ }}diff --git a/engine/crates/xerj-cluster/src/lib.rs b/engine/crates/xerj-cluster/src/lib.rsindex a44322f07..360c07a0d 100644--- a/engine/crates/xerj-cluster/src/lib.rs+++ b/engine/crates/xerj-cluster/src/lib.rs@@ -58,6 +58,7 @@ pub mod replication;pub mod router;pub mod runner;pub mod search;+pub mod shard;pub mod transport;// ── Convenience re-exports ────────────────────────────────────────────────────@@ -78,4 +79,8 @@ pub use router::jump_hash;pub use router::ShardRouter;pub use runner::{ClusterRunner, ClusterRunnerBuilder};pub use search::{merge_search_responses, SearchHit, SearchMessage};+pub use shard::{+ primary_shard_plan, shard_plan_from_schema_json, MemoryShardDir, MemoryShardRpc, NullShardRpc,+ ShardAssignment, ShardError, ShardHandler, ShardOp, ShardResult, ShardRpc,+};pub use transport::TcpTransport;diff --git a/engine/crates/xerj-cluster/src/metadata.rs b/engine/crates/xerj-cluster/src/metadata.rsindex 04636a5a5..a58820dfc 100644--- a/engine/crates/xerj-cluster/src/metadata.rs+++ b/engine/crates/xerj-cluster/src/metadata.rs@@ -178,10 +178,16 @@ impl ClusterMetadata {IndexMetadata {name: name.to_string(),schema_json: schema_json.to_string(),- num_shards: 1, // default; set by shard assignment commands+ num_shards: 1, // widened by the shard plan below, when one is presentcreated_at: now,},);+ // The plan rides in the same log entry as the create, so the index+ // and its owners become visible together. A schema with no plan+ // (older entries, tests) leaves the table empty.+ for assignment in crate::shard::shard_plan_from_schema_json(schema_json) {+ self.apply_assign_shard(name, assignment.shard, &assignment.node_id);+ }}fn apply_delete_index(&mut self, name: &str) {diff --git a/engine/crates/xerj-cluster/src/node.rs b/engine/crates/xerj-cluster/src/node.rsindex e75229a22..257b77aab 100644--- a/engine/crates/xerj-cluster/src/node.rs+++ b/engine/crates/xerj-cluster/src/node.rs@@ -22,6 +22,7 @@ use crate::propose::{ApplyEffect, ClusterHandle, ClusterView, MetadataApplier, ProposalRequest, ProposeError,};use crate::raft::{ClusterCommand, OutboundMessage, RaftMessage, RaftNode, RaftState};+use crate::shard::NullShardRpc;// ── Transport abstraction ─────────────────────────────────────────────────────@@ -36,6 +37,12 @@ pub trait ClusterTransport: Send + Sync {/// Receive the next inbound message. Returns `(sender_id, message)`.async fn recv(&self) -> Result<(String, RaftMessage)>;++ /// Data-plane handle for document forwarding. The default fails every+ /// call; TCP and the in-memory bus override it.+ fn shard_rpc(&self) -> Arc<dyn crate::shard::ShardRpc> {+ Arc::new(NullShardRpc)+ }}// ── Cluster node ──────────────────────────────────────────────────────────────@@ -89,7 +96,12 @@ impl ClusterNode {state: RaftState::Follower,leader_id: None,}));- let handle = ClusterHandle::new(id.clone(), tx, Arc::clone(&view));+ let shard = transport.shard_rpc();+ let mut members = peers.clone();+ members.push(id.clone());+ members.sort();+ members.dedup();+ let handle = ClusterHandle::new(id.clone(), members, tx, Arc::clone(&view), shard);ClusterNode {raft: RaftNode::new(id, peers),metadata: ClusterMetadata::new(),@@ -472,6 +484,8 @@ pub mod in_memory {use std::sync::Arc;use tokio::sync::{mpsc, Mutex};+ use crate::shard::{MemoryShardDir, MemoryShardRpc};+type Sender = mpsc::UnboundedSender<(String, RaftMessage)>;type Receiver = mpsc::UnboundedReceiver<(String, RaftMessage)>;@@ -479,12 +493,14 @@ pub mod in_memory {#[derive(Clone)]pub struct InMemoryBus {senders: Arc<Mutex<HashMap<String, Sender>>>,+ shards: Arc<MemoryShardDir>,}impl InMemoryBus {pub fn new() -> Self {InMemoryBus {senders: Arc::new(Mutex::new(HashMap::new())),+ shards: MemoryShardDir::new(),}}@@ -523,15 +539,18 @@ pub mod in_memory {pub node_id: String,bus: InMemoryBus,receiver: Arc<Mutex<Receiver>>,+ shard_rpc: Arc<MemoryShardRpc>,}impl InMemoryTransport {pub async fn new(node_id: String, bus: InMemoryBus) -> Self {let receiver = bus.register(&node_id).await;+ let shard_rpc = MemoryShardRpc::new(node_id.clone(), Arc::clone(&bus.shards));InMemoryTransport {node_id,bus,receiver: Arc::new(Mutex::new(receiver)),+ shard_rpc,}}}@@ -548,6 +567,10 @@ pub mod in_memory {.await.ok_or_else(|| anyhow::anyhow!("channel closed"))}++ fn shard_rpc(&self) -> Arc<dyn crate::shard::ShardRpc> {+ Arc::clone(&self.shard_rpc) as Arc<dyn crate::shard::ShardRpc>+ }}}diff --git a/engine/crates/xerj-cluster/src/propose.rs b/engine/crates/xerj-cluster/src/propose.rsindex 646eea79f..3775f7f2b 100644--- a/engine/crates/xerj-cluster/src/propose.rs+++ b/engine/crates/xerj-cluster/src/propose.rs@@ -13,6 +13,7 @@ use std::time::{Duration, Instant};use tokio::sync::{mpsc, oneshot};use crate::raft::{ClusterCommand, RaftState};+use crate::shard::{ShardError, ShardHandler, ShardOp, ShardResult, ShardRpc};/// How a committed metadata command changed the local engine.///@@ -75,25 +76,33 @@ pub struct ClusterHandle {struct ClusterHandleInner {node_id: String,+ /// This node plus its peers, sorted, deduped. The index-create proposal+ /// builds the shard plan from this list.+ members: Vec<String>,tx: mpsc::UnboundedSender<ProposalRequest>,next: AtomicU64,timeout_ms: AtomicU64,view: Arc<RwLock<ClusterView>>,+ shard: Arc<dyn ShardRpc>,}impl ClusterHandle {pub(crate) fn new(node_id: String,+ members: Vec<String>,tx: mpsc::UnboundedSender<ProposalRequest>,view: Arc<RwLock<ClusterView>>,+ shard: Arc<dyn ShardRpc>,) -> Self {ClusterHandle {inner: Arc::new(ClusterHandleInner {node_id,+ members,tx,next: AtomicU64::new(1),timeout_ms: AtomicU64::new(5_000),view,+ shard,}),}}@@ -102,6 +111,26 @@ impl ClusterHandle {&self.inner.node_id}+ /// Members the shard plan is built from: this node and the peers it was+ /// started with, sorted.+ pub fn members(&self) -> Vec<String> {+ self.inner.members.clone()+ }++ /// Install the handler that runs forwarded shard operations on this node.+ ///+ /// The server does this once the engine exists, before it accepts HTTP.+ /// A call that arrives with no handler fails; it is not applied locally+ /// on the sender.+ pub fn set_shard_handler(&self, handler: Arc<dyn ShardHandler>) {+ self.inner.shard.set_handler(handler);+ }++ /// Forward `op` to `to` and wait for that member's result.+ pub async fn shard_call(&self, to: &str, op: ShardOp) -> Result<ShardResult, ShardError> {+ self.inner.shard.call(to, op).await+ }+/// How long [`ClusterHandle::propose`] waits for a commit.////// The run loop uses the same budget: past the deadline it fails thediff --git a/engine/crates/xerj-cluster/src/raft.rs b/engine/crates/xerj-cluster/src/raft.rsindex fbbcb4220..afe5b65d1 100644--- a/engine/crates/xerj-cluster/src/raft.rs+++ b/engine/crates/xerj-cluster/src/raft.rs@@ -21,6 +21,8 @@ use rand::Rng;use serde::{Deserialize, Serialize};use tracing::{debug, info, warn};+use crate::shard::{ShardOp, ShardResult};+/// State a Raft node can be in.#[derive(Debug, Clone, PartialEq)]pub enum RaftState {@@ -189,6 +191,15 @@ pub enum RaftMessage {leader_id: Option<String>,reason: String,},+ /// Caller → shard owner, on its own connection. Not a Raft RPC and not+ /// appended to the log. The owner answers with [`RaftMessage::ShardResponse`]+ /// on the same connection.+ ShardRequest { request_id: u64, op: ShardOp },+ /// Shard owner → caller. See [`RaftMessage::ShardRequest`].+ ShardResponse {+ request_id: u64,+ result: ShardResult,+ },}fn message_term(msg: &RaftMessage) -> u64 {@@ -199,6 +210,9 @@ fn message_term(msg: &RaftMessage) -> u64 {| RaftMessage::RequestVoteResponse { term, .. }| RaftMessage::ForwardProposal { term, .. }| RaftMessage::ForwardRejected { term, .. } => *term,+ // Not a Raft term. A shard frame that reaches the state machine must+ // not step a leader down.+ RaftMessage::ShardRequest { .. } | RaftMessage::ShardResponse { .. } => 0,}}@@ -458,6 +472,10 @@ impl RaftNode {// the request id to a waiting caller; the state machine has// nothing else to record.}+ RaftMessage::ShardRequest { .. } | RaftMessage::ShardResponse { .. } => {+ // The transport answers these on the connection that carried+ // them. They are not log entries.+ }}out}diff --git a/engine/crates/xerj-cluster/src/router.rs b/engine/crates/xerj-cluster/src/router.rsindex cdf42bb8d..3a327f520 100644--- a/engine/crates/xerj-cluster/src/router.rs+++ b/engine/crates/xerj-cluster/src/router.rs@@ -73,6 +73,20 @@ impl ShardRouter {self.num_shards}+ /// Widen or set the hash width. `route_doc` uses this for every index,+ /// so a plan that assigns shard `n` must set the width to at least `n + 1`+ /// or that shard is unreachable.+ pub fn set_num_shards(&mut self, num_shards: u32) {+ assert!(num_shards > 0, "num_shards must be > 0");+ self.num_shards = num_shards;+ }++ /// Drop every primary and replica row for `index`.+ pub fn clear_index(&mut self, index: &str) {+ self.routing_table.retain(|(idx, _), _| idx != index);+ self.replica_table.retain(|(idx, _), _| idx != index);+ }+/// Route a document to its shard and owning node.////// Returns `(shard_id, Option<node_id>)`. `node_id` is `None` when thediff --git a/engine/crates/xerj-cluster/src/shard.rs b/engine/crates/xerj-cluster/src/shard.rsnew file mode 100644index 000000000..d8c131b09--- /dev/null+++ b/engine/crates/xerj-cluster/src/shard.rs@@ -0,0 +1,267 @@+//! Data-plane calls between shard owners.+//!+//! Metadata (index create, the shard plan) goes through the Raft log.+//! A document does not. The node that accepts the HTTP call asks+//! [`crate::router::ShardRouter`] which member owns the document and, when+//! that member is someone else, invokes [`ShardRpc::call`]. The owner's+//! [`ShardHandler`] applies the operation to local storage and returns the+//! JSON the caller's HTTP response forwards.+//!+//! In-process tests share a [`MemoryShardDir`]. The TCP transport opens a+//! fresh authenticated connection, writes one [`crate::raft::RaftMessage::ShardRequest`],+//! and reads one [`crate::raft::RaftMessage::ShardResponse`] before closing.+//! That exchange is not the Raft outbound queue: a dropped heartbeat is+//! retransmitted, a dropped document write must not be.++use std::collections::HashMap;+use std::sync::{Arc, Mutex};++use serde::{Deserialize, Serialize};++/// One primary-shard assignment carried in a committed index-create entry.+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]+pub struct ShardAssignment {+ pub shard: u32,+ pub node_id: String,+}++/// One primary per member, shard `i` on the `i`th member id in sorted order.+///+/// The plan is computed by the node that proposes the create and stored in+/// the log entry. Every member applies that list; nobody recomputes it.+pub fn primary_shard_plan(members: &[String]) -> Vec<ShardAssignment> {+ let mut members: Vec<&str> = members+ .iter()+ .map(String::as_str)+ .filter(|id| !id.is_empty())+ .collect();+ members.sort_unstable();+ members.dedup();+ if members.is_empty() {+ return Vec::new();+ }+ (0..members.len() as u32)+ .map(|shard| ShardAssignment {+ shard,+ node_id: members[shard as usize].to_string(),+ })+ .collect()+}++/// Assignments embedded in a `CreateIndex` schema envelope, if any.+///+/// A schema that is not the replicated envelope — including the `"{}"`+/// fixtures the Raft tests propose — yields an empty plan.+pub fn shard_plan_from_schema_json(schema_json: &str) -> Vec<ShardAssignment> {+ let Ok(value) = serde_json::from_str::<serde_json::Value>(schema_json) else {+ return Vec::new();+ };+ let Some(items) = value.get("shard_plan").and_then(|v| v.as_array()) else {+ return Vec::new();+ };+ items+ .iter()+ .filter_map(|item| {+ let shard = item.get("shard")?.as_u64()?;+ let node_id = item.get("node_id")?.as_str()?.to_string();+ if node_id.is_empty() || shard > u32::MAX as u64 {+ return None;+ }+ Some(ShardAssignment {+ shard: shard as u32,+ node_id,+ })+ })+ .collect()+}++/// A document or search operation forwarded to the shard owner.+#[derive(Debug, Clone, Serialize, Deserialize)]+pub enum ShardOp {+ /// Index or create one document. `source_json` is the body after the+ /// accepting node has run pipelines and field checks.+ IndexDoc {+ index: String,+ id: String,+ source_json: String,+ /// `"index"` or `"create"`.+ op: String,+ /// Internal (1-based) sequence, already translated from the ES wire.+ if_seq_no: Option<u64>,+ if_primary_term: Option<u64>,+ version: Option<u64>,+ version_type: Option<String>,+ },+ GetDoc {+ index: String,+ id: String,+ source: Option<String>,+ source_includes: Option<String>,+ source_excludes: Option<String>,+ },+ DeleteDoc {+ index: String,+ id: String,+ if_seq_no: Option<u64>,+ if_primary_term: Option<u64>,+ },+ /// Search this member's local shard only. The caller merges hits.+ Search {+ index: String,+ /// JSON object, the search body, with `from`/`size` already widened+ /// to the window the merger needs.+ body_json: String,+ /// JSON [`crate`] search query params. `from` and `size` match the body.+ params_json: String,+ },+}++/// What the owner sends back. `status` is an HTTP status; `body` is JSON.+#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]+pub struct ShardResult {+ pub status: u16,+ pub body: String,+}++impl ShardResult {+ pub fn http(status: u16, body: impl Into<String>) -> Self {+ ShardResult {+ status,+ body: body.into(),+ }+ }+}++/// Why a shard call did not produce a [`ShardResult`].+///+/// The document was not acknowledged. Callers must not write it locally+/// instead.+#[derive(Debug, thiserror::Error)]+pub enum ShardError {+ #[error("shard call to {to} failed: {reason}")]+ Unavailable { to: String, reason: String },+ #[error("shard call to {to} timed out")]+ Timeout { to: String },+}++/// Executes a [`ShardOp`] against this node's engine.+#[async_trait::async_trait]+pub trait ShardHandler: Send + Sync {+ async fn handle(&self, op: ShardOp) -> ShardResult;+}++/// Sends a [`ShardOp`] to the named member and waits for its result.+#[async_trait::async_trait]+pub trait ShardRpc: Send + Sync {+ async fn call(&self, to: &str, op: ShardOp) -> Result<ShardResult, ShardError>;+ fn set_handler(&self, handler: Arc<dyn ShardHandler>);+}++/// Transport that has no data plane. Calls fail; nothing is written locally.+pub struct NullShardRpc;++#[async_trait::async_trait]+impl ShardRpc for NullShardRpc {+ async fn call(&self, to: &str, _op: ShardOp) -> Result<ShardResult, ShardError> {+ Err(ShardError::Unavailable {+ to: to.to_string(),+ reason: "this transport has no shard path".to_string(),+ })+ }++ fn set_handler(&self, _handler: Arc<dyn ShardHandler>) {}+}++/// Handler map shared by every in-process member of one ring.+#[derive(Default)]+pub struct MemoryShardDir {+ handlers: Mutex<HashMap<String, Arc<dyn ShardHandler>>>,+}++impl MemoryShardDir {+ pub fn new() -> Arc<Self> {+ Arc::new(MemoryShardDir {+ handlers: Mutex::new(HashMap::new()),+ })+ }++ fn lock(&self) -> std::sync::MutexGuard<'_, HashMap<String, Arc<dyn ShardHandler>>> {+ self.handlers.lock().unwrap_or_else(|e| e.into_inner())+ }+}++/// In-process [`ShardRpc`]. A call invokes the target's handler directly.+pub struct MemoryShardRpc {+ node_id: String,+ dir: Arc<MemoryShardDir>,+}++impl MemoryShardRpc {+ pub fn new(node_id: impl Into<String>, dir: Arc<MemoryShardDir>) -> Arc<Self> {+ Arc::new(MemoryShardRpc {+ node_id: node_id.into(),+ dir,+ })+ }+}++#[async_trait::async_trait]+impl ShardRpc for MemoryShardRpc {+ async fn call(&self, to: &str, op: ShardOp) -> Result<ShardResult, ShardError> {+ let handler = self.dir.lock().get(to).cloned();+ match handler {+ Some(handler) => Ok(handler.handle(op).await),+ None => Err(ShardError::Unavailable {+ to: to.to_string(),+ reason: "no shard handler registered on that member".to_string(),+ }),+ }+ }++ fn set_handler(&self, handler: Arc<dyn ShardHandler>) {+ self.dir.lock().insert(self.node_id.clone(), handler);+ }+}++#[cfg(test)]+mod tests {+ use super::*;++ #[test]+ fn plan_is_one_primary_per_sorted_member() {+ let plan = primary_shard_plan(&["n3".into(), "n1".into(), "n2".into(), "n1".into()]);+ assert_eq!(+ plan,+ vec![+ ShardAssignment {+ shard: 0,+ node_id: "n1".into()+ },+ ShardAssignment {+ shard: 1,+ node_id: "n2".into()+ },+ ShardAssignment {+ shard: 2,+ node_id: "n3".into()+ },+ ]+ );+ }++ #[test]+ fn schema_without_a_plan_assigns_nothing() {+ assert!(shard_plan_from_schema_json("{}").is_empty());+ assert!(shard_plan_from_schema_json("not-json").is_empty());+ let parsed = shard_plan_from_schema_json(+ r#"{"v":1,"shard_plan":[{"shard":1,"node_id":"n2"},{"shard":0,"node_id":""}]}"#,+ );+ assert_eq!(+ parsed,+ vec![ShardAssignment {+ shard: 1,+ node_id: "n2".into()+ }]+ );+ }+}diff --git a/engine/crates/xerj-cluster/src/transport.rs b/engine/crates/xerj-cluster/src/transport.rsindex 060c3640a..2f4b390b5 100644--- a/engine/crates/xerj-cluster/src/transport.rs+++ b/engine/crates/xerj-cluster/src/transport.rs@@ -54,6 +54,7 @@ use tracing::{debug, info, warn};use crate::auth::{tags_match, ClusterSecret, CHALLENGE_LEN, TAG_LEN, WIRE_MAGIC, WIRE_VERSION};use crate::node::ClusterTransport;use crate::raft::RaftMessage;+use crate::shard::{ShardError, ShardHandler, ShardOp, ShardResult, ShardRpc};/// Largest accepted frame payload (10 MiB).const MAX_FRAME_BYTES: usize = 10 * 1024 * 1024;@@ -67,6 +68,12 @@ const MAX_NODE_ID_BYTES: usize = 256;/// (and its buffers) indefinitely — cheap for an attacker, expensive for us.const HANDSHAKE_TIMEOUT: Duration = Duration::from_secs(10);+/// How long a shard RPC waits for the owner's response, including connect.+///+/// Distinct from [`SEND_TIMEOUT`]: a document index can outlast a heartbeat+/// frame, and a timeout here is a failed write, not a dropped heartbeat.+const SHARD_RPC_TIMEOUT: Duration = Duration::from_secs(10);+/// How long a send may take once the TCP connection is established.////// Authentication makes the sender *read* before it writes (it needs the@@ -317,6 +324,20 @@ pub struct TcpTransport {// background listener task terminates.#[allow(dead_code)]sender: mpsc::Sender<(String, RaftMessage)>,+ /// Document/search RPC. Shared with the accept loop, which answers+ /// [`RaftMessage::ShardRequest`] on the connection that brought it.+ shard_rpc: Arc<TcpShardRpc>,+}++/// Authenticated request/response for one shard operation.+///+/// Not the Raft outbound queue. Each call opens a connection, writes one+/// request frame, and reads one response frame.+struct TcpShardRpc {+ node_id: String,+ peers: Arc<HashMap<String, SocketAddr>>,+ secret: ClusterSecret,+ handler: std::sync::Mutex<Option<Arc<dyn ShardHandler>>>,}impl TcpTransport {@@ -373,14 +394,22 @@ impl TcpTransport {outbound.insert(peer_id.clone(), peer_tx);}+ let peers = Arc::new(peers);+ let shard_rpc = Arc::new(TcpShardRpc {+ node_id: node_id.clone(),+ peers: Arc::clone(&peers),+ secret: secret.clone(),+ handler: std::sync::Mutex::new(None),+ });let transport = TcpTransport {node_id: node_id.clone(),listen_addr,- peers: Arc::new(peers),+ peers,secret,incoming: Arc::new(Mutex::new(rx)),outbound,sender: tx.clone(),+ shard_rpc,};// Spawn the background listener task.@@ -397,6 +426,7 @@ impl TcpTransport {let node_id = self.node_id.clone();let secret = self.secret.clone();+ let shard_rpc = Arc::clone(&self.shard_rpc);info!(node = %node_id, addr = %self.listen_addr, "TCP transport listening (authenticated)");tokio::spawn(async move {@@ -407,8 +437,10 @@ impl TcpTransport {let tx2 = tx.clone();let nid = node_id.clone();let secret = secret.clone();+ let shard_rpc = Arc::clone(&shard_rpc);tokio::spawn(async move {- if let Err(e) = handle_connection(stream, secret, tx2).await {+ if let Err(e) = handle_connection(stream, secret, tx2, shard_rpc).await+ {debug!(node = %nid, %peer_addr, error = %e, "TCP connection closed");}});@@ -524,6 +556,70 @@ async fn peer_sender_task(}}+#[async_trait]+impl ShardRpc for TcpShardRpc {+ async fn call(&self, to: &str, op: ShardOp) -> Result<ShardResult, ShardError> {+ if to == self.node_id {+ let handler = self+ .handler+ .lock()+ .unwrap_or_else(|e| e.into_inner())+ .clone();+ return match handler {+ Some(handler) => Ok(handler.handle(op).await),+ None => Err(ShardError::Unavailable {+ to: to.to_string(),+ reason: "no shard handler registered on this node".to_string(),+ }),+ };+ }+ let Some(addr) = self.peers.get(to).copied() else {+ return Err(ShardError::Unavailable {+ to: to.to_string(),+ reason: "unknown peer".to_string(),+ });+ };+ match tokio::time::timeout(+ SHARD_RPC_TIMEOUT,+ shard_exchange(&self.node_id, to, addr, &self.secret, op),+ )+ .await+ {+ Ok(Ok(result)) => Ok(result),+ Ok(Err(e)) => Err(ShardError::Unavailable {+ to: to.to_string(),+ reason: e.to_string(),+ }),+ Err(_) => Err(ShardError::Timeout { to: to.to_string() }),+ }+ }++ fn set_handler(&self, handler: Arc<dyn ShardHandler>) {+ *self.handler.lock().unwrap_or_else(|e| e.into_inner()) = Some(handler);+ }+}++async fn shard_exchange(+ node_id: &str,+ peer_id: &str,+ addr: SocketAddr,+ secret: &ClusterSecret,+ op: ShardOp,+) -> Result<ShardResult> {+ let mut stream = TcpStream::connect(addr)+ .await+ .with_context(|| format!("connect to shard peer {peer_id} at {addr}"))?;+ let challenge = read_handshake(&mut stream).await?;+ write_hello(&mut stream, secret, &challenge, node_id).await?;+ let request = RaftMessage::ShardRequest { request_id: 1, op };+ write_frame(&mut stream, secret, &challenge, node_id, 0, &request).await?;+ match read_frame(&mut stream, secret, &challenge, peer_id, 0).await? {+ Some(RaftMessage::ShardResponse { result, .. }) => Ok(result),+ Some(_) => anyhow::bail!("peer {peer_id} answered a shard call with a raft frame"),+ None => anyhow::bail!("peer {peer_id} closed before the shard response"),+ }+}+#[async_trait]impl ClusterTransport for TcpTransport {/// Enqueue `msg` on the peer's outbound queue — returns as soon as the@@ -558,6 +654,10 @@ impl ClusterTransport for TcpTransport {.await.ok_or_else(|| anyhow::anyhow!("TCP transport incoming channel closed"))}++ fn shard_rpc(&self) -> Arc<dyn ShardRpc> {+ Arc::clone(&self.shard_rpc) as Arc<dyn ShardRpc>+ }}// ── Connection handler ────────────────────────────────────────────────────────@@ -571,6 +671,7 @@ async fn handle_connection(mut stream: TcpStream,secret: ClusterSecret,tx: mpsc::Sender<(String, RaftMessage)>,+ shard_rpc: Arc<TcpShardRpc>,) -> Result<()> {let challenge = write_handshake(&mut stream).await?;@@ -586,6 +687,33 @@ async fn handle_connection(let mut seq: u64 = 0;loop {match read_frame(&mut stream, &secret, &challenge, &from, seq).await {+ Ok(Some(RaftMessage::ShardRequest { request_id, op })) => {+ let handler = shard_rpc+ .handler+ .lock()+ .unwrap_or_else(|e| e.into_inner())+ .clone();+ let result = match handler {+ Some(handler) => handler.handle(op).await,+ None => ShardResult::http(+ 503,+ r#"{"error":{"type":"shard_not_ready","reason":"no shard handler registered"},"status":503}"#,+ ),+ };+ let response = RaftMessage::ShardResponse { request_id, result };+ // seq 0 on the way back: the caller reads the first frame this+ // socket writes. The tag is over this node's id.+ write_frame(+ &mut stream,+ &secret,+ &challenge,+ &shard_rpc.node_id,+ 0,+ &response,+ )+ .await?;+ break;+ }Ok(Some(msg)) => {if tx.send((from.clone(), msg)).await.is_err() {break; // receiver droppeddiff --git a/engine/crates/xerj-engine/src/cluster_apply.rs b/engine/crates/xerj-engine/src/cluster_apply.rsindex 1a5875c28..7c4cffd09 100644--- a/engine/crates/xerj-engine/src/cluster_apply.rs+++ b/engine/crates/xerj-engine/src/cluster_apply.rs@@ -92,7 +92,7 @@ impl Engine {)));};spec.schema = self.apply_index_template(&spec.name, spec.schema);- let envelope = serde_json::json!({+ let mut envelope = serde_json::json!({"v": 1,"kind": "index_create","schema": spec.schema,@@ -101,6 +101,10 @@ impl Engine {"index_settings": spec.index_settings,"aliases": spec.aliases,});+ let plan = xerj_cluster::primary_shard_plan(&handle.members());+ if !plan.is_empty() {+ envelope["shard_plan"] = serde_json::to_value(&plan)?;+ }let schema_json = serde_json::to_string(&envelope)?;let effect = handle.propose(ClusterCommand::CreateIndex {@@ -112,6 +116,46 @@ impl Engine {effect_result(&spec.name, effect)}+ /// Create `name` through the log when this node does not have it yet.+ ///+ /// An `AlreadyExists` commit is success: a concurrent create won, and+ /// this node applied that entry before its own. Used by document+ /// auto-create so a clustered write does not invent a node-local index.+ pub async fn ensure_replicated_index(&self, name: &str) -> Result<()> {+ if self.get_index(name).is_ok() {+ return Ok(());+ }+ let spec = ReplicatedIndexSpec {+ name: name.to_string(),+ schema: Schema::empty(),+ settings: Value::Null,+ mappings: Value::Null,+ index_settings: Value::Null,+ aliases: Value::Null,+ };+ match self.replicate_index_create(spec).await {+ Ok(()) => Ok(()),+ Err(e) if is_already_exists(&e) => Ok(()),+ Err(e) => Err(e),+ }+ }++ /// Forward one shard operation to `owner`. A failure is not a local write.+ pub async fn forward_shard(+ &self,+ owner: &str,+ op: xerj_cluster::ShardOp,+ ) -> Result<xerj_cluster::ShardResult> {+ let Some(handle) = self.cluster_handle() else {+ return Err(EngineError::Common(xerj_common::XerjError::internal(+ "forward_shard called with no cluster handle",+ )));+ };+ handle.shard_call(owner, op).await.map_err(|err| {+ EngineError::Common(xerj_common::XerjError::cluster_unavailable(err.to_string()))+ })+ }+/// Propose one index's merged mapping. Callers validate before this.pub async fn replicate_mapping(&self,@@ -270,9 +314,28 @@ struct MappingEnvelope {async fn apply_committed(engine: &Engine, command: &ClusterCommand) -> ApplyEffect {match command {ClusterCommand::CreateIndex { name, schema_json } => {- apply_create(engine, name, schema_json).await+ // Install owners before the index is visible. A GET that observes+ // the new index otherwise races a router that is still empty and+ // treats the write as local.+ let plan = xerj_cluster::shard_plan_from_schema_json(schema_json);+ let installed_early = !plan.is_empty() && engine.shard_search_targets(name).is_empty();+ if installed_early {+ engine.install_shard_plan(name, &plan);+ }+ let effect = apply_create(engine, name, schema_json).await;+ if installed_early+ && matches!(effect, ApplyEffect::Failed(_))+ && engine.get_index(name).is_err()+ {+ engine.clear_shard_plan(name);+ }+ effect+ }+ ClusterCommand::DeleteIndex { name } => {+ let effect = apply_delete(engine, name).await;+ engine.clear_shard_plan(name);+ effect}- ClusterCommand::DeleteIndex { name } => apply_delete(engine, name).await,ClusterCommand::UpdateMapping {index,mapping_json,@@ -281,8 +344,16 @@ async fn apply_committed(engine: &Engine, command: &ClusterCommand) -> ApplyEffeindex,settings_json,} => apply_settings(engine, index, settings_json).await,- // Shard assignment, membership, and generic config are stored by- // ClusterMetadata. They do not change this node's indices.+ ClusterCommand::AssignShard {+ index,+ shard,+ node_id,+ } => {+ engine.assign_one_shard(index, *shard, node_id);+ ApplyEffect::Applied+ }+ // Membership and generic config are stored by ClusterMetadata.+ // They do not change this node's indices._ => ApplyEffect::Applied,}}diff --git a/engine/crates/xerj-engine/src/engine.rs b/engine/crates/xerj-engine/src/engine.rsindex 80b4bd5c5..2da91959d 100644--- a/engine/crates/xerj-engine/src/engine.rs+++ b/engine/crates/xerj-engine/src/engine.rs@@ -3538,28 +3538,90 @@ impl Engine {/// V4 M5.2 — route a document to the node that owns its shard.///- /// Returns `Ok(None)` when the doc belongs to this node (handle it- /// locally via the existing `Index::index_document_with_version`- /// path). Returns `Ok(Some(node_id))` when the doc belongs to a- /// peer and must be forwarded via the cluster transport.+ /// Returns `None` when this node should write the document itself:+ /// there is no shard assignment (single-node mode, and any index that+ /// was not created through the replicated shard plan), or the assigned+ /// owner is this node. Returns `Some(node_id)` when a peer owns the+ /// shard. Callers forward that write; they do not also store it here.///- /// In single-node mode (`num_shards == 1` and the router has no- /// assignments) this always resolves to "local" and the caller- /// takes the existing fast path — zero overhead for single-node- /// deployments.+ /// With no cluster handle the owner comparison uses `"local"`, which+ /// no replicated plan names, and the empty routing table makes every+ /// call `None`.pub fn route_write(&self, index: &str, doc_id: &str) -> Option<String> {+ let me = self.routing_node_id();let router = self.shard_router.read();- let (_shard, owner) = router.route_doc(index, doc_id);- match owner {- // No assignment yet — treat as local. This is the- // single-node default path.+ match router.route_doc(index, doc_id).1 {None => None,- Some(owner) if owner == self.node_id.as_str() => None,+ Some(owner) if owner == me => None,Some(owner) => Some(owner.to_string()),}}- /// Read-only view of the local node id.+ /// `true` when a replicated shard plan names this node as the primary+ /// for `doc_id`. An index with no plan is `false` — that is not the+ /// same as [`Engine::route_write`] returning `None`, which also covers+ /// the unassigned case.+ pub fn owns_shard(&self, index: &str, doc_id: &str) -> bool {+ let me = self.routing_node_id();+ let router = self.shard_router.read();+ matches!(router.route_doc(index, doc_id).1, Some(owner) if owner == me)+ }++ /// Node ids that hold at least one primary of `index`, sorted.+ pub fn shard_search_targets(&self, index: &str) -> Vec<String> {+ self.shard_router.read().search_targets(index)+ }++ /// Identity [`Engine::route_write`] compares against shard owners.+ ///+ /// The cluster handle's node id when one is attached, otherwise the+ /// single-node `"local"` id.+ pub fn routing_node_id(&self) -> String {+ self.cluster+ .lock()+ .as_ref()+ .map(|handle| handle.node_id().to_string())+ .unwrap_or_else(|| self.node_id.to_string())+ }++ /// Replace `index`'s primary assignments and widen the hash width so+ /// every shard in `plan` is reachable.+ pub fn install_shard_plan(&self, index: &str, plan: &[xerj_cluster::ShardAssignment]) {+ if plan.is_empty() {+ return;+ }+ let mut router = self.shard_router.write();+ let width = plan+ .iter()+ .map(|assignment| assignment.shard.saturating_add(1))+ .max()+ .unwrap_or(1);+ if width > router.num_shards() {+ router.set_num_shards(width);+ }+ router.clear_index(index);+ for assignment in plan {+ router.assign(index, assignment.shard, &assignment.node_id);+ }+ }++ /// Record one primary without dropping the index's other shards.+ pub fn assign_one_shard(&self, index: &str, shard: u32, node_id: &str) {+ let mut router = self.shard_router.write();+ let width = shard.saturating_add(1);+ if width > router.num_shards() {+ router.set_num_shards(width);+ }+ router.assign(index, shard, node_id);+ }++ /// Drop `index` from the routing table. Called when a delete commits.+ pub fn clear_shard_plan(&self, index: &str) {+ self.shard_router.write().clear_index(index);+ }++ /// Read-only view of the single-node id (`"local"` unless a test+ /// replaced the field). Cluster routing uses [`Engine::routing_node_id`].pub fn local_node_id(&self) -> &str {self.node_id.as_str()}diff --git a/engine/crates/xerj-server/src/main.rs b/engine/crates/xerj-server/src/main.rsindex 2bb8078cd..d1578ddc8 100644--- a/engine/crates/xerj-server/src/main.rs+++ b/engine/crates/xerj-server/src/main.rs@@ -4173,7 +4173,13 @@ async fn async_main() -> Result<()> {engine: std::sync::Arc::clone(&state.engine),}));let node_id = node.raft.id.clone();- state.engine.attach_cluster(node.handle());+ let handle = node.handle();+ state.engine.attach_cluster(handle.clone());+ // Document and search forwarding. Installed before the runner and+ // before HTTP accept, so a committed shard plan can be served.+ handle.set_shard_handler(std::sync::Arc::new(+ xerj_api::cluster_shard::ApiShardHandler::new(state.clone()),+ ));let mut runner = ClusterRunner::new(node, tick, shutdown_rx);info!(node = %node_id, "Starting cluster runner (Raft mode)");tokio::spawn(async move { runner.run().await });diff --git a/landing/docs/clustering.html b/landing/docs/clustering.htmlindex d1ff0cb6d..9deeeb90d 100644--- a/landing/docs/clustering.html+++ b/landing/docs/clustering.html@@ -234,16 +234,17 @@<h1>Clustering</h1>- <p>XERJ ships an <strong>embedded Raft implementation</strong> — no etcd, no ZooKeeper, no Consul. One binary is still one binary; cluster mode is a config switch, not a separate process. <strong>Where the ring stands today, plainly:</strong> leader election, heartbeats, and the authenticated inter-node transport are real, and <strong>index metadata</strong> (create, delete, mapping, and settings) is proposed through the Raft log and applied on every member. Documents are still node-local. A document indexed on one node is not visible on another (public tracking for the rest of the wiring: <a href="https://github.com/xerj-org/xerj/issues/1170">xerj-org/xerj#1170</a>). If you need a second live copy of your data today, use the <a href="/docs/config#wal_tap">WAL tap</a> or per-node backups.</p>+ <p>XERJ ships an <strong>embedded Raft implementation</strong> — no etcd, no ZooKeeper, no Consul. One binary is still one binary; cluster mode is a config switch, not a separate process. <strong>Where the ring stands today, plainly:</strong> leader election, heartbeats, and the authenticated inter-node transport are real. <strong>Index metadata</strong> (create, delete, mapping, and settings) and the <strong>primary shard plan</strong> for that index are proposed through the Raft log and applied on every member. A document written with <code class="inline">PUT</code>, <code class="inline">POST /{index}/_doc</code>, or <code class="inline">PUT /{index}/_create</code> is stored on the member that owns its shard, whichever node accepted the call. <code class="inline">GET</code> and <code class="inline">DELETE /{index}/_doc/{id}</code> follow the same owner. <code class="inline">GET</code> or <code class="inline">POST /{index}/_search</code> fans out to every owner and merges hits. There is still one copy of each document — no replicas — and a node that missed the log does not catch up (public tracking: <a href="https://github.com/xerj-org/xerj/issues/1170">xerj-org/xerj#1170</a>). For a second live copy, use the <a href="/docs/config#wal_tap">WAL tap</a> or per-node backups.</p><h2>What cluster mode does not give you yet</h2><div class="enum-list">- <div class="item">No document replication<span class="dsc">The index exists on every member after create. The documents do not. <code class="inline">PUT n1/idx/_doc/1</code> then <code class="inline">GET n2/idx/_doc/1</code> is <code class="inline">404</code>. <code class="inline">GET n2/idx</code> is <code class="inline">200</code>.</span></div>- <div class="item">No shard routing<span class="dsc">No forwarding to a primary, no replica reads. Each node's API serves that node's storage only. <code class="inline">Engine::route_write</code> is unchanged.</span></div>- <div class="item">No snapshot catch-up<span class="dsc">A node that missed a metadata entry does not install it later. The server's Raft log is in memory; restart keeps indices already applied in that node's data directory and does not replay entries it never saw.</span></div>+ <div class="item">No replicas and no quorum<span class="dsc">Each shard has one primary. <code class="inline">index.number_of_replicas</code> is stored as a setting and does not place a second copy. The write is acknowledged when that owner has indexed it. If the owner is unreachable the caller gets <code class="inline">503</code> and the document is not stored on the node that accepted the request.</span></div>+ <div class="item">Single-document routing only<span class="dsc"><code class="inline">_bulk</code>, <code class="inline">_update</code>, update-by-query, and the native <code class="inline">/v1</code> ingest paths still write on the node that received them. They do not consult the shard plan.</span></div>+ <div class="item">Search merges hits, not every clause<span class="dsc">A query, including <code class="inline">match_all</code>, is run on each owner and the hits are merged by the per-shard score (not a global IDF), then <code class="inline">from</code>/<code class="inline">size</code> is applied. Aggregations, suggest, collapse, rescore, profile, <code class="inline">search_after</code>, and scroll are refused with <code class="inline">400</code> when the index has more than one owner — the response is not a partial merge. <code class="inline">/_search</code> with no index, and a comma-separated or wildcard index list, still search only the node that received them. The per-shard window is capped at 10,000 hits.</span></div>+ <div class="item">No snapshot catch-up<span class="dsc">A node that missed a metadata entry does not install it later. The server's Raft log is in memory. Restart keeps indices and documents already stored in that node's data directory and does not replay the log, so the shard plan is gone and later writes are not routed. Documents that lived only on a member that is gone are gone with it.</span></div><div class="item">No node-lifecycle APIs<span class="dsc">There is no drain / activate / remove endpoint and no region split-merge. Earlier revisions of this page documented them; the binary has never shipped them.</span></div></div>- <p>Treat a clustered node's <em>document</em> durability exactly like a single node's. Index metadata is the exception, and only while a leader can commit.</p>+ <p>A clustered document has one home. Losing that member loses the documents it owned. Index metadata is on every member that applied the log, and only while a leader can commit.</p><h2>What you get</h2><ul>@@ -251,7 +252,9 @@<li><strong>Authenticated transport</strong> — every inter-node connection is HMAC-SHA256 over a per-connection challenge using the cluster secret. There is no unauthenticated mode; the node refuses to start without a secret.</li><li><strong>Dead-peer isolation</strong> — a hung or dead peer costs one <code class="inline">WARN</code> on the transition and a retry ladder backing off 100 ms → 6.4 s; it never delays messages to live peers.</li><li><strong>Index metadata replication</strong> — <code class="inline">PUT /{index}</code>, <code class="inline">DELETE /{index}</code>, <code class="inline">PUT /{index}/_mapping</code>, and <code class="inline">PUT /{index}/_settings</code> are proposed on the member that received them. A follower forwards to the leader. The HTTP call succeeds only after the entry commits on a majority and this node has applied it. No elected leader, or a commit that does not land in time, is <code class="inline">503</code> and the change is not applied on the caller. With <code class="inline">[cluster] enabled = false</code> these calls stay local, as before.</li>- <li><strong>In-memory Raft log in the server</strong> — the on-disk log type exists and is tested, and the server does not open it. Applied indices live in each node's data directory, so a restart still has the indices it already applied. It does not catch up on a metadata change it missed.</li>+ <li><strong>Primary shard plan</strong> — the create entry names one primary per member (member ids sorted, shard <code class="inline">i</code> on the <code class="inline">i</code>th). <code class="inline">Engine::route_write</code> returns that owner, or nothing when this node is the owner or the index has no plan. Document id is the routing key (jump consistent hash).</li>+ <li><strong>Document routing</strong> — <code class="inline">PUT</code>/<code class="inline">POST /{index}/_doc</code>, <code class="inline">PUT /{index}/_create/{id}</code>, <code class="inline">GET</code> and <code class="inline">DELETE /{index}/_doc/{id}</code> run on the primary. A search of one concrete index fans out to every primary and merges hits. The owner is reached on the cluster port (authenticated, one request and one response per connection), not by writing the document into the Raft log.</li>+ <li><strong>In-memory Raft log in the server</strong> — the on-disk log type exists and is tested, and the server does not open it. Applied indices and the documents a member owned live in that node's data directory, so a restart still has those bytes. The shard plan does not. It lives in the log entry and in each process's routing table, and neither is reloaded. After a restart, <code class="inline">route_write</code> treats the index as unassigned and a new write stays on the node that accepted it. Creating the index again does not rebuild the plan: the index already exists. Deleting it rebuilds a plan and deletes the documents.</li></ul><h2>Bootstrapping a 3-node cluster</h2>@@ -316,16 +319,17 @@ INFO xerj_cluster::raft: Became leader node=10.0.0.12:9300 term=1</pre>"green"3</pre>- <h2>What replicates, and what stays on the node that wrote it</h2>- <p><code class="inline">PUT /orders</code> on any member — leader or follower — creates <code class="inline">orders</code> on every member, including its mapping and the settings and aliases in that request. <code class="inline">GET /orders</code> and <code class="inline">GET /orders/_mapping</code> on another member succeed after the call returns. <code class="inline">DELETE /orders</code> removes it on every member. A settings or mapping change is one log entry per concrete index. A follower that cannot reach a leader returns <code class="inline">503</code>; it does not create the index locally and report success.</p>- <p>Several things around that create stay on the accepting node, on purpose for this step:</p>+ <h2>What replicates, and what stays on one node</h2>+ <p><code class="inline">PUT /orders</code> on any member — leader or follower — creates <code class="inline">orders</code> on every member, including its mapping, the settings and aliases in that request, and a primary on each member. <code class="inline">GET /orders</code> and <code class="inline">GET /orders/_mapping</code> on another member succeed after the call returns. <code class="inline">DELETE /orders</code> removes it on every member. A settings or mapping change is one log entry per concrete index. A follower that cannot reach a leader returns <code class="inline">503</code>; it does not create the index locally and report success.</p>+ <p><code class="inline">PUT /orders/_doc/1</code> on any member stores the document on the primary for that id. <code class="inline">GET /orders/_doc/1</code> from any other member returns it. <code class="inline">POST /orders/_search</code> from any member returns documents stored on the other primaries. The bytes live on the owner only; the other members forward. A document write that names an index which does not exist yet proposes the create (empty schema, plus the accepting node's templates) and then routes. With no elected leader that write is <code class="inline">503</code> and the index is not created.</p>+ <p>Several things around that path stay local, on purpose for this step:</p><ul>- <li><strong>Documents</strong>, including an index that a document write auto-creates, are local. The replicated create is the explicit index API.</li>+ <li><strong><code class="inline">_bulk</code>, <code class="inline">_update</code>, and the native ingest API</strong> do not route. A bulk body accepted by one member is indexed there, even when the shard plan says another member owns the id.</li><li><strong>Index templates</strong> are not replicated. The accepting node applies its templates once and the effective schema is what the log carries.</li><li><strong>ILM/ISM attach</strong> from <code class="inline">index.lifecycle.name</code> runs on the accepting node only.</li>- <li><strong>Console system indices</strong> created at boot, before the ring is attached, stay local.</li>+ <li><strong>Console system indices</strong> created at boot, before the ring is attached, have no shard plan and stay local.</li></ul>- <p>There is still no shard assignment to other members and no replica set. For a second live copy of your documents, point the <a href="/docs/config#wal_tap">WAL tap</a> at another XERJ or any ES-compatible target: it ships indexed writes near-real-time from the WAL, one-directional, without touching the write path.</p>+ <p>There is still no replica set. For a second live copy of your documents, point the <a href="/docs/config#wal_tap">WAL tap</a> at another XERJ or any ES-compatible target: it ships indexed writes near-real-time from the WAL, one-directional, without touching the write path.</p><h2>Failure modes</h2><div class="enum-list">@@ -336,7 +340,7 @@ INFO xerj_cluster::raft: Became leader node=10.0.0.12:9300 term=1</pre></div><h2>When not to cluster</h2>- <p>When you need copies of documents. Cluster mode now keeps index metadata in step and still does not copy documents, route shards, or catch a rejoining node up. One node avoids network round-trips, needs no quorum, and is trivial to back up; add a WAL tap to a second node if you want a live spare of the data. The default config ships with <code class="inline">[cluster] enabled = false</code>.</p>+ <p>When you need two copies of a document, or a search that aggregates across shards. Cluster mode keeps index metadata in step and places each document on one primary; it does not copy that document, and it does not catch a rejoining node up. One node avoids the forwarding hop, needs no quorum, and is trivial to back up; add a WAL tap to a second node if you want a live spare of the data. The default config ships with <code class="inline">[cluster] enabled = false</code>.</p><p class="source-note">Source · engine/crates/xerj-cluster/src/raft.rs · engine/crates/xerj-cluster/src/transport.rs · engine/crates/xerj-cluster/src/runner.rs</p>@@ -411,7 +415,7 @@ INFO xerj_cluster::raft: Became leader node=10.0.0.12:9300 term=1</pre><span><a href="/">HOME</a> · <a href="/product">PRODUCT</a> · <a href="/brand">BRAND</a> · <a href="/playground">DASHBOARDS</a> · <a href="/llms.txt">LLMS.TXT</a></span></footer>-<script type="application/json" id="docs-index">[{"title":"Corpus Hub","slug":"/docs/corpus-hub","kicker":"CORPUS HUB","snip":"Ask a pinned, signed, pre-indexed corpus: xerj ask, xerj hub pull/sync, the MCP hub tools, and how to contribute a corpus."},{"title":"Overview","slug":"/docs/","kicker":"START","snip":""},{"title":"Quickstart","slug":"/docs/quickstart","kicker":"START","snip":""},{"title":"Install","slug":"/docs/install","kicker":"START","snip":""},{"title":"Migrate from ES","slug":"/docs/migration-from-es","kicker":"START","snip":""},{"title":"CLI","slug":"/docs/cli","kicker":"REFERENCE","snip":""},{"title":"Config TOML","slug":"/docs/config","kicker":"REFERENCE","snip":""},{"title":"Environment","slug":"/docs/env","kicker":"REFERENCE","snip":""},{"title":"Native REST API","slug":"/docs/api-native","kicker":"REFERENCE","snip":""},{"title":"ES-compat API","slug":"/docs/api-es-compat","kicker":"REFERENCE","snip":""},{"title":"Query types","slug":"/docs/queries","kicker":"DATA MODEL","snip":""},{"title":"Aggregations","slug":"/docs/aggregations","kicker":"DATA MODEL","snip":""},{"title":"Analyzers","slug":"/docs/analyzers","kicker":"DATA MODEL","snip":""},{"title":"Vectors & kNN","slug":"/docs/vectors","kicker":"DATA MODEL","snip":""},{"title":"Ingest pipelines","slug":"/docs/ingest","kicker":"ENGINE","snip":""},{"title":"Storage & WAL","slug":"/docs/storage","kicker":"ENGINE","snip":""},{"title":"Compression","slug":"/docs/compression","kicker":"ENGINE","snip":""},{"title":"Clustering","slug":"/docs/clustering","kicker":"ENGINE","snip":""},{"title":"Metrics","slug":"/docs/metrics","kicker":"ENGINE","snip":""},{"title":"Auth & TLS","slug":"/docs/security","kicker":"ENGINE","snip":""},{"title":"Running in production","slug":"/docs/operations","kicker":"OPERATIONS","snip":""},{"title":"Backup & restore","slug":"/docs/backup-restore","kicker":"OPERATIONS","snip":""},{"title":"Upgrades","slug":"/docs/upgrades","kicker":"OPERATIONS","snip":""},{"title":"Troubleshooting","slug":"/docs/troubleshooting","kicker":"OPERATIONS","snip":""},{"title":"SIEM","slug":"/docs/playbooks/siem","kicker":"PLAYBOOKS","snip":""},{"title":"Log analytics","slug":"/docs/playbooks/log-analytics","kicker":"PLAYBOOKS","snip":""},{"title":"Vector · RAG","slug":"/docs/playbooks/vector-search","kicker":"PLAYBOOKS","snip":""},{"title":"Full-text search","slug":"/docs/playbooks/full-text","kicker":"PLAYBOOKS","snip":""},{"title":"Observability","slug":"/docs/playbooks/observability","kicker":"PLAYBOOKS","snip":""},{"title":"CLI · --config","slug":"/docs/cli#config-flag","kicker":"CLI","snip":"-c, --config <PATH> Path to the TOML config file."},{"title":"CLI · --data-dir","slug":"/docs/cli#data-dir-flag","kicker":"CLI","snip":"-d, --data-dir <PATH> Override [server] data_dir at the command line."},{"title":"CLI · --insecure","slug":"/docs/cli#insecure-flag","kicker":"CLI","snip":"-k, --insecure Disable TLS and API-key auth. Dev only."},{"title":"[server] rest_port","slug":"/docs/config#server","kicker":"CONFIG","snip":"u16 · default 8080 · Native REST API listener port."},{"title":"[server] es_compat_port","slug":"/docs/config#server","kicker":"CONFIG","snip":"u16 · default 9200 · Elasticsearch-wire-format port. 0 = OS-assigned port."},{"title":"[storage] wal_sync","slug":"/docs/config#storage","kicker":"CONFIG","snip":"enum · default batched · sync · batched · async. Durability knob."},{"title":"[vector] hnsw_ef_search","slug":"/docs/config#vector","kicker":"CONFIG","snip":"u32 · default 100 · Accepted but not wired — the beam width comes from the request num_candidates."},{"title":"[vector] default_quantization","slug":"/docs/config#vector","kicker":"CONFIG","snip":"enum · none (default) · scalar8. binary is refused at startup; scalar4 is not accepted."},{"title":"GET /v1/health","slug":"/docs/api-native#health","kicker":"NATIVE API","snip":"Liveness + readiness. {status, uptime_s, version}."},{"title":"GET /v1/metrics","slug":"/docs/api-native#metrics","kicker":"NATIVE API","snip":"Prometheus text format. Scrape-friendly."},{"title":"POST /v1/indices/:name/turbo-ingest","slug":"/docs/api-native#turbo-ingest","kicker":"NATIVE API","snip":"NDJSON bulk path with parallel tokenization."},{"title":"POST /v1/indices/:name/search","slug":"/docs/api-native#search","kicker":"NATIVE API","snip":"Unified query endpoint — full-text, term, range, KNN, hybrid."},{"title":"POST /:index/_bulk","slug":"/docs/api-es-compat#bulk","kicker":"ES API","snip":"NDJSON bulk. Actions: index, create, update, delete."},{"title":"POST /:index/_search","slug":"/docs/api-es-compat#search","kicker":"ES API","snip":"ES query DSL. Drop-in migration path."},{"title":"match_all","slug":"/docs/queries#match-all","kicker":"QUERY","snip":"Everything in the index."},{"title":"bool","slug":"/docs/queries#bool","kicker":"QUERY","snip":"must / should / must_not / filter clauses."},{"title":"knn","slug":"/docs/queries#knn","kicker":"QUERY","snip":"Vector kNN — HNSW-served when unfiltered (exact-rescored); filtered runs the exact scan."},{"title":"semantic","slug":"/docs/queries#semantic","kicker":"QUERY","snip":"Embed the query text at query time and search the vector field."},{"title":"hybrid","slug":"/docs/queries#hybrid","kicker":"QUERY","snip":"RRF / linear fusion of sub-queries."},{"title":"terms aggregation","slug":"/docs/aggregations#terms","kicker":"AGG","snip":"Exact, pre-computed terms histograms over keyword fields."},{"title":"date_histogram","slug":"/docs/aggregations#date-histogram","kicker":"AGG","snip":"Time-bucketed counts."},{"title":"kNN serving","slug":"/docs/vectors#knn-query","kicker":"VECTORS","snip":"Unfiltered kNN via persisted HNSW + exact rescore; filtered kNN exact scan."},{"title":"SQ8 quantization","slug":"/docs/vectors#quantization","kicker":"VECTORS","snip":"8-bit scalar quantization. int8 scoring precision, not a memory saving (#392)."},{"title":"turbo-ingest","slug":"/docs/ingest#turbo","kicker":"INGEST","snip":"NDJSON bulk path. Parallel tokenization across cores."},{"title":"delta-of-delta","slug":"/docs/compression#delta-of-delta","kicker":"COMPRESSION","snip":"Timestamp encoding. Apache/Nginx/ISO auto-detect."},{"title":"dictionary encoding","slug":"/docs/compression#dictionary","kicker":"COMPRESSION","snip":"Up to 256 unique strings per column."},{"title":"api_key","slug":"/docs/security#api-key","kicker":"SECURITY","snip":"Static admin key. Auto-generated on first run."},{"title":"SIEM playbook","slug":"/docs/playbooks/siem","kicker":"PLAYBOOK","snip":"Terms agg over source IPs for SIEM, with detection rules."},{"title":"RAG playbook","slug":"/docs/playbooks/vector-search","kicker":"PLAYBOOK","snip":"Embeddings, kNN, hybrid search, BM25 + kNN fusion."},{"title":"[cluster] enabled","slug":"/docs/config#cluster","kicker":"CONFIG","snip":"bool · default false · Enable multi-node Raft cluster mode."},{"title":"[cluster] peers","slug":"/docs/config#cluster","kicker":"CONFIG","snip":"array · \"host:port=host:port\" · label must equal address (#1171)."},{"title":"[cluster] port","slug":"/docs/config#cluster","kicker":"CONFIG","snip":"u16 · default 9300 · Intra-cluster Raft + search port."},{"title":"[embedding] default_endpoint","slug":"/docs/config#embedding","kicker":"CONFIG","snip":"string · OpenAI-compatible embeddings URL for auto-embedding on ingest."},{"title":"[embedding] default_model","slug":"/docs/config#embedding","kicker":"CONFIG","snip":"string · Model name (e.g. text-embedding-3-small, nomic-embed-text)."},{"title":"Raft","slug":"/docs/clustering","kicker":"CLUSTER","snip":"Embedded Raft ring — leader election, heartbeats, authenticated transport."},{"title":"Leader failover","slug":"/docs/clustering","kicker":"CLUSTER","snip":"Kill the leader of a 3-node ring: survivors elect a new leader in ~250 ms."},{"title":"Cluster honesty","slug":"/docs/clustering","kicker":"CLUSTER","snip":"What the ring does today; metadata and data replication are roadmap (#1170)."},{"title":"Systemd unit","slug":"/docs/operations","kicker":"OPS","snip":"Production unit file with hardening directives."},{"title":"Readiness probe","slug":"/docs/operations","kicker":"OPS","snip":"GET /v1/health/ready — only 200 once WAL replay is complete."},{"title":"Config reload","slug":"/docs/operations","kicker":"OPS","snip":"SIGHUP reloads most keys without restart."},{"title":"Hot backup","slug":"/docs/backup-restore","kicker":"OPS","snip":"rsync the data dir after POST /v1/admin/flush."},{"title":"S3 backup","slug":"/docs/backup-restore","kicker":"OPS","snip":"POST /v1/admin/backup → any S3-compatible endpoint."},{"title":"Cluster upgrade","slug":"/docs/upgrades","kicker":"OPS","snip":"The cluster wire has no version negotiation — full stop/start, not rolling."},{"title":"Merge pressure","slug":"/docs/troubleshooting","kicker":"OPS","snip":"Segment count climbing → raise merge.max_concurrent + io_rate."},{"title":"Cluster flapping","slug":"/docs/troubleshooting","kicker":"OPS","snip":"Sustained leader churn is a defect — capture RUST_LOG=xerj_cluster=debug and file it."},{"title":"WAL warnings after a crash","slug":"/docs/troubleshooting","kicker":"OPS","snip":"Repaired on startup; no manual repair command."},{"title": "Recipes", "slug": "/docs/recipes/", "kicker": "RECIPES", "snip": "Task-oriented AI & search recipes, each verified end-to-end against a live XERJ."},{"title": "Semantic search & RAG", "slug": "/docs/recipes/semantic-search-rag", "kicker": "RECIPE", "snip": "Retrieval by meaning with semantic_text — auto-embed on ingest, no separate vector DB."},{"title": "Agent memory", "slug": "/docs/recipes/agentic-memory", "kicker": "RECIPE", "snip": "Long-term memory for agents via /_memory — store, recall, forget, per-agent isolation."},{"title": "Passage retrieval", "slug": "/docs/recipes/passage-retrieval", "kicker": "RECIPE", "snip": "Auto-chunk long docs; best-passage (max-sim) scoring — 98% top-3."},{"title": "Vector search · kNN", "slug": "/docs/recipes/vector-search-knn", "kicker": "RECIPE", "snip": "Nearest-neighbor over dense_vector — HNSW-served unfiltered, exact with filters."},{"title": "Vector quantization", "slug": "/docs/recipes/vector-quantization", "kicker": "RECIPE", "snip": "scalar8 (accepts ES int8_hnsw; exact code scan) — int8 scoring at recall@10 = 0.998. Precision, not memory."},{"title": "Hybrid search", "slug": "/docs/recipes/hybrid-search", "kicker": "RECIPE", "snip": "Keyword + vector in one query — RRF fusion."},{"title": "Log analytics", "slug": "/docs/recipes/log-analytics", "kicker": "RECIPE", "snip": "Raw logs to dashboards — error rates, p95 latency, top services."},{"title": "Anomaly detection", "slug": "/docs/recipes/anomaly-detection", "kicker": "RECIPE", "snip": "Statistical _ml detectors that flag spikes in metrics/logs."},{"title": "Anomaly datafeeds", "slug": "/docs/recipes/continuous-anomaly-datafeeds", "kicker": "RECIPE", "snip": "Live _ml datafeed re-scores an index on a timer, stores anomaly records."},{"title": "Migrate from ES", "slug": "/docs/recipes/migrate-from-elasticsearch", "kicker": "RECIPE", "snip": "Point your existing ES client at XERJ — same wire, change the URL."}]</script>+<script type="application/json" id="docs-index">[{"title":"Corpus Hub","slug":"/docs/corpus-hub","kicker":"CORPUS HUB","snip":"Ask a pinned, signed, pre-indexed corpus: xerj ask, xerj hub pull/sync, the MCP hub tools, and how to contribute a corpus."},{"title":"Overview","slug":"/docs/","kicker":"START","snip":""},{"title":"Quickstart","slug":"/docs/quickstart","kicker":"START","snip":""},{"title":"Install","slug":"/docs/install","kicker":"START","snip":""},{"title":"Migrate from ES","slug":"/docs/migration-from-es","kicker":"START","snip":""},{"title":"CLI","slug":"/docs/cli","kicker":"REFERENCE","snip":""},{"title":"Config TOML","slug":"/docs/config","kicker":"REFERENCE","snip":""},{"title":"Environment","slug":"/docs/env","kicker":"REFERENCE","snip":""},{"title":"Native REST API","slug":"/docs/api-native","kicker":"REFERENCE","snip":""},{"title":"ES-compat API","slug":"/docs/api-es-compat","kicker":"REFERENCE","snip":""},{"title":"Query types","slug":"/docs/queries","kicker":"DATA MODEL","snip":""},{"title":"Aggregations","slug":"/docs/aggregations","kicker":"DATA MODEL","snip":""},{"title":"Analyzers","slug":"/docs/analyzers","kicker":"DATA MODEL","snip":""},{"title":"Vectors & kNN","slug":"/docs/vectors","kicker":"DATA MODEL","snip":""},{"title":"Ingest pipelines","slug":"/docs/ingest","kicker":"ENGINE","snip":""},{"title":"Storage & WAL","slug":"/docs/storage","kicker":"ENGINE","snip":""},{"title":"Compression","slug":"/docs/compression","kicker":"ENGINE","snip":""},{"title":"Clustering","slug":"/docs/clustering","kicker":"ENGINE","snip":""},{"title":"Metrics","slug":"/docs/metrics","kicker":"ENGINE","snip":""},{"title":"Auth & TLS","slug":"/docs/security","kicker":"ENGINE","snip":""},{"title":"Running in production","slug":"/docs/operations","kicker":"OPERATIONS","snip":""},{"title":"Backup & restore","slug":"/docs/backup-restore","kicker":"OPERATIONS","snip":""},{"title":"Upgrades","slug":"/docs/upgrades","kicker":"OPERATIONS","snip":""},{"title":"Troubleshooting","slug":"/docs/troubleshooting","kicker":"OPERATIONS","snip":""},{"title":"SIEM","slug":"/docs/playbooks/siem","kicker":"PLAYBOOKS","snip":""},{"title":"Log analytics","slug":"/docs/playbooks/log-analytics","kicker":"PLAYBOOKS","snip":""},{"title":"Vector · RAG","slug":"/docs/playbooks/vector-search","kicker":"PLAYBOOKS","snip":""},{"title":"Full-text search","slug":"/docs/playbooks/full-text","kicker":"PLAYBOOKS","snip":""},{"title":"Observability","slug":"/docs/playbooks/observability","kicker":"PLAYBOOKS","snip":""},{"title":"CLI · --config","slug":"/docs/cli#config-flag","kicker":"CLI","snip":"-c, --config <PATH> Path to the TOML config file."},{"title":"CLI · --data-dir","slug":"/docs/cli#data-dir-flag","kicker":"CLI","snip":"-d, --data-dir <PATH> Override [server] data_dir at the command line."},{"title":"CLI · --insecure","slug":"/docs/cli#insecure-flag","kicker":"CLI","snip":"-k, --insecure Disable TLS and API-key auth. Dev only."},{"title":"[server] rest_port","slug":"/docs/config#server","kicker":"CONFIG","snip":"u16 · default 8080 · Native REST API listener port."},{"title":"[server] es_compat_port","slug":"/docs/config#server","kicker":"CONFIG","snip":"u16 · default 9200 · Elasticsearch-wire-format port. 0 = OS-assigned port."},{"title":"[storage] wal_sync","slug":"/docs/config#storage","kicker":"CONFIG","snip":"enum · default batched · sync · batched · async. Durability knob."},{"title":"[vector] hnsw_ef_search","slug":"/docs/config#vector","kicker":"CONFIG","snip":"u32 · default 100 · Accepted but not wired — the beam width comes from the request num_candidates."},{"title":"[vector] default_quantization","slug":"/docs/config#vector","kicker":"CONFIG","snip":"enum · none (default) · scalar8. binary is refused at startup; scalar4 is not accepted."},{"title":"GET /v1/health","slug":"/docs/api-native#health","kicker":"NATIVE API","snip":"Liveness + readiness. {status, uptime_s, version}."},{"title":"GET /v1/metrics","slug":"/docs/api-native#metrics","kicker":"NATIVE API","snip":"Prometheus text format. Scrape-friendly."},{"title":"POST /v1/indices/:name/turbo-ingest","slug":"/docs/api-native#turbo-ingest","kicker":"NATIVE API","snip":"NDJSON bulk path with parallel tokenization."},{"title":"POST /v1/indices/:name/search","slug":"/docs/api-native#search","kicker":"NATIVE API","snip":"Unified query endpoint — full-text, term, range, KNN, hybrid."},{"title":"POST /:index/_bulk","slug":"/docs/api-es-compat#bulk","kicker":"ES API","snip":"NDJSON bulk. Actions: index, create, update, delete."},{"title":"POST /:index/_search","slug":"/docs/api-es-compat#search","kicker":"ES API","snip":"ES query DSL. Drop-in migration path."},{"title":"match_all","slug":"/docs/queries#match-all","kicker":"QUERY","snip":"Everything in the index."},{"title":"bool","slug":"/docs/queries#bool","kicker":"QUERY","snip":"must / should / must_not / filter clauses."},{"title":"knn","slug":"/docs/queries#knn","kicker":"QUERY","snip":"Vector kNN — HNSW-served when unfiltered (exact-rescored); filtered runs the exact scan."},{"title":"semantic","slug":"/docs/queries#semantic","kicker":"QUERY","snip":"Embed the query text at query time and search the vector field."},{"title":"hybrid","slug":"/docs/queries#hybrid","kicker":"QUERY","snip":"RRF / linear fusion of sub-queries."},{"title":"terms aggregation","slug":"/docs/aggregations#terms","kicker":"AGG","snip":"Exact, pre-computed terms histograms over keyword fields."},{"title":"date_histogram","slug":"/docs/aggregations#date-histogram","kicker":"AGG","snip":"Time-bucketed counts."},{"title":"kNN serving","slug":"/docs/vectors#knn-query","kicker":"VECTORS","snip":"Unfiltered kNN via persisted HNSW + exact rescore; filtered kNN exact scan."},{"title":"SQ8 quantization","slug":"/docs/vectors#quantization","kicker":"VECTORS","snip":"8-bit scalar quantization. int8 scoring precision, not a memory saving (#392)."},{"title":"turbo-ingest","slug":"/docs/ingest#turbo","kicker":"INGEST","snip":"NDJSON bulk path. Parallel tokenization across cores."},{"title":"delta-of-delta","slug":"/docs/compression#delta-of-delta","kicker":"COMPRESSION","snip":"Timestamp encoding. Apache/Nginx/ISO auto-detect."},{"title":"dictionary encoding","slug":"/docs/compression#dictionary","kicker":"COMPRESSION","snip":"Up to 256 unique strings per column."},{"title":"api_key","slug":"/docs/security#api-key","kicker":"SECURITY","snip":"Static admin key. Auto-generated on first run."},{"title":"SIEM playbook","slug":"/docs/playbooks/siem","kicker":"PLAYBOOK","snip":"Terms agg over source IPs for SIEM, with detection rules."},{"title":"RAG playbook","slug":"/docs/playbooks/vector-search","kicker":"PLAYBOOK","snip":"Embeddings, kNN, hybrid search, BM25 + kNN fusion."},{"title":"[cluster] enabled","slug":"/docs/config#cluster","kicker":"CONFIG","snip":"bool · default false · Enable multi-node Raft cluster mode."},{"title":"[cluster] peers","slug":"/docs/config#cluster","kicker":"CONFIG","snip":"array · \"host:port=host:port\" · label must equal address (#1171)."},{"title":"[cluster] port","slug":"/docs/config#cluster","kicker":"CONFIG","snip":"u16 · default 9300 · Intra-cluster Raft + search port."},{"title":"[embedding] default_endpoint","slug":"/docs/config#embedding","kicker":"CONFIG","snip":"string · OpenAI-compatible embeddings URL for auto-embedding on ingest."},{"title":"[embedding] default_model","slug":"/docs/config#embedding","kicker":"CONFIG","snip":"string · Model name (e.g. text-embedding-3-small, nomic-embed-text)."},{"title":"Raft","slug":"/docs/clustering","kicker":"CLUSTER","snip":"Embedded Raft ring — leader election, heartbeats, authenticated transport."},{"title":"Leader failover","slug":"/docs/clustering","kicker":"CLUSTER","snip":"Kill the leader of a 3-node ring: survivors elect a new leader in ~250 ms."},{"title":"Cluster honesty","slug":"/docs/clustering","kicker":"CLUSTER","snip":"Metadata and primary shard routing are live. No replicas, no bulk routing, no snapshot catch-up (#1170)."},{"title":"Systemd unit","slug":"/docs/operations","kicker":"OPS","snip":"Production unit file with hardening directives."},{"title":"Readiness probe","slug":"/docs/operations","kicker":"OPS","snip":"GET /v1/health/ready — only 200 once WAL replay is complete."},{"title":"Config reload","slug":"/docs/operations","kicker":"OPS","snip":"SIGHUP reloads most keys without restart."},{"title":"Hot backup","slug":"/docs/backup-restore","kicker":"OPS","snip":"rsync the data dir after POST /v1/admin/flush."},{"title":"S3 backup","slug":"/docs/backup-restore","kicker":"OPS","snip":"POST /v1/admin/backup → any S3-compatible endpoint."},{"title":"Cluster upgrade","slug":"/docs/upgrades","kicker":"OPS","snip":"The cluster wire has no version negotiation — full stop/start, not rolling."},{"title":"Merge pressure","slug":"/docs/troubleshooting","kicker":"OPS","snip":"Segment count climbing → raise merge.max_concurrent + io_rate."},{"title":"Cluster flapping","slug":"/docs/troubleshooting","kicker":"OPS","snip":"Sustained leader churn is a defect — capture RUST_LOG=xerj_cluster=debug and file it."},{"title":"WAL warnings after a crash","slug":"/docs/troubleshooting","kicker":"OPS","snip":"Repaired on startup; no manual repair command."},{"title": "Recipes", "slug": "/docs/recipes/", "kicker": "RECIPES", "snip": "Task-oriented AI & search recipes, each verified end-to-end against a live XERJ."},{"title": "Semantic search & RAG", "slug": "/docs/recipes/semantic-search-rag", "kicker": "RECIPE", "snip": "Retrieval by meaning with semantic_text — auto-embed on ingest, no separate vector DB."},{"title": "Agent memory", "slug": "/docs/recipes/agentic-memory", "kicker": "RECIPE", "snip": "Long-term memory for agents via /_memory — store, recall, forget, per-agent isolation."},{"title": "Passage retrieval", "slug": "/docs/recipes/passage-retrieval", "kicker": "RECIPE", "snip": "Auto-chunk long docs; best-passage (max-sim) scoring — 98% top-3."},{"title": "Vector search · kNN", "slug": "/docs/recipes/vector-search-knn", "kicker": "RECIPE", "snip": "Nearest-neighbor over dense_vector — HNSW-served unfiltered, exact with filters."},{"title": "Vector quantization", "slug": "/docs/recipes/vector-quantization", "kicker": "RECIPE", "snip": "scalar8 (accepts ES int8_hnsw; exact code scan) — int8 scoring at recall@10 = 0.998. Precision, not memory."},{"title": "Hybrid search", "slug": "/docs/recipes/hybrid-search", "kicker": "RECIPE", "snip": "Keyword + vector in one query — RRF fusion."},{"title": "Log analytics", "slug": "/docs/recipes/log-analytics", "kicker": "RECIPE", "snip": "Raw logs to dashboards — error rates, p95 latency, top services."},{"title": "Anomaly detection", "slug": "/docs/recipes/anomaly-detection", "kicker": "RECIPE", "snip": "Statistical _ml detectors that flag spikes in metrics/logs."},{"title": "Anomaly datafeeds", "slug": "/docs/recipes/continuous-anomaly-datafeeds", "kicker": "RECIPE", "snip": "Live _ml datafeed re-scores an index on a timer, stores anomaly records."},{"title": "Migrate from ES", "slug": "/docs/recipes/migrate-from-elasticsearch", "kicker": "RECIPE", "snip": "Point your existing ES client at XERJ — same wire, change the URL."}]</script><script>// IIFE — js-auth navigates between decrypted pages in the same