AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
//! Engine: manages multiple named indices.
use std::path::PathBuf;
use std::sync::Arc;
use std::time::Instant;
use dashmap::DashMap;
use serde::{Deserialize, Serialize};
use serde_json::Value;
use tokio::sync::RwLock;
use tracing::{error, info, warn};
use xerj_common::config::Config;
use xerj_common::types::{IndexName, Schema};
use crate::cluster_state::{
ClusterStateBootStatus, PersistedClusterStateV1, PersistedDataStreamV1,
PersistedIndexTemplateV1, PreparedClusterState, CLUSTER_STATE_V1,
};
use crate::index::{Index, IndexStats};
use crate::{EngineError, Result};
// ── Memory-pressure drain plumbing (issue #948) ─────────────────────────────
//
// The parent breaker used to only REJECT writes (429). Under a workload
// whose caches hold gigabytes of re-hydratable state and whose memtable
// holds raw write payloads, rejection frees nothing: the breaker stays
// engaged, the client retries, and the run never finishes — exactly
// the #948 mailbox ingest (16 GiB cap, 26.5 GiB demand, 118-147 s engaged
// stretches, 3-4 s releases) and the #2238 corpus-index loop (8 GiB auto
// tier, dozens of indices under their own flush line but over the 2 GiB
// summed-memtable budget, indexed count stuck at 0). The sampler requests
// a DRAIN on either latch: flush every index, release rebuildable caches,
// purge allocator pages. The latches do not share a timer. RSS uses
// [`crate::governor::MemoryDrainSchedule`], which doubles its spacing up to
// 15 minutes when a drain does not lower RSS (#1981). The summed-memtable
// latch keeps the fixed 2 s window (#2238): that flush is what lets corpus
// index move, and a stable RSS must not stretch it.
/// Minimum spacing between automatic drains while the breaker stays engaged.
/// The engage edge always fires immediately; repeats are rate-limited so a
/// long pinned stretch drains progressively (caches refill under continued
/// reads) without the drain worker competing with ingest for every tick.
const MEMORY_DRAIN_MIN_INTERVAL: std::time::Duration = std::time::Duration::from_secs(2);
/// Whether this sampler tick should ask the drain worker to run on the fixed
/// window (#948, #2238).
///
/// The sampler passes the summed-memtable latch. A memtable-only trip used to
/// be ignored, so a multi-index ingest that had filled the auto tier's 2 GiB
/// memtable budget kept receiving HTTP 429 with nothing flushed and the
/// indexed count stayed at 0. The engage edge fires immediately; while the
/// latch stays up, repeats wait `min_interval`. Releasing the latch clears
/// the timer so the next engage drains at once.
///
/// The RSS latch does not use this function. A drain that cannot shrink RSS
/// backs off ([`crate::governor::MemoryDrainSchedule`], #1981). Applying that
/// backoff to the memtable latch would stall corpus index while RSS sits
/// still, which is the normal shape of a memtable-only trip.
fn pressure_drain_due(
needs_drain: bool,
last_drain_at: &mut Option<std::time::Instant>,
now: std::time::Instant,
min_interval: std::time::Duration,
) -> bool {
if !needs_drain {
*last_drain_at = None;
return false;
}
let due = match *last_drain_at {
None => true,
Some(at) => now.saturating_duration_since(at) >= min_interval,
};
if due {
*last_drain_at = Some(now);
}
due
}
/// Whether this sample should request one drain for either parent latch.
///
/// RSS and the summed-memtable budget each keep their own clock. A flat RSS
/// backs off ([`crate::governor::MemoryDrainSchedule`]); memtable pressure
/// stays on [`MEMORY_DRAIN_MIN_INTERVAL`], including when the RSS clock is
/// already stretched. One `true` is one request — the channel coalesces a
/// second — so a staggered engage can ask slightly sooner than one window,
/// and neither latch waits out the other's gap.
fn sample_drain_due(
now: std::time::Instant,
rss: u64,
rss_engaged: bool,
memtable_engaged: bool,
rss_drains: &mut crate::governor::MemoryDrainSchedule,
last_memtable_drain_at: &mut Option<std::time::Instant>,
) -> bool {
let rss_due = rss_drains.observe(now, rss, rss_engaged);
let memtable_due = pressure_drain_due(
memtable_engaged,
last_memtable_drain_at,
now,
MEMORY_DRAIN_MIN_INTERVAL,
);
rss_due || memtable_due
}
/// Bounded request channel from the sampler thread to the drain worker.
/// Capacity 1 + try_send ⇒ requests coalesce: one arrives mid-drain is
/// dropped, and the next sampler tick re-requests if still engaged.
static MEMORY_DRAIN_TX: std::sync::OnceLock<std::sync::mpsc::SyncSender<()>> =
std::sync::OnceLock::new();
/// Ask the drain worker to flush, at most once per [`MEMORY_DRAIN_MIN_INTERVAL`].
///
/// The sampler requests a drain on its own clock (`pressure_drain_due`). A
/// write the breaker has just rejected also requests one (#2321): the 429
/// itself does not free the memtable, and waiting for the next sample is how
/// a client that retries immediately spends its patience against a budget
/// nothing is flushing. This clock is separate, so the first rejection is
/// immediate even when the sampler just fired. Later rejections from this
/// path are spaced by the same interval, so a stream of 429s does not queue a
/// drain per bulk. No worker (a library `Engine` that never called
/// [`Engine::spawn_resource_sampler`]) is a no-op; callers that must flush
/// inline use [`Engine::publish_memtables_if_budget_engaged`].
pub(crate) fn request_memory_pressure_drain() {
if !drain_request_due() {
return;
}
if let Some(tx) = MEMORY_DRAIN_TX.get() {
let _ = tx.try_send(());
}
}
fn drain_request_due() -> bool {
use std::sync::atomic::{AtomicU64, Ordering};
static ORIGIN: std::sync::OnceLock<std::time::Instant> = std::sync::OnceLock::new();
// `u64::MAX` is "never requested". Elapsed millis from process start do
// not reach it.
static LAST_MS: AtomicU64 = AtomicU64::new(u64::MAX);
let origin = ORIGIN.get_or_init(std::time::Instant::now);
let now_ms = u64::try_from(origin.elapsed().as_millis()).unwrap_or(u64::MAX);
let prev = LAST_MS.load(Ordering::Relaxed);
let window_ms = u64::try_from(MEMORY_DRAIN_MIN_INTERVAL.as_millis()).unwrap_or(2_000);
if prev != u64::MAX && now_ms.saturating_sub(prev) < window_ms {
return false;
}
LAST_MS.store(now_ms, Ordering::Relaxed);
true
}
/// Spawn attempts (and the first back-off) for the governor's sampler and
/// drain threads. They are started once, at boot; losing one disables the
/// parent memory breaker for the life of the process, so a transient
/// `EAGAIN` is worth riding out.
const GOVERNOR_THREAD_SPAWN_ATTEMPTS: u32 = 3;
const GOVERNOR_THREAD_SPAWN_BACKOFF: std::time::Duration = std::time::Duration::from_millis(50);
/// Start one of the governor's background threads (#1122).
///
/// These used to be `let _ = Builder::spawn(..)`: a failed spawn was dropped
/// on the floor, and the node silently ran with no memory breaker. Now the
/// spawn is retried with back-off through `xerj_common::thread_spawn` (which
/// never panics), and a final failure is logged at ERROR naming the thread
/// and what the node loses. Returns whether the thread is running.
///
/// These threads deliberately do NOT consult the governor's spawn gate
/// (`thread_spawn_safe`). The gate is a soft RSS watermark, not an OS
/// allocation failure, and these are the threads that sample RSS, engage the
/// breaker and drain memory. A node that boots above its watermark (a large
/// WAL replay under a low `XERJ_MAX_PROCESS_MEMORY_MB`) would otherwise have
/// every attempt refused and run with no breaker for the life of the process,
/// which is exactly when the breaker is needed. A real spawn failure still
/// comes back as an `Err` and is logged, not aborted.
fn spawn_governor_thread<F>(name: &str, f: F) -> bool
where
F: FnOnce() + Send + 'static,
{
spawn_governor_thread_using(&xerj_common::thread_spawn::StdSpawn, || true, name, f)
}
fn spawn_governor_thread_using<S, G, F>(spawner: &S, gate: G, name: &str, f: F) -> bool
where
S: xerj_common::thread_spawn::Spawn,
G: Fn() -> bool,
F: FnOnce() + Send + 'static,
{
match xerj_common::thread_spawn::spawn_with_retry_using(
spawner,
gate,
name,
GOVERNOR_THREAD_SPAWN_ATTEMPTS,
GOVERNOR_THREAD_SPAWN_BACKOFF,
f,
) {
Ok(_detached) => true,
Err(failure) => {
tracing::error!(
thread = name,
attempts = GOVERNOR_THREAD_SPAWN_ATTEMPTS,
error = %failure.error,
"could not start a resource-governor thread; the parent memory/disk \
breaker is degraded until restart"
);
false
}
}
}
/// Allocator page purge hook, installed by the binary that owns the global
/// allocator (xerj-server installs a jemalloc epoch+purge closure). Freeing
/// heap is not enough: the RSS breaker keys on resident pages, and jemalloc
/// keeps freed pages resident until they are purged back to the OS. The
/// engine crate itself is allocator-agnostic, so the default is a no-op.
type AllocatorPurgeHook = Arc<dyn Fn() + Send + Sync + 'static>;
static ALLOCATOR_PURGE_HOOK: std::sync::RwLock<Option<AllocatorPurgeHook>> =
std::sync::RwLock::new(None);
/// Install the allocator page-purge hook (see [`ALLOCATOR_PURGE_HOOK`]).
/// Called once by the server binary during boot, before any drain can fire.
pub fn set_allocator_purge_hook(hook: AllocatorPurgeHook) {
let mut slot = ALLOCATOR_PURGE_HOOK
.write()
.unwrap_or_else(|e| e.into_inner());
*slot = Some(hook);
}
/// Spaces breaker-driven cache releases (#2427/#1122). The first engaged
/// tick releases; later ticks within `interval` only flush and purge.
/// `XERJ_BREAKER_CACHE_RELEASE_SECS` sets the interval (default 30 s; `0`
/// releases on every tick, the pre-fix behaviour).
#[derive(Debug, Clone)]
pub struct PressureCacheReleaseGate {
interval: std::time::Duration,
last: Option<std::time::Instant>,
}
impl PressureCacheReleaseGate {
pub const DEFAULT_INTERVAL: std::time::Duration = std::time::Duration::from_secs(30);
pub fn new(interval: std::time::Duration) -> Self {
Self {
interval,
last: None,
}
}
pub fn from_env() -> Self {
let interval = std::env::var("XERJ_BREAKER_CACHE_RELEASE_SECS")
.ok()
.and_then(|v| v.trim().parse::<u64>().ok())
.map(std::time::Duration::from_secs)
.unwrap_or(Self::DEFAULT_INTERVAL);
Self::new(interval)
}
pub fn interval(&self) -> std::time::Duration {
self.interval
}
/// Whether a drain at `now` should release caches; records it if so.
pub fn should_release(&mut self, now: std::time::Instant) -> bool {
let due = self
.last
.is_none_or(|last| now.saturating_duration_since(last) >= self.interval);
if due {
self.last = Some(now);
}
due
}
}
#[cfg(test)]
mod pressure_cache_release_gate_tests {
use super::PressureCacheReleaseGate;
use std::time::{Duration, Instant};
/// #2427/#1122: the breaker requests a drain every sampler tick while
/// engaged. Only the first tick in each interval may release caches.
#[test]
fn engaged_ticks_release_caches_at_most_once_per_interval() {
let mut gate = PressureCacheReleaseGate::new(Duration::from_secs(30));
let t0 = Instant::now();
assert!(gate.should_release(t0), "the first engaged tick releases");
for tick in 1..15u64 {
assert!(
!gate.should_release(t0 + Duration::from_secs(2 * tick)),
"tick at {}s must only flush and purge",
2 * tick
);
}
assert!(gate.should_release(t0 + Duration::from_secs(30)));
assert!(!gate.should_release(t0 + Duration::from_secs(31)));
}
#[test]
fn a_zero_interval_releases_on_every_tick() {
let mut gate = PressureCacheReleaseGate::new(Duration::ZERO);
let t0 = Instant::now();
assert!(gate.should_release(t0));
assert!(gate.should_release(t0));
}
}
/// Run the installed allocator purge, if any. Never panics: a broken hook
/// must not take the drain (or the process) down — purging is an
/// optimisation for RSS return, not a correctness step.
pub(crate) fn purge_allocator_pages() {
let hook = ALLOCATOR_PURGE_HOOK
.read()
.unwrap_or_else(|e| e.into_inner());
if let Some(hook) = hook.as_ref() {
hook();
}
}
/// Privacy-safe identity of the embedding execution contract exposed to
/// remote ingestion clients. The digest is opaque: model paths, provider URLs,
/// credentials, and model names never cross the API boundary.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct EmbeddingExecutionIdentity {
pub version: u32,
pub backend: String,
pub identity_sha256: String,
/// Vector width, but only for the backends where the server actually
/// pins it: `lexical` always emits [`xerj_ai::local::DEFAULT_DIMS`],
/// `onnx-experimental` is constrained to 384 by `validate_onnx_dimensions`,
/// and `neural` reads `hidden_size` out of the model's own `config.json`
/// while content-addressing it (#487) — the same number
/// `NeuralEmbedder::dims` reports, so it is known without loading the
/// model. It stays `None` for a `neural` node whose assets cannot be
/// resolved at all, and for a `proxy` whose endpoint has not confirmed a
/// width yet (#2306). A confirmed proxy reports the probed width. A reader
/// must not treat an absent width as 384.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub dimensions: Option<usize>,
pub semantic_contract: String,
pub resumable: bool,
#[serde(skip_serializing_if = "Option::is_none")]
pub non_resumable_reason: Option<String>,
/// Measured neural-ingest probe. Advisory and omitted when unset.
/// Not part of [`Self::identity_sha256`]: a different machine or a
/// warmer CPU must still resume the same index. Zero is never sent.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub ingest_probe_bytes: Option<u64>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub ingest_probe_passages: Option<u32>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub ingest_probe_millis: Option<u64>,
}
// ── Clustering types ─────────────────────────────────────────────────────────
/// Node identity and cluster membership configuration.
///
/// # Clustering Roadmap
///
/// xerj is currently single-node (single-shard per index). The fields below
/// are the first step toward a distributed cluster:
///
/// 1. **Node identity** — `ClusterConfig` establishes a stable node ID, human
/// name, and cluster name. This is the minimum required to join a cluster.
///
/// 2. **Shard routing** (next)
/// - Each index will support N primary shards, distributed across nodes via
/// consistent hashing of the document ID.
/// - The routing table will be propagated via a Raft-backed cluster state.
///
/// 3. **Leader election** (next)
/// - One node is designated the "master" (cluster coordinator).
/// - `GET /_cluster/health` will report `relocating_shards` / `unassigned_shards`.
///
/// 4. **Replication** (future)
/// - Each primary shard will have M replica shards on different nodes.
/// - Writes propagate via async WAL shipping to replica nodes.
/// - Reads can be served from any replica.
///
/// 5. **Allocation** (future)
/// - `GET /_cluster/allocation/explain` will return real allocation decisions.
/// - Shard rebalancing will be triggered automatically when nodes join/leave.
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ClusterConfig {
/// Stable unique identifier for this node (UUID recommended).
pub node_id: String,
/// Human-readable node name shown in `_cat/nodes`.
pub node_name: String,
/// Logical cluster name; nodes with different names will not join each other.
pub cluster_name: String,
}
impl Default for ClusterConfig {
fn default() -> Self {
Self {
node_id: uuid::Uuid::new_v4().to_string(),
node_name: "xerj-node-1".to_string(),
cluster_name: "xerj".to_string(),
}
}
}
// ── Public types ─────────────────────────────────────────────────────────────
/// Summary information about one index (for the list endpoint).
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct IndexInfo {
pub name: String,
pub doc_count: u64,
pub segment_count: usize,
pub schema_version: u64,
}
/// An index directory that is present on disk but could not be opened.
///
/// Before issue #206 a failed open left only `name → reason` in a private map
/// that nothing but [`Engine::health`] ever read: the index was invisible to
/// `_cat/indices` and `_cluster/state`, `DELETE` answered 404, and the only
/// recovery was to stop the server and edit the data directory by hand. A
/// failed index is now a real, addressable state — it is listed, it carries
/// the open error verbatim, it can be deleted, and it can be retried once the
/// operator has fixed the cause.
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct FailedIndex {
/// Index name, i.e. the directory name under `data_dir`.
pub name: String,
/// The verbatim error from the last open attempt. Never summarised —
/// the storage layer's message already names the file and the fix.
pub reason: String,
/// Wall-clock ms of the first failure (boot, restore, or a failed retry).
pub failed_at_ms: i64,
/// How many explicit retries have been attempted since (0 at first boot).
pub retries: u32,
}
/// Overall engine health.
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct HealthStatus {
pub status: String,
pub index_count: usize,
pub total_docs: u64,
pub version: String,
}
/// Index template — applied when a new index matching the pattern is created.
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct IndexTemplate {
pub index_patterns: Vec<String>,
pub settings: Value,
pub mappings: Value,
pub priority: i32,
}
/// Active scroll context holding all matching hits.
///
/// Each context pins a fully-hydrated `Vec<Hit>` snapshot, so its lifetime
/// must be bounded: pre-rc.4 contexts lived until an explicit
/// `DELETE /_search/scroll` and accumulated forever otherwise (RC4
/// blocker 11). Every context now carries an `expires_at` deadline
/// (from the request's `scroll=…` keep-alive, capped by
/// `Config.search_context.scroll_max_keep_alive_secs`), refreshed on each
/// continuation, enforced on access, and swept in the background.
pub struct ScrollContext {
pub index: String,
/// Each hit paired with the name of the index it actually came from.
/// Pairing is structural on purpose (#414): a multi-index scroll used to
/// store bare hits plus one context-level `index`, so every continuation
/// page stamped the same name onto hits from different indices and
/// `(_index, _id)` stopped being distinct.
pub hits: Vec<(String, xerj_query::executor::Hit)>,
pub position: usize,
pub page_size: usize,
/// Whether to emit `_seq_no`/`_primary_term` on every page. Real ES
/// fixes this at the scroll-opening request and does not re-read it
/// from continuation bodies — the flag governs the whole scroll's
/// lifetime, not each page — so it's captured once here, not re-parsed
/// per continuation.
pub seq_no_primary_term: bool,
/// Whether to emit `_version` on every page. Same once-at-open semantics
/// as `seq_no_primary_term`.
pub version: bool,
/// Whether the opening request suppressed `_source` at the REQUEST level —
/// `"_source": false`, or `stored_fields` implying suppression when
/// `_source` was unspecified. These are constant across the whole snapshot
/// (index-independent), so a single captured boolean is correct. Captured
/// once at open — like `seq_no_primary_term`/`version`, ES fixes the
/// `_source` selection at the scroll-opening request — so the continuation
/// render omits `_source` when this is set, matching the first page
/// (#624/#637).
pub source_disabled: bool,
/// Whether the opening route's first page also applies mapping-level
/// `_source.enabled: false` suppression (which is PER-HIT: a scroll can span
/// indices with divergent `_source` settings). `search_impl` (`_search?scroll=`)
/// does, so the continuation re-derives it per hit from each hit's own index;
/// `search_with_scroll` (`_search_scroll`) does NOT suppress it on its first
/// page, so its continuation must not either, or it would introduce a new
/// first-page/continuation split on that route (#637; that route's own gap is
/// tracked in #659).
pub mapping_source_check: bool,
pub created: Instant,
/// The keep-alive window last requested for this context. A
/// continuation without an explicit `scroll` parameter re-arms the
/// deadline with this same duration (ES keeps the context alive for
/// the previous keep-alive in that case).
pub keep_alive: std::time::Duration,
/// Wall-clock deadline after which the context is dead: continuations
/// return the same 404 as an unknown id, and the background sweeper
/// (or an opportunistic sweep on open) frees the pinned hits.
pub expires_at: Instant,
}
/// Point-in-time search context — snapshots the set of indices and the
/// max seq_no visible at open time so later searches against the PIT
/// ignore docs that arrived after the snapshot.
#[derive(Debug, Clone)]
pub struct PitContext {
/// Indices the PIT was opened against (resolved from wildcard).
pub indices: Vec<String>,
/// Optional index_filter query AST (applied on each participating
/// index — matching it narrows the snapshot).
pub index_filter: Option<Value>,
/// Per-index snapshot of max visible seq_no at open time.
pub index_max_seq: std::collections::HashMap<String, u64>,
pub created: Instant,
/// Wall-clock deadline after which the background sweeper drops
/// this PIT. Computed at open time as `created + keep_alive`.
/// Pre-v0.6.2 PITs had no TTL and accumulated forever — trivial
/// memory leak vector. ES requires `keep_alive`; we default to
/// `Config.pit.default_keep_alive` (5 min) when missing and
/// silently cap at `Config.pit.max_keep_alive` (24 h).
pub expires_at: Instant,
}
/// A data stream backed by one or more time-series indices.
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct DataStream {
pub name: String,
pub backing_indices: Vec<String>,
pub timestamp_field: String,
pub generation: u64,
}
/// A created API key, kept in memory so the key can be re-authenticated by
/// the auth middleware.
///
/// Persisted across restarts (item 6): every mutation rewrites
/// `<data_dir>/api_keys.json` (0600, atomic) and the file is reloaded on boot,
/// so a key minted by `POST /_security/api_key` keeps working after the node
/// restarts — before this, the map was in-memory only and every restart
/// silently invalidated all minted keys (Kibana/agents would 401 until re-set).
/// Serialized as JSON, so the fields must stay `serde`-round-trippable.
///
/// # The secret is a hash (issue #201)
///
/// It used to be the credential itself, in the clear, in a file. 0600 and an
/// atomic rename are the right handling for a secret *while the process owns
/// it*, but a file has more readers than a process has: a backup, a volume
/// snapshot, a container layer, a support bundle, a decommissioned disk. Any
/// one of those handed over every live credential on the node.
///
/// Now only [`crate::secret_hash`]'s salted-SHA-256 digest is stored, and the
/// secret fields are **private**: outside this module the struct cannot be
/// built with a struct literal at all, so the only way to make a record is
/// [`ApiKeyRecord::new`], which hashes. That is deliberate — it makes storing
/// plaintext again a compile error rather than a code-review catch.
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ApiKeyRecord {
/// Caller-supplied key name (informational).
pub name: String,
/// Salted hash of the secret half of the credential (the `api_key` value
/// returned to the caller, i.e. the part after `id:` in the decoded
/// `ApiKey` header). Never the secret itself.
///
/// `#[serde(default)]` so a pre-#201 `api_keys.json` — which has `secret`
/// and no `secret_hash` — still deserializes; [`Engine::load_persisted_api_keys`]
/// then migrates it. Anything here that
/// [`crate::secret_hash::is_usable_hash`] rejects — empty (never migrated), or
/// present but unparseable (truncated write, hand edit, a scheme this
/// build does not know) — means "no usable credential": the load path
/// drops such a record and [`ApiKeyRecord::verify_secret`] denies it if
/// one ever reaches the auth path anyway.
#[serde(default)]
secret_hash: String,
/// Pre-#201 plaintext secret, read only so it can be migrated away.
///
/// Deserialized from the old `secret` field, never written back: the load
/// path hashes it into `secret_hash`, clears this, and rewrites the file,
/// after which the plaintext exists nowhere. `skip_serializing_if` means
/// even a partially-migrated in-memory record never re-emits plaintext.
#[serde(default, rename = "secret", skip_serializing_if = "Option::is_none")]
legacy_plaintext_secret: Option<String>,
/// Creation time in epoch milliseconds.
pub creation_ms: u64,
/// Absolute expiration in epoch milliseconds, or `None` if the key never
/// expires.
pub expiration_ms: Option<u64>,
/// Set once the key has been invalidated (revoked).
pub invalidated: bool,
/// When the key was invalidated, in epoch milliseconds — `None` while the
/// key is live. Reported as `invalidation` by `GET /_security/api_key`
/// (ES stamps the analogous `invalidation_time` on its key doc in the
/// same update that flips `api_key_invalidated`). `#[serde(default)]` so
/// an `api_keys.json` written before this field existed still loads.
#[serde(default)]
pub invalidation_ms: Option<u64>,
/// Index-scoped grants parsed from the `role_descriptors` supplied at
/// creation (`crate::rbac::roles_from_role_descriptors`).
///
/// **Empty means "no grant", not "all grants."** A key with no roles is an
/// *unscoped* legacy credential: it keeps the historical superuser reach
/// over the general ES-compat surface, but it holds no privilege at all on
/// the reserved `.xerj-memory-*` namespace (agent-memory namespaces and
/// second brains), which is the fail-closed half of the per-brain
/// authorization model. A key with a non-empty list is *scoped* and may
/// touch only what these roles allow.
///
/// `#[serde(default)]` so an `api_keys.json` written before this field
/// existed still loads — those keys deserialize as unscoped, which is the
/// safe reading of "was minted when nothing was enforced".
#[serde(default)]
pub roles: Vec<crate::rbac::Role>,
}
impl ApiKeyRecord {
/// Build a live key record from the plaintext secret handed to the caller.
///
/// The plaintext is hashed here and dropped on return — this function is
/// the only way to construct a record, so there is no path by which a
/// secret reaches [`Engine::flush_api_keys`] and therefore the disk.
pub fn new(
name: impl Into<String>,
secret: &str,
creation_ms: u64,
expiration_ms: Option<u64>,
roles: Vec<crate::rbac::Role>,
) -> Self {
Self {
name: name.into(),
secret_hash: crate::secret_hash::hash_secret(secret),
legacy_plaintext_secret: None,
creation_ms,
expiration_ms,
invalidated: false,
invalidation_ms: None,
roles,
}
}
/// Does `presented` match this record's secret?
///
/// Fail-closed on a record that carries no usable hash — never migrated
/// (empty), or present but not one of our encodings.
/// [`Engine::load_persisted_api_keys`] drops those before they can reach
/// here, but if one ever did, "no usable stored hash" must mean "no",
/// never "yes". [`crate::secret_hash::is_usable_hash`] is the discriminator,
/// the same one the load path uses, so the two cannot disagree about what
/// counts as a credential.
pub fn verify_secret(&self, presented: &str) -> bool {
if !crate::secret_hash::is_usable_hash(&self.secret_hash) {
return false;
}
crate::secret_hash::verify_secret(presented, &self.secret_hash)
}
/// Migrate a record loaded from a pre-#201 `api_keys.json`.
///
/// Returns `true` when the record changed and the store must be rewritten.
/// `Err(())` means the record carries no usable credential in either form
/// and must be dropped rather than kept as a key that silently never
/// authenticates.
///
/// "Usable credential" is [`crate::secret_hash::is_usable_hash`], which is
/// a full decode — **not** `!secret_hash.is_empty()`, and not a check of
/// the `$ssha256$` tag either. A `secret_hash` that carries the tag but
/// does not decode (`"$ssha256$truncated"` from a hand edit, a scheme a
/// future build writes and this one cannot read) is denied by every
/// verifier, so a record holding only that can never authenticate.
/// Restoring it would leave a key `GET /_security/api_key` lists as live
/// while nothing can ever use it — the accept-then-ignore shape issue #204
/// tracks, and exactly what dropping the empty case already avoids. Only
/// the two shapes that really are credentials survive:
///
/// * a `secret_hash` that decodes — the post-#201 shape, and the winner
/// whenever both forms are present;
/// * a non-empty `secret` and **no** `secret_hash` at all — the exact
/// pre-#201 shape, which is what migration is for.
///
/// Anything else is dropped rather than repaired. A record carrying both a
/// plaintext and a hash-shaped value that does not decode is a store
/// nobody can explain; deriving a live credential from the half of it that
/// #201 exists to delete is the fail-open reading of an ambiguous record,
/// and this is a credential path.
///
/// The drop is in memory. `load_persisted_api_keys` only rewrites the file
/// when something migrated, so a dropped record normally stays on disk for
/// the operator to inspect after the error log points at it.
fn migrate_from_plaintext(&mut self) -> std::result::Result<bool, ()> {
let plaintext = self.legacy_plaintext_secret.take();
if crate::secret_hash::is_usable_hash(&self.secret_hash) {
// The hash is the credential. A leftover plaintext beside it is
// discarded — dropping it is the whole point, and re-deriving from
// it could silently swap which secret the record authenticates.
// `Some` means the file still held plaintext, so it must be
// rewritten.
return Ok(plaintext.is_some());
}
match plaintext {
Some(plain) if !plain.is_empty() && self.secret_hash.is_empty() => {
self.secret_hash = crate::secret_hash::hash_secret(&plain);
Ok(true)
}
_ => Err(()),
}
}
}
/// Atomically write a **secret** file (API-key store) with owner-only (0600)
/// permissions. The temp file is created 0600 *before* any bytes are written so
/// the secret is never briefly world-readable, then renamed over the target
/// (the rename carries the 0600 inode). On non-unix, perms are left to the OS
/// default. Mirrors `index::write_file_atomic` but hardens the mode.
pub fn write_secret_file_atomic(path: &std::path::Path, bytes: &[u8]) -> std::io::Result<()> {
use std::io::Write as _;
// Unique staging name, for the same reason as `index::write_file_atomic`:
// two concurrent key mints sharing one `api_keys.tmp` can interleave into a
// key store that no longer parses, and a corrupt key store silently drops
// every persisted key at the next boot.
let tmp = crate::index::staging_path(path);
let staged = (|| -> std::io::Result<()> {
let mut opts = std::fs::OpenOptions::new();
opts.write(true).create(true).truncate(true);
#[cfg(unix)]
{
use std::os::unix::fs::OpenOptionsExt as _;
opts.mode(0o600);
}
let mut f = opts.open(&tmp)?;
f.write_all(bytes)?;
f.sync_all()?;
// Belt and braces: pin 0600 even if the platform ignored the open mode.
#[cfg(unix)]
{
use std::os::unix::fs::PermissionsExt as _;
std::fs::set_permissions(&tmp, std::fs::Permissions::from_mode(0o600))?;
}
std::fs::rename(&tmp, path)
})();
// Never leave a staging file behind on failure — it holds key material.
if let Err(e) = staged {
let _ = std::fs::remove_file(&tmp);
return Err(e);
}
if let Some(parent) = path.parent() {
if let Ok(dir) = std::fs::File::open(parent) {
let _ = dir.sync_all();
}
}
Ok(())
}
// ── Engine ────────────────────────────────────────────────────────────────────
/// Top-level engine — manages multiple named indices.
///
/// `Engine` is cheaply clonable (`Arc`-backed) and safe to share across
/// async tasks.
#[derive(Clone)]
pub struct Engine {
config: Arc<Config>,
indices: Arc<DashMap<String, Arc<Index>>>,
data_dir: PathBuf,
/// alias_name → list of index names
pub aliases: Arc<DashMap<String, Vec<String>>>,
/// template_name → IndexTemplate. Persisted in
/// `<data_dir>/cluster_state.json`; mutate only through
/// [`Engine::put_index_template`] / [`Engine::delete_index_template`],
/// or the change will not survive a restart (issue #203).
pub templates: Arc<DashMap<String, IndexTemplate>>,
/// scroll_id → ScrollContext
pub scrolls: Arc<DashMap<String, ScrollContext>>,
/// pipeline_id → pipeline definition JSON. Persisted in
/// `<data_dir>/cluster_state.json`; mutate only through
/// [`Engine::put_pipeline`] / [`Engine::delete_pipeline`], or the change
/// will not survive a restart (issue #203).
pub pipelines: Arc<DashMap<String, Value>>,
/// pipeline_id → why the stored definition has no compiled counterpart.
///
/// Issue #204. A definition that carries a stage type this build does not
/// implement is still stored (Elasticsearch accepts it, and
/// `_ingest/pipeline/{id}/_simulate` still walks it), but nothing may be
/// ingested through it: `process_through_pipeline` refuses with the reason
/// recorded here instead of quietly indexing the untransformed document.
pub unrunnable_pipelines: Arc<DashMap<String, String>>,
/// pipeline_id → the reason a **gate-bearing build refused this exact
/// definition at PUT time**, persisted so a restart cannot overturn the
/// answer the operator already received.
///
/// Issue #204. This is PROVENANCE, and it plays the same role for a
/// pipeline that `analysis-binding.json` plays for an index: its ABSENCE is
/// the load-bearing signal. A definition in `cluster_state.json` with no
/// entry here was written by a build that had no gate, so the boot replay
/// may soften it ([`CompileMode::ReplayPersisted`](xerj_wasm::pipeline::CompileMode)).
/// A definition WITH an entry here was already answered "no" by this
/// contract, and the replay must not resurrect it — without this marker,
/// `ReplayPersisted` cannot tell the two apart, and a plain restart turned a
/// PUT-time refusal back into a `201` that wrote the wrong document
/// (`GET` hands the operator the stored shape, they edit it, PUT it back,
/// get the refusal, restart, and the refusal is gone).
///
/// Persisted in `<data_dir>/ingest-pipeline-refusals.json`, deliberately
/// NOT in `cluster_state.json`: that envelope is a closed format-1 contract
/// with `deny_unknown_fields` (see [`crate::cluster_state`]), so adding a
/// field to it would make every file this build writes unreadable to an
/// older binary. A sidecar an older build simply ignores does not.
///
/// Reference: Lucene records the same kind of fact for the same reason.
/// `SegmentInfos.indexCreatedVersionMajor`
/// (`lucene/core/src/java/org/apache/lucene/index/SegmentInfos.java:172`,
/// "The Lucene version major that was used to create the index") is written
/// once into the commit
/// (`SegmentInfos.java:633`, `out.writeVInt(indexCreatedVersionMajor)`) and
/// then *decides how the bytes are read back* rather than being re-derived:
/// `readCommit` throws `CorruptIndexException` when a segment predates the
/// recorded creation version (`SegmentInfos.java:491-501`), and
/// `readLatestCommit(dir, minSupportedMajor)` refuses a commit whose
/// recorded creation major is below the caller's floor
/// (`SegmentInfos.java:572-578`). The lesson taken here is the shape, not
/// the code: what a *later* reader may assume about persisted bytes has to
/// be recorded WITH them at write time, because it cannot be recovered
/// afterwards. Lucene is Apache-2.0, the same licence as XERJ; no code is
/// reproduced.
pub definition_time_refusals: Arc<DashMap<String, String>>,
/// Set when `ingest-pipeline-refusals.json` exists but could not be read or
/// parsed, so no pipeline's provenance is known.
///
/// Issue #204. "Unknown provenance" is not "no refusal": replaying softly
/// there would resurrect exactly the refusals the marker exists to keep.
/// Every persisted pipeline is replayed at DEFINITION strictness instead —
/// fail closed, and loudly, with the repair named in the reason.
pub pipeline_refusals_unreadable: Arc<std::sync::atomic::AtomicBool>,
/// pipeline_id → configuration the running build cannot honour but that
/// was already persisted by an older one.
///
/// Issue #204. The pipeline RUNS — exactly as the build that stored it ran
/// it — because failing it at boot would turn a working cluster's writes
/// from 201 into 400 on a binary upgrade alone. This map is what stops that
/// compatibility door from being silent: it is populated only by the boot
/// replay ([`Engine::compile_pipeline_with_mode`] with
/// [`CompileMode::ReplayPersisted`](xerj_wasm::pipeline::CompileMode)), and
/// a re-`PUT` of the same definition answers 400 with the same reason.
pub degraded_pipelines: Arc<DashMap<String, String>>,
/// index_name → open/closed state (true = closed)
pub closed_indices: Arc<DashMap<String, bool>>,
/// data stream name → DataStream. Persisted in
/// `<data_dir>/cluster_state.json`; mutate only through the
/// `*_data_stream` methods, or the change will not survive a restart
/// (issue #203).
pub data_streams: Arc<DashMap<String, DataStream>>,
/// ILM policy name → raw policy JSON, kept verbatim for `GET _ilm/policy`
/// round-trip fidelity. The REAL execution engine reads `ism_policies`
/// instead — `PUT _ilm/policy/{name}` writes to BOTH maps (see
/// `xerj_api::es_compat::put_ilm_policy`), translating into the shared
/// internal model. See `crate::lifecycle` for why one engine drives both
/// surfaces.
///
/// This map — the verbatim ILM document — is persisted in
/// `<data_dir>/cluster_state.json`; mutate only through
/// [`Engine::put_ilm_policy`] / [`Engine::delete_ilm_policy`], or the
/// change will not survive a restart (issue #203). The translated
/// `ism_policies` model has its own file (`ism_policies.json`, issue
/// #199) — the two never write each other's file.
pub ilm_policies: Arc<DashMap<String, Value>>,
/// policy id → the internal ISM-shaped model every managed index is
/// actually driven by, regardless of whether the policy was created via
/// `_plugins/_ism/policies` (native) or `_ilm/policy` (translated).
/// Persisted separately in `<data_dir>/ism_policies.json` — see
/// [`Engine::put_ism_policy`].
pub ism_policies: Arc<DashMap<String, crate::lifecycle::LifecyclePolicy>>,
/// index name → lifecycle execution cursor (current state, pending
/// action, timestamps). Presence in this map is what "managed" means —
/// an index with no entry here is not touched by the background job.
pub managed_indices: Arc<DashMap<String, crate::lifecycle::ManagedIndexState>>,
/// Persisted detach tombstones (issue #282, ported from #262): an index
/// name lands here when an operator explicitly detached it — `PUT
/// /{index}/_settings {"index.lifecycle.name": null}`, `POST
/// /{index}/_ilm/remove`, or `POST /_plugins/_ism/remove/{index}`. An
/// acknowledged detach must be authoritative across restarts: mere
/// *absence* from `managed_indices` cannot distinguish "never managed"
/// from "operator said stop", and any future code path that re-derives
/// attachments (e.g. from persisted index settings, which may still
/// carry a stale `index.lifecycle.name`) MUST consult this set first.
/// Cleared by an explicit re-attach or by deleting the index. Persisted
/// in `ism_managed_indices.json` alongside the cursors.
pub lifecycle_detached: Arc<DashMap<String, ()>>,
/// Operator kill switch for lifecycle execution (`POST /_ilm/stop` /
/// `/_ilm/start`): while false, `lifecycle::tick` returns without
/// acting. In-memory only, matching #262's rc — a restart resumes
/// execution, which errs on the side of retention running.
pub lifecycle_running: Arc<std::sync::atomic::AtomicBool>,
/// index name → creation time (epoch ms), for `min_index_age` and for
/// `GET /{index}`'s `creation_date` (previously synthesized as
/// `Utc::now()` on every request — see `record_index_created_at`).
pub index_created_at: Arc<DashMap<String, i64>>,
/// component template name → template JSON. Persisted in
/// `<data_dir>/cluster_state.json`; mutate only through
/// [`Engine::put_component_template`] /
/// [`Engine::delete_component_template`], or the change will not survive
/// a restart (issue #203).
pub component_templates: Arc<DashMap<String, Value>>,
/// snapshot repository name → repo config JSON
pub snapshot_repos: Arc<DashMap<String, Value>>,
/// snapshot repo/snapshot_name → snapshot info JSON
pub snapshots: Arc<DashMap<String, Value>>,
/// cluster-level settings (persistent + transient)
pub cluster_settings: Arc<RwLock<Value>>,
/// enrich policy name → policy JSON
pub enrich_policies: Arc<DashMap<String, Value>>,
/// watcher id → watch definition JSON
pub watches: Arc<DashMap<String, Value>>,
/// script/template id → template source JSON (for _search/template)
pub search_templates: Arc<DashMap<String, Value>>,
/// async search id → stored result JSON
pub async_searches: Arc<DashMap<String, Value>>,
/// Index directories that are present but could not be opened, keyed by
/// index name (health = red). See [`FailedIndex`] — these are inspectable
/// (`GET /_cluster/indices/failed`), deletable (`DELETE /{index}`) and
/// retryable (`POST /_cluster/indices/failed/{index}/_retry`).
pub failed_indices: Arc<DashMap<String, FailedIndex>>,
/// transform id → transform definition JSON
pub transforms: Arc<DashMap<String, Value>>,
/// index_name → frozen state (true = frozen / read-only)
pub frozen_indices: Arc<DashMap<String, bool>>,
/// rollup job id → job definition JSON
pub rollup_jobs: Arc<DashMap<String, Value>>,
/// CCR auto-follow pattern name → pattern JSON
pub ccr_auto_follow: Arc<DashMap<String, Value>>,
/// Single-node WAL tap (issue #320): live config, durable cursors and
/// counters for the one-directional push of allowlisted indices to an
/// external ES-compatible target. Always present; inert until
/// `wal_tap.enabled` and a target URL are set. Not CCR — see
/// [`crate::wal_tap`] for why the two are different features.
pub wal_tap: Arc<crate::wal_tap::WalTap>,
/// API key id → record. Populated by `POST /_security/api_key` so the
/// auth middleware can re-authenticate `Authorization: ApiKey <encoded>`.
/// In-memory only (lost on restart).
pub api_keys: Arc<DashMap<String, ApiKeyRecord>>,
/// `"{application}\0{name}"` → the ES-shaped privilege object
/// (`application`/`name`/`actions`/`metadata`). Populated by
/// `PUT /_security/privilege`, read by `GET /_security/privilege*`.
/// Not enforced (same honest-surface convention as `roles`/`api_keys`'
/// role_descriptors) — Kibana registers its `kibana-.kibana` privileges
/// here at startup so `GET` stops reporting an empty set on every
/// subsequent poll, but nothing actually gates on them yet.
/// In-memory only (lost on restart).
pub application_privileges: Arc<DashMap<String, Value>>,
/// legacy index template name (v1 /_template) → template JSON.
/// Persisted in `<data_dir>/cluster_state.json`; mutate only through
/// [`Engine::put_legacy_template`] / [`Engine::delete_legacy_template`],
/// or the change will not survive a restart (issue #203).
pub legacy_templates: Arc<DashMap<String, Value>>,
/// pipeline_name → compiled, executable Pipeline (typed transform pipeline)
pub transform_pipelines: Arc<DashMap<String, xerj_wasm::pipeline::Pipeline>>,
/// PIT id → PitContext. Records the max seq_no per index at PIT
/// open time so searches using `pit.id: ...` filter out any docs
/// that appeared after the snapshot was taken.
pub pits: Arc<DashMap<String, PitContext>>,
/// index_name → opaque settings blob as last written by the user.
/// Stored as-is so `GET /{index}/_settings` can round-trip what was
/// provided on creation or updated by PUT /{index}/_settings. Keys
/// include `number_of_replicas`, `refresh_interval`, `max_result_window`
/// and anything else ES accepts.
pub index_settings: Arc<DashMap<String, Value>>,
/// index_name → mapping properties JSON, also stored as written so
/// `GET /{index}/_mapping` and `indices.create` round-trip.
pub index_mappings: Arc<DashMap<String, Value>>,
/// index_name → aliases object as seen at create-time (so we can
/// round-trip filter, routing, is_write_index etc. that would
/// otherwise be collapsed to `{}` in the simple alias map).
pub index_alias_metadata: Arc<DashMap<String, Value>>,
/// Slow query log — v0.8 8-P6. Per-process bounded ring buffer of
/// queries that exceeded the configured wall-clock threshold.
pub slow_query: Arc<crate::slow_query::SlowQueryLog>,
/// Tamper-evident audit log — v0.9 9-P4. Hash-chained append-only
/// log of every search / index / delete / admin op.
pub audit: Arc<crate::audit::AuditLog>,
/// Role store — v0.9 9-P2. In-memory map of role name → Role.
/// Wired into auth middleware in v0.9.0-beta.1.
pub roles: Arc<crate::rbac::RoleStore>,
// ── V4 M5.2: cluster routing ────────────────────────────────────────────
/// Local node id (matches `cluster.peers` entry name when clustering
/// is enabled). Used by the write path to decide if a doc belongs
/// to this node or must be forwarded to a peer.
///
/// When clustering is disabled, this is set to `"local"` and every
/// routing decision resolves to "this node" — so the write path is
/// 100 % backward compatible with single-node deployments.
pub node_id: Arc<String>,
/// Shard router shared across all indices. For single-node clusters
/// this has `num_shards = 1` and always routes to `node_id`, which
/// is a no-op. In multi-node mode, populated via
/// `ShardRouter::update_from_metadata` whenever the Raft log
/// commits a new shard assignment.
pub shard_router: Arc<parking_lot::RwLock<xerj_cluster::router::ShardRouter>>,
/// One lock per `(index, shard)` held across a replica copy and the
/// in-sync publish that follows it. Document writes for a shard all run
/// on its primary, so this is what keeps two of those writes from
/// publishing different in-sync sets.
replica_sync: Arc<DashMap<(String, u32), Arc<tokio::sync::Mutex<()>>>>,
/// Exclusive lock on `<data_dir>/node.lock`, held for the engine's
/// whole lifetime (RC4 blocker 13). Acquired in [`Engine::new`] BEFORE
/// any index is opened (i.e. before any WAL replay or segment flush can
/// touch the directory), so a second xerj process pointed at a live data
/// dir fails fast instead of replaying the WAL and flushing duplicate
/// segments into it — the classic systemd double-start corruption.
///
/// The OS lock dies with the process on Unix. A crash on Windows can
/// leave `LockFileEx` stuck on the file; `node_lock` reclaims it when
/// the PID it records is not running, and logs that reclaim. A live
/// holder still blocks a second start.
///
/// Drop unlocks explicitly (`NodeLock`) before close. A `fork` in another
/// thread inherits the descriptor until that child execs; close alone
/// does not release `flock` across that window, so a second engine on
/// this directory would refuse to start after the holder was dropped.
_node_lock: Arc<crate::node_lock::NodeLock>,
/// Serializes rewrites of `<data_dir>/cluster_state.json` (issue #203).
///
/// The atomic rewrite stages through a single fixed path
/// (`cluster_state.tmp`), so two concurrent flushes — two provisioning
/// PUTs arriving together is all it takes — would create/truncate the
/// *same* temp file and rename each other's half-written bytes over the
/// real one. Holding this across snapshot-and-write makes the whole
/// rewrite one critical section; the last writer's snapshot necessarily
/// includes every mutation that had already been applied to the maps, so
/// serializing loses nothing.
///
/// Never taken while holding a `DashMap` guard.
cluster_state_write: Arc<parking_lot::Mutex<()>>,
/// Immutable classification of the one `cluster_state.json` document read
/// after `node.lock` and before any index/WAL open. A blocked status keeps
/// readiness false and refuses every metadata mutation for this process
/// lifetime; recovery requires replacing the bytes and restarting.
///
/// Without this the failure is silent *and* destructive. The maps boot
/// empty, so the first `PUT /_index_template/...` snapshots six empty
/// maps and `write_file_atomic` renames `cluster_state.tmp` over the
/// target — and `rename(2)` needs only write permission on the
/// *directory*, so an unreadable-but-perfectly-intact document is
/// unlinked by a write that answers `{"acknowledged": true}`. Every
/// trigger listed above is transient or trivially repairable *until*
/// xerj overwrites the file, which is the one outcome that is not.
///
/// Refusing is also the honest answer rather than merely the safe one:
/// the node is running without the templates that shape new indices and
/// the pipelines `?pipeline=x` names, so accepting more configuration on
/// top of a config that is silently a subset is the same
/// accepted-and-ignored shape issue #204 tracks.
///
/// Cleared only by a boot that loads cleanly — fix or move the file
/// aside and restart. redb takes the same position after a failed
/// integrity check or a failed I/O: an `AtomicBool` on the backend makes
/// every later write return `StorageError::PreviousIo`, whose message is
/// "Please close and re-open the database"
/// (`redb/src/tree_store/page_store/cached_file.rs:125-145`,
/// `redb/src/error.rs:65`; Apache-2.0/MIT, shape adapted, no code
/// copied).
cluster_state_boot: Arc<ClusterStateBootStatus>,
/// Raft proposer for cluster metadata. `None` unless
/// [`Engine::attach_cluster`] ran, which the server does only after the
/// cluster transport is up. Single-node boots leave this empty and every
/// index create/delete/mapping/settings path stays local.
cluster: Arc<parking_lot::Mutex<Option<xerj_cluster::ClusterHandle>>>,
}
impl Engine {
pub fn segment_hydration_cache_capacities(
&self,
) -> [usize; crate::segment_cache_budget::CATEGORY_COUNT] {
let mut total = [0_usize; crate::segment_cache_budget::CATEGORY_COUNT];
for index in self.indices.iter() {
let capacities = index.value().segment_hydration_cache_capacities();
for (sum, capacity) in total.iter_mut().zip(capacities) {
*sum = sum.saturating_add(capacity);
}
}
total
}
/// Create a new engine, opening any existing indices from disk.
pub fn new(mut config: Config) -> Result<Self> {
// Runtime asset ownership belongs to this Engine, not to the caller's
// clone lineage. This ensures a later Engine re-reads same-path ONNX
// assets while all Index config clones within one Engine share bytes.
config.embedding.runtime_onnx_assets = Arc::new(std::sync::OnceLock::new());
// Same ownership rule for the proxy width cache: this engine probes
// into its own cell, and every index cloned from `config` shares it.
config.embedding.runtime_proxy_width = Arc::new(std::sync::RwLock::new(None));
let data_dir = PathBuf::from(&config.server.data_dir);
std::fs::create_dir_all(&data_dir)?;
// Data-dir exclusivity (RC4 blocker 13): take the node lock BEFORE
// scanning/opening any index below — Index::open replays the WAL
// and can flush segments, which must never happen while another
// process serves the same directory.
let node_lock = Arc::new(crate::node_lock::acquire(&data_dir)?);
// Classify durable cluster metadata before Index::open can replay a
// WAL or startup reconciliation can rewrite it. The parsed document
// is retained and applied later; the path is never reread this boot.
let PreparedClusterState {
status: cluster_state_status,
document: prepared_cluster_state,
} = crate::cluster_state::preflight(&data_dir);
let storage_available = cluster_state_status.is_writable();
// Cloned before `config` is moved into the Arc below — the WAL tap
// owns a live, runtime-editable copy (PUT /_xerj/wal_tap) rather than
// reading the frozen boot config.
let wal_tap_config = config.wal_tap.clone();
// Built HERE, ahead of the struct literal, because `WalTap::new` is
// what loads the persisted runtime configuration and it can override
// `min_retained_generations` — which `store_config_from` reads out of
// this `Config` when every index opens a few lines below. Freezing the
// *file's* floor into the `Arc` while the tap ran with a different one
// is how a floor set through `PUT /_xerj/wal_tap` survived a restart
// in `_stats` and still reached no `WalWriter`.
let wal_tap = Arc::new(crate::wal_tap::WalTap::new(&data_dir, wal_tap_config));
config.wal_tap = wal_tap.config();
// Apply operator-tunable aggregation bucket cap. Stored in a static
// AtomicUsize inside aggs.rs so all per-bucket-allocator hot loops
// can read it with no plumbing through every agg signature.
crate::aggs::set_max_buckets(config.limits.max_buckets);
// Initialise the process-wide resource governor (parent circuit
// breaker) from config: the memtable byte budget, RSS watermark,
// per-query memory guard, global search pool, and disk flood-stage
// block. Idempotent — a second in-process Engine (tests) re-uses the
// first governor. The background sampler that drives its trip flags
// is started by `spawn_resource_sampler`, called once the engine is
// Arc-wrapped.
crate::governor::init(&config);
// Install the engine's pool widths from `engine.{flush,merge,search}
// _workers` before any pool is built. Idempotent, same as the governor
// above: the first engine in a process fixes the widths, and a later
// engine asking for different ones is told its request was ignored
// rather than left to assume it took effect (#240 §4).
crate::pools::init(&config.engine);
let engine = Self {
config: Arc::new(config),
indices: Arc::new(DashMap::new()),
data_dir: data_dir.clone(),
aliases: Arc::new(DashMap::new()),
templates: Arc::new(DashMap::new()),
scrolls: Arc::new(DashMap::new()),
pipelines: Arc::new(DashMap::new()),
unrunnable_pipelines: Arc::new(DashMap::new()),
definition_time_refusals: Arc::new(DashMap::new()),
pipeline_refusals_unreadable: Arc::new(std::sync::atomic::AtomicBool::new(false)),
degraded_pipelines: Arc::new(DashMap::new()),
closed_indices: Arc::new(DashMap::new()),
data_streams: Arc::new(DashMap::new()),
ilm_policies: Arc::new(DashMap::new()),
ism_policies: Arc::new(DashMap::new()),
managed_indices: Arc::new(DashMap::new()),
lifecycle_detached: Arc::new(DashMap::new()),
lifecycle_running: Arc::new(std::sync::atomic::AtomicBool::new(true)),
index_created_at: Arc::new(DashMap::new()),
component_templates: Arc::new(DashMap::new()),
snapshot_repos: Arc::new(DashMap::new()),
snapshots: Arc::new(DashMap::new()),
cluster_settings: Arc::new(RwLock::new(serde_json::json!({
"persistent": {},
"transient": {}
}))),
enrich_policies: Arc::new(DashMap::new()),
watches: Arc::new(DashMap::new()),
search_templates: Arc::new(DashMap::new()),
async_searches: Arc::new(DashMap::new()),
failed_indices: Arc::new(DashMap::new()),
transforms: Arc::new(DashMap::new()),
frozen_indices: Arc::new(DashMap::new()),
rollup_jobs: Arc::new(DashMap::new()),
ccr_auto_follow: Arc::new(DashMap::new()),
wal_tap,
api_keys: Arc::new(DashMap::new()),
application_privileges: Arc::new(DashMap::new()),
legacy_templates: Arc::new(DashMap::new()),
transform_pipelines: Arc::new(DashMap::new()),
pits: Arc::new(DashMap::new()),
index_settings: Arc::new(DashMap::new()),
index_mappings: Arc::new(DashMap::new()),
index_alias_metadata: Arc::new(DashMap::new()),
slow_query: crate::slow_query::SlowQueryLog::new(
crate::slow_query::DEFAULT_SLOW_QUERY_CAPACITY,
crate::slow_query::DEFAULT_SLOW_QUERY_MS,
),
// Issue #201: durable, so the evidence outlives the incident it
// is evidence of. Falls back to in-memory (with a warning) if the
// file cannot be opened — an unwritable audit log must not stop
// the node booting.
// A blocked compatibility-fence boot is a diagnostic shell, not a
// partially-open storage node. Opening the durable audit sink
// rewrites audit.jsonl from its ring during construction, so keep
// even that non-index store memory-only until a supported restart.
audit: if storage_available {
crate::audit::AuditLog::open(
crate::audit::DEFAULT_AUDIT_CAPACITY,
data_dir.join("audit.jsonl"),
)
} else {
crate::audit::AuditLog::new(crate::audit::DEFAULT_AUDIT_CAPACITY)
},
roles: crate::rbac::RoleStore::new(),
// Single-node default: 1 shard, "local" owner. Writes never
// forward; multi-node mode overrides these via the Raft
// commit handler when shard assignments change.
node_id: Arc::new("local".to_string()),
shard_router: Arc::new(parking_lot::RwLock::new(
xerj_cluster::router::ShardRouter::new(1),
)),
replica_sync: Arc::new(DashMap::new()),
_node_lock: node_lock,
cluster_state_write: Arc::new(parking_lot::Mutex::new(())),
cluster_state_boot: Arc::new(cluster_state_status),
cluster: Arc::new(parking_lot::Mutex::new(None)),
};
// A rejected cluster-state document means this binary cannot know the
// storage topology that document describes. Do not open *any* index in
// that state: Index::open replays WALs and may publish a segment. This
// includes `.xerj_*` system indices; treating them as harmless was the
// exact path that mutated `.xerj_dashboards` during the causal A/B.
if storage_available {
// Scan data_dir for existing index directories.
if let Ok(read_dir) = std::fs::read_dir(&data_dir) {
for entry in read_dir.flatten() {
let path = entry.path();
if !path.is_dir() {
continue;
}
// Check if this looks like an index directory (has a WAL subdirectory).
if !path.join("wal").exists() {
continue;
}
let name_str = match path.file_name().and_then(|n| n.to_str()) {
Some(n) => n.to_string(),
None => continue,
};
let index_name = match IndexName::new(&name_str) {
Ok(n) => n,
Err(_) => {
warn!("Skipping directory '{}': not a valid index name", name_str);
continue;
}
};
match Index::open(index_name.clone(), &engine.config, &data_dir) {
Ok(idx) => {
info!(name = name_str.as_str(), "opened existing index");
// #320 — `engine.config.wal_tap` is already the
// tap's effective configuration (see `WalTap::new`
// above), so the store opened with the right
// floor; this makes that true by construction
// rather than by argument, and covers a tap
// reconfigured between two boot-loop iterations.
idx.set_wal_min_retained_generations(engine.wal_retention_floor());
// Restore the raw ES mapping blob (analyzers, formats,
// dims — full fidelity) BEFORE any ingest/query can run,
// so GET /_mapping and mapping-dependent code paths see
// the same mapping as pre-restart. A corrupt blob fails
// the index (#202) rather than serving it with a
// silently reduced mapping.
if let Err(e) = engine.load_persisted_es_mapping(&name_str) {
warn!(name = name_str.as_str(), error = %e, "failed to open index");
engine.record_failed_index(&name_str, e.to_string());
continue;
}
// The index isn't registered yet, so the propagation
// inside load can't find it — set the toggles on the
// local handle instead.
if let Some(m) = engine.index_mappings.get(name_str.as_str()) {
Engine::apply_date_mapping_flags(&idx, m.value());
}
engine.load_or_backfill_index_created_at(&name_str);
// Before the index is visible: a write that lands
// in the same instant must already see the
// persisted default pipeline (#2294).
engine.rehydrate_index_settings_from_disk(&name_str);
engine.indices.insert(name_str, idx);
}
Err(e) => {
warn!(name = name_str.as_str(), error = %e, "failed to open index");
// The mapping blob lives beside the data, not inside
// the store, so it is readable even when the store
// refuses to open. Load it so the metadata surfaces
// (`GET /{index}`, `GET /{index}/_mapping`) can still
// tell the operator what was in the index they are
// trying to recover. Propagation into the (absent)
// handle no-ops. Since #202 this load can itself fail
// (unreadable/unparseable `es_mapping.json`); the index
// is already being quarantined for the store error, so
// the extra failure is reported rather than dropped —
// it tells the operator a second file needs repairing.
if let Err(map_err) = engine.load_persisted_es_mapping(&name_str) {
warn!(
name = name_str.as_str(),
error = %map_err,
"es_mapping.json is also unreadable; the mapping surfaces \
cannot show what this index held"
);
}
engine.record_failed_index(&name_str, e.to_string());
}
}
}
}
// Restore persisted API keys (item 6) so keys minted before a restart
// still authenticate. Must run before the server starts accepting
// requests (i.e. here in `new`), and is cheap (one small JSON file).
engine.load_persisted_api_keys();
// Restore persisted aliases so e.g. `.kibana` (always an alias,
// never a bare index) still resolves after a restart — see
// `load_persisted_aliases`'s doc comment for the concrete failure
// this caused (OpenSearch Dashboards stuck indefinitely on every
// restart, mistaking a missing-alias 404 for a still-in-progress
// migration by another instance).
engine.load_persisted_aliases();
// …and the alias filter metadata that pairs with that topology, or
// a filtered alias comes back unfiltered — wrong reads, and a
// by-query that empties the whole backing index (#524).
engine.load_alias_metadata();
// Restore the cluster-level management state — index templates,
// legacy templates, component templates, ingest pipelines, data
// streams, ILM policies (issue #203). Before this, "replace the
// binary and restart" silently reverted every one of them: the
// documents kept flowing, into indices that no longer had the shape
// the operator designed. Must run after the index scan above, so a
// restored data stream can report which of its backing indices are
// actually present, and before the server accepts requests.
// Issue #204: the refusal markers are read BEFORE the pipelines
// they classify, and unconditionally — not from inside
// `apply_cluster_state`, which returns early when there is no
// `cluster_state.json`. On that path an unreadable sidecar left
// `pipeline_refusals_unreadable` false, so the first refused `PUT`
// would have overwritten the file with a one-entry map and erased
// every marker still in it.
engine.load_persisted_pipeline_refusals();
engine.apply_cluster_state(prepared_cluster_state);
// Then say plainly which `.ds-*` indices no restored stream claims.
// Nothing is deleted — see the doc comment; this is the log line that
// stops an unreachable backing index from being invisible.
engine.warn_orphaned_backing_indices();
// Restore ISM/ILM policies and managed-index execution state so a
// policy attached before a restart keeps running afterward instead
// of silently going idle. Separate files (`ism_policies.json`,
// `ism_managed_indices.json`) from `cluster_state.json` above, and
// deliberately so: that file holds the verbatim documents an
// operator PUT, this holds the executor's own cursor. Loaded after
// `apply_cluster_state` because a restored data stream is what a
// rollover action operates on.
engine.load_persisted_ism_policies();
engine.load_persisted_managed_indices();
// Spawn the PIT sweeper. Pre-v0.6.2 PITs accumulated forever;
// every open without close was a memory leak. The sweeper
// walks `engine.pits` every `pit.sweep_interval_secs` and
// drops any with `expires_at < now`. Cheap (DashMap iter +
// Instant compare) and bounded by the live PIT count.
engine.spawn_pit_sweeper();
// Spawn the scroll + async-search context sweeper (RC4 blocker
// 11) — the scroll/async twin of the PIT sweeper above. Each
// scroll pins a full Vec<Hit> and each async search pins its
// response JSON; without a sweeper both maps grow until an
// explicit client DELETE, i.e. forever under normal client
// behavior.
engine.spawn_search_context_sweeper();
} else {
warn!(
"storage activation skipped because the boot-time cluster-state format fence is latched"
);
}
Ok(engine)
}
/// Drop PIT contexts whose `expires_at` is in the past. Cheap
/// O(N) walk; runs on the background sweeper task and is also
/// invoked opportunistically inside `open_pit` so a tight
/// open-without-close loop self-bounds without waiting for the
/// next sweep tick.
pub fn sweep_expired_pits(&self) -> usize {
let now = Instant::now();
let expired: Vec<String> = self
.pits
.iter()
.filter(|e| e.value().expires_at <= now)
.map(|e| e.key().clone())
.collect();
for id in &expired {
self.pits.remove(id);
}
expired.len()
}
fn spawn_pit_sweeper(&self) {
let pits = Arc::clone(&self.pits);
let interval = std::time::Duration::from_secs(self.config.pit.sweep_interval_secs.max(1));
tokio::spawn(async move {
let mut tick = tokio::time::interval(interval);
// Skip the immediate first tick — Engine::new just ran, so
// the pits map is empty.
tick.tick().await;
loop {
tick.tick().await;
let now = Instant::now();
let expired: Vec<String> = pits
.iter()
.filter(|e| e.value().expires_at <= now)
.map(|e| e.key().clone())
.collect();
if expired.is_empty() {
continue;
}
for id in &expired {
pits.remove(id);
}
tracing::debug!(
swept = expired.len(),
remaining = pits.len(),
"PIT sweep dropped expired contexts",
);
}
});
}
/// Drop scroll contexts whose `expires_at` is in the past. Mirrors
/// [`Engine::sweep_expired_pits`]: cheap O(N) walk, run by the
/// background sweeper and opportunistically before opening a new
/// scroll so a tight open-without-clear loop self-bounds.
pub fn sweep_expired_scrolls(&self) -> usize {
let now = Instant::now();
let expired: Vec<String> = self
.scrolls
.iter()
.filter(|e| e.value().expires_at <= now)
.map(|e| e.key().clone())
.collect();
for id in &expired {
self.scrolls.remove(id);
}
expired.len()
}
/// Drop stored async-search results whose `expiration_time_in_millis`
/// is in the past. The expiry lives inside the stored response JSON
/// (it is part of the ES wire format), so the sweep reads it from
/// there; a malformed/missing field counts as expired — every writer
/// sets it, so that arm only fires on corruption.
pub fn sweep_expired_async_searches(&self) -> usize {
let now_ms = std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.map(|d| d.as_millis() as i64)
.unwrap_or(i64::MAX);
let expired: Vec<String> = self
.async_searches
.iter()
.filter(|e| {
e.value()
.get("expiration_time_in_millis")
.and_then(Value::as_i64)
.map(|exp| exp <= now_ms)
.unwrap_or(true)
})
.map(|e| e.key().clone())
.collect();
for id in &expired {
self.async_searches.remove(id);
}
expired.len()
}
/// Background sweeper for scroll + async-search contexts (RC4
/// blocker 11) — the twin of [`Engine::spawn_pit_sweeper`].
///
/// Deliberately captures ONLY the two context maps (never a full
/// `Engine` clone): the engine holds the data-dir `node.lock`, and a
/// long-lived task owning an `Engine` clone would keep that lock
/// alive after the last user-visible engine is dropped.
fn spawn_search_context_sweeper(&self) {
let scrolls = Arc::clone(&self.scrolls);
let async_searches = Arc::clone(&self.async_searches);
let interval =
std::time::Duration::from_secs(self.config.search_context.sweep_interval_secs.max(1));
tokio::spawn(async move {
let mut tick = tokio::time::interval(interval);
// Skip the immediate first tick — Engine::new just ran, so
// both maps are empty.
tick.tick().await;
loop {
tick.tick().await;
let now = Instant::now();
let expired_scrolls: Vec<String> = scrolls
.iter()
.filter(|e| e.value().expires_at <= now)
.map(|e| e.key().clone())
.collect();
for id in &expired_scrolls {
scrolls.remove(id);
}
let now_ms = std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.map(|d| d.as_millis() as i64)
.unwrap_or(i64::MAX);
let expired_async: Vec<String> = async_searches
.iter()
.filter(|e| {
e.value()
.get("expiration_time_in_millis")
.and_then(Value::as_i64)
.map(|exp| exp <= now_ms)
.unwrap_or(true)
})
.map(|e| e.key().clone())
.collect();
for id in &expired_async {
async_searches.remove(id);
}
if !expired_scrolls.is_empty() || !expired_async.is_empty() {
tracing::debug!(
swept_scrolls = expired_scrolls.len(),
swept_async = expired_async.len(),
remaining_scrolls = scrolls.len(),
remaining_async = async_searches.len(),
"search-context sweep dropped expired contexts",
);
}
}
});
}
/// Create a new index, applying any matching index template.
pub fn create_index(&self, name: &str, schema: Schema) -> Result<()> {
self.create_index_with_settings(name, schema, Value::Null)
}
/// Merge the highest-priority matching index template's declared fields
/// into `schema` (adding any the schema does not already define). Applied
/// on every create path so template behavior is independent of whether the
/// caller supplied settings.
///
/// The cluster proposer calls this once and replicates the effective
/// schema. The applier must not call it again: nodes can have different
/// local templates, and a second pass would diverge.
///
/// `semantic_text` stays a text field with an embedding contract. It used
/// to fall through to `object` here, so a template that matched (its
/// `keyword` fields appeared) still could not embed (#2281). The raw ES
/// mapping and settings are merged separately by
/// [`Engine::merge_index_template`] — this pass only builds the native
/// schema, and `schema_to_es_properties` cannot spell `semantic_text`.
pub(crate) fn apply_index_template(&self, name: &str, schema: Schema) -> Schema {
let mut effective_schema = schema;
let mut new_semantic_targets = Vec::new();
// #2407: a template's bare `semantic_text` is sized at the width this
// server's embedding backend actually produces, not always 384.
// Resolved once per create and only when a bare field appears, so a
// template without one never pays for the proxy width probe (#2306).
let mut confirmed_dimensions: Option<usize> = None;
if let Some(tmpl) = self.best_matching_template(name) {
if let Some(props) = tmpl.mappings.get("properties").and_then(Value::as_object) {
for (field_name, field_def) in props {
if effective_schema
.fields
.iter()
.any(|f| &f.name == field_name)
{
continue;
}
let bare_semantic = field_def.get("type").and_then(Value::as_str)
== Some("semantic_text")
&& field_def.get("dimensions").is_none()
&& field_def.get("dims").is_none();
let semantic_dimensions = if bare_semantic {
*confirmed_dimensions
.get_or_insert_with(|| self.confirmed_semantic_dimensions())
} else {
xerj_ai::local::DEFAULT_DIMS
};
let fc =
field_from_template_property(field_name, field_def, semantic_dimensions);
if let Some(embedding) = fc.embedding.as_ref() {
new_semantic_targets.push(
embedding
.target_field
.clone()
.unwrap_or_else(|| format!("{field_name}_vector")),
);
}
let _ = effective_schema.add_field(fc);
}
}
}
// Only companions of fields this pass added. A caller that already
// built the schema (the ES create handler) has its own companions;
// adding them again would fail the duplicate-name check, and running
// this for every create would change indexes that have no template.
for target in new_semantic_targets {
if effective_schema.fields.iter().any(|f| f.name == target) {
continue;
}
let source = effective_schema.fields.iter().find(|f| {
f.embedding.as_ref().and_then(|e| e.target_field.as_deref())
== Some(target.as_str())
});
let Some(source) = source else {
continue;
};
let mut companion =
xerj_common::types::FieldConfig::new(target, xerj_common::types::FieldType::Vector);
companion.options.dimensions = source.options.dimensions;
companion.options.similarity = source.options.similarity.clone();
let _ = effective_schema.add_field(companion);
}
effective_schema
}
/// Merge the highest-priority matching template into a create body.
///
/// Request mappings and settings win where both sides name the same field
/// or setting. Template values fill the gaps. With no matching template
/// the inputs are returned unchanged.
///
/// Callers that persist the ES mapping blob and `index_settings` must use
/// this result. [`Engine::apply_index_template`] only updates the native
/// schema, which cannot round-trip `semantic_text` or
/// `index.default_pipeline` (#2281).
pub fn merge_index_template(
&self,
index_name: &str,
mappings: Value,
settings: Value,
) -> (Value, Value) {
let Some(tmpl) = self.best_matching_template(index_name) else {
return (mappings, settings);
};
(
merge_template_mappings(&tmpl.mappings, mappings),
merge_template_settings(&tmpl.settings, settings),
)
}
/// `true` when `index` does not exist yet and a matching template declares
/// `semantic_text`. Bulk classifies the embed path before auto-create, so
/// a template-only field would otherwise take the turbo path and never be
/// embedded (#2281).
pub(crate) fn absent_index_template_has_semantic_text(&self, index: &str) -> bool {
if self.indices.contains_key(index) || self.failed_indices.contains_key(index) {
return false;
}
self.best_matching_template(index)
.and_then(|t| t.mappings.get("properties").cloned())
.is_some_and(|props| json_properties_have_semantic_text(&props))
}
/// Create a new index with explicit settings (e.g. custom analysis configuration).
///
/// The `settings` value is stored alongside the index and used to configure
/// the analyzer registry (custom analyzers, synonym filters, ngram tokenizers, etc.).
pub fn create_index_with_settings(
&self,
name: &str,
schema: Schema,
settings: serde_json::Value,
) -> Result<()> {
// Index creation is storage activation: it creates the directory and
// WAL before any cluster-state serializer is involved. A blocked boot
// must reject here so HTTP, gRPC, direct CLI, and Console callers share
// the same fail-closed boundary.
self.ensure_cluster_state_writable()?;
// Same rejection order as before the exact-create split: a bad name
// fails before the visibility and already-exists checks.
let _ = IndexName::new(name).map_err(EngineError::Common)?;
// Per-index authorization backstop (issue #79): a caller that may not
// see this name must not be able to bring it into existence either —
// otherwise it could squat a brain name it cannot read. Reported as a
// create failure rather than a distinct refusal so the two are
// indistinguishable. See `crate::index_guard`.
if !crate::index_guard::visible(name) {
return Err(EngineError::Common(
xerj_common::XerjError::index_not_found(name),
));
}
if self.indices.contains_key(name) {
return Err(EngineError::Common(
xerj_common::XerjError::index_already_exists(name),
));
}
// The name is taken by an index that exists on disk but would not
// open. Creating over it would run `Index::create` across the existing
// store and overwrite `schema.json` with an empty mapping — the very
// mapping loss #202 is about, reached through the other door, and it
// would destroy the evidence too. Refuse here with the recorded reason
// instead (issue #206).
if let Some(f) = self.failed_indices.get(name) {
return Err(EngineError::Common(
xerj_common::XerjError::index_unavailable(name, f.reason.clone()),
));
}
// Apply matching template (highest priority wins) on every create path.
let effective_schema = self.apply_index_template(name, schema);
self.create_index_exact(name, effective_schema, settings)
}
/// Create `name` with `schema` exactly as given.
///
/// Does not apply index templates and does not consult the per-request
/// visibility guard. The single-node create path applies the template and
/// checks visibility first. The cluster applier calls this so every member
/// materialises the schema the accepting node already resolved.
pub(crate) fn create_index_exact(
&self,
name: &str,
schema: Schema,
settings: serde_json::Value,
) -> Result<()> {
self.ensure_cluster_state_writable()?;
let index_name = IndexName::new(name).map_err(EngineError::Common)?;
if self.indices.contains_key(name) {
return Err(EngineError::Common(
xerj_common::XerjError::index_already_exists(name),
));
}
if let Some(f) = self.failed_indices.get(name) {
return Err(EngineError::Common(
xerj_common::XerjError::index_unavailable(name, f.reason.clone()),
));
}
let idx = Index::create_with_settings(
index_name,
schema,
settings,
&self.config,
&self.data_dir,
)?;
// #320 — the store was seeded from the boot `Config`; the tap's live
// floor is what an operator last asked for. Applied before the index
// is published, so no ingest (and therefore no prune) can run against
// the wrong floor.
idx.set_wal_min_retained_generations(self.wal_retention_floor());
self.indices.insert(name.to_string(), idx);
self.record_index_created_at(name);
info!(name, "index created with custom settings");
Ok(())
}
/// Register the raw ES mapping blob for `name` and persist it into the
/// index data dir (atomic temp-file + rename) so `GET /{index}/_mapping`
/// round-trips the exact user-provided mapping (analyzers, date formats,
/// dense_vector dims/similarity, multi-fields) across restarts.
///
/// This is the single write path for `engine.index_mappings` — both
/// index-create-with-mappings and PUT /_mapping go through here.
pub fn put_index_mapping(&self, name: &str, mapping: Value) -> Result<()> {
self.ensure_cluster_state_writable()?;
let index_dir = self.data_dir.join(name);
if index_dir.is_dir() {
match serde_json::to_vec_pretty(&mapping) {
Ok(bytes) => {
if let Err(e) =
crate::index::write_file_atomic(&index_dir.join("es_mapping.json"), &bytes)
{
warn!(index = name, error = %e, "failed to persist es_mapping.json");
}
}
Err(e) => {
warn!(index = name, error = %e, "failed to serialize index mapping for persistence");
}
}
}
self.propagate_date_detection(name, &mapping);
self.index_mappings.insert(name.to_string(), mapping);
Ok(())
}
/// Push the mapping's `date_detection` toggle (default true) down to the
/// open `Index` so dynamic inference honors it. The blob shape varies by
/// caller (`{"date_detection": ..}` from PUT /_mapping, or nested under
/// `"mappings"` from index-create), so both levels are checked.
fn propagate_date_detection(&self, name: &str, mapping: &Value) {
if let Ok(idx) = self.get_index(name) {
Self::apply_date_mapping_flags(&idx, mapping);
}
}
/// Push both date-related mapping toggles down to an open index:
/// the `date_detection` bool and the set of date fields excluded from
/// default-format ingest validation (explicit `format` — those are
/// validated against their own format by the bulk path — or
/// `ignore_malformed`).
pub(crate) fn apply_date_mapping_flags(idx: &crate::index::Index, mapping: &Value) {
idx.set_date_detection(Self::mapping_date_detection(mapping));
idx.set_date_format_exclusions(Self::mapping_date_exclusions(mapping));
}
/// Read the `date_detection` toggle out of a raw mapping blob
/// (defaulting to true, like ES).
pub(crate) fn mapping_date_detection(mapping: &Value) -> bool {
mapping
.get("date_detection")
.or_else(|| {
mapping
.get("mappings")
.and_then(|m| m.get("date_detection"))
})
.and_then(Value::as_bool)
.unwrap_or(true)
}
/// Top-level date fields carrying an explicit `format` or
/// `ignore_malformed` in the raw mapping blob (either shape).
fn mapping_date_exclusions(mapping: &Value) -> std::collections::HashSet<String> {
let props = mapping
.get("properties")
.or_else(|| mapping.get("mappings").and_then(|m| m.get("properties")));
let mut out = std::collections::HashSet::new();
let Some(obj) = props.and_then(Value::as_object) else {
return out;
};
for (fname, spec) in obj {
let ftype = spec.get("type").and_then(Value::as_str).unwrap_or("");
if ftype != "date" && ftype != "date_nanos" {
continue;
}
let has_format = spec
.get("format")
.and_then(Value::as_str)
.is_some_and(|f| !f.is_empty());
let ignore_malformed = spec
.get("ignore_malformed")
.and_then(Value::as_bool)
.unwrap_or(false);
if has_format || ignore_malformed {
out.insert(fname.clone());
}
}
out
}
/// Load a previously-persisted raw ES mapping blob for `name` (if any)
/// into the in-memory `index_mappings` map. Called whenever an index is
/// (re)opened from disk — boot scan and snapshot restore. A missing
/// file is fine (pre-fix indices, dynamic-only indices): readers fall
/// back to schema-derived properties from `schema.json`.
///
/// A file that is present but unreadable or unparseable is **not** fine and
/// is no longer logged-and-ignored (#202). This blob is the full-fidelity
/// mapping — analyzers, date formats, `dense_vector` dims — and is what
/// `GET /{index}/_mapping` answers with; dropping it leaves the index
/// serving a quietly emptier mapping than the one its own data was written
/// under. The caller fails the index instead, which turns cluster health
/// red and keeps the corrupt index out of the served set.
fn load_persisted_es_mapping(&self, name: &str) -> Result<()> {
let path = self.data_dir.join(name).join("es_mapping.json");
let bytes = match std::fs::read(&path) {
Ok(bytes) => bytes,
Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(()),
Err(e) => {
return Err(EngineError::CorruptIndexMetadata {
file: path.display().to_string(),
reason: e.to_string(),
})
}
};
match serde_json::from_slice::<Value>(&bytes) {
Ok(mapping) => {
self.propagate_date_detection(name, &mapping);
self.index_mappings.insert(name.to_string(), mapping);
Ok(())
}
Err(e) => Err(EngineError::CorruptIndexMetadata {
file: path.display().to_string(),
reason: format!("{e} ({} bytes on disk)", bytes.len()),
}),
}
}
/// Copy `<data_dir>/<name>/settings.json` into `index_settings`.
///
/// Create and `PUT /_settings` fill that map, and [`Self::default_pipeline_for`]
/// reads only the map. `GET /_settings` falls back to the file when the
/// entry is missing, so after a restart the binding was still reported
/// and ordinary writes no longer ran it (#2294). Boot, failed-index
/// retry, and snapshot restore all open an existing index through
/// `Index::open` and must rebuild the map from the same file `open` loaded.
///
/// A missing file, or a JSON `null`, drops any stale entry: restoring a
/// snapshot that was created without settings must not keep the pipeline
/// the previous index had in memory. An unreadable or unparseable file is
/// logged and left as-is. `Index::open` already refuses an unparseable
/// `settings.json` before this runs, so a parse error here is a change
/// between those two reads, not a second copy of that refusal.
fn rehydrate_index_settings_from_disk(&self, name: &str) {
let path = self.data_dir.join(name).join("settings.json");
match std::fs::read(&path) {
Ok(bytes) => match serde_json::from_slice::<Value>(&bytes) {
Ok(settings) if !settings.is_null() => {
self.index_settings.insert(name.to_string(), settings);
}
Ok(_) => {
self.index_settings.remove(name);
}
Err(e) => {
warn!(
name,
error = %e,
"settings.json did not parse while rebuilding index settings; \
leaving the in-memory binding unchanged"
);
}
},
Err(e) if e.kind() == std::io::ErrorKind::NotFound => {
self.index_settings.remove(name);
}
Err(e) => {
warn!(
name,
error = %e,
"settings.json unreadable while rebuilding index settings; \
leaving the in-memory binding unchanged"
);
}
}
}
/// Path of the persisted API-key store (`<data_dir>/api_keys.json`).
fn api_keys_path(&self) -> PathBuf {
self.data_dir.join("api_keys.json")
}
/// Insert (or overwrite) an API key and durably persist the whole store
/// (item 6). The auth middleware re-authenticates `Authorization: ApiKey
/// <encoded>` against `api_keys`; before this, that map lived only in
/// memory, so a restart silently invalidated every minted key. Now each
/// mutation snapshots the full map to `<data_dir>/api_keys.json` (0600,
/// atomic temp+rename), reloaded on boot.
///
/// A blocked boot returns the typed compatibility error before inserting
/// the key. Once storage is writable, an ordinary persistence I/O failure
/// remains best-effort: it is logged but does not fail the create — the key
/// still works until the next restart, matching the old behavior.
pub fn persist_api_key(&self, id: String, record: ApiKeyRecord) -> Result<()> {
self.ensure_cluster_state_writable()?;
self.api_keys.insert(id, record);
self.flush_api_keys_best_effort();
Ok(())
}
/// Invalidate (revoke) minted API keys by id — issue #208's missing half.
/// `ApiKeyRecord.invalidated` has been honoured by the auth path since the
/// field existed, but nothing could ever set it, so a leaked key was
/// permanent and rotation impossible.
///
/// Returns `(invalidated, previously_invalidated)` ids — the two non-error
/// buckets of ES's `DELETE /_security/api_key` response. An id that
/// matches no record lands in **neither** list: ES resolves selectors to
/// keys first and answers with an empty response when nothing matches
/// (`ApiKeyService#invalidateApiKeys`), it does not error per unknown id.
///
/// The flag and `invalidation_ms` are flipped in-memory — the auth
/// middleware reads this same map, so revocation takes effect on the very
/// next request — and the store is flushed to `api_keys.json` once at the
/// end, same durability contract as [`Self::persist_api_key`].
pub fn invalidate_api_keys(
&self,
ids: &[String],
now_ms: u64,
) -> Result<(Vec<String>, Vec<String>)> {
self.ensure_cluster_state_writable()?;
let mut invalidated = Vec::new();
let mut previously = Vec::new();
for id in ids {
let Some(mut rec) = self.api_keys.get_mut(id) else {
continue;
};
if rec.invalidated {
previously.push(id.clone());
} else {
rec.invalidated = true;
rec.invalidation_ms = Some(now_ms);
invalidated.push(id.clone());
}
// `rec` (a DashMap guard) drops here, before `flush_api_keys`
// re-iterates the map below.
}
if !invalidated.is_empty() {
self.flush_api_keys_best_effort();
}
Ok((invalidated, previously))
}
/// Serialize the current `api_keys` map to `<data_dir>/api_keys.json`
/// atomically with owner-only (0600) permissions.
///
/// Since #201 the file holds only salted hashes, but 0600 stays: a hash
/// plus a key id is still an offline target and still tells a reader
/// exactly which credentials exist.
fn flush_api_keys(&self) -> std::io::Result<()> {
let snapshot: std::collections::HashMap<String, ApiKeyRecord> = self
.api_keys
.iter()
.map(|e| (e.key().clone(), e.value().clone()))
.collect();
let bytes = serde_json::to_vec_pretty(&snapshot)
.map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e))?;
write_secret_file_atomic(&self.api_keys_path(), &bytes)
}
/// [`Self::flush_api_keys`] for the mutation paths, where a persistence
/// failure must not fail the request: the key still works until the next
/// restart, which is the pre-persistence behaviour rather than a
/// regression of key creation.
fn flush_api_keys_best_effort(&self) {
if let Err(e) = self.flush_api_keys() {
warn!(error = %e, "failed to persist api_keys.json (keys work until restart)");
}
}
/// Load persisted API keys from `<data_dir>/api_keys.json` into the
/// in-memory map on boot. A missing file is normal (fresh node / no keys
/// ever minted); a corrupt file is logged and ignored (the node still
/// boots — the admin key path is unaffected).
///
/// # Migration off plaintext (issue #201)
///
/// A store written before #201 holds `"secret": "<the credential>"`. Those
/// records are hashed here and the file is rewritten **during boot**,
/// before the server accepts a request, so an upgraded node stops leaking
/// on first start rather than on first key rotation. The migration is
/// one-way and needs no flag day: [`crate::secret_hash::is_usable_hash`]
/// is the discriminator ([`ApiKeyRecord::migrate_from_plaintext`] calls
/// it), so a store whose hashes decode is untouched, and only the exact
/// pre-#201 shape — a `secret` and no `secret_hash` at all — is migrated.
///
/// A record that is neither of those two shapes is **dropped with an
/// error**, not kept: keeping it would leave an entry that `GET
/// /_security/api_key` lists as a live credential while nothing can ever
/// authenticate as it — precisely the accept-then-ignore behaviour that
/// issue #204 tracks. Since "usable" is a full decode, a `secret_hash`
/// that is present but unparseable is dropped exactly like an absent one;
/// both are equally unauthenticatable.
fn load_persisted_api_keys(&self) {
let path = self.api_keys_path();
let Ok(bytes) = std::fs::read(&path) else {
return;
};
match serde_json::from_slice::<std::collections::HashMap<String, ApiKeyRecord>>(&bytes) {
Ok(map) => {
let mut restored = 0usize;
let mut migrated = 0usize;
let mut dropped = 0usize;
for (id, mut rec) in map {
match rec.migrate_from_plaintext() {
Ok(changed) => {
if changed {
migrated += 1;
}
restored += 1;
self.api_keys.insert(id, rec);
}
Err(()) => {
dropped += 1;
error!(
key_id = %id,
"api_keys.json record is not a credential this build can \
use: its secret_hash does not decode (absent, truncated, \
or an encoding this build does not know) and it is not \
the pre-#201 plaintext shape either — dropping it; \
nothing could ever authenticate as this key and listing \
it would be a lie"
);
}
}
}
if restored > 0 {
info!(count = restored, "restored persisted API keys");
}
if migrated > 0 {
// Rewrite immediately: until this lands, the plaintext is
// still on disk. A failure here is not cosmetic, so it is
// an error, not a warning — the operator has to know the
// node is still leaking and why.
match self.flush_api_keys() {
Ok(()) => info!(
count = migrated,
"migrated API key secrets to salted hashes; plaintext removed \
from api_keys.json"
),
Err(e) => error!(
error = %e,
count = migrated,
path = %path.display(),
"could not rewrite api_keys.json after hashing — PLAINTEXT API \
KEY SECRETS REMAIN ON DISK. Fix the permissions/space problem \
and restart, or rotate the keys."
),
}
}
if dropped > 0 {
error!(count = dropped, "dropped unusable API key records");
}
}
Err(e) => {
warn!(error = %e, "ignoring corrupt api_keys.json");
}
}
}
/// Find the highest-priority template matching `index_name`.
fn best_matching_template(&self, index_name: &str) -> Option<IndexTemplate> {
let mut best: Option<(i32, IndexTemplate)> = None;
for entry in self.templates.iter() {
let tmpl = entry.value();
let matches = tmpl
.index_patterns
.iter()
.any(|pat| glob_match(pat, index_name));
if matches {
let priority = tmpl.priority;
if best.as_ref().map(|(p, _)| priority > *p).unwrap_or(true) {
best = Some((priority, tmpl.clone()));
}
}
}
best.map(|(_, t)| t)
}
// ── Alias methods ─────────────────────────────────────────────────────────
/// Path of the persisted alias catalog (`<data_dir>/aliases.json`).
fn aliases_path(&self) -> PathBuf {
self.data_dir.join("aliases.json")
}
/// Serialize the current `aliases` map to `<data_dir>/aliases.json`
/// atomically (temp-file + rename), mirroring `flush_api_keys`. Every
/// alias mutation snapshots the full map; a write failure is logged but
/// non-fatal (the alias still works in-memory until the next restart).
fn flush_aliases(&self) {
let snapshot: std::collections::HashMap<String, Vec<String>> = self
.aliases
.iter()
.map(|e| (e.key().clone(), e.value().clone()))
.collect();
let bytes = match serde_json::to_vec_pretty(&snapshot) {
Ok(b) => b,
Err(e) => {
warn!(error = %e, "failed to serialize aliases for persistence");
return;
}
};
if let Err(e) = crate::index::write_file_atomic(&self.aliases_path(), &bytes) {
warn!(error = %e, "failed to persist aliases.json (aliases work until restart)");
}
}
/// Load persisted aliases from `<data_dir>/aliases.json` into the
/// in-memory map on boot. A missing file is normal (fresh node / no
/// aliases ever created); a corrupt file is logged and ignored.
///
/// Without this, `.kibana` (always an alias, never a bare index —
/// Kibana/OpenSearch Dashboards create it via `PUT .kibana_1` +
/// `POST _aliases`) silently stopped resolving on every restart: the
/// backing index's data survived, but the alias pointing at it didn't,
/// so `GET /.kibana` 404'd and OSD's saved-objects migrator concluded
/// the migration was still in progress (never actually completing) —
/// found empirically via a real OpenSearch Dashboards container stuck
/// on "Another OpenSearch Dashboards instance appears to be migrating
/// the index" on every restart, indefinitely.
fn load_persisted_aliases(&self) {
let path = self.aliases_path();
let Ok(bytes) = std::fs::read(&path) else {
return;
};
match serde_json::from_slice::<std::collections::HashMap<String, Vec<String>>>(&bytes) {
Ok(map) => {
let n = map.len();
for (alias, indices) in map {
self.aliases.insert(alias, indices);
}
if n > 0 {
info!(count = n, "restored persisted aliases");
}
}
Err(e) => {
warn!(error = %e, "ignoring corrupt aliases.json");
}
}
}
// ── Alias filter metadata durability (issue #524) ───────────────────────
/// Path of the persisted alias-metadata sidecar
/// (`<data_dir>/alias_metadata.json`).
///
/// A SEPARATE file from `aliases.json` (which persists only the
/// alias→members topology) on purpose. Each alias's `filter` (and the
/// other create-time metadata) lived only in the in-memory
/// `index_alias_metadata` map and evaporated on restart, so a filtered
/// alias came back with its members but no filter — wrong `_search` /
/// `_count` results, and once the by-query paths honour the filter, a
/// `_delete_by_query` that empties the whole backing index instead of the
/// slice the filter names (#524). A separate file keeps `aliases.json`'s
/// shape unchanged; an older binary simply ignores this one, the same
/// downgrade-safe call the ingest-pipeline-refusals sidecar makes.
fn alias_metadata_path(&self) -> PathBuf {
self.data_dir.join("alias_metadata.json")
}
/// Serialize `index_alias_metadata` to disk atomically (temp-file +
/// rename), mirroring [`Engine::flush_aliases`]. Called after every
/// mutation of the map; a write failure is logged but non-fatal (the
/// filter still applies in-memory until the next restart).
pub fn flush_alias_metadata(&self) {
let snapshot: std::collections::HashMap<String, serde_json::Value> = self
.index_alias_metadata
.iter()
.map(|e| (e.key().clone(), e.value().clone()))
.collect();
let bytes = match serde_json::to_vec_pretty(&snapshot) {
Ok(b) => b,
Err(e) => {
warn!(error = %e, "failed to serialize alias metadata for persistence");
return;
}
};
if let Err(e) = crate::index::write_file_atomic(&self.alias_metadata_path(), &bytes) {
warn!(error = %e, "failed to persist alias_metadata.json (alias filters work until restart)");
}
}
/// Load persisted alias metadata from `<data_dir>/alias_metadata.json` into
/// the in-memory map on boot. A missing file is normal (fresh node, or a
/// node whose aliases were all created by a build that predates this
/// sidecar); a corrupt file is logged and ignored.
fn load_alias_metadata(&self) {
let path = self.alias_metadata_path();
let Ok(bytes) = std::fs::read(&path) else {
return;
};
match serde_json::from_slice::<std::collections::HashMap<String, serde_json::Value>>(&bytes)
{
Ok(map) => {
let n = map.len();
for (index, meta) in map {
self.index_alias_metadata.insert(index, meta);
}
if n > 0 {
info!(count = n, "restored persisted alias metadata");
}
}
Err(e) => {
warn!(error = %e, "ignoring corrupt alias_metadata.json");
}
}
}
// ── Ingest-pipeline refusal provenance (issue #204) ─────────────────────
/// Sidecar holding [`Engine::definition_time_refusals`].
///
/// A separate file rather than an eighth field in `cluster_state.json`:
/// that envelope is `deny_unknown_fields`, so widening it would make every
/// file this build writes unreadable to an older binary — a downgrade
/// break traded for an upgrade fix. An older binary ignores this file.
fn pipeline_refusals_path(&self) -> PathBuf {
self.data_dir.join("ingest-pipeline-refusals.json")
}
/// Write the whole refusal map atomically.
///
/// `Err` means the marker did not reach disk, and the caller MUST NOT
/// acknowledge: without it the next boot cannot tell this definition from
/// one an older, gate-less build wrote, and would soften the refusal back
/// into a `201` (issue #204). Same rule `flush_cluster_state` already
/// enforces for the definition itself (issue #203).
///
/// # Why the snapshot is taken inside the lock
///
/// This writes the WHOLE map, so two concurrent refused `PUT`s are a
/// lost-update race, not a merge: each inserts its own marker, each
/// snapshots, and the one that writes SECOND with the OLDER snapshot
/// erases the other's marker from disk — while both callers are told
/// `{"acknowledged": true}`, which is exactly the promise this function's
/// `Err` contract exists to keep. Measured on the unlocked version at a
/// live node: 40 concurrent refused `PUT`s, 40 acknowledged, 37 markers on
/// disk; after a plain restart the three lost ones replayed at
/// `ReplayPersisted`, answered `201`, and dropped a field from a document
/// their `if` guard excluded. Sequential tests never see it.
///
/// `cluster_state_write` is the mutex, not a second one, because the two
/// files are written back to back by the same callers and one order for
/// both is easier to reason about than two. It is taken and released here,
/// never held across `flush_cluster_state` — `parking_lot::Mutex` is not
/// reentrant.
fn flush_pipeline_refusals(&self) -> Result<()> {
// A boot that could not READ this file has an empty in-memory map, so
// writing it would erase every marker still on disk: the next boot
// would parse the truncated file cleanly, conclude those definitions
// have no provenance, and replay them at `ReplayPersisted` — the very
// hole the marker closes, reopened by the fail-closed path itself.
// Refuse instead, and name the file, so the operator's repair is to
// move it aside rather than to discover the loss at the next restart
// (issue #204).
if self
.pipeline_refusals_unreadable
.load(std::sync::atomic::Ordering::SeqCst)
{
return Err(EngineError::Common(xerj_common::XerjError::internal(
format!(
"{} could not be read at boot, so its markers are not in memory and \
rewriting it here would erase every refusal it still holds; every \
persisted pipeline is being replayed at definition strictness until \
the file is moved aside and this node restarted",
self.pipeline_refusals_path().display()
),
)));
}
// One writer at a time, snapshot included — see the doc comment.
let _writing = self.cluster_state_write.lock();
let snapshot: std::collections::BTreeMap<String, String> = self
.definition_time_refusals
.iter()
.map(|e| (e.key().clone(), e.value().clone()))
.collect();
let bytes = serde_json::to_vec_pretty(&snapshot).map_err(|e| {
xerj_common::XerjError::internal(format!(
"could not serialise the ingest-pipeline refusal markers: {e}"
))
})?;
crate::index::write_file_atomic(&self.pipeline_refusals_path(), &bytes).map_err(|e| {
EngineError::Common(xerj_common::XerjError::internal(format!(
"could not persist {}: {e}",
self.pipeline_refusals_path().display()
)))
})
}
/// Clearing a refusal marker is not on the acknowledge path — a marker
/// that outlives its cause makes a pipeline *more* refused, never less —
/// so a write failure is logged rather than propagated.
fn flush_pipeline_refusals_best_effort(&self) {
if let Err(e) = self.flush_pipeline_refusals() {
warn!(
error = %e,
"could not clear an ingest-pipeline refusal marker; the pipeline will be \
refused again after the next restart until a re-PUT succeeds"
);
}
}
/// Read the refusal markers back at boot, before any pipeline is replayed.
///
/// A missing file is the normal case for a data dir written before this
/// build — every pipeline in it is then treated as "no provenance", which
/// is exactly the compatibility door the replay mode exists for. A corrupt
/// file is NOT treated as "no refusals": that would silently reopen the
/// hole this marker closes, so the boot refuses to run any pipeline it
/// cannot classify.
fn load_persisted_pipeline_refusals(&self) {
let path = self.pipeline_refusals_path();
let bytes = match std::fs::read(&path) {
Ok(b) => b,
Err(e) if e.kind() == std::io::ErrorKind::NotFound => return,
Err(e) => {
error!(
path = %path.display(), error = %e,
"ingest-pipeline refusal markers are unreadable; every persisted pipeline \
will be replayed at DEFINITION strictness so a refusal cannot be lost"
);
self.pipeline_refusals_unreadable
.store(true, std::sync::atomic::Ordering::SeqCst);
return;
}
};
match serde_json::from_slice::<std::collections::BTreeMap<String, String>>(&bytes) {
Ok(map) => {
let n = map.len();
for (id, reason) in map {
self.definition_time_refusals.insert(id, reason);
}
if n > 0 {
info!(
count = n,
"restored ingest-pipeline refusals recorded at definition time"
);
}
}
Err(e) => {
error!(
path = %path.display(), error = %e,
"ingest-pipeline refusal markers are corrupt; every persisted pipeline will \
be replayed at DEFINITION strictness so a refusal cannot be lost"
);
self.pipeline_refusals_unreadable
.store(true, std::sync::atomic::Ordering::SeqCst);
}
}
}
// ── Lifecycle management (ISM/ILM) persistence ──────────────────────────
fn ism_policies_path(&self) -> PathBuf {
self.data_dir.join("ism_policies.json")
}
fn managed_indices_path(&self) -> PathBuf {
self.data_dir.join("ism_managed_indices.json")
}
/// Insert (or overwrite) an ISM-shaped policy and persist the whole
/// store, mirroring `flush_aliases`. Both `_plugins/_ism/policies` (as
/// entered) and `_ilm/policy` (after translation) write through here —
/// see the `ism_policies` field doc.
pub fn put_ism_policy(
&self,
id: String,
policy: crate::lifecycle::LifecyclePolicy,
) -> Result<()> {
self.ensure_cluster_state_writable()?;
self.ism_policies.insert(id, policy);
self.flush_ism_policies();
Ok(())
}
pub fn remove_ism_policy(&self, id: &str) -> Result<bool> {
self.ensure_cluster_state_writable()?;
let removed = self.ism_policies.remove(id).is_some();
if removed {
self.flush_ism_policies();
}
Ok(removed)
}
fn flush_ism_policies(&self) {
let snapshot: std::collections::HashMap<String, crate::lifecycle::LifecyclePolicy> = self
.ism_policies
.iter()
.map(|e| (e.key().clone(), e.value().clone()))
.collect();
let bytes = match serde_json::to_vec_pretty(&snapshot) {
Ok(b) => b,
Err(e) => {
warn!(error = %e, "failed to serialize ism_policies for persistence");
return;
}
};
if let Err(e) = crate::index::write_file_atomic(&self.ism_policies_path(), &bytes) {
warn!(error = %e, "failed to persist ism_policies.json (policies work until restart)");
}
}
fn load_persisted_ism_policies(&self) {
let Ok(bytes) = std::fs::read(self.ism_policies_path()) else {
return;
};
match serde_json::from_slice::<
std::collections::HashMap<String, crate::lifecycle::LifecyclePolicy>,
>(&bytes)
{
Ok(map) => {
let n = map.len();
let mut restored = 0;
for (id, policy) in map {
// Every REST-reachable insertion path (native ISM PUT and
// the ILM-shape translation) already runs `validate()`
// before a policy lands here — this is the one path that
// doesn't, since it reads back whatever was on disk. A
// hand-edited or corrupted single entry in an otherwise
// parseable file would silently satisfy `change_policy`'s
// "does this policy_id exist" check without ever having
// been checked structurally. Skip just that entry rather
// than discarding the whole file, matching the
// per-entry granularity of the corruption it's guarding
// against.
if let Err(reason) = policy.validate() {
warn!(policy_id = %id, error = %reason, "skipping invalid persisted ISM/ILM policy");
continue;
}
self.ism_policies.insert(id, policy);
restored += 1;
}
if restored > 0 {
info!(count = restored, "restored persisted ISM/ILM policies");
}
if restored < n {
warn!(
skipped = n - restored,
"some persisted ISM/ILM policies failed validation and were not restored"
);
}
}
Err(e) => warn!(error = %e, "ignoring corrupt ism_policies.json"),
}
}
/// Persist the whole `managed_indices` map plus the detach tombstones
/// (issue #282). Called by `lifecycle::tick` after any change, and by
/// the attach/detach handlers.
///
/// On-disk shape is an envelope — `{"managed": {...}, "detached":
/// [...]}` — so the acknowledged detaches survive a restart alongside
/// the cursors. `load_persisted_managed_indices` still accepts the
/// pre-#282 bare-map form.
pub fn persist_managed_indices(&self) -> Result<()> {
self.ensure_cluster_state_writable()?;
let managed: std::collections::HashMap<String, crate::lifecycle::ManagedIndexState> = self
.managed_indices
.iter()
.map(|e| (e.key().clone(), e.value().clone()))
.collect();
let mut detached: Vec<String> = self
.lifecycle_detached
.iter()
.map(|e| e.key().clone())
.collect();
detached.sort();
let envelope = serde_json::json!({ "managed": managed, "detached": detached });
let bytes = match serde_json::to_vec_pretty(&envelope) {
Ok(b) => b,
Err(e) => {
warn!(error = %e, "failed to serialize managed_indices for persistence");
return Ok(());
}
};
if let Err(e) = crate::index::write_file_atomic(&self.managed_indices_path(), &bytes) {
warn!(error = %e, "failed to persist ism_managed_indices.json (state works until restart)");
}
Ok(())
}
fn load_persisted_managed_indices(&self) {
let Ok(bytes) = std::fs::read(self.managed_indices_path()) else {
return;
};
// `deny_unknown_fields` is what keeps the two on-disk shapes
// unambiguous: without it, a pre-#282 bare-map file (index names as
// top-level keys) would "successfully" parse as an envelope with
// every cursor silently ignored as an unknown field — the exact
// accepted-and-ignored class this issue removes.
#[derive(serde::Deserialize)]
#[serde(deny_unknown_fields)]
struct Envelope {
#[serde(default)]
managed: std::collections::HashMap<String, crate::lifecycle::ManagedIndexState>,
#[serde(default)]
detached: Vec<String>,
}
let (map, detached) = match serde_json::from_slice::<Envelope>(&bytes) {
Ok(env) => (env.managed, env.detached),
Err(_) => match serde_json::from_slice::<
std::collections::HashMap<String, crate::lifecycle::ManagedIndexState>,
>(&bytes)
{
Ok(map) => (map, Vec::new()),
Err(e) => {
warn!(error = %e, "ignoring corrupt ism_managed_indices.json");
return;
}
},
};
let n = map.len();
for (index_name, state) in map {
self.managed_indices.insert(index_name, state);
}
for index_name in detached {
self.lifecycle_detached.insert(index_name, ());
}
if n > 0 {
info!(count = n, "restored persisted ISM managed-index state");
}
}
/// Detach `index` from lifecycle management and record a persisted
/// tombstone for it (issue #282). This is THE detach path — `PUT
/// /{index}/_settings {"index.lifecycle.name": null}`, `POST
/// /{index}/_ilm/remove` and `POST /_plugins/_ism/remove/{index}` all
/// land here, so a detach is recorded once and the executor cannot
/// disagree with the operator about which route it honoured (#262's
/// `set_index_lifecycle_policy` made the same call).
///
/// Also scrubs `index.lifecycle.name` from the stored display settings:
/// ES drops the setting on removal rather than reporting a name that is
/// no longer in force, and a stale name here is exactly the input a
/// future settings-derived re-attach would trip over.
///
/// Returns whether the index was managed before the call.
pub fn detach_lifecycle(&self, index: &str) -> Result<bool> {
self.ensure_cluster_state_writable()?;
let was_managed = self.managed_indices.remove(index).is_some();
self.lifecycle_detached.insert(index.to_string(), ());
if let Some(mut stored) = self.index_settings.get_mut(index) {
let v = stored.value_mut();
if let Some(obj) = v.as_object_mut() {
obj.remove("index.lifecycle.name");
}
if let Some(lifecycle) = v
.pointer_mut("/index/lifecycle")
.and_then(Value::as_object_mut)
{
lifecycle.remove("name");
}
}
self.persist_managed_indices()?;
Ok(was_managed)
}
/// Whether `lifecycle::tick` is allowed to act — the `POST /_ilm/stop`
/// kill switch (issue #282).
pub fn lifecycle_execution_running(&self) -> bool {
self.lifecycle_running
.load(std::sync::atomic::Ordering::Relaxed)
}
/// `POST /_ilm/start` / `POST /_ilm/stop`.
pub fn set_lifecycle_execution_running(&self, running: bool) {
self.lifecycle_running
.store(running, std::sync::atomic::Ordering::Relaxed);
}
/// Spawn the background lifecycle-execution job: every
/// `config.lifecycle.tick_interval_secs` (default 300s = 5 minutes,
/// matching OpenSearch ISM's own default job interval), runs
/// `lifecycle::tick` over every managed index. Not latency-critical
/// (unlike the resource governor's sampler), so a plain `tokio::spawn`
/// on an interval is fine — uses a `Weak` self-pointer so it exits
/// cleanly when the engine is dropped (matters for tests, which create
/// and drop many `Engine`s).
pub fn spawn_lifecycle_manager(self: &Arc<Self>) {
let interval =
std::time::Duration::from_secs(self.config.lifecycle.tick_interval_secs.max(1));
let weak = Arc::downgrade(self);
tokio::spawn(async move {
let mut tick = tokio::time::interval(interval);
tick.tick().await; // skip the immediate first tick
loop {
tick.tick().await;
let Some(engine) = weak.upgrade() else {
return; // engine dropped
};
crate::lifecycle::tick(&engine).await;
}
});
}
// ── Index creation-date tracking ─────────────────────────────────────────
fn index_created_at_path(&self, name: &str) -> PathBuf {
self.data_dir.join(name).join("created_at.json")
}
/// Record `name`'s creation time (now) and persist it — called exactly
/// once, at the end of `create_index_with_settings`. Without this,
/// `min_index_age` has no ground truth, and `GET /{index}`'s
/// `creation_date` was synthesized fresh on every single request
/// (always reporting "now"), which is what this replaces.
fn record_index_created_at(&self, name: &str) {
let now = crate::lifecycle::now_ms();
self.index_created_at.insert(name.to_string(), now);
match serde_json::to_vec_pretty(&serde_json::json!({ "creation_date_ms": now })) {
Ok(bytes) => {
if let Err(e) =
crate::index::write_file_atomic(&self.index_created_at_path(name), &bytes)
{
warn!(index = name, error = %e, "failed to persist created_at.json");
}
}
Err(e) => warn!(index = name, error = %e, "failed to serialize created_at.json"),
}
}
/// Load `name`'s persisted creation time on startup. An index directory
/// that predates this feature (no `created_at.json`) gets a synthetic
/// baseline of "now" recorded and persisted on first post-upgrade boot —
/// its true history is unknown, so `min_index_age` starts counting from
/// the upgrade rather than from the index's real (unrecorded) creation.
/// Documented tradeoff, not silently wrong: every index that existed
/// before this shipped gets exactly one synthetic reset, once.
fn load_or_backfill_index_created_at(&self, name: &str) {
let path = self.index_created_at_path(name);
if let Ok(bytes) = std::fs::read(&path) {
if let Ok(v) = serde_json::from_slice::<Value>(&bytes) {
if let Some(ms) = v.get("creation_date_ms").and_then(Value::as_i64) {
self.index_created_at.insert(name.to_string(), ms);
return;
}
}
}
self.record_index_created_at(name);
}
// ── Cluster metadata persistence (issue #203) ────────────────────────────
//
// Index templates, legacy (v1) templates, component templates, ingest
// pipelines, data streams and ILM policies used to live only in the
// in-memory maps above: `PUT /_index_template/logs` answered `200
// {"acknowledged": true}` and the template was gone after the next boot,
// with no error anywhere and the next matching index silently created
// unshaped. They are now snapshotted, as one consistent document, into
// `<data_dir>/cluster_state.json` on every mutation and reloaded in
// `Engine::new`.
//
// Durability contract: a full-file rewrite through
// `index::write_file_atomic` — write to `cluster_state.tmp`, `fsync` it,
// `rename` over the target, then `fsync` the parent directory so the
// rename itself survives power loss. The rename is the commit point, so
// a `kill -9` at any instant leaves either the previous complete
// document or the new one, never a half-written mix. The same shape peer
// engines use for their manifests (cf. `fjall/src/file.rs:17`
// `fsync_directory`, which fjall calls after every atomic rewrite;
// `sled/src/metadata_store.rs:696` discards leftover `*.tmp` files from
// an interrupted rewrite on the next boot, which `apply_cluster_state`
// mirrors below).
//
// One file rather than six keeps the maps mutually consistent — a data
// stream and the template that shaped it can never be restored from
// different generations.
/// Path of the persisted cluster-metadata document
/// (`<data_dir>/cluster_state.json`).
fn cluster_state_path(&self) -> PathBuf {
self.data_dir.join("cluster_state.json")
}
/// `Err` when boot did not classify `cluster_state.json` as absent or as
/// the exact supported format 1.
///
/// Every management mutation calls this, not only the ones that reach
/// `flush_cluster_state` with something to write. A `DELETE` consults the
/// in-memory map first, and while the load has failed that map is empty,
/// so an unguarded delete answers `404 not found` about an object that is
/// sitting in the document on disk — the same lie in a different shape.
/// A typed 503 is the only honest answer while the node cannot see the
/// operator's configuration.
/// Install the Raft handle the server built for this process.
///
/// Called once, after console bootstrap, so system indices created during
/// boot stay node-local. Later calls replace the handle.
pub fn attach_cluster(&self, handle: xerj_cluster::ClusterHandle) {
*self.cluster.lock() = Some(handle);
}
/// `true` when metadata writes must go through the Raft log.
pub fn has_cluster(&self) -> bool {
self.cluster.lock().is_some()
}
/// Clone of the proposer. The lock is not held across the subsequent await.
pub fn cluster_handle(&self) -> Option<xerj_cluster::ClusterHandle> {
self.cluster.lock().clone()
}
pub fn ensure_cluster_state_writable(&self) -> Result<()> {
if self.cluster_state_boot.is_writable() {
return Ok(());
}
let path = self.cluster_state_path();
let detail = self
.cluster_state_boot
.detail()
.unwrap_or("cluster state was blocked at boot");
error!(
path = %path.display(),
detail,
"refusing a cluster-metadata write because the boot-time format fence is latched"
);
Err(EngineError::Common(
xerj_common::XerjError::cluster_state_unavailable(),
))
}
/// Snapshot every persisted management map into one document.
fn cluster_state_snapshot(&self) -> PersistedClusterStateV1 {
fn dump<V: Clone>(map: &DashMap<String, V>) -> std::collections::BTreeMap<String, V> {
map.iter()
.map(|e| (e.key().clone(), e.value().clone()))
.collect()
}
PersistedClusterStateV1 {
version: CLUSTER_STATE_V1,
index_templates: self
.templates
.iter()
.map(|entry| {
(
entry.key().clone(),
PersistedIndexTemplateV1::from(entry.value().clone()),
)
})
.collect(),
legacy_templates: dump(&self.legacy_templates),
component_templates: dump(&self.component_templates),
pipelines: dump(&self.pipelines),
data_streams: self
.data_streams
.iter()
.map(|entry| {
(
entry.key().clone(),
PersistedDataStreamV1::from(entry.value().clone()),
)
})
.collect(),
ilm_policies: dump(&self.ilm_policies),
}
}
/// Durably write the current cluster metadata.
///
/// Unlike `flush_aliases`, a failure here is **returned**, not swallowed:
/// every caller is an API write that would otherwise answer
/// `{"acknowledged": true}` for a change it cannot keep. Callers roll the
/// in-memory change back and surface a 500, so the operator finds out at
/// the moment of the write instead of at the next restart.
///
/// Refuses outright when boot could not load the existing document — see
/// the boot-time cluster-state fence. The snapshot below is taken from the live maps,
/// which in that state hold only what this process was told after boot, so
/// writing it would rename an empty-ish document over configuration that is
/// still intact on disk.
fn flush_cluster_state(&self) -> Result<()> {
self.ensure_cluster_state_writable()?;
// One writer at a time — see `cluster_state_write`. Snapshotting
// inside the lock also means the bytes that land are never older
// than a rewrite that has already returned.
let _writing = self.cluster_state_write.lock();
// The final writer owns the fence too. A future caller that misses a
// preguard still cannot normalize an unsupported document.
self.ensure_cluster_state_writable()?;
let snapshot = self.cluster_state_snapshot();
let bytes = serde_json::to_vec_pretty(&snapshot)?;
crate::index::write_file_atomic(&self.cluster_state_path(), &bytes)?;
Ok(())
}
/// Highest generation `N` for which a backing index `.ds-<stream>-<N>` is
/// actually open in this data dir; `0` when there is none.
///
/// Read off the open indices rather than the stream document, because the
/// point is to catch the case where the two disagree. Suffix parsing keeps
/// neighbouring streams apart on its own: for stream `a`, `.ds-a-b-000001`
/// leaves `b-000001`, which is not a number.
fn highest_backing_generation(&self, stream: &str) -> u64 {
let prefix = format!(".ds-{stream}-");
self.indices
.iter()
.filter_map(|e| {
e.key()
.strip_prefix(&prefix)
.and_then(|gen| gen.parse::<u64>().ok())
})
.max()
.unwrap_or(0)
}
/// Apply the exact format-1 document already read and validated before
/// index discovery. `None` means either a fresh node or a blocked boot;
/// the immutable boot status distinguishes those cases.
fn apply_cluster_state(&self, state: Option<PersistedClusterStateV1>) {
let Some(state) = state else {
return;
};
for (name, tmpl) in state.index_templates {
self.templates.insert(name, tmpl.into());
}
for (name, body) in state.legacy_templates {
self.legacy_templates.insert(name, body);
}
for (name, body) in state.component_templates {
self.component_templates.insert(name, body);
}
for (name, body) in state.ilm_policies {
self.ilm_policies.insert(name, body);
}
// Data streams: restore the record, reconcile it against what is
// actually on disk, then say plainly when a backing index it names is
// not there. Reporting a stream whose data is gone as
// `"status": "GREEN"` without a word in the log is the same quiet lie
// this issue is about.
let mut reconciled_a_stream = false;
for (name, persisted) in state.data_streams {
let mut ds: DataStream = persisted.into();
// `rollover_data_stream` creates the new backing index before it
// records the new generation, so `kill -9` in that window leaves
// `.ds-<name>-00000N` on disk while the document still says N-1.
// Restoring N-1 verbatim would wedge the stream permanently: the
// next rollover computes the same name, and `create_index`
// refuses it as `index_already_exists` — forever. Adopt the
// highest generation that actually exists instead, which is also
// what stops a second rollover writing into an index that already
// holds the first one's documents.
let on_disk = self.highest_backing_generation(&name);
if on_disk > ds.generation {
let adopted = format!(".ds-{name}-{on_disk:06}");
warn!(
data_stream = name.as_str(),
recorded = ds.generation,
found = on_disk,
adopted = adopted.as_str(),
"adopting a backing index left behind by a rollover that \
was interrupted before its generation reached disk"
);
if !ds.backing_indices.contains(&adopted) {
ds.backing_indices.push(adopted);
}
ds.generation = on_disk;
reconciled_a_stream = true;
}
let missing: Vec<&str> = ds
.backing_indices
.iter()
.filter(|b| !self.indices.contains_key(b.as_str()))
.map(|b| b.as_str())
.collect();
if !missing.is_empty() {
warn!(
data_stream = name.as_str(),
missing = missing.join(","),
"restored data stream references backing indices that are \
not present in the data dir"
);
}
self.data_streams.insert(name, ds);
}
// Pipelines carry behaviour, not just a document to hand back from
// `GET /_ingest/pipeline`: the executable form has to be rebuilt or
// `?pipeline=x` would be accepted after a restart and quietly do
// nothing. Keep the stored document either way so GET round-trips
// exactly as it did before the restart.
//
// Which strictness a definition is replayed at is decided by
// PROVENANCE, never by the calendar (issue #204). The markers are
// already loaded — `Engine::new` does it before this runs, so the
// no-`cluster_state.json` path sees them too — and loading is
// idempotent, so this call keeps every other caller of
// `apply_cluster_state` correct on its own.
self.load_persisted_pipeline_refusals();
let provenance_unknown = self
.pipeline_refusals_unreadable
.load(std::sync::atomic::Ordering::SeqCst);
let mut pipelines_restored = 0usize;
for (name, config) in state.pipelines {
self.pipelines.insert(name.clone(), config.clone());
let refused_at_definition_time = self
.definition_time_refusals
.get(&name)
.map(|r| r.value().clone());
// Replay strictness is for definitions written by a build with NO
// gate: those were accepted by whichever build wrote them and
// failing them at boot would flip a working cluster's writes from
// 201 to 400 on a binary upgrade alone. A definition THIS contract
// already refused is the opposite case — the operator asked, was
// told 400, and a restart must not answer differently. Softening
// it here dropped `secret` from documents an `if` guard excluded,
// under a 201, with no PUT in between.
let mode = if refused_at_definition_time.is_some() || provenance_unknown {
xerj_wasm::pipeline::CompileMode::Strict
} else {
xerj_wasm::pipeline::CompileMode::ReplayPersisted
};
match self.compile_pipeline_with_mode(&name, config, mode) {
Ok(()) => {
if refused_at_definition_time.is_some() {
// This build honours what the refusing one could not,
// so the marker has outlived its cause. Clearing it is
// what stops a capability upgrade needing a re-PUT.
info!(
pipeline = name.as_str(),
"a pipeline refused at definition time by an earlier build compiles \
cleanly here; the recorded refusal is cleared and it now runs"
);
self.definition_time_refusals.remove(&name);
self.flush_pipeline_refusals_best_effort();
}
pipelines_restored += 1
}
Err(e) => {
// Issue #204: record WHY there is no executable form, so
// every ingest path refuses loudly with the reason
// instead of reporting a pipeline GET plainly shows as
// "not found" — the same contract as a fresh PUT.
let reason = match (&refused_at_definition_time, provenance_unknown) {
// Answer the operator with the words they already
// read at PUT time, so the restart is visibly a
// continuation rather than a new, different failure.
(Some(recorded), _) => recorded.clone(),
(None, true) => format!(
"the persisted definition could not be recompiled at startup ({e}), \
and `ingest-pipeline-refusals.json` could not be read, so this \
build cannot tell an already-refused definition from one an \
older build accepted; no document can be ingested through \
pipeline [{name}] — repair or delete that file, or re-PUT the \
definition"
),
(None, false) => format!(
"the persisted definition could not be recompiled \
at startup ({e}); no document can be ingested \
through pipeline [{name}] — re-PUT the definition"
),
};
self.unrunnable_pipelines.insert(name.clone(), reason);
error!(
pipeline = name.as_str(), error = %e,
refused_at_definition_time = refused_at_definition_time.is_some(),
"persisted ingest pipeline could not be recompiled — it is \
still visible to GET /_ingest/pipeline but will NOT \
transform anything; re-PUT the definition"
)
}
}
}
// Make an adopted generation durable straight away, so the recovery
// happens once rather than on every boot from here on.
if reconciled_a_stream {
if let Err(e) = self.flush_cluster_state() {
warn!(
error = %e,
"could not persist the reconciled data-stream generation; \
it will be recomputed from disk on the next boot"
);
}
}
let n = self.templates.len()
+ self.legacy_templates.len()
+ self.component_templates.len()
+ self.pipelines.len()
+ self.data_streams.len()
+ self.ilm_policies.len();
if n > 0 {
info!(
index_templates = self.templates.len(),
legacy_templates = self.legacy_templates.len(),
component_templates = self.component_templates.len(),
pipelines = pipelines_restored,
data_streams = self.data_streams.len(),
ilm_policies = self.ilm_policies.len(),
"restored persisted cluster metadata"
);
}
}
/// Name every `.ds-*` index on disk that no restored data stream claims.
///
/// This does **not** reconcile anything — nothing is deleted, adopted or
/// repaired, because an orphan may hold the only copy of somebody's data.
/// It exists so the state is not *silent*: an orphaned backing index is
/// unreachable through the data-stream API (`GET` and `DELETE` on the
/// stream answer 404, while `PUT /_data_stream/<name>` answers
/// `409 resource_already_exists_exception` naming the orphan), so without
/// a line in the boot log the operator has no way to learn the name they
/// have to pass to `DELETE /<backing-index>` to clear it.
///
/// Expected sources: a data dir written by a build that predates
/// `cluster_state.json` (data streams were not persisted at all then), or
/// a `DELETE /_data_stream` interrupted by a build that recorded the
/// removal before destroying the backing indices. Skipped entirely when
/// the cluster state did not load — then every stream is unknown and
/// every backing index would look orphaned.
fn warn_orphaned_backing_indices(&self) {
if !self.cluster_state_boot.is_writable() {
return;
}
// Snapshot both maps before comparing them: nothing here may hold a
// shard guard on one map while taking one on the other.
let backing: Vec<String> = self
.indices
.iter()
.map(|e| e.key().clone())
.filter(|n| n.starts_with(".ds-"))
.collect();
if backing.is_empty() {
return;
}
let streams: Vec<(String, Vec<String>)> = self
.data_streams
.iter()
.map(|s| (s.key().clone(), s.value().backing_indices.clone()))
.collect();
let claimed = |index: &str| {
streams.iter().any(|(stream, indices)| {
indices.iter().any(|b| b == index)
|| index
.strip_prefix(&format!(".ds-{stream}-"))
.is_some_and(|gen| gen.parse::<u64>().is_ok())
})
};
let mut orphans: Vec<String> = backing.into_iter().filter(|n| !claimed(n)).collect();
if orphans.is_empty() {
return;
}
orphans.sort();
warn!(
count = orphans.len(),
indices = orphans.join(","),
"backing indices on disk belong to no known data stream — they are \
unreachable through the data-stream API and are NOT deleted \
automatically; remove each with DELETE /<index> once you have \
confirmed the data is not wanted"
);
}
/// Insert into a persisted map and durably record the result, rolling the
/// in-memory change back if the write fails so a 500 never leaves a
/// change that only this process can see.
fn persisted_insert<V: Clone>(
&self,
map: &DashMap<String, V>,
name: String,
value: V,
) -> Result<()> {
self.ensure_cluster_state_writable()?;
let previous = map.insert(name.clone(), value);
if let Err(e) = self.flush_cluster_state() {
match previous {
Some(old) => map.insert(name, old),
None => map.remove(&name).map(|(_, v)| v),
};
return Err(e);
}
Ok(())
}
/// Remove from a persisted map and durably record the result. Returns
/// `false` when there was nothing to remove (the caller answers 404);
/// an `Err` means the removal did not reach disk and was undone.
fn persisted_remove<V: Clone>(&self, map: &DashMap<String, V>, name: &str) -> Result<bool> {
// Before the map lookup: an empty map after a failed load would
// otherwise turn "I cannot see your configuration" into "it does not
// exist" — see `ensure_cluster_state_writable`.
self.ensure_cluster_state_writable()?;
let Some((key, removed)) = map.remove(name) else {
return Ok(false);
};
if let Err(e) = self.flush_cluster_state() {
map.insert(key, removed);
return Err(e);
}
Ok(true)
}
/// Create or replace a v2 index template (`PUT /_index_template/{name}`).
pub fn put_index_template(&self, name: String, template: IndexTemplate) -> Result<()> {
self.persisted_insert(&self.templates, name, template)
}
/// Delete a v2 index template; `false` when it did not exist.
pub fn delete_index_template(&self, name: &str) -> Result<bool> {
self.persisted_remove(&self.templates, name)
}
/// Create or replace a legacy v1 template (`PUT /_template/{name}`).
pub fn put_legacy_template(&self, name: String, body: Value) -> Result<()> {
self.persisted_insert(&self.legacy_templates, name, body)
}
/// Delete a legacy v1 template; `false` when it did not exist.
pub fn delete_legacy_template(&self, name: &str) -> Result<bool> {
self.persisted_remove(&self.legacy_templates, name)
}
/// Create or replace a component template.
pub fn put_component_template(&self, name: String, body: Value) -> Result<()> {
self.persisted_insert(&self.component_templates, name, body)
}
/// Delete a component template; `false` when it did not exist.
pub fn delete_component_template(&self, name: &str) -> Result<bool> {
self.persisted_remove(&self.component_templates, name)
}
/// Create or replace an ILM policy.
pub fn put_ilm_policy(&self, name: String, body: Value) -> Result<()> {
self.persisted_insert(&self.ilm_policies, name, body)
}
/// Delete an ILM policy; `false` when it did not exist.
pub fn delete_ilm_policy(&self, name: &str) -> Result<bool> {
self.persisted_remove(&self.ilm_policies, name)
}
/// Durably store an ingest pipeline definition and (re)build its
/// executable form. This is the only way a user-visible pipeline should
/// ever be created — `compile_pipeline` alone does not reach disk.
///
/// `config` is the xerj-shaped pipeline config to compile, and is what
/// gets stored (and handed back by `GET /_ingest/pipeline/{id}`) when it
/// compiles — the shape this endpoint has always returned.
///
/// A definition that does not compile is NOT stored: `Ok(Some(err))` is
/// returned, nothing is flushed, and any pipeline already registered under
/// this id is left untouched. A caller that must keep an uncompilable
/// definition readable — the ES-compat surface does, because Elasticsearch
/// accepts processors this build does not implement — calls
/// [`Self::register_unrunnable_pipeline`] instead, which stores the
/// definition *and* the reason it cannot run, and persists the refusal
/// marker that makes that reason survive a restart.
///
/// (This used to take a third `keep_if_uncompilable: Option<Value>`
/// argument whose `Some` arm stored the definition here. Every caller in
/// the workspace passed `None` — the ES surface routes through
/// `register_unrunnable_pipeline` — so the arm was unreachable, untested,
/// and held its own copy of the refusal-marker bookkeeping that the live
/// path already does. Issue #204: a security-relevant branch with no
/// caller is a liability, not a spare part.)
///
/// A compile failure drops the *executable* form under this id. Leaving
/// the old one behind would mean `?pipeline={id}` kept running a
/// definition `GET` no longer shows — and stopped running it at the next
/// restart. A definition must behave the same before and after a reboot;
/// that is the whole point of persisting it.
///
/// `Err` means the definition did not reach disk. The in-memory change is
/// rolled back first, so the caller can report the write as failed
/// without leaving behind a pipeline only this process can see.
pub fn put_pipeline(&self, id: &str, config: Value) -> Result<Option<xerj_wasm::WasmError>> {
// Up front, so a refused write never disturbs the compiled form of a
// pipeline that is still running.
self.ensure_cluster_state_writable()?;
let previous_doc = self.pipelines.get(id).map(|v| v.value().clone());
// Taken out up front so a failed compile cannot leave a stale
// executable behind; put back verbatim on every abort path.
let previous_compiled = self.transform_pipelines.remove(id).map(|(_, p)| p);
// `compile_pipeline` clears this on success; snapshot it so a failed
// flush can put the previous definition's unrunnable reason back too
// (issue #204 — a rolled-back write must not turn "stored but not
// runnable, and here is why" into a bare "not found").
let previous_reason = self.unrunnable_pipelines.get(id).map(|r| r.value().clone());
// Same for the persisted refusal marker (issue #204): a rolled-back
// write must not silently un-refuse a definition that is still the one
// on disk.
let previous_marker = self
.definition_time_refusals
.get(id)
.map(|r| r.value().clone());
// Same for the boot-replay degradation note: a rolled-back write must
// leave the previous definition described exactly as it was.
let previous_degradation = self.degraded_pipelines.get(id).map(|r| r.value().clone());
// `compile_pipeline` mutates nothing when it fails, so an abort only
// has to undo the two lines above.
if let Err(e) = self.compile_pipeline(id, config) {
if let Some(p) = previous_compiled {
self.transform_pipelines.insert(id.to_string(), p);
}
return Ok(Some(e));
}
// Every abort path restores exactly the state this call found, so a
// write that did not reach disk leaves nothing only this process can
// see. Each expansion sits in a branch that returns, so the moves out
// of the `previous_*` snapshots never overlap.
macro_rules! rollback {
() => {{
match previous_doc {
Some(d) => self.pipelines.insert(id.to_string(), d),
None => self.pipelines.remove(id).map(|(_, v)| v),
};
self.transform_pipelines.remove(id);
if let Some(p) = previous_compiled {
self.transform_pipelines.insert(id.to_string(), p);
}
match previous_reason {
Some(r) => self.unrunnable_pipelines.insert(id.to_string(), r),
None => self.unrunnable_pipelines.remove(id).map(|(_, v)| v),
};
match previous_marker {
Some(r) => self.definition_time_refusals.insert(id.to_string(), r),
None => self.definition_time_refusals.remove(id).map(|(_, v)| v),
};
match previous_degradation {
Some(r) => self.degraded_pipelines.insert(id.to_string(), r),
None => self.degraded_pipelines.remove(id).map(|(_, v)| v),
};
}};
}
// Issue #204: order the two files so the crash window fails CLOSED.
//
// Reaching here means the definition compiled, so the only marker
// change this function can make is a REMOVAL — `compile_pipeline`
// clears the marker on success, and nothing here sets one (a
// definition that does not compile returned above without storing
// anything; the caller that keeps such a definition readable goes
// through `register_unrunnable_pipeline`, which inserts, and therefore
// orders the two files the other way round for the same reason).
//
// A removal must land AFTER the definition:
//
// definition without the clear — the marker is stale, so the boot
// replays the NEW definition at definition strictness. It compiled
// here, so it compiles there, and the self-heal clears the marker
// and logs it. Loud at worst, and it repairs itself.
// clear without the definition — the marker is gone while the OLD,
// refused definition is still the one on disk, so the boot replays
// it softly and the pipeline silently starts running with the
// refused option ignored. That is exactly the defect this marker
// exists to close, reached through the rollback path.
//
// Which is also why clearing is best-effort rather than on the
// acknowledge path: a failed clear only ever leaves a pipeline MORE
// refused, and the next boot undoes it without an operator.
if let Err(e) = self.flush_cluster_state() {
rollback!();
return Err(e);
}
let marker_changed = self
.definition_time_refusals
.get(id)
.map(|r| r.value().clone())
!= previous_marker;
if marker_changed {
self.flush_pipeline_refusals_best_effort();
}
info!(name = id, "transform pipeline created");
// `Ok(None)` is now the only success: a definition that does not
// compile returned `Ok(Some(err))` above without storing anything.
Ok(None)
}
/// Delete an ingest pipeline — both the stored definition and the
/// compiled form, so `?pipeline={name}` stops working at the same moment
/// `GET /_ingest/pipeline/{name}` starts answering 404. `false` when it
/// did not exist.
pub fn delete_pipeline(&self, name: &str) -> Result<bool> {
let removed = self.persisted_remove(&self.pipelines, name)?;
if removed {
self.transform_pipelines.remove(name);
// Issue #204: a deleted pipeline must not leave a stale
// "stored but not runnable" reason behind either — the next
// definition under this id starts from a clean slate.
self.unrunnable_pipelines.remove(name);
self.degraded_pipelines.remove(name);
// …and neither may the persisted refusal marker outlive the
// definition it refers to, or a later PUT of a *different*
// definition under the same id would inherit its 400 at the next
// boot (issue #204).
if self.definition_time_refusals.remove(name).is_some() {
self.flush_pipeline_refusals_best_effort();
}
}
Ok(removed)
}
/// Add an alias pointing to an index. A blocked boot fails before the map
/// or alias file changes.
pub fn add_alias(&self, alias: &str, index: &str) -> Result<()> {
self.ensure_cluster_state_writable()?;
let mut entry = self.aliases.entry(alias.to_string()).or_default();
if !entry.contains(&index.to_string()) {
entry.push(index.to_string());
}
drop(entry);
self.flush_aliases();
Ok(())
}
/// Remove an alias's association with an index. A blocked boot fails
/// before the map or alias file changes.
pub fn remove_alias(&self, alias: &str, index: &str) -> Result<()> {
self.ensure_cluster_state_writable()?;
if let Some(mut entry) = self.aliases.get_mut(alias) {
entry.retain(|i| i != index);
}
// Clean up empty alias entries.
self.aliases.retain(|_, v| !v.is_empty());
self.flush_aliases();
Ok(())
}
/// Resolve a name: if it's an alias, return the aliased index names;
/// otherwise return the name itself (if the index exists).
pub fn resolve_alias(&self, name: &str) -> Vec<String> {
if let Some(indices) = self.aliases.get(name) {
return indices.clone();
}
vec![name.to_string()]
}
/// The concrete index a write to `name` must target when `name` is an
/// alias: a single backing index unless explicitly disabled, or exactly one marked
/// `is_write_index: true`. `Ok(None)` means `name` is not an alias (a
/// concrete index or a name the caller may auto-create). More than one
/// candidate is ambiguous and refused with ES's message.
///
/// `get_index` deliberately resolves an alias to its FIRST backing index,
/// which is fine for reads by name but wrong for a write: a two-index
/// alias silently sent the document to whichever member was listed first,
/// ignoring `is_write_index`. Write paths resolve through this function
/// before they touch a mapping, pipeline, or store.
pub fn write_target(&self, name: &str) -> std::result::Result<Option<String>, String> {
let Some(backings) = self.aliases.get(name) else {
return Ok(None);
};
let single_write_disabled = backings.len() == 1
&& self
.index_alias_metadata
.get(backings[0].as_str())
.is_some_and(|meta| {
meta.get(name)
.and_then(|alias_meta| alias_meta.get("is_write_index"))
.and_then(serde_json::Value::as_bool)
== Some(false)
});
if backings.len() == 1 && !single_write_disabled {
return Ok(Some(backings[0].clone()));
}
let mut writes = backings.iter().filter(|backing| {
let Some(meta) = self.index_alias_metadata.get(backing.as_str()) else {
return false;
};
meta.get(name)
.and_then(|alias_meta| alias_meta.get("is_write_index"))
.and_then(serde_json::Value::as_bool)
.unwrap_or(false)
});
let first = writes.next().cloned();
let second = writes.next();
match (first, second) {
(Some(target), None) => Ok(Some(target)),
(None, _) => Err(format!(
"no write index is defined for alias [{name}]. The write index may be \
explicitly disabled using is_write_index=false or the alias points to \
multiple indices without one being designated as a write index"
)),
(Some(_), Some(_)) => Err(format!("alias [{name}] has more than one write index")),
}
}
/// Delete an index and all its data.
///
/// Also drops any aliases that pointed only at this index (matching ES
/// semantics) and clears the `closed_indices` flag so the name is
/// truly gone when another test recreates it.
///
/// A **failed** index (present on disk, refused at open — see
/// [`FailedIndex`]) is deletable through this same door. It used to answer
/// 404 `index_not_found`, which left an operator with no lever but
/// stopping the server and removing the directory by hand (issue #206).
pub async fn delete_index(&self, name: &str) -> Result<()> {
self.ensure_cluster_state_writable()?;
// Per-index authorization backstop (issue #79) — "destroy the brain"
// is the loudest door, so it is checked before the index is removed
// from the map. See `crate::index_guard`.
if !crate::index_guard::visible(name) {
return Err(EngineError::Common(
xerj_common::XerjError::index_not_found(name),
));
}
if let Some(handle) = self.cluster_handle() {
// A name this node does not have is a local 404. Proposing it
// would delete the index on peers and, on the ES surface, come
// back as acknowledged once the not-found is swallowed.
if !self.indices.contains_key(name) && !self.failed_indices.contains_key(name) {
return Err(EngineError::Common(
xerj_common::XerjError::index_not_found(name),
));
}
let effect = handle
.propose(xerj_cluster::ClusterCommand::DeleteIndex {
name: name.to_string(),
})
.await
.map_err(crate::cluster_apply::propose_error)?;
return crate::cluster_apply::effect_result(name, effect);
}
self.delete_index_local(name).await
}
/// Remove `name` from this node's storage.
///
/// Does not propose. The cluster applier calls this after the delete has
/// committed. The single-node [`Engine::delete_index`] falls through here
/// after the visibility check.
pub(crate) async fn delete_index_local(&self, name: &str) -> Result<()> {
self.ensure_cluster_state_writable()?;
match self.indices.remove(name).map(|(_, v)| v) {
Some(idx) => {
// The handle is pulled out of the map first so no write can
// land in a directory that is being removed — but if the
// removal then fails (read-only mount, EACCES, EROFS), the
// name has been freed while the bytes are still there. That
// is exactly the stuck state issue #206 is about, arrived at
// from the other side: `Engine::indices` no longer holds it,
// `failed_indices` never did, so `_cat/indices` cannot show
// it, `DELETE` answers 404 and none of the three recovery
// levers this module adds can name it. Put the handle back so
// a delete that did not happen leaves the index addressable
// and the operator can retry it.
if let Err(e) = idx.delete_all_data().await {
self.indices.insert(name.to_string(), idx);
warn!(name, error = %e, "index delete failed; index left in service");
return Err(e);
}
}
None => {
// Not open. If it is a known failed index, its bytes are still
// on disk and removing them is exactly what the operator asked
// for; anything else is a genuine 404.
if !self.failed_indices.contains_key(name) {
return Err(EngineError::Common(
xerj_common::XerjError::index_not_found(name),
));
}
// Drop the bookkeeping only after the bytes are gone. Removing
// it first would take a delete that failed (read-only mount,
// fs error) out of the failed list while its directory
// survives — the operator would believe the name was freed and
// the index would reappear on the next boot.
self.remove_index_dir(name)?;
self.failed_indices.remove(name);
}
}
self.forget_index_metadata(name);
info!(name, "index deleted");
Ok(())
}
/// Drop every piece of engine-side metadata that names `index` — aliases
/// that pointed only at it, the closed flag, settings, mappings, alias
/// metadata. Shared by the open-index and failed-index delete paths so the
/// two cannot drift.
fn forget_index_metadata(&self, name: &str) {
// Remove this index from every alias that references it; drop the
// alias entirely when its backing list becomes empty.
let empty_aliases: Vec<String> = self
.aliases
.iter_mut()
.filter_map(|mut entry| {
entry.value_mut().retain(|n| n != name);
if entry.value().is_empty() {
Some(entry.key().clone())
} else {
None
}
})
.collect();
for a in empty_aliases {
self.aliases.remove(&a);
}
self.flush_aliases();
self.closed_indices.remove(name);
self.index_settings.remove(name);
self.index_mappings.remove(name);
self.index_alias_metadata.remove(name);
self.flush_alias_metadata();
// The WAL tap's cursor is a byte offset into a WAL stream that has
// just ceased to exist. Dropping it HERE, rather than waiting for a
// poll to notice the index is gone, is what makes delete-and-recreate
// inside one poll interval safe — see `wal_tap::WalTap::forget_index`.
self.wal_tap.forget_index(name);
// Lifecycle bookkeeping: a deleted index needs neither an execution
// cursor nor a detach tombstone (#282) — and clearing the tombstone
// here means a *recreated* index with the same name starts fresh
// rather than inheriting a years-old "operator said stop".
let had_lifecycle_state = self.managed_indices.remove(name).is_some()
| self.lifecycle_detached.remove(name).is_some();
if had_lifecycle_state {
// `delete_index` checked the compatibility fence before reaching
// this private cleanup path.
let _ = self.persist_managed_indices();
}
}
/// Remove `<data_dir>/<name>` from disk, refusing anything that does not
/// resolve inside `data_dir`.
///
/// The open-index path deletes through `Index::delete_all_data`, which can
/// only ever point at a directory the engine itself built. The failed-index
/// path has no `Index` to ask, so the name is re-validated here and the
/// resolved path is proven to be under `data_dir` before anything is
/// removed — the same canonicalisation rule the snapshot-restore path
/// applies before it writes.
fn remove_index_dir(&self, name: &str) -> Result<()> {
// Reject traversal/absolute forms up front: only a legal index name
// can name a directory we own.
IndexName::new(name).map_err(EngineError::Common)?;
let dir = self.data_dir.join(name);
if !dir.exists() {
return Ok(());
}
let dir_canon = dir.canonicalize().map_err(EngineError::Io)?;
let root_canon = self.data_dir.canonicalize().map_err(EngineError::Io)?;
if !dir_canon.starts_with(&root_canon) {
return Err(EngineError::Common(xerj_common::XerjError::storage(
format!("refusing to delete index [{name}] outside data_dir (canonical)"),
)));
}
std::fs::remove_dir_all(&dir_canon).map_err(EngineError::Io)?;
Ok(())
}
// ── Failed-index recovery (issue #206) ────────────────────────────────────
/// Record (or re-record) an index that could not be opened.
///
/// Preserves `failed_at_ms` across repeated failures so "since when" stays
/// truthful, and counts retries so a flapping directory is visible as
/// such.
fn record_failed_index(&self, name: &str, reason: String) {
match self.failed_indices.get_mut(name) {
Some(mut existing) => {
existing.reason = reason;
existing.retries = existing.retries.saturating_add(1);
}
None => {
self.failed_indices.insert(
name.to_string(),
FailedIndex {
name: name.to_string(),
reason,
failed_at_ms: now_millis(),
retries: 0,
},
);
}
}
}
/// Every failed index the caller is allowed to see, sorted by name.
pub fn list_failed_indices(&self) -> Vec<FailedIndex> {
let mut out: Vec<FailedIndex> = self
.failed_indices
.iter()
.filter(|e| crate::index_guard::visible(e.key()))
.map(|e| e.value().clone())
.collect();
out.sort_by(|a, b| a.name.cmp(&b.name));
out
}
/// Re-attempt the open of a failed index.
///
/// On success the index becomes a normal, serving index (mapping and date
/// flags restored exactly as at boot) and leaves the failed set. On failure
/// the recorded reason is refreshed, the retry counter advances, and the
/// new error is returned — the operator gets the *current* reason, not the
/// one from boot.
pub fn retry_failed_index(&self, name: &str) -> Result<()> {
self.ensure_cluster_state_writable()?;
if !crate::index_guard::visible(name) || !self.failed_indices.contains_key(name) {
return Err(EngineError::Common(
xerj_common::XerjError::index_not_found(name),
));
}
let index_name = IndexName::new(name).map_err(EngineError::Common)?;
match Index::open(index_name, &self.config, &self.data_dir) {
Ok(idx) => {
// #320 — a reopened index gets the tap's live retention floor,
// not the boot file's.
idx.set_wal_min_retained_generations(self.wal_retention_floor());
// The store opened, but the full-fidelity mapping blob is part
// of what makes this index serveable (#202). Putting the index
// back with a silently reduced mapping is exactly the defect
// the open refusal exists to prevent, so a retry that cannot
// read `es_mapping.json` stays failed and says why.
if let Err(e) = self.load_persisted_es_mapping(name) {
let reason = e.to_string();
self.record_failed_index(name, reason.clone());
warn!(name, error = %reason, "retry of failed index did not succeed");
return Err(EngineError::Common(
xerj_common::XerjError::index_unavailable(name, reason),
));
}
if let Some(m) = self.index_mappings.get(name) {
Engine::apply_date_mapping_flags(&idx, m.value());
}
self.rehydrate_index_settings_from_disk(name);
self.indices.insert(name.to_string(), idx);
self.failed_indices.remove(name);
info!(name, "failed index reopened");
Ok(())
}
Err(e) => {
let reason = e.to_string();
self.record_failed_index(name, reason.clone());
warn!(name, error = %reason, "retry of failed index did not succeed");
Err(EngineError::Common(
xerj_common::XerjError::index_unavailable(name, reason),
))
}
}
}
/// Whether the concrete index `name` is currently held in memory (its
/// `Arc<Index>` is live in `self.indices`). A closed index stays loaded —
/// `_close` frees the index's rebuildable caches but keeps the handle here
/// (#463), so this returns `true` for a closed index too; it is `false` only
/// for a name that was never opened or has been deleted. Not alias-resolving
/// — callers pass a concrete index name.
pub fn is_index_loaded(&self, name: &str) -> bool {
self.indices.contains_key(name)
}
/// `_close` that actually reclaims RAM (#463). Marks the index unqueryable
/// AND frees its rebuildable in-memory caches via [`Index::release_memory`]
/// (the ~15 per-segment cache DashMaps; reads re-hydrate from disk). The
/// `Arc<Index>` handle STAYS in `self.indices`, so every loaded-index view
/// (`_cat/indices`, `_cluster/health`, the `expand_wildcards` filtering) is
/// unchanged — a closed index is loaded-and-flagged, matching the rest of the
/// codebase's model. [`reopen_index`](Self::reopen_index) just clears the flag.
///
/// The memtable is flushed to disk FIRST: freeing the caches without flushing
/// would be harmless (caches only mirror durable data), but the flush also
/// drains the memtable so the closed index holds no unpublished writes. A
/// flush failure aborts the close and leaves the index fully in service.
pub async fn close_index(&self, name: &str) -> Result<()> {
self.ensure_cluster_state_writable()?;
if let Some(idx) = self.indices.get(name).map(|r| Arc::clone(r.value())) {
// Durability barrier: flush the memtable to disk BEFORE releasing the
// caches, so no not-yet-published document is lost. A flush failure
// aborts the close and leaves the index fully in service.
idx.flush().await?;
// Free the rebuildable in-memory caches (memtable is now empty; the
// per-segment caches re-hydrate from disk on the next read). The
// `Arc<Index>` shell STAYS in `self.indices`, so every loaded-index
// view (`_cat/indices`, `_cluster/health`, `_cluster/state`, the
// `expand_wildcards` filtering) is unchanged — a closed index is
// loaded-and-flagged, matching the rest of the codebase's model.
idx.release_memory();
}
self.closed_indices.insert(name.to_string(), true);
info!(name, "index closed; in-memory caches released");
Ok(())
}
/// `_open` counterpart to [`close_index`](Self::close_index): clears the
/// closed flag. The index handle was never removed (only its caches were
/// released), so there is nothing to reconstruct — subsequent reads
/// re-hydrate segments from disk on demand.
pub fn reopen_index(&self, name: &str) -> Result<()> {
self.ensure_cluster_state_writable()?;
self.closed_indices.remove(name);
info!(name, "index reopened");
Ok(())
}
/// Get a reference to an index by name, resolving aliases first.
/// If the name is an alias pointing to multiple indices, returns the first one.
///
/// This is the single funnel every handler goes through to reach index
/// data, whether it read the name from the URL path, from a `_bulk` action
/// line, from an `_mget` `docs[]._index`, from a `terms` lookup or from the
/// table name inside an `_sql` statement — which is exactly why the
/// per-index authorization backstop is here (issue #79, see
/// [`crate::index_guard`]). The check runs **after** alias resolution, on
/// the concrete name, so pointing an alias at someone else's index cannot
/// launder access to it.
pub fn get_index(&self, name: &str) -> Result<Arc<Index>> {
// A blocked boot deliberately did not open any index or replay any
// WAL. Report that unavailable state instead of turning every real
// on-disk index into a misleading 404.
if !self.cluster_state_boot.is_writable() {
return Err(EngineError::Common(
xerj_common::XerjError::cluster_state_unavailable(),
));
}
// Check if name is an alias — if so, resolve to the first backing index.
if let Some(aliased) = self.aliases.get(name) {
if let Some(real_name) = aliased.first() {
if !crate::index_guard::visible(real_name) {
return Err(EngineError::Common(
xerj_common::XerjError::index_not_found(name),
));
}
return self
.indices
.get(real_name.as_str())
.map(|r| Arc::clone(r.value()))
.ok_or_else(|| EngineError::Common(self.missing_index_error(real_name)));
}
}
if !crate::index_guard::visible(name) {
return Err(EngineError::Common(
xerj_common::XerjError::index_not_found(name),
));
}
self.indices
.get(name)
.map(|r| Arc::clone(r.value()))
.ok_or_else(|| EngineError::Common(self.missing_index_error(name)))
}
/// The error for a name that is not in `indices`.
///
/// A name that failed to open is **not** "not found": the directory is
/// still there and the operator has a lever. Reporting a 404 for it is how
/// issue #206's stuck state stayed invisible, so a failed index answers
/// 503 `no_shard_available_action_exception` carrying the open error and
/// the three commands that act on it.
fn missing_index_error(&self, name: &str) -> xerj_common::XerjError {
match self.failed_indices.get(name) {
Some(f) => xerj_common::XerjError::index_unavailable(name, f.reason.clone()),
None => xerj_common::XerjError::index_not_found(name),
}
}
/// Return an index by name, creating it if it doesn't exist (ES behaviour).
///
/// The visibility check is repeated here rather than left to
/// [`Engine::get_index`]: that call reports "not found" for a denied name,
/// and "not found" is precisely the branch this function answers by
/// **creating** the index. Without its own check, auto-create would be the
/// bypass (`create_index` catches it too — belt and braces on the write
/// door).
pub fn get_or_create_index(&self, name: &str) -> Result<Arc<Index>> {
if !crate::index_guard::visible(name) {
return Err(EngineError::Common(
xerj_common::XerjError::index_not_found(name),
));
}
if let Ok(idx) = self.get_index(name) {
return Ok(idx);
}
// A failed index already occupies this name and its bytes are still on
// disk. Auto-creating over it would either destroy recoverable data or
// fail deep inside the store with an opaque message — refuse loudly
// with the open reason and the operator's options instead (issue #206).
if let Some(f) = self.failed_indices.get(name) {
return Err(EngineError::Common(
xerj_common::XerjError::index_unavailable(name, f.reason.clone()),
));
}
// Auto-create. A matching template supplies the mapping and settings
// an explicit `PUT /{index}` would have merged in (#2281). The schema
// is filled by `apply_index_template` inside `create_index_with_settings`;
// the raw mapping and the display settings are what `GET /_mapping`
// and `default_pipeline_for` read, and neither is implied by the schema.
let (mappings, settings) = self.merge_index_template(name, Value::Null, Value::Null);
self.create_index_with_settings(name, Schema::empty(), settings.clone())?;
if mappings.as_object().is_some_and(|m| !m.is_empty()) {
self.put_index_mapping(name, mappings)?;
}
if settings.as_object().is_some_and(|m| !m.is_empty()) {
self.index_settings.insert(name.to_string(), settings);
}
self.get_index(name)
}
/// V4 M5.2 — route a document to the node that owns its shard.
///
/// Returns `None` when this node should write the document itself:
/// there is no shard assignment (single-node mode, and any index that
/// was not created through the replicated shard plan), or the assigned
/// owner is this node. Returns `Some(node_id)` when a peer owns the
/// shard. Callers forward that write; they do not also store it here.
///
/// With no cluster handle the owner comparison uses `"local"`, which
/// no replicated plan names, and the empty routing table makes every
/// call `None`.
pub fn route_write(&self, index: &str, doc_id: &str) -> Option<String> {
let me = self.routing_node_id();
let router = self.shard_router.read();
match router.route_doc(index, doc_id).1 {
None => None,
Some(owner) if owner == me => None,
Some(owner) => Some(owner.to_string()),
}
}
/// `true` when a replicated shard plan names this node as the primary
/// for `doc_id`. An index with no plan is `false` — that is not the
/// same as [`Engine::route_write`] returning `None`, which also covers
/// the unassigned case.
pub fn owns_shard(&self, index: &str, doc_id: &str) -> bool {
let me = self.routing_node_id();
let router = self.shard_router.read();
matches!(router.route_doc(index, doc_id).1, Some(owner) if owner == me)
}
/// Node ids that hold at least one primary of `index`, sorted.
pub fn shard_search_targets(&self, index: &str) -> Vec<String> {
self.shard_router.read().search_targets(index)
}
/// Identity [`Engine::route_write`] compares against shard owners.
///
/// The cluster handle's node id when one is attached, otherwise the
/// single-node `"local"` id.
pub fn routing_node_id(&self) -> String {
self.cluster
.lock()
.as_ref()
.map(|handle| handle.node_id().to_string())
.unwrap_or_else(|| self.node_id.to_string())
}
/// Replace `index`'s primary assignments and widen the hash width so
/// every shard in `plan` is reachable.
pub fn install_shard_plan(&self, index: &str, plan: &[xerj_cluster::ShardAssignment]) {
if plan.is_empty() {
return;
}
let mut router = self.shard_router.write();
let width = plan
.iter()
.map(|assignment| assignment.shard.saturating_add(1))
.max()
.unwrap_or(1);
if width > router.num_shards() {
router.set_num_shards(width);
}
router.clear_index(index);
for assignment in plan {
router.assign(index, assignment.shard, &assignment.node_id);
for replica in &assignment.replicas {
if replica != &assignment.node_id {
router.add_replica(index, assignment.shard, replica);
}
}
}
}
/// `(shard, primary node)` for `doc_id`, when the index has a plan.
pub fn primary_of(&self, index: &str, doc_id: &str) -> Option<(u32, String)> {
let router = self.shard_router.read();
let (shard, owner) = router.route_doc(index, doc_id);
owner.map(|owner| (shard, owner.to_string()))
}
/// Primary node id for one shard of `index`, when the plan names one.
pub fn shard_primary(&self, index: &str, shard: u32) -> Option<String> {
self.shard_router.read().primary_node(index, shard)
}
/// Replica node ids a read or a copy uses, excluding the primary.
///
/// The create-time list until an in-sync update commits, then the
/// replicas that confirmed the last acknowledged write.
pub fn replica_nodes(&self, index: &str, shard: u32) -> Vec<String> {
self.shard_router.read().get_replicas(index, shard)
}
/// Replicas named at create, excluding the primary. The quorum uses this
/// count, which does not shrink when a replica falls out of sync.
pub fn placed_replica_nodes(&self, index: &str, shard: u32) -> Vec<String> {
self.shard_router.read().placed_replicas(index, shard)
}
/// Record the replicas that may serve a later read of this shard.
///
/// Ids outside the create-time list are ignored. A shard with no placed
/// replicas is left as it is.
pub fn set_shard_in_sync(&self, index: &str, shard: u32, nodes: &[String]) {
self.shard_router.write().set_in_sync(index, shard, nodes);
}
/// Lock that serializes replica copies of one shard on this process.
pub fn replica_sync_lock(&self, index: &str, shard: u32) -> Arc<tokio::sync::Mutex<()>> {
self.replica_sync
.entry((index.to_string(), shard))
.or_insert_with(|| Arc::new(tokio::sync::Mutex::new(())))
.clone()
}
/// Primary shards of `index` owned by `node_id`, sorted.
pub fn shards_owned_by(&self, index: &str, node_id: &str) -> Vec<u32> {
self.shard_router.read().shards_owned_by(index, node_id)
}
/// Shards of `index` for which this node is a replica, sorted.
pub fn local_replica_shards(&self, index: &str) -> Vec<u32> {
let me = self.routing_node_id();
self.shard_router.read().replica_shards_for(index, &me)
}
/// Record one primary without dropping the index's other shards.
pub fn assign_one_shard(&self, index: &str, shard: u32, node_id: &str) {
let mut router = self.shard_router.write();
let width = shard.saturating_add(1);
if width > router.num_shards() {
router.set_num_shards(width);
}
router.assign(index, shard, node_id);
}
/// Drop `index` from the routing table. Called when a delete commits.
pub fn clear_shard_plan(&self, index: &str) {
self.shard_router.write().clear_index(index);
}
/// Read-only view of the single-node id (`"local"` unless a test
/// replaced the field). Cluster routing uses [`Engine::routing_node_id`].
pub fn local_node_id(&self) -> &str {
self.node_id.as_str()
}
/// List all indices with summary info.
/// Cheap sync accessor for the set of currently-open index names.
/// Used by PIT expansion and other handlers that need to iterate
/// the live index list without paying for the `list_indices`
/// snapshot or `get_settings()` call.
/// Filtered to the current request's visible set (issue #79) — this and
/// [`Engine::list_indices`] are where `*`, `_all` and `logs-*` are turned
/// into concrete names, so filtering here is what makes a wildcard resolve
/// to "the indices you may see" instead of having to be refused outright.
/// Unfiltered outside a request; see [`crate::index_guard`].
pub fn index_name_list(&self) -> Vec<String> {
self.indices
.iter()
.map(|e| e.key().clone())
.filter(|n| crate::index_guard::visible(n))
.collect()
}
/// Cheap O(1) sync check whether an open index with `name` exists and is
/// visible to the current request (issue #79), avoiding `list_indices()`
/// or full-map enumeration on single-index operations (#1226).
pub fn has_open_index(&self, name: &str) -> bool {
crate::index_guard::visible(name) && self.indices.contains_key(name)
}
/// The WAL-consumer retention floor the tap is currently configured with.
///
/// Read from the tap, never from `self.config`: the tap's copy is the one
/// an operator can change at runtime, and after a restart it is the one
/// that was persisted. `self.config` is the boot file, frozen in an `Arc`.
fn wal_retention_floor(&self) -> u64 {
self.wal_tap.config().min_retained_generations
}
/// Push the tap's live retention floor onto every open index's WAL shards.
///
/// Returns the number of indices it reached, so the `PUT /_xerj/wal_tap`
/// handler can report what actually happened instead of guessing.
///
/// #320. A floor set through the API previously reached nothing: it lived
/// in the tap's own `RwLock` and in the state file, while every
/// `WalWriter` had been seeded from `Engine.config` — an `Arc<Config>`
/// written once at boot from `xerj.toml` and never mutated. The API
/// answered `200` with the new value and a warning that it would apply
/// after a restart; it did not apply after a restart either. This is the
/// half that makes it true for indices that are already open.
///
/// Re-applying a policy to already-live state on change, rather than only
/// at construction, is Lucene's `IndexFileDeleter.revisitPolicy`
/// (lucene/core/src/java/org/apache/lucene/index/IndexFileDeleter.java:516-543,
/// Apache-2.0), which re-runs `policy.onCommit(commits)` over the commits
/// that already exist rather than waiting for the next one.
pub fn apply_wal_retention_floor(&self) -> usize {
let floor = self.wal_retention_floor();
// Snapshot the handles first, then take the WAL mutexes — never both
// at once. `DashMap::iter` holds a read guard on each map shard for as
// long as the iterator lives, and `set_wal_min_retained_generations`
// locks every `WalWriter` of the index; holding the first while
// acquiring the second would put a lock-order edge between two
// subsystems that otherwise have none, for the sake of one avoided
// allocation on an operator-triggered path.
let indices: Vec<Arc<Index>> = self.indices.iter().map(|e| e.value().clone()).collect();
for index in &indices {
index.set_wal_min_retained_generations(floor);
}
let applied = indices.len();
tracing::debug!(
floor,
indices = applied,
"WAL tap: retention floor applied to open indices"
);
applied
}
/// Sum the internal query-result cache hit/miss counters across every open
/// index (RC4-W4 item 4). Returns `(hits, misses)`. Reconciled into the
/// `xerj_query_cache_{hits,misses}` gauges by the `/v1/metrics` handler at
/// scrape time — the engine owns the truth, the API layer owns Prometheus.
pub fn query_cache_totals(&self) -> (u64, u64) {
let mut hits = 0u64;
let mut misses = 0u64;
for entry in self.indices.iter() {
hits = hits.saturating_add(entry.value().query_cache_hit_count());
misses = misses.saturating_add(entry.value().query_cache_miss_count());
}
(hits, misses)
}
/// Filtered to the current request's visible set — see
/// [`Engine::index_name_list`] and [`crate::index_guard`]. Every metadata
/// listing (`_cat/indices`, `_mapping`, `_alias`, `_resolve/index`,
/// `/v1/dashboard/summary`, cluster health) enumerates through here, so a
/// scoped credential gets a filtered view of the cluster rather than a
/// blanket refusal.
pub async fn list_indices(&self) -> Vec<IndexInfo> {
let mut list = Vec::new();
for (name, idx) in self.open_index_handles() {
if !crate::index_guard::visible(&name) {
continue;
}
let stats = idx.stats().await;
list.push(IndexInfo {
name: stats.name,
doc_count: stats.doc_count,
segment_count: stats.segment_count,
schema_version: stats.schema_version,
});
}
list
}
/// Snapshot handles for read-only statistics, releasing every DashMap
/// iterator guard before awaiting an index. A stalled schema lock must
/// not hold the map shard against an index deletion or insertion (#2405).
/// This preserves the existing observational (non-atomic) stats contract.
fn open_index_handles(&self) -> Vec<(String, Arc<Index>)> {
self.indices
.iter()
.map(|entry| (entry.key().clone(), Arc::clone(entry.value())))
.collect()
}
/// Get the stats for a single index.
pub async fn index_stats(&self, name: &str) -> Result<IndexStats> {
let idx = self.get_index(name)?;
Ok(idx.stats().await)
}
/// Flush the in-memory memtable for an index to a durable segment on disk.
///
/// After a flush the WAL checkpoint is advanced and old WAL generations are
/// pruned, so the data survives future restarts without WAL replay.
pub async fn flush_index(&self, name: &str) -> Result<()> {
let idx = self.get_index(name)?;
idx.flush().await
}
/// Flush all indices whose memtable exceeds the size threshold.
///
/// Called periodically by the background flush timer.
pub async fn flush_all_if_needed(&self) {
for entry in self.indices.iter() {
let idx = Arc::clone(entry.value());
if idx.needs_flush().await {
if let Err(e) = idx.flush().await {
tracing::warn!(
index = entry.key().as_str(),
error = %e,
"background flush failed"
);
}
}
}
}
/// Force-flush every index regardless of memtable size.
///
/// Called from the SIGTERM/SIGINT shutdown hook so that any data still
/// in the memtable at the moment we stop accepting requests gets a
/// chance to land on disk as a segment before the process exits.
/// Without this, anything that was bulk-ingested after the last
/// auto-flush threshold crossing lives only in the WAL until the next
/// startup — and if startup index-discovery doesn't pick the index up
/// (e.g. WAL-only indexes), the data is lost.
///
/// First stops every index's background merge scheduling (#871:
/// event-driven debounced checks; historically a per-index tick task)
/// and aborts any armed check task — a pending debounce
/// `tokio::time::sleep` keeps the tokio runtime alive even after axum
/// has stopped accepting connections. Without stopping them up-front,
/// the process waits for the sleep to wake (or a merge fires
/// post-shutdown — either way SIGTERM hangs). Stopping FIRST also
/// guarantees the shutdown flushes below cannot arm new checks through
/// the segments-changed hook. See bench
/// `engine/reports/2026-04-25T03-30-00` for the captured regression
/// introduced by B-2b (commit 605ac7b).
pub async fn flush_all_force(&self) {
// 1. Stop all background merge scheduling so the runtime can exit
// once the flush is done. Aborts are non-blocking; the armed
// task is unwound by tokio without us needing to await it.
for entry in self.indices.iter() {
entry.value().abort_background_tasks();
}
// 2. Final synchronous flush across every index.
for entry in self.indices.iter() {
let idx = Arc::clone(entry.value());
if let Err(e) = idx.flush().await {
tracing::warn!(
index = entry.key().as_str(),
error = %e,
"shutdown flush failed"
);
}
}
}
/// Engine health status.
///
/// Returns `"green"` when all indices are fully flushed to durable segments.
/// Returns `"yellow"` when one or more indices have unflushed memtable data
/// (data is safe in the WAL but not yet in a segment — a crash would require
/// WAL replay).
/// Returns `"red"` when one or more indices failed to open on startup
/// (tracked in [`failed_indices`]).
pub async fn health(&self) -> HealthStatus {
let mut total_docs = 0u64;
let mut has_memtable_only = false;
for (_, idx) in self.open_index_handles() {
let stats = idx.stats().await;
total_docs += stats.doc_count;
// Yellow condition: any index has in-memory data that hasn't been
// flushed to a segment yet.
if stats.segment_count == 0 && stats.memtable_doc_count > 0 {
has_memtable_only = true;
}
}
// Red condition: any index directory could not be opened on startup.
let has_failed = !self.failed_indices.is_empty();
let status = if has_failed {
"red"
} else if has_memtable_only {
"yellow"
} else {
"green"
};
HealthStatus {
status: status.to_string(),
index_count: self.indices.len(),
total_docs,
version: env!("CARGO_PKG_VERSION").to_string(),
}
}
// ── Data Stream methods ───────────────────────────────────────────────────
/// Create a new data stream with its first backing index.
pub fn create_data_stream(&self, name: &str) -> Result<()> {
// Before `create_index` below, not just before the flush: refusing
// after the backing index exists would leave a `.ds-*` directory with
// no stream in front of it.
self.ensure_cluster_state_writable()?;
if self.data_streams.contains_key(name) {
return Err(EngineError::Common(
xerj_common::XerjError::index_already_exists(name),
));
}
let backing_name = format!(".ds-{}-000001", name);
self.create_index(&backing_name, Schema::empty())?;
// Alias: writing to the stream name → first backing index.
self.add_alias(name, &backing_name)?;
let ds = DataStream {
name: name.to_string(),
backing_indices: vec![backing_name],
timestamp_field: "@timestamp".to_string(),
generation: 1,
};
// Durable before the caller is told it worked (issue #203): the
// backing index is already on disk, so a stream record that lives
// only in memory would come back after a restart as an orphaned
// `.ds-*` index with no stream in front of it.
self.persisted_insert(&self.data_streams, name.to_string(), ds)?;
info!(name, "data stream created");
Ok(())
}
/// Drop a deleted backing index from its data stream's generation list
/// so the stream does not keep advertising an index that no longer
/// exists (#282, ported from #262's `detach_data_stream_backing_index`)
/// — and persist the shrunken list, or a restart would reload the
/// deleted generation from `cluster_state.json` as an orphan.
pub(crate) fn detach_data_stream_backing_index(&self, index: &str) -> Result<()> {
// This helper is crate-visible because lifecycle deletion calls it
// after removing the physical backing index. Keep the compatibility
// fence here as well: a future caller must not be able to edit the
// live stream map merely because it bypassed the lifecycle entry
// point's guard.
self.ensure_cluster_state_writable()?;
let mut changed = false;
for mut entry in self.data_streams.iter_mut() {
let before = entry.backing_indices.len();
entry.backing_indices.retain(|b| b != index);
changed |= entry.backing_indices.len() != before;
}
if changed {
self.flush_cluster_state()?;
}
Ok(())
}
/// Set (or clear) `is_write_index` on one alias member while preserving
/// that member's filter/routing metadata.
fn set_alias_write_flag(&self, index: &str, alias: &str, write: bool) {
let mut entry = self
.index_alias_metadata
.entry(index.to_string())
.or_insert_with(|| serde_json::json!({}));
if !entry.is_object() {
*entry = serde_json::json!({});
}
let aliases = entry.as_object_mut().expect("alias metadata object");
let alias_meta = aliases
.entry(alias.to_string())
.or_insert_with(|| serde_json::json!({}));
if !alias_meta.is_object() {
*alias_meta = serde_json::json!({});
}
alias_meta
.as_object_mut()
.expect("alias metadata object")
.insert("is_write_index".to_string(), serde_json::json!(write));
}
/// Roll over a data stream: create the next backing index and update the alias.
pub fn rollover_data_stream(&self, name: &str) -> Result<String> {
// Same reason as `create_data_stream`, plus: the generation counter is
// the one value that must never be handed out twice, so a rollover
// that cannot record it must not create its backing index at all.
self.ensure_cluster_state_writable()?;
let mut ds = self
.data_streams
.get_mut(name)
.ok_or_else(|| EngineError::Common(xerj_common::XerjError::index_not_found(name)))?;
let previous_backings = ds.backing_indices.clone();
ds.generation += 1;
let new_backing = format!(".ds-{}-{:06}", name, ds.generation);
drop(ds); // release borrow before calling create_index
self.create_index(&new_backing, Schema::empty())?;
// Update alias to point at the new (write) backing index.
// Keep old backing indices accessible for reads via the alias list.
if let Some(mut entry) = self.aliases.get_mut(name) {
if !entry.contains(&new_backing) {
entry.push(new_backing.clone());
}
drop(entry);
self.flush_aliases();
} else {
self.add_alias(name, &new_backing)?;
}
// Data-stream writes resolve through this alias, not a separate
// stream-write path. With two members and no write flag the alias is
// ambiguous, so the first write after the rollover used to answer 400
// "no write index is defined". Mark the new backing as the writer and
// clear every older member (preserving their filter/routing).
for old_backing in &previous_backings {
self.set_alias_write_flag(old_backing, name, false);
}
self.set_alias_write_flag(&new_backing, name, true);
self.flush_alias_metadata();
if let Some(mut ds) = self.data_streams.get_mut(name) {
ds.backing_indices.push(new_backing.clone());
}
// The generation counter is the thing that must not be lost: an
// unpersisted rollover would hand `.ds-<name>-000002` out twice after
// a restart and write into the previous generation's index.
self.flush_cluster_state()?;
info!(
name,
new_backing = new_backing.as_str(),
"data stream rolled over"
);
Ok(new_backing)
}
/// Delete a data stream and all its backing indices.
pub async fn delete_data_stream(&self, name: &str) -> Result<()> {
// Before the map lookup, so an unloaded state cannot answer
// `index_not_found` for a stream that is recorded on disk — and, more
// importantly, cannot go on to destroy its backing indices.
self.ensure_cluster_state_writable()?;
let ds = self
.data_streams
.remove(name)
.map(|(_, v)| v)
.ok_or_else(|| EngineError::Common(xerj_common::XerjError::index_not_found(name)))?;
// Destroy the backing indices FIRST, and only then record the removal.
// The two orderings fail differently and only one of them is
// recoverable:
//
// destroy → record (this one): a crash in the window leaves
// `cluster_state.json` still describing the stream while some of its
// `.ds-*` directories are already gone. The next boot restores the
// stream and warns `restored data stream references backing indices
// that are not present in the data dir` (see `apply_cluster_state`),
// `GET /_data_stream/<name>` still answers, and re-issuing the
// DELETE finishes the job.
//
// record → destroy: a crash in the window strands
// `.ds-<name>-00000N` directories that no data-stream API can reach
// — GET and DELETE answer 404 while `PUT /_data_stream/<name>`
// answers `409 resource_already_exists_exception` forever, and boot
// does not reconcile it. That is the same permanent wedge this PR
// fixes for an interrupted rollover, so it is not the ordering to
// ship.
//
// A backing index that cannot be destroyed therefore fails the
// delete: the stream is put back, the removal is never recorded, and
// the caller gets the error rather than being told `acknowledged`
// about data that is still on disk.
//
// Note what "fails" does *not* mean. The loop does not stop at the
// first error — every backing index that can be destroyed is
// destroyed, and only the ones that could not are still there. The
// 500 says the delete did not finish, not that nothing happened; the
// caller asked for the whole stream to go, so making the progress we
// can is the useful half. A retry is safe and completes the job — an
// already-deleted backing index is simply absent from `self.indices`
// on the next pass.
let mut delete_err: Option<EngineError> = None;
for backing in &ds.backing_indices {
let Some((_, idx)) = self.indices.remove(backing) else {
// Not open: nothing left to destroy under this name (a
// previous attempt already removed it).
continue;
};
match idx.delete_all_data().await {
Ok(()) => {}
// `remove_dir_all` reports an already-absent directory as
// NotFound, which is the outcome we were asking for.
Err(EngineError::Io(e)) if e.kind() == std::io::ErrorKind::NotFound => {}
Err(e) => {
warn!(
data_stream = name,
backing = backing.as_str(), error = %e,
"backing index could not be deleted — the data stream \
is left in place so the DELETE can be retried"
);
// Put it back so a retry can try again, and so the stream
// we are about to restore does not point at an index the
// engine has forgotten how to reach.
self.indices.insert(backing.clone(), idx);
if delete_err.is_none() {
delete_err = Some(e);
}
}
}
}
if let Some(e) = delete_err {
self.data_streams.insert(name.to_string(), ds);
return Err(e);
}
// Only now is the removal safe to make durable. A write failure here
// means the data is gone but the document still names the stream —
// the recoverable half of the window above, so put the stream back in
// memory as well, keeping memory and disk saying the same thing, and
// report the error instead of acknowledging.
if let Err(e) = self.flush_cluster_state() {
self.data_streams.insert(name.to_string(), ds);
return Err(e);
}
// Remove the alias. Last, so a failure anywhere above leaves the
// stream fully addressable for the retry.
self.aliases.remove(name);
self.flush_aliases();
info!(name, "data stream deleted");
Ok(())
}
/// Return a reference to the engine configuration.
///
/// Useful for handlers that need to read turbo-mode settings without
/// coupling to the full engine internals.
pub fn config(&self) -> &Config {
&self.config
}
/// Immutable boot classification used by readiness and diagnostics.
pub fn cluster_state_boot_status(&self) -> &ClusterStateBootStatus {
&self.cluster_state_boot
}
/// Return the effective embedding identity without exposing configuration
/// secrets or local asset paths.
/// The width a bare `semantic_text` is sized at (#2407): the backend's
/// confirmed width when one exists, else the built-in default, with a
/// warning, since that fallback only matches a 384-wide backend.
pub fn confirmed_semantic_dimensions(&self) -> usize {
let unconfirmed = match self.embedding_execution_identity() {
Ok(identity) => match identity.dimensions {
Some(dimensions) => return dimensions,
None => format!("the {} backend has not confirmed a width", identity.backend),
},
Err(error) => error.to_string(),
};
warn!(
reason = %unconfirmed,
fallback = xerj_ai::local::DEFAULT_DIMS,
"bare semantic_text sized at the built-in default width; declare `dimensions` \
if the embedding backend returns another width"
);
xerj_ai::local::DEFAULT_DIMS
}
pub fn embedding_execution_identity(&self) -> Result<EmbeddingExecutionIdentity> {
// #2306: a proxy resume needs a confirmed vector width. The probe runs
// here, on the identity request autoindex makes before it writes, and
// not on index open — open compares the model+endpoint digest, which
// does not wait on the endpoint.
crate::index::confirm_proxy_embedding_width(&self.config.embedding, &self.data_dir);
crate::index::embedding_execution_identity(&self.config.embedding)
}
/// Sum of every open index's live memtable footprint, in bytes. This is
/// the quantity the process-wide memtable budget (item 1) is checked
/// against — per-index back-pressure only ever sees one index's slice.
pub fn total_memtable_bytes(&self) -> u64 {
self.indices
.iter()
.map(|e| e.value().memtable_bytes() as u64)
.sum()
}
/// Sum of every open index's in-memory HNSW graph estimate (#2427),
/// reported as the `vector_graph` ingest-memory category.
pub fn total_vector_graph_bytes(&self) -> u64 {
self.indices
.iter()
.map(|e| e.value().vector_graph_bytes() as u64)
.sum()
}
/// Sum of every open index's version-map estimate (#2427), reported as
/// the `version_map` ingest-memory category.
pub fn total_version_map_bytes(&self) -> u64 {
self.indices
.iter()
.map(|e| e.value().version_map_bytes() as u64)
.sum()
}
/// Memory-pressure drain (issue #948): flush every loaded index's
/// memtable, then release its rebuildable caches via
/// [`crate::index::Index::release_memory`], then hand the freed pages
/// back to the OS through the allocator-purge hook (installed by the
/// server binary; a no-op elsewhere).
///
/// Why the flush comes first: the memtable holds not-yet-published
/// writes that no cache release can reclaim, and on the raw-bytes bulk
/// path its live heap is the payload itself — the exact bytes a pinned
/// breaker needs gone. Caches only ever mirror durable data, so this is
/// the same ordering [`Self::close_index`] uses (#463), applied to every
/// index without closing anything: reads re-hydrate from disk.
///
/// Best-effort by design: an index whose flush fails is still
/// cache-released and never aborts the drain — a drain must not become a
/// new way to wedge. Returns the number of indexes drained; closed
/// indexes are skipped (their caches were already released at close).
pub async fn drain_rebuildable_memory(&self) -> usize {
let loaded = self.open_indices_largest_memtable_first();
self.release_rebuildable_memory(&loaded).await
}
/// The breaker-driven drain (#2427/#1122): always flush memtables and
/// purge the allocator, but release the rebuildable caches only when
/// `release_caches` is set. The sampler requests a drain on every tick
/// while the breaker is engaged; releasing every cache each time made a
/// read-heavy node re-hydrate cold on every query, which refilled RSS and
/// re-engaged the breaker — a ~2 s thrash loop that drained nothing
/// durable. The worker spaces cache releases with
/// [`PressureCacheReleaseGate`]; `_cache/clear` and the direct
/// [`Self::drain_rebuildable_memory`] still release every time.
pub async fn pressure_drain(&self, release_caches: bool) -> usize {
let loaded = self.open_indices_largest_memtable_first();
if release_caches {
return self.release_rebuildable_memory(&loaded).await;
}
// One shard at a time (#2427): the flush working set is uncharged,
// and building every shard at once is what pushed RSS past the cap
// in the middle of the drain meant to bring it down.
for idx in &loaded {
if idx.memtable_bytes() > 0 && idx.flush_one_shard_at_a_time().await.is_err() {
tracing::warn!("memory pressure drain: flush failed");
}
}
purge_allocator_pages();
loaded.len()
}
/// Open indices, largest live memtable first (#2321).
///
/// The summed budget stays engaged until enough bytes are retired. DashMap
/// iteration order can flush a tiny index first and leave the breaker up
/// for the rest of the walk; the index holding the bytes goes first so
/// the next sample can admit writes while the smaller ones are still
/// publishing.
fn open_indices_largest_memtable_first(&self) -> Vec<Arc<Index>> {
let mut loaded: Vec<Arc<Index>> = self
.indices
.iter()
.filter(|e| !self.closed_indices.contains_key(e.key().as_str()))
.map(|e| Arc::clone(e.value()))
.collect();
loaded.sort_by_key(|idx| std::cmp::Reverse(idx.memtable_bytes()));
loaded
}
/// Publish memtables that are holding the process budget, before a search
/// walks them (#2321).
///
/// A `{prefix}-*` search scans every index's memtable, frozen generations
/// included. On the 8 GiB auto tier that sum sits at 2 GiB while each
/// index is still under its own flush line — the same shape that answers
/// writes with HTTP 429. Scanning it is the multi-minute query a restart
/// (flush to segments) brought down to seconds. Flush, largest first, and
/// wait until the charged bytes drop or 30 s pass. Cache release is
/// intentionally not done here: the search that follows has to read the
/// segments just published.
///
/// A no-op when the memtable breaker is off, so an ordinary search does
/// not flush. No governor (an `Index` built directly) is also a no-op.
pub async fn publish_memtables_if_budget_engaged(&self) -> usize {
let Some(governor) = crate::governor::global() else {
return 0;
};
if !governor.memtable_breaker_engaged() {
return 0;
}
let loaded = self.open_indices_largest_memtable_first();
let mut published = 0usize;
for idx in &loaded {
if idx.memtable_bytes() > 0 && idx.flush().await.is_ok() {
published += 1;
}
}
let deadline = std::time::Instant::now() + std::time::Duration::from_secs(30);
loop {
governor.refresh_memtable(self.total_memtable_bytes());
if !governor.memtable_breaker_engaged() || std::time::Instant::now() >= deadline {
break;
}
tokio::time::sleep(std::time::Duration::from_millis(25)).await;
}
published
}
/// Flush then release the rebuildable caches of exactly `indices`, then
/// purge the allocator (issue #950). The shared body of the
/// memory-pressure drain and of `POST /{index}/_cache/clear`, so the
/// operator-facing clear and the breaker's drain cannot drift apart: same
/// flush-first ordering as [`Self::close_index`], nothing closed, reads
/// re-hydrate from disk. Best-effort per index. Returns how many were
/// released.
pub async fn release_rebuildable_memory(&self, indices: &[Arc<Index>]) -> usize {
for idx in indices {
// Serial for the same reason as `pressure_drain` (#2427): a
// release must not spike RSS on its way down.
if idx.memtable_bytes() > 0 && idx.flush_one_shard_at_a_time().await.is_err() {
tracing::warn!("memory release: flush failed, releasing caches anyway");
}
idx.release_memory();
}
purge_allocator_pages();
indices.len()
}
/// Spawn the background resource sampler (item 1/3): every
/// [`crate::governor::SAMPLE_INTERVAL_MS`] it refreshes the governor's
/// summed-memtable / RSS / disk-usage atomics, which drive the hot-path
/// admission checks. Uses a `Weak` self-pointer so it exits when the
/// engine is dropped. Idempotent-safe to call once after Arc-wrapping.
///
/// LOAD-BEARING: runs on a dedicated OS thread, NOT a `tokio::spawn` task.
/// The whole point of the parent breaker is to fire under a runaway ingest
/// load — exactly the condition that saturates the tokio worker pool. A
/// tokio-task sampler starves under that load: its `interval.tick()` wakes
/// but never gets scheduled, so the trip flags never flip and the process
/// OOMs anyway (observed: a MemoryMax=2G ingest reached the 2G cap and was
/// OOM-killed while `memory_tripped` stayed false). A plain thread with
/// `std::thread::sleep` is immune to runtime starvation. All work here is
/// sync (DashMap scan + two syscalls), so no runtime is needed.
pub fn spawn_resource_sampler(self: &Arc<Self>) {
// Nothing to drive the flags if the governor was never installed.
if crate::governor::global().is_none() {
return;
}
let period = std::time::Duration::from_millis(crate::governor::SAMPLE_INTERVAL_MS);
// ── Thread C: memory-pressure drain worker (#948) ─────────────
// Thread A (below) must touch no engine lock, so it cannot run the
// drain itself: it only requests one over this bounded channel. The
// worker owns a single-threaded tokio runtime so the drain's async
// flushes work even though this thread lives outside the server
// runtime; a capacity-1 channel with try_send makes coalescing
// automatic — requests that arrive mid-drain are dropped, and the
// next sampler tick re-requests if the breaker is still engaged.
// Requires a tokio handle at spawn time so flush internals that
// spawn detached work (merge handoff) land on the REAL runtime
// instead of dying with this thread's runtime.
let drain_handle = tokio::runtime::Handle::try_current().ok();
let (drain_tx, drain_rx) = std::sync::mpsc::sync_channel::<()>(1);
let weak_c = Arc::downgrade(self);
let drain_worker = spawn_governor_thread("xerj-mem-drain", move || {
let Some(handle) = drain_handle else {
// Spawned outside a runtime (unit tests): the drain
// method stays directly callable; only the sampler's
// automatic wiring is inert.
return;
};
let mut release_gate = PressureCacheReleaseGate::from_env();
while let Ok(()) = drain_rx.recv() {
let Some(engine) = weak_c.upgrade() else {
return; // engine dropped
};
let release_caches = release_gate.should_release(std::time::Instant::now());
let drained = handle.block_on(engine.pressure_drain(release_caches));
if release_caches {
tracing::info!(
indexes = drained,
"memory breaker engaged — drained rebuildable state (flush + cache release + purge)"
);
} else {
tracing::debug!(
indexes = drained,
"memory breaker engaged — flushed and purged; caches kept (released within the last {:?})",
release_gate.interval()
);
}
}
});
// Publish the request channel only once a worker is draining it: with
// no worker, the sampler's requests have nowhere to go.
if drain_worker {
let _ = MEMORY_DRAIN_TX.set(drain_tx);
}
// ── Thread A: memory + disk ──────────────────────────────────────
// Touches NO engine lock — just `/proc/self/statm` + `statvfs`. Kept
// on its own thread so a memtable-sum stall (below) can NEVER delay
// the memory breaker: the observed OOM was exactly RSS climbing past
// the watermark while the sampler was blocked summing memtables under
// a turbo batch's shard write-locks. Uses a `Weak` liveness check to
// exit when the engine is dropped.
let weak_a = Arc::downgrade(self);
let data_dir = self.data_dir.to_string_lossy().to_string();
let cluster_settings = self.cluster_settings.clone();
const FLOOD_STAGE_SETTING: &str = "cluster.routing.allocation.disk.watermark.flood_stage";
const BREAKER_LIMIT_SETTING: &str = "indices.breaker.total.limit";
const DISK_THRESHOLD_SETTING: &str = "cluster.routing.allocation.disk.threshold_enabled";
// Snapshot the uncapped machine memory once: a percentage override is
// resolved against it, and it is the hard clamp for an absolute one.
let machine_memory_bytes = crate::governor::effective_memory_limit_bytes();
spawn_governor_thread("xerj-mem-sampler", move || {
tracing::info!(
period_ms = crate::governor::SAMPLE_INTERVAL_MS,
"memory/disk sampler thread started (parent circuit breaker)"
);
// Two clocks (#948). RSS backs off when a drain does not lower
// it (#1981). The summed-memtable latch stays on the 2 s window
// (#2238), including while that RSS clock is already stretched:
// a stable RSS is the normal shape of a memtable-only trip, and
// backing off there leaves corpus index at 429 with nothing
// flushed. try_send never blocks, so Thread A's no-lock contract
// still holds.
let mut rss_drains =
crate::governor::MemoryDrainSchedule::new(MEMORY_DRAIN_MIN_INTERVAL);
let mut last_memtable_drain_at: Option<std::time::Instant> = None;
loop {
std::thread::sleep(period);
if weak_a.strong_count() == 0 {
return; // engine dropped
}
let governor = match crate::governor::global() {
Some(g) => g,
None => return,
};
// Working set on Linux and macOS. On Windows, the max of the
// working set and commit charge, plus available commit, so a
// no-pagefile box trips before VirtualAlloc aborts (#1122).
let observed = crate::governor::current_process_memory_bytes();
let available_commit = xerj_common::resource::available_commit_bytes();
let (disk_pct, disk_avail) = crate::governor::disk_used_pct_and_avail(&data_dir);
governor.refresh_memory_observing(observed, disk_pct, disk_avail, available_commit);
if sample_drain_due(
std::time::Instant::now(),
// Same figure the breaker just compared: working set, or
// on Windows the max of working set and commit charge.
// Backing off on the working set alone would stretch
// drains while commit charge is the value still climbing.
observed,
governor.memory_breaker_engaged(),
governor.memtable_breaker_engaged(),
&mut rss_drains,
&mut last_memtable_drain_at,
) {
if let Some(tx) = MEMORY_DRAIN_TX.get() {
let _ = tx.try_send(());
}
}
// Runtime disk flood-stage override, matching ES's
// `PUT _cluster/settings` unblock without a restart. A
// non-blocking read: on contention we just keep last
// tick's threshold and retry next sample, never stall
// the memory/disk breaker on this lock.
if let Ok(settings) = cluster_settings.try_read() {
let flood_override = settings
.get("persistent")
.and_then(|p| p.get(FLOOD_STAGE_SETTING))
.or_else(|| {
settings
.get("transient")
.and_then(|t| t.get(FLOOD_STAGE_SETTING))
})
.and_then(crate::governor::parse_flood_stage_pct);
governor.set_disk_flood_pct_override(flood_override);
// `threshold_enabled: false` turns the flood stage
// off and releases an engaged block, as in ES.
let threshold_enabled = settings
.get("persistent")
.and_then(|p| p.get(DISK_THRESHOLD_SETTING))
.or_else(|| {
settings
.get("transient")
.and_then(|t| t.get(DISK_THRESHOLD_SETTING))
})
.and_then(crate::governor::parse_setting_bool);
governor.set_disk_threshold_enabled(threshold_enabled);
// Runtime parent-memory-breaker watermark override
// (#1250): raise the watermark on a big box live via
// `PUT _cluster/settings {indices.breaker.total.limit}`,
// no restart — the live equivalent of the boot-only
// XERJ_MAX_PROCESS_MEMORY_MB cap.
let breaker_override = settings
.get("persistent")
.and_then(|p| p.get(BREAKER_LIMIT_SETTING))
.or_else(|| {
settings
.get("transient")
.and_then(|t| t.get(BREAKER_LIMIT_SETTING))
})
.and_then(|v| {
crate::governor::parse_breaker_total_limit(v, machine_memory_bytes)
});
governor.set_memory_watermark_override(breaker_override);
}
}
});
// ── Thread B: summed memtable budget ─────────────────────────────
// Each per-index `size_bytes()` is one relaxed atomic load of the
// aggregate the shards maintain incrementally — no shard locks, so
// this never blocks under a turbo batch (#872).
let weak_b = Arc::downgrade(self);
spawn_governor_thread("xerj-memtable-sampler", move || loop {
std::thread::sleep(period);
let engine = match weak_b.upgrade() {
Some(e) => e,
None => return, // engine dropped
};
let governor = match crate::governor::global() {
Some(g) => g,
None => return,
};
governor.refresh_memtable(engine.total_memtable_bytes());
});
}
// ── Transform pipeline methods ────────────────────────────────────────────
/// The `index.default_pipeline` setting for `index`, if it has one.
///
/// One implementation for every write path. It used to live only in the
/// `_doc` handlers, which is why `_bulk` ignored the setting entirely and
/// stored documents untransformed under a `201` while the single-document
/// form of the same write refused (issue #204).
///
/// Handles the three shapes settings arrive in — nested
/// `{index: {default_pipeline: ..}}`, dotted `{"index.default_pipeline": ..}`
/// and flat `{default_pipeline: ..}` — mirroring the `number_of_shards`
/// lookup pattern used elsewhere.
///
/// `index` is matched literally: an alias has no settings of its own, so a
/// write addressed to an alias does not pick up its backing index's default
/// pipeline. That is a known gap, identical on every write path.
pub fn default_pipeline_for(&self, index: &str) -> Option<String> {
if let Some(p) = self
.index_settings
.get(index)
.and_then(|v| pipeline_setting(v.value()))
{
return Some(p);
}
// A write that auto-creates the index resolves the pipeline before
// that create (#2281). The template's default applies only then.
// An index that already exists keeps the pipeline persisted in its
// `settings.json`, which boot, retry, and snapshot restore copy back
// into this map (#2294). Falling through to the template would attach
// a pipeline the index was not created with.
if self.indices.contains_key(index) || self.failed_indices.contains_key(index) {
return None;
}
self.best_matching_template(index)
.and_then(|t| pipeline_setting(&t.settings))
}
/// Compile and register a typed transform pipeline from a JSON config,
/// **in memory only**.
///
/// `config_json` must be a valid [`PipelineConfig`](xerj_wasm::pipeline::PipelineConfig)
/// object. The compiled pipeline is stored in `transform_pipelines` and
/// the raw JSON in `pipelines`, so the ES-compatible ingest API can hand
/// it back.
///
/// This is the primitive underneath [`Engine::put_pipeline`] and the boot
/// path; it deliberately does not touch disk, so it is private. Anything
/// that serves a user request must go through `put_pipeline`, or the
/// pipeline is gone at the next restart (issue #203).
fn compile_pipeline(
&self,
name: &str,
config_json: Value,
) -> std::result::Result<(), xerj_wasm::WasmError> {
self.compile_pipeline_with_mode(name, config_json, xerj_wasm::pipeline::CompileMode::Strict)
}
/// [`Self::compile_pipeline`], with the strictness chosen by the caller.
///
/// Issue #204. The boot replay passes
/// [`CompileMode::ReplayPersisted`](xerj_wasm::pipeline::CompileMode) — a
/// definition already in `cluster_state.json` was accepted by whichever
/// build wrote it, and there is no caller left to answer 400. Refusing it
/// at boot would flip a working cluster's writes from 201 to 400 on a
/// binary upgrade alone, with no PUT and no operator action. Whatever the
/// replay had to wave through is recorded in `degraded_pipelines` and
/// logged at ERROR, so it is loud instead of silent.
fn compile_pipeline_with_mode(
&self,
name: &str,
config_json: Value,
mode: xerj_wasm::pipeline::CompileMode,
) -> std::result::Result<(), xerj_wasm::WasmError> {
let cfg: xerj_wasm::pipeline::PipelineConfig =
serde_json::from_value(config_json.clone()).map_err(xerj_wasm::WasmError::Json)?;
let pipeline = match mode {
xerj_wasm::pipeline::CompileMode::Strict => {
xerj_wasm::pipeline::Pipeline::from_config(name, &cfg)?
}
xerj_wasm::pipeline::CompileMode::ReplayPersisted => {
xerj_wasm::pipeline::Pipeline::from_config_replaying_persisted(name, &cfg)?
}
};
if pipeline.degradations.is_empty() {
self.degraded_pipelines.remove(name);
} else {
let summary = pipeline.degradations.join("; ");
error!(
pipeline = name,
degradations = summary.as_str(),
"persisted ingest pipeline uses configuration this build cannot honour — it \
is still running exactly as the build that stored it ran it, but it is NOT \
doing what the definition says; re-PUT the definition to see the error"
);
self.degraded_pipelines.insert(name.to_string(), summary);
}
self.pipelines.insert(name.to_string(), config_json);
self.transform_pipelines.insert(name.to_string(), pipeline);
// Issue #204: a successful compile clears any recorded "stored but
// not runnable" reason left by a previous definition under this id —
// including the persisted refusal marker, whose only job is to keep a
// refusal alive across a restart. Persisting the removal is the
// caller's (`put_pipeline`, and the boot replay's self-heal); an
// orphaned marker only ever makes a pipeline more refused, and the
// next successful PUT clears it again.
self.unrunnable_pipelines.remove(name);
self.definition_time_refusals.remove(name);
Ok(())
}
/// Store a pipeline definition that could not be compiled, together with
/// the reason, and make sure nothing stale is left behind.
///
/// Issue #204. This exists for one case: a definition xerj accepts for wire
/// compatibility (Elasticsearch does) but cannot execute — because it names
/// a processor this build does not implement, or sets an option on one that
/// this build cannot honour (a processor-level `if` guard, an `on_failure`
/// chain, a `grok` `patterns` array). Storing it keeps
/// `GET`/`_simulate` truthful about what was submitted; recording the
/// reason is what stops [`Self::process_through_pipeline`] from silently
/// passing documents through untransformed. A previously-compiled pipeline
/// under the same id is REMOVED — re-defining a pipeline must not leave the
/// old stages quietly running under the new definition's name.
///
/// Like [`Engine::put_pipeline`], the stored definition is flushed to
/// `cluster_state.json` before the caller may acknowledge it (issue
/// #203). `Err` means it did not reach disk; the in-memory change is
/// rolled back first.
///
/// The *reason* is persisted too, in
/// `<data_dir>/ingest-pipeline-refusals.json`, and that is not
/// belt-and-braces. This docstring used to claim the reason could stay in
/// memory "because the boot replay fails to recompile the stored
/// definition and re-records why" — which is false for the shape that
/// matters. A `stages`-shaped definition (the one `GET` hands an operator
/// to edit and re-`PUT`) recompiles at boot in the compatibility mode that
/// exists for definitions written by a gate-less build, so the refusal was
/// SOFTENED by a restart: measured, a refused `{"remove": {"field":
/// "secret", "if": …}}` went from `400` to `201` across a plain restart,
/// with `secret` dropped from a document the guard excludes. The marker is
/// what tells the replay "this build already answered 400 for this exact
/// definition", so the flush below is on the acknowledge path exactly like
/// the definition's own.
pub fn register_unrunnable_pipeline(
&self,
name: &str,
definition: Value,
reason: String,
) -> Result<()> {
self.ensure_cluster_state_writable()?;
let previous_doc = self.pipelines.get(name).map(|v| v.value().clone());
let previous_compiled = self.transform_pipelines.remove(name).map(|(_, p)| p);
let previous_reason = self
.unrunnable_pipelines
.get(name)
.map(|r| r.value().clone());
let previous_marker = self
.definition_time_refusals
.get(name)
.map(|r| r.value().clone());
self.pipelines.insert(name.to_string(), definition);
self.unrunnable_pipelines
.insert(name.to_string(), reason.clone());
// The marker that makes this refusal survive a restart. Without it the
// boot replay softens the refusal and the pipeline silently starts
// running, ignoring the very option it was refused for (issue #204).
self.definition_time_refusals
.insert(name.to_string(), reason.clone());
// Nothing is running under this id any more, so nothing is running
// degraded either.
let previous_degradation = self.degraded_pipelines.remove(name).map(|(_, v)| v);
macro_rules! rollback {
() => {{
match previous_doc {
Some(d) => self.pipelines.insert(name.to_string(), d),
None => self.pipelines.remove(name).map(|(_, v)| v),
};
match previous_reason {
Some(r) => self.unrunnable_pipelines.insert(name.to_string(), r),
None => self.unrunnable_pipelines.remove(name).map(|(_, v)| v),
};
match previous_marker {
Some(r) => self.definition_time_refusals.insert(name.to_string(), r),
None => self.definition_time_refusals.remove(name).map(|(_, v)| v),
};
if let Some(p) = previous_compiled {
self.transform_pipelines.insert(name.to_string(), p);
}
if let Some(d) = previous_degradation {
self.degraded_pipelines.insert(name.to_string(), d);
}
}};
}
// Marker before definition — see `put_pipeline` for why that order is
// load-bearing. An `Err` from either is NOT acknowledgeable: a refusal
// the next boot cannot see is the whole defect.
if let Err(e) = self.flush_pipeline_refusals() {
rollback!();
return Err(e);
}
if let Err(e) = self.flush_cluster_state() {
rollback!();
self.flush_pipeline_refusals_best_effort();
return Err(e);
}
warn!(
name,
reason = reason.as_str(),
"ingest pipeline stored but NOT runnable — documents may not be ingested through it"
);
Ok(())
}
/// Run `docs` through a named pipeline, returning `(action, doc)` pairs.
///
/// Returns [`xerj_wasm::WasmError::PipelineNotFound`] when `pipeline_name`
/// does not exist. Documents with a [`ProcessAction::Drop`] action are
/// still returned in the output — callers decide whether to skip indexing.
pub fn process_through_pipeline(
&self,
pipeline_name: &str,
mut docs: Vec<Value>,
) -> std::result::Result<Vec<(xerj_wasm::pipeline::ProcessAction, Value)>, xerj_wasm::WasmError>
{
let pipeline = self.transform_pipelines.get(pipeline_name).ok_or_else(|| {
// Issue #204: distinguish "no such pipeline" from "stored but this
// build cannot run it". Both must fail — what must not happen is a
// document being indexed untransformed — but reporting the second
// as the first sent operators hunting for a pipeline that is
// plainly there in `GET /_ingest/pipeline`.
match self.unrunnable_pipelines.get(pipeline_name) {
Some(reason) => xerj_wasm::WasmError::PipelineNotRunnable {
pipeline: pipeline_name.to_string(),
reason: reason.clone(),
},
None => xerj_wasm::WasmError::PipelineNotFound(pipeline_name.to_string()),
}
})?;
let actions = pipeline.process_batch(&mut docs);
Ok(actions.into_iter().zip(docs).collect())
}
// ── Snapshot / Restore ────────────────────────────────────────────────────
/// Create a filesystem snapshot of all (or named) indices.
///
/// For each index this copies:
/// - WAL files (`<index>/wal/`)
/// - Segment files (`<index>/segments/`)
/// - Schema and settings JSON files
///
/// A `manifest.json` is written at the snapshot root listing every index
/// and its files so that `restore_snapshot` can replay them.
pub async fn create_snapshot(
&self,
repo_path: &str,
name: &str,
indices: Option<Vec<String>>,
) -> Result<Value> {
self.ensure_cluster_state_writable()?;
let snap_dir = std::path::Path::new(repo_path).join(name);
// No hub-pack root here: the pack cache is a restore-only source, so a
// client can never write a snapshot into the user's home directory.
let snap_dir = validate_snapshot_path(
repo_path,
name,
&snap_dir,
&self.data_dir,
&self.config.limits.snapshot_repo_allowlist,
None,
)?;
std::fs::create_dir_all(&snap_dir).map_err(EngineError::Io)?;
// Wall-clock start, captured BEFORE the flush+copy work so
// duration_in_millis reflects the real elapsed time (it was hardcoded
// to 0 because start_time == end_time were sampled at the same instant
// after the copy).
let start_ms = now_millis();
let target_indices: Vec<String> = match indices {
Some(list) if !list.is_empty() => list,
// ES excludes system indices from an all-indices snapshot; exclude
// our `.xerj_*` internals (auth, sessions, dashboards, …) by
// default so a plain snapshot captures only user data.
_ => self
.indices
.iter()
.map(|e| e.key().clone())
.filter(|n| !is_system_index(n))
.collect(),
};
for idx_name in &target_indices {
let idx = match self.indices.get(idx_name.as_str()) {
Some(i) => i,
None => continue,
};
// Flush memtable so all data is on disk before copying.
let _ = idx.flush().await;
let src_dir = idx.data_dir().to_path_buf();
let dst_dir = snap_dir.join(idx_name);
std::fs::create_dir_all(&dst_dir).map_err(EngineError::Io)?;
// Copy everything recursively (WAL + segments + schema). The
// per-file list is intentionally discarded: it was serialized into
// the response as `index_files` (~170 KB of dead weight that ES
// never returns and `restore_snapshot` never reads — restore copies
// the snapshot dir back wholesale and reopens by index name).
let mut files: Vec<String> = Vec::new();
copy_dir_recursive(&src_dir, &dst_dir, &mut files).map_err(EngineError::Io)?;
}
let end_ms = now_millis();
let manifest = serde_json::json!({
"snapshot": name,
"uuid": uuid::Uuid::new_v4().to_string(),
"version": "8.13.0",
"indices": target_indices,
"state": "SUCCESS",
"start_time_in_millis": start_ms,
"end_time_in_millis": end_ms,
"duration_in_millis": (end_ms - start_ms).max(0),
"failures": [],
"shards": {
"total": target_indices.len(),
"failed": 0,
"successful": target_indices.len(),
},
});
let manifest_path = snap_dir.join("manifest.json");
let bytes = serde_json::to_vec_pretty(&manifest).map_err(EngineError::Serde)?;
std::fs::write(&manifest_path, bytes).map_err(EngineError::Io)?;
info!(snapshot = name, repo = repo_path, "snapshot created");
Ok(manifest)
}
/// Restore a snapshot: copies files back and reopens the indices.
///
/// `indices` is the ES restore-body `indices` filter: a list of index
/// names / wildcard patterns (each entry may itself be a comma-separated
/// multi-target expression, ES accepts both the string and array forms).
/// `None` / empty restores every index in the snapshot (ES default).
/// A non-wildcard entry that matches nothing in the snapshot is an
/// error — silently restoring nothing (or worse, everything) would
/// misreport what was rewritten. Returns the list of index names
/// actually restored.
///
/// This filter used to be IGNORED entirely: a restore request naming
/// one index rewrote EVERY index in the snapshot with snapshot-time
/// state, destroying all writes made since (live-verified 2026-07-12).
pub async fn restore_snapshot(
&self,
repo_path: &str,
name: &str,
indices: Option<Vec<String>>,
) -> Result<Vec<String>> {
self.ensure_cluster_state_writable()?;
let snap_dir = std::path::Path::new(repo_path).join(name);
// `limits.hub_pack_restore`: the Corpus Hub pack cache of the user this
// server runs as is a restore source by default, so `xerj hub pull`
// works against a server the user started by hand (not only against the
// node `xerj ask` starts, which allowlists it in its own config).
let hub_packs = if self.config.limits.hub_pack_restore {
xerj_common::hub::hub_packs_dir()
} else {
None
};
let snap_dir = validate_snapshot_path(
repo_path,
name,
&snap_dir,
&self.data_dir,
&self.config.limits.snapshot_repo_allowlist,
hub_packs.as_deref(),
)?;
let manifest_path = snap_dir.join("manifest.json");
let manifest_bytes = std::fs::read(&manifest_path).map_err(EngineError::Io)?;
let manifest: Value =
serde_json::from_slice(&manifest_bytes).map_err(EngineError::Serde)?;
let snapshot_indices: Vec<String> = manifest
.get("indices")
.and_then(Value::as_array)
.map(|arr| {
arr.iter()
.filter_map(|v| v.as_str().map(|s| s.to_string()))
.collect::<Vec<String>>()
})
.unwrap_or_default();
// Apply the `indices` filter. Split each entry on commas, then match
// each pattern against the snapshot's indices (glob `*` wildcards,
// same matcher the search path uses for index selectors).
let patterns: Vec<String> = indices
.unwrap_or_default()
.iter()
.flat_map(|entry| entry.split(','))
.map(|s| s.trim().to_string())
.filter(|s| !s.is_empty())
.collect();
let index_names: Vec<String> = if patterns.is_empty() {
// Default restore: every index in the snapshot EXCEPT `.xerj_*`
// system indices (ES excludes system indices from a default
// restore). Snapshots taken by this build already omit them, but a
// snapshot written by an older build may still carry them.
snapshot_indices
.into_iter()
.filter(|n| !is_system_index(n))
.collect()
} else {
let mut selected: Vec<String> = Vec::new();
for pat in &patterns {
// A wildcard never matches a `.xerj_*` system index (ES `*`
// semantics); only a pattern that explicitly targets the
// dot-namespace (`.xerj_*`, `.xerj_users`, …) may.
let targets_system = pat.starts_with('.');
let matched: Vec<&String> = snapshot_indices
.iter()
.filter(|n| glob_match(pat, n))
.filter(|n| targets_system || !is_system_index(n))
.collect();
if matched.is_empty() && !pat.contains('*') {
// ES: restore of an index absent from the snapshot fails
// loud (snapshot_restore_exception). Mirror that. The
// repo's filesystem path is deliberately NOT echoed.
return Err(EngineError::Common(xerj_common::XerjError::invalid_query(
format!("[{name}] no index matches [{pat}] in snapshot"),
)));
}
for m in matched {
if !selected.contains(m) {
selected.push(m.clone());
}
}
}
selected
};
// Canonicalize the data_dir once for the per-index containment check
// below. `self.data_dir` always exists at this point (the engine opens
// indices under it), but canonicalize defensively in case a test mounts
// a relative path that hasn't been created yet.
let data_dir_canon = self
.data_dir
.canonicalize()
.unwrap_or_else(|_| self.data_dir.clone());
for idx_name in &index_names {
// Validate the index name BEFORE any filesystem op. Previously the
// `IndexName::new` check ran AFTER `remove_dir_all`, so a manifest
// carrying `..` as an index name deleted the parent of `data_dir`.
// `IndexName::validate` now rejects `.`/`..`/separators/`..`
// substrings, so this is belt-and-suspenders against a future
// regression in the validator.
let index_name = match IndexName::new(idx_name) {
Ok(n) => n,
Err(e) => {
warn!(index = idx_name, error = %e, "skipping restore of invalid index name");
continue;
}
};
let src_dir = snap_dir.join(idx_name.as_str());
if !src_dir.exists() {
warn!(index = idx_name, "snapshot directory missing, skipping");
continue;
}
let dst_dir = self.data_dir.join(idx_name);
// Containment: lexical check BEFORE any filesystem op. `IndexName`
// validation already rejects `.`/`..`/separators/`..` substrings,
// so this is a cheap belt-and-suspenders guard against a future
// regression in the validator that would let `dst_dir` escape the
// data_dir.
if !dst_dir.starts_with(&self.data_dir) {
return Err(EngineError::Common(
xerj_common::XerjError::invalid_mapping(format!(
"refusing to restore index [{idx_name}] outside data_dir"
)),
));
}
// Remove existing index data (if any) and close it.
if self.indices.contains_key(idx_name.as_str()) {
self.indices.remove(idx_name.as_str());
}
if dst_dir.exists() {
std::fs::remove_dir_all(&dst_dir).map_err(EngineError::Io)?;
}
std::fs::create_dir_all(&dst_dir).map_err(EngineError::Io)?;
// After create_dir_all, re-verify the canonicalized path is still
// inside the canonicalized data_dir (catches symlink-based escapes
// that the lexical check misses).
if let Ok(dst_canon) = dst_dir.canonicalize() {
if !dst_canon.starts_with(&data_dir_canon) {
return Err(EngineError::Common(
xerj_common::XerjError::invalid_mapping(format!(
"refusing to restore index [{idx_name}] outside data_dir (canonical)"
)),
));
}
}
// Copy snapshot files back.
let mut _files: Vec<String> = Vec::new();
copy_dir_recursive(&src_dir, &dst_dir, &mut _files).map_err(EngineError::Io)?;
// Reopen the index.
match Index::open(index_name, &self.config, &self.data_dir) {
Ok(idx) => {
// #320 — a restored index gets the tap's live retention
// floor, not the boot file's.
idx.set_wal_min_retained_generations(self.wal_retention_floor());
// Snapshot dirs carry es_mapping.json — reload it so the
// restored index serves the same mapping it was saved with.
// A corrupt blob in the snapshot fails the restore of that
// index instead of restoring it with a reduced mapping.
if let Err(e) = self.load_persisted_es_mapping(idx_name) {
warn!(index = idx_name, error = %e, "failed to reopen restored index");
self.record_failed_index(idx_name, e.to_string());
continue;
}
if let Some(m) = self.index_mappings.get(idx_name) {
Engine::apply_date_mapping_flags(&idx, m.value());
}
// Replace whatever the live map held. A snapshot created
// without settings must not keep the previous binding.
self.rehydrate_index_settings_from_disk(idx_name);
self.indices.insert(idx_name.clone(), idx);
// Restoring from a snapshot is *the* repair for an index
// that failed to open, so clear the recorded failure —
// otherwise cluster health stays red after the repair
// actually worked.
self.failed_indices.remove(idx_name);
info!(index = idx_name, "index restored from snapshot");
}
Err(e) => {
warn!(index = idx_name, error = %e, "failed to reopen restored index");
self.record_failed_index(idx_name, e.to_string());
}
}
}
info!(
snapshot = name,
repo = repo_path,
indices = ?index_names,
"snapshot restored"
);
Ok(index_names)
}
} // end impl Engine
// ── Private helpers ───────────────────────────────────────────────────────────
/// Milliseconds since the Unix epoch (wall clock).
fn now_millis() -> i64 {
std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.unwrap_or_default()
.as_millis() as i64
}
/// A XERJ-internal system index (`.xerj_*`: auth, sessions, dashboards,
/// audit, …). ES hides its own system indices from default snapshot/restore;
/// we do the same for ours so `_snapshot`/`_restore` operate on user data.
pub(crate) fn is_system_index(name: &str) -> bool {
name.starts_with(".xerj_")
}
/// Validate a snapshot repo path and snapshot name before any filesystem op.
///
/// Returns the canonicalized `snap_dir` (creating it is the caller's job) or
/// an error. This is the chokepoint for the snapshot path-traversal surface:
/// - the snapshot `name` must not contain `..`, `/`, `\`, or NUL, so
/// `Path::new(repo).join(name)` cannot escape the repo;
/// - the canonicalized `snap_dir` must remain inside the canonicalized
/// `repo_path`, so a `repo_path` like `/tmp/repo/../..` is rejected;
/// - the `repo_path` itself must not be empty (defensive).
///
/// Pre-fix, `PUT /_snapshot/evil` with `location: "/etc"` plus a snapshot
/// `name = ".."` made `snap_dir = "/etc/.."`, so `create_dir_all` and
/// `manifest.json` landed in the parent of `/etc`. The snapshot name was
/// otherwise unvalidated.
fn validate_snapshot_path(
repo_path: &str,
name: &str,
snap_dir: &std::path::Path,
data_dir: &std::path::Path,
allowlist: &[String],
hub_packs: Option<&std::path::Path>,
) -> Result<std::path::PathBuf> {
if repo_path.is_empty() {
return Err(EngineError::Common(
xerj_common::XerjError::invalid_mapping(
"snapshot repository location must not be empty",
),
));
}
// F-PATH-02: the repo `location` is operator/attacker-supplied via
// `PUT /_snapshot/{repo}`. The name-containment checks below only prove the
// snapshot stays inside THAT location — they say nothing about whether the
// location itself is sane. Bound the location to `data_dir` (default) or a
// configured allowlist (an ES `path.repo` equivalent), so a repo cannot point
// at `/etc`, `/root/.ssh`, another tenant's dir, etc.
// F-PATH-02 (residual): a location whose fully-resolved target does not yet
// exist makes `canonicalize()` below fail, so the lexical fallback keeps any
// `..` components. A component-based `starts_with` then treats
// `<data_dir>/../../escape` as *inside* `<data_dir>` (it never normalizes
// `..`), and `create_dir_all` lets the OS walk out. Reject `..` outright —
// no legitimate snapshot root needs parent-dir traversal — so neither the
// location containment check nor the snapshot-name check can be bypassed
// with an unresolved parent reference.
if std::path::Path::new(repo_path)
.components()
.any(|c| matches!(c, std::path::Component::ParentDir))
{
return Err(EngineError::Common(
xerj_common::XerjError::invalid_mapping(format!(
"snapshot repository location [{repo_path}] must not contain '..' path components"
)),
));
}
let base_canon = |p: &std::path::Path| p.canonicalize().unwrap_or_else(|_| p.to_path_buf());
let repo_base = base_canon(std::path::Path::new(repo_path));
let mut allowed_bases: Vec<std::path::PathBuf> = vec![base_canon(data_dir)];
allowed_bases.extend(
allowlist
.iter()
.map(|b| base_canon(std::path::Path::new(b))),
);
// The hub pack cache (restore only — create passes `None`). Unlike an
// operator's allowlist entry it is implicit, so it is trusted only in the
// shape `xerj hub pull` makes it; see `trusted_hub_pack_root`.
allowed_bases.extend(hub_packs.and_then(trusted_hub_pack_root));
if !allowed_bases.iter().any(|base| repo_base.starts_with(base)) {
return Err(EngineError::Common(
xerj_common::XerjError::invalid_mapping(format!(
"snapshot repository location [{repo_path}] is outside data_dir and not in \
limits.snapshot_repo_allowlist; refusing (set the allowlist to permit external \
snapshot roots)"
)),
));
}
if name.is_empty()
|| name.contains("..")
|| name.contains('/')
|| name.contains('\\')
|| name.contains('\0')
{
return Err(EngineError::Common(xerj_common::XerjError::invalid_mapping(
format!("invalid snapshot name [{name}]: must not be empty or contain '..', '/', '\\', or NUL"),
)));
}
// Canonicalize repo_path. The repo may not yet exist (create_snapshot
// creates it on the next line), so fall back to the lexically-cleaned
// path. `Path::canonicalize` requires the path to exist.
let repo_canon = std::path::Path::new(repo_path)
.canonicalize()
.unwrap_or_else(|_| std::path::PathBuf::from(repo_path));
// If snap_dir already exists, canonicalize it and verify containment;
// if it does not yet exist (the create_snapshot case), verify lexically
// that it starts with the repo.
if let Ok(snap_canon) = snap_dir.canonicalize() {
if !snap_canon.starts_with(&repo_canon) {
return Err(EngineError::Common(
xerj_common::XerjError::invalid_mapping(format!(
"snapshot [{name}] resolves outside its repository; refusing"
)),
));
}
Ok(snap_canon)
} else {
// Not yet on disk: lexical containment check. This catches `..` in
// repo_path that canonicalize couldn't resolve because the parent
// exists but the repo dir does not.
let snap_lex = snap_dir.to_path_buf();
if !snap_lex.starts_with(&repo_canon) {
return Err(EngineError::Common(
xerj_common::XerjError::invalid_mapping(format!(
"snapshot [{name}] resolves outside its repository; refusing"
)),
));
}
Ok(snap_lex)
}
}
/// The canonical hub pack-cache root a restore may read from, or `None` when
/// `packs` is not safe to trust implicitly.
///
/// `limits.snapshot_repo_allowlist` entries are canonicalized as given: an
/// operator who allowlists a symlink means its target. The hub root is NOT an
/// operator's choice — it is derived from the environment — so it is held to
/// the shape `xerj hub pull` creates (`create_dir_all`, by the same user):
///
/// - it exists and is a real directory, not a symlink (`symlink_metadata`),
/// so `~/.xerj/hub/packs -> /` cannot widen the grant to the whole disk;
/// - canonicalizing it changes nothing but its parent's resolution (its own
/// last component is not a link);
/// - it is not a filesystem root;
/// - on Unix it is owned by the server's effective user, so a shared or
/// pre-created directory (a `HOME` under a world-writable dir) is not
/// honoured.
///
/// What is inside it is still checked per request: the caller canonicalizes the
/// repo location, so `packs/x -> /etc` resolves outside this root and is
/// refused, and `..` components are rejected before this is consulted.
fn trusted_hub_pack_root(packs: &std::path::Path) -> Option<std::path::PathBuf> {
if !packs.is_absolute() {
return None;
}
let meta = std::fs::symlink_metadata(packs).ok()?;
if meta.file_type().is_symlink() || !meta.is_dir() {
return None;
}
#[cfg(unix)]
{
use std::os::unix::fs::MetadataExt;
// SAFETY: geteuid has no preconditions and cannot fail.
if meta.uid() != unsafe { libc::geteuid() } {
return None;
}
}
let canon = packs.canonicalize().ok()?;
let parent = packs.parent()?.canonicalize().ok()?;
if canon != parent.join(packs.file_name()?) || canon.parent().is_none() {
return None;
}
Some(canon)
}
/// Recursively copy all files from `src` to `dst`, recording relative paths in `files`.
fn copy_dir_recursive(
src: &std::path::Path,
dst: &std::path::Path,
files: &mut Vec<String>,
) -> std::io::Result<()> {
for entry in std::fs::read_dir(src)? {
let entry = entry?;
let src_path = entry.path();
let file_name = entry.file_name();
let dst_path = dst.join(&file_name);
if src_path.is_dir() {
std::fs::create_dir_all(&dst_path)?;
copy_dir_recursive(&src_path, &dst_path, files)?;
} else {
std::fs::copy(&src_path, &dst_path)?;
files.push(file_name.to_string_lossy().to_string());
}
}
Ok(())
}
/// Simple glob pattern matching (supports `*` and `?`).
pub(crate) fn glob_match(pattern: &str, text: &str) -> bool {
let pat: Vec<char> = pattern.chars().collect();
let txt: Vec<char> = text.chars().collect();
let (m, n) = (pat.len(), txt.len());
let mut dp = vec![vec![false; n + 1]; m + 1];
dp[0][0] = true;
for i in 1..=m {
if pat[i - 1] == '*' {
dp[i][0] = dp[i - 1][0];
}
}
for i in 1..=m {
for j in 1..=n {
if pat[i - 1] == '*' {
dp[i][j] = dp[i - 1][j] || dp[i][j - 1];
} else if pat[i - 1] == '?' || pat[i - 1] == txt[j - 1] {
dp[i][j] = dp[i - 1][j - 1];
}
}
}
dp[m][n]
}
/// `index.default_pipeline` in any of the three spellings settings arrive in.
fn pipeline_setting(v: &Value) -> Option<String> {
v.get("index")
.and_then(|ix| ix.get("default_pipeline"))
.or_else(|| v.get("index.default_pipeline"))
.or_else(|| v.get("default_pipeline"))
.and_then(Value::as_str)
.filter(|s| !s.is_empty())
.map(str::to_string)
}
fn value_is_empty_object(v: &Value) -> bool {
v.is_null() || v.as_object().is_some_and(|m| m.is_empty())
}
/// Template mappings fill fields the request did not declare. A request
/// property replaces the template's property of the same name. Other mapping
/// keys (`dynamic`, `_source`, …) follow the same rule.
fn merge_template_mappings(template: &Value, request: Value) -> Value {
let Some(template_obj) = template.as_object().filter(|m| !m.is_empty()) else {
return request;
};
let Some(request_obj) = request.as_object().filter(|m| !m.is_empty()) else {
return template.clone();
};
let mut out = template_obj.clone();
for (key, value) in request_obj {
if key == "properties" {
let mut props = out
.get("properties")
.and_then(Value::as_object)
.cloned()
.unwrap_or_default();
if let Some(req_props) = value.as_object() {
for (field, def) in req_props {
props.insert(field.clone(), def.clone());
}
out.insert("properties".to_string(), Value::Object(props));
} else {
out.insert(key.clone(), value.clone());
}
} else {
out.insert(key.clone(), value.clone());
}
}
Value::Object(out)
}
/// Deep-merge `over` onto `base`. Objects combine; `over` wins on a leaf.
fn deep_merge_request_wins(base: &Value, over: &Value) -> Value {
match (base.as_object(), over.as_object()) {
(Some(base), Some(over)) => {
let mut out = base.clone();
for (k, v) in over {
match out.get(k) {
Some(existing) if existing.is_object() && v.is_object() => {
out.insert(k.clone(), deep_merge_request_wins(existing, v));
}
_ => {
out.insert(k.clone(), v.clone());
}
}
}
Value::Object(out)
}
_ => over.clone(),
}
}
/// Write `pipeline` in both spellings `default_pipeline_for` and
/// `GET /_settings` read. Nested `index.default_pipeline` is what the settings
/// response shows once an `index` object exists; a sibling dotted key is
/// dropped by that response, which is how a template pipeline disappeared
/// beside `xerj autoindex`'s `index.xerj_ingest_shards` (#2281).
fn write_default_pipeline(settings: &mut Value, pipeline: &str) {
if !settings.is_object() {
*settings = serde_json::json!({});
}
let Some(obj) = settings.as_object_mut() else {
return;
};
obj.insert(
"index.default_pipeline".to_string(),
Value::String(pipeline.to_string()),
);
let index = obj
.entry("index".to_string())
.or_insert_with(|| serde_json::json!({}));
if let Some(index_obj) = index.as_object_mut() {
index_obj.insert(
"default_pipeline".to_string(),
Value::String(pipeline.to_string()),
);
}
}
/// Copy single-segment `index.<key>` entries into the nested `index` object
/// when that object does not already name them. Keys with a further dot
/// (`index.query.strict_unknown_fields`) stay as dotted keys: that is a
/// spelling the reader looks up literally, and folding it would hide it.
fn hoist_single_segment_index_keys(mut settings: Value) -> Value {
let dotted: Vec<(String, Value)> = {
let Some(obj) = settings.as_object() else {
return settings;
};
obj.iter()
.filter_map(|(k, v)| {
let rest = k.strip_prefix("index.")?;
if rest.is_empty() || rest.contains('.') {
return None;
}
Some((rest.to_string(), v.clone()))
})
.collect()
};
if dotted.is_empty() {
return settings;
}
let Some(obj) = settings.as_object_mut() else {
return settings;
};
let index = obj
.entry("index".to_string())
.or_insert_with(|| serde_json::json!({}));
if let Some(index_obj) = index.as_object_mut() {
for (k, v) in dotted {
index_obj.entry(k).or_insert(v);
}
}
settings
}
fn merge_template_settings(template: &Value, request: Value) -> Value {
let request_pipeline = pipeline_setting(&request);
let template_pipeline = pipeline_setting(template);
let Some(template_obj) = template.as_object().filter(|m| !m.is_empty()) else {
return request;
};
let mut merged = if value_is_empty_object(&request) {
Value::Object(template_obj.clone())
} else {
deep_merge_request_wins(&Value::Object(template_obj.clone()), &request)
};
// The request's spelling wins over the template's, whichever of the two
// forms each side used. Filling gaps via deep-merge alone keeps a
// template nested value when the request only set the dotted key.
if let Some(pipeline) = request_pipeline.or(template_pipeline) {
write_default_pipeline(&mut merged, &pipeline);
}
hoist_single_segment_index_keys(merged)
}
/// One template property, in the same native shape `es_properties_to_fields`
/// builds for an explicit mapping. `semantic_text` is text plus an embedding
/// contract — not `object` (#2281).
fn field_from_template_property(
name: &str,
field_def: &Value,
semantic_dimensions: usize,
) -> xerj_common::types::FieldConfig {
use xerj_common::types::{EmbeddingConfig, FieldConfig, FieldType};
let es_type = field_def
.get("type")
.and_then(Value::as_str)
.unwrap_or("object");
let native_type = es_type_to_field_type(es_type);
let mut fc = FieldConfig::new(name.to_string(), native_type);
// #790: a template's `date_nanos` keeps nanosecond precision;
// `FieldConfig::new` already made a plain `date` millisecond-scaled.
if native_type == FieldType::Date && es_type == "date_nanos" {
fc.options.date_precision = Some(xerj_common::types::DatePrecision::Nanos);
}
if es_type == "semantic_text" {
// Same default as `es_properties_to_fields`: analyzed text has no
// doc values unless the mapping asks. An explicit `true` is kept.
fc.options.doc_values = field_def
.get("doc_values")
.and_then(Value::as_bool)
.unwrap_or(false);
let dims = field_def
.get("dimensions")
.or_else(|| field_def.get("dims"))
.and_then(Value::as_u64)
.map(|d| d as usize)
.unwrap_or(semantic_dimensions);
fc.options.dimensions = Some(dims);
fc.options.similarity = Some(
field_def
.get("similarity")
.and_then(Value::as_str)
.unwrap_or("cosine")
.to_string(),
);
let target_field = field_def
.get("target_field")
.and_then(Value::as_str)
.map(str::to_string)
.unwrap_or_else(|| format!("{name}_vector"));
fc.embedding = Some(EmbeddingConfig {
endpoint: field_def
.get("inference_endpoint")
.and_then(Value::as_str)
.map(str::to_string),
model: field_def
.get("inference_id")
.or_else(|| field_def.get("model_id"))
.and_then(Value::as_str)
.map(str::to_string),
target_field: Some(target_field),
});
}
fc
}
fn json_properties_have_semantic_text(props: &Value) -> bool {
let Some(obj) = props.as_object() else {
return false;
};
obj.values().any(|spec| {
spec.get("type").and_then(Value::as_str) == Some("semantic_text")
|| spec
.get("properties")
.is_some_and(json_properties_have_semantic_text)
})
}
/// Convert an ES field type string to a native FieldType.
fn es_type_to_field_type(es_type: &str) -> xerj_common::types::FieldType {
use xerj_common::types::FieldType;
match es_type {
// `semantic_text` carries the source text. The embedding lives on the
// companion vector. Falling through to `Object` is what made a
// template field report `type: object` (#2281).
"text" | "semantic_text" => FieldType::Text,
"keyword" | "constant_keyword" | "wildcard" => FieldType::Keyword,
"long" | "integer" | "short" | "byte" | "unsigned_long" => FieldType::Long,
"double" | "float" | "half_float" | "scaled_float" => FieldType::Double,
"boolean" => FieldType::Boolean,
"date" | "date_nanos" => FieldType::Date,
"ip" => FieldType::Ip,
"dense_vector" => FieldType::Vector,
"geo_point" => FieldType::GeoPoint,
"binary" => FieldType::Binary,
"nested" => FieldType::Nested,
_ => FieldType::Object,
}
}
#[cfg(test)]
mod snapshot_path_security_tests {
use super::validate_snapshot_path;
#[test]
fn f_path_02_repo_location_outside_data_dir_is_rejected_by_default() {
let data = tempfile::TempDir::new().unwrap();
// A repo location pointing outside data_dir (the F-PATH-02 exploit shape)
// with an empty allowlist must be refused.
let outside = tempfile::TempDir::new().unwrap();
let repo = outside.path().to_str().unwrap();
let snap = std::path::Path::new(repo).join("s1");
let err = validate_snapshot_path(repo, "s1", &snap, data.path(), &[], None)
.expect_err("external repo location must be rejected");
assert!(err.to_string().contains("outside data_dir"), "{err}");
}
#[test]
fn location_inside_data_dir_is_allowed() {
let data = tempfile::TempDir::new().unwrap();
let repo_dir = data.path().join("backups");
std::fs::create_dir_all(&repo_dir).unwrap();
let repo = repo_dir.to_str().unwrap();
let snap = repo_dir.join("s1");
validate_snapshot_path(repo, "s1", &snap, data.path(), &[], None)
.expect("a repo under data_dir must be allowed");
}
#[test]
fn explicit_allowlist_permits_an_external_root() {
let data = tempfile::TempDir::new().unwrap();
let ext = tempfile::TempDir::new().unwrap();
let repo = ext.path().to_str().unwrap().to_string();
let snap = std::path::Path::new(&repo).join("s1");
validate_snapshot_path(
&repo,
"s1",
&snap,
data.path(),
std::slice::from_ref(&repo),
None,
)
.expect("an allowlisted external root must be permitted");
}
#[test]
fn traversal_name_still_rejected_even_inside_an_allowed_base() {
let data = tempfile::TempDir::new().unwrap();
let snap = data.path().join("..");
let repo = data.path().to_str().unwrap();
assert!(validate_snapshot_path(repo, "..", &snap, data.path(), &[], None).is_err());
}
#[test]
fn f_path_02_residual_parent_ref_to_nonexistent_target_is_rejected() {
// The residual escape (#73): a location containing `..` whose fully
// resolved target does NOT yet exist. canonicalize() fails, the lexical
// fallback keeps the `..`, and a component-based starts_with(data_dir)
// would pass — letting create_dir_all write outside data_dir. It must be
// refused before any directory is created.
let data = tempfile::TempDir::new().unwrap();
let repo = format!("{}/../../xerj-escape", data.path().display());
let snap = std::path::Path::new(&repo).join("s1");
let err = validate_snapshot_path(&repo, "s1", &snap, data.path(), &[], None)
.expect_err("a location containing `..` must be rejected");
assert!(err.to_string().contains(".."), "unexpected error: {err}");
// And it stays rejected even if the operator allowlists a broad root:
// the `..` guard runs before containment, so a parent-ref cannot be
// laundered through the allowlist either.
let err2 =
validate_snapshot_path(&repo, "s1", &snap, data.path(), &["/".to_string()], None)
.expect_err("`..` must be rejected even with a permissive allowlist");
assert!(err2.to_string().contains(".."), "unexpected error: {err2}");
}
}
#[cfg(test)]
mod hub_pack_restore_root_tests {
//! `limits.hub_pack_restore`: the Corpus Hub pack cache is a restore-only
//! snapshot root, trusted only in the shape `xerj hub pull` creates it.
use super::{trusted_hub_pack_root, validate_snapshot_path};
use std::path::{Path, PathBuf};
/// `<tmp>/hub/packs/lz4/0774d055/snapshot/<snap>` as `xerj hub pull`
/// extracts it, plus a data_dir elsewhere. Returns (tmp, data, packs, repo).
fn layout() -> (tempfile::TempDir, PathBuf, PathBuf, PathBuf) {
let tmp = tempfile::TempDir::new().unwrap();
let root = tmp.path().canonicalize().unwrap();
let data = root.join("data");
let packs = root.join("hub").join("packs");
let repo = packs.join("lz4").join("0774d055").join("snapshot");
std::fs::create_dir_all(&data).unwrap();
std::fs::create_dir_all(repo.join("snap-1")).unwrap();
(tmp, data, packs, repo)
}
fn check(repo: &Path, data: &Path, hub: Option<&Path>) -> Result<PathBuf, String> {
let repo_s = repo.to_str().unwrap();
validate_snapshot_path(repo_s, "snap-1", &repo.join("snap-1"), data, &[], hub)
.map_err(|e| e.to_string())
}
#[test]
fn a_pulled_pack_is_restorable_from_the_hub_cache_without_an_allowlist() {
let (_t, data, packs, repo) = layout();
let snap = check(&repo, &data, Some(&packs)).expect("restore from the pack cache");
assert!(snap.starts_with(&packs), "{}", snap.display());
}
#[test]
fn without_the_hub_root_the_same_location_is_refused_as_before() {
// Create passes `None`, and so does `hub_pack_restore = false`: the
// pack cache is then exactly as foreign as any other path.
let (_t, data, _packs, repo) = layout();
let err = check(&repo, &data, None).expect_err("no hub root, no grant");
assert!(err.contains("snapshot_repo_allowlist"), "{err}");
}
#[test]
fn the_grant_covers_the_pack_cache_only_not_its_parent_or_a_sibling() {
let (_t, data, packs, _repo) = layout();
let hub = packs.parent().unwrap();
// The hub root itself (catalogue, node data, config) is not a restore source.
std::fs::create_dir_all(hub.join("node").join("snap-1")).unwrap();
assert!(check(&hub.join("node"), &data, Some(&packs)).is_err());
// A sibling sharing the name as a string prefix is not inside it.
let sibling = hub.join("packs-evil");
std::fs::create_dir_all(sibling.join("snap-1")).unwrap();
assert!(check(&sibling, &data, Some(&packs)).is_err());
// Nor is an unrelated absolute path.
assert!(check(Path::new("/etc"), &data, Some(&packs)).is_err());
}
#[test]
fn traversal_out_of_the_pack_cache_is_refused() {
let (_t, data, packs, _repo) = layout();
let outside = packs.parent().unwrap().parent().unwrap().join("outside");
std::fs::create_dir_all(outside.join("snap-1")).unwrap();
let escape = packs
.join("lz4")
.join("..")
.join("..")
.join("..")
.join("outside");
let err = check(&escape, &data, Some(&packs)).expect_err("`..` refused");
assert!(err.contains(".."), "{err}");
}
#[cfg(unix)]
#[test]
fn a_symlink_inside_the_pack_cache_cannot_point_the_restore_elsewhere() {
let (_t, data, packs, _repo) = layout();
let outside = packs.parent().unwrap().parent().unwrap().join("outside");
std::fs::create_dir_all(outside.join("snap-1")).unwrap();
let link = packs.join("evil");
std::os::unix::fs::symlink(&outside, &link).unwrap();
let err = check(&link, &data, Some(&packs)).expect_err("escaping link refused");
assert!(err.contains("snapshot_repo_allowlist"), "{err}");
}
#[cfg(unix)]
#[test]
fn a_pack_cache_that_is_itself_a_symlink_is_not_trusted() {
// `~/.xerj/hub/packs -> /` would otherwise turn the grant into the
// whole disk; `-> <elsewhere>` would move it somewhere nobody chose.
let tmp = tempfile::TempDir::new().unwrap();
let root = tmp.path().canonicalize().unwrap();
let data = root.join("data");
std::fs::create_dir_all(&data).unwrap();
let hub = root.join("hub");
std::fs::create_dir_all(&hub).unwrap();
let packs = hub.join("packs");
std::os::unix::fs::symlink("/", &packs).unwrap();
assert_eq!(trusted_hub_pack_root(&packs), None);
assert!(check(Path::new("/etc"), &data, Some(&packs)).is_err());
let real = root.join("elsewhere");
std::fs::create_dir_all(real.join("snap-1")).unwrap();
std::fs::remove_file(&packs).unwrap();
std::os::unix::fs::symlink(&real, &packs).unwrap();
assert_eq!(trusted_hub_pack_root(&packs), None);
assert!(check(&packs, &data, Some(&packs)).is_err());
assert!(check(&real, &data, Some(&packs)).is_err());
}
#[test]
fn a_missing_relative_or_non_directory_pack_cache_is_not_trusted() {
let tmp = tempfile::TempDir::new().unwrap();
let root = tmp.path().canonicalize().unwrap();
assert_eq!(trusted_hub_pack_root(&root.join("absent")), None);
assert_eq!(trusted_hub_pack_root(Path::new("relative/packs")), None);
let file = root.join("packs");
std::fs::write(&file, b"not a dir").unwrap();
assert_eq!(trusted_hub_pack_root(&file), None);
std::fs::remove_file(&file).unwrap();
std::fs::create_dir_all(&file).unwrap();
assert_eq!(trusted_hub_pack_root(&file), Some(file.clone()));
}
#[cfg(unix)]
#[test]
fn the_filesystem_root_is_never_a_pack_cache() {
assert_eq!(trusted_hub_pack_root(Path::new("/")), None);
}
}
#[cfg(test)]
mod ism_policy_persistence_validation_tests {
use crate::lifecycle::{LifecyclePolicy, LifecycleState};
use crate::Engine;
use xerj_common::config::Config;
fn valid_policy() -> LifecyclePolicy {
LifecyclePolicy {
description: None,
default_state: "only".to_string(),
states: vec![LifecycleState {
name: "only".to_string(),
actions: vec![],
transitions: vec![],
}],
}
}
/// `load_persisted_ism_policies` is the one path into `ism_policies`
/// that doesn't go through the REST layer's `validate()` call (native
/// PUT and the ILM-translation path both validate before ever storing
/// a policy) — it just reads back whatever was written to disk. A
/// hand-edited or otherwise corrupted single entry must not silently
/// load: `change_ism_policy` only checks "does this policy_id exist",
/// so an unvalidated entry here would be reachable through it without
/// ever having passed the same structural checks a direct PUT enforces.
#[tokio::test]
async fn corrupt_single_entry_is_skipped_valid_entries_still_restored() {
let dir = tempfile::tempdir().unwrap();
let mut config = Config::default();
config.server.data_dir = dir.path().to_string_lossy().into_owned();
// "broken" transitions to a state that doesn't exist — exactly what
// `LifecyclePolicy::validate()` rejects, but nothing stops it being
// written to disk directly (bypassing the REST layer entirely).
let mut policies = std::collections::HashMap::new();
policies.insert("good-policy".to_string(), valid_policy());
policies.insert(
"broken-policy".to_string(),
LifecyclePolicy {
description: None,
default_state: "start".to_string(),
states: vec![LifecycleState {
name: "start".to_string(),
actions: vec![],
transitions: vec![crate::lifecycle::LifecycleTransition {
state_name: "does-not-exist".to_string(),
conditions: None,
}],
}],
},
);
std::fs::write(
dir.path().join("ism_policies.json"),
serde_json::to_vec(&policies).unwrap(),
)
.unwrap();
let engine = Engine::new(config).unwrap();
assert!(
engine.ism_policies.contains_key("good-policy"),
"a structurally valid persisted policy must still load"
);
assert!(
!engine.ism_policies.contains_key("broken-policy"),
"a policy that would fail validate() must not be silently restored"
);
}
}
#[cfg(test)]
mod api_key_record_migration_tests {
use super::ApiKeyRecord;
/// A record as it comes off disk, before the load path touches it.
fn loaded(secret_hash: &str, plaintext: Option<&str>) -> ApiKeyRecord {
ApiKeyRecord {
name: "loaded".to_string(),
secret_hash: secret_hash.to_string(),
legacy_plaintext_secret: plaintext.map(str::to_string),
creation_ms: 1_753_600_000_000,
expiration_ms: None,
invalidated: false,
invalidation_ms: None,
roles: Vec::new(),
}
}
/// The discriminator is `is_usable_hash`, not `is_empty` and not a check
/// of the `$ssha256$` tag: a `secret_hash` that does not decode can never
/// verify against anything, so a record carrying only that is as unusable
/// as one carrying nothing and must be dropped, not restored as a live
/// key. Three of these carry the tag — a prefix test would call them
/// migrated and keep them.
#[test]
fn an_unparseable_hash_with_no_plaintext_is_dropped() {
for stored in [
"",
"$ssha256$",
"$ssha256$truncated",
"$ssha256$deadbeef$cafe",
"$argon2id$v=19$m=1,t=1,p=1$c2FsdA$aGFzaA",
"left-over-plaintext",
] {
for plaintext in [None, Some("")] {
let mut rec = loaded(stored, plaintext);
assert_eq!(
rec.migrate_from_plaintext(),
Err(()),
"{stored:?} + {plaintext:?} must be dropped, not restored"
);
}
}
}
/// A plaintext beside a hash-shaped value that does not decode is a store
/// nobody can explain — not the pre-#201 shape (that has no `secret_hash`
/// at all), and not a migrated one. Deriving a live credential from the
/// half of it #201 exists to delete is the fail-open reading, so the
/// record is dropped instead.
#[test]
fn a_plaintext_beside_an_unparseable_hash_is_dropped_not_repaired() {
for stored in [
"$ssha256$truncated",
"$argon2id$v=19$m=1,t=1,p=1$c2FsdA$aGFzaA",
"left-over-plaintext",
] {
let mut rec = loaded(stored, Some("the-plaintext"));
assert_eq!(
rec.migrate_from_plaintext(),
Err(()),
"{stored:?} + a plaintext is ambiguous and must be dropped"
);
}
}
/// A usable hash still wins over a leftover plaintext, and the plaintext is
/// discarded — re-deriving there could silently swap which secret the
/// record authenticates.
#[test]
fn a_usable_hash_wins_over_a_leftover_plaintext() {
let hash = crate::secret_hash::hash_secret("hashed-secret");
let mut rec = loaded(&hash, Some("stale-plaintext"));
assert_eq!(rec.migrate_from_plaintext(), Ok(true));
assert_eq!(rec.secret_hash, hash);
assert!(rec.legacy_plaintext_secret.is_none());
assert!(rec.verify_secret("hashed-secret"));
assert!(!rec.verify_secret("stale-plaintext"));
}
/// An already-migrated record is left alone and reports no rewrite.
#[test]
fn an_already_hashed_record_is_untouched() {
let hash = crate::secret_hash::hash_secret("s");
let mut rec = loaded(&hash, None);
assert_eq!(rec.migrate_from_plaintext(), Ok(false));
assert_eq!(rec.secret_hash, hash);
}
/// The pre-#201 shape: plaintext only.
#[test]
fn a_pre_201_plaintext_record_is_hashed() {
let mut rec = loaded("", Some("legacy-secret"));
assert_eq!(rec.migrate_from_plaintext(), Ok(true));
assert!(crate::secret_hash::is_usable_hash(&rec.secret_hash));
assert!(rec.verify_secret("legacy-secret"));
}
/// Even if one reached the auth path unmigrated, an unusable hash denies.
#[test]
fn verify_secret_denies_an_unusable_hash() {
for stored in ["", "$ssha256$truncated", "plaintext-secret"] {
let rec = loaded(stored, None);
assert!(
!rec.verify_secret(stored),
"{stored:?} must not verify against itself"
);
assert!(!rec.verify_secret("anything"));
}
}
#[tokio::test]
async fn has_open_index_tracks_created_and_deleted_indices() {
let dir = tempfile::tempdir().unwrap();
let mut config = xerj_common::config::Config::default();
config.server.data_dir = dir.path().to_string_lossy().into_owned();
let engine = crate::Engine::new(config).unwrap();
assert!(!engine.has_open_index("alpha"));
engine.get_or_create_index("alpha").unwrap();
assert!(engine.has_open_index("alpha"));
engine.delete_index("alpha").await.unwrap();
assert!(!engine.has_open_index("alpha"));
}
}
#[cfg(test)]
mod governor_thread_spawn_tests {
use super::spawn_governor_thread_using;
use xerj_common::thread_spawn::{BoxedJob, Spawn};
/// A spawner that always fails the way `pthread_create` does at a
/// thread or memory limit.
struct AlwaysEagain;
impl Spawn for AlwaysEagain {
fn spawn(
&self,
_name: &str,
_job: BoxedJob,
) -> std::io::Result<xerj_common::thread_spawn::SpawnedThread> {
Err(std::io::Error::from_raw_os_error(libc::EAGAIN))
}
}
#[test]
fn a_governor_thread_that_cannot_spawn_reports_false_instead_of_aborting() {
let ran = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false));
let job_ran = std::sync::Arc::clone(&ran);
let started = spawn_governor_thread_using(
&AlwaysEagain,
|| true,
"xerj-test",
move || {
job_ran.store(true, std::sync::atomic::Ordering::SeqCst);
},
);
assert!(!started);
assert!(!ran.load(std::sync::atomic::Ordering::SeqCst));
}
#[test]
fn a_closed_spawn_gate_is_respected() {
// A working spawner, but the gate is closed. Production passes an
// always-open gate for these threads (see `spawn_governor_thread`);
// this pins the plumbing for a caller that does gate.
let started = spawn_governor_thread_using(
&xerj_common::thread_spawn::StdSpawn,
|| false,
"xerj-test",
|| {},
);
assert!(!started);
}
}
#[cfg(test)]
mod index_template_merge_tests {
//! #2281: a template's `semantic_text` and `index.default_pipeline` have
//! to survive the merge with the body `xerj autoindex` actually sends.
use super::{field_from_template_property, merge_template_mappings, merge_template_settings};
use serde_json::json;
#[test]
fn dotted_template_pipeline_survives_an_autoindex_settings_body() {
let template = json!({ "index.default_pipeline": "jsqlite-copy" });
let request = json!({
"index": { "xerj_ingest_shards": 1 },
"index.query.strict_unknown_fields": true,
"analysis": { "analyzer": { "default": { "type": "code" } } }
});
let merged = merge_template_settings(&template, request);
assert_eq!(
merged.pointer("/index/default_pipeline"),
Some(&json!("jsqlite-copy")),
"{merged}"
);
assert_eq!(
merged.get("index.default_pipeline"),
Some(&json!("jsqlite-copy")),
"{merged}"
);
assert_eq!(
merged.pointer("/index/xerj_ingest_shards"),
Some(&json!(1)),
"{merged}"
);
assert_eq!(
merged.get("index.query.strict_unknown_fields"),
Some(&json!(true)),
"{merged}"
);
assert!(
merged
.pointer("/index/query.strict_unknown_fields")
.is_none(),
"a multi-dot key must stay dotted: {merged}"
);
assert_eq!(
merged.pointer("/analysis/analyzer/default/type"),
Some(&json!("code")),
"{merged}"
);
}
#[test]
fn the_request_pipeline_wins_in_either_spelling() {
let dotted_request = merge_template_settings(
&json!({ "index": { "default_pipeline": "from-template" } }),
json!({ "index.default_pipeline": "from-request" }),
);
assert_eq!(
dotted_request.pointer("/index/default_pipeline"),
Some(&json!("from-request")),
"{dotted_request}"
);
let nested_request = merge_template_settings(
&json!({ "index.default_pipeline": "from-template" }),
json!({ "index": { "default_pipeline": "from-request" } }),
);
assert_eq!(
nested_request.pointer("/index/default_pipeline"),
Some(&json!("from-request")),
"{nested_request}"
);
assert_eq!(
nested_request.get("index.default_pipeline"),
Some(&json!("from-request")),
"{nested_request}"
);
}
#[test]
fn request_properties_replace_template_properties_of_the_same_name() {
let merged = merge_template_mappings(
&json!({
"properties": {
"diagnostic_semantic": { "type": "semantic_text" },
"diagnostic_template_marker": { "type": "keyword" }
}
}),
json!({
"properties": {
"impl": { "type": "text" },
"diagnostic_semantic": { "type": "keyword" }
}
}),
);
assert_eq!(
merged.pointer("/properties/diagnostic_semantic/type"),
Some(&json!("keyword"))
);
assert_eq!(
merged.pointer("/properties/diagnostic_template_marker/type"),
Some(&json!("keyword"))
);
assert_eq!(
merged.pointer("/properties/impl/type"),
Some(&json!("text"))
);
}
#[test]
fn semantic_text_is_text_with_an_embedding_contract() {
use xerj_common::types::FieldType;
let semantic = field_from_template_property(
"diagnostic_semantic",
&json!({ "type": "semantic_text" }),
768, // #2407: a bare semantic_text takes the caller's backend width
);
assert_eq!(semantic.field_type, FieldType::Text);
assert!(!semantic.options.doc_values);
let embedding = semantic.embedding.expect("embedding");
assert_eq!(
embedding.target_field.as_deref(),
Some("diagnostic_semantic_vector")
);
assert_eq!(semantic.options.dimensions, Some(768));
let keyword = field_from_template_property(
"diagnostic_template_marker",
&json!({ "type": "keyword" }),
xerj_ai::local::DEFAULT_DIMS,
);
assert_eq!(keyword.field_type, FieldType::Keyword);
assert!(keyword.embedding.is_none());
}
}
#[cfg(test)]
mod pressure_drain_tests {
use super::{pressure_drain_due, sample_drain_due, MEMORY_DRAIN_MIN_INTERVAL};
use crate::governor::MemoryDrainSchedule;
use std::time::{Duration, Instant};
/// #2238: the engage edge drains immediately, a still-engaged latch does
/// not drain again inside the window, and releasing the latch arms the
/// next engage. The sampler passes the memtable latch; the RSS latch
/// has its own schedule.
#[test]
fn pressure_drain_fires_on_the_engage_edge_and_respects_the_window() {
let mut last = None;
let t0 = Instant::now();
let window = Duration::from_secs(2);
assert!(pressure_drain_due(true, &mut last, t0, window));
assert!(
!pressure_drain_due(true, &mut last, t0 + Duration::from_millis(500), window),
"one engaged stretch drains once per window"
);
assert!(pressure_drain_due(
true,
&mut last,
t0 + Duration::from_secs(2),
window
));
assert!(!pressure_drain_due(
false,
&mut last,
t0 + Duration::from_secs(3),
window
));
assert!(last.is_none(), "a released latch clears the timer");
assert!(
pressure_drain_due(true, &mut last, t0 + Duration::from_secs(3), window),
"the next engage drains immediately"
);
}
/// #1981 x #2238: a flat RSS backs off, and that backoff must not hide a
/// memtable trip. Corpus index hits the summed budget while RSS is
/// stable; the flush that frees it stays on the 2 s window. A memtable
/// engage that arrives during an RSS gap drains on that sample, not at
/// the end of the gap.
#[test]
fn memtable_pressure_is_not_stretched_by_the_rss_backoff() {
let min = MEMORY_DRAIN_MIN_INTERVAL;
let t0 = Instant::now();
let stuck = 64 * 1024 * 1024;
let mut rss_drains = MemoryDrainSchedule::new(min);
let mut last_memtable = None;
let mut rss_fires = 0u32;
let mut t = Duration::ZERO;
let warmup = Duration::from_secs(60);
while t < warmup {
if sample_drain_due(
t0 + t,
stuck,
true,
false,
&mut rss_drains,
&mut last_memtable,
) {
rss_fires += 1;
}
t += min;
}
assert!(
rss_fires < 20,
"a flat RSS must back off across 60s, fired {rss_fires}"
);
assert!(
!sample_drain_due(
t0 + warmup,
stuck,
true,
false,
&mut rss_drains,
&mut last_memtable,
),
"the RSS clock is still inside its backed-off gap"
);
assert!(
sample_drain_due(
t0 + warmup,
stuck,
true,
true,
&mut rss_drains,
&mut last_memtable,
),
"a memtable engage drains immediately during that gap"
);
assert!(!sample_drain_due(
t0 + warmup + Duration::from_millis(500),
stuck,
true,
true,
&mut rss_drains,
&mut last_memtable,
));
assert!(sample_drain_due(
t0 + warmup + min,
stuck,
true,
true,
&mut rss_drains,
&mut last_memtable,
));
// Memtable only, RSS stable and disengaged: every 2 s step fires.
// Folding this latch into the RSS backoff would drop fires well
// below the step count.
let mut rss_drains = MemoryDrainSchedule::new(min);
let mut last_memtable = None;
let horizon = Duration::from_secs(120);
let mut mem_fires = 0u32;
let mut t = Duration::ZERO;
while t < horizon {
if sample_drain_due(
t0 + t,
stuck,
false,
true,
&mut rss_drains,
&mut last_memtable,
) {
mem_fires += 1;
}
t += min;
}
let steps = (horizon.as_secs() / min.as_secs()) as u32;
assert_eq!(
mem_fires, steps,
"memtable pressure keeps the fixed window while RSS does not fall"
);
}
}
#[cfg(test)]
mod serial_pressure_flush_tests {
use super::Engine;
use crate::index::FlushShardProbe;
use serde_json::json;
use tempfile::TempDir;
use xerj_common::config::Config;
use xerj_common::types::{FieldConfig, FieldType, Schema};
async fn seed(engine: &Engine, name: &str, n: usize) {
let mut schema = Schema::empty();
schema
.fields
.push(FieldConfig::new("body", FieldType::Text));
engine.create_index(name, schema).expect("create");
let idx = engine.get_index(name).expect("get");
for i in 0..n {
idx.index_document(
Some(format!("d{i}")),
json!({ "body": format!("document number {i}") }),
)
.await
.expect("index");
}
}
async fn count_all(engine: &Engine, name: &str) -> u64 {
let idx = engine.get_index(name).expect("get index");
let req = xerj_query::parse_request(
&json!({ "query": { "match_all": {} }, "size": 0, "from": 0 }),
)
.expect("parse_request");
idx.search(&req).await.expect("search").total.value
}
/// #2427: a breaker drain builds one shard segment at a time. The flush
/// working set (parsed docs, columns, postings, stored blocks) is not
/// charged to the memtable budget, and a drain that built every shard at
/// once took RSS from 1,460 MB to 2,323 MB on a 1,536 MB-capped node.
/// The control index shows the ordinary flush still fans out, so the
/// probe does see concurrency when there is some. FAIL-BEFORE:
/// `pressure_drain` called `Index::flush`, which runs every non-empty
/// shard at once (peak > 1 here).
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
async fn a_pressure_drain_flushes_one_shard_at_a_time() {
let dir = TempDir::new().unwrap();
let mut config = Config::default();
config.server.data_dir = dir.path().to_str().unwrap().to_string();
let engine = Engine::new(config).expect("engine::new");
seed(&engine, "serial-drain-control", 400).await;
let control = engine.get_index("serial-drain-control").unwrap();
control.flush().await.expect("flush");
assert!(
FlushShardProbe::peak("serial-drain-control") > 1,
"the ordinary flush runs shards concurrently"
);
seed(&engine, "serial-drain", 400).await;
let idx = engine.get_index("serial-drain").unwrap();
assert!(idx.memtable_bytes() > 0);
engine.pressure_drain(false).await;
assert_eq!(idx.memtable_bytes(), 0, "the drain published the memtable");
assert_eq!(
FlushShardProbe::peak("serial-drain"),
1,
"a pressure drain must build one shard at a time"
);
assert_eq!(count_all(&engine, "serial-drain").await, 400);
}
}