Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

19.4 KB
Raw Blame
//! # xerj-engine
//!
//! Integration crate that wires together all xerj subsystems into a working
//! search engine:
//!
//! - [`Engine`]    — manages multiple named indices
//! - [`Index`]     — per-index coordinator: WAL + storage + FTS
//! - [`FtsMemtable`] — in-memory inverted index for unflushed documents
//! - [`bulk`]      — NDJSON bulk operation processing

// The engine wires aggregation, segment, and memtable pipelines whose closures
// and columnar accumulators carry deeply-nested generic types (e.g.
// `Vec<(String, HashMap<String, String>, Arc<Value>)>`). Naming each with a
// type alias hurts more than it helps at these call sites, so we opt out of
// `clippy::type_complexity` crate-wide — the standard idiom for data-plane
// internals.
#![allow(clippy::type_complexity)]

pub mod aggs;
pub mod audit;
pub mod bulk;
pub mod cluster_apply;
mod cluster_state;
mod code_blocks;
mod collection_publication;
pub mod engine;
pub mod governor;
pub mod index;
pub mod index_guard;
pub mod ingest_memory;
pub mod lifecycle;
pub mod memtable;
mod merge_budget;
mod merge_reserve;
mod node_lock;
pub mod painless;
pub mod pools;
pub mod rbac;
pub mod read_budget;
pub mod secret_hash;
pub mod segment_cache_budget;
pub mod segment_cache_estimates;
pub mod slow_query;
pub mod sql;
pub mod turbo_ingest;
mod vector_column;
pub mod wal_tap;
mod write_publication;

#[cfg(test)]
mod bounded_ingest_resources_tests;

// ── Re-exports ────────────────────────────────────────────────────────────────

pub use cluster_apply::{EngineMetadataApplier, ReplicatedIndexSpec};
pub use cluster_state::{ClusterStateBlockKind, ClusterStateBootStatus};
pub use engine::{EmbeddingExecutionIdentity, Engine, HealthStatus, IndexInfo};
pub use index::{
    detect_log_format, filter_object, resolve_date_math, resolve_field_alias,
    storage_mode_from_config, EnrichTable, FieldEncodingInfo, Index, IndexResponse, IndexStats,
    LogFormat,
};
pub use node_lock::data_dir_lock_held;
// Second-brain graph expansion (bounded columnar traversal over an edges
// index). Lives under `index` for private-field access; re-exported at the
// crate root so callers write `xerj_engine::graph::GraphExpandRequest`.
pub use index::graph;
pub use memtable::FtsMemtable;

// ── Process-global metrics handle ─────────────────────────────────────────────
//
// Flush, merge, and WAL writes happen deep in the engine, whose types are
// constructed (`Index::open`) before the server's `Metrics` instance exists and
// without a handle to it. Rather than thread `Arc<Metrics>` through every
// constructor and the free-fn flush pipeline, the server installs the single
// process-wide `Metrics` here once at startup (there is exactly one per
// process). Call sites record via `engine_metrics()` — a no-op when unset
// (e.g. in unit tests), so the engine never fabricates observations.
//
// The cell itself lives in `xerj_common::metrics` (#819) so `xerj-storage`
// call sites (segment open) can record into the same process-wide instance;
// these wrappers keep the engine-facing API unchanged.

/// Install the process-wide metrics handle the engine records into. Idempotent;
/// the first call wins (later calls are ignored). Called once by the server at
/// startup with the same `Metrics` the HTTP `/metrics` endpoint scrapes.
pub fn set_engine_metrics(metrics: std::sync::Arc<xerj_common::metrics::Metrics>) {
    xerj_common::metrics::set_global_metrics(metrics);
}

/// The installed metrics handle, or `None` if the server has not installed one
/// (unit tests, embedded uses). Engine call sites guard on this so recording is
/// a no-op when metrics are not wired.
pub fn engine_metrics() -> Option<&'static std::sync::Arc<xerj_common::metrics::Metrics>> {
    xerj_common::metrics::global_metrics()
}

// ── Ingest/flush/merge rayon pool ────────────────────────────────────────────
//
// A pool that cannot be started does not abort the process (#1122). Release
// builds are `panic = "abort"`, and the old `.build().expect(..)` turned an
// `EAGAIN` from `pthread_create` into a dead node at the moment the machine
// was already out of threads. Workers are started with
// `pools::safe_rayon_spawn` rather than `std::thread::Builder`: std panics
// inside the new thread when its alternate signal stack cannot be mapped,
// after `build()` has already been told the thread exists, so the `Err`
// fallback never got to run. The call runs on the calling thread instead,
// and the next call tries to start the pool again.

static INGEST_POOL: std::sync::OnceLock<rayon::ThreadPool> = std::sync::OnceLock::new();
static BACKGROUND_POOL: std::sync::OnceLock<rayon::ThreadPool> = std::sync::OnceLock::new();
static MERGE_POOL: std::sync::OnceLock<rayon::ThreadPool> = std::sync::OnceLock::new();

/// Dedicated rayon pool for ingest-side CPU work: bulk-body parse, P2.1
/// per-batch doc parse/analyze/insert, flush FTS+DV side-car builds, and
/// merge FTS rebuilds.
///
/// Pre-fix all of that ran on the GLOBAL rayon pool — the same pool the
/// search path uses for segment fan-out (`search_segments`) and the
/// fast-agg dv-warm par_iter.  A single long flush/merge job therefore
/// queued every search's rayon work behind it: with a background bulk
/// writer running, foreground `match_all` p99 was measured at 15 s+ while
/// steady-state reads were ~2 ms (BEAT_ES_MASTER_PLAN Phase 2, read-under-
/// write).  Ingest work now runs here; the global pool stays free for
/// search/agg fan-out, so neither side can starve the other's queue (the
/// OS scheduler time-slices the two pools fairly).
///
/// Sized to all available cores (`crate::pools`, `Workload::Latency`): ingest
/// throughput is unchanged when no searches run, and under mixed load the
/// kernel — not the rayon job queue — arbitrates.
pub(crate) fn ingest_pool() -> pools::PoolCall {
    fn build(
        threads: usize,
    ) -> std::result::Result<rayon::ThreadPool, rayon::ThreadPoolBuildError> {
        rayon::ThreadPoolBuilder::new()
            .num_threads(threads.max(1))
            .thread_name(|i| format!("xerj-ingest-{i}"))
            .spawn_handler(crate::pools::safe_rayon_spawn)
            // nice(5): bulk parse/analyze/insert are request-latency
            // sensitive for the WRITER, but under mixed load they were
            // the biggest nice(0) CPU block competing head-to-head with
            // query threads — every flush-storm/bulk burst showed up as
            // a read p95/p99 episode.  One CFS step below reads keeps
            // read tails flat while the writer retains far more
            // throughput than the sustained-ingest target (measured
            // 115 k docs/s at nice 0 vs a 40 k floor; nice 5 trades a
            // slice of that for read-tail stability).  Flush side-cars
            // are nice(10) (`background_pool`), merges nice(15)
            // (`merge_pool`) — the maintenance ladder stays below both.
            //
            // The ladder is platform-specific: `nice()` is per-thread on Linux
            // but per-PROCESS on Darwin, where calling it from a start handler
            // would deprioritize the whole server, search included (#240 §5).
            // `resource::deprioritize_current_thread` owns that decision.
            .start_handler(|_| {
                xerj_common::resource::deprioritize_current_thread(
                    xerj_common::resource::Deprioritize::Ingest,
                );
            })
            .build()
    }
    pools::PoolCall::new(
        &INGEST_POOL,
        "xerj-ingest",
        || pools::sizing().ingest,
        build,
    )
}

/// Deprioritised rayon pool for BACKGROUND index maintenance: flush
/// side-car builds (FTS + doc-values), segment finalisation, and merge
/// rebuilds.
///
/// Separate from `ingest_pool` (foreground, normal priority — bulk parse
/// and memtable insert are request-latency-critical) and from the global
/// pool (search fan-out).  nice(10) lets CFS schedule foreground search
/// and bulk threads ahead of maintenance encode work, which otherwise
/// saturates every core for seconds per merged segment; with an idle
/// foreground the pool still gets every core, so writer-only throughput
/// is unchanged.
pub(crate) fn background_pool() -> pools::PoolCall {
    fn build(
        threads: usize,
    ) -> std::result::Result<rayon::ThreadPool, rayon::ThreadPoolBuildError> {
        rayon::ThreadPoolBuilder::new()
            .num_threads(threads.max(1))
            .thread_name(|i| format!("xerj-bg-{i}"))
            .spawn_handler(crate::pools::safe_rayon_spawn)
            .start_handler(|_| {
                xerj_common::resource::deprioritize_current_thread(
                    xerj_common::resource::Deprioritize::Background,
                );
            })
            .build()
    }
    pools::PoolCall::new(
        &BACKGROUND_POOL,
        "xerj-bg",
        || pools::sizing().background,
        build,
    )
}

/// Dedicated rayon pool for **merge** encode work (byte-copy re-encode +
/// FTS side-car rebuild of merged segments).
///
/// Pre-fix merges ran on `ingest_pool()` — the same pool every `_bulk`
/// request's parse/analyze/insert par_iters use.  A single 16-segment
/// (~500 k-doc) merge saturates all N workers for many seconds, so every
/// concurrent bulk request's `install()` queued behind it: measured on the
/// 1 M×c8 ingest benchmark as a 4.7 s → 17.5 s FULL STALL of ingest
/// (window throughput 370 k docs/s → 7.8 k docs/s) the moment the first
/// background merge tick fired.  100 k-doc runs finish before the first
/// 5 s merge tick, which is why the regression only appeared at 1 M scale.
///
/// Merges are pure background work with no client waiting on them — ES
/// likewise runs merges on a small dedicated pool and throttles them under
/// indexing pressure.  Sizing: `max(2, ncores/8)` keeps a merge from ever
/// occupying more than a sliver of the machine, while `nice(15)` (one step
/// below ingest's `nice(10)`) lets both foreground reads AND ingest encode
/// pre-empt merge threads when cores are scarce.  Merges take a few times
/// longer to converge — acceptable: the 5 s merge loop just picks up where
/// it left off, and `merge_in_progress` already serialises passes.
pub(crate) fn merge_pool() -> pools::PoolCall {
    fn build(
        threads: usize,
    ) -> std::result::Result<rayon::ThreadPool, rayon::ThreadPoolBuildError> {
        rayon::ThreadPoolBuilder::new()
            .num_threads(threads.max(1))
            .thread_name(|i| format!("xerj-merge-{i}"))
            .spawn_handler(crate::pools::safe_rayon_spawn)
            .start_handler(|_| {
                xerj_common::resource::deprioritize_current_thread(
                    xerj_common::resource::Deprioritize::Maintenance,
                );
            })
            .build()
    }
    pools::PoolCall::new(&MERGE_POOL, "xerj-merge", || pools::sizing().merge, build)
}

/// Global bound on how many shard flushes may run their (CPU-heavy,
/// ~600 ms) segment finalize concurrently.
///
/// Under sustained ingest all 16 memtable shards cross their per-shard
/// flush threshold at nearly the same instant (round-robin doc routing),
/// so `maybe_spawn_flush` dispatches a 16-wide flush STORM.  Pre-fix each
/// finalize ran inline via `background_pool().install(...)`, which BLOCKS
/// the calling tokio worker for the whole finalize; a 16-wide storm parked
/// up to 16 async workers at once, starving the concurrent `_bulk`
/// clients' HTTP handling and the 300/s reader's search dispatch.  It also
/// oversubscribed the 32-thread background pool 16-deep (512 queued rayon
/// tasks), inflating per-finalize wall time into 700 ms+ outliers and the
/// memtable-back-pressure that capped ingest.
///
/// The finalize now runs on `spawn_blocking` (never parks an async worker)
/// under this semaphore, so at most N finalizes are in flight.  Measured
/// on 1M×c8 (32 cores): unbounded+spawn_blocking gained little (337k),
/// but N=4 staggered the storm into a smoother band — ingest 338k→370k
/// (+9%, peak 377k) with all five mixed read p99s neutral-to-better.
/// Total flush CPU is unchanged; the win is scheduling, not less work.
///
/// N comes from `engine.flush_workers`, which defaults to the maintenance
/// share of the machine, `max(2, ncores/8)` (=4 on the 32-core bench, matching
/// the measured optimum); `XERJ_FIN_CONC` still overrides it for one run.
pub(crate) fn flush_finalize_gate() -> &'static tokio::sync::Semaphore {
    static GATE: std::sync::OnceLock<tokio::sync::Semaphore> = std::sync::OnceLock::new();
    GATE.get_or_init(|| tokio::sync::Semaphore::new(crate::pools::sizing().flush_finalize))
}

/// One permit: while the parent RSS breaker is engaged, flush finalizes
/// (segment build, FST build, doc-values encode) also take this, so only one
/// runs at a time (#2427).
///
/// [`flush_finalize_gate`] allows `max(2, ncores/8)` concurrent finalizes.
/// Each holds a frozen memtable plus its FST and column build buffers. Under
/// the breaker, those buffers stacked on a node that was already at the
/// watermark: RSS climbed about 1 GB past a 3,000 MB cap while writes were
/// already getting 429s. Serializing them while the breaker is engaged bounds
/// the overshoot to one build. The drain still makes progress; it just does
/// not multiply the peak.
pub(crate) fn flush_pressure_gate() -> &'static tokio::sync::Semaphore {
    static GATE: std::sync::OnceLock<tokio::sync::Semaphore> = std::sync::OnceLock::new();
    GATE.get_or_init(|| tokio::sync::Semaphore::new(1))
}

// ── Analyzer helpers ──────────────────────────────────────────────────────────

use std::sync::Arc;
// `AnalyzerRegistry` is `pub use`d, not merely imported, so the API layer can
// gate `PUT /{index}/_settings` on exactly the `analysis` spellings the
// registry resolves without xerj-api taking a direct dependency on xerj-fts.
// The create-time and settings-time gates drifted apart once already — one
// refused four spellings, the other two — and one shared symbol is what stops
// that (issue #204).
pub use xerj_fts::analyzer::{AnalyzerRegistry, Token};
use xerj_fts::analyzer::{
    LowercaseFilter, StandardTokenizer, StemmerFilter, StopwordsFilter, TokenFilter, Tokenizer,
};

/// Return a default [`AnalyzerRegistry`] pre-populated with all built-in analyzers.
pub fn analyzer_registry() -> Arc<AnalyzerRegistry> {
    Arc::new(AnalyzerRegistry::with_defaults())
}

/// Step-by-step output of the standard analyzer pipeline.
pub struct AnalyzeExplainSteps {
    /// Token texts after the raw tokenizer (no filters applied yet).
    pub after_tokenizer: Vec<serde_json::Value>,
    /// Token texts after the lowercase filter.
    pub after_lowercase: Vec<serde_json::Value>,
    /// Token texts after the stop-word filter.
    pub after_stopwords: Vec<serde_json::Value>,
    /// Token texts after the stemmer (final output).
    pub after_stemmer: Vec<serde_json::Value>,
    /// Alias for `after_stemmer`.
    pub final_tokens: Vec<serde_json::Value>,
}

/// Run the standard analyzer pipeline step-by-step on `input`.
///
/// Returns each intermediate token list so the caller can expose a
/// `"explain": true` analysis response (Item #8 from user feedback).
pub fn analyze_explain_steps(input: &str) -> AnalyzeExplainSteps {
    fn token_json(t: &xerj_fts::analyzer::Token) -> serde_json::Value {
        serde_json::json!({
            "token": t.text,
            "start_offset": t.start_offset,
            "end_offset": t.end_offset,
            "position": t.position,
        })
    }

    // Stage 0: raw tokenizer
    let raw = StandardTokenizer.tokenize(input);
    let after_tokenizer: Vec<_> = raw.iter().map(token_json).collect();

    // Stage 1: lowercase
    let lc_tokens = LowercaseFilter.filter(raw.clone());
    let after_lowercase: Vec<_> = lc_tokens.iter().map(token_json).collect();

    // Stage 2: stop words
    let sw_tokens = StopwordsFilter::english().filter(lc_tokens);
    let after_stopwords: Vec<_> = sw_tokens.iter().map(token_json).collect();

    // Stage 3: stemmer (final)
    let stemmed = StemmerFilter::english().filter(sw_tokens);
    let after_stemmer: Vec<_> = stemmed.iter().map(token_json).collect();
    let final_tokens = after_stemmer.clone();

    AnalyzeExplainSteps {
        after_tokenizer,
        after_lowercase,
        after_stopwords,
        after_stemmer,
        final_tokens,
    }
}

// ── Error type ────────────────────────────────────────────────────────────────

use thiserror::Error;

#[derive(Debug, Error)]
pub enum EngineError {
    #[error(transparent)]
    Common(#[from] xerj_common::XerjError),

    #[error("storage error: {0}")]
    Storage(#[from] xerj_storage::StorageError),

    #[error("query error: {0}")]
    Query(#[from] xerj_query::QueryError),

    #[error("FTS error: {0}")]
    Fts(anyhow::Error),

    #[error("IO error: {0}")]
    Io(#[from] std::io::Error),

    #[error("serialization error: {0}")]
    Serde(#[from] serde_json::Error),

    /// An index metadata sidecar (`schema.json`, `settings.json`,
    /// `es_mapping.json`) is present on disk but could not be read or parsed.
    ///
    /// This is deliberately distinct from the file being **absent**, which is a
    /// legitimate state (indices created before create-time schema persistence
    /// have no `schema.json`; an index created without settings has no
    /// `settings.json`) and still opens with a dynamic mapping. Unparseable
    /// means the recorded mapping is *lost*, and defaulting there silently
    /// re-infers every field type from whatever documents arrive next — so the
    /// open is refused instead, the same way `xerj-storage` refuses a segment
    /// whose header does not match (`StorageError::UnsupportedVersion`).
    #[error("corrupt index metadata: {file} exists but could not be loaded ({reason}); refusing to open the index rather than fall back to a default mapping — restore the file or restore the index from a snapshot")]
    CorruptIndexMetadata { file: String, reason: String },
}

/// Engine-level result alias.
pub type Result<T> = std::result::Result<T, EngineError>;

impl From<EngineError> for xerj_common::XerjError {
    fn from(e: EngineError) -> Self {
        match e {
            EngineError::Common(z) => z,
            EngineError::Storage(s) => xerj_common::XerjError::storage(s.to_string()),
            EngineError::Query(q) => xerj_common::XerjError::invalid_query(q.to_string()),
            EngineError::Fts(f) => xerj_common::XerjError::internal(f.to_string()),
            EngineError::Io(io) => xerj_common::XerjError::storage_io("IO error", io),
            EngineError::Serde(s) => xerj_common::XerjError::serialization(s.to_string()),
            // `store_exception` (500) is the ES type for "this shard's on-disk
            // state is unusable" — the closest honest mapping for a corrupt
            // sidecar, and it keeps the reason string intact for the client.
            corrupt @ EngineError::CorruptIndexMetadata { .. } => {
                xerj_common::XerjError::storage(corrupt.to_string())
            }
        }
    }
}