AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
fix(engine): apply persisted default_pipeline after restart (#2296)
xerj-orgd53bc2e44f29
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 13e54266a..b9e8eaeb3 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -45,6 +45,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0### Fixed+- An existing index's persisted `index.default_pipeline` runs again after a+ clean restart. `GET /_settings` still reported the binding, and the+ pipeline definition was still there, but an ordinary document write no+ longer copied the field: the in-memory binding was filled on create and+ on settings update, and opening the index from disk did not rebuild it.+ Boot, a failed-index retry, and snapshot restore now copy `settings.json`+ back into that map. Explicit `?pipeline=` was already unaffected+ ([#2294](https://github.com/xerj-org/xerj/issues/2294)).+- `xerj autoindex` no longer aborts the node on one large JSON file (#1122).A bulk whose analysis would cross the memory watermark is rejected withHTTP 429 before the documents are parsed, so several in-flight bulks cannotdiff --git a/engine/crates/xerj-api/src/es_compat.rs b/engine/crates/xerj-api/src/es_compat.rsindex 8f7ba1063..be45777ae 100644--- a/engine/crates/xerj-api/src/es_compat.rs+++ b/engine/crates/xerj-api/src/es_compat.rs@@ -2975,10 +2975,11 @@ async fn get_index_inner(////// Two fixes over replaying the display map alone (#1092):///-/// * The display map (`engine.index_settings`) lives in memory only, so after a-/// restart it is empty and every create-time setting vanished from the-/// response. A live index with no display entry falls back to its persisted-/// `settings.json` copy — what the index was created with.+/// * The display map (`engine.index_settings`) is rebuilt from each index's+/// `settings.json` when that index is opened (#2294). A live index whose+/// display entry is still missing falls back to the same file — what the+/// index was created with — so `GET /_settings` and `default_pipeline_for`+/// agree./// * `analysis` is reported from the engine, not echoed: whatever spelling the/// caller used (`settings.analysis` beside an `index` block was dropped/// outright), the response carries `index.analysis` = the block the index'sdiff --git a/engine/crates/xerj-api/tests/default_pipeline_survives_restart.rs b/engine/crates/xerj-api/tests/default_pipeline_survives_restart.rsnew file mode 100644index 000000000..21a5a94b7--- /dev/null+++ b/engine/crates/xerj-api/tests/default_pipeline_survives_restart.rs@@ -0,0 +1,328 @@+//! Issue #2294: an index created with a persisted `index.default_pipeline`+//! kept reporting the binding from `GET /_settings` after a clean restart+//! (the response falls back to `settings.json`), and the pipeline definition+//! survived in cluster state, but an ordinary write no longer ran it.+//!+//! `Engine::default_pipeline_for` reads the in-memory `index_settings` map.+//! That map was filled on create and on `PUT /_settings`, and the boot,+//! failed-index retry, and snapshot-restore open paths never copied+//! `settings.json` back into it. Explicit `?pipeline=` still worked because+//! it does not consult the map.+//!+//! `index.final_pipeline` is not a setting this build reads.++use axum::body::Body;+use axum::http::{Request, StatusCode};+use serde_json::{json, Value};+use tower::ServiceExt;++fn app_on(dir: &std::path::Path) -> (axum::Router, xerj_api::state::AppState) {+ let mut config = xerj_common::config::Config::default();+ config.server.data_dir = dir.to_string_lossy().into_owned();+ config.storage.wal_sync = xerj_common::config::WalSync::Async;+ let metrics = xerj_common::metrics::Metrics::new().expect("metrics");+ let engine = xerj_engine::Engine::new(config.clone()).expect("engine");+ let state = xerj_api::state::AppState::new(config, engine, metrics);+ let app = xerj_api::router::build_es_compat_router(state.clone());+ (app, state)+}++async fn send(app: &axum::Router, req: Request<Body>) -> (StatusCode, Value) {+ let res = app.clone().oneshot(req).await.expect("response");+ let status = res.status();+ let bytes = axum::body::to_bytes(res.into_body(), 8 * 1024 * 1024)+ .await+ .expect("body");+ let body = serde_json::from_slice(&bytes).unwrap_or(Value::Null);+ (status, body)+}++fn put(path: &str, body: Value) -> Request<Body> {+ Request::builder()+ .method("PUT")+ .uri(path)+ .header("content-type", "application/json")+ .body(Body::from(body.to_string()))+ .expect("request")+}++fn get(path: &str) -> Request<Body> {+ Request::builder()+ .method("GET")+ .uri(path)+ .body(Body::empty())+ .expect("request")+}++fn ndjson(path: &str, body: &str) -> Request<Body> {+ Request::builder()+ .method("POST")+ .uri(path)+ .header("content-type", "application/x-ndjson")+ .body(Body::from(body.to_string()))+ .expect("request")+}++/// The pipeline from the report: copy `source_marker` onto `copied_marker`.+/// The ES `copy` processor is stored as `copy_field`.+async fn define_copy_pipeline(app: &axum::Router) {+ let (status, body) = send(+ app,+ put(+ "/_ingest/pipeline/restart-copy",+ json!({+ "processors": [{+ "copy": { "source": "source_marker", "target": "copied_marker" }+ }]+ }),+ ),+ )+ .await;+ assert_eq!(status, StatusCode::OK, "{body}");+}++fn assert_copied(doc: &Value, marker: &str) {+ assert_eq!(+ doc["_source"]["source_marker"].as_str(),+ Some(marker),+ "source field missing: {doc}"+ );+ assert_eq!(+ doc["_source"]["copied_marker"].as_str(),+ Some(marker),+ "default pipeline did not copy the field: {doc}"+ );+}++/// Dotted `index.default_pipeline` (the spelling in #2294) and the nested+/// spelling both have to keep transforming writes after the process that+/// created the index is gone.+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]+async fn persisted_default_pipeline_runs_after_clean_restart() {+ let dir = tempfile::tempdir().expect("data dir");++ {+ let (app, state) = app_on(dir.path());+ define_copy_pipeline(&app).await;++ let (status, body) = send(+ &app,+ put(+ "/restart-probe",+ json!({+ "settings": { "index.default_pipeline": "restart-copy" },+ "mappings": { "properties": {+ "source_marker": { "type": "keyword" },+ "copied_marker": { "type": "keyword" }+ }}+ }),+ ),+ )+ .await;+ assert_eq!(status, StatusCode::OK, "{body}");++ let (status, body) = send(+ &app,+ put(+ "/nested-probe",+ json!({+ "settings": { "index": { "default_pipeline": "restart-copy" } },+ "mappings": { "properties": {+ "source_marker": { "type": "keyword" },+ "copied_marker": { "type": "keyword" }+ }}+ }),+ ),+ )+ .await;+ assert_eq!(status, StatusCode::OK, "{body}");++ // #2281 stored a template pipeline on the index at create. That copy+ // lives in settings.json and has to come back on open too.+ let (status, body) = send(+ &app,+ put(+ "/_index_template/restart-template",+ json!({+ "index_patterns": ["from-template-*"],+ "priority": 100,+ "template": {+ "settings": { "index.default_pipeline": "restart-copy" }+ }+ }),+ ),+ )+ .await;+ assert_eq!(status, StatusCode::OK, "{body}");+ let (status, body) = send(&app, put("/from-template-probe", json!({}))).await;+ assert_eq!(status, StatusCode::OK, "{body}");++ // A template registered after the index exists must not become that+ // index's pipeline on the next open.+ let (status, body) = send(&app, put("/plain-probe", json!({}))).await;+ assert_eq!(status, StatusCode::OK, "{body}");+ let (status, body) = send(+ &app,+ put(+ "/_index_template/plain-template",+ json!({+ "index_patterns": ["plain-*"],+ "priority": 100,+ "template": {+ "settings": { "index.default_pipeline": "restart-copy" }+ }+ }),+ ),+ )+ .await;+ assert_eq!(status, StatusCode::OK, "{body}");++ let (status, body) = send(+ &app,+ put(+ "/restart-probe/_doc/before?refresh=true",+ json!({ "source_marker": "before-clean-restart" }),+ ),+ )+ .await;+ assert_eq!(status, StatusCode::CREATED, "{body}");+ let (status, doc) = send(&app, get("/restart-probe/_doc/before")).await;+ assert_eq!(status, StatusCode::OK, "{doc}");+ assert_copied(&doc, "before-clean-restart");++ assert_eq!(+ state+ .engine+ .default_pipeline_for("restart-probe")+ .as_deref(),+ Some("restart-copy")+ );+ assert_eq!(+ state.engine.default_pipeline_for("nested-probe").as_deref(),+ Some("restart-copy")+ );+ assert_eq!(+ state+ .engine+ .default_pipeline_for("from-template-probe")+ .as_deref(),+ Some("restart-copy")+ );+ assert_eq!(+ state.engine.default_pipeline_for("plain-probe"),+ None,+ "a template added after create must not bind the existing index"+ );++ drop(app);+ drop(state);+ }++ let (app, state) = app_on(dir.path());++ let (status, settings) = send(&app, get("/restart-probe/_settings")).await;+ assert_eq!(status, StatusCode::OK, "{settings}");+ assert_eq!(+ settings.pointer("/restart-probe/settings/index/index.default_pipeline"),+ Some(&json!("restart-copy")),+ "GET _settings must still report the persisted binding: {settings}"+ );+ let (status, settings) = send(&app, get("/nested-probe/_settings")).await;+ assert_eq!(status, StatusCode::OK, "{settings}");+ assert_eq!(+ settings.pointer("/nested-probe/settings/index/default_pipeline"),+ Some(&json!("restart-copy")),+ "nested default_pipeline must still be reported: {settings}"+ );++ let (status, pipeline) = send(&app, get("/_ingest/pipeline/restart-copy")).await;+ assert_eq!(status, StatusCode::OK, "{pipeline}");++ assert_eq!(+ state+ .engine+ .default_pipeline_for("restart-probe")+ .as_deref(),+ Some("restart-copy"),+ "the in-memory binding must be rebuilt from settings.json"+ );+ assert_eq!(+ state.engine.default_pipeline_for("nested-probe").as_deref(),+ Some("restart-copy"),+ "the nested spelling must be rebuilt too"+ );+ assert_eq!(+ state+ .engine+ .default_pipeline_for("from-template-probe")+ .as_deref(),+ Some("restart-copy"),+ "a pipeline copied from a template at create must survive restart"+ );+ assert_eq!(+ state.engine.default_pipeline_for("plain-probe"),+ None,+ "reopen must not attach a template the index was not created with"+ );++ let (status, before) = send(&app, get("/restart-probe/_doc/before")).await;+ assert_eq!(status, StatusCode::OK, "{before}");+ assert_copied(&before, "before-clean-restart");++ let (status, body) = send(+ &app,+ put(+ "/restart-probe/_doc/after?refresh=true",+ json!({ "source_marker": "after-clean-restart" }),+ ),+ )+ .await;+ assert_eq!(status, StatusCode::CREATED, "{body}");+ let (status, doc) = send(&app, get("/restart-probe/_doc/after")).await;+ assert_eq!(status, StatusCode::OK, "{doc}");+ assert_copied(&doc, "after-clean-restart");++ let (status, body) = send(+ &app,+ ndjson(+ "/_bulk?refresh=true",+ "{\"index\":{\"_index\":\"nested-probe\",\"_id\":\"after\"}}\n{\"source_marker\":\"after-nested\"}\n",+ ),+ )+ .await;+ assert_eq!(status, StatusCode::OK, "{body}");+ assert_eq!(body["errors"], false, "{body}");+ let (status, doc) = send(&app, get("/nested-probe/_doc/after")).await;+ assert_eq!(status, StatusCode::OK, "{doc}");+ assert_copied(&doc, "after-nested");++ let (status, body) = send(+ &app,+ put(+ "/from-template-probe/_doc/after?refresh=true",+ json!({ "source_marker": "after-template" }),+ ),+ )+ .await;+ assert_eq!(status, StatusCode::CREATED, "{body}");+ let (status, doc) = send(&app, get("/from-template-probe/_doc/after")).await;+ assert_eq!(status, StatusCode::OK, "{doc}");+ assert_copied(&doc, "after-template");++ let (status, body) = send(+ &app,+ put(+ "/plain-probe/_doc/after?refresh=true",+ json!({ "source_marker": "after-plain" }),+ ),+ )+ .await;+ assert_eq!(status, StatusCode::CREATED, "{body}");+ let (status, doc) = send(&app, get("/plain-probe/_doc/after")).await;+ assert_eq!(status, StatusCode::OK, "{doc}");+ assert_eq!(doc["_source"]["source_marker"], "after-plain", "{doc}");+ assert!(+ doc["_source"].get("copied_marker").is_none(),+ "plain-probe must stay unbound after restart: {doc}"+ );+}diff --git a/engine/crates/xerj-engine/src/engine.rs b/engine/crates/xerj-engine/src/engine.rsindex 00a4884e6..506488586 100644--- a/engine/crates/xerj-engine/src/engine.rs+++ b/engine/crates/xerj-engine/src/engine.rs@@ -1128,6 +1128,10 @@ impl Engine {Engine::apply_date_mapping_flags(&idx, m.value());}engine.load_or_backfill_index_created_at(&name_str);+ // Before the index is visible: a write that lands+ // in the same instant must already see the+ // persisted default pipeline (#2294).+ engine.rehydrate_index_settings_from_disk(&name_str);engine.indices.insert(name_str, idx);}Err(e) => {@@ -1723,6 +1727,54 @@ impl Engine {}}+ /// Copy `<data_dir>/<name>/settings.json` into `index_settings`.+ ///+ /// Create and `PUT /_settings` fill that map, and [`Self::default_pipeline_for`]+ /// reads only the map. `GET /_settings` falls back to the file when the+ /// entry is missing, so after a restart the binding was still reported+ /// and ordinary writes no longer ran it (#2294). Boot, failed-index+ /// retry, and snapshot restore all open an existing index through+ /// `Index::open` and must rebuild the map from the same file `open` loaded.+ ///+ /// A missing file, or a JSON `null`, drops any stale entry: restoring a+ /// snapshot that was created without settings must not keep the pipeline+ /// the previous index had in memory. An unreadable or unparseable file is+ /// logged and left as-is. `Index::open` already refuses an unparseable+ /// `settings.json` before this runs, so a parse error here is a change+ /// between those two reads, not a second copy of that refusal.+ fn rehydrate_index_settings_from_disk(&self, name: &str) {+ let path = self.data_dir.join(name).join("settings.json");+ match std::fs::read(&path) {+ Ok(bytes) => match serde_json::from_slice::<Value>(&bytes) {+ Ok(settings) if !settings.is_null() => {+ self.index_settings.insert(name.to_string(), settings);+ }+ Ok(_) => {+ self.index_settings.remove(name);+ }+ Err(e) => {+ warn!(+ name,+ error = %e,+ "settings.json did not parse while rebuilding index settings; \+ leaving the in-memory binding unchanged"+ );+ }+ },+ Err(e) if e.kind() == std::io::ErrorKind::NotFound => {+ self.index_settings.remove(name);+ }+ Err(e) => {+ warn!(+ name,+ error = %e,+ "settings.json unreadable while rebuilding index settings; \+ leaving the in-memory binding unchanged"+ );+ }+ }+ }+/// Path of the persisted API-key store (`<data_dir>/api_keys.json`).fn api_keys_path(&self) -> PathBuf {self.data_dir.join("api_keys.json")@@ -3426,6 +3478,7 @@ impl Engine {if let Some(m) = self.index_mappings.get(name) {Engine::apply_date_mapping_flags(&idx, m.value());}+ self.rehydrate_index_settings_from_disk(name);self.indices.insert(name.to_string(), idx);self.failed_indices.remove(name);info!(name, "failed index reopened");@@ -4455,9 +4508,11 @@ impl Engine {return Some(p);}// A write that auto-creates the index resolves the pipeline before- // that create (#2281). The template's default applies only then: an- // index that already exists keeps whatever it was created with, even- // when this in-memory map was not rehydrated after a restart.+ // that create (#2281). The template's default applies only then.+ // An index that already exists keeps the pipeline persisted in its+ // `settings.json`, which boot, retry, and snapshot restore copy back+ // into this map (#2294). Falling through to the template would attach+ // a pipeline the index was not created with.if self.indices.contains_key(index) || self.failed_indices.contains_key(index) {return None;}@@ -4963,6 +5018,9 @@ impl Engine {if let Some(m) = self.index_mappings.get(idx_name) {Engine::apply_date_mapping_flags(&idx, m.value());}+ // Replace whatever the live map held. A snapshot created+ // without settings must not keep the previous binding.+ self.rehydrate_index_settings_from_disk(idx_name);self.indices.insert(idx_name.clone(), idx);// Restoring from a snapshot is *the* repair for an index// that failed to open, so clear the recorded failure —