Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

fix(engine): apply persisted default_pipeline after restart (#2296)

xerj-orgd53bc2e44f29
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 13e54266a..b9e8eaeb3 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -45,6 +45,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
### Fixed
+- An existing index's persisted `index.default_pipeline` runs again after a
+ clean restart. `GET /_settings` still reported the binding, and the
+ pipeline definition was still there, but an ordinary document write no
+ longer copied the field: the in-memory binding was filled on create and
+ on settings update, and opening the index from disk did not rebuild it.
+ Boot, a failed-index retry, and snapshot restore now copy `settings.json`
+ back into that map. Explicit `?pipeline=` was already unaffected
+ ([#2294](https://github.com/xerj-org/xerj/issues/2294)).
+
- `xerj autoindex` no longer aborts the node on one large JSON file (#1122).
A bulk whose analysis would cross the memory watermark is rejected with
HTTP 429 before the documents are parsed, so several in-flight bulks cannot
diff --git a/engine/crates/xerj-api/src/es_compat.rs b/engine/crates/xerj-api/src/es_compat.rs
index 8f7ba1063..be45777ae 100644
--- a/engine/crates/xerj-api/src/es_compat.rs
+++ b/engine/crates/xerj-api/src/es_compat.rs
@@ -2975,10 +2975,11 @@ async fn get_index_inner(
///
/// Two fixes over replaying the display map alone (#1092):
///
-/// * The display map (`engine.index_settings`) lives in memory only, so after a
-/// restart it is empty and every create-time setting vanished from the
-/// response. A live index with no display entry falls back to its persisted
-/// `settings.json` copy — what the index was created with.
+/// * The display map (`engine.index_settings`) is rebuilt from each index's
+/// `settings.json` when that index is opened (#2294). A live index whose
+/// display entry is still missing falls back to the same file — what the
+/// index was created with — so `GET /_settings` and `default_pipeline_for`
+/// agree.
/// * `analysis` is reported from the engine, not echoed: whatever spelling the
/// caller used (`settings.analysis` beside an `index` block was dropped
/// outright), the response carries `index.analysis` = the block the index's
diff --git a/engine/crates/xerj-api/tests/default_pipeline_survives_restart.rs b/engine/crates/xerj-api/tests/default_pipeline_survives_restart.rs
new file mode 100644
index 000000000..21a5a94b7
--- /dev/null
+++ b/engine/crates/xerj-api/tests/default_pipeline_survives_restart.rs
@@ -0,0 +1,328 @@
+//! Issue #2294: an index created with a persisted `index.default_pipeline`
+//! kept reporting the binding from `GET /_settings` after a clean restart
+//! (the response falls back to `settings.json`), and the pipeline definition
+//! survived in cluster state, but an ordinary write no longer ran it.
+//!
+//! `Engine::default_pipeline_for` reads the in-memory `index_settings` map.
+//! That map was filled on create and on `PUT /_settings`, and the boot,
+//! failed-index retry, and snapshot-restore open paths never copied
+//! `settings.json` back into it. Explicit `?pipeline=` still worked because
+//! it does not consult the map.
+//!
+//! `index.final_pipeline` is not a setting this build reads.
+
+use axum::body::Body;
+use axum::http::{Request, StatusCode};
+use serde_json::{json, Value};
+use tower::ServiceExt;
+
+fn app_on(dir: &std::path::Path) -> (axum::Router, xerj_api::state::AppState) {
+ let mut config = xerj_common::config::Config::default();
+ config.server.data_dir = dir.to_string_lossy().into_owned();
+ config.storage.wal_sync = xerj_common::config::WalSync::Async;
+ let metrics = xerj_common::metrics::Metrics::new().expect("metrics");
+ let engine = xerj_engine::Engine::new(config.clone()).expect("engine");
+ let state = xerj_api::state::AppState::new(config, engine, metrics);
+ let app = xerj_api::router::build_es_compat_router(state.clone());
+ (app, state)
+}
+
+async fn send(app: &axum::Router, req: Request<Body>) -> (StatusCode, Value) {
+ let res = app.clone().oneshot(req).await.expect("response");
+ let status = res.status();
+ let bytes = axum::body::to_bytes(res.into_body(), 8 * 1024 * 1024)
+ .await
+ .expect("body");
+ let body = serde_json::from_slice(&bytes).unwrap_or(Value::Null);
+ (status, body)
+}
+
+fn put(path: &str, body: Value) -> Request<Body> {
+ Request::builder()
+ .method("PUT")
+ .uri(path)
+ .header("content-type", "application/json")
+ .body(Body::from(body.to_string()))
+ .expect("request")
+}
+
+fn get(path: &str) -> Request<Body> {
+ Request::builder()
+ .method("GET")
+ .uri(path)
+ .body(Body::empty())
+ .expect("request")
+}
+
+fn ndjson(path: &str, body: &str) -> Request<Body> {
+ Request::builder()
+ .method("POST")
+ .uri(path)
+ .header("content-type", "application/x-ndjson")
+ .body(Body::from(body.to_string()))
+ .expect("request")
+}
+
+/// The pipeline from the report: copy `source_marker` onto `copied_marker`.
+/// The ES `copy` processor is stored as `copy_field`.
+async fn define_copy_pipeline(app: &axum::Router) {
+ let (status, body) = send(
+ app,
+ put(
+ "/_ingest/pipeline/restart-copy",
+ json!({
+ "processors": [{
+ "copy": { "source": "source_marker", "target": "copied_marker" }
+ }]
+ }),
+ ),
+ )
+ .await;
+ assert_eq!(status, StatusCode::OK, "{body}");
+}
+
+fn assert_copied(doc: &Value, marker: &str) {
+ assert_eq!(
+ doc["_source"]["source_marker"].as_str(),
+ Some(marker),
+ "source field missing: {doc}"
+ );
+ assert_eq!(
+ doc["_source"]["copied_marker"].as_str(),
+ Some(marker),
+ "default pipeline did not copy the field: {doc}"
+ );
+}
+
+/// Dotted `index.default_pipeline` (the spelling in #2294) and the nested
+/// spelling both have to keep transforming writes after the process that
+/// created the index is gone.
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn persisted_default_pipeline_runs_after_clean_restart() {
+ let dir = tempfile::tempdir().expect("data dir");
+
+ {
+ let (app, state) = app_on(dir.path());
+ define_copy_pipeline(&app).await;
+
+ let (status, body) = send(
+ &app,
+ put(
+ "/restart-probe",
+ json!({
+ "settings": { "index.default_pipeline": "restart-copy" },
+ "mappings": { "properties": {
+ "source_marker": { "type": "keyword" },
+ "copied_marker": { "type": "keyword" }
+ }}
+ }),
+ ),
+ )
+ .await;
+ assert_eq!(status, StatusCode::OK, "{body}");
+
+ let (status, body) = send(
+ &app,
+ put(
+ "/nested-probe",
+ json!({
+ "settings": { "index": { "default_pipeline": "restart-copy" } },
+ "mappings": { "properties": {
+ "source_marker": { "type": "keyword" },
+ "copied_marker": { "type": "keyword" }
+ }}
+ }),
+ ),
+ )
+ .await;
+ assert_eq!(status, StatusCode::OK, "{body}");
+
+ // #2281 stored a template pipeline on the index at create. That copy
+ // lives in settings.json and has to come back on open too.
+ let (status, body) = send(
+ &app,
+ put(
+ "/_index_template/restart-template",
+ json!({
+ "index_patterns": ["from-template-*"],
+ "priority": 100,
+ "template": {
+ "settings": { "index.default_pipeline": "restart-copy" }
+ }
+ }),
+ ),
+ )
+ .await;
+ assert_eq!(status, StatusCode::OK, "{body}");
+ let (status, body) = send(&app, put("/from-template-probe", json!({}))).await;
+ assert_eq!(status, StatusCode::OK, "{body}");
+
+ // A template registered after the index exists must not become that
+ // index's pipeline on the next open.
+ let (status, body) = send(&app, put("/plain-probe", json!({}))).await;
+ assert_eq!(status, StatusCode::OK, "{body}");
+ let (status, body) = send(
+ &app,
+ put(
+ "/_index_template/plain-template",
+ json!({
+ "index_patterns": ["plain-*"],
+ "priority": 100,
+ "template": {
+ "settings": { "index.default_pipeline": "restart-copy" }
+ }
+ }),
+ ),
+ )
+ .await;
+ assert_eq!(status, StatusCode::OK, "{body}");
+
+ let (status, body) = send(
+ &app,
+ put(
+ "/restart-probe/_doc/before?refresh=true",
+ json!({ "source_marker": "before-clean-restart" }),
+ ),
+ )
+ .await;
+ assert_eq!(status, StatusCode::CREATED, "{body}");
+ let (status, doc) = send(&app, get("/restart-probe/_doc/before")).await;
+ assert_eq!(status, StatusCode::OK, "{doc}");
+ assert_copied(&doc, "before-clean-restart");
+
+ assert_eq!(
+ state
+ .engine
+ .default_pipeline_for("restart-probe")
+ .as_deref(),
+ Some("restart-copy")
+ );
+ assert_eq!(
+ state.engine.default_pipeline_for("nested-probe").as_deref(),
+ Some("restart-copy")
+ );
+ assert_eq!(
+ state
+ .engine
+ .default_pipeline_for("from-template-probe")
+ .as_deref(),
+ Some("restart-copy")
+ );
+ assert_eq!(
+ state.engine.default_pipeline_for("plain-probe"),
+ None,
+ "a template added after create must not bind the existing index"
+ );
+
+ drop(app);
+ drop(state);
+ }
+
+ let (app, state) = app_on(dir.path());
+
+ let (status, settings) = send(&app, get("/restart-probe/_settings")).await;
+ assert_eq!(status, StatusCode::OK, "{settings}");
+ assert_eq!(
+ settings.pointer("/restart-probe/settings/index/index.default_pipeline"),
+ Some(&json!("restart-copy")),
+ "GET _settings must still report the persisted binding: {settings}"
+ );
+ let (status, settings) = send(&app, get("/nested-probe/_settings")).await;
+ assert_eq!(status, StatusCode::OK, "{settings}");
+ assert_eq!(
+ settings.pointer("/nested-probe/settings/index/default_pipeline"),
+ Some(&json!("restart-copy")),
+ "nested default_pipeline must still be reported: {settings}"
+ );
+
+ let (status, pipeline) = send(&app, get("/_ingest/pipeline/restart-copy")).await;
+ assert_eq!(status, StatusCode::OK, "{pipeline}");
+
+ assert_eq!(
+ state
+ .engine
+ .default_pipeline_for("restart-probe")
+ .as_deref(),
+ Some("restart-copy"),
+ "the in-memory binding must be rebuilt from settings.json"
+ );
+ assert_eq!(
+ state.engine.default_pipeline_for("nested-probe").as_deref(),
+ Some("restart-copy"),
+ "the nested spelling must be rebuilt too"
+ );
+ assert_eq!(
+ state
+ .engine
+ .default_pipeline_for("from-template-probe")
+ .as_deref(),
+ Some("restart-copy"),
+ "a pipeline copied from a template at create must survive restart"
+ );
+ assert_eq!(
+ state.engine.default_pipeline_for("plain-probe"),
+ None,
+ "reopen must not attach a template the index was not created with"
+ );
+
+ let (status, before) = send(&app, get("/restart-probe/_doc/before")).await;
+ assert_eq!(status, StatusCode::OK, "{before}");
+ assert_copied(&before, "before-clean-restart");
+
+ let (status, body) = send(
+ &app,
+ put(
+ "/restart-probe/_doc/after?refresh=true",
+ json!({ "source_marker": "after-clean-restart" }),
+ ),
+ )
+ .await;
+ assert_eq!(status, StatusCode::CREATED, "{body}");
+ let (status, doc) = send(&app, get("/restart-probe/_doc/after")).await;
+ assert_eq!(status, StatusCode::OK, "{doc}");
+ assert_copied(&doc, "after-clean-restart");
+
+ let (status, body) = send(
+ &app,
+ ndjson(
+ "/_bulk?refresh=true",
+ "{\"index\":{\"_index\":\"nested-probe\",\"_id\":\"after\"}}\n{\"source_marker\":\"after-nested\"}\n",
+ ),
+ )
+ .await;
+ assert_eq!(status, StatusCode::OK, "{body}");
+ assert_eq!(body["errors"], false, "{body}");
+ let (status, doc) = send(&app, get("/nested-probe/_doc/after")).await;
+ assert_eq!(status, StatusCode::OK, "{doc}");
+ assert_copied(&doc, "after-nested");
+
+ let (status, body) = send(
+ &app,
+ put(
+ "/from-template-probe/_doc/after?refresh=true",
+ json!({ "source_marker": "after-template" }),
+ ),
+ )
+ .await;
+ assert_eq!(status, StatusCode::CREATED, "{body}");
+ let (status, doc) = send(&app, get("/from-template-probe/_doc/after")).await;
+ assert_eq!(status, StatusCode::OK, "{doc}");
+ assert_copied(&doc, "after-template");
+
+ let (status, body) = send(
+ &app,
+ put(
+ "/plain-probe/_doc/after?refresh=true",
+ json!({ "source_marker": "after-plain" }),
+ ),
+ )
+ .await;
+ assert_eq!(status, StatusCode::CREATED, "{body}");
+ let (status, doc) = send(&app, get("/plain-probe/_doc/after")).await;
+ assert_eq!(status, StatusCode::OK, "{doc}");
+ assert_eq!(doc["_source"]["source_marker"], "after-plain", "{doc}");
+ assert!(
+ doc["_source"].get("copied_marker").is_none(),
+ "plain-probe must stay unbound after restart: {doc}"
+ );
+}
diff --git a/engine/crates/xerj-engine/src/engine.rs b/engine/crates/xerj-engine/src/engine.rs
index 00a4884e6..506488586 100644
--- a/engine/crates/xerj-engine/src/engine.rs
+++ b/engine/crates/xerj-engine/src/engine.rs
@@ -1128,6 +1128,10 @@ impl Engine {
Engine::apply_date_mapping_flags(&idx, m.value());
}
engine.load_or_backfill_index_created_at(&name_str);
+ // Before the index is visible: a write that lands
+ // in the same instant must already see the
+ // persisted default pipeline (#2294).
+ engine.rehydrate_index_settings_from_disk(&name_str);
engine.indices.insert(name_str, idx);
}
Err(e) => {
@@ -1723,6 +1727,54 @@ impl Engine {
}
}
+ /// Copy `<data_dir>/<name>/settings.json` into `index_settings`.
+ ///
+ /// Create and `PUT /_settings` fill that map, and [`Self::default_pipeline_for`]
+ /// reads only the map. `GET /_settings` falls back to the file when the
+ /// entry is missing, so after a restart the binding was still reported
+ /// and ordinary writes no longer ran it (#2294). Boot, failed-index
+ /// retry, and snapshot restore all open an existing index through
+ /// `Index::open` and must rebuild the map from the same file `open` loaded.
+ ///
+ /// A missing file, or a JSON `null`, drops any stale entry: restoring a
+ /// snapshot that was created without settings must not keep the pipeline
+ /// the previous index had in memory. An unreadable or unparseable file is
+ /// logged and left as-is. `Index::open` already refuses an unparseable
+ /// `settings.json` before this runs, so a parse error here is a change
+ /// between those two reads, not a second copy of that refusal.
+ fn rehydrate_index_settings_from_disk(&self, name: &str) {
+ let path = self.data_dir.join(name).join("settings.json");
+ match std::fs::read(&path) {
+ Ok(bytes) => match serde_json::from_slice::<Value>(&bytes) {
+ Ok(settings) if !settings.is_null() => {
+ self.index_settings.insert(name.to_string(), settings);
+ }
+ Ok(_) => {
+ self.index_settings.remove(name);
+ }
+ Err(e) => {
+ warn!(
+ name,
+ error = %e,
+ "settings.json did not parse while rebuilding index settings; \
+ leaving the in-memory binding unchanged"
+ );
+ }
+ },
+ Err(e) if e.kind() == std::io::ErrorKind::NotFound => {
+ self.index_settings.remove(name);
+ }
+ Err(e) => {
+ warn!(
+ name,
+ error = %e,
+ "settings.json unreadable while rebuilding index settings; \
+ leaving the in-memory binding unchanged"
+ );
+ }
+ }
+ }
+
/// Path of the persisted API-key store (`<data_dir>/api_keys.json`).
fn api_keys_path(&self) -> PathBuf {
self.data_dir.join("api_keys.json")
@@ -3426,6 +3478,7 @@ impl Engine {
if let Some(m) = self.index_mappings.get(name) {
Engine::apply_date_mapping_flags(&idx, m.value());
}
+ self.rehydrate_index_settings_from_disk(name);
self.indices.insert(name.to_string(), idx);
self.failed_indices.remove(name);
info!(name, "failed index reopened");
@@ -4455,9 +4508,11 @@ impl Engine {
return Some(p);
}
// A write that auto-creates the index resolves the pipeline before
- // that create (#2281). The template's default applies only then: an
- // index that already exists keeps whatever it was created with, even
- // when this in-memory map was not rehydrated after a restart.
+ // that create (#2281). The template's default applies only then.
+ // An index that already exists keeps the pipeline persisted in its
+ // `settings.json`, which boot, retry, and snapshot restore copy back
+ // into this map (#2294). Falling through to the template would attach
+ // a pipeline the index was not created with.
if self.indices.contains_key(index) || self.failed_indices.contains_key(index) {
return None;
}
@@ -4963,6 +5018,9 @@ impl Engine {
if let Some(m) = self.index_mappings.get(idx_name) {
Engine::apply_date_mapping_flags(&idx, m.value());
}
+ // Replace whatever the live map held. A snapshot created
+ // without settings must not keep the previous binding.
+ self.rehydrate_index_settings_from_disk(idx_name);
self.indices.insert(idx_name.clone(), idx);
// Restoring from a snapshot is *the* repair for an index
// that failed to open, so clear the recorded failure —