AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
fix(api): _reindex applies source filtering and refuses scripts (#2719)
ordinary being22cd023993be
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 8ddf475bc..3bda406fc 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -47,6 +47,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0### Fixed+- `_reindex` applies `source._source` field filtering to the documents it+ copies, so an excluded secret or large field no longer reaches the+ destination. `_source:false` is refused with ES's sentence, and a `script`+ body is refused before any work instead of silently copying the+ untransformed document (this build has no reindex script context).+- A node whose resting memory sits between the merge-pool ceiling and thememory watermark merges again. Every background merge's output write wasrefused there, so small batches were retried every 5 s without everdiff --git a/engine/crates/xerj-api/src/es_compat.rs b/engine/crates/xerj-api/src/es_compat.rsindex 7a91279b8..1d480705d 100644--- a/engine/crates/xerj-api/src/es_compat.rs+++ b/engine/crates/xerj-api/src/es_compat.rs@@ -28334,6 +28334,12 @@ pub struct ReindexBody {/// source or destination work.#[serde(default)]pub max_docs: Option<i64>,+ /// `script`: a Painless/expression transformation run on each source+ /// document before it is written. This build has no reindex script+ /// context; the field is captured so the handler can refuse it instead+ /// of silently copying the untransformed (possibly unredacted) source.+ #[serde(default)]+ pub script: Option<Value>,}#[derive(Debug, Deserialize, Default)]@@ -28348,6 +28354,11 @@ pub struct ReindexSource {pub index: String,#[serde(default)]pub query: Option<Value>,+ /// `source._source`: field filtering for the documents copied into the+ /// destination. Serde used to drop this, so a caller excluding a secret+ /// or large field still copied it verbatim (a data-minimization leak).+ #[serde(default)]+ pub _source: Option<Value>,#[serde(default = "reindex_default_size")]pub size: usize,/// ES reindex-from-remote spec (`{"host": "http://...", ...}`).@@ -28472,6 +28483,34 @@ pub async fn reindex(.into_response();}+ // A reindex script would change the document source/metadata before the+ // write. This build has no script context for that operation, and serde+ // used to drop `script` entirely, so a redaction/rename script answered+ // 200 having copied the untransformed document. Refuse it instead of+ // silently copying bytes the caller asked to change (#204).+ if body.script.as_ref().is_some_and(|script| !script.is_null()) {+ return es_bad_request(+ "illegal_argument_exception",+ "reindex scripts are not supported by this XERJ version; refusing the request \+ rather than copying documents without the requested transformation",+ None,+ );+ }+ // `_source: false` asks to copy no fields, which `_reindex` cannot honor:+ // the source search must return a document to copy. ES rejects it too.+ let source_filter = match body.source._source.clone() {+ Some(value) if value.is_null() => json!(true),+ Some(value) => value,+ None => json!(true),+ };+ if source_filter == Value::Bool(false) {+ return es_bad_request(+ "illegal_argument_exception",+ "_source:false is not supported in this context",+ None,+ );+ }+// Validate request controls before any source is flushed or the// destination is created. Serde used to drop all three, so a caller// asking for create-only or a capped copy silently got an overwriting@@ -28626,8 +28665,9 @@ pub async fn reindex("query": effective_query,"size": page_size,"sort": [{ "_id": "asc" }],- // Copies must retain engine-generated vector companions too.- "_source": true,+ // Apply the caller's `source._source` filter. With no filter,+ // `true` retains engine-generated vector companions too.+ "_source": source_filter.clone(),})).map_err(|e| xerj_common::XerjError::invalid_query(e.to_string())){@@ -50036,6 +50076,7 @@ mod reindex_keyset_tests {source: ReindexSource {index: "src".into(),query: None,+ _source: None,size: 1000,remote: None,},@@ -50046,6 +50087,7 @@ mod reindex_keyset_tests {},conflicts: None,max_docs: None,+ script: None,};let resp = reindex(State(state.clone()),diff --git a/engine/crates/xerj-api/tests/reindex_source_filtering.rs b/engine/crates/xerj-api/tests/reindex_source_filtering.rsnew file mode 100644index 000000000..d4f810005--- /dev/null+++ b/engine/crates/xerj-api/tests/reindex_source_filtering.rs@@ -0,0 +1,251 @@+//! `_reindex` must copy only the fields the caller asked for, and must not+//! silently ignore a transformation it cannot run.+//!+//! `source._source` was not in `ReindexSource` and `script` was not in+//! `ReindexBody`, so serde dropped both. A caller excluding a secret or large+//! field still copied it verbatim, and a caller asking for a redaction/rename+//! script got the untransformed document copied under a 200. Both are+//! data-minimization failures, not a missing convenience.+//!+//! Elasticsearch semantics pinned here:+//! - `_source: [fields]` and `_source: {excludes: [...]}` limit the bytes that+//! reach the destination;+//! - `_source: false` is refused by the reindex context;+//! - `script` is part of the copy contract. This build has no reindex script+//! context, so the request is refused instead of copying raw documents.+//!+//! Elasticsearch is referenced for wire semantics only; no ES code is+//! reproduced here.++use axum::body::Body;+use axum::http::{Request, StatusCode};+use serde_json::{json, Value};+use tower::ServiceExt;++async fn app() -> (axum::Router, tempfile::TempDir) {+ let dir = tempfile::tempdir().expect("tempdir");+ let mut config = xerj_common::config::Config::default();+ config.server.data_dir = dir.path().to_string_lossy().into_owned();+ config.storage.wal_sync = xerj_common::config::WalSync::Async;+ let metrics = xerj_common::metrics::Metrics::new().expect("metrics");+ let engine = xerj_engine::Engine::new(config.clone()).expect("engine");+ let state = xerj_api::state::AppState::new(config, engine, metrics);+ (xerj_api::router::build_es_compat_router(state), dir)+}++async fn call(app: &axum::Router, method: &str, path: &str, body: Value) -> (StatusCode, Value) {+ let mut req = Request::builder().method(method).uri(path);+ let body = if body.is_null() {+ Body::empty()+ } else {+ req = req.header("content-type", "application/json");+ Body::from(body.to_string())+ };+ let response = app.clone().oneshot(req.body(body).unwrap()).await.unwrap();+ let status = response.status();+ let bytes = axum::body::to_bytes(response.into_body(), usize::MAX)+ .await+ .unwrap();+ (+ status,+ serde_json::from_slice(&bytes).unwrap_or(Value::Null),+ )+}++async fn create_index(app: &axum::Router, index: &str) {+ let (status, body) = call(+ app,+ "PUT",+ &format!("/{index}"),+ json!({"mappings": {"properties": {+ "text": {"type": "keyword"},+ "secret": {"type": "keyword"}+ }}}),+ )+ .await;+ assert_eq!(status, StatusCode::OK, "{body}");+}++async fn source_of(app: &axum::Router, index: &str, id: &str) -> Option<Value> {+ let response = app+ .clone()+ .oneshot(+ Request::builder()+ .method("GET")+ .uri(format!("/{index}/_doc/{id}"))+ .body(Body::empty())+ .unwrap(),+ )+ .await+ .unwrap();+ if response.status() != StatusCode::OK {+ return None;+ }+ let bytes = axum::body::to_bytes(response.into_body(), usize::MAX)+ .await+ .unwrap();+ let body: Value = serde_json::from_slice(&bytes).unwrap_or(Value::Null);+ Some(body["_source"].clone())+}++async fn reindex(app: &axum::Router, body: Value) -> (StatusCode, Value) {+ call(app, "POST", "/_reindex", body).await+}++#[tokio::test]+async fn source_include_list_limits_copied_fields() {+ let (app, _dir) = app().await;+ create_index(&app, "s").await;+ call(+ &app,+ "PUT",+ "/s/_doc/1",+ json!({"text": "keep", "secret": "drop"}),+ )+ .await;++ let (status, body) = reindex(+ &app,+ json!({+ "source": {"index": "s", "_source": ["text"]},+ "dest": {"index": "d1"}+ }),+ )+ .await;+ assert_eq!(status, StatusCode::OK, "{body}");+ assert_eq!(body["created"], 1, "{body}");+ assert_eq!(+ source_of(&app, "d1", "1").await,+ Some(json!({"text": "keep"})),+ "the excluded field must not reach the destination"+ );+}++#[tokio::test]+async fn source_exclude_object_limits_copied_fields() {+ let (app, _dir) = app().await;+ create_index(&app, "s").await;+ call(+ &app,+ "PUT",+ "/s/_doc/1",+ json!({"text": "keep", "secret": "drop"}),+ )+ .await;++ let (status, body) = reindex(+ &app,+ json!({+ "source": {"index": "s", "_source": {"excludes": ["secret"]}},+ "dest": {"index": "d1"}+ }),+ )+ .await;+ assert_eq!(status, StatusCode::OK, "{body}");+ assert_eq!(body["created"], 1, "{body}");+ assert_eq!(+ source_of(&app, "d1", "1").await,+ Some(json!({"text": "keep"})),+ "the excluded field must not reach the destination"+ );+}++#[tokio::test]+async fn source_false_is_refused_before_any_work() {+ let (app, _dir) = app().await;+ create_index(&app, "s").await;+ call(&app, "PUT", "/s/_doc/1", json!({"text": "keep"})).await;++ let (status, body) = reindex(+ &app,+ json!({+ "source": {"index": "s", "_source": false},+ "dest": {"index": "d"}+ }),+ )+ .await;+ assert_eq!(status, StatusCode::BAD_REQUEST, "{body}");+ assert_eq!(+ body.pointer("/error/type").and_then(Value::as_str),+ Some("illegal_argument_exception"),+ "{body}"+ );+ let reason = body+ .pointer("/error/root_cause/0/reason")+ .and_then(Value::as_str)+ .unwrap_or_default();+ assert!(reason.contains("_source:false"), "{body}");++ let (status, _) = call(&app, "GET", "/d", Value::Null).await;+ assert_eq!(+ status,+ StatusCode::NOT_FOUND,+ "no destination may be created"+ );+}++#[tokio::test]+async fn script_is_refused_instead_of_copying_untransformed() {+ let (app, _dir) = app().await;+ create_index(&app, "s").await;+ call(+ &app,+ "PUT",+ "/s/_doc/1",+ json!({"text": "keep", "secret": "drop"}),+ )+ .await;++ let (status, body) = reindex(+ &app,+ json!({+ "source": {"index": "s"},+ "dest": {"index": "d"},+ "script": {"source": "ctx._source.remove('secret')", "lang": "painless"}+ }),+ )+ .await;+ assert_eq!(status, StatusCode::BAD_REQUEST, "{body}");+ assert_eq!(+ body.pointer("/error/type").and_then(Value::as_str),+ Some("illegal_argument_exception"),+ "{body}"+ );+ let reason = body+ .pointer("/error/root_cause/0/reason")+ .and_then(Value::as_str)+ .unwrap_or_default();+ assert!(reason.contains("scripts are not supported"), "{body}");++ let (status, _) = call(&app, "GET", "/d", Value::Null).await;+ assert_eq!(+ status,+ StatusCode::NOT_FOUND,+ "no destination may be created"+ );+}++#[tokio::test]+async fn default_copies_the_whole_source() {+ let (app, _dir) = app().await;+ create_index(&app, "s").await;+ call(+ &app,+ "PUT",+ "/s/_doc/1",+ json!({"text": "keep", "secret": "also-kept"}),+ )+ .await;++ let (status, body) = reindex(+ &app,+ json!({"source": {"index": "s"}, "dest": {"index": "d"}}),+ )+ .await;+ assert_eq!(status, StatusCode::OK, "{body}");+ assert_eq!(+ source_of(&app, "d", "1").await,+ Some(json!({"text": "keep", "secret": "also-kept"})),+ "with no filter the whole source is copied"+ );+}