Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

fix(api): _reindex applies source filtering and refuses scripts (#2719)

ordinary being22cd023993be
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 8ddf475bc..3bda406fc 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -47,6 +47,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
### Fixed
+- `_reindex` applies `source._source` field filtering to the documents it
+ copies, so an excluded secret or large field no longer reaches the
+ destination. `_source:false` is refused with ES's sentence, and a `script`
+ body is refused before any work instead of silently copying the
+ untransformed document (this build has no reindex script context).
+
- A node whose resting memory sits between the merge-pool ceiling and the
memory watermark merges again. Every background merge's output write was
refused there, so small batches were retried every 5 s without ever
diff --git a/engine/crates/xerj-api/src/es_compat.rs b/engine/crates/xerj-api/src/es_compat.rs
index 7a91279b8..1d480705d 100644
--- a/engine/crates/xerj-api/src/es_compat.rs
+++ b/engine/crates/xerj-api/src/es_compat.rs
@@ -28334,6 +28334,12 @@ pub struct ReindexBody {
/// source or destination work.
#[serde(default)]
pub max_docs: Option<i64>,
+ /// `script`: a Painless/expression transformation run on each source
+ /// document before it is written. This build has no reindex script
+ /// context; the field is captured so the handler can refuse it instead
+ /// of silently copying the untransformed (possibly unredacted) source.
+ #[serde(default)]
+ pub script: Option<Value>,
}
#[derive(Debug, Deserialize, Default)]
@@ -28348,6 +28354,11 @@ pub struct ReindexSource {
pub index: String,
#[serde(default)]
pub query: Option<Value>,
+ /// `source._source`: field filtering for the documents copied into the
+ /// destination. Serde used to drop this, so a caller excluding a secret
+ /// or large field still copied it verbatim (a data-minimization leak).
+ #[serde(default)]
+ pub _source: Option<Value>,
#[serde(default = "reindex_default_size")]
pub size: usize,
/// ES reindex-from-remote spec (`{"host": "http://...", ...}`).
@@ -28472,6 +28483,34 @@ pub async fn reindex(
.into_response();
}
+ // A reindex script would change the document source/metadata before the
+ // write. This build has no script context for that operation, and serde
+ // used to drop `script` entirely, so a redaction/rename script answered
+ // 200 having copied the untransformed document. Refuse it instead of
+ // silently copying bytes the caller asked to change (#204).
+ if body.script.as_ref().is_some_and(|script| !script.is_null()) {
+ return es_bad_request(
+ "illegal_argument_exception",
+ "reindex scripts are not supported by this XERJ version; refusing the request \
+ rather than copying documents without the requested transformation",
+ None,
+ );
+ }
+ // `_source: false` asks to copy no fields, which `_reindex` cannot honor:
+ // the source search must return a document to copy. ES rejects it too.
+ let source_filter = match body.source._source.clone() {
+ Some(value) if value.is_null() => json!(true),
+ Some(value) => value,
+ None => json!(true),
+ };
+ if source_filter == Value::Bool(false) {
+ return es_bad_request(
+ "illegal_argument_exception",
+ "_source:false is not supported in this context",
+ None,
+ );
+ }
+
// Validate request controls before any source is flushed or the
// destination is created. Serde used to drop all three, so a caller
// asking for create-only or a capped copy silently got an overwriting
@@ -28626,8 +28665,9 @@ pub async fn reindex(
"query": effective_query,
"size": page_size,
"sort": [{ "_id": "asc" }],
- // Copies must retain engine-generated vector companions too.
- "_source": true,
+ // Apply the caller's `source._source` filter. With no filter,
+ // `true` retains engine-generated vector companions too.
+ "_source": source_filter.clone(),
}))
.map_err(|e| xerj_common::XerjError::invalid_query(e.to_string()))
{
@@ -50036,6 +50076,7 @@ mod reindex_keyset_tests {
source: ReindexSource {
index: "src".into(),
query: None,
+ _source: None,
size: 1000,
remote: None,
},
@@ -50046,6 +50087,7 @@ mod reindex_keyset_tests {
},
conflicts: None,
max_docs: None,
+ script: None,
};
let resp = reindex(
State(state.clone()),
diff --git a/engine/crates/xerj-api/tests/reindex_source_filtering.rs b/engine/crates/xerj-api/tests/reindex_source_filtering.rs
new file mode 100644
index 000000000..d4f810005
--- /dev/null
+++ b/engine/crates/xerj-api/tests/reindex_source_filtering.rs
@@ -0,0 +1,251 @@
+//! `_reindex` must copy only the fields the caller asked for, and must not
+//! silently ignore a transformation it cannot run.
+//!
+//! `source._source` was not in `ReindexSource` and `script` was not in
+//! `ReindexBody`, so serde dropped both. A caller excluding a secret or large
+//! field still copied it verbatim, and a caller asking for a redaction/rename
+//! script got the untransformed document copied under a 200. Both are
+//! data-minimization failures, not a missing convenience.
+//!
+//! Elasticsearch semantics pinned here:
+//! - `_source: [fields]` and `_source: {excludes: [...]}` limit the bytes that
+//! reach the destination;
+//! - `_source: false` is refused by the reindex context;
+//! - `script` is part of the copy contract. This build has no reindex script
+//! context, so the request is refused instead of copying raw documents.
+//!
+//! Elasticsearch is referenced for wire semantics only; no ES code is
+//! reproduced here.
+
+use axum::body::Body;
+use axum::http::{Request, StatusCode};
+use serde_json::{json, Value};
+use tower::ServiceExt;
+
+async fn app() -> (axum::Router, tempfile::TempDir) {
+ let dir = tempfile::tempdir().expect("tempdir");
+ let mut config = xerj_common::config::Config::default();
+ config.server.data_dir = dir.path().to_string_lossy().into_owned();
+ config.storage.wal_sync = xerj_common::config::WalSync::Async;
+ let metrics = xerj_common::metrics::Metrics::new().expect("metrics");
+ let engine = xerj_engine::Engine::new(config.clone()).expect("engine");
+ let state = xerj_api::state::AppState::new(config, engine, metrics);
+ (xerj_api::router::build_es_compat_router(state), dir)
+}
+
+async fn call(app: &axum::Router, method: &str, path: &str, body: Value) -> (StatusCode, Value) {
+ let mut req = Request::builder().method(method).uri(path);
+ let body = if body.is_null() {
+ Body::empty()
+ } else {
+ req = req.header("content-type", "application/json");
+ Body::from(body.to_string())
+ };
+ let response = app.clone().oneshot(req.body(body).unwrap()).await.unwrap();
+ let status = response.status();
+ let bytes = axum::body::to_bytes(response.into_body(), usize::MAX)
+ .await
+ .unwrap();
+ (
+ status,
+ serde_json::from_slice(&bytes).unwrap_or(Value::Null),
+ )
+}
+
+async fn create_index(app: &axum::Router, index: &str) {
+ let (status, body) = call(
+ app,
+ "PUT",
+ &format!("/{index}"),
+ json!({"mappings": {"properties": {
+ "text": {"type": "keyword"},
+ "secret": {"type": "keyword"}
+ }}}),
+ )
+ .await;
+ assert_eq!(status, StatusCode::OK, "{body}");
+}
+
+async fn source_of(app: &axum::Router, index: &str, id: &str) -> Option<Value> {
+ let response = app
+ .clone()
+ .oneshot(
+ Request::builder()
+ .method("GET")
+ .uri(format!("/{index}/_doc/{id}"))
+ .body(Body::empty())
+ .unwrap(),
+ )
+ .await
+ .unwrap();
+ if response.status() != StatusCode::OK {
+ return None;
+ }
+ let bytes = axum::body::to_bytes(response.into_body(), usize::MAX)
+ .await
+ .unwrap();
+ let body: Value = serde_json::from_slice(&bytes).unwrap_or(Value::Null);
+ Some(body["_source"].clone())
+}
+
+async fn reindex(app: &axum::Router, body: Value) -> (StatusCode, Value) {
+ call(app, "POST", "/_reindex", body).await
+}
+
+#[tokio::test]
+async fn source_include_list_limits_copied_fields() {
+ let (app, _dir) = app().await;
+ create_index(&app, "s").await;
+ call(
+ &app,
+ "PUT",
+ "/s/_doc/1",
+ json!({"text": "keep", "secret": "drop"}),
+ )
+ .await;
+
+ let (status, body) = reindex(
+ &app,
+ json!({
+ "source": {"index": "s", "_source": ["text"]},
+ "dest": {"index": "d1"}
+ }),
+ )
+ .await;
+ assert_eq!(status, StatusCode::OK, "{body}");
+ assert_eq!(body["created"], 1, "{body}");
+ assert_eq!(
+ source_of(&app, "d1", "1").await,
+ Some(json!({"text": "keep"})),
+ "the excluded field must not reach the destination"
+ );
+}
+
+#[tokio::test]
+async fn source_exclude_object_limits_copied_fields() {
+ let (app, _dir) = app().await;
+ create_index(&app, "s").await;
+ call(
+ &app,
+ "PUT",
+ "/s/_doc/1",
+ json!({"text": "keep", "secret": "drop"}),
+ )
+ .await;
+
+ let (status, body) = reindex(
+ &app,
+ json!({
+ "source": {"index": "s", "_source": {"excludes": ["secret"]}},
+ "dest": {"index": "d1"}
+ }),
+ )
+ .await;
+ assert_eq!(status, StatusCode::OK, "{body}");
+ assert_eq!(body["created"], 1, "{body}");
+ assert_eq!(
+ source_of(&app, "d1", "1").await,
+ Some(json!({"text": "keep"})),
+ "the excluded field must not reach the destination"
+ );
+}
+
+#[tokio::test]
+async fn source_false_is_refused_before_any_work() {
+ let (app, _dir) = app().await;
+ create_index(&app, "s").await;
+ call(&app, "PUT", "/s/_doc/1", json!({"text": "keep"})).await;
+
+ let (status, body) = reindex(
+ &app,
+ json!({
+ "source": {"index": "s", "_source": false},
+ "dest": {"index": "d"}
+ }),
+ )
+ .await;
+ assert_eq!(status, StatusCode::BAD_REQUEST, "{body}");
+ assert_eq!(
+ body.pointer("/error/type").and_then(Value::as_str),
+ Some("illegal_argument_exception"),
+ "{body}"
+ );
+ let reason = body
+ .pointer("/error/root_cause/0/reason")
+ .and_then(Value::as_str)
+ .unwrap_or_default();
+ assert!(reason.contains("_source:false"), "{body}");
+
+ let (status, _) = call(&app, "GET", "/d", Value::Null).await;
+ assert_eq!(
+ status,
+ StatusCode::NOT_FOUND,
+ "no destination may be created"
+ );
+}
+
+#[tokio::test]
+async fn script_is_refused_instead_of_copying_untransformed() {
+ let (app, _dir) = app().await;
+ create_index(&app, "s").await;
+ call(
+ &app,
+ "PUT",
+ "/s/_doc/1",
+ json!({"text": "keep", "secret": "drop"}),
+ )
+ .await;
+
+ let (status, body) = reindex(
+ &app,
+ json!({
+ "source": {"index": "s"},
+ "dest": {"index": "d"},
+ "script": {"source": "ctx._source.remove('secret')", "lang": "painless"}
+ }),
+ )
+ .await;
+ assert_eq!(status, StatusCode::BAD_REQUEST, "{body}");
+ assert_eq!(
+ body.pointer("/error/type").and_then(Value::as_str),
+ Some("illegal_argument_exception"),
+ "{body}"
+ );
+ let reason = body
+ .pointer("/error/root_cause/0/reason")
+ .and_then(Value::as_str)
+ .unwrap_or_default();
+ assert!(reason.contains("scripts are not supported"), "{body}");
+
+ let (status, _) = call(&app, "GET", "/d", Value::Null).await;
+ assert_eq!(
+ status,
+ StatusCode::NOT_FOUND,
+ "no destination may be created"
+ );
+}
+
+#[tokio::test]
+async fn default_copies_the_whole_source() {
+ let (app, _dir) = app().await;
+ create_index(&app, "s").await;
+ call(
+ &app,
+ "PUT",
+ "/s/_doc/1",
+ json!({"text": "keep", "secret": "also-kept"}),
+ )
+ .await;
+
+ let (status, body) = reindex(
+ &app,
+ json!({"source": {"index": "s"}, "dest": {"index": "d"}}),
+ )
+ .await;
+ assert_eq!(status, StatusCode::OK, "{body}");
+ assert_eq!(
+ source_of(&app, "d", "1").await,
+ Some(json!({"text": "keep", "secret": "also-kept"})),
+ "with no filter the whole source is copied"
+ );
+}