Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

fix(api): _reindex honours op_type, conflicts and max_docs (#2653)

ordinary being4ec65b2fa754
diff --git a/CHANGELOG.md b/CHANGELOG.md
index abc2a0cb1..55d48c523 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -35,6 +35,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
### Fixed
+- `_reindex` honours `dest.op_type: create`, `conflicts`, and `max_docs`
+ instead of silently dropping them. `create` no longer overwrites an
+ existing destination document: the default abort returns 409 with ES's
+ version-conflict failure, and `conflicts: proceed` skips and counts it.
+ `max_docs` caps successful writes (the query-parameter form is accepted
+ too); invalid values are refused before any source or destination work.
+
- Boolean text queries with one `must` match now keep `filter` and `must_not`
clauses non-scoring before a flush and on mixed memory/segment indices.
Their retained hits use the same BM25 scores as the bare match (#2403).
diff --git a/engine/crates/xerj-api/src/es_compat.rs b/engine/crates/xerj-api/src/es_compat.rs
index 270da3dca..c42faceed 100644
--- a/engine/crates/xerj-api/src/es_compat.rs
+++ b/engine/crates/xerj-api/src/es_compat.rs
@@ -28325,6 +28325,22 @@ pub async fn clear_scroll_path(
pub struct ReindexBody {
pub source: ReindexSource,
pub dest: ReindexDest,
+ /// `conflicts`: `abort` (default) stops on the first version conflict;
+ /// `proceed` skips it, counts it, and keeps copying.
+ #[serde(default)]
+ pub conflicts: Option<String>,
+ /// `max_docs`: cap on documents successfully written. `-1` or absent
+ /// means no cap. The handler validates the positive/-1 range before any
+ /// source or destination work.
+ #[serde(default)]
+ pub max_docs: Option<i64>,
+}
+
+#[derive(Debug, Deserialize, Default)]
+pub struct ReindexParams {
+ /// `_reindex?max_docs=N` — the query-parameter form ES also accepts.
+ #[serde(default)]
+ pub max_docs: Option<i64>,
}
#[derive(Debug, Deserialize)]
@@ -28357,6 +28373,12 @@ pub struct ReindexDest {
/// `{"created": N}` having copied every document VERBATIM.
#[serde(default)]
pub pipeline: Option<String>,
+ /// `dest.op_type`: `index` (default) overwrites an existing id; `create`
+ /// only creates missing documents and reports a version conflict for an
+ /// existing id. Serde used to drop this field, so `create` silently
+ /// overwrote the destination and answered 200.
+ #[serde(default)]
+ pub op_type: Option<String>,
}
/// The filter attached to `alias` on one backing index, if any. Alias
@@ -28375,8 +28397,41 @@ fn alias_filter_for(state: &AppState, backing: &str, alias: &str) -> Option<Valu
})
}
+/// The ES `_reindex` summary shared by the success and version-conflict-abort
+/// responses. `total` counts documents successfully processed, not the number
+/// of source hits returned: a conflict is counted separately in
+/// `version_conflicts`, and an abort carries it in `failures` as a 409 bulk
+/// item while the HTTP status is also 409.
+#[allow(clippy::too_many_arguments)]
+fn reindex_summary(
+ started: Instant,
+ total: usize,
+ updated: usize,
+ created: usize,
+ noops: usize,
+ batches: usize,
+ version_conflicts: usize,
+ failures: &[Value],
+) -> Value {
+ json!({
+ "took": started.elapsed().as_millis() as u64,
+ "timed_out": false,
+ "total": total,
+ "updated": updated,
+ "created": created,
+ "deleted": 0,
+ "batches": batches,
+ "failures": failures,
+ "throttled_millis": 0,
+ "noops": noops,
+ "version_conflicts": version_conflicts,
+ "retries": { "bulk": 0, "search": 0 },
+ })
+}
+
pub async fn reindex(
State(state): State<AppState>,
+ Query(params): Query<ReindexParams>,
Json(body): Json<ReindexBody>,
) -> impl IntoResponse {
let started = Instant::now();
@@ -28417,6 +28472,53 @@ pub async fn reindex(
.into_response();
}
+ // Validate request controls before any source is flushed or the
+ // destination is created. Serde used to drop all three, so a caller
+ // asking for create-only or a capped copy silently got an overwriting
+ // full copy (and a 200).
+ let op_type = body.dest.op_type.as_deref().unwrap_or("index");
+ if !matches!(op_type, "index" | "create") {
+ return es_bad_request(
+ "illegal_argument_exception",
+ &format!("op_type [{op_type}] is not valid; must be one of [index, create]"),
+ None,
+ );
+ }
+ let conflicts = body.conflicts.as_deref().unwrap_or("abort");
+ if !matches!(conflicts, "abort" | "proceed") {
+ return es_bad_request(
+ "illegal_argument_exception",
+ &format!("conflicts [{conflicts}] is not valid; must be one of [abort, proceed]"),
+ None,
+ );
+ }
+ let max_docs = match body.max_docs.or(params.max_docs) {
+ None | Some(-1) => None,
+ Some(n) if n > 0 => match usize::try_from(n) {
+ Ok(n) => Some(n),
+ Err(_) => {
+ return es_bad_request(
+ "illegal_argument_exception",
+ &format!(
+ "maxDocs should be greater than 0 if the request is limited to some \
+ number of documents or -1 if it isn't but it was [{n}]"
+ ),
+ None,
+ )
+ }
+ },
+ Some(n) => {
+ return es_bad_request(
+ "illegal_argument_exception",
+ &format!(
+ "maxDocs should be greater than 0 if the request is limited to some \
+ number of documents or -1 if it isn't but it was [{n}]"
+ ),
+ None,
+ )
+ }
+ };
+
let source_name = &body.source.index;
// The destination may be an alias. ES routes a write through an alias to
// its single write index and rejects an ambiguous alias;
@@ -28559,13 +28661,31 @@ pub async fn reindex(
let mut created = 0usize;
let mut updated = 0usize;
let mut noops = 0usize;
+ let mut version_conflicts = 0usize;
let mut failures: Vec<Value> = Vec::new();
let mut batches = 0usize;
+ // Set when `max_docs` has been reached mid-page, so both the hit loop and
+ // the member/page loops stop without counting the rest of that page.
+ let mut stop = false;
for (_, source_idx, mut search_req) in prepared_sources {
// IDs are ordered only within one source member. Each parsed plan
// starts with no cursor and advances after its own completed page.
loop {
+ if let Some(limit) = max_docs {
+ if created + updated >= limit {
+ stop = true;
+ break;
+ }
+ }
+ // `max_docs` caps SUCCESSFUL writes. With `conflicts=proceed` a
+ // page may contain conflicts, so request only the writes still
+ // allowed and keep paging when conflicts consume the page.
+ search_req.size = match max_docs {
+ Some(limit) => limit.saturating_sub(created + updated).min(page_size),
+ None => page_size,
+ };
+ let requested = search_req.size;
let results = match source_idx.search(&search_req).await {
Ok(r) => r,
Err(e) => return ApiError::new(xerj_common::XerjError::from(e)).into_response(),
@@ -28584,12 +28704,12 @@ pub async fn reindex(
}
batches += 1;
- total_fetched += batch_size;
// Capture the keyset cursor before the `for` loop consumes `results.hits`.
let last_id = results.hits.last().map(|h| h.id.clone());
for hit in results.hits {
+ total_fetched += 1;
if !hit.source.is_null() {
// `dest.pipeline`, if the request named one. Already known to
// exist and to be runnable (checked before any copy started),
@@ -28635,30 +28755,88 @@ pub async fn reindex(
}
},
};
- // Check if doc already exists in dest to track created vs updated.
- let exists = dest_idx
- .get_document(&hit.id)
- .await
- .ok()
- .flatten()
- .is_some();
- match dest_idx
- .index_document(Some(hit.id.clone()), doc_source)
- .await
- {
- Ok(_) => {
- if exists {
- updated += 1;
- } else {
- created += 1;
+ if op_type == "create" {
+ // `create_document` is the engine's atomic
+ // op_type=create: the existence check runs under the
+ // write lock, so a concurrent writer cannot slip a
+ // document between a get_document() and the write.
+ match dest_idx.create_document(hit.id.clone(), doc_source).await {
+ Ok(_) => created += 1,
+ Err(xerj_engine::EngineError::Common(
+ xerj_common::XerjError::VersionConflict { id, actual, .. },
+ )) => {
+ version_conflicts += 1;
+ if conflicts == "proceed" {
+ continue;
+ }
+ // ES aborts the run on the first conflict and
+ // answers 409 with one bulk failure entry; the
+ // document already in the destination is not
+ // overwritten.
+ failures.push(json!({
+ "index": dest_name.as_str(),
+ "id": hit.id,
+ "status": 409,
+ "cause": {
+ "type": "version_conflict_engine_exception",
+ "reason": format!(
+ "[{id}]: version conflict, document already exists \
+ (current version [{actual}])"
+ ),
+ "index": dest_name.as_str(),
+ "shard": "0",
+ },
+ }));
+ return (
+ StatusCode::CONFLICT,
+ Json(reindex_summary(
+ started,
+ created + updated + noops,
+ updated,
+ created,
+ noops,
+ batches,
+ version_conflicts,
+ &failures,
+ )),
+ )
+ .into_response();
}
- }
- Err(e) => {
- failures.push(json!({
+ Err(e) => failures.push(json!({
"id": hit.id,
"cause": { "type": "reindex_error", "reason": e.to_string() },
- }));
+ })),
}
+ } else {
+ // Check if doc already exists in dest to track created vs updated.
+ let exists = dest_idx
+ .get_document(&hit.id)
+ .await
+ .ok()
+ .flatten()
+ .is_some();
+ match dest_idx
+ .index_document(Some(hit.id.clone()), doc_source)
+ .await
+ {
+ Ok(_) => {
+ if exists {
+ updated += 1;
+ } else {
+ created += 1;
+ }
+ }
+ Err(e) => {
+ failures.push(json!({
+ "id": hit.id,
+ "cause": { "type": "reindex_error", "reason": e.to_string() },
+ }));
+ }
+ }
+ }
+ if max_docs.is_some_and(|limit| created + updated >= limit) {
+ stop = true;
+ break;
}
}
}
@@ -28666,33 +28844,30 @@ pub async fn reindex(
// Advance the keyset cursor to the last `_id` of this page.
search_req.search_after = last_id.map(|id| vec![json!(id)]);
- // Stop if we've reached the safety backstop or fetched fewer docs than
- // requested (meaning we hit the end of the source).
- if batch_size < page_size || total_fetched >= max_total {
+ // Stop if the write cap was reached, the safety backstop fired, or
+ // this page was short (the member is exhausted). `requested` is
+ // the size actually sent, which may be below `page_size` because
+ // of `max_docs`.
+ if stop || batch_size < requested || total_fetched >= max_total {
break;
}
}
- if total_fetched >= max_total {
+ if stop || total_fetched >= max_total {
break;
}
}
- let took = started.elapsed().as_millis() as u64;
- Json(json!({
- "took": took,
- "timed_out": false,
- "total": total_fetched,
- "updated": updated,
- "created": created,
- "deleted": 0,
- "batches": batches,
- "failures": failures,
- "throttled_millis": 0,
- "noops": noops,
- "version_conflicts": 0,
- "retries": { "bulk": 0, "search": 0 },
- }))
+ Json(reindex_summary(
+ started,
+ created + updated + noops,
+ updated,
+ created,
+ noops,
+ batches,
+ version_conflicts,
+ &failures,
+ ))
.into_response()
}
@@ -49853,11 +50028,18 @@ mod reindex_keyset_tests {
dest: ReindexDest {
index: "dst".into(),
pipeline: None,
+ op_type: None,
},
+ conflicts: None,
+ max_docs: None,
};
- let resp = reindex(State(state.clone()), Json(body))
- .await
- .into_response();
+ let resp = reindex(
+ State(state.clone()),
+ Query(ReindexParams::default()),
+ Json(body),
+ )
+ .await
+ .into_response();
assert!(
resp.status().is_success(),
"reindex returned {}",
diff --git a/engine/crates/xerj-api/tests/reindex_request_controls.rs b/engine/crates/xerj-api/tests/reindex_request_controls.rs
new file mode 100644
index 000000000..006d39f3c
--- /dev/null
+++ b/engine/crates/xerj-api/tests/reindex_request_controls.rs
@@ -0,0 +1,282 @@
+//! `_reindex` must honour the request controls that bound or guard a copy.
+//!
+//! `dest.op_type=create`, top-level `conflicts`, and `max_docs` were not in the
+//! request structs, so serde dropped them. A caller asking `create` got an
+//! overwrite of the existing destination document; a caller asking
+//! `max_docs: 1` got the whole source. Both answered 200, so the wrong result
+//! was silent.
+//!
+//! Elasticsearch semantics pinned here:
+//! - `create` conflicts are version conflicts; `conflicts: abort` (default)
+//! stops on the first one and answers 409 with the normal reindex summary,
+//! `version_conflicts`, and one `failures[]` entry whose item `status` is 409;
+//! - `conflicts: proceed` skips the conflict, counts it, and continues;
+//! - `max_docs` caps SUCCESSFUL writes, so with `proceed` the source may be
+//! read past the cap until that many documents have been written.
+//!
+//! Elasticsearch is referenced for wire semantics only; no ES code is
+//! reproduced here.
+
+use axum::body::Body;
+use axum::http::{Request, StatusCode};
+use serde_json::{json, Value};
+use tower::ServiceExt;
+
+async fn app() -> (axum::Router, tempfile::TempDir) {
+ let dir = tempfile::tempdir().expect("tempdir");
+ let mut config = xerj_common::config::Config::default();
+ config.server.data_dir = dir.path().to_string_lossy().into_owned();
+ config.storage.wal_sync = xerj_common::config::WalSync::Async;
+ let metrics = xerj_common::metrics::Metrics::new().expect("metrics");
+ let engine = xerj_engine::Engine::new(config.clone()).expect("engine");
+ let state = xerj_api::state::AppState::new(config, engine, metrics);
+ (xerj_api::router::build_es_compat_router(state), dir)
+}
+
+async fn call(app: &axum::Router, method: &str, path: &str, body: Value) -> (StatusCode, Value) {
+ let mut req = Request::builder().method(method).uri(path);
+ let body = if body.is_null() {
+ Body::empty()
+ } else {
+ req = req.header("content-type", "application/json");
+ Body::from(body.to_string())
+ };
+ let response = app.clone().oneshot(req.body(body).unwrap()).await.unwrap();
+ let status = response.status();
+ let bytes = axum::body::to_bytes(response.into_body(), usize::MAX)
+ .await
+ .unwrap();
+ (
+ status,
+ serde_json::from_slice(&bytes).unwrap_or(Value::Null),
+ )
+}
+
+async fn create_index(app: &axum::Router, index: &str) {
+ let (status, body) = call(
+ app,
+ "PUT",
+ &format!("/{index}"),
+ json!({"mappings": {"properties": {"v": {"type": "keyword"}}}}),
+ )
+ .await;
+ assert_eq!(status, StatusCode::OK, "{body}");
+}
+
+async fn doc(app: &axum::Router, index: &str, id: &str, v: &str) {
+ let (status, body) = call(app, "PUT", &format!("/{index}/_doc/{id}"), json!({"v": v})).await;
+ assert_eq!(status, StatusCode::CREATED, "{body}");
+}
+
+async fn value_of(app: &axum::Router, index: &str, id: &str) -> Option<String> {
+ let response = app
+ .clone()
+ .oneshot(
+ Request::builder()
+ .method("GET")
+ .uri(format!("/{index}/_doc/{id}"))
+ .body(Body::empty())
+ .unwrap(),
+ )
+ .await
+ .unwrap();
+ if response.status() != StatusCode::OK {
+ return None;
+ }
+ let bytes = axum::body::to_bytes(response.into_body(), usize::MAX)
+ .await
+ .unwrap();
+ let body: Value = serde_json::from_slice(&bytes).unwrap_or(Value::Null);
+ body["_source"]["v"].as_str().map(str::to_string)
+}
+
+async fn count(app: &axum::Router, index: &str) -> u64 {
+ let (status, body) = call(app, "POST", &format!("/{index}/_refresh"), Value::Null).await;
+ assert_eq!(status, StatusCode::OK, "{body}");
+ let (status, body) = call(app, "GET", &format!("/{index}/_count"), Value::Null).await;
+ assert_eq!(status, StatusCode::OK, "{body}");
+ body["count"].as_u64().unwrap_or(0)
+}
+
+async fn reindex(app: &axum::Router, path: &str, body: Value) -> (StatusCode, Value) {
+ call(app, "POST", path, body).await
+}
+
+#[tokio::test]
+async fn op_type_create_aborts_without_overwriting() {
+ let (app, _dir) = app().await;
+ create_index(&app, "s").await;
+ create_index(&app, "d").await;
+ doc(&app, "s", "1", "source").await;
+ doc(&app, "d", "1", "existing").await;
+
+ let (status, body) = reindex(
+ &app,
+ "/_reindex",
+ json!({"source": {"index": "s"}, "dest": {"index": "d", "op_type": "create"}}),
+ )
+ .await;
+ assert_eq!(status, StatusCode::CONFLICT, "{body}");
+ assert_eq!(body["created"], 0, "{body}");
+ assert_eq!(body["updated"], 0, "{body}");
+ assert_eq!(body["version_conflicts"], 1, "{body}");
+ assert_eq!(body["total"], 0, "{body}");
+ assert_eq!(
+ body.pointer("/failures/0/status").and_then(Value::as_u64),
+ Some(409),
+ "{body}"
+ );
+ assert_eq!(
+ body.pointer("/failures/0/cause/type")
+ .and_then(Value::as_str),
+ Some("version_conflict_engine_exception"),
+ "{body}"
+ );
+ assert_eq!(
+ value_of(&app, "d", "1").await.as_deref(),
+ Some("existing"),
+ "the existing destination document must not be overwritten"
+ );
+}
+
+#[tokio::test]
+async fn op_type_create_with_conflicts_proceed_skips_existing() {
+ let (app, _dir) = app().await;
+ create_index(&app, "s").await;
+ create_index(&app, "d").await;
+ doc(&app, "s", "1", "source-one").await;
+ doc(&app, "s", "2", "source-two").await;
+ doc(&app, "d", "1", "existing").await;
+
+ let (status, body) = reindex(
+ &app,
+ "/_reindex",
+ json!({
+ "conflicts": "proceed",
+ "source": {"index": "s"},
+ "dest": {"index": "d", "op_type": "create"}
+ }),
+ )
+ .await;
+ assert_eq!(status, StatusCode::OK, "{body}");
+ assert_eq!(body["created"], 1, "{body}");
+ assert_eq!(body["updated"], 0, "{body}");
+ assert_eq!(body["version_conflicts"], 1, "{body}");
+ assert_eq!(body["total"], 1, "{body}");
+ assert_eq!(body["failures"], json!([]), "{body}");
+ assert_eq!(value_of(&app, "d", "1").await.as_deref(), Some("existing"));
+ assert_eq!(
+ value_of(&app, "d", "2").await.as_deref(),
+ Some("source-two")
+ );
+}
+
+#[tokio::test]
+async fn max_docs_caps_successful_writes() {
+ let (app, _dir) = app().await;
+ create_index(&app, "s").await;
+ for (id, v) in [("1", "one"), ("2", "two"), ("3", "three")] {
+ doc(&app, "s", id, v).await;
+ }
+
+ let (status, body) = reindex(
+ &app,
+ "/_reindex",
+ json!({"max_docs": 1, "source": {"index": "s"}, "dest": {"index": "d1"}}),
+ )
+ .await;
+ assert_eq!(status, StatusCode::OK, "{body}");
+ assert_eq!(body["created"], 1, "{body}");
+ assert_eq!(body["total"], 1, "{body}");
+ assert_eq!(count(&app, "d1").await, 1);
+
+ let (status, body) = reindex(
+ &app,
+ "/_reindex",
+ json!({"max_docs": 2, "source": {"index": "s"}, "dest": {"index": "d2"}}),
+ )
+ .await;
+ assert_eq!(status, StatusCode::OK, "{body}");
+ assert_eq!(body["created"], 2, "{body}");
+ assert_eq!(body["total"], 2, "{body}");
+ assert_eq!(count(&app, "d2").await, 2);
+
+ // The query-parameter form is part of the API too.
+ let (status, body) = reindex(
+ &app,
+ "/_reindex?max_docs=1",
+ json!({"source": {"index": "s"}, "dest": {"index": "d3"}}),
+ )
+ .await;
+ assert_eq!(status, StatusCode::OK, "{body}");
+ assert_eq!(body["created"], 1, "{body}");
+ assert_eq!(count(&app, "d3").await, 1);
+
+ // -1 explicitly means "no cap".
+ let (status, body) = reindex(
+ &app,
+ "/_reindex",
+ json!({"max_docs": -1, "source": {"index": "s"}, "dest": {"index": "d4"}}),
+ )
+ .await;
+ assert_eq!(status, StatusCode::OK, "{body}");
+ assert_eq!(body["created"], 3, "{body}");
+ assert_eq!(count(&app, "d4").await, 3);
+}
+
+#[tokio::test]
+async fn max_docs_counts_successful_writes_when_conflicts_proceed() {
+ let (app, _dir) = app().await;
+ create_index(&app, "s").await;
+ create_index(&app, "d").await;
+ for (id, v) in [("1", "one"), ("2", "two"), ("3", "three")] {
+ doc(&app, "s", id, v).await;
+ }
+ doc(&app, "d", "1", "existing").await;
+
+ let (status, body) = reindex(
+ &app,
+ "/_reindex",
+ json!({
+ "conflicts": "proceed",
+ "max_docs": 2,
+ "source": {"index": "s"},
+ "dest": {"index": "d", "op_type": "create"}
+ }),
+ )
+ .await;
+ assert_eq!(status, StatusCode::OK, "{body}");
+ assert_eq!(body["version_conflicts"], 1, "{body}");
+ assert_eq!(body["created"], 2, "{body}");
+ assert_eq!(body["total"], 2, "{body}");
+ assert_eq!(body["failures"], json!([]), "{body}");
+ assert_eq!(value_of(&app, "d", "1").await.as_deref(), Some("existing"));
+ assert_eq!(value_of(&app, "d", "2").await.as_deref(), Some("two"));
+ assert_eq!(value_of(&app, "d", "3").await.as_deref(), Some("three"));
+}
+
+#[tokio::test]
+async fn invalid_request_controls_are_refused_before_any_copy() {
+ let (app, _dir) = app().await;
+ create_index(&app, "s").await;
+ doc(&app, "s", "1", "one").await;
+
+ for body in [
+ json!({"source": {"index": "s"}, "dest": {"index": "d", "op_type": "upsert"}}),
+ json!({"conflicts": "ignore", "source": {"index": "s"}, "dest": {"index": "d"}}),
+ json!({"max_docs": 0, "source": {"index": "s"}, "dest": {"index": "d"}}),
+ json!({"max_docs": -2, "source": {"index": "s"}, "dest": {"index": "d"}}),
+ ] {
+ let (status, response) = reindex(&app, "/_reindex", body.clone()).await;
+ assert_eq!(status, StatusCode::BAD_REQUEST, "{body} -> {response}");
+ assert_eq!(
+ response.pointer("/error/type").and_then(Value::as_str),
+ Some("illegal_argument_exception"),
+ "{body} -> {response}"
+ );
+ }
+
+ // None of the refused requests may leave a destination index behind.
+ let (status, body) = call(&app, "GET", "/d", Value::Null).await;
+ assert_eq!(status, StatusCode::NOT_FOUND, "{body}");
+}