AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
fix(api): _reindex honours op_type, conflicts and max_docs (#2653)
ordinary being4ec65b2fa754
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex abc2a0cb1..55d48c523 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -35,6 +35,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0### Fixed+- `_reindex` honours `dest.op_type: create`, `conflicts`, and `max_docs`+ instead of silently dropping them. `create` no longer overwrites an+ existing destination document: the default abort returns 409 with ES's+ version-conflict failure, and `conflicts: proceed` skips and counts it.+ `max_docs` caps successful writes (the query-parameter form is accepted+ too); invalid values are refused before any source or destination work.+- Boolean text queries with one `must` match now keep `filter` and `must_not`clauses non-scoring before a flush and on mixed memory/segment indices.Their retained hits use the same BM25 scores as the bare match (#2403).diff --git a/engine/crates/xerj-api/src/es_compat.rs b/engine/crates/xerj-api/src/es_compat.rsindex 270da3dca..c42faceed 100644--- a/engine/crates/xerj-api/src/es_compat.rs+++ b/engine/crates/xerj-api/src/es_compat.rs@@ -28325,6 +28325,22 @@ pub async fn clear_scroll_path(pub struct ReindexBody {pub source: ReindexSource,pub dest: ReindexDest,+ /// `conflicts`: `abort` (default) stops on the first version conflict;+ /// `proceed` skips it, counts it, and keeps copying.+ #[serde(default)]+ pub conflicts: Option<String>,+ /// `max_docs`: cap on documents successfully written. `-1` or absent+ /// means no cap. The handler validates the positive/-1 range before any+ /// source or destination work.+ #[serde(default)]+ pub max_docs: Option<i64>,+}++#[derive(Debug, Deserialize, Default)]+pub struct ReindexParams {+ /// `_reindex?max_docs=N` — the query-parameter form ES also accepts.+ #[serde(default)]+ pub max_docs: Option<i64>,}#[derive(Debug, Deserialize)]@@ -28357,6 +28373,12 @@ pub struct ReindexDest {/// `{"created": N}` having copied every document VERBATIM.#[serde(default)]pub pipeline: Option<String>,+ /// `dest.op_type`: `index` (default) overwrites an existing id; `create`+ /// only creates missing documents and reports a version conflict for an+ /// existing id. Serde used to drop this field, so `create` silently+ /// overwrote the destination and answered 200.+ #[serde(default)]+ pub op_type: Option<String>,}/// The filter attached to `alias` on one backing index, if any. Alias@@ -28375,8 +28397,41 @@ fn alias_filter_for(state: &AppState, backing: &str, alias: &str) -> Option<Valu})}+/// The ES `_reindex` summary shared by the success and version-conflict-abort+/// responses. `total` counts documents successfully processed, not the number+/// of source hits returned: a conflict is counted separately in+/// `version_conflicts`, and an abort carries it in `failures` as a 409 bulk+/// item while the HTTP status is also 409.+#[allow(clippy::too_many_arguments)]+fn reindex_summary(+ started: Instant,+ total: usize,+ updated: usize,+ created: usize,+ noops: usize,+ batches: usize,+ version_conflicts: usize,+ failures: &[Value],+) -> Value {+ json!({+ "took": started.elapsed().as_millis() as u64,+ "timed_out": false,+ "total": total,+ "updated": updated,+ "created": created,+ "deleted": 0,+ "batches": batches,+ "failures": failures,+ "throttled_millis": 0,+ "noops": noops,+ "version_conflicts": version_conflicts,+ "retries": { "bulk": 0, "search": 0 },+ })+}+pub async fn reindex(State(state): State<AppState>,+ Query(params): Query<ReindexParams>,Json(body): Json<ReindexBody>,) -> impl IntoResponse {let started = Instant::now();@@ -28417,6 +28472,53 @@ pub async fn reindex(.into_response();}+ // Validate request controls before any source is flushed or the+ // destination is created. Serde used to drop all three, so a caller+ // asking for create-only or a capped copy silently got an overwriting+ // full copy (and a 200).+ let op_type = body.dest.op_type.as_deref().unwrap_or("index");+ if !matches!(op_type, "index" | "create") {+ return es_bad_request(+ "illegal_argument_exception",+ &format!("op_type [{op_type}] is not valid; must be one of [index, create]"),+ None,+ );+ }+ let conflicts = body.conflicts.as_deref().unwrap_or("abort");+ if !matches!(conflicts, "abort" | "proceed") {+ return es_bad_request(+ "illegal_argument_exception",+ &format!("conflicts [{conflicts}] is not valid; must be one of [abort, proceed]"),+ None,+ );+ }+ let max_docs = match body.max_docs.or(params.max_docs) {+ None | Some(-1) => None,+ Some(n) if n > 0 => match usize::try_from(n) {+ Ok(n) => Some(n),+ Err(_) => {+ return es_bad_request(+ "illegal_argument_exception",+ &format!(+ "maxDocs should be greater than 0 if the request is limited to some \+ number of documents or -1 if it isn't but it was [{n}]"+ ),+ None,+ )+ }+ },+ Some(n) => {+ return es_bad_request(+ "illegal_argument_exception",+ &format!(+ "maxDocs should be greater than 0 if the request is limited to some \+ number of documents or -1 if it isn't but it was [{n}]"+ ),+ None,+ )+ }+ };+let source_name = &body.source.index;// The destination may be an alias. ES routes a write through an alias to// its single write index and rejects an ambiguous alias;@@ -28559,13 +28661,31 @@ pub async fn reindex(let mut created = 0usize;let mut updated = 0usize;let mut noops = 0usize;+ let mut version_conflicts = 0usize;let mut failures: Vec<Value> = Vec::new();let mut batches = 0usize;+ // Set when `max_docs` has been reached mid-page, so both the hit loop and+ // the member/page loops stop without counting the rest of that page.+ let mut stop = false;for (_, source_idx, mut search_req) in prepared_sources {// IDs are ordered only within one source member. Each parsed plan// starts with no cursor and advances after its own completed page.loop {+ if let Some(limit) = max_docs {+ if created + updated >= limit {+ stop = true;+ break;+ }+ }+ // `max_docs` caps SUCCESSFUL writes. With `conflicts=proceed` a+ // page may contain conflicts, so request only the writes still+ // allowed and keep paging when conflicts consume the page.+ search_req.size = match max_docs {+ Some(limit) => limit.saturating_sub(created + updated).min(page_size),+ None => page_size,+ };+ let requested = search_req.size;let results = match source_idx.search(&search_req).await {Ok(r) => r,Err(e) => return ApiError::new(xerj_common::XerjError::from(e)).into_response(),@@ -28584,12 +28704,12 @@ pub async fn reindex(}batches += 1;- total_fetched += batch_size;// Capture the keyset cursor before the `for` loop consumes `results.hits`.let last_id = results.hits.last().map(|h| h.id.clone());for hit in results.hits {+ total_fetched += 1;if !hit.source.is_null() {// `dest.pipeline`, if the request named one. Already known to// exist and to be runnable (checked before any copy started),@@ -28635,30 +28755,88 @@ pub async fn reindex(}},};- // Check if doc already exists in dest to track created vs updated.- let exists = dest_idx- .get_document(&hit.id)- .await- .ok()- .flatten()- .is_some();- match dest_idx- .index_document(Some(hit.id.clone()), doc_source)- .await- {- Ok(_) => {- if exists {- updated += 1;- } else {- created += 1;+ if op_type == "create" {+ // `create_document` is the engine's atomic+ // op_type=create: the existence check runs under the+ // write lock, so a concurrent writer cannot slip a+ // document between a get_document() and the write.+ match dest_idx.create_document(hit.id.clone(), doc_source).await {+ Ok(_) => created += 1,+ Err(xerj_engine::EngineError::Common(+ xerj_common::XerjError::VersionConflict { id, actual, .. },+ )) => {+ version_conflicts += 1;+ if conflicts == "proceed" {+ continue;+ }+ // ES aborts the run on the first conflict and+ // answers 409 with one bulk failure entry; the+ // document already in the destination is not+ // overwritten.+ failures.push(json!({+ "index": dest_name.as_str(),+ "id": hit.id,+ "status": 409,+ "cause": {+ "type": "version_conflict_engine_exception",+ "reason": format!(+ "[{id}]: version conflict, document already exists \+ (current version [{actual}])"+ ),+ "index": dest_name.as_str(),+ "shard": "0",+ },+ }));+ return (+ StatusCode::CONFLICT,+ Json(reindex_summary(+ started,+ created + updated + noops,+ updated,+ created,+ noops,+ batches,+ version_conflicts,+ &failures,+ )),+ )+ .into_response();}- }- Err(e) => {- failures.push(json!({+ Err(e) => failures.push(json!({"id": hit.id,"cause": { "type": "reindex_error", "reason": e.to_string() },- }));+ })),}+ } else {+ // Check if doc already exists in dest to track created vs updated.+ let exists = dest_idx+ .get_document(&hit.id)+ .await+ .ok()+ .flatten()+ .is_some();+ match dest_idx+ .index_document(Some(hit.id.clone()), doc_source)+ .await+ {+ Ok(_) => {+ if exists {+ updated += 1;+ } else {+ created += 1;+ }+ }+ Err(e) => {+ failures.push(json!({+ "id": hit.id,+ "cause": { "type": "reindex_error", "reason": e.to_string() },+ }));+ }+ }+ }+ if max_docs.is_some_and(|limit| created + updated >= limit) {+ stop = true;+ break;}}}@@ -28666,33 +28844,30 @@ pub async fn reindex(// Advance the keyset cursor to the last `_id` of this page.search_req.search_after = last_id.map(|id| vec![json!(id)]);- // Stop if we've reached the safety backstop or fetched fewer docs than- // requested (meaning we hit the end of the source).- if batch_size < page_size || total_fetched >= max_total {+ // Stop if the write cap was reached, the safety backstop fired, or+ // this page was short (the member is exhausted). `requested` is+ // the size actually sent, which may be below `page_size` because+ // of `max_docs`.+ if stop || batch_size < requested || total_fetched >= max_total {break;}}- if total_fetched >= max_total {+ if stop || total_fetched >= max_total {break;}}- let took = started.elapsed().as_millis() as u64;- Json(json!({- "took": took,- "timed_out": false,- "total": total_fetched,- "updated": updated,- "created": created,- "deleted": 0,- "batches": batches,- "failures": failures,- "throttled_millis": 0,- "noops": noops,- "version_conflicts": 0,- "retries": { "bulk": 0, "search": 0 },- }))+ Json(reindex_summary(+ started,+ created + updated + noops,+ updated,+ created,+ noops,+ batches,+ version_conflicts,+ &failures,+ )).into_response()}@@ -49853,11 +50028,18 @@ mod reindex_keyset_tests {dest: ReindexDest {index: "dst".into(),pipeline: None,+ op_type: None,},+ conflicts: None,+ max_docs: None,};- let resp = reindex(State(state.clone()), Json(body))- .await- .into_response();+ let resp = reindex(+ State(state.clone()),+ Query(ReindexParams::default()),+ Json(body),+ )+ .await+ .into_response();assert!(resp.status().is_success(),"reindex returned {}",diff --git a/engine/crates/xerj-api/tests/reindex_request_controls.rs b/engine/crates/xerj-api/tests/reindex_request_controls.rsnew file mode 100644index 000000000..006d39f3c--- /dev/null+++ b/engine/crates/xerj-api/tests/reindex_request_controls.rs@@ -0,0 +1,282 @@+//! `_reindex` must honour the request controls that bound or guard a copy.+//!+//! `dest.op_type=create`, top-level `conflicts`, and `max_docs` were not in the+//! request structs, so serde dropped them. A caller asking `create` got an+//! overwrite of the existing destination document; a caller asking+//! `max_docs: 1` got the whole source. Both answered 200, so the wrong result+//! was silent.+//!+//! Elasticsearch semantics pinned here:+//! - `create` conflicts are version conflicts; `conflicts: abort` (default)+//! stops on the first one and answers 409 with the normal reindex summary,+//! `version_conflicts`, and one `failures[]` entry whose item `status` is 409;+//! - `conflicts: proceed` skips the conflict, counts it, and continues;+//! - `max_docs` caps SUCCESSFUL writes, so with `proceed` the source may be+//! read past the cap until that many documents have been written.+//!+//! Elasticsearch is referenced for wire semantics only; no ES code is+//! reproduced here.++use axum::body::Body;+use axum::http::{Request, StatusCode};+use serde_json::{json, Value};+use tower::ServiceExt;++async fn app() -> (axum::Router, tempfile::TempDir) {+ let dir = tempfile::tempdir().expect("tempdir");+ let mut config = xerj_common::config::Config::default();+ config.server.data_dir = dir.path().to_string_lossy().into_owned();+ config.storage.wal_sync = xerj_common::config::WalSync::Async;+ let metrics = xerj_common::metrics::Metrics::new().expect("metrics");+ let engine = xerj_engine::Engine::new(config.clone()).expect("engine");+ let state = xerj_api::state::AppState::new(config, engine, metrics);+ (xerj_api::router::build_es_compat_router(state), dir)+}++async fn call(app: &axum::Router, method: &str, path: &str, body: Value) -> (StatusCode, Value) {+ let mut req = Request::builder().method(method).uri(path);+ let body = if body.is_null() {+ Body::empty()+ } else {+ req = req.header("content-type", "application/json");+ Body::from(body.to_string())+ };+ let response = app.clone().oneshot(req.body(body).unwrap()).await.unwrap();+ let status = response.status();+ let bytes = axum::body::to_bytes(response.into_body(), usize::MAX)+ .await+ .unwrap();+ (+ status,+ serde_json::from_slice(&bytes).unwrap_or(Value::Null),+ )+}++async fn create_index(app: &axum::Router, index: &str) {+ let (status, body) = call(+ app,+ "PUT",+ &format!("/{index}"),+ json!({"mappings": {"properties": {"v": {"type": "keyword"}}}}),+ )+ .await;+ assert_eq!(status, StatusCode::OK, "{body}");+}++async fn doc(app: &axum::Router, index: &str, id: &str, v: &str) {+ let (status, body) = call(app, "PUT", &format!("/{index}/_doc/{id}"), json!({"v": v})).await;+ assert_eq!(status, StatusCode::CREATED, "{body}");+}++async fn value_of(app: &axum::Router, index: &str, id: &str) -> Option<String> {+ let response = app+ .clone()+ .oneshot(+ Request::builder()+ .method("GET")+ .uri(format!("/{index}/_doc/{id}"))+ .body(Body::empty())+ .unwrap(),+ )+ .await+ .unwrap();+ if response.status() != StatusCode::OK {+ return None;+ }+ let bytes = axum::body::to_bytes(response.into_body(), usize::MAX)+ .await+ .unwrap();+ let body: Value = serde_json::from_slice(&bytes).unwrap_or(Value::Null);+ body["_source"]["v"].as_str().map(str::to_string)+}++async fn count(app: &axum::Router, index: &str) -> u64 {+ let (status, body) = call(app, "POST", &format!("/{index}/_refresh"), Value::Null).await;+ assert_eq!(status, StatusCode::OK, "{body}");+ let (status, body) = call(app, "GET", &format!("/{index}/_count"), Value::Null).await;+ assert_eq!(status, StatusCode::OK, "{body}");+ body["count"].as_u64().unwrap_or(0)+}++async fn reindex(app: &axum::Router, path: &str, body: Value) -> (StatusCode, Value) {+ call(app, "POST", path, body).await+}++#[tokio::test]+async fn op_type_create_aborts_without_overwriting() {+ let (app, _dir) = app().await;+ create_index(&app, "s").await;+ create_index(&app, "d").await;+ doc(&app, "s", "1", "source").await;+ doc(&app, "d", "1", "existing").await;++ let (status, body) = reindex(+ &app,+ "/_reindex",+ json!({"source": {"index": "s"}, "dest": {"index": "d", "op_type": "create"}}),+ )+ .await;+ assert_eq!(status, StatusCode::CONFLICT, "{body}");+ assert_eq!(body["created"], 0, "{body}");+ assert_eq!(body["updated"], 0, "{body}");+ assert_eq!(body["version_conflicts"], 1, "{body}");+ assert_eq!(body["total"], 0, "{body}");+ assert_eq!(+ body.pointer("/failures/0/status").and_then(Value::as_u64),+ Some(409),+ "{body}"+ );+ assert_eq!(+ body.pointer("/failures/0/cause/type")+ .and_then(Value::as_str),+ Some("version_conflict_engine_exception"),+ "{body}"+ );+ assert_eq!(+ value_of(&app, "d", "1").await.as_deref(),+ Some("existing"),+ "the existing destination document must not be overwritten"+ );+}++#[tokio::test]+async fn op_type_create_with_conflicts_proceed_skips_existing() {+ let (app, _dir) = app().await;+ create_index(&app, "s").await;+ create_index(&app, "d").await;+ doc(&app, "s", "1", "source-one").await;+ doc(&app, "s", "2", "source-two").await;+ doc(&app, "d", "1", "existing").await;++ let (status, body) = reindex(+ &app,+ "/_reindex",+ json!({+ "conflicts": "proceed",+ "source": {"index": "s"},+ "dest": {"index": "d", "op_type": "create"}+ }),+ )+ .await;+ assert_eq!(status, StatusCode::OK, "{body}");+ assert_eq!(body["created"], 1, "{body}");+ assert_eq!(body["updated"], 0, "{body}");+ assert_eq!(body["version_conflicts"], 1, "{body}");+ assert_eq!(body["total"], 1, "{body}");+ assert_eq!(body["failures"], json!([]), "{body}");+ assert_eq!(value_of(&app, "d", "1").await.as_deref(), Some("existing"));+ assert_eq!(+ value_of(&app, "d", "2").await.as_deref(),+ Some("source-two")+ );+}++#[tokio::test]+async fn max_docs_caps_successful_writes() {+ let (app, _dir) = app().await;+ create_index(&app, "s").await;+ for (id, v) in [("1", "one"), ("2", "two"), ("3", "three")] {+ doc(&app, "s", id, v).await;+ }++ let (status, body) = reindex(+ &app,+ "/_reindex",+ json!({"max_docs": 1, "source": {"index": "s"}, "dest": {"index": "d1"}}),+ )+ .await;+ assert_eq!(status, StatusCode::OK, "{body}");+ assert_eq!(body["created"], 1, "{body}");+ assert_eq!(body["total"], 1, "{body}");+ assert_eq!(count(&app, "d1").await, 1);++ let (status, body) = reindex(+ &app,+ "/_reindex",+ json!({"max_docs": 2, "source": {"index": "s"}, "dest": {"index": "d2"}}),+ )+ .await;+ assert_eq!(status, StatusCode::OK, "{body}");+ assert_eq!(body["created"], 2, "{body}");+ assert_eq!(body["total"], 2, "{body}");+ assert_eq!(count(&app, "d2").await, 2);++ // The query-parameter form is part of the API too.+ let (status, body) = reindex(+ &app,+ "/_reindex?max_docs=1",+ json!({"source": {"index": "s"}, "dest": {"index": "d3"}}),+ )+ .await;+ assert_eq!(status, StatusCode::OK, "{body}");+ assert_eq!(body["created"], 1, "{body}");+ assert_eq!(count(&app, "d3").await, 1);++ // -1 explicitly means "no cap".+ let (status, body) = reindex(+ &app,+ "/_reindex",+ json!({"max_docs": -1, "source": {"index": "s"}, "dest": {"index": "d4"}}),+ )+ .await;+ assert_eq!(status, StatusCode::OK, "{body}");+ assert_eq!(body["created"], 3, "{body}");+ assert_eq!(count(&app, "d4").await, 3);+}++#[tokio::test]+async fn max_docs_counts_successful_writes_when_conflicts_proceed() {+ let (app, _dir) = app().await;+ create_index(&app, "s").await;+ create_index(&app, "d").await;+ for (id, v) in [("1", "one"), ("2", "two"), ("3", "three")] {+ doc(&app, "s", id, v).await;+ }+ doc(&app, "d", "1", "existing").await;++ let (status, body) = reindex(+ &app,+ "/_reindex",+ json!({+ "conflicts": "proceed",+ "max_docs": 2,+ "source": {"index": "s"},+ "dest": {"index": "d", "op_type": "create"}+ }),+ )+ .await;+ assert_eq!(status, StatusCode::OK, "{body}");+ assert_eq!(body["version_conflicts"], 1, "{body}");+ assert_eq!(body["created"], 2, "{body}");+ assert_eq!(body["total"], 2, "{body}");+ assert_eq!(body["failures"], json!([]), "{body}");+ assert_eq!(value_of(&app, "d", "1").await.as_deref(), Some("existing"));+ assert_eq!(value_of(&app, "d", "2").await.as_deref(), Some("two"));+ assert_eq!(value_of(&app, "d", "3").await.as_deref(), Some("three"));+}++#[tokio::test]+async fn invalid_request_controls_are_refused_before_any_copy() {+ let (app, _dir) = app().await;+ create_index(&app, "s").await;+ doc(&app, "s", "1", "one").await;++ for body in [+ json!({"source": {"index": "s"}, "dest": {"index": "d", "op_type": "upsert"}}),+ json!({"conflicts": "ignore", "source": {"index": "s"}, "dest": {"index": "d"}}),+ json!({"max_docs": 0, "source": {"index": "s"}, "dest": {"index": "d"}}),+ json!({"max_docs": -2, "source": {"index": "s"}, "dest": {"index": "d"}}),+ ] {+ let (status, response) = reindex(&app, "/_reindex", body.clone()).await;+ assert_eq!(status, StatusCode::BAD_REQUEST, "{body} -> {response}");+ assert_eq!(+ response.pointer("/error/type").and_then(Value::as_str),+ Some("illegal_argument_exception"),+ "{body} -> {response}"+ );+ }++ // None of the refused requests may leave a destination index behind.+ let (status, body) = call(&app, "GET", "/d", Value::Null).await;+ assert_eq!(status, StatusCode::NOT_FOUND, "{body}");+}