AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
fix(api): _update_by_query/_delete_by_query honor wait_for_completion=false
Vincenzo Lombardo69a26b5a29aa
diff --git a/engine/crates/xerj-api/src/error.rs b/engine/crates/xerj-api/src/error.rsindex 174b3083b..9a87b94bc 100644--- a/engine/crates/xerj-api/src/error.rs+++ b/engine/crates/xerj-api/src/error.rs@@ -21,6 +21,7 @@ use axum::{Json,};use serde::Serialize;+use serde_json::{json, Value};use xerj_common::XerjError;// ─────────────────────────────────────────────────────────────────────────────@@ -106,8 +107,16 @@ impl From<XerjError> for ApiError {}}-impl IntoResponse for ApiError {- fn into_response(self) -> Response {+impl ApiError {+ /// ES-compatible error body as a plain `Value`, for contexts that never+ /// go through axum's `Response` type — e.g. storing the `response` of a+ /// completed `_tasks/{id}` entry for a detached background task.+ pub fn into_value(self) -> Value {+ let (_, body) = self.into_parts();+ serde_json::to_value(body).unwrap_or_else(|_| json!({}))+ }++ fn into_parts(self) -> (StatusCode, EsErrorResponse) {let mut status_code = self.inner.http_status();// ES parity: an explicit write/metadata block is 403@@ -228,6 +237,13 @@ impl IntoResponse for ApiError {status: status_code,};+ (http_status, body)+ }+}++impl IntoResponse for ApiError {+ fn into_response(self) -> Response {+ let (http_status, body) = self.into_parts();(http_status, Json(body)).into_response()}}diff --git a/engine/crates/xerj-api/src/es_compat.rs b/engine/crates/xerj-api/src/es_compat.rsindex c27161890..8ab7331a7 100644--- a/engine/crates/xerj-api/src/es_compat.rs+++ b/engine/crates/xerj-api/src/es_compat.rs@@ -17107,11 +17107,9 @@ pub struct DeleteByQueryBody {pub async fn delete_by_query(State(state): State<AppState>,Path(index): Path<String>,+ Query(params): Query<HashMap<String, String>>,Json(body): Json<DeleteByQueryBody>,) -> impl IntoResponse {- let started = Instant::now();- let _task = state.tasks.register("indices:data/write/delete/byquery");-let idx = match state.engine.get_index(&index) {Ok(i) => i,Err(e) => return ApiError::new(xerj_common::XerjError::from(e)).into_response(),@@ -17126,9 +17124,40 @@ pub async fn delete_by_query(Err(e) => return ApiError::new(e).into_response(),};- let results = match idx.search(&search_req).await {+ // ES defaults `wait_for_completion` to true (synchronous); only an+ // explicit `false` switches to the async `{"task": "node:id"}` form.+ let wait_for_completion = params+ .get("wait_for_completion")+ .map(|v| v != "false")+ .unwrap_or(true);+ let handle = state.tasks.register("indices:data/write/delete/byquery");++ if !wait_for_completion {+ let task_key = handle.key().to_string();+ let spawned_key = task_key.clone();+ let tasks = state.tasks.clone();+ tokio::spawn(async move {+ let response = run_delete_by_query(&idx, &search_req).await;+ tasks.complete(&spawned_key, response);+ drop(handle);+ });+ return Json(json!({ "task": task_key })).into_response();+ }++ let response = run_delete_by_query(&idx, &search_req).await;+ drop(handle);+ Json(response).into_response()+}++async fn run_delete_by_query(+ idx: &xerj_engine::Index,+ search_req: &xerj_query::SearchRequest,+) -> Value {+ let started = Instant::now();++ let results = match idx.search(search_req).await {Ok(r) => r,- Err(e) => return ApiError::new(xerj_common::XerjError::from(e)).into_response(),+ Err(e) => return ApiError::new(xerj_common::XerjError::from(e)).into_value(),};let total = results.hits.len() as u64;@@ -17148,7 +17177,7 @@ pub async fn delete_by_query(}let took = started.elapsed().as_millis() as u64;- Json(json!({+ json!({"took": took,"timed_out": false,"total": total,@@ -17160,8 +17189,7 @@ pub async fn delete_by_query("throttled_millis": 0,"requests_per_second": -1,"throttled_until_millis": 0,- }))- .into_response()+ })}// ─────────────────────────────────────────────────────────────────────────────@@ -17178,11 +17206,9 @@ pub struct UpdateByQueryBody {pub async fn update_by_query(State(state): State<AppState>,Path(index): Path<String>,+ Query(params): Query<HashMap<String, String>>,Json(body): Json<UpdateByQueryBody>,) -> impl IntoResponse {- let started = Instant::now();- let _task = state.tasks.register("indices:data/write/update/byquery");-let idx = match state.engine.get_index(&index) {Ok(i) => i,Err(e) => return ApiError::new(xerj_common::XerjError::from(e)).into_response(),@@ -17197,20 +17223,52 @@ pub async fn update_by_query(Err(e) => return ApiError::new(e).into_response(),};- let results = match idx.search(&search_req).await {+ // Optional painless script — when present, each matched hit's source is+ // mutated by the script and re-indexed under its EXISTING `_id`, so the+ // update happens in place (no duplicate-`_id` docs are appended).+ let script = body.script.as_ref().map(extract_update_script);++ // ES defaults `wait_for_completion` to true (synchronous); only an+ // explicit `false` switches to the async `{"task": "node:id"}` form.+ let wait_for_completion = params+ .get("wait_for_completion")+ .map(|v| v != "false")+ .unwrap_or(true);+ let handle = state.tasks.register("indices:data/write/update/byquery");++ if !wait_for_completion {+ let task_key = handle.key().to_string();+ let spawned_key = task_key.clone();+ let tasks = state.tasks.clone();+ tokio::spawn(async move {+ let response = run_update_by_query(&idx, &search_req, script).await;+ tasks.complete(&spawned_key, response);+ drop(handle);+ });+ return Json(json!({ "task": task_key })).into_response();+ }++ let response = run_update_by_query(&idx, &search_req, script).await;+ drop(handle);+ Json(response).into_response()+}++async fn run_update_by_query(+ idx: &xerj_engine::Index,+ search_req: &xerj_query::SearchRequest,+ script: Option<(String, Value)>,+) -> Value {+ let started = Instant::now();++ let results = match idx.search(search_req).await {Ok(r) => r,- Err(e) => return ApiError::new(xerj_common::XerjError::from(e)).into_response(),+ Err(e) => return ApiError::new(xerj_common::XerjError::from(e)).into_value(),};let total = results.hits.len() as u64;let mut updated = 0u64;let mut failures: Vec<Value> = Vec::new();- // Optional painless script — when present, each matched hit's source is- // mutated by the script and re-indexed under its EXISTING `_id`, so the- // update happens in place (no duplicate-`_id` docs are appended).- let script = body.script.as_ref().map(extract_update_script);-for hit in results.hits {if hit.source.is_null() {continue;@@ -17241,7 +17299,7 @@ pub async fn update_by_query(}let took = started.elapsed().as_millis() as u64;- Json(json!({+ json!({"took": took,"timed_out": false,"total": total,@@ -17254,8 +17312,7 @@ pub async fn update_by_query("throttled_millis": 0,"requests_per_second": -1,"throttled_until_millis": 0,- }))- .into_response()+ })}// ─────────────────────────────────────────────────────────────────────────────@@ -21789,26 +21846,38 @@ pub async fn get_task_by_id(State(state): State<AppState>,Path(task_id): Path<String>,) -> impl IntoResponse {- match state.tasks.get(&task_id) {- Some(entry) => Json(json!({+ if let Some(entry) = state.tasks.get(&task_id) {+ return Json(json!({"completed": false,"task": task_to_json(&entry),}))- .into_response(),- None => (- StatusCode::NOT_FOUND,- Json(json!({- "error": {- "type": "resource_not_found_exception",- "reason": format!(- "task [{task_id}] isn't running and hasn't stored its results"- ),- },- "status": 404- })),- )- .into_response(),+ .into_response();+ }+ if let Some(completed) = state.tasks.get_completed(&task_id) {+ return Json(json!({+ "completed": true,+ "task": task_to_json_with_nanos(+ &completed.entry,+ completed.running_nanos,+ completed.entry.is_cancelled(),+ ),+ "response": completed.response,+ }))+ .into_response();}+ (+ StatusCode::NOT_FOUND,+ Json(json!({+ "error": {+ "type": "resource_not_found_exception",+ "reason": format!(+ "task [{task_id}] isn't running and hasn't stored its results"+ ),+ },+ "status": 404+ })),+ )+ .into_response()}pub async fn cancel_task(@@ -27194,6 +27263,17 @@ fn dir_size_bytes(p: &std::path::Path) -> u64 {/// and order match the previous hard-coded `get_task_by_id` response — only the/// values are now real.fn task_to_json(entry: &crate::state::TaskEntry) -> Value {+ task_to_json_with_nanos(entry, entry.running_nanos(), entry.is_cancelled())+}++/// Same as [`task_to_json`], but with `running_time_in_nanos`/`cancelled`+/// supplied explicitly — used for completed tasks, whose running time must+/// stay frozen at completion rather than keep growing via `entry.start_instant`.+fn task_to_json_with_nanos(+ entry: &crate::state::TaskEntry,+ running_nanos: u64,+ cancelled: bool,+) -> Value {json!({"node": entry.node.as_str(),"id": entry.id,@@ -27202,9 +27282,9 @@ fn task_to_json(entry: &crate::state::TaskEntry) -> Value {"status": {},"description": entry.action,"start_time_in_millis": entry.start_time_ms,- "running_time_in_nanos": entry.running_nanos(),+ "running_time_in_nanos": running_nanos,"cancellable": true,- "cancelled": entry.is_cancelled(),+ "cancelled": cancelled,"headers": {}})}diff --git a/engine/crates/xerj-api/src/state.rs b/engine/crates/xerj-api/src/state.rsindex 4bb896b75..34d7534bb 100644--- a/engine/crates/xerj-api/src/state.rs+++ b/engine/crates/xerj-api/src/state.rs@@ -140,7 +140,9 @@ impl TaskEntry {/// RAII handle returned by [`TaskRegistry::register`]. Dropping it (at the end/// of the long-running handler) removes the task from the registry, so the-/// Tasks API only ever reflects genuinely in-flight operations.+/// Tasks API only ever reflects genuinely in-flight operations. If the task+/// was already finalized via [`TaskRegistry::complete`], the removal here is+/// a no-op (the entry has already moved to the completed store).pub struct TaskHandle {inner: Arc<DashMap<String, TaskEntry>>,key: String,@@ -151,6 +153,10 @@ impl TaskHandle {pub fn is_cancelled(&self) -> bool {self.cancelled.load(Ordering::Relaxed)}+ /// The ES task key (`{node}:{id}`) this handle was registered under.+ pub fn key(&self) -> &str {+ &self.key+ }}impl Drop for TaskHandle {@@ -159,10 +165,24 @@ impl Drop for TaskHandle {}}+/// Snapshot of a finished long-running task kept around so `GET+/// /_tasks/{id}` can report `completed: true` with a final `response` —+/// mirroring ES, which persists completed-task docs (`.tasks` index) after+/// the in-flight entry disappears instead of turning into a 404.+#[derive(Clone)]+pub struct CompletedTask {+ pub entry: TaskEntry,+ /// `running_time_in_nanos` frozen at completion time (the live+ /// `TaskEntry::running_nanos` would otherwise keep growing forever).+ pub running_nanos: u64,+ pub response: serde_json::Value,+}+/// Registry of in-flight long-running tasks. Cheaply cloneable (`Arc` inside).#[derive(Clone)]pub struct TaskRegistry {inner: Arc<DashMap<String, TaskEntry>>,+ completed: Arc<DashMap<String, CompletedTask>>,next_id: Arc<AtomicU64>,node_id: Arc<String>,}@@ -171,6 +191,7 @@ impl TaskRegistry {pub fn new(node_id: Arc<String>) -> Self {Self {inner: Arc::new(DashMap::new()),+ completed: Arc::new(DashMap::new()),next_id: Arc::new(AtomicU64::new(1)),node_id,}@@ -210,6 +231,26 @@ impl TaskRegistry {pub fn list(&self) -> Vec<TaskEntry> {self.inner.iter().map(|e| e.value().clone()).collect()}+ /// Move a finished task from the in-flight registry into the completed+ /// store, freezing its running time and attaching the final `response`.+ /// A no-op if `key` isn't currently in-flight (e.g. already completed,+ /// or never registered).+ pub fn complete(&self, key: &str, response: serde_json::Value) {+ if let Some((_, entry)) = self.inner.remove(key) {+ let running_nanos = entry.running_nanos();+ self.completed.insert(+ key.to_string(),+ CompletedTask {+ entry,+ running_nanos,+ response,+ },+ );+ }+ }+ pub fn get_completed(&self, id: &str) -> Option<CompletedTask> {+ self.completed.get(id).map(|e| e.value().clone())+ }}// ─────────────────────────────────────────────────────────────────────────────