Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

fix(api): report refresh publication failures

bugerb52c20b9c357
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 23cc2c8fc..9005bbdbc 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -56,6 +56,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
tolerance made the enforced floor `interval/2` shorter — 12.5 s at the
shipped defaults. Measured at `--progress-interval 10`: 11 of 11 same-phase
gaps under 15 s (min 10.0 s) before, none after (min 19.3 s).
+- **`/{index}/_refresh` reports segment-publication failures instead of
+ claiming every shard succeeded.** The endpoint previously discarded every
+ error returned by the synchronous engine flush and always answered HTTP 200
+ with `successful == total`. It now includes the index, status, and underlying
+ reason in `_shards.failures`, attempts every resolved index, and returns the
+ first failed shard's HTTP status as Elasticsearch 8.13.4 does for refresh.
## [1.0.0-rc.15] - 2026-08-10
diff --git a/engine/crates/xerj-api/src/es_compat.rs b/engine/crates/xerj-api/src/es_compat.rs
index c451efb85..9db58a00d 100644
--- a/engine/crates/xerj-api/src/es_compat.rs
+++ b/engine/crates/xerj-api/src/es_compat.rs
@@ -15879,14 +15879,61 @@ pub async fn refresh_index(
Ok(h) => h,
Err(e) => return ApiError::new(e).into_response(),
};
- for (_, idx) in &handles {
- let _ = idx.flush().await;
+ let total = handles.len() as u64;
+ let mut successful = 0u64;
+ let mut failures = Vec::new();
+ let mut response_status = StatusCode::OK;
+ for (name, idx) in &handles {
+ match idx.flush().await {
+ Ok(()) => successful += 1,
+ Err(error) => {
+ let rendered = ApiError::new(xerj_common::XerjError::from(error)).into_value();
+ let status = rendered
+ .get("status")
+ .and_then(Value::as_u64)
+ .and_then(|code| StatusCode::from_u16(code as u16).ok())
+ .unwrap_or(StatusCode::INTERNAL_SERVER_ERROR);
+ if failures.is_empty() {
+ response_status = status;
+ }
+ let reason = rendered
+ .pointer("/error/root_cause/0")
+ .or_else(|| rendered.pointer("/error"))
+ .cloned()
+ .unwrap_or_else(|| json!({ "type": "exception", "reason": "refresh failed" }));
+ let status_name = status
+ .canonical_reason()
+ .unwrap_or("Internal Server Error")
+ .to_ascii_uppercase()
+ .replace([' ', '-'], "_");
+ failures.push(json!({
+ "shard": 0,
+ "index": name,
+ "status": status_name,
+ "reason": reason,
+ }));
+ }
+ }
}
- let n = handles.len() as u32;
- Json(json!({
- "_shards": { "total": n, "successful": n, "failed": 0 }
- }))
- .into_response()
+ let failed = failures.len() as u64;
+ let mut body = json!({
+ "_shards": { "total": total, "successful": successful, "failed": failed }
+ });
+ if failed > 0 {
+ body["_shards"]["failures"] = Value::Array(failures);
+ }
+
+ // Refresh is a synchronous publication barrier. Tantivy's analogous
+ // blocking commit contract returns its publication error to the caller
+ // (`tantivy/src/indexer/index_writer.rs:651-666`, MIT); dropping it here
+ // would turn "not published" into an affirmative success response.
+ // Match the refresh-specific ES 8.13.4 wire contract, whose REST handler
+ // uses `BaseBroadcastResponse::getStatus`: any failed shard makes the HTTP
+ // response use the first shard's status, even if another shard succeeded
+ // (`RestRefreshAction.java:47-52`, `BaseBroadcastResponse.java:114-122`;
+ // semantics only, no ES code). The body still reports every attempted
+ // shard (`RestActions.java:79-103`).
+ (response_status, Json(body)).into_response()
}
// ─────────────────────────────────────────────────────────────────────────────
diff --git a/engine/crates/xerj-api/tests/refresh_reports_flush_failure.rs b/engine/crates/xerj-api/tests/refresh_reports_flush_failure.rs
new file mode 100644
index 000000000..9c7506315
--- /dev/null
+++ b/engine/crates/xerj-api/tests/refresh_reports_flush_failure.rs
@@ -0,0 +1,162 @@
+//! A refresh is a synchronous visibility barrier: its HTTP response must not
+//! claim success when the engine could not publish the requested segment.
+
+use axum::body::Body;
+use axum::http::{Request, StatusCode};
+use serde_json::{json, Value};
+use tower::ServiceExt;
+
+async fn request(app: &axum::Router, method: &str, path: &str, body: Value) -> (StatusCode, Value) {
+ let response = app
+ .clone()
+ .oneshot(
+ Request::builder()
+ .method(method)
+ .uri(path)
+ .header("content-type", "application/json")
+ .body(Body::from(body.to_string()))
+ .expect("request"),
+ )
+ .await
+ .expect("response");
+ let status = response.status();
+ let bytes = axum::body::to_bytes(response.into_body(), usize::MAX)
+ .await
+ .expect("response body");
+ let body = serde_json::from_slice(&bytes).expect("JSON response");
+ (status, body)
+}
+
+fn app() -> (axum::Router, tempfile::TempDir) {
+ let dir = tempfile::tempdir().expect("data dir");
+ let mut config = xerj_common::config::Config::default();
+ config.server.data_dir = dir.path().to_string_lossy().into_owned();
+ config.storage.wal_sync = xerj_common::config::WalSync::Async;
+ let metrics = xerj_common::metrics::Metrics::new().expect("metrics");
+ let engine = xerj_engine::Engine::new(config.clone()).expect("engine");
+ let app = xerj_api::router::build_es_compat_router(xerj_api::state::AppState::new(
+ config, engine, metrics,
+ ));
+ (app, dir)
+}
+
+async fn create_text_index_with_document(app: &axum::Router, name: &str) {
+ let (status, body) = request(
+ app,
+ "PUT",
+ &format!("/{name}"),
+ json!({"mappings": {"properties": {"body": {"type": "text"}}}}),
+ )
+ .await;
+ assert_eq!(status, StatusCode::OK, "create {name}: {body}");
+
+ let (status, body) = request(
+ app,
+ "PUT",
+ &format!("/{name}/_doc/1"),
+ json!({"body": "must remain searchable after a successful refresh"}),
+ )
+ .await;
+ assert!(status.is_success(), "index document: {status} {body}");
+}
+
+fn block_segment_publication(dir: &std::path::Path, name: &str) {
+ let segments = dir.join(name).join("segments");
+ std::fs::remove_dir(&segments).expect("empty segments directory");
+ std::fs::write(&segments, b"not a directory").expect("segments blocker");
+}
+
+#[tokio::test]
+async fn refresh_reports_a_segment_publication_failure() {
+ let (app, dir) = app();
+ create_text_index_with_document(&app, "refresh-failure").await;
+
+ // Make segment publication fail independently of any particular encoder:
+ // replacing the still-empty segments directory with a regular file makes
+ // the real storage/FTS finalizer return an I/O error. This is deterministic
+ // even as individual side-car filename bugs are fixed.
+ block_segment_publication(dir.path(), "refresh-failure");
+
+ let (status, body) = request(&app, "POST", "/refresh-failure/_refresh", json!({})).await;
+ assert_eq!(
+ status,
+ StatusCode::INTERNAL_SERVER_ERROR,
+ "an all-shards refresh failure must not be HTTP 200: {body}"
+ );
+ assert_eq!(body.pointer("/_shards/total"), Some(&json!(1)), "{body}");
+ assert_eq!(
+ body.pointer("/_shards/successful"),
+ Some(&json!(0)),
+ "{body}"
+ );
+ assert_eq!(body.pointer("/_shards/failed"), Some(&json!(1)), "{body}");
+ assert_eq!(
+ body.pointer("/_shards/failures/0/index"),
+ Some(&json!("refresh-failure")),
+ "{body}"
+ );
+ assert_eq!(
+ body.pointer("/_shards/failures/0/status"),
+ Some(&json!("INTERNAL_SERVER_ERROR")),
+ "{body}"
+ );
+ let failure = body
+ .pointer("/_shards/failures/0")
+ .and_then(Value::as_object)
+ .expect("failure object");
+ let mut failure_fields: Vec<&str> = failure.keys().map(String::as_str).collect();
+ failure_fields.sort_unstable();
+ assert_eq!(
+ failure_fields,
+ ["index", "reason", "shard", "status"],
+ "ES 8.13.4 DefaultShardOperationFailedException shape: {body}"
+ );
+ assert_eq!(
+ body.pointer("/_shards/failures/0/reason/type"),
+ Some(&json!("store_exception")),
+ "{body}"
+ );
+ let reason = body
+ .pointer("/_shards/failures/0/reason/reason")
+ .and_then(Value::as_str)
+ .unwrap_or_default();
+ assert!(
+ !reason.is_empty() && reason != "refresh failed" && reason.contains("I/O error:"),
+ "segment publication must preserve the real storage I/O reason: {body}"
+ );
+}
+
+#[tokio::test]
+async fn multi_index_refresh_attempts_every_index_after_a_failure() {
+ let (app, dir) = app();
+ create_text_index_with_document(&app, "refresh-good").await;
+ create_text_index_with_document(&app, "refresh-bad").await;
+ block_segment_publication(dir.path(), "refresh-bad");
+
+ // The broken index is explicitly first: `successful == 1` proves the
+ // handler did not abort before attempting the second resolved index.
+ let (status, body) = request(
+ &app,
+ "POST",
+ "/refresh-bad,refresh-good/_refresh",
+ json!({}),
+ )
+ .await;
+ assert_eq!(
+ status,
+ StatusCode::INTERNAL_SERVER_ERROR,
+ "ES 8.13.4 refresh uses the first failed shard's status: {body}"
+ );
+ assert_eq!(body.pointer("/_shards/total"), Some(&json!(2)), "{body}");
+ assert_eq!(
+ body.pointer("/_shards/successful"),
+ Some(&json!(1)),
+ "{body}"
+ );
+ assert_eq!(body.pointer("/_shards/failed"), Some(&json!(1)), "{body}");
+ assert_eq!(
+ body.pointer("/_shards/failures/0/index"),
+ Some(&json!("refresh-bad")),
+ "{body}"
+ );
+}