AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
server: answer "starting" while indices open; name the fix in 415s (#2662)
xerj-orgb6941cf43140
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 8a229a30c..fb4ed655c 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -35,6 +35,20 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0### Fixed+- The node binds its ES-compat and REST listeners before it opens its+ indices, and on plain-HTTP ports answers every request with+ `503 node_starting_exception` (with `Retry-After: 1`) until the engine is+ up. `/health/live` answers 200 `{"status":"starting"}` and `/health/ready`+ answers 503, so a health check during a long startup no longer sees+ "connection refused" and takes the node for dead. TLS ports bind early and+ queue connections until the node is ready (#2575).++- A body sent without a JSON `Content-Type` (the `curl -d` default,+ `application/x-www-form-urlencoded`) is still refused with 415, as+ Elasticsearch 8 does, but the reason now names the fix:+ `-H 'Content-Type: application/json'`. The curl recipes that left the+ header out now send it (#2575).+- `xerj autoindex` now says why a source file over the 2 MiB code cap isskipped (#2483). Such a file (generated stubs, minified bundles) used to bereported as "no records extracted (code candidate family, 1 junk lines)",diff --git a/docs/examples/rust-ast-audit/QUERY_COOKBOOK.md b/docs/examples/rust-ast-audit/QUERY_COOKBOOK.mdindex 0d7f0b479..0316f850a 100644--- a/docs/examples/rust-ast-audit/QUERY_COOKBOOK.md+++ b/docs/examples/rust-ast-audit/QUERY_COOKBOOK.md@@ -16,14 +16,14 @@ that a file-scoped grep cannot.```bash# Every axum-handler-shaped function (takes Path/Query/Json/State/... + async).-curl -s "$X/rustfns/_count" -d '{"query":{"bool":{"filter":[+curl -s -H 'Content-Type: application/json' "$X/rustfns/_count" -d '{"query":{"bool":{"filter":[{"term":{"is_handler_shaped":true}},{"term":{"is_test":false}}]}}}'# → 298``````bash# The route table itself: method + path → handler.-curl -s "$X/rustroutes/_search" -d '{"query":{"wildcard":{"path":"*sql*"}},+curl -s -H 'Content-Type: application/json' "$X/rustroutes/_search" -d '{"query":{"wildcard":{"path":"*sql*"}},"_source":["method","path","handler","file","line"]}'# → POST /_sql → sql_query (crates/xerj-api/src/router.rs:637)```@@ -33,7 +33,7 @@ curl -s "$X/rustroutes/_search" -d '{"query":{"wildcard":{"path":"*sql*"}},```bash# Recursion with NO depth bound — the stack-overflow shape. (Direct self-calls;# the call-graph CYCLE version is find_recursion_cycles.py, below.)-curl -s "$X/rustfns/_count" -d '{"query":{"bool":{+curl -s -H 'Content-Type: application/json' "$X/rustfns/_count" -d '{"query":{"bool":{"filter":[{"term":{"calls_self":true}},{"term":{"is_test":false}}],"must_not":[{"term":{"has_depth_guard":true}}]}}}'# → 377 (broad; the cycle finder is the sharp tool)@@ -41,7 +41,7 @@ curl -s "$X/rustfns/_count" -d '{"query":{"bool":{```bash# Narrowing `as` casts on a codebase with a documented truncation-bug history.-curl -s "$X/rustfns/_count" -d '{"query":{"bool":{"filter":[+curl -s -H 'Content-Type: application/json' "$X/rustfns/_count" -d '{"query":{"bool":{"filter":[{"term":{"has_narrowing_cast":true}},{"term":{"is_test":false}}]}}}'# → 375```@@ -50,7 +50,7 @@ curl -s "$X/rustfns/_count" -d '{"query":{"bool":{"filter":[```bash# Allocation whose size is derived from a parameter (with_capacity/reserve/vec!/repeat).-curl -s "$X/rustfns/_count" -d '{"query":{"bool":{"filter":[+curl -s -H 'Content-Type: application/json' "$X/rustfns/_count" -d '{"query":{"bool":{"filter":[{"range":{"alloc_from_param_count":{"gte":1}}},{"term":{"is_test":false}}]}}}'# → 137```@@ -58,7 +58,7 @@ curl -s "$X/rustfns/_count" -d '{"query":{"bool":{"filter":[```bash# The cross-product blowup shape: an allocation argument that MULTIPLIES two values# (fields.len() * like.len() — the exact shape of PR #69's F6).-curl -s "$X/rustfns/_count" -d '{"query":{"term":{"alloc_product":true}}}'+curl -s -H 'Content-Type: application/json' "$X/rustfns/_count" -d '{"query":{"term":{"alloc_product":true}}}'```## Path traversal / filesystem escape@@ -66,7 +66,7 @@ curl -s "$X/rustfns/_count" -d '{"query":{"term":{"alloc_product":true}}}'```bash# `.join(param)` — a path built from an attacker-influenced component — with NO# lexical/canonical containment check in the same function.-curl -s "$X/rustfns/_count" -d '{"query":{"bool":{+curl -s -H 'Content-Type: application/json' "$X/rustfns/_count" -d '{"query":{"bool":{"filter":[{"term":{"path_join_from_param":true}},{"term":{"is_test":false}}],"must_not":[{"term":{"validators":"containment"}}]}}}'# → 17 (down from 56 with the blunt sink filter; the argument-provenance signal@@ -77,12 +77,12 @@ curl -s "$X/rustfns/_count" -d '{"query":{"bool":{```bash# Every function containing unsafe (block or unsafe fn), non-test.-curl -s "$X/rustfns/_count" -d '{"query":{"bool":{"filter":[+curl -s -H 'Content-Type: application/json' "$X/rustfns/_count" -d '{"query":{"bool":{"filter":[{"term":{"unsafe_any":true}},{"term":{"is_test":false}}]}}}'# → 19 (+2 test-only = 21 total; matches the extractor's independent count)# Group by the unsafe operation kind (ffi_call, uninit, from_raw, ptr_arith, …).-curl -s "$X/rustfns/_search" -d '{"size":0,"query":{"term":{"unsafe_any":true}},+curl -s -H 'Content-Type: application/json' "$X/rustfns/_search" -d '{"size":0,"query":{"term":{"unsafe_any":true}},"aggs":{"ops":{"terms":{"field":"unsafe_ops","size":20}}}}'```@@ -91,7 +91,7 @@ curl -s "$X/rustfns/_search" -d '{"size":0,"query":{"term":{"unsafe_any":true}},```bash# A lock guard bound to a name, with an `.await` later in the function body:# a lock potentially held across a suspension point.-curl -s "$X/rustfns/_count" -d '{"query":{"bool":{"filter":[+curl -s -H 'Content-Type: application/json' "$X/rustfns/_count" -d '{"query":{"bool":{"filter":[{"term":{"lock_across_await":true}},{"term":{"is_test":false}}]}}}'# → 15```@@ -99,7 +99,7 @@ curl -s "$X/rustfns/_count" -d '{"query":{"bool":{"filter":[## Deserialization surface```bash-curl -s "$X/rustfns/_count" -d '{"query":{"bool":{"filter":[+curl -s -H 'Content-Type: application/json' "$X/rustfns/_count" -d '{"query":{"bool":{"filter":[{"term":{"sinks":"deserialize"}},{"term":{"is_test":false}}]}}}'# → 147```diff --git a/docs/recipes/production-deployment.md b/docs/recipes/production-deployment.mdindex 9cb2cd65a..8b35143d1 100644--- a/docs/recipes/production-deployment.md+++ b/docs/recipes/production-deployment.md@@ -573,7 +573,7 @@ Elasticsearch**, use the scroll + bulk pattern below.**fails loud** instead of silently doing the wrong thing (verified):```bash-curl -k -H "Authorization: ApiKey $KEY" -X POST https://xerj:9200/_reindex -d '{+curl -k -H "Authorization: ApiKey $KEY" -H "Content-Type: application/json" -X POST https://xerj:9200/_reindex -d '{"source": { "remote": { "host": "http://old-es:9200" }, "index": "src" },"dest": { "index": "dst" } }'# HTTP 400diff --git a/engine/crates/xerj-api/src/extract.rs b/engine/crates/xerj-api/src/extract.rsindex 5039406aa..3a3521c9b 100644--- a/engine/crates/xerj-api/src/extract.rs+++ b/engine/crates/xerj-api/src/extract.rs@@ -89,7 +89,12 @@ impl IntoResponse for OptionalJsonRejection {),OptionalJsonRejection::UnsupportedMediaType(ct) => (StatusCode::UNSUPPORTED_MEDIA_TYPE,- format!("unsupported Content-Type: {ct}"),+ // Name the fix (#2575): `curl -d` sends form encoding by+ // default, and the bare media type does not say what to change.+ format!(+ "unsupported Content-Type: {ct}. Send the body as JSON with the header \+ `Content-Type: application/json` (curl: -H 'Content-Type: application/json')"+ ),),};let body = serde_json::json!({@@ -174,3 +179,32 @@ impl<T> std::ops::DerefMut for OptionalJson<T> {&mut self.0}}++#[cfg(test)]+mod tests {+ use super::*;++ /// #2575: the 415 names the header to send, not just the media type.+ #[tokio::test]+ async fn unsupported_media_type_names_the_json_header_fix() {+ let response =+ OptionalJsonRejection::UnsupportedMediaType("application/x-www-form-urlencoded".into())+ .into_response();+ assert_eq!(response.status(), StatusCode::UNSUPPORTED_MEDIA_TYPE);+ let bytes = axum::body::to_bytes(response.into_body(), usize::MAX)+ .await+ .unwrap();+ let body: serde_json::Value = serde_json::from_slice(&bytes).unwrap();+ let reason = body["error"]["reason"].as_str().unwrap();+ assert!(+ reason.starts_with("unsupported Content-Type: application/x-www-form-urlencoded"),+ "{reason}"+ );+ assert!(+ reason.contains("Content-Type: application/json"),+ "{reason}"+ );+ assert_eq!(body["error"]["type"], "parse_exception");+ assert_eq!(body["status"], 415);+ }+}diff --git a/engine/crates/xerj-server/src/main.rs b/engine/crates/xerj-server/src/main.rsindex b4cd7db90..f7451d440 100644--- a/engine/crates/xerj-server/src/main.rs+++ b/engine/crates/xerj-server/src/main.rs@@ -1623,6 +1623,59 @@ fn bind_listener(addr: SocketAddr, name: &str) -> Result<std::net::TcpListener>Ok(listener)}+#[cfg(test)]+mod starting_responder_tests {+ use super::*;+ use tokio::io::{AsyncReadExt, AsyncWriteExt};++ async fn get(addr: SocketAddr, path: &str) -> String {+ let mut stream = tokio::net::TcpStream::connect(addr).await.unwrap();+ stream+ .write_all(+ format!("GET {path} HTTP/1.1\r\nHost: x\r\nConnection: close\r\n\r\n").as_bytes(),+ )+ .await+ .unwrap();+ let mut out = String::new();+ stream.read_to_string(&mut out).await.unwrap();+ out+ }++ /// #2575: while the engine opens, the bound port answers "starting"+ /// instead of refusing; afterwards the real router owns the listener.+ #[tokio::test(flavor = "multi_thread", worker_threads = 2)]+ async fn the_port_answers_starting_then_hands_over_to_the_real_router() {+ let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();+ listener.set_nonblocking(true).unwrap();+ let addr = listener.local_addr().unwrap();++ let starting = StartingResponder::start(&[(&listener, "ES-compat")]);+ let search = get(addr, "/idx/_search").await;+ assert!(search.starts_with("HTTP/1.1 503"), "{search}");+ assert!(+ search.to_ascii_lowercase().contains("retry-after: 1"),+ "{search}"+ );+ assert!(search.contains("node_starting_exception"), "{search}");+ let ready = get(addr, "/health/ready").await;+ assert!(ready.starts_with("HTTP/1.1 503"), "{ready}");+ let live = get(addr, "/health/live").await;+ assert!(live.starts_with("HTTP/1.1 200"), "{live}");+ starting.stop().await;++ let real = Router::new().route("/", axum::routing::get(|| async { "real" }));+ let server = tokio::spawn(async move {+ axum::serve(TcpListener::from_std(listener).unwrap(), real)+ .await+ .unwrap();+ });+ let after = get(addr, "/").await;+ assert!(after.starts_with("HTTP/1.1 200"), "{after}");+ assert!(after.ends_with("real"), "{after}");+ server.abort();+ }+}+#[cfg(test)]mod bind_listener_tests {use super::{bind_error_message, bind_listener};@@ -1803,6 +1856,96 @@ fn bind_listener_candidate(/// How far past the requested port `--auto-port` scans.const AUTO_PORT_SCAN: u16 = 1000;+/// Answers on the bound ES-compat and REST listeners while `Engine::new`+/// opens the data dir (#2575). Every path returns 503 with an ES-shaped+/// `node_starting_exception` and `Retry-After: 1`, except `/health/live`,+/// which returns 200 because the process is alive. `/health/ready` gets the+/// 503 like everything else.+struct StartingResponder {+ stop: tokio::sync::watch::Sender<bool>,+ tasks: Vec<tokio::task::JoinHandle<()>>,+}++const STARTING_REASON: &str = "XERJ is starting: the node is opening the indices in its data \+ directory and will serve requests when that finishes. Retry shortly; GET /health/ready \+ returns 200 once the node is ready.";++fn starting_router() -> Router {+ use axum::http::{header, StatusCode};+ use axum::routing::get;+ Router::new()+ .route(+ "/health/live",+ get(|| async {+ (+ StatusCode::OK,+ axum::Json(serde_json::json!({"status": "starting"})),+ )+ }),+ )+ .fallback(|| async {+ (+ StatusCode::SERVICE_UNAVAILABLE,+ [(header::RETRY_AFTER, "1")],+ axum::Json(serde_json::json!({+ "error": {"type": "node_starting_exception", "reason": STARTING_REASON},+ "status": 503,+ })),+ )+ })+}++impl StartingResponder {+ /// Serve [`starting_router`] on a duplicate of each listener's socket.+ /// A listener that cannot be duplicated is skipped: its clients wait in+ /// the backlog, which is still better than a refusal.+ fn start(listeners: &[(&std::net::TcpListener, &'static str)]) -> Self {+ let (stop, rx) = tokio::sync::watch::channel(false);+ let mut tasks = Vec::new();+ for (listener, name) in listeners {+ let clone = match listener.try_clone() {+ Ok(clone) => clone,+ Err(error) => {+ warn!("{name}: no starting responder ({error}); clients wait until the engine is ready");+ continue;+ }+ };+ let mut rx = rx.clone();+ let name = *name;+ tasks.push(tokio::spawn(async move {+ let listener = match TcpListener::from_std(clone) {+ Ok(listener) => listener,+ Err(error) => {+ warn!("{name}: no starting responder ({error})");+ return;+ }+ };+ let serve =+ axum::serve(listener, starting_router()).with_graceful_shutdown(async move {+ let _ = rx.wait_for(|stopped| *stopped).await;+ });+ let _ = serve.await;+ }));+ }+ StartingResponder { stop, tasks }+ }++ /// Stop answering and release the duplicate sockets before the real+ /// routers take the listeners. In-flight "starting" answers get 2 s.+ async fn stop(self) {+ let _ = self.stop.send(true);+ for task in self.tasks {+ let abort = task.abort_handle();+ if tokio::time::timeout(std::time::Duration::from_secs(2), task)+ .await+ .is_err()+ {+ abort.abort();+ }+ }+ }+}+async fn serve_on(router: Router,listener: std::net::TcpListener,@@ -4447,8 +4590,109 @@ async fn async_main() -> Result<()> {// 7. Metricslet metrics = Metrics::new().context("initialise metrics")?;- // 8. Engine (opens existing indices from disk)- let engine = Engine::new(cfg.clone()).context("initialise engine")?;+ // 8c. Bind all three listeners NOW — before the engine opens its indices,+ // before the console setup link, before the banner, before anything+ // reports readiness, and fatally.+ //+ // Binding ahead of `Engine::new` (#2575): opening every index can take+ // minutes on a large data dir, and a probe in that window used to get+ // "connection refused", indistinguishable from a node that is not+ // running. With the ports held, the starting responder below answers+ // 503 `node_starting_exception` until the engine is ready.+ //+ // This used to happen at step 10/10a below, well after the console+ // bootstrap at (old) step 9 had already built and printed its+ // first-launch setup link from `cfg.server.es_compat_port` — the+ // *requested* port, not the one the kernel handed back. With an+ // ephemeral port (`es_compat_port = 0`) that produced+ // `http://127.0.0.1:0/_xerj-console/setup#token=…`, a link the single+ // action a first-run operator is told to take, and unreachable. It+ // also ran ahead of the bind, so it printed to stderr whether or not+ // the bind that follows ever succeeds (issue #469; #466 fixed the+ // same class for the startup banner, printed from `local_addr()`+ // right below).+ //+ // Moving the bind here — ahead of the console bootstrap rather than+ // just ahead of the banner — means both the setup link and the banner+ // can only ever be built from a port this process actually holds; see+ // `bind_listener` for why binding eagerly and fatally is itself+ // load-bearing (issue #465).+ //+ // Addresses are composed from the parsed IP, never by formatting+ // `"{bind}:{port}"` — that string is unparseable for every IPv6+ // literal (`"::1:9200"`), which made `bind_address = "::1"` a node+ // that could not start at all. `socket_addr` cannot be `None` here:+ // step 3a rejected an unparseable `bind_address` before the data+ // directory existed, which is where that failure belongs. The+ // context is kept as a backstop so that if a future edit reorders or+ // drops 3a, this still names the setting rather than surfacing a bare+ // `Option` unwrap.+ let ip = cfg.bind_ip().with_context(|| bad_bind_address(&cfg))?;++ let mut avoid: Vec<u16> = Vec::new();+ if cfg.cluster.enabled {+ avoid.push(cfg.cluster.port);+ }++ let (es_listener, bound_es_port) = bind_listener_candidate(+ ip,+ cfg.server.es_compat_port,+ "ES-compat",+ cfg.server.auto_port,+ &avoid,+ )?;+ avoid.push(bound_es_port);++ let (rest_listener, bound_rest_port) = bind_listener_candidate(+ ip,+ cfg.server.rest_port,+ "native REST",+ cfg.server.auto_port,+ &avoid,+ )?;+ avoid.push(bound_rest_port);++ let (grpc_listener, bound_grpc_port) = bind_listener_candidate(+ ip,+ cfg.server.grpc_port,+ "gRPC",+ cfg.server.auto_port,+ &avoid,+ )?;++ cfg.server.es_compat_port = bound_es_port;+ cfg.server.rest_port = bound_rest_port;+ cfg.server.grpc_port = bound_grpc_port;++ let bound = BoundAddrs {+ rest: rest_listener+ .local_addr()+ .context("native REST: read the bound address")?,+ es: es_listener+ .local_addr()+ .context("ES-compat: read the bound address")?,+ grpc: grpc_listener+ .local_addr()+ .context("gRPC: read the bound address")?,+ };++ // 8. Engine (opens existing indices from disk). While it opens, the+ // ES-compat and REST ports answer "starting" (#2575). Not on TLS+ // ports: a cleartext answer there would be a downgrade, so those+ // connections wait in the accept backlog instead of being refused.+ let starting = if tls_config.is_none() {+ Some(StartingResponder::start(&[+ (&es_listener, "ES-compat"),+ (&rest_listener, "native REST"),+ ]))+ } else {+ None+ };+ let engine = Engine::new(cfg.clone()).context("initialise engine");+ if let Some(starting) = starting {+ starting.stop().await;+ }+ let engine = engine?;let storage_available = engine.cluster_state_boot_status().is_writable();// This node's own cluster address, composed from the parsed bind IP.@@ -4603,85 +4847,6 @@ async fn async_main() -> Result<()> {None};- // 8c. Bind all three listeners NOW — before the console setup link, before- // the banner, before anything reports readiness, and fatally.- //- // This used to happen at step 10/10a below, well after the console- // bootstrap at (old) step 9 had already built and printed its- // first-launch setup link from `cfg.server.es_compat_port` — the- // *requested* port, not the one the kernel handed back. With an- // ephemeral port (`es_compat_port = 0`) that produced- // `http://127.0.0.1:0/_xerj-console/setup#token=…`, a link the single- // action a first-run operator is told to take, and unreachable. It- // also ran ahead of the bind, so it printed to stderr whether or not- // the bind that follows ever succeeds (issue #469; #466 fixed the- // same class for the startup banner, printed from `local_addr()`- // right below).- //- // Moving the bind here — ahead of the console bootstrap rather than- // just ahead of the banner — means both the setup link and the banner- // can only ever be built from a port this process actually holds; see- // `bind_listener` for why binding eagerly and fatally is itself- // load-bearing (issue #465).- //- // Addresses are composed from the parsed IP, never by formatting- // `"{bind}:{port}"` — that string is unparseable for every IPv6- // literal (`"::1:9200"`), which made `bind_address = "::1"` a node- // that could not start at all. `socket_addr` cannot be `None` here:- // step 3a rejected an unparseable `bind_address` before the data- // directory existed, which is where that failure belongs. The- // context is kept as a backstop so that if a future edit reorders or- // drops 3a, this still names the setting rather than surfacing a bare- // `Option` unwrap.- let ip = cfg.bind_ip().with_context(|| bad_bind_address(&cfg))?;-- let mut avoid: Vec<u16> = Vec::new();- if cfg.cluster.enabled {- avoid.push(cfg.cluster.port);- }-- let (es_listener, bound_es_port) = bind_listener_candidate(- ip,- cfg.server.es_compat_port,- "ES-compat",- cfg.server.auto_port,- &avoid,- )?;- avoid.push(bound_es_port);-- let (rest_listener, bound_rest_port) = bind_listener_candidate(- ip,- cfg.server.rest_port,- "native REST",- cfg.server.auto_port,- &avoid,- )?;- avoid.push(bound_rest_port);-- let (grpc_listener, bound_grpc_port) = bind_listener_candidate(- ip,- cfg.server.grpc_port,- "gRPC",- cfg.server.auto_port,- &avoid,- )?;-- cfg.server.es_compat_port = bound_es_port;- cfg.server.rest_port = bound_rest_port;- cfg.server.grpc_port = bound_grpc_port;-- let bound = BoundAddrs {- rest: rest_listener- .local_addr()- .context("native REST: read the bound address")?,- es: es_listener- .local_addr()- .context("ES-compat: read the bound address")?,- grpc: grpc_listener- .local_addr()- .context("gRPC: read the bound address")?,- };-// 9. Xerj Console bootstrap. Creates `.xerj_*` system indices on first// boot, persists a 32-byte master key under data_dir/.xerj_master_key// (mode 0600), and prints the first-launch magic-link banner to