Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

server: answer "starting" while indices open; name the fix in 415s (#2662)

xerj-orgb6941cf43140
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 8a229a30c..fb4ed655c 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -35,6 +35,20 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
### Fixed
+- The node binds its ES-compat and REST listeners before it opens its
+ indices, and on plain-HTTP ports answers every request with
+ `503 node_starting_exception` (with `Retry-After: 1`) until the engine is
+ up. `/health/live` answers 200 `{"status":"starting"}` and `/health/ready`
+ answers 503, so a health check during a long startup no longer sees
+ "connection refused" and takes the node for dead. TLS ports bind early and
+ queue connections until the node is ready (#2575).
+
+- A body sent without a JSON `Content-Type` (the `curl -d` default,
+ `application/x-www-form-urlencoded`) is still refused with 415, as
+ Elasticsearch 8 does, but the reason now names the fix:
+ `-H 'Content-Type: application/json'`. The curl recipes that left the
+ header out now send it (#2575).
+
- `xerj autoindex` now says why a source file over the 2 MiB code cap is
skipped (#2483). Such a file (generated stubs, minified bundles) used to be
reported as "no records extracted (code candidate family, 1 junk lines)",
diff --git a/docs/examples/rust-ast-audit/QUERY_COOKBOOK.md b/docs/examples/rust-ast-audit/QUERY_COOKBOOK.md
index 0d7f0b479..0316f850a 100644
--- a/docs/examples/rust-ast-audit/QUERY_COOKBOOK.md
+++ b/docs/examples/rust-ast-audit/QUERY_COOKBOOK.md
@@ -16,14 +16,14 @@ that a file-scoped grep cannot.
```bash
# Every axum-handler-shaped function (takes Path/Query/Json/State/... + async).
-curl -s "$X/rustfns/_count" -d '{"query":{"bool":{"filter":[
+curl -s -H 'Content-Type: application/json' "$X/rustfns/_count" -d '{"query":{"bool":{"filter":[
{"term":{"is_handler_shaped":true}},{"term":{"is_test":false}}]}}}'
# → 298
```
```bash
# The route table itself: method + path → handler.
-curl -s "$X/rustroutes/_search" -d '{"query":{"wildcard":{"path":"*sql*"}},
+curl -s -H 'Content-Type: application/json' "$X/rustroutes/_search" -d '{"query":{"wildcard":{"path":"*sql*"}},
"_source":["method","path","handler","file","line"]}'
# → POST /_sql → sql_query (crates/xerj-api/src/router.rs:637)
```
@@ -33,7 +33,7 @@ curl -s "$X/rustroutes/_search" -d '{"query":{"wildcard":{"path":"*sql*"}},
```bash
# Recursion with NO depth bound — the stack-overflow shape. (Direct self-calls;
# the call-graph CYCLE version is find_recursion_cycles.py, below.)
-curl -s "$X/rustfns/_count" -d '{"query":{"bool":{
+curl -s -H 'Content-Type: application/json' "$X/rustfns/_count" -d '{"query":{"bool":{
"filter":[{"term":{"calls_self":true}},{"term":{"is_test":false}}],
"must_not":[{"term":{"has_depth_guard":true}}]}}}'
# → 377 (broad; the cycle finder is the sharp tool)
@@ -41,7 +41,7 @@ curl -s "$X/rustfns/_count" -d '{"query":{"bool":{
```bash
# Narrowing `as` casts on a codebase with a documented truncation-bug history.
-curl -s "$X/rustfns/_count" -d '{"query":{"bool":{"filter":[
+curl -s -H 'Content-Type: application/json' "$X/rustfns/_count" -d '{"query":{"bool":{"filter":[
{"term":{"has_narrowing_cast":true}},{"term":{"is_test":false}}]}}}'
# → 375
```
@@ -50,7 +50,7 @@ curl -s "$X/rustfns/_count" -d '{"query":{"bool":{"filter":[
```bash
# Allocation whose size is derived from a parameter (with_capacity/reserve/vec!/repeat).
-curl -s "$X/rustfns/_count" -d '{"query":{"bool":{"filter":[
+curl -s -H 'Content-Type: application/json' "$X/rustfns/_count" -d '{"query":{"bool":{"filter":[
{"range":{"alloc_from_param_count":{"gte":1}}},{"term":{"is_test":false}}]}}}'
# → 137
```
@@ -58,7 +58,7 @@ curl -s "$X/rustfns/_count" -d '{"query":{"bool":{"filter":[
```bash
# The cross-product blowup shape: an allocation argument that MULTIPLIES two values
# (fields.len() * like.len() — the exact shape of PR #69's F6).
-curl -s "$X/rustfns/_count" -d '{"query":{"term":{"alloc_product":true}}}'
+curl -s -H 'Content-Type: application/json' "$X/rustfns/_count" -d '{"query":{"term":{"alloc_product":true}}}'
```
## Path traversal / filesystem escape
@@ -66,7 +66,7 @@ curl -s "$X/rustfns/_count" -d '{"query":{"term":{"alloc_product":true}}}'
```bash
# `.join(param)` — a path built from an attacker-influenced component — with NO
# lexical/canonical containment check in the same function.
-curl -s "$X/rustfns/_count" -d '{"query":{"bool":{
+curl -s -H 'Content-Type: application/json' "$X/rustfns/_count" -d '{"query":{"bool":{
"filter":[{"term":{"path_join_from_param":true}},{"term":{"is_test":false}}],
"must_not":[{"term":{"validators":"containment"}}]}}}'
# → 17 (down from 56 with the blunt sink filter; the argument-provenance signal
@@ -77,12 +77,12 @@ curl -s "$X/rustfns/_count" -d '{"query":{"bool":{
```bash
# Every function containing unsafe (block or unsafe fn), non-test.
-curl -s "$X/rustfns/_count" -d '{"query":{"bool":{"filter":[
+curl -s -H 'Content-Type: application/json' "$X/rustfns/_count" -d '{"query":{"bool":{"filter":[
{"term":{"unsafe_any":true}},{"term":{"is_test":false}}]}}}'
# → 19 (+2 test-only = 21 total; matches the extractor's independent count)
# Group by the unsafe operation kind (ffi_call, uninit, from_raw, ptr_arith, …).
-curl -s "$X/rustfns/_search" -d '{"size":0,"query":{"term":{"unsafe_any":true}},
+curl -s -H 'Content-Type: application/json' "$X/rustfns/_search" -d '{"size":0,"query":{"term":{"unsafe_any":true}},
"aggs":{"ops":{"terms":{"field":"unsafe_ops","size":20}}}}'
```
@@ -91,7 +91,7 @@ curl -s "$X/rustfns/_search" -d '{"size":0,"query":{"term":{"unsafe_any":true}},
```bash
# A lock guard bound to a name, with an `.await` later in the function body:
# a lock potentially held across a suspension point.
-curl -s "$X/rustfns/_count" -d '{"query":{"bool":{"filter":[
+curl -s -H 'Content-Type: application/json' "$X/rustfns/_count" -d '{"query":{"bool":{"filter":[
{"term":{"lock_across_await":true}},{"term":{"is_test":false}}]}}}'
# → 15
```
@@ -99,7 +99,7 @@ curl -s "$X/rustfns/_count" -d '{"query":{"bool":{"filter":[
## Deserialization surface
```bash
-curl -s "$X/rustfns/_count" -d '{"query":{"bool":{"filter":[
+curl -s -H 'Content-Type: application/json' "$X/rustfns/_count" -d '{"query":{"bool":{"filter":[
{"term":{"sinks":"deserialize"}},{"term":{"is_test":false}}]}}}'
# → 147
```
diff --git a/docs/recipes/production-deployment.md b/docs/recipes/production-deployment.md
index 9cb2cd65a..8b35143d1 100644
--- a/docs/recipes/production-deployment.md
+++ b/docs/recipes/production-deployment.md
@@ -573,7 +573,7 @@ Elasticsearch**, use the scroll + bulk pattern below.
**fails loud** instead of silently doing the wrong thing (verified):
```bash
-curl -k -H "Authorization: ApiKey $KEY" -X POST https://xerj:9200/_reindex -d '{
+curl -k -H "Authorization: ApiKey $KEY" -H "Content-Type: application/json" -X POST https://xerj:9200/_reindex -d '{
"source": { "remote": { "host": "http://old-es:9200" }, "index": "src" },
"dest": { "index": "dst" } }'
# HTTP 400
diff --git a/engine/crates/xerj-api/src/extract.rs b/engine/crates/xerj-api/src/extract.rs
index 5039406aa..3a3521c9b 100644
--- a/engine/crates/xerj-api/src/extract.rs
+++ b/engine/crates/xerj-api/src/extract.rs
@@ -89,7 +89,12 @@ impl IntoResponse for OptionalJsonRejection {
),
OptionalJsonRejection::UnsupportedMediaType(ct) => (
StatusCode::UNSUPPORTED_MEDIA_TYPE,
- format!("unsupported Content-Type: {ct}"),
+ // Name the fix (#2575): `curl -d` sends form encoding by
+ // default, and the bare media type does not say what to change.
+ format!(
+ "unsupported Content-Type: {ct}. Send the body as JSON with the header \
+ `Content-Type: application/json` (curl: -H 'Content-Type: application/json')"
+ ),
),
};
let body = serde_json::json!({
@@ -174,3 +179,32 @@ impl<T> std::ops::DerefMut for OptionalJson<T> {
&mut self.0
}
}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ /// #2575: the 415 names the header to send, not just the media type.
+ #[tokio::test]
+ async fn unsupported_media_type_names_the_json_header_fix() {
+ let response =
+ OptionalJsonRejection::UnsupportedMediaType("application/x-www-form-urlencoded".into())
+ .into_response();
+ assert_eq!(response.status(), StatusCode::UNSUPPORTED_MEDIA_TYPE);
+ let bytes = axum::body::to_bytes(response.into_body(), usize::MAX)
+ .await
+ .unwrap();
+ let body: serde_json::Value = serde_json::from_slice(&bytes).unwrap();
+ let reason = body["error"]["reason"].as_str().unwrap();
+ assert!(
+ reason.starts_with("unsupported Content-Type: application/x-www-form-urlencoded"),
+ "{reason}"
+ );
+ assert!(
+ reason.contains("Content-Type: application/json"),
+ "{reason}"
+ );
+ assert_eq!(body["error"]["type"], "parse_exception");
+ assert_eq!(body["status"], 415);
+ }
+}
diff --git a/engine/crates/xerj-server/src/main.rs b/engine/crates/xerj-server/src/main.rs
index b4cd7db90..f7451d440 100644
--- a/engine/crates/xerj-server/src/main.rs
+++ b/engine/crates/xerj-server/src/main.rs
@@ -1623,6 +1623,59 @@ fn bind_listener(addr: SocketAddr, name: &str) -> Result<std::net::TcpListener>
Ok(listener)
}
+#[cfg(test)]
+mod starting_responder_tests {
+ use super::*;
+ use tokio::io::{AsyncReadExt, AsyncWriteExt};
+
+ async fn get(addr: SocketAddr, path: &str) -> String {
+ let mut stream = tokio::net::TcpStream::connect(addr).await.unwrap();
+ stream
+ .write_all(
+ format!("GET {path} HTTP/1.1\r\nHost: x\r\nConnection: close\r\n\r\n").as_bytes(),
+ )
+ .await
+ .unwrap();
+ let mut out = String::new();
+ stream.read_to_string(&mut out).await.unwrap();
+ out
+ }
+
+ /// #2575: while the engine opens, the bound port answers "starting"
+ /// instead of refusing; afterwards the real router owns the listener.
+ #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+ async fn the_port_answers_starting_then_hands_over_to_the_real_router() {
+ let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
+ listener.set_nonblocking(true).unwrap();
+ let addr = listener.local_addr().unwrap();
+
+ let starting = StartingResponder::start(&[(&listener, "ES-compat")]);
+ let search = get(addr, "/idx/_search").await;
+ assert!(search.starts_with("HTTP/1.1 503"), "{search}");
+ assert!(
+ search.to_ascii_lowercase().contains("retry-after: 1"),
+ "{search}"
+ );
+ assert!(search.contains("node_starting_exception"), "{search}");
+ let ready = get(addr, "/health/ready").await;
+ assert!(ready.starts_with("HTTP/1.1 503"), "{ready}");
+ let live = get(addr, "/health/live").await;
+ assert!(live.starts_with("HTTP/1.1 200"), "{live}");
+ starting.stop().await;
+
+ let real = Router::new().route("/", axum::routing::get(|| async { "real" }));
+ let server = tokio::spawn(async move {
+ axum::serve(TcpListener::from_std(listener).unwrap(), real)
+ .await
+ .unwrap();
+ });
+ let after = get(addr, "/").await;
+ assert!(after.starts_with("HTTP/1.1 200"), "{after}");
+ assert!(after.ends_with("real"), "{after}");
+ server.abort();
+ }
+}
+
#[cfg(test)]
mod bind_listener_tests {
use super::{bind_error_message, bind_listener};
@@ -1803,6 +1856,96 @@ fn bind_listener_candidate(
/// How far past the requested port `--auto-port` scans.
const AUTO_PORT_SCAN: u16 = 1000;
+/// Answers on the bound ES-compat and REST listeners while `Engine::new`
+/// opens the data dir (#2575). Every path returns 503 with an ES-shaped
+/// `node_starting_exception` and `Retry-After: 1`, except `/health/live`,
+/// which returns 200 because the process is alive. `/health/ready` gets the
+/// 503 like everything else.
+struct StartingResponder {
+ stop: tokio::sync::watch::Sender<bool>,
+ tasks: Vec<tokio::task::JoinHandle<()>>,
+}
+
+const STARTING_REASON: &str = "XERJ is starting: the node is opening the indices in its data \
+ directory and will serve requests when that finishes. Retry shortly; GET /health/ready \
+ returns 200 once the node is ready.";
+
+fn starting_router() -> Router {
+ use axum::http::{header, StatusCode};
+ use axum::routing::get;
+ Router::new()
+ .route(
+ "/health/live",
+ get(|| async {
+ (
+ StatusCode::OK,
+ axum::Json(serde_json::json!({"status": "starting"})),
+ )
+ }),
+ )
+ .fallback(|| async {
+ (
+ StatusCode::SERVICE_UNAVAILABLE,
+ [(header::RETRY_AFTER, "1")],
+ axum::Json(serde_json::json!({
+ "error": {"type": "node_starting_exception", "reason": STARTING_REASON},
+ "status": 503,
+ })),
+ )
+ })
+}
+
+impl StartingResponder {
+ /// Serve [`starting_router`] on a duplicate of each listener's socket.
+ /// A listener that cannot be duplicated is skipped: its clients wait in
+ /// the backlog, which is still better than a refusal.
+ fn start(listeners: &[(&std::net::TcpListener, &'static str)]) -> Self {
+ let (stop, rx) = tokio::sync::watch::channel(false);
+ let mut tasks = Vec::new();
+ for (listener, name) in listeners {
+ let clone = match listener.try_clone() {
+ Ok(clone) => clone,
+ Err(error) => {
+ warn!("{name}: no starting responder ({error}); clients wait until the engine is ready");
+ continue;
+ }
+ };
+ let mut rx = rx.clone();
+ let name = *name;
+ tasks.push(tokio::spawn(async move {
+ let listener = match TcpListener::from_std(clone) {
+ Ok(listener) => listener,
+ Err(error) => {
+ warn!("{name}: no starting responder ({error})");
+ return;
+ }
+ };
+ let serve =
+ axum::serve(listener, starting_router()).with_graceful_shutdown(async move {
+ let _ = rx.wait_for(|stopped| *stopped).await;
+ });
+ let _ = serve.await;
+ }));
+ }
+ StartingResponder { stop, tasks }
+ }
+
+ /// Stop answering and release the duplicate sockets before the real
+ /// routers take the listeners. In-flight "starting" answers get 2 s.
+ async fn stop(self) {
+ let _ = self.stop.send(true);
+ for task in self.tasks {
+ let abort = task.abort_handle();
+ if tokio::time::timeout(std::time::Duration::from_secs(2), task)
+ .await
+ .is_err()
+ {
+ abort.abort();
+ }
+ }
+ }
+}
+
async fn serve_on(
router: Router,
listener: std::net::TcpListener,
@@ -4447,8 +4590,109 @@ async fn async_main() -> Result<()> {
// 7. Metrics
let metrics = Metrics::new().context("initialise metrics")?;
- // 8. Engine (opens existing indices from disk)
- let engine = Engine::new(cfg.clone()).context("initialise engine")?;
+ // 8c. Bind all three listeners NOW — before the engine opens its indices,
+ // before the console setup link, before the banner, before anything
+ // reports readiness, and fatally.
+ //
+ // Binding ahead of `Engine::new` (#2575): opening every index can take
+ // minutes on a large data dir, and a probe in that window used to get
+ // "connection refused", indistinguishable from a node that is not
+ // running. With the ports held, the starting responder below answers
+ // 503 `node_starting_exception` until the engine is ready.
+ //
+ // This used to happen at step 10/10a below, well after the console
+ // bootstrap at (old) step 9 had already built and printed its
+ // first-launch setup link from `cfg.server.es_compat_port` — the
+ // *requested* port, not the one the kernel handed back. With an
+ // ephemeral port (`es_compat_port = 0`) that produced
+ // `http://127.0.0.1:0/_xerj-console/setup#token=…`, a link the single
+ // action a first-run operator is told to take, and unreachable. It
+ // also ran ahead of the bind, so it printed to stderr whether or not
+ // the bind that follows ever succeeds (issue #469; #466 fixed the
+ // same class for the startup banner, printed from `local_addr()`
+ // right below).
+ //
+ // Moving the bind here — ahead of the console bootstrap rather than
+ // just ahead of the banner — means both the setup link and the banner
+ // can only ever be built from a port this process actually holds; see
+ // `bind_listener` for why binding eagerly and fatally is itself
+ // load-bearing (issue #465).
+ //
+ // Addresses are composed from the parsed IP, never by formatting
+ // `"{bind}:{port}"` — that string is unparseable for every IPv6
+ // literal (`"::1:9200"`), which made `bind_address = "::1"` a node
+ // that could not start at all. `socket_addr` cannot be `None` here:
+ // step 3a rejected an unparseable `bind_address` before the data
+ // directory existed, which is where that failure belongs. The
+ // context is kept as a backstop so that if a future edit reorders or
+ // drops 3a, this still names the setting rather than surfacing a bare
+ // `Option` unwrap.
+ let ip = cfg.bind_ip().with_context(|| bad_bind_address(&cfg))?;
+
+ let mut avoid: Vec<u16> = Vec::new();
+ if cfg.cluster.enabled {
+ avoid.push(cfg.cluster.port);
+ }
+
+ let (es_listener, bound_es_port) = bind_listener_candidate(
+ ip,
+ cfg.server.es_compat_port,
+ "ES-compat",
+ cfg.server.auto_port,
+ &avoid,
+ )?;
+ avoid.push(bound_es_port);
+
+ let (rest_listener, bound_rest_port) = bind_listener_candidate(
+ ip,
+ cfg.server.rest_port,
+ "native REST",
+ cfg.server.auto_port,
+ &avoid,
+ )?;
+ avoid.push(bound_rest_port);
+
+ let (grpc_listener, bound_grpc_port) = bind_listener_candidate(
+ ip,
+ cfg.server.grpc_port,
+ "gRPC",
+ cfg.server.auto_port,
+ &avoid,
+ )?;
+
+ cfg.server.es_compat_port = bound_es_port;
+ cfg.server.rest_port = bound_rest_port;
+ cfg.server.grpc_port = bound_grpc_port;
+
+ let bound = BoundAddrs {
+ rest: rest_listener
+ .local_addr()
+ .context("native REST: read the bound address")?,
+ es: es_listener
+ .local_addr()
+ .context("ES-compat: read the bound address")?,
+ grpc: grpc_listener
+ .local_addr()
+ .context("gRPC: read the bound address")?,
+ };
+
+ // 8. Engine (opens existing indices from disk). While it opens, the
+ // ES-compat and REST ports answer "starting" (#2575). Not on TLS
+ // ports: a cleartext answer there would be a downgrade, so those
+ // connections wait in the accept backlog instead of being refused.
+ let starting = if tls_config.is_none() {
+ Some(StartingResponder::start(&[
+ (&es_listener, "ES-compat"),
+ (&rest_listener, "native REST"),
+ ]))
+ } else {
+ None
+ };
+ let engine = Engine::new(cfg.clone()).context("initialise engine");
+ if let Some(starting) = starting {
+ starting.stop().await;
+ }
+ let engine = engine?;
let storage_available = engine.cluster_state_boot_status().is_writable();
// This node's own cluster address, composed from the parsed bind IP.
@@ -4603,85 +4847,6 @@ async fn async_main() -> Result<()> {
None
};
- // 8c. Bind all three listeners NOW — before the console setup link, before
- // the banner, before anything reports readiness, and fatally.
- //
- // This used to happen at step 10/10a below, well after the console
- // bootstrap at (old) step 9 had already built and printed its
- // first-launch setup link from `cfg.server.es_compat_port` — the
- // *requested* port, not the one the kernel handed back. With an
- // ephemeral port (`es_compat_port = 0`) that produced
- // `http://127.0.0.1:0/_xerj-console/setup#token=…`, a link the single
- // action a first-run operator is told to take, and unreachable. It
- // also ran ahead of the bind, so it printed to stderr whether or not
- // the bind that follows ever succeeds (issue #469; #466 fixed the
- // same class for the startup banner, printed from `local_addr()`
- // right below).
- //
- // Moving the bind here — ahead of the console bootstrap rather than
- // just ahead of the banner — means both the setup link and the banner
- // can only ever be built from a port this process actually holds; see
- // `bind_listener` for why binding eagerly and fatally is itself
- // load-bearing (issue #465).
- //
- // Addresses are composed from the parsed IP, never by formatting
- // `"{bind}:{port}"` — that string is unparseable for every IPv6
- // literal (`"::1:9200"`), which made `bind_address = "::1"` a node
- // that could not start at all. `socket_addr` cannot be `None` here:
- // step 3a rejected an unparseable `bind_address` before the data
- // directory existed, which is where that failure belongs. The
- // context is kept as a backstop so that if a future edit reorders or
- // drops 3a, this still names the setting rather than surfacing a bare
- // `Option` unwrap.
- let ip = cfg.bind_ip().with_context(|| bad_bind_address(&cfg))?;
-
- let mut avoid: Vec<u16> = Vec::new();
- if cfg.cluster.enabled {
- avoid.push(cfg.cluster.port);
- }
-
- let (es_listener, bound_es_port) = bind_listener_candidate(
- ip,
- cfg.server.es_compat_port,
- "ES-compat",
- cfg.server.auto_port,
- &avoid,
- )?;
- avoid.push(bound_es_port);
-
- let (rest_listener, bound_rest_port) = bind_listener_candidate(
- ip,
- cfg.server.rest_port,
- "native REST",
- cfg.server.auto_port,
- &avoid,
- )?;
- avoid.push(bound_rest_port);
-
- let (grpc_listener, bound_grpc_port) = bind_listener_candidate(
- ip,
- cfg.server.grpc_port,
- "gRPC",
- cfg.server.auto_port,
- &avoid,
- )?;
-
- cfg.server.es_compat_port = bound_es_port;
- cfg.server.rest_port = bound_rest_port;
- cfg.server.grpc_port = bound_grpc_port;
-
- let bound = BoundAddrs {
- rest: rest_listener
- .local_addr()
- .context("native REST: read the bound address")?,
- es: es_listener
- .local_addr()
- .context("ES-compat: read the bound address")?,
- grpc: grpc_listener
- .local_addr()
- .context("gRPC: read the bound address")?,
- };
-
// 9. Xerj Console bootstrap. Creates `.xerj_*` system indices on first
// boot, persists a 32-byte master key under data_dir/.xerj_master_key
// (mode 0600), and prints the first-launch magic-link banner to