Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

feat(api): auto-sense OpenSearch vs Elasticsearch clients per request

Vincenzo Lombardo3db8b707aa4c
diff --git a/engine/crates/xerj-api/src/es_compat.rs b/engine/crates/xerj-api/src/es_compat.rs
index 0360d4511..33ba73a32 100644
--- a/engine/crates/xerj-api/src/es_compat.rs
+++ b/engine/crates/xerj-api/src/es_compat.rs
@@ -24,7 +24,7 @@ use crate::{
responses::{
BulkItemError, EsBulkItem, EsBulkItemAction, EsBulkItemResult, EsBulkResponse,
EsDeleteDocResponse, EsDeleteIndexResponse, EsDocResponse, EsGetResponse, EsHit, EsHits,
- EsHitsTotal, EsIndexResponse, EsInfoResponse, EsSearchResponse,
+ EsHitsTotal, EsIndexResponse, EsInfoResponse, EsSearchResponse, EsVersion,
},
state::AppState,
};
@@ -33,14 +33,87 @@ use crate::{
// GET / — cluster info
// ─────────────────────────────────────────────────────────────────────────────
-pub async fn es_info(State(state): State<AppState>) -> impl IntoResponse {
+pub async fn es_info(
+ State(state): State<AppState>,
+ headers: axum::http::HeaderMap,
+) -> impl IntoResponse {
// Stable real node identity (matches _cat/nodes and _cat/master) instead of
// a fresh random UUID per call.
let node_name = state.engine.node_id.as_str().to_string();
- let resp = EsInfoResponse::new(node_name, "xerj".to_string());
+ let mut resp = EsInfoResponse::new(node_name, "xerj".to_string());
+ resp.version = resolve_compat_version(&state, &headers);
Json(resp).into_response()
}
+/// Last-known-good OpenSearch version to advertise when a caller is (or is
+/// forced to be) treated as OpenSearch but neither an explicit
+/// Version to advertise when a caller is (or is forced to be) treated as
+/// OpenSearch and no explicit `--compat-version` override is set — mirrors
+/// the role `EsVersion::default()`'s hardcoded `"8.13.0"` already plays for
+/// the Elasticsearch path.
+///
+/// NOT derived from the calling client's own self-reported library version
+/// — that was the original plan (mirror whatever version the client says
+/// ITS OWN library is), but empirical testing against a real OpenSearch
+/// Dashboards 2.11.1 container disproved it: OSD's backend talks to the
+/// cluster via `opensearch-js`, and its own internal `opensearch-js`
+/// version (`1.1.0` at time of writing) has no relationship to a server
+/// version OSD would consider compatible — mirroring it verbatim made OSD
+/// log "This version of OpenSearch Dashboards (v2.11.1) is incompatible
+/// with the following OpenSearch nodes in your cluster: v1.1.0 ..." and
+/// refuse to start. A client library's own version number is simply not a
+/// reliable proxy for a compatible SERVER version. `"2.11.0"` (OSD's own
+/// version) verified empirically to pass OSD's compatibility gate.
+const FALLBACK_OPENSEARCH_VERSION: &str = "2.11.0";
+
+/// Resolve the `version` block (including the OpenSearch-only
+/// `distribution` field) to report to THIS caller, per request.
+///
+/// Two-tier priority, most explicit wins — see `CompatConfig`'s doc comment
+/// for the full rationale:
+/// 1. `state.config.compat.distribution`/`.version`, set via
+/// `--compat-distribution`/`--compat-version` (or the matching env
+/// vars) — always wins, no header inspection at all.
+/// 2. Unset: inspect the caller's `User-Agent` header. A recognized
+/// OpenSearch client (`opensearch-py`, `opensearch-js`, or anything
+/// else whose name contains "opensearch") gets `distribution:
+/// "opensearch"` and, absent a version override, `FALLBACK_OPENSEARCH_VERSION`
+/// (see its doc comment for why this isn't the client's own version). A
+/// recognized Elasticsearch client, or no recognizable client at all,
+/// is unchanged pre-existing behavior — plain Elasticsearch,
+/// `EsVersion::default()`.
+fn resolve_compat_version(state: &AppState, headers: &axum::http::HeaderMap) -> EsVersion {
+ let mut version = EsVersion::default();
+
+ let cfg = &state.config.compat;
+ let user_agent = headers
+ .get(axum::http::header::USER_AGENT)
+ .and_then(|v| v.to_str().ok());
+
+ let is_opensearch = if !cfg.distribution.is_empty() {
+ cfg.distribution == "opensearch"
+ } else {
+ user_agent
+ .map(|ua| ua.to_ascii_lowercase().contains("opensearch"))
+ .unwrap_or(false)
+ };
+
+ if is_opensearch {
+ version.distribution = Some("opensearch".to_string());
+ version.number = if !cfg.version.is_empty() {
+ cfg.version.clone()
+ } else {
+ FALLBACK_OPENSEARCH_VERSION.to_string()
+ };
+ } else if !cfg.version.is_empty() {
+ // Elasticsearch-shaped (no `distribution` field), but the operator
+ // still wants a specific version.number reported.
+ version.number = cfg.version.clone();
+ }
+
+ version
+}
+
// ─────────────────────────────────────────────────────────────────────────────
// GET /_cluster/health
// ─────────────────────────────────────────────────────────────────────────────
@@ -20314,7 +20387,20 @@ pub async fn cat_master(State(state): State<AppState>) -> impl IntoResponse {
// GET /_nodes/{node_id}/stats
// ─────────────────────────────────────────────────────────────────────────────
-pub async fn nodes_info(State(state): State<AppState>) -> impl IntoResponse {
+pub async fn nodes_info(
+ State(state): State<AppState>,
+ headers: axum::http::HeaderMap,
+) -> impl IntoResponse {
+ // OpenSearch Dashboards' own initial-handshake compatibility check
+ // hits THIS endpoint (GET /_nodes?filter_path=nodes.*.version,...), not
+ // GET / — found empirically: a real OSD 2.11.1 container pointed at
+ // xerj logged "This version of OpenSearch Dashboards (v2.11.1) is
+ // incompatible with the following OpenSearch nodes in your cluster:
+ // v8.13.0 ..." even though OSD's own request carried a recognizable
+ // `opensearch-js/...` User-Agent — this handler just wasn't reading it.
+ // Same resolution `es_info` (GET /) uses.
+ let compat_version = resolve_compat_version(&state, &headers);
+ let reported_version = compat_version.number.clone();
let (_idx_count, total_docs, store_bytes) = real_index_totals(&state).await;
let rss_bytes = read_rss_bytes().unwrap_or(0);
// Real host memory total from /proc/meminfo (falls back to an RSS-derived
@@ -20338,7 +20424,7 @@ pub async fn nodes_info(State(state): State<AppState>) -> impl IntoResponse {
"transport_address": "127.0.0.1:9300",
"host": "127.0.0.1",
"ip": "127.0.0.1",
- "version": "8.13.0",
+ "version": reported_version,
"build_flavor": "default",
"build_type": "tar",
"build_hash": "xerj",
diff --git a/engine/crates/xerj-api/src/responses.rs b/engine/crates/xerj-api/src/responses.rs
index 3130c3e75..02e167f70 100644
--- a/engine/crates/xerj-api/src/responses.rs
+++ b/engine/crates/xerj-api/src/responses.rs
@@ -527,6 +527,12 @@ pub struct EsVersion {
pub lucene_version: String,
pub minimum_wire_compatibility_version: String,
pub minimum_index_compatibility_version: String,
+ /// OpenSearch-only field — real Elasticsearch never sends this key at
+ /// all, so it's `None`/omitted for an Elasticsearch-shaped response,
+ /// `Some("opensearch")` when the caller is (or is forced to be) treated
+ /// as an OpenSearch client. See `es_compat::resolve_compat_version`.
+ #[serde(skip_serializing_if = "Option::is_none")]
+ pub distribution: Option<String>,
}
impl Default for EsVersion {
@@ -542,6 +548,7 @@ impl Default for EsVersion {
lucene_version: "9.10.0".to_string(),
minimum_wire_compatibility_version: "7.17.0".to_string(),
minimum_index_compatibility_version: "7.0.0".to_string(),
+ distribution: None,
}
}
}
diff --git a/engine/crates/xerj-common/src/config.rs b/engine/crates/xerj-common/src/config.rs
index 0c058e55c..d7ec0fea3 100644
--- a/engine/crates/xerj-common/src/config.rs
+++ b/engine/crates/xerj-common/src/config.rs
@@ -82,6 +82,8 @@ pub struct Config {
pub search_context: SearchContextConfig,
/// Structured logging + access log — 2 settings.
pub logging: LoggingConfig,
+ /// Elasticsearch/OpenSearch wire-compatibility identity — 2 settings.
+ pub compat: CompatConfig,
}
// Total: 5+3+2+3+10+5+3+1+6+2+4+3+4+3+2 = 56 fields (incl. cors: 2, auth: 3,
@@ -283,6 +285,43 @@ impl Default for ServerConfig {
// ─────────────────────────────────────────────────────────────────────────────
+/// Elasticsearch/OpenSearch wire-compatibility identity — controls what the
+/// `version`/`distribution` block in `GET /` (and any other endpoint that
+/// reports the same block) says about xerj to the calling client.
+///
+/// Three-tier resolution, most explicit wins:
+/// 1. `distribution`/`version` set here (via `--compat-distribution` /
+/// `XERJ_COMPAT_DISTRIBUTION`, `--compat-version` / `XERJ_COMPAT_VERSION`)
+/// — always wins, no per-request inspection at all. For a client that
+/// sends no `User-Agent`, or an unrecognized one, or an operator who just
+/// wants deterministic behavior without relying on request sniffing.
+/// 2. Left unset (default): xerj inspects the request's `User-Agent` header
+/// and answers per-request — an OpenSearch client and an Elasticsearch
+/// client hitting the SAME running instance each see the block shaped
+/// for their own ecosystem.
+/// 3. Neither of the above resolves anything (no override, no recognizable
+/// `User-Agent`): unchanged pre-existing behavior — plain Elasticsearch.
+///
+/// **2 settings.**
+#[derive(Debug, Clone, Default, Serialize, Deserialize)]
+#[serde(default, deny_unknown_fields)]
+pub struct CompatConfig {
+ /// Force the reported client distribution: `"elasticsearch"` or
+ /// `"opensearch"`. Empty (default) — resolve per-request from
+ /// `User-Agent` instead (falls back to `elasticsearch`-shaped output,
+ /// i.e. no `distribution` field, when nothing is detected either way).
+ pub distribution: String,
+ /// Force the reported `version.number`. Empty (default) — when the
+ /// caller is auto-detected as OpenSearch (see `distribution` above),
+ /// reports a fixed, empirically-verified compatible OpenSearch version
+ /// instead (NOT the client's own self-reported library version — tried
+ /// that first, a real OpenSearch Dashboards container rejected it, see
+ /// `es_compat::FALLBACK_OPENSEARCH_VERSION`'s doc comment for why).
+ pub version: String,
+}
+
+// ─────────────────────────────────────────────────────────────────────────────
+
/// Authentication settings.
///
/// **3 settings.**
diff --git a/engine/crates/xerj-server/src/main.rs b/engine/crates/xerj-server/src/main.rs
index d68995fd8..503f4e9ee 100644
--- a/engine/crates/xerj-server/src/main.rs
+++ b/engine/crates/xerj-server/src/main.rs
@@ -80,6 +80,8 @@ struct CliArgs {
embed_mode: Option<String>,
onnx_model: Option<String>,
onnx_tokenizer: Option<String>,
+ compat_distribution: Option<String>,
+ compat_version: Option<String>,
}
fn parse_args() -> CliArgs {
@@ -90,6 +92,8 @@ fn parse_args() -> CliArgs {
let mut embed_mode = None;
let mut onnx_model = None;
let mut onnx_tokenizer = None;
+ let mut compat_distribution = None;
+ let mut compat_version = None;
while let Some(arg) = args.next() {
match arg.as_str() {
@@ -107,6 +111,8 @@ fn parse_args() -> CliArgs {
}
"--onnx-model" => onnx_model = args.next(),
"--onnx-tokenizer" => onnx_tokenizer = args.next(),
+ "--compat-distribution" => compat_distribution = args.next(),
+ "--compat-version" => compat_version = args.next(),
"--help" | "-h" => {
print_help();
std::process::exit(0);
@@ -129,6 +135,8 @@ fn parse_args() -> CliArgs {
embed_mode,
onnx_model,
onnx_tokenizer,
+ compat_distribution,
+ compat_version,
}
}
@@ -157,6 +165,24 @@ fn print_help() {
no auto-download\n\
--onnx-model <PATH> compatible FP32 MiniLM ONNX model\n\
--onnx-tokenizer <PATH> tokenizer.json from the same model/export\n\
+ --compat-distribution <DIST> Force the wire-compat client identity: elasticsearch |\n\
+ opensearch. Default: unset — xerj auto-detects per request\n\
+ from the caller's User-Agent header instead (an\n\
+ opensearch-py/opensearch-js client — including OpenSearch\n\
+ Dashboards itself — and an elasticsearch-py/elasticsearch-js\n\
+ client hitting the SAME running instance each see GET / and\n\
+ GET /_nodes shaped for their own ecosystem). Setting this\n\
+ pins ALL clients to one identity regardless of User-Agent —\n\
+ useful if a client sends no recognizable User-Agent, or you\n\
+ just want deterministic behavior.\n\
+ Example: --compat-distribution opensearch\n\
+ --compat-version <VER> Force the reported version.number (e.g. 2.11.0). Default:\n\
+ unset — when auto-detected as OpenSearch, reports a fixed\n\
+ version verified to pass OpenSearch Dashboards' own\n\
+ compatibility check (not the calling client's own version —\n\
+ client library versions aren't a reliable stand-in for a\n\
+ compatible server version, confirmed against a real OSD\n\
+ container). Example: --compat-version 2.11.0\n\
--help, -h Show this help\n\
--version, -V Print version and exit\n\
\n\
@@ -170,7 +196,29 @@ fn print_help() {
XERJ_CONFIG Config file path\n\
XERJ_EMBED_MODE Embedding backend (lexical|neural|proxy|auto|onnx-experimental)\n\
XERJ_ONNX_MODEL FP32 ONNX model path\n\
- XERJ_ONNX_TOKENIZER matching tokenizer.json path\n",
+ XERJ_ONNX_TOKENIZER matching tokenizer.json path\n\
+ XERJ_COMPAT_DISTRIBUTION elasticsearch|opensearch — same as --compat-distribution\n\
+ XERJ_COMPAT_VERSION same as --compat-version\n\
+ \n\
+ WIRE-COMPAT IDENTITY — three ways to run this, pick what fits:\n\
+ \n\
+ 1. Fully automatic (default, no flags):\n\
+ xerj -d ./data\n\
+ Every client gets a response shaped for its own ecosystem, detected from its\n\
+ User-Agent on every request. One instance serves ES and OpenSearch clients at\n\
+ once.\n\
+ \n\
+ 2. Force one identity for every client, every request:\n\
+ xerj -d ./data --compat-distribution opensearch --compat-version 2.11.0\n\
+ Or via env (handy in docker-compose):\n\
+ XERJ_COMPAT_DISTRIBUTION=opensearch XERJ_COMPAT_VERSION=2.11.0 xerj -d ./data\n\
+ Ignores User-Agent entirely — useful for a client that doesn't send one, or when\n\
+ you want fully deterministic behavior.\n\
+ \n\
+ 3. Pin only the distribution, let the version auto-detect per client:\n\
+ xerj -d ./data --compat-distribution opensearch\n\
+ (or set only XERJ_COMPAT_VERSION and leave distribution to auto-detect — the two\n\
+ settings are resolved independently, mix and match as needed.)\n",
env!("CARGO_PKG_VERSION")
);
}
@@ -339,6 +387,38 @@ fn load_config(args: &CliArgs) -> Result<Config> {
}
}
+ // ES/OpenSearch wire-compatibility override: `--compat-distribution` /
+ // `XERJ_COMPAT_DISTRIBUTION` (flag wins). Left unset, xerj auto-detects
+ // per request from the caller's User-Agent instead — this is only for
+ // pinning deterministic behavior (or covering a client whose
+ // User-Agent isn't recognized).
+ if let Some(dist) = args
+ .compat_distribution
+ .clone()
+ .or_else(|| std::env::var("XERJ_COMPAT_DISTRIBUTION").ok())
+ {
+ let dist = dist.trim().to_ascii_lowercase();
+ match dist.as_str() {
+ "elasticsearch" | "opensearch" => {
+ info!("compat.distribution = {dist} (forced via CLI/env)");
+ cfg.compat.distribution = dist;
+ }
+ other => {
+ anyhow::bail!(
+ "unknown --compat-distribution '{other}'; use elasticsearch|opensearch"
+ );
+ }
+ }
+ }
+ if let Some(version) = args
+ .compat_version
+ .clone()
+ .or_else(|| std::env::var("XERJ_COMPAT_VERSION").ok())
+ {
+ info!("compat.version = {version} (forced via CLI/env)");
+ cfg.compat.version = version;
+ }
+
Ok(cfg)
}
@@ -813,6 +893,8 @@ async fn run_cli_index(cmd: IndexCmdArgs) -> Result<()> {
embed_mode: None,
onnx_model: None,
onnx_tokenizer: None,
+ compat_distribution: None,
+ compat_version: None,
};
let mut cfg = load_config(&fake_cli)?;
// Tracing after config so the [logging] format applies (RC4-W4 item 6).