Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

fix(api): _count honours min_score on single and multi-index selectors (#2277)

ordinary being038924cdddbd
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 3f63e272a..4607a93e1 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -22,6 +22,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
### Fixed
+- `POST /{index}/_count` honours `min_score`. The handler rebuilds a `size: 0`
+ search body from the query alone, so the threshold used to be dropped: for
+ `{"query":{"match":…},"min_score":10}` `_search` returned 0 while `_count`
+ returned every match. Both the single-index and the multi-index/alias paths
+ now set it on the request, and a non-numeric value is the same 400
+ `parse_exception` `_search` gives.
+
- x86_64 release binaries no longer require AVX2. The build baseline is
`x86-64-v2` (SSE4.2). The HNSW eight-accumulator dot product — the kernel
`x86-64-v3` was actually vectorizing — compiles an AVX2+FMA variant and
diff --git a/engine/crates/xerj-api/src/es_compat.rs b/engine/crates/xerj-api/src/es_compat.rs
index 5fd14462a..2bba1e0e0 100644
--- a/engine/crates/xerj-api/src/es_compat.rs
+++ b/engine/crates/xerj-api/src/es_compat.rs
@@ -23113,6 +23113,32 @@ pub async fn count_docs(
Query(params): Query<CountParams>,
body: OptionalJson<Value>,
) -> impl IntoResponse {
+ // #193-style parity: ES's `_count` body accepts `min_score` and filters the
+ // counted set by it. The handler rebuilds a `size: 0` search body from the
+ // query alone, so this field used to be dropped and `_count` answered the
+ // unfiltered total while `_search` with the same body answered 0. A
+ // non-numeric value is the same 400 parse_exception `_search` gives.
+ let min_score = match body.as_ref().and_then(|b| b.get("min_score")) {
+ None | Some(Value::Null) => None,
+ Some(value) => match value.as_f64() {
+ Some(ms) => Some(ms),
+ None => {
+ return (
+ StatusCode::BAD_REQUEST,
+ Json(json!({
+ "error": {
+ "type": "parse_exception",
+ "reason": format!(
+ "malformed JSON in request body: invalid value for [min_score]: {value}"
+ ),
+ "status": 400,
+ }
+ })),
+ )
+ .into_response();
+ }
+ },
+ };
// Multi-index / all selector: sum counts from every participating index.
//
// An ALIAS belongs in this branch too. It used not to, so a multi-index
@@ -23182,6 +23208,7 @@ pub async fn count_docs(
}
}
let has_query = !alias_filters.is_empty()
+ || min_score.is_some()
|| body
.as_ref()
.and_then(|b| b.get("query"))
@@ -23235,13 +23262,19 @@ pub async fn count_docs(
let search_body = json!({ "query": query_val, "size": 0, "from": 0 });
// A body that does not parse is not per-index: it is the same
// 400 the single-index arm returns, for the same reason.
- let req = match xerj_query::parse_request(&search_body) {
+ let mut req = match xerj_query::parse_request(&search_body) {
Ok(r) => r,
Err(e) => {
let ze = xerj_common::XerjError::invalid_query(e.to_string());
return ApiError::new(ze).into_response();
}
};
+ // `parse_request` leaves `min_score` at None (the `_search`
+ // handler fills it from `EsSearchBody`), so the count path has
+ // to set it explicitly or the threshold is silently dropped.
+ if let Some(ms) = min_score {
+ req.min_score = Some(ms);
+ }
match idx.search(&req).await {
Ok(result) => {
// A script resource limit in the filter (terms_set's
@@ -23304,12 +23337,15 @@ pub async fn count_docs(
Err(e) => return ApiError::new(xerj_common::XerjError::from(e)).into_response(),
};
- // If no query or match_all, use the stored stats for speed.
- let has_query = body
- .as_ref()
- .and_then(|b| b.get("query"))
- .map(|q| !q.is_null())
- .unwrap_or(false);
+ // If no query or match_all, use the stored stats for speed. A body that
+ // carries `min_score` still has to run the query: match_all scores 1.0, so
+ // a threshold above it counts zero.
+ let has_query = min_score.is_some()
+ || body
+ .as_ref()
+ .and_then(|b| b.get("query"))
+ .map(|q| !q.is_null())
+ .unwrap_or(false);
let count = if has_query {
// Run a search with size=0 to get the total.
@@ -23324,15 +23360,25 @@ pub async fn count_docs(
resolve_terms_lookups(&mut query_val, &state).await;
let search_body = json!({ "query": query_val, "size": 0, "from": 0 });
match xerj_query::parse_request(&search_body) {
- Ok(req) => match idx.search(&req).await {
- Ok(result) => {
- if let Some(reason) = &result.script_failure {
- return script_limit_response(reason);
+ Ok(mut req) => {
+ // `parse_request` leaves `min_score` at None (the `_search`
+ // handler fills it from `EsSearchBody`), so the count path has
+ // to set it explicitly or the threshold is silently dropped.
+ if let Some(ms) = min_score {
+ req.min_score = Some(ms);
+ }
+ match idx.search(&req).await {
+ Ok(result) => {
+ if let Some(reason) = &result.script_failure {
+ return script_limit_response(reason);
+ }
+ result.total.value
+ }
+ Err(e) => {
+ return ApiError::new(xerj_common::XerjError::from(e)).into_response()
}
- result.total.value
}
- Err(e) => return ApiError::new(xerj_common::XerjError::from(e)).into_response(),
- },
+ }
Err(e) => {
let ze = xerj_common::XerjError::invalid_query(e.to_string());
return ApiError::new(ze).into_response();
diff --git a/engine/crates/xerj-api/tests/count_honours_min_score.rs b/engine/crates/xerj-api/tests/count_honours_min_score.rs
new file mode 100644
index 000000000..6af6244be
--- /dev/null
+++ b/engine/crates/xerj-api/tests/count_honours_min_score.rs
@@ -0,0 +1,159 @@
+//! `POST /{index}/_count` honours the `min_score` field of its body.
+//!
+//! It rebuilds a `size: 0` search body from `query` alone, so the field used to
+//! be dropped: `_search` with `min_score` returned 0 while `_count` with the
+//! same body returned every match — a wrong count, not a slower one. The
+//! single-index and multi-index/alias paths both carry it now, and a body with
+//! `min_score` but no `query` runs `match_all` through the scoring path because
+//! match_all scores 1.0 (a threshold above it counts zero).
+//!
+//! Elasticsearch is referenced for semantics only. It is AGPL-3.0/SSPL-1.0/
+//! Elastic-2.0 licensed and no code from it is reproduced here.
+
+use axum::body::Body;
+use axum::http::{Request, StatusCode};
+use serde_json::{json, Value};
+use tower::ServiceExt;
+
+async fn app() -> (axum::Router, tempfile::TempDir) {
+ let dir = tempfile::tempdir().expect("tempdir");
+ let mut config = xerj_common::config::Config::default();
+ config.server.data_dir = dir.path().to_string_lossy().into_owned();
+ config.storage.wal_sync = xerj_common::config::WalSync::Async;
+ let metrics = xerj_common::metrics::Metrics::new().expect("metrics");
+ let engine = xerj_engine::Engine::new(config.clone()).expect("engine");
+ let state = xerj_api::state::AppState::new(config, engine, metrics);
+ (xerj_api::router::build_es_compat_router(state), dir)
+}
+
+async fn json_req(
+ app: &axum::Router,
+ method: &str,
+ path: &str,
+ body: Value,
+) -> (StatusCode, Value) {
+ let response = app
+ .clone()
+ .oneshot(
+ Request::builder()
+ .method(method)
+ .uri(path)
+ .header("content-type", "application/json")
+ .body(Body::from(body.to_string()))
+ .expect("request"),
+ )
+ .await
+ .expect("response");
+ let status = response.status();
+ let bytes = axum::body::to_bytes(response.into_body(), usize::MAX)
+ .await
+ .expect("body");
+ (
+ status,
+ serde_json::from_slice(&bytes).unwrap_or(Value::Null),
+ )
+}
+
+async fn search_total(app: &axum::Router, index: &str, body: &Value) -> u64 {
+ let (status, resp) = json_req(app, "POST", &format!("/{index}/_search"), body.clone()).await;
+ assert_eq!(status, StatusCode::OK, "{resp}");
+ resp["hits"]["total"]["value"].as_u64().unwrap_or(0)
+}
+
+async fn count(app: &axum::Router, index: &str, body: &Value) -> u64 {
+ let (status, resp) = json_req(app, "POST", &format!("/{index}/_count"), body.clone()).await;
+ assert_eq!(status, StatusCode::OK, "{resp}");
+ resp["count"].as_u64().unwrap_or(0)
+}
+
+#[tokio::test]
+async fn count_honours_min_score_on_single_and_multi_index_selectors() {
+ let (app, _dir) = app().await;
+ for (index, docs) in [
+ (
+ "c1",
+ vec![
+ json!({"t": "alpha", "k": "a"}),
+ json!({"t": "alpha alpha alpha", "k": "b"}),
+ json!({"t": "beta", "k": "c"}),
+ ],
+ ),
+ ("c2", vec![json!({"t": "alpha", "k": "a"})]),
+ ] {
+ let (status, body) = json_req(
+ &app,
+ "PUT",
+ &format!("/{index}"),
+ json!({"mappings": {"properties": {"t": {"type": "text"}, "k": {"type": "keyword"}}}}),
+ )
+ .await;
+ assert_eq!(status, StatusCode::OK, "{body}");
+ for (i, doc) in docs.into_iter().enumerate() {
+ let (status, body) =
+ json_req(&app, "PUT", &format!("/{index}/_doc/{}", i + 1), doc).await;
+ assert!(status.is_success(), "{body}");
+ }
+ let (status, body) =
+ json_req(&app, "POST", &format!("/{index}/_refresh"), Value::Null).await;
+ assert_eq!(status, StatusCode::OK, "{body}");
+ }
+
+ // Single index: the same body must give the same answer from both
+ // endpoints, including the zero case that exposed the bug.
+ for query in [
+ json!({"query": {"match": {"t": "alpha"}}, "min_score": 10.0}),
+ json!({"query": {"match": {"t": "alpha"}}, "min_score": 0.1}),
+ // No query: match_all scores 1.0, so a threshold above it is zero and
+ // one below it keeps every document.
+ json!({"min_score": 2.0}),
+ json!({"min_score": 0.1}),
+ ] {
+ assert_eq!(
+ search_total(&app, "c1", &query).await,
+ count(&app, "c1", &query).await,
+ "single-index parity for {query}"
+ );
+ }
+ assert_eq!(
+ count(
+ &app,
+ "c1",
+ &json!({"query": {"match": {"t": "alpha"}}, "min_score": 10.0})
+ )
+ .await,
+ 0,
+ "min_score above every score must count zero"
+ );
+
+ // Comma selector: the multi-index path must carry the same threshold.
+ for query in [
+ json!({"query": {"match": {"t": "alpha"}}, "min_score": 10.0}),
+ json!({"query": {"match": {"t": "alpha"}}, "min_score": 0.1}),
+ json!({"min_score": 2.0}),
+ json!({"min_score": 0.1}),
+ ] {
+ assert_eq!(
+ search_total(&app, "c1,c2", &query).await,
+ count(&app, "c1,c2", &query).await,
+ "multi-index parity for {query}"
+ );
+ }
+ assert_eq!(
+ count(&app, "c1,c2", &json!({"min_score": 0.1})).await,
+ 4,
+ "c1 has 3 docs, c2 has 1, and match_all scores 1.0"
+ );
+
+ // A non-numeric min_score is the same 400 `_search` gives, not a silently
+ // ignored filter.
+ for endpoint in ["_search", "_count"] {
+ let (status, body) = json_req(
+ &app,
+ "POST",
+ &format!("/c1/{endpoint}"),
+ json!({"query": {"match_all": {}}, "min_score": "x"}),
+ )
+ .await;
+ assert_eq!(status, StatusCode::BAD_REQUEST, "{endpoint}: {body}");
+ }
+}