Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

fix(search): apply post_filter to the hits of _search (#2404)

Ahmet Adaklı9cf3f29cb8a1
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 2427cf3df..1bd033cfe 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -79,6 +79,17 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
### Fixed
+- `POST /{index}/_search` applies `post_filter` to the hits, as
+ Elasticsearch does. It used to be accepted and ignored: every document
+ the query matched came back and `hits.total` counted them all.
+ Aggregations still see the unfiltered query, which is the point of the
+ parameter: a faceted UI keeps offering a facet's other values while one
+ is selected. A request that aggregates now runs as two searches, one for
+ the filtered hits and one for the aggregations. `post_filter` beside a
+ hybrid query is still a 400 (#943). `_msearch`, async search, scroll and
+ a `?q=` query still ignore `post_filter`
+ ([#2400](https://github.com/xerj-org/xerj/issues/2400)).
+
- `xerj mcp` no longer holds every message behind a slow tool call. Each
`tools/call` runs on its own task, so `ping`, `tools/list` and other
calls are answered while one call waits on the node. A ping sent 0.2 s
diff --git a/engine/crates/xerj-api/src/es_compat.rs b/engine/crates/xerj-api/src/es_compat.rs
index f0baf2c3a..5a10e6c51 100644
--- a/engine/crates/xerj-api/src/es_compat.rs
+++ b/engine/crates/xerj-api/src/es_compat.rs
@@ -13093,6 +13093,21 @@ async fn search_dispatch(
params: EsSearchQueryParams,
principal: crate::auth::Principal,
body: EsSearchJson,
+) -> Response {
+ // #2400: `post_filter` narrows the hits, never the aggregations.
+ if let Some(split) = split_post_filter(&params, &body) {
+ return search_with_post_filter(state, index, params, principal, split).await;
+ }
+ search_routed(state, index, params, principal, body).await
+}
+
+/// One search through the cluster fan-out when it applies, else on this node.
+async fn search_routed(
+ state: AppState,
+ index: String,
+ params: EsSearchQueryParams,
+ principal: crate::auth::Principal,
+ body: EsSearchJson,
) -> Response {
if let Some(response) =
crate::cluster_shard::fanout_search(&state, &index, &params, &principal, &body).await
@@ -13102,6 +13117,218 @@ async fn search_dispatch(
search_impl(state, index, params, principal, body).await
}
+/// The searches a request with `post_filter` becomes (#2400). Elasticsearch
+/// applies `post_filter` to the hits after the aggregations are computed, so
+/// a faceted UI can show a facet's other values while one is selected.
+///
+/// - `hits`: the query wrapped as `bool{must: [query], filter: [post_filter]}`,
+/// without aggregations. A filter clause does not score, so the hits, their
+/// scores and `hits.total` are what Elasticsearch returns.
+/// - `aggs`: when the request aggregates, the original request without
+/// `post_filter` and with `size: 0`, so the aggregations are exactly what
+/// they were before.
+struct PostFilterSplit {
+ hits: EsSearchBody,
+ aggs: Option<(EsSearchQueryParams, EsSearchBody)>,
+}
+
+/// `None` leaves the request as it is: no `post_filter`; a hybrid query, which
+/// `build_search_request` rejects beside `post_filter` (#943); or a `?q=`
+/// query, which replaces the body query this wraps.
+fn split_post_filter(params: &EsSearchQueryParams, body: &EsSearchJson) -> Option<PostFilterSplit> {
+ let original = body.0.as_ref()?;
+ let filter = original.post_filter.clone()?;
+ if params.q.is_some() {
+ return None;
+ }
+ let query = original
+ .query
+ .clone()
+ .unwrap_or_else(|| json!({ "match_all": {} }));
+ let hybrid = parse_request(&json!({ "query": query }))
+ .map(|req| req.query.contains_hybrid())
+ .unwrap_or(false);
+ if hybrid {
+ return None;
+ }
+
+ let mut hits = original.clone();
+ hits.post_filter = None;
+ hits.query = Some(json!({ "bool": { "must": [query], "filter": [filter] } }));
+ let had_aggs = hits.aggs.take().is_some();
+ let had_aggregations = hits.aggregations.take().is_some();
+
+ let aggs = (had_aggs || had_aggregations).then(|| {
+ let mut body = original.clone();
+ body.post_filter = None;
+ body.size = 0;
+ body.from = 0;
+ let mut params = params.clone();
+ // Query-string paging overrides the body, and a second scroll context
+ // would leak: this search only reads the aggregations.
+ params.size = Some("0".to_string());
+ params.from = None;
+ params.scroll = None;
+ (params, body)
+ });
+ Some(PostFilterSplit { hits, aggs })
+}
+
+/// Run the split searches and answer with the hits of one and the
+/// aggregations of the other. An error from either is returned as it is.
+async fn search_with_post_filter(
+ state: AppState,
+ index: String,
+ params: EsSearchQueryParams,
+ principal: crate::auth::Principal,
+ split: PostFilterSplit,
+) -> Response {
+ let hits_search = search_routed(
+ state.clone(),
+ index.clone(),
+ params,
+ principal.clone(),
+ EsSearchJson(Some(split.hits)),
+ );
+ let Some((aggs_params, aggs_body)) = split.aggs else {
+ return hits_search.await;
+ };
+ let aggs_search = search_routed(
+ state,
+ index,
+ aggs_params,
+ principal,
+ EsSearchJson(Some(aggs_body)),
+ );
+ let (hits_response, aggs_response) = tokio::join!(Box::pin(hits_search), Box::pin(aggs_search));
+ if !hits_response.status().is_success() {
+ return hits_response;
+ }
+ if !aggs_response.status().is_success() {
+ return aggs_response;
+ }
+
+ let (mut parts, hits_body) = hits_response.into_parts();
+ let read = |body: axum::body::Body| async move {
+ axum::body::to_bytes(body, usize::MAX)
+ .await
+ .ok()
+ .and_then(|bytes| serde_json::from_slice::<Value>(&bytes).ok())
+ };
+ let (Some(mut merged), Some(aggs_value)) =
+ (read(hits_body).await, read(aggs_response.into_body()).await)
+ else {
+ return ApiError::new(xerj_common::XerjError::internal(
+ "post_filter: could not read the search responses to merge them",
+ ))
+ .into_response();
+ };
+ if let (Some(obj), Some(aggregations)) =
+ (merged.as_object_mut(), aggs_value.get("aggregations"))
+ {
+ obj.insert("aggregations".to_string(), aggregations.clone());
+ }
+ parts.headers.remove(axum::http::header::CONTENT_LENGTH);
+ Response::from_parts(parts, axum::body::Body::from(merged.to_string()))
+}
+
+#[cfg(test)]
+mod post_filter_split_tests {
+ use super::*;
+
+ fn body(v: Value) -> EsSearchJson {
+ EsSearchJson(Some(serde_json::from_value(v).expect("search body")))
+ }
+
+ #[test]
+ fn no_post_filter_is_left_alone() {
+ let params = EsSearchQueryParams::default();
+ assert!(split_post_filter(&params, &body(json!({"query": {"match_all": {}}}))).is_none());
+ assert!(split_post_filter(&params, &EsSearchJson(None)).is_none());
+ }
+
+ #[test]
+ fn hits_search_wraps_the_query_and_drops_aggregations() {
+ let params = EsSearchQueryParams::default();
+ let split = split_post_filter(
+ &params,
+ &body(json!({
+ "query": {"match": {"body": "alpha"}},
+ "post_filter": {"term": {"provenance": "draft"}},
+ })),
+ )
+ .expect("split");
+ assert_eq!(
+ split.hits.query,
+ Some(json!({"bool": {
+ "must": [{"match": {"body": "alpha"}}],
+ "filter": [{"term": {"provenance": "draft"}}],
+ }}))
+ );
+ assert!(split.hits.post_filter.is_none());
+ assert!(split.aggs.is_none(), "no aggregations, no second search");
+ }
+
+ #[test]
+ fn a_missing_query_is_match_all() {
+ let split = split_post_filter(
+ &EsSearchQueryParams::default(),
+ &body(json!({"post_filter": {"term": {"provenance": "draft"}}})),
+ )
+ .expect("split");
+ assert_eq!(
+ split.hits.query.unwrap()["bool"]["must"][0],
+ json!({"match_all": {}})
+ );
+ }
+
+ #[test]
+ fn aggregations_come_from_the_unfiltered_request_with_no_hits_and_no_scroll() {
+ let params = EsSearchQueryParams {
+ size: Some("25".into()),
+ from: Some("5".into()),
+ scroll: Some("1m".into()),
+ ..Default::default()
+ };
+ let split = split_post_filter(
+ &params,
+ &body(json!({
+ "query": {"match_all": {}},
+ "size": 25,
+ "post_filter": {"term": {"provenance": "draft"}},
+ "aggs": {"p": {"terms": {"field": "provenance"}}},
+ })),
+ )
+ .expect("split");
+ assert!(split.hits.aggs.is_none() && split.hits.aggregations.is_none());
+ let (aggs_params, aggs_body) = split.aggs.expect("an aggregation search");
+ assert!(aggs_body.post_filter.is_none());
+ assert_eq!(aggs_body.query, Some(json!({"match_all": {}})));
+ assert_eq!((aggs_body.size, aggs_body.from), (0, 0));
+ assert!(aggs_body.aggs.is_some());
+ assert_eq!(aggs_params.size.as_deref(), Some("0"));
+ assert!(aggs_params.from.is_none() && aggs_params.scroll.is_none());
+ }
+
+ #[test]
+ fn hybrid_and_query_string_requests_are_not_split() {
+ let hybrid = body(json!({
+ "query": {"hybrid": {"queries": [
+ {"query": {"match": {"body": "alpha"}}},
+ {"query": {"match": {"body": "beta"}}},
+ ]}},
+ "post_filter": {"term": {"provenance": "draft"}},
+ }));
+ assert!(split_post_filter(&EsSearchQueryParams::default(), &hybrid).is_none());
+ let q_param = EsSearchQueryParams {
+ q: Some("body:alpha".into()),
+ ..Default::default()
+ };
+ let with_filter = body(json!({"post_filter": {"term": {"provenance": "draft"}}}));
+ assert!(split_post_filter(&q_param, &with_filter).is_none());
+ }
+}
+
/// One shard's search, used by the cluster fan-out. Does not fan out again.
///
/// `Box::pin` keeps this future pointer-sized. `search_impl`'s state is
diff --git a/engine/crates/xerj-api/tests/hybrid_bool_filter_applied.rs b/engine/crates/xerj-api/tests/hybrid_bool_filter_applied.rs
index faf04c9ff..ac1cd8e10 100644
--- a/engine/crates/xerj-api/tests/hybrid_bool_filter_applied.rs
+++ b/engine/crates/xerj-api/tests/hybrid_bool_filter_applied.rs
@@ -230,11 +230,11 @@ async fn post_filter_beside_hybrid_is_a_400_naming_the_spellings() {
);
}
-/// Guard against over-rejecting: `post_filter` beside a PLAIN query keeps its
-/// documented accepted-and-ignored behaviour (#204 tracks implementing it) —
-/// the 400 fires only when the query contains a hybrid.
+/// Guard against over-rejecting: `post_filter` beside a PLAIN query is a 200
+/// — the 400 fires only when the query contains a hybrid — and, since #2400,
+/// it is applied: only the two eml docs come back.
#[tokio::test]
-async fn post_filter_beside_a_plain_match_stays_accepted_and_ignored() {
+async fn post_filter_beside_a_plain_match_is_accepted_and_applied() {
let (app, _dir) = hyb943_app_with_docs().await;
let (status, body) = json_req(
@@ -251,7 +251,7 @@ async fn post_filter_beside_a_plain_match_stays_accepted_and_ignored() {
assert_eq!(
status,
StatusCode::OK,
- "post_filter beside a plain query must stay 200 (accepted-and-ignored): {body}"
+ "post_filter beside a plain query must stay 200: {body}"
);
let ids: Vec<String> = body
.pointer("/hits/hits")
@@ -264,9 +264,7 @@ async fn post_filter_beside_a_plain_match_stays_accepted_and_ignored() {
.unwrap_or_default();
assert_eq!(
ids.len(),
- 4,
- "the plain query must return its UNfiltered hit set (all four docs, not \
- just the two eml ones — post_filter is not applied to plain queries yet, \
- see #204): {ids:?}"
+ 2,
+ "post_filter must narrow the plain query's hits to the two eml docs (#2400): {ids:?}"
);
}
diff --git a/engine/tests/es-compat-yaml/yaml/search/780_post_filter.yml b/engine/tests/es-compat-yaml/yaml/search/780_post_filter.yml
new file mode 100644
index 000000000..ca0d4d169
--- /dev/null
+++ b/engine/tests/es-compat-yaml/yaml/search/780_post_filter.yml
@@ -0,0 +1,70 @@
+---
+setup:
+ - do:
+ indices.create:
+ index: post_filter_test
+ body:
+ settings:
+ number_of_shards: 1
+ mappings:
+ properties:
+ provenance:
+ type: keyword
+ body:
+ type: text
+ - do:
+ bulk:
+ refresh: true
+ body:
+ - '{"index": {"_index": "post_filter_test", "_id": "1"}}'
+ - '{"provenance": "draft", "body": "alpha beta"}'
+ - '{"index": {"_index": "post_filter_test", "_id": "2"}}'
+ - '{"provenance": "human", "body": "alpha"}'
+ - '{"index": {"_index": "post_filter_test", "_id": "3"}}'
+ - '{"provenance": "human", "body": "gamma"}'
+
+---
+"post_filter filters the hits and leaves aggregations unfiltered":
+ - do:
+ search:
+ rest_total_hits_as_int: true
+ index: post_filter_test
+ body:
+ size: 10
+ query: { match_all: {} }
+ post_filter: { term: { provenance: draft } }
+ aggs: { by_provenance: { terms: { field: provenance } } }
+
+ - match: { hits.total: 1 }
+ - length: { hits.hits: 1 }
+ - match: { hits.hits.0._id: "1" }
+ - length: { aggregations.by_provenance.buckets: 2 }
+ - match: { aggregations.by_provenance.buckets.0.key: human }
+ - match: { aggregations.by_provenance.buckets.0.doc_count: 2 }
+ - match: { aggregations.by_provenance.buckets.1.key: draft }
+ - match: { aggregations.by_provenance.buckets.1.doc_count: 1 }
+
+---
+"post_filter without a query":
+ - do:
+ search:
+ rest_total_hits_as_int: true
+ index: post_filter_test
+ body:
+ post_filter: { term: { provenance: human } }
+
+ - match: { hits.total: 2 }
+ - length: { hits.hits: 2 }
+
+---
+"post_filter applies to hits.total with size 0":
+ - do:
+ search:
+ rest_total_hits_as_int: true
+ index: post_filter_test
+ body:
+ size: 0
+ post_filter: { term: { provenance: draft } }
+
+ - match: { hits.total: 1 }
+ - length: { hits.hits: 0 }