AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
fix(search): apply post_filter to the hits of _search (#2404)
Ahmet Adaklı9cf3f29cb8a1
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 2427cf3df..1bd033cfe 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -79,6 +79,17 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0### Fixed+- `POST /{index}/_search` applies `post_filter` to the hits, as+ Elasticsearch does. It used to be accepted and ignored: every document+ the query matched came back and `hits.total` counted them all.+ Aggregations still see the unfiltered query, which is the point of the+ parameter: a faceted UI keeps offering a facet's other values while one+ is selected. A request that aggregates now runs as two searches, one for+ the filtered hits and one for the aggregations. `post_filter` beside a+ hybrid query is still a 400 (#943). `_msearch`, async search, scroll and+ a `?q=` query still ignore `post_filter`+ ([#2400](https://github.com/xerj-org/xerj/issues/2400)).+- `xerj mcp` no longer holds every message behind a slow tool call. Each`tools/call` runs on its own task, so `ping`, `tools/list` and othercalls are answered while one call waits on the node. A ping sent 0.2 sdiff --git a/engine/crates/xerj-api/src/es_compat.rs b/engine/crates/xerj-api/src/es_compat.rsindex f0baf2c3a..5a10e6c51 100644--- a/engine/crates/xerj-api/src/es_compat.rs+++ b/engine/crates/xerj-api/src/es_compat.rs@@ -13093,6 +13093,21 @@ async fn search_dispatch(params: EsSearchQueryParams,principal: crate::auth::Principal,body: EsSearchJson,+) -> Response {+ // #2400: `post_filter` narrows the hits, never the aggregations.+ if let Some(split) = split_post_filter(¶ms, &body) {+ return search_with_post_filter(state, index, params, principal, split).await;+ }+ search_routed(state, index, params, principal, body).await+}++/// One search through the cluster fan-out when it applies, else on this node.+async fn search_routed(+ state: AppState,+ index: String,+ params: EsSearchQueryParams,+ principal: crate::auth::Principal,+ body: EsSearchJson,) -> Response {if let Some(response) =crate::cluster_shard::fanout_search(&state, &index, ¶ms, &principal, &body).await@@ -13102,6 +13117,218 @@ async fn search_dispatch(search_impl(state, index, params, principal, body).await}+/// The searches a request with `post_filter` becomes (#2400). Elasticsearch+/// applies `post_filter` to the hits after the aggregations are computed, so+/// a faceted UI can show a facet's other values while one is selected.+///+/// - `hits`: the query wrapped as `bool{must: [query], filter: [post_filter]}`,+/// without aggregations. A filter clause does not score, so the hits, their+/// scores and `hits.total` are what Elasticsearch returns.+/// - `aggs`: when the request aggregates, the original request without+/// `post_filter` and with `size: 0`, so the aggregations are exactly what+/// they were before.+struct PostFilterSplit {+ hits: EsSearchBody,+ aggs: Option<(EsSearchQueryParams, EsSearchBody)>,+}++/// `None` leaves the request as it is: no `post_filter`; a hybrid query, which+/// `build_search_request` rejects beside `post_filter` (#943); or a `?q=`+/// query, which replaces the body query this wraps.+fn split_post_filter(params: &EsSearchQueryParams, body: &EsSearchJson) -> Option<PostFilterSplit> {+ let original = body.0.as_ref()?;+ let filter = original.post_filter.clone()?;+ if params.q.is_some() {+ return None;+ }+ let query = original+ .query+ .clone()+ .unwrap_or_else(|| json!({ "match_all": {} }));+ let hybrid = parse_request(&json!({ "query": query }))+ .map(|req| req.query.contains_hybrid())+ .unwrap_or(false);+ if hybrid {+ return None;+ }++ let mut hits = original.clone();+ hits.post_filter = None;+ hits.query = Some(json!({ "bool": { "must": [query], "filter": [filter] } }));+ let had_aggs = hits.aggs.take().is_some();+ let had_aggregations = hits.aggregations.take().is_some();++ let aggs = (had_aggs || had_aggregations).then(|| {+ let mut body = original.clone();+ body.post_filter = None;+ body.size = 0;+ body.from = 0;+ let mut params = params.clone();+ // Query-string paging overrides the body, and a second scroll context+ // would leak: this search only reads the aggregations.+ params.size = Some("0".to_string());+ params.from = None;+ params.scroll = None;+ (params, body)+ });+ Some(PostFilterSplit { hits, aggs })+}++/// Run the split searches and answer with the hits of one and the+/// aggregations of the other. An error from either is returned as it is.+async fn search_with_post_filter(+ state: AppState,+ index: String,+ params: EsSearchQueryParams,+ principal: crate::auth::Principal,+ split: PostFilterSplit,+) -> Response {+ let hits_search = search_routed(+ state.clone(),+ index.clone(),+ params,+ principal.clone(),+ EsSearchJson(Some(split.hits)),+ );+ let Some((aggs_params, aggs_body)) = split.aggs else {+ return hits_search.await;+ };+ let aggs_search = search_routed(+ state,+ index,+ aggs_params,+ principal,+ EsSearchJson(Some(aggs_body)),+ );+ let (hits_response, aggs_response) = tokio::join!(Box::pin(hits_search), Box::pin(aggs_search));+ if !hits_response.status().is_success() {+ return hits_response;+ }+ if !aggs_response.status().is_success() {+ return aggs_response;+ }++ let (mut parts, hits_body) = hits_response.into_parts();+ let read = |body: axum::body::Body| async move {+ axum::body::to_bytes(body, usize::MAX)+ .await+ .ok()+ .and_then(|bytes| serde_json::from_slice::<Value>(&bytes).ok())+ };+ let (Some(mut merged), Some(aggs_value)) =+ (read(hits_body).await, read(aggs_response.into_body()).await)+ else {+ return ApiError::new(xerj_common::XerjError::internal(+ "post_filter: could not read the search responses to merge them",+ ))+ .into_response();+ };+ if let (Some(obj), Some(aggregations)) =+ (merged.as_object_mut(), aggs_value.get("aggregations"))+ {+ obj.insert("aggregations".to_string(), aggregations.clone());+ }+ parts.headers.remove(axum::http::header::CONTENT_LENGTH);+ Response::from_parts(parts, axum::body::Body::from(merged.to_string()))+}++#[cfg(test)]+mod post_filter_split_tests {+ use super::*;++ fn body(v: Value) -> EsSearchJson {+ EsSearchJson(Some(serde_json::from_value(v).expect("search body")))+ }++ #[test]+ fn no_post_filter_is_left_alone() {+ let params = EsSearchQueryParams::default();+ assert!(split_post_filter(¶ms, &body(json!({"query": {"match_all": {}}}))).is_none());+ assert!(split_post_filter(¶ms, &EsSearchJson(None)).is_none());+ }++ #[test]+ fn hits_search_wraps_the_query_and_drops_aggregations() {+ let params = EsSearchQueryParams::default();+ let split = split_post_filter(+ ¶ms,+ &body(json!({+ "query": {"match": {"body": "alpha"}},+ "post_filter": {"term": {"provenance": "draft"}},+ })),+ )+ .expect("split");+ assert_eq!(+ split.hits.query,+ Some(json!({"bool": {+ "must": [{"match": {"body": "alpha"}}],+ "filter": [{"term": {"provenance": "draft"}}],+ }}))+ );+ assert!(split.hits.post_filter.is_none());+ assert!(split.aggs.is_none(), "no aggregations, no second search");+ }++ #[test]+ fn a_missing_query_is_match_all() {+ let split = split_post_filter(+ &EsSearchQueryParams::default(),+ &body(json!({"post_filter": {"term": {"provenance": "draft"}}})),+ )+ .expect("split");+ assert_eq!(+ split.hits.query.unwrap()["bool"]["must"][0],+ json!({"match_all": {}})+ );+ }++ #[test]+ fn aggregations_come_from_the_unfiltered_request_with_no_hits_and_no_scroll() {+ let params = EsSearchQueryParams {+ size: Some("25".into()),+ from: Some("5".into()),+ scroll: Some("1m".into()),+ ..Default::default()+ };+ let split = split_post_filter(+ ¶ms,+ &body(json!({+ "query": {"match_all": {}},+ "size": 25,+ "post_filter": {"term": {"provenance": "draft"}},+ "aggs": {"p": {"terms": {"field": "provenance"}}},+ })),+ )+ .expect("split");+ assert!(split.hits.aggs.is_none() && split.hits.aggregations.is_none());+ let (aggs_params, aggs_body) = split.aggs.expect("an aggregation search");+ assert!(aggs_body.post_filter.is_none());+ assert_eq!(aggs_body.query, Some(json!({"match_all": {}})));+ assert_eq!((aggs_body.size, aggs_body.from), (0, 0));+ assert!(aggs_body.aggs.is_some());+ assert_eq!(aggs_params.size.as_deref(), Some("0"));+ assert!(aggs_params.from.is_none() && aggs_params.scroll.is_none());+ }++ #[test]+ fn hybrid_and_query_string_requests_are_not_split() {+ let hybrid = body(json!({+ "query": {"hybrid": {"queries": [+ {"query": {"match": {"body": "alpha"}}},+ {"query": {"match": {"body": "beta"}}},+ ]}},+ "post_filter": {"term": {"provenance": "draft"}},+ }));+ assert!(split_post_filter(&EsSearchQueryParams::default(), &hybrid).is_none());+ let q_param = EsSearchQueryParams {+ q: Some("body:alpha".into()),+ ..Default::default()+ };+ let with_filter = body(json!({"post_filter": {"term": {"provenance": "draft"}}}));+ assert!(split_post_filter(&q_param, &with_filter).is_none());+ }+}+/// One shard's search, used by the cluster fan-out. Does not fan out again.////// `Box::pin` keeps this future pointer-sized. `search_impl`'s state isdiff --git a/engine/crates/xerj-api/tests/hybrid_bool_filter_applied.rs b/engine/crates/xerj-api/tests/hybrid_bool_filter_applied.rsindex faf04c9ff..ac1cd8e10 100644--- a/engine/crates/xerj-api/tests/hybrid_bool_filter_applied.rs+++ b/engine/crates/xerj-api/tests/hybrid_bool_filter_applied.rs@@ -230,11 +230,11 @@ async fn post_filter_beside_hybrid_is_a_400_naming_the_spellings() {);}-/// Guard against over-rejecting: `post_filter` beside a PLAIN query keeps its-/// documented accepted-and-ignored behaviour (#204 tracks implementing it) —-/// the 400 fires only when the query contains a hybrid.+/// Guard against over-rejecting: `post_filter` beside a PLAIN query is a 200+/// — the 400 fires only when the query contains a hybrid — and, since #2400,+/// it is applied: only the two eml docs come back.#[tokio::test]-async fn post_filter_beside_a_plain_match_stays_accepted_and_ignored() {+async fn post_filter_beside_a_plain_match_is_accepted_and_applied() {let (app, _dir) = hyb943_app_with_docs().await;let (status, body) = json_req(@@ -251,7 +251,7 @@ async fn post_filter_beside_a_plain_match_stays_accepted_and_ignored() {assert_eq!(status,StatusCode::OK,- "post_filter beside a plain query must stay 200 (accepted-and-ignored): {body}"+ "post_filter beside a plain query must stay 200: {body}");let ids: Vec<String> = body.pointer("/hits/hits")@@ -264,9 +264,7 @@ async fn post_filter_beside_a_plain_match_stays_accepted_and_ignored() {.unwrap_or_default();assert_eq!(ids.len(),- 4,- "the plain query must return its UNfiltered hit set (all four docs, not \- just the two eml ones — post_filter is not applied to plain queries yet, \- see #204): {ids:?}"+ 2,+ "post_filter must narrow the plain query's hits to the two eml docs (#2400): {ids:?}");}diff --git a/engine/tests/es-compat-yaml/yaml/search/780_post_filter.yml b/engine/tests/es-compat-yaml/yaml/search/780_post_filter.ymlnew file mode 100644index 000000000..ca0d4d169--- /dev/null+++ b/engine/tests/es-compat-yaml/yaml/search/780_post_filter.yml@@ -0,0 +1,70 @@+---+setup:+ - do:+ indices.create:+ index: post_filter_test+ body:+ settings:+ number_of_shards: 1+ mappings:+ properties:+ provenance:+ type: keyword+ body:+ type: text+ - do:+ bulk:+ refresh: true+ body:+ - '{"index": {"_index": "post_filter_test", "_id": "1"}}'+ - '{"provenance": "draft", "body": "alpha beta"}'+ - '{"index": {"_index": "post_filter_test", "_id": "2"}}'+ - '{"provenance": "human", "body": "alpha"}'+ - '{"index": {"_index": "post_filter_test", "_id": "3"}}'+ - '{"provenance": "human", "body": "gamma"}'++---+"post_filter filters the hits and leaves aggregations unfiltered":+ - do:+ search:+ rest_total_hits_as_int: true+ index: post_filter_test+ body:+ size: 10+ query: { match_all: {} }+ post_filter: { term: { provenance: draft } }+ aggs: { by_provenance: { terms: { field: provenance } } }++ - match: { hits.total: 1 }+ - length: { hits.hits: 1 }+ - match: { hits.hits.0._id: "1" }+ - length: { aggregations.by_provenance.buckets: 2 }+ - match: { aggregations.by_provenance.buckets.0.key: human }+ - match: { aggregations.by_provenance.buckets.0.doc_count: 2 }+ - match: { aggregations.by_provenance.buckets.1.key: draft }+ - match: { aggregations.by_provenance.buckets.1.doc_count: 1 }++---+"post_filter without a query":+ - do:+ search:+ rest_total_hits_as_int: true+ index: post_filter_test+ body:+ post_filter: { term: { provenance: human } }++ - match: { hits.total: 2 }+ - length: { hits.hits: 2 }++---+"post_filter applies to hits.total with size 0":+ - do:+ search:+ rest_total_hits_as_int: true+ index: post_filter_test+ body:+ size: 0+ post_filter: { term: { provenance: draft } }++ - match: { hits.total: 1 }+ - length: { hits.hits: 0 }