AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
feat(query): implement percolate query (reverse search)
xerj-orgbd7513d09bc3
diff --git a/engine/crates/xerj-engine/src/index.rs b/engine/crates/xerj-engine/src/index.rsindex 0ef4a7b7c..7d7723669 100644--- a/engine/crates/xerj-engine/src/index.rs+++ b/engine/crates/xerj-engine/src/index.rs@@ -12110,6 +12110,7 @@ fn is_doc_scan_query(q: &QueryNode) -> bool {| QueryNode::Constant { .. }| QueryNode::Boosted { .. }| QueryNode::Intervals { .. }+ | QueryNode::Percolate { .. })}@@ -13345,6 +13346,22 @@ fn doc_matches_query(q: &QueryNode, source: &Value) -> bool {!intervals.is_empty()}+ // Percolate (reverse search): this stored doc's `field` holds a+ // serialized ES query. Parse it and report a match iff that stored+ // query matches at least one of the supplied inline documents.+ // A missing field or an unparseable stored query => no match.+ QueryNode::Percolate { field, documents } => {+ let stored = match get_field_value(source, field) {+ Some(v) if v.is_object() => v,+ _ => return false,+ };+ let stored_q = match xerj_query::parse_query(&stored) {+ Ok(q) => q,+ Err(_) => return false,+ };+ documents.iter().any(|d| doc_matches_query(&stored_q, d))+ }+_ => false,}}@@ -16482,3 +16499,85 @@ mod fts_projection_tests {assert_eq!(out, vec!["model".to_string(), "status".to_string()]);}}++#[cfg(test)]+mod percolate_tests {+ use super::*;+ use serde_json::json;+ use xerj_query::ast::QueryNode;++ /// A stored percolator doc holds a serialized query in `query`; the+ /// percolate query matches that stored doc iff the stored query matches+ /// one of the supplied inline documents.+ #[test]+ fn percolate_term_match_and_miss() {+ let stored = json!({ "query": { "term": { "message": "bonsai" } } });++ let q_hit = QueryNode::Percolate {+ field: "query".into(),+ documents: vec![json!({ "message": "bonsai" })],+ };+ assert!(+ doc_matches_query(&q_hit, &stored),+ "term matches inline doc"+ );++ let q_miss = QueryNode::Percolate {+ field: "query".into(),+ documents: vec![json!({ "message": "cactus" })],+ };+ assert!(+ !doc_matches_query(&q_miss, &stored),+ "term does not match a different value"+ );+ }++ #[test]+ fn percolate_range_and_bool() {+ let stored_range = json!({ "query": { "range": { "price": { "gte": 10, "lte": 20 } } } });+ let q = QueryNode::Percolate {+ field: "query".into(),+ documents: vec![json!({ "price": 15 })],+ };+ assert!(doc_matches_query(&q, &stored_range));+ let q_out = QueryNode::Percolate {+ field: "query".into(),+ documents: vec![json!({ "price": 25 })],+ };+ assert!(!doc_matches_query(&q_out, &stored_range));++ let stored_bool = json!({+ "query": {+ "bool": {+ "must": [+ { "term": { "message": "foo" } },+ { "range": { "price": { "gte": 5 } } }+ ]+ }+ }+ });+ let q_bool = QueryNode::Percolate {+ field: "query".into(),+ documents: vec![json!({ "message": "foo", "price": 9 })],+ };+ assert!(doc_matches_query(&q_bool, &stored_bool));+ let q_bool_miss = QueryNode::Percolate {+ field: "query".into(),+ documents: vec![json!({ "message": "foo", "price": 1 })],+ };+ assert!(!doc_matches_query(&q_bool_miss, &stored_bool));+ }++ #[test]+ fn percolate_missing_or_unparseable_field_is_no_match() {+ let stored = json!({ "message": "no query here" });+ let q = QueryNode::Percolate {+ field: "query".into(),+ documents: vec![json!({ "message": "anything" })],+ };+ assert!(!doc_matches_query(&q, &stored));++ let stored2 = json!({ "query": "not-an-object" });+ assert!(!doc_matches_query(&q, &stored2));+ }+}diff --git a/engine/crates/xerj-query/src/ast.rs b/engine/crates/xerj-query/src/ast.rsindex 70605f68d..670a8dd64 100644--- a/engine/crates/xerj-query/src/ast.rs+++ b/engine/crates/xerj-query/src/ast.rs@@ -521,6 +521,17 @@ pub enum QueryNode {max_query_terms: u32,},+ /// Percolate query — reverse search. Each document stored in the index+ /// holds a serialized query in its `field` (a `percolator`-typed field);+ /// this query matches those stored documents whose stored query matches+ /// one of the supplied inline `documents`.+ Percolate {+ /// Name of the percolator field holding the stored query object.+ field: String,+ /// One or more inline documents to test the stored queries against.+ documents: Vec<serde_json::Value>,+ },+/// Pinned query — ensures specific document IDs appear first in results.Pinned {/// IDs to pin to the top of results.diff --git a/engine/crates/xerj-query/src/parser.rs b/engine/crates/xerj-query/src/parser.rsindex 5c0645bb9..3e15b4bdb 100644--- a/engine/crates/xerj-query/src/parser.rs+++ b/engine/crates/xerj-query/src/parser.rs@@ -307,7 +307,7 @@ pub fn parse_query(json: &Value) -> Result<QueryNode> {"has_child" => parse_has_child(params),"has_parent" => parse_has_parent(params),"more_like_this" => parse_more_like_this(params),- "percolate" => Ok(QueryNode::MatchNone), // percolate not supported+ "percolate" => parse_percolate(params),"pinned" => parse_pinned(params),// ── Span queries ───────────────────────────────────────────────────────"span_term" => parse_span_term(params),@@ -2824,6 +2824,49 @@ fn parse_more_like_this(params: &Value) -> Result<QueryNode> {/// }/// }/// ```+/// Parse a `percolate` query.+///+/// Only the inline-document form is supported:+/// ```json+/// { "field": "query", "document": { … } }+/// { "field": "query", "documents": [ { … }, … ] }+/// ```+/// The index/id fetch form (`{ "field": "query", "index": "...", "id": "..." }`)+/// is rejected with a 400 rather than silently returning nothing.+fn parse_percolate(params: &Value) -> Result<QueryNode> {+ let obj = params+ .as_object()+ .ok_or_else(|| qerr("`percolate` must be an object"))?;++ let field = string_field(obj, "field")?;++ let documents: Vec<Value> = if let Some(doc) = obj.get("document") {+ if !doc.is_object() {+ return invalid("`percolate.document` must be an object");+ }+ vec![doc.clone()]+ } else if let Some(docs) = obj.get("documents") {+ let arr = docs+ .as_array()+ .ok_or_else(|| qerr("`percolate.documents` must be an array"))?;+ if arr.iter().any(|d| !d.is_object()) {+ return invalid("`percolate.documents` must contain only objects");+ }+ arr.clone()+ } else {+ return invalid(+ "`percolate` requires an inline `document` or `documents`; the \+ index/id document-fetch form is not supported",+ );+ };++ if documents.is_empty() {+ return invalid("`percolate` requires at least one document");+ }++ Ok(QueryNode::Percolate { field, documents })+}+fn parse_pinned(params: &Value) -> Result<QueryNode> {let obj = params.as_object()@@ -4376,4 +4419,51 @@ mod tests {}));assert!(matches!(node, QueryNode::Bool { .. }));}++ // ── percolate ─────────────────────────────────────────────────────────────++ #[test]+ fn test_percolate_single_document() {+ let node = q(json!({+ "percolate": { "field": "query", "document": { "message": "hi" } }+ }));+ match node {+ QueryNode::Percolate { field, documents } => {+ assert_eq!(field, "query");+ assert_eq!(documents.len(), 1);+ assert_eq!(documents[0]["message"], json!("hi"));+ }+ other => panic!("expected percolate, got {:?}", other),+ }+ }++ #[test]+ fn test_percolate_multiple_documents() {+ let node = q(json!({+ "percolate": {+ "field": "query",+ "documents": [ { "message": "a" }, { "message": "b" } ]+ }+ }));+ match node {+ QueryNode::Percolate { documents, .. } => assert_eq!(documents.len(), 2),+ other => panic!("expected percolate, got {:?}", other),+ }+ }++ #[test]+ fn test_percolate_missing_document_is_400() {+ // Neither `document` nor `documents` present.+ assert!(parse_query(&json!({ "percolate": { "field": "query" } })).is_err());+ // Index/id fetch form is rejected.+ assert!(parse_query(&json!({+ "percolate": { "field": "query", "index": "i", "id": "1" }+ }))+ .is_err());+ // Missing field is rejected.+ assert!(parse_query(&json!({+ "percolate": { "document": { "m": "x" } }+ }))+ .is_err());+ }}diff --git a/engine/crates/xerj-query/src/planner.rs b/engine/crates/xerj-query/src/planner.rsindex 23ef86dec..5f6082661 100644--- a/engine/crates/xerj-query/src/planner.rs+++ b/engine/crates/xerj-query/src/planner.rs@@ -586,6 +586,10 @@ fn plan_node(query: QueryNode, schema: &Schema) -> ExecutionPlan {// Named: plan the inner query; name annotation has no effect on planning.QueryNode::Named { query, .. } => plan_node(*query, schema),+ // Percolate: pure doc-scan (reverse match). The plan is ignored by the+ // engine's doc-scan path; MatchAll satisfies exhaustiveness here.+ QueryNode::Percolate { .. } => ExecutionPlan::MatchAll,+// ── Span queries — treat as doc-scans ────────────────────────────────QueryNode::SpanTerm { field, value } => ExecutionPlan::FtsScan {field,diff --git a/engine/tests/es-compat-yaml/yaml/search/percolate/10_basic.yml b/engine/tests/es-compat-yaml/yaml/search/percolate/10_basic.ymlnew file mode 100644index 000000000..2505fcf03--- /dev/null+++ b/engine/tests/es-compat-yaml/yaml/search/percolate/10_basic.yml@@ -0,0 +1,78 @@+setup:+ - do:+ indices.create:+ index: queries+ body:+ mappings:+ properties:+ query:+ type: percolator+ message:+ type: text+ price:+ type: integer++ # Index four percolator documents: a match, a term, a range and a bool.+ - do:+ bulk:+ refresh: true+ body:+ - '{ "index" : { "_index" : "queries", "_id" : "match" } }'+ - '{ "query": { "match": { "message": "bonsai tree" } } }'+ - '{ "index" : { "_index" : "queries", "_id" : "term" } }'+ - '{ "query": { "term": { "message": "cactus" } } }'+ - '{ "index" : { "_index" : "queries", "_id" : "range" } }'+ - '{ "query": { "range": { "price": { "gte": 10, "lte": 20 } } } }'+ - '{ "index" : { "_index" : "queries", "_id" : "bool" } }'+ - '{ "query": { "bool": { "must": [ { "match": { "message": "bonsai" } }, { "range": { "price": { "gte": 5 } } } ] } } }'++---+"percolate matches the match + bool stored queries":++ - do:+ search:+ rest_total_hits_as_int: true+ index: queries+ body:+ query:+ percolate:+ field: query+ document:+ message: "a lovely bonsai tree"+ price: 15++ # The `match` percolator (message contains bonsai/tree) and the `bool`+ # percolator (bonsai AND price>=5) match; the `term`(cactus) and `range`+ # (10..20 only, but bool also needs price>=5 which 15 satisfies) —+ # note range 10..20 with price 15 also matches.+ - match: { hits.total: 3 }++---+"percolate matches only the term stored query":++ - do:+ search:+ rest_total_hits_as_int: true+ index: queries+ body:+ query:+ percolate:+ field: query+ document:+ message: "cactus"+ price: 100++ - match: { hits.total: 1 }+ - match: { hits.hits.0._id: "term" }++---+"percolate missing document is a 400":++ - do:+ catch: bad_request+ search:+ index: queries+ body:+ query:+ percolate:+ field: query