Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

feat(query): implement percolate query (reverse search)

xerj-orgbd7513d09bc3
diff --git a/engine/crates/xerj-engine/src/index.rs b/engine/crates/xerj-engine/src/index.rs
index 0ef4a7b7c..7d7723669 100644
--- a/engine/crates/xerj-engine/src/index.rs
+++ b/engine/crates/xerj-engine/src/index.rs
@@ -12110,6 +12110,7 @@ fn is_doc_scan_query(q: &QueryNode) -> bool {
| QueryNode::Constant { .. }
| QueryNode::Boosted { .. }
| QueryNode::Intervals { .. }
+ | QueryNode::Percolate { .. }
)
}
@@ -13345,6 +13346,22 @@ fn doc_matches_query(q: &QueryNode, source: &Value) -> bool {
!intervals.is_empty()
}
+ // Percolate (reverse search): this stored doc's `field` holds a
+ // serialized ES query. Parse it and report a match iff that stored
+ // query matches at least one of the supplied inline documents.
+ // A missing field or an unparseable stored query => no match.
+ QueryNode::Percolate { field, documents } => {
+ let stored = match get_field_value(source, field) {
+ Some(v) if v.is_object() => v,
+ _ => return false,
+ };
+ let stored_q = match xerj_query::parse_query(&stored) {
+ Ok(q) => q,
+ Err(_) => return false,
+ };
+ documents.iter().any(|d| doc_matches_query(&stored_q, d))
+ }
+
_ => false,
}
}
@@ -16482,3 +16499,85 @@ mod fts_projection_tests {
assert_eq!(out, vec!["model".to_string(), "status".to_string()]);
}
}
+
+#[cfg(test)]
+mod percolate_tests {
+ use super::*;
+ use serde_json::json;
+ use xerj_query::ast::QueryNode;
+
+ /// A stored percolator doc holds a serialized query in `query`; the
+ /// percolate query matches that stored doc iff the stored query matches
+ /// one of the supplied inline documents.
+ #[test]
+ fn percolate_term_match_and_miss() {
+ let stored = json!({ "query": { "term": { "message": "bonsai" } } });
+
+ let q_hit = QueryNode::Percolate {
+ field: "query".into(),
+ documents: vec![json!({ "message": "bonsai" })],
+ };
+ assert!(
+ doc_matches_query(&q_hit, &stored),
+ "term matches inline doc"
+ );
+
+ let q_miss = QueryNode::Percolate {
+ field: "query".into(),
+ documents: vec![json!({ "message": "cactus" })],
+ };
+ assert!(
+ !doc_matches_query(&q_miss, &stored),
+ "term does not match a different value"
+ );
+ }
+
+ #[test]
+ fn percolate_range_and_bool() {
+ let stored_range = json!({ "query": { "range": { "price": { "gte": 10, "lte": 20 } } } });
+ let q = QueryNode::Percolate {
+ field: "query".into(),
+ documents: vec![json!({ "price": 15 })],
+ };
+ assert!(doc_matches_query(&q, &stored_range));
+ let q_out = QueryNode::Percolate {
+ field: "query".into(),
+ documents: vec![json!({ "price": 25 })],
+ };
+ assert!(!doc_matches_query(&q_out, &stored_range));
+
+ let stored_bool = json!({
+ "query": {
+ "bool": {
+ "must": [
+ { "term": { "message": "foo" } },
+ { "range": { "price": { "gte": 5 } } }
+ ]
+ }
+ }
+ });
+ let q_bool = QueryNode::Percolate {
+ field: "query".into(),
+ documents: vec![json!({ "message": "foo", "price": 9 })],
+ };
+ assert!(doc_matches_query(&q_bool, &stored_bool));
+ let q_bool_miss = QueryNode::Percolate {
+ field: "query".into(),
+ documents: vec![json!({ "message": "foo", "price": 1 })],
+ };
+ assert!(!doc_matches_query(&q_bool_miss, &stored_bool));
+ }
+
+ #[test]
+ fn percolate_missing_or_unparseable_field_is_no_match() {
+ let stored = json!({ "message": "no query here" });
+ let q = QueryNode::Percolate {
+ field: "query".into(),
+ documents: vec![json!({ "message": "anything" })],
+ };
+ assert!(!doc_matches_query(&q, &stored));
+
+ let stored2 = json!({ "query": "not-an-object" });
+ assert!(!doc_matches_query(&q, &stored2));
+ }
+}
diff --git a/engine/crates/xerj-query/src/ast.rs b/engine/crates/xerj-query/src/ast.rs
index 70605f68d..670a8dd64 100644
--- a/engine/crates/xerj-query/src/ast.rs
+++ b/engine/crates/xerj-query/src/ast.rs
@@ -521,6 +521,17 @@ pub enum QueryNode {
max_query_terms: u32,
},
+ /// Percolate query — reverse search. Each document stored in the index
+ /// holds a serialized query in its `field` (a `percolator`-typed field);
+ /// this query matches those stored documents whose stored query matches
+ /// one of the supplied inline `documents`.
+ Percolate {
+ /// Name of the percolator field holding the stored query object.
+ field: String,
+ /// One or more inline documents to test the stored queries against.
+ documents: Vec<serde_json::Value>,
+ },
+
/// Pinned query — ensures specific document IDs appear first in results.
Pinned {
/// IDs to pin to the top of results.
diff --git a/engine/crates/xerj-query/src/parser.rs b/engine/crates/xerj-query/src/parser.rs
index 5c0645bb9..3e15b4bdb 100644
--- a/engine/crates/xerj-query/src/parser.rs
+++ b/engine/crates/xerj-query/src/parser.rs
@@ -307,7 +307,7 @@ pub fn parse_query(json: &Value) -> Result<QueryNode> {
"has_child" => parse_has_child(params),
"has_parent" => parse_has_parent(params),
"more_like_this" => parse_more_like_this(params),
- "percolate" => Ok(QueryNode::MatchNone), // percolate not supported
+ "percolate" => parse_percolate(params),
"pinned" => parse_pinned(params),
// ── Span queries ───────────────────────────────────────────────────────
"span_term" => parse_span_term(params),
@@ -2824,6 +2824,49 @@ fn parse_more_like_this(params: &Value) -> Result<QueryNode> {
/// }
/// }
/// ```
+/// Parse a `percolate` query.
+///
+/// Only the inline-document form is supported:
+/// ```json
+/// { "field": "query", "document": { … } }
+/// { "field": "query", "documents": [ { … }, … ] }
+/// ```
+/// The index/id fetch form (`{ "field": "query", "index": "...", "id": "..." }`)
+/// is rejected with a 400 rather than silently returning nothing.
+fn parse_percolate(params: &Value) -> Result<QueryNode> {
+ let obj = params
+ .as_object()
+ .ok_or_else(|| qerr("`percolate` must be an object"))?;
+
+ let field = string_field(obj, "field")?;
+
+ let documents: Vec<Value> = if let Some(doc) = obj.get("document") {
+ if !doc.is_object() {
+ return invalid("`percolate.document` must be an object");
+ }
+ vec![doc.clone()]
+ } else if let Some(docs) = obj.get("documents") {
+ let arr = docs
+ .as_array()
+ .ok_or_else(|| qerr("`percolate.documents` must be an array"))?;
+ if arr.iter().any(|d| !d.is_object()) {
+ return invalid("`percolate.documents` must contain only objects");
+ }
+ arr.clone()
+ } else {
+ return invalid(
+ "`percolate` requires an inline `document` or `documents`; the \
+ index/id document-fetch form is not supported",
+ );
+ };
+
+ if documents.is_empty() {
+ return invalid("`percolate` requires at least one document");
+ }
+
+ Ok(QueryNode::Percolate { field, documents })
+}
+
fn parse_pinned(params: &Value) -> Result<QueryNode> {
let obj = params
.as_object()
@@ -4376,4 +4419,51 @@ mod tests {
}));
assert!(matches!(node, QueryNode::Bool { .. }));
}
+
+ // ── percolate ─────────────────────────────────────────────────────────────
+
+ #[test]
+ fn test_percolate_single_document() {
+ let node = q(json!({
+ "percolate": { "field": "query", "document": { "message": "hi" } }
+ }));
+ match node {
+ QueryNode::Percolate { field, documents } => {
+ assert_eq!(field, "query");
+ assert_eq!(documents.len(), 1);
+ assert_eq!(documents[0]["message"], json!("hi"));
+ }
+ other => panic!("expected percolate, got {:?}", other),
+ }
+ }
+
+ #[test]
+ fn test_percolate_multiple_documents() {
+ let node = q(json!({
+ "percolate": {
+ "field": "query",
+ "documents": [ { "message": "a" }, { "message": "b" } ]
+ }
+ }));
+ match node {
+ QueryNode::Percolate { documents, .. } => assert_eq!(documents.len(), 2),
+ other => panic!("expected percolate, got {:?}", other),
+ }
+ }
+
+ #[test]
+ fn test_percolate_missing_document_is_400() {
+ // Neither `document` nor `documents` present.
+ assert!(parse_query(&json!({ "percolate": { "field": "query" } })).is_err());
+ // Index/id fetch form is rejected.
+ assert!(parse_query(&json!({
+ "percolate": { "field": "query", "index": "i", "id": "1" }
+ }))
+ .is_err());
+ // Missing field is rejected.
+ assert!(parse_query(&json!({
+ "percolate": { "document": { "m": "x" } }
+ }))
+ .is_err());
+ }
}
diff --git a/engine/crates/xerj-query/src/planner.rs b/engine/crates/xerj-query/src/planner.rs
index 23ef86dec..5f6082661 100644
--- a/engine/crates/xerj-query/src/planner.rs
+++ b/engine/crates/xerj-query/src/planner.rs
@@ -586,6 +586,10 @@ fn plan_node(query: QueryNode, schema: &Schema) -> ExecutionPlan {
// Named: plan the inner query; name annotation has no effect on planning.
QueryNode::Named { query, .. } => plan_node(*query, schema),
+ // Percolate: pure doc-scan (reverse match). The plan is ignored by the
+ // engine's doc-scan path; MatchAll satisfies exhaustiveness here.
+ QueryNode::Percolate { .. } => ExecutionPlan::MatchAll,
+
// ── Span queries — treat as doc-scans ────────────────────────────────
QueryNode::SpanTerm { field, value } => ExecutionPlan::FtsScan {
field,
diff --git a/engine/tests/es-compat-yaml/yaml/search/percolate/10_basic.yml b/engine/tests/es-compat-yaml/yaml/search/percolate/10_basic.yml
new file mode 100644
index 000000000..2505fcf03
--- /dev/null
+++ b/engine/tests/es-compat-yaml/yaml/search/percolate/10_basic.yml
@@ -0,0 +1,78 @@
+setup:
+ - do:
+ indices.create:
+ index: queries
+ body:
+ mappings:
+ properties:
+ query:
+ type: percolator
+ message:
+ type: text
+ price:
+ type: integer
+
+ # Index four percolator documents: a match, a term, a range and a bool.
+ - do:
+ bulk:
+ refresh: true
+ body:
+ - '{ "index" : { "_index" : "queries", "_id" : "match" } }'
+ - '{ "query": { "match": { "message": "bonsai tree" } } }'
+ - '{ "index" : { "_index" : "queries", "_id" : "term" } }'
+ - '{ "query": { "term": { "message": "cactus" } } }'
+ - '{ "index" : { "_index" : "queries", "_id" : "range" } }'
+ - '{ "query": { "range": { "price": { "gte": 10, "lte": 20 } } } }'
+ - '{ "index" : { "_index" : "queries", "_id" : "bool" } }'
+ - '{ "query": { "bool": { "must": [ { "match": { "message": "bonsai" } }, { "range": { "price": { "gte": 5 } } } ] } } }'
+
+---
+"percolate matches the match + bool stored queries":
+
+ - do:
+ search:
+ rest_total_hits_as_int: true
+ index: queries
+ body:
+ query:
+ percolate:
+ field: query
+ document:
+ message: "a lovely bonsai tree"
+ price: 15
+
+ # The `match` percolator (message contains bonsai/tree) and the `bool`
+ # percolator (bonsai AND price>=5) match; the `term`(cactus) and `range`
+ # (10..20 only, but bool also needs price>=5 which 15 satisfies) —
+ # note range 10..20 with price 15 also matches.
+ - match: { hits.total: 3 }
+
+---
+"percolate matches only the term stored query":
+
+ - do:
+ search:
+ rest_total_hits_as_int: true
+ index: queries
+ body:
+ query:
+ percolate:
+ field: query
+ document:
+ message: "cactus"
+ price: 100
+
+ - match: { hits.total: 1 }
+ - match: { hits.hits.0._id: "term" }
+
+---
+"percolate missing document is a 400":
+
+ - do:
+ catch: bad_request
+ search:
+ index: queries
+ body:
+ query:
+ percolate:
+ field: query