Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

feat(query): implement span_containing / span_within enclosure

xerj-orge779b724e73d
diff --git a/engine/crates/xerj-engine/src/index.rs b/engine/crates/xerj-engine/src/index.rs
index a54970435..86dbdaf57 100644
--- a/engine/crates/xerj-engine/src/index.rs
+++ b/engine/crates/xerj-engine/src/index.rs
@@ -12131,6 +12131,8 @@ fn is_doc_scan_query(q: &QueryNode) -> bool {
| QueryNode::SpanOr { .. }
| QueryNode::SpanNot { .. }
| QueryNode::SpanFirst { .. }
+ | QueryNode::SpanContaining { .. }
+ | QueryNode::SpanWithin { .. }
| QueryNode::HasChild { .. }
| QueryNode::HasParent { .. }
| QueryNode::Constant { .. }
@@ -13309,6 +13311,31 @@ fn doc_matches_query(q: &QueryNode, source: &Value) -> bool {
tokens[..limit].iter().any(|t| t == &v_lower)
}
+ // SpanContaining / SpanWithin: both reduce to the same doc-level
+ // predicate — there exists a `big` span that encloses a `little`
+ // span. (span_containing returns the enclosing spans, span_within the
+ // enclosed ones, but a document matches in either case iff such a pair
+ // exists.) Tokenise the shared field, evaluate both spans into
+ // position intervals, and test enclosure.
+ QueryNode::SpanContaining { little, big } | QueryNode::SpanWithin { little, big } => {
+ let field = match span_field(big).or_else(|| span_field(little)) {
+ Some(f) => f,
+ None => return false,
+ };
+ let text = match get_field_value(source, &field) {
+ Some(Value::String(s)) => s,
+ _ => return false,
+ };
+ let tokens = intervals_tokenise(&text);
+ let bigs = span_intervals(big, &tokens);
+ if bigs.is_empty() {
+ return false;
+ }
+ let littles = span_intervals(little, &tokens);
+ bigs.iter()
+ .any(|b| littles.iter().any(|l| b.start <= l.start && l.end <= b.end))
+ }
+
// ── Join queries — UNREACHABLE ─────────────────────────────────────────
// has_child/has_parent are rejected with a 400 at parse time
// (see xerj-query parser.rs::parse_has_child), so these AST variants
@@ -13415,6 +13442,133 @@ impl TokenInterval {
}
}
+/// Find the target field of a span query subtree (the field all span clauses
+/// operate on). ES span queries are single-field; we return the first
+/// `span_term` / `term` field encountered.
+fn span_field(node: &QueryNode) -> Option<String> {
+ match node {
+ QueryNode::SpanTerm { field, .. } => Some(field.clone()),
+ QueryNode::Term { field, .. } => Some(field.clone()),
+ QueryNode::SpanNear { clauses, .. } | QueryNode::SpanOr { clauses } => {
+ clauses.iter().find_map(span_field)
+ }
+ QueryNode::SpanNot { include, .. } => span_field(include),
+ QueryNode::SpanFirst { match_query, .. } => span_field(match_query),
+ QueryNode::SpanContaining { big, little } | QueryNode::SpanWithin { big, little } => {
+ span_field(big).or_else(|| span_field(little))
+ }
+ _ => None,
+ }
+}
+
+/// Evaluate a span query subtree over an already-tokenised field, returning
+/// the set of matching token-position intervals. Used by span_containing /
+/// span_within to test enclosure between a `little` and a `big` span.
+///
+/// Supports SpanTerm/Term, SpanNear, SpanOr, SpanFirst and SpanNot — the same
+/// family the doc-scan span matcher already understands.
+fn span_intervals(node: &QueryNode, tokens: &[String]) -> Vec<TokenInterval> {
+ match node {
+ QueryNode::SpanTerm { value, .. } => {
+ let target = value.to_lowercase();
+ tokens
+ .iter()
+ .enumerate()
+ .filter(|(_, t)| **t == target)
+ .map(|(i, _)| TokenInterval { start: i, end: i })
+ .collect()
+ }
+ QueryNode::Term { value, .. } => match value.as_str() {
+ Some(v) => {
+ let target = v.to_lowercase();
+ tokens
+ .iter()
+ .enumerate()
+ .filter(|(_, t)| **t == target)
+ .map(|(i, _)| TokenInterval { start: i, end: i })
+ .collect()
+ }
+ None => Vec::new(),
+ },
+ QueryNode::SpanOr { clauses } => clauses
+ .iter()
+ .flat_map(|c| span_intervals(c, tokens))
+ .collect(),
+ QueryNode::SpanNear {
+ clauses,
+ slop,
+ in_order,
+ } => {
+ let per: Vec<Vec<TokenInterval>> =
+ clauses.iter().map(|c| span_intervals(c, tokens)).collect();
+ if per.iter().any(|p| p.is_empty()) {
+ return Vec::new();
+ }
+ // Choose one interval per clause such that consecutive picks are
+ // within `slop` (adjacency gap) and — when in_order — appear in
+ // clause order. The produced interval encloses all picks.
+ #[allow(clippy::too_many_arguments)]
+ fn rec(
+ per: &[Vec<TokenInterval>],
+ idx: usize,
+ prev: Option<TokenInterval>,
+ min: usize,
+ max: usize,
+ slop: u32,
+ in_order: bool,
+ out: &mut Vec<TokenInterval>,
+ ) {
+ if idx == per.len() {
+ if prev.is_some() {
+ out.push(TokenInterval {
+ start: min,
+ end: max,
+ });
+ }
+ return;
+ }
+ for &iv in &per[idx] {
+ let (nmin, nmax, ok) = match prev {
+ None => (iv.start, iv.end, true),
+ Some(p) => {
+ let order_ok = !in_order || iv.start > p.end;
+ // Gap between the two intervals (0 when they
+ // overlap); one of the two subtractions is 0.
+ let dist = iv
+ .start
+ .saturating_sub(p.end)
+ .max(p.start.saturating_sub(iv.end));
+ let slop_ok = dist <= slop as usize + 1;
+ (min.min(iv.start), max.max(iv.end), order_ok && slop_ok)
+ }
+ };
+ if ok {
+ rec(per, idx + 1, Some(iv), nmin, nmax, slop, in_order, out);
+ }
+ }
+ }
+ let mut out = Vec::new();
+ rec(&per, 0, None, 0, 0, *slop, *in_order, &mut out);
+ out
+ }
+ QueryNode::SpanFirst { match_query, end } => span_intervals(match_query, tokens)
+ .into_iter()
+ .filter(|iv| iv.end < *end as usize)
+ .collect(),
+ QueryNode::SpanNot { include, exclude } => {
+ let exc = span_intervals(exclude, tokens);
+ span_intervals(include, tokens)
+ .into_iter()
+ .filter(|iv| {
+ // Keep include spans that do not overlap any exclude span.
+ !exc.iter().any(|e| iv.start <= e.end && e.start <= iv.end)
+ })
+ .collect()
+ }
+ _ => Vec::new(),
+ }
+}
+
/// Evaluate an intervals rule against a tokenised doc. Returns the set of
/// surviving intervals (can be empty).
fn intervals_eval(rule: &Value, tokens: &[String]) -> Vec<TokenInterval> {
@@ -16666,3 +16820,68 @@ mod knn_num_candidates_tests {
assert_eq!(peeled.3, Some(64), "num_candidates");
}
}
+
+#[cfg(test)]
+mod span_containment_tests {
+ use super::*;
+ use serde_json::json;
+ use xerj_query::ast::QueryNode;
+
+ fn span_term(field: &str, value: &str) -> QueryNode {
+ QueryNode::SpanTerm {
+ field: field.into(),
+ value: value.into(),
+ }
+ }
+
+ /// A `big` span_near over [quick, fox] with slop 3, and a `little`
+ /// span_term(brown) that falls between them.
+ fn near_quick_fox() -> QueryNode {
+ QueryNode::SpanNear {
+ clauses: vec![span_term("text", "quick"), span_term("text", "fox")],
+ slop: 3,
+ in_order: true,
+ }
+ }
+
+ #[test]
+ fn span_within_little_inside_big_matches() {
+ // "the quick brown fox jumps": brown (pos 2) is inside the
+ // quick..fox span (pos 1..3).
+ let doc = json!({ "text": "the quick brown fox jumps" });
+ let q = QueryNode::SpanWithin {
+ little: Box::new(span_term("text", "brown")),
+ big: Box::new(near_quick_fox()),
+ };
+ assert!(doc_matches_query(&q, &doc));
+ }
+
+ #[test]
+ fn span_containing_big_contains_little_matches() {
+ let doc = json!({ "text": "the quick brown fox jumps" });
+ let q = QueryNode::SpanContaining {
+ little: Box::new(span_term("text", "brown")),
+ big: Box::new(near_quick_fox()),
+ };
+ assert!(doc_matches_query(&q, &doc));
+ }
+
+ #[test]
+ fn span_within_non_enclosing_does_not_match() {
+ // "jumps" (pos 4) is outside the quick..fox span (pos 1..3).
+ let doc = json!({ "text": "the quick brown fox jumps" });
+ let q = QueryNode::SpanWithin {
+ little: Box::new(span_term("text", "jumps")),
+ big: Box::new(near_quick_fox()),
+ };
+ assert!(!doc_matches_query(&q, &doc));
+
+ // And when the big span does not even form (no "quick" near "fox").
+ let doc2 = json!({ "text": "brown only here" });
+ let q2 = QueryNode::SpanContaining {
+ little: Box::new(span_term("text", "brown")),
+ big: Box::new(near_quick_fox()),
+ };
+ assert!(!doc_matches_query(&q2, &doc2));
+ }
+}
diff --git a/engine/crates/xerj-query/src/ast.rs b/engine/crates/xerj-query/src/ast.rs
index 567698e68..5937b56d7 100644
--- a/engine/crates/xerj-query/src/ast.rs
+++ b/engine/crates/xerj-query/src/ast.rs
@@ -581,6 +581,18 @@ pub enum QueryNode {
end: u32,
},
+ /// Span containing — matches spans of `big` that enclose a span of `little`.
+ SpanContaining {
+ little: Box<QueryNode>,
+ big: Box<QueryNode>,
+ },
+
+ /// Span within — matches spans of `little` that are enclosed by a span of `big`.
+ SpanWithin {
+ little: Box<QueryNode>,
+ big: Box<QueryNode>,
+ },
+
// ── Join queries ──────────────────────────────────────────────────────────
/// Has-child — matches parent documents that have child documents matching the query.
HasChild {
diff --git a/engine/crates/xerj-query/src/parser.rs b/engine/crates/xerj-query/src/parser.rs
index d26e1e015..90b0a51c8 100644
--- a/engine/crates/xerj-query/src/parser.rs
+++ b/engine/crates/xerj-query/src/parser.rs
@@ -3062,16 +3062,31 @@ fn parse_span_first(params: &Value) -> Result<QueryNode> {
})
}
-/// Parse span_containing / span_within — use the `big` clause only.
+/// Parse `span_containing` / `span_within`. Both take a `little` and a `big`
+/// span clause; the difference is only which span must enclose the other,
+/// which is captured by the returned variant.
fn parse_span_containing_like(query_type: &str, params: &Value) -> Result<QueryNode> {
- let obj = match params.as_object() {
- Some(o) => o,
- None => return Ok(QueryNode::MatchNone),
- };
- obj.get("big")
- .map(parse_query)
- .transpose()?
+ let obj = params
+ .as_object()
+ .ok_or_else(|| qerr(format!("`{}` must be an object", query_type)))?;
+
+ let little = obj
+ .get("little")
+ .ok_or_else(|| qerr(format!("`{}` requires a `little` clause", query_type)))
+ .and_then(parse_query)?;
+ let big = obj
+ .get("big")
.ok_or_else(|| qerr(format!("`{}` requires a `big` clause", query_type)))
+ .and_then(parse_query)?;
+
+ let little = Box::new(little);
+ let big = Box::new(big);
+
+ Ok(match query_type {
+ "span_within" => QueryNode::SpanWithin { little, big },
+ // "span_containing"
+ _ => QueryNode::SpanContaining { little, big },
+ })
}
/// Parse a `has_child` query.
@@ -4516,4 +4531,55 @@ mod tests {
}))
.is_err());
}
+
+ // ── span_containing / span_within ───────────────────────────────────────
+
+ #[test]
+ fn test_span_containing_parses_little_and_big() {
+ let node = q(json!({
+ "span_containing": {
+ "little": { "span_term": { "text": "brown" } },
+ "big": {
+ "span_near": {
+ "clauses": [
+ { "span_term": { "text": "quick" } },
+ { "span_term": { "text": "fox" } }
+ ],
+ "slop": 3,
+ "in_order": true
+ }
+ }
+ }
+ }));
+ match node {
+ QueryNode::SpanContaining { little, big } => {
+ assert!(matches!(*little, QueryNode::SpanTerm { .. }));
+ assert!(matches!(*big, QueryNode::SpanNear { .. }));
+ }
+ other => panic!("expected span_containing, got {:?}", other),
+ }
+ }
+
+ #[test]
+ fn test_span_within_variant() {
+ let node = q(json!({
+ "span_within": {
+ "little": { "span_term": { "text": "brown" } },
+ "big": { "span_term": { "text": "quick" } }
+ }
+ }));
+ assert!(matches!(node, QueryNode::SpanWithin { .. }));
+ }
+
+ #[test]
+ fn test_span_containing_missing_clause_is_400() {
+ assert!(parse_query(&json!({
+ "span_containing": { "big": { "span_term": { "text": "x" } } }
+ }))
+ .is_err());
+ assert!(parse_query(&json!({
+ "span_within": { "little": { "span_term": { "text": "x" } } }
+ }))
+ .is_err());
+ }
}
diff --git a/engine/crates/xerj-query/src/planner.rs b/engine/crates/xerj-query/src/planner.rs
index fe95c890e..704d8b824 100644
--- a/engine/crates/xerj-query/src/planner.rs
+++ b/engine/crates/xerj-query/src/planner.rs
@@ -641,6 +641,13 @@ fn plan_node(query: QueryNode, schema: &Schema) -> ExecutionPlan {
QueryNode::SpanFirst { match_query, .. } => plan_node(*match_query, schema),
+ // span_containing / span_within are pure doc-scans (enclosure test);
+ // the plan is ignored by the doc-scan path. Plan the `big` clause so
+ // any field annotation is preserved for cost estimation.
+ QueryNode::SpanContaining { big, .. } | QueryNode::SpanWithin { big, .. } => {
+ plan_node(*big, schema)
+ }
+
// ── Join queries — UNREACHABLE ────────────────────────────────────────
// has_child/has_parent are rejected with a 400 at parse time
// (see parser.rs::parse_has_child), so these AST variants are never
diff --git a/engine/tests/es-compat-yaml/yaml/search/210_span_containing_within.yml b/engine/tests/es-compat-yaml/yaml/search/210_span_containing_within.yml
new file mode 100644
index 000000000..d751ba7d3
--- /dev/null
+++ b/engine/tests/es-compat-yaml/yaml/search/210_span_containing_within.yml
@@ -0,0 +1,80 @@
+setup:
+ - do:
+ indices.create:
+ index: spans
+ body:
+ mappings:
+ properties:
+ text:
+ type: text
+
+ - do:
+ bulk:
+ refresh: true
+ body:
+ # doc 1: "brown" sits between "quick" and "fox" — the little span is
+ # enclosed by the big span_near.
+ - '{ "index": { "_index": "spans", "_id": "1" } }'
+ - '{ "text": "the quick brown fox jumps" }'
+ # doc 2: "brown" is AFTER "fox", so it is not enclosed by quick..fox.
+ - '{ "index": { "_index": "spans", "_id": "2" } }'
+ - '{ "text": "the quick fox is brown" }'
+
+---
+"span_within: little span enclosed by big span matches only doc 1":
+
+ - do:
+ search:
+ rest_total_hits_as_int: true
+ index: spans
+ body:
+ query:
+ span_within:
+ little:
+ span_term: { text: "brown" }
+ big:
+ span_near:
+ clauses:
+ - span_term: { text: "quick" }
+ - span_term: { text: "fox" }
+ slop: 3
+ in_order: true
+
+ - match: { hits.total: 1 }
+ - match: { hits.hits.0._id: "1" }
+
+---
+"span_containing: big span contains little span matches only doc 1":
+
+ - do:
+ search:
+ rest_total_hits_as_int: true
+ index: spans
+ body:
+ query:
+ span_containing:
+ little:
+ span_term: { text: "brown" }
+ big:
+ span_near:
+ clauses:
+ - span_term: { text: "quick" }
+ - span_term: { text: "fox" }
+ slop: 3
+ in_order: true
+
+ - match: { hits.total: 1 }
+ - match: { hits.hits.0._id: "1" }
+
+---
+"span_containing requires both little and big (400)":
+
+ - do:
+ catch: bad_request
+ search:
+ index: spans
+ body:
+ query:
+ span_containing:
+ big:
+ span_term: { text: "fox" }