AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
feat(query): implement span_containing / span_within enclosure
xerj-orge779b724e73d
diff --git a/engine/crates/xerj-engine/src/index.rs b/engine/crates/xerj-engine/src/index.rsindex a54970435..86dbdaf57 100644--- a/engine/crates/xerj-engine/src/index.rs+++ b/engine/crates/xerj-engine/src/index.rs@@ -12131,6 +12131,8 @@ fn is_doc_scan_query(q: &QueryNode) -> bool {| QueryNode::SpanOr { .. }| QueryNode::SpanNot { .. }| QueryNode::SpanFirst { .. }+ | QueryNode::SpanContaining { .. }+ | QueryNode::SpanWithin { .. }| QueryNode::HasChild { .. }| QueryNode::HasParent { .. }| QueryNode::Constant { .. }@@ -13309,6 +13311,31 @@ fn doc_matches_query(q: &QueryNode, source: &Value) -> bool {tokens[..limit].iter().any(|t| t == &v_lower)}+ // SpanContaining / SpanWithin: both reduce to the same doc-level+ // predicate — there exists a `big` span that encloses a `little`+ // span. (span_containing returns the enclosing spans, span_within the+ // enclosed ones, but a document matches in either case iff such a pair+ // exists.) Tokenise the shared field, evaluate both spans into+ // position intervals, and test enclosure.+ QueryNode::SpanContaining { little, big } | QueryNode::SpanWithin { little, big } => {+ let field = match span_field(big).or_else(|| span_field(little)) {+ Some(f) => f,+ None => return false,+ };+ let text = match get_field_value(source, &field) {+ Some(Value::String(s)) => s,+ _ => return false,+ };+ let tokens = intervals_tokenise(&text);+ let bigs = span_intervals(big, &tokens);+ if bigs.is_empty() {+ return false;+ }+ let littles = span_intervals(little, &tokens);+ bigs.iter()+ .any(|b| littles.iter().any(|l| b.start <= l.start && l.end <= b.end))+ }+// ── Join queries — UNREACHABLE ─────────────────────────────────────────// has_child/has_parent are rejected with a 400 at parse time// (see xerj-query parser.rs::parse_has_child), so these AST variants@@ -13415,6 +13442,133 @@ impl TokenInterval {}}+/// Find the target field of a span query subtree (the field all span clauses+/// operate on). ES span queries are single-field; we return the first+/// `span_term` / `term` field encountered.+fn span_field(node: &QueryNode) -> Option<String> {+ match node {+ QueryNode::SpanTerm { field, .. } => Some(field.clone()),+ QueryNode::Term { field, .. } => Some(field.clone()),+ QueryNode::SpanNear { clauses, .. } | QueryNode::SpanOr { clauses } => {+ clauses.iter().find_map(span_field)+ }+ QueryNode::SpanNot { include, .. } => span_field(include),+ QueryNode::SpanFirst { match_query, .. } => span_field(match_query),+ QueryNode::SpanContaining { big, little } | QueryNode::SpanWithin { big, little } => {+ span_field(big).or_else(|| span_field(little))+ }+ _ => None,+ }+}++/// Evaluate a span query subtree over an already-tokenised field, returning+/// the set of matching token-position intervals. Used by span_containing /+/// span_within to test enclosure between a `little` and a `big` span.+///+/// Supports SpanTerm/Term, SpanNear, SpanOr, SpanFirst and SpanNot — the same+/// family the doc-scan span matcher already understands.+fn span_intervals(node: &QueryNode, tokens: &[String]) -> Vec<TokenInterval> {+ match node {+ QueryNode::SpanTerm { value, .. } => {+ let target = value.to_lowercase();+ tokens+ .iter()+ .enumerate()+ .filter(|(_, t)| **t == target)+ .map(|(i, _)| TokenInterval { start: i, end: i })+ .collect()+ }+ QueryNode::Term { value, .. } => match value.as_str() {+ Some(v) => {+ let target = v.to_lowercase();+ tokens+ .iter()+ .enumerate()+ .filter(|(_, t)| **t == target)+ .map(|(i, _)| TokenInterval { start: i, end: i })+ .collect()+ }+ None => Vec::new(),+ },+ QueryNode::SpanOr { clauses } => clauses+ .iter()+ .flat_map(|c| span_intervals(c, tokens))+ .collect(),+ QueryNode::SpanNear {+ clauses,+ slop,+ in_order,+ } => {+ let per: Vec<Vec<TokenInterval>> =+ clauses.iter().map(|c| span_intervals(c, tokens)).collect();+ if per.iter().any(|p| p.is_empty()) {+ return Vec::new();+ }+ // Choose one interval per clause such that consecutive picks are+ // within `slop` (adjacency gap) and — when in_order — appear in+ // clause order. The produced interval encloses all picks.+ #[allow(clippy::too_many_arguments)]+ fn rec(+ per: &[Vec<TokenInterval>],+ idx: usize,+ prev: Option<TokenInterval>,+ min: usize,+ max: usize,+ slop: u32,+ in_order: bool,+ out: &mut Vec<TokenInterval>,+ ) {+ if idx == per.len() {+ if prev.is_some() {+ out.push(TokenInterval {+ start: min,+ end: max,+ });+ }+ return;+ }+ for &iv in &per[idx] {+ let (nmin, nmax, ok) = match prev {+ None => (iv.start, iv.end, true),+ Some(p) => {+ let order_ok = !in_order || iv.start > p.end;+ // Gap between the two intervals (0 when they+ // overlap); one of the two subtractions is 0.+ let dist = iv+ .start+ .saturating_sub(p.end)+ .max(p.start.saturating_sub(iv.end));+ let slop_ok = dist <= slop as usize + 1;+ (min.min(iv.start), max.max(iv.end), order_ok && slop_ok)+ }+ };+ if ok {+ rec(per, idx + 1, Some(iv), nmin, nmax, slop, in_order, out);+ }+ }+ }+ let mut out = Vec::new();+ rec(&per, 0, None, 0, 0, *slop, *in_order, &mut out);+ out+ }+ QueryNode::SpanFirst { match_query, end } => span_intervals(match_query, tokens)+ .into_iter()+ .filter(|iv| iv.end < *end as usize)+ .collect(),+ QueryNode::SpanNot { include, exclude } => {+ let exc = span_intervals(exclude, tokens);+ span_intervals(include, tokens)+ .into_iter()+ .filter(|iv| {+ // Keep include spans that do not overlap any exclude span.+ !exc.iter().any(|e| iv.start <= e.end && e.start <= iv.end)+ })+ .collect()+ }+ _ => Vec::new(),+ }+}+/// Evaluate an intervals rule against a tokenised doc. Returns the set of/// surviving intervals (can be empty).fn intervals_eval(rule: &Value, tokens: &[String]) -> Vec<TokenInterval> {@@ -16666,3 +16820,68 @@ mod knn_num_candidates_tests {assert_eq!(peeled.3, Some(64), "num_candidates");}}++#[cfg(test)]+mod span_containment_tests {+ use super::*;+ use serde_json::json;+ use xerj_query::ast::QueryNode;++ fn span_term(field: &str, value: &str) -> QueryNode {+ QueryNode::SpanTerm {+ field: field.into(),+ value: value.into(),+ }+ }++ /// A `big` span_near over [quick, fox] with slop 3, and a `little`+ /// span_term(brown) that falls between them.+ fn near_quick_fox() -> QueryNode {+ QueryNode::SpanNear {+ clauses: vec![span_term("text", "quick"), span_term("text", "fox")],+ slop: 3,+ in_order: true,+ }+ }++ #[test]+ fn span_within_little_inside_big_matches() {+ // "the quick brown fox jumps": brown (pos 2) is inside the+ // quick..fox span (pos 1..3).+ let doc = json!({ "text": "the quick brown fox jumps" });+ let q = QueryNode::SpanWithin {+ little: Box::new(span_term("text", "brown")),+ big: Box::new(near_quick_fox()),+ };+ assert!(doc_matches_query(&q, &doc));+ }++ #[test]+ fn span_containing_big_contains_little_matches() {+ let doc = json!({ "text": "the quick brown fox jumps" });+ let q = QueryNode::SpanContaining {+ little: Box::new(span_term("text", "brown")),+ big: Box::new(near_quick_fox()),+ };+ assert!(doc_matches_query(&q, &doc));+ }++ #[test]+ fn span_within_non_enclosing_does_not_match() {+ // "jumps" (pos 4) is outside the quick..fox span (pos 1..3).+ let doc = json!({ "text": "the quick brown fox jumps" });+ let q = QueryNode::SpanWithin {+ little: Box::new(span_term("text", "jumps")),+ big: Box::new(near_quick_fox()),+ };+ assert!(!doc_matches_query(&q, &doc));++ // And when the big span does not even form (no "quick" near "fox").+ let doc2 = json!({ "text": "brown only here" });+ let q2 = QueryNode::SpanContaining {+ little: Box::new(span_term("text", "brown")),+ big: Box::new(near_quick_fox()),+ };+ assert!(!doc_matches_query(&q2, &doc2));+ }+}diff --git a/engine/crates/xerj-query/src/ast.rs b/engine/crates/xerj-query/src/ast.rsindex 567698e68..5937b56d7 100644--- a/engine/crates/xerj-query/src/ast.rs+++ b/engine/crates/xerj-query/src/ast.rs@@ -581,6 +581,18 @@ pub enum QueryNode {end: u32,},+ /// Span containing — matches spans of `big` that enclose a span of `little`.+ SpanContaining {+ little: Box<QueryNode>,+ big: Box<QueryNode>,+ },++ /// Span within — matches spans of `little` that are enclosed by a span of `big`.+ SpanWithin {+ little: Box<QueryNode>,+ big: Box<QueryNode>,+ },+// ── Join queries ──────────────────────────────────────────────────────────/// Has-child — matches parent documents that have child documents matching the query.HasChild {diff --git a/engine/crates/xerj-query/src/parser.rs b/engine/crates/xerj-query/src/parser.rsindex d26e1e015..90b0a51c8 100644--- a/engine/crates/xerj-query/src/parser.rs+++ b/engine/crates/xerj-query/src/parser.rs@@ -3062,16 +3062,31 @@ fn parse_span_first(params: &Value) -> Result<QueryNode> {})}-/// Parse span_containing / span_within — use the `big` clause only.+/// Parse `span_containing` / `span_within`. Both take a `little` and a `big`+/// span clause; the difference is only which span must enclose the other,+/// which is captured by the returned variant.fn parse_span_containing_like(query_type: &str, params: &Value) -> Result<QueryNode> {- let obj = match params.as_object() {- Some(o) => o,- None => return Ok(QueryNode::MatchNone),- };- obj.get("big")- .map(parse_query)- .transpose()?+ let obj = params+ .as_object()+ .ok_or_else(|| qerr(format!("`{}` must be an object", query_type)))?;++ let little = obj+ .get("little")+ .ok_or_else(|| qerr(format!("`{}` requires a `little` clause", query_type)))+ .and_then(parse_query)?;+ let big = obj+ .get("big").ok_or_else(|| qerr(format!("`{}` requires a `big` clause", query_type)))+ .and_then(parse_query)?;++ let little = Box::new(little);+ let big = Box::new(big);++ Ok(match query_type {+ "span_within" => QueryNode::SpanWithin { little, big },+ // "span_containing"+ _ => QueryNode::SpanContaining { little, big },+ })}/// Parse a `has_child` query.@@ -4516,4 +4531,55 @@ mod tests {})).is_err());}++ // ── span_containing / span_within ───────────────────────────────────────++ #[test]+ fn test_span_containing_parses_little_and_big() {+ let node = q(json!({+ "span_containing": {+ "little": { "span_term": { "text": "brown" } },+ "big": {+ "span_near": {+ "clauses": [+ { "span_term": { "text": "quick" } },+ { "span_term": { "text": "fox" } }+ ],+ "slop": 3,+ "in_order": true+ }+ }+ }+ }));+ match node {+ QueryNode::SpanContaining { little, big } => {+ assert!(matches!(*little, QueryNode::SpanTerm { .. }));+ assert!(matches!(*big, QueryNode::SpanNear { .. }));+ }+ other => panic!("expected span_containing, got {:?}", other),+ }+ }++ #[test]+ fn test_span_within_variant() {+ let node = q(json!({+ "span_within": {+ "little": { "span_term": { "text": "brown" } },+ "big": { "span_term": { "text": "quick" } }+ }+ }));+ assert!(matches!(node, QueryNode::SpanWithin { .. }));+ }++ #[test]+ fn test_span_containing_missing_clause_is_400() {+ assert!(parse_query(&json!({+ "span_containing": { "big": { "span_term": { "text": "x" } } }+ }))+ .is_err());+ assert!(parse_query(&json!({+ "span_within": { "little": { "span_term": { "text": "x" } } }+ }))+ .is_err());+ }}diff --git a/engine/crates/xerj-query/src/planner.rs b/engine/crates/xerj-query/src/planner.rsindex fe95c890e..704d8b824 100644--- a/engine/crates/xerj-query/src/planner.rs+++ b/engine/crates/xerj-query/src/planner.rs@@ -641,6 +641,13 @@ fn plan_node(query: QueryNode, schema: &Schema) -> ExecutionPlan {QueryNode::SpanFirst { match_query, .. } => plan_node(*match_query, schema),+ // span_containing / span_within are pure doc-scans (enclosure test);+ // the plan is ignored by the doc-scan path. Plan the `big` clause so+ // any field annotation is preserved for cost estimation.+ QueryNode::SpanContaining { big, .. } | QueryNode::SpanWithin { big, .. } => {+ plan_node(*big, schema)+ }+// ── Join queries — UNREACHABLE ────────────────────────────────────────// has_child/has_parent are rejected with a 400 at parse time// (see parser.rs::parse_has_child), so these AST variants are neverdiff --git a/engine/tests/es-compat-yaml/yaml/search/210_span_containing_within.yml b/engine/tests/es-compat-yaml/yaml/search/210_span_containing_within.ymlnew file mode 100644index 000000000..d751ba7d3--- /dev/null+++ b/engine/tests/es-compat-yaml/yaml/search/210_span_containing_within.yml@@ -0,0 +1,80 @@+setup:+ - do:+ indices.create:+ index: spans+ body:+ mappings:+ properties:+ text:+ type: text++ - do:+ bulk:+ refresh: true+ body:+ # doc 1: "brown" sits between "quick" and "fox" — the little span is+ # enclosed by the big span_near.+ - '{ "index": { "_index": "spans", "_id": "1" } }'+ - '{ "text": "the quick brown fox jumps" }'+ # doc 2: "brown" is AFTER "fox", so it is not enclosed by quick..fox.+ - '{ "index": { "_index": "spans", "_id": "2" } }'+ - '{ "text": "the quick fox is brown" }'++---+"span_within: little span enclosed by big span matches only doc 1":++ - do:+ search:+ rest_total_hits_as_int: true+ index: spans+ body:+ query:+ span_within:+ little:+ span_term: { text: "brown" }+ big:+ span_near:+ clauses:+ - span_term: { text: "quick" }+ - span_term: { text: "fox" }+ slop: 3+ in_order: true++ - match: { hits.total: 1 }+ - match: { hits.hits.0._id: "1" }++---+"span_containing: big span contains little span matches only doc 1":++ - do:+ search:+ rest_total_hits_as_int: true+ index: spans+ body:+ query:+ span_containing:+ little:+ span_term: { text: "brown" }+ big:+ span_near:+ clauses:+ - span_term: { text: "quick" }+ - span_term: { text: "fox" }+ slop: 3+ in_order: true++ - match: { hits.total: 1 }+ - match: { hits.hits.0._id: "1" }++---+"span_containing requires both little and big (400)":++ - do:+ catch: bad_request+ search:+ index: spans+ body:+ query:+ span_containing:+ big:+ span_term: { text: "fox" }