AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
fix(engine): filtered kNN evaluates its filter schema-aware (#423)
xerj-org8a5791b29ccd
diff --git a/engine/crates/xerj-engine/src/index.rs b/engine/crates/xerj-engine/src/index.rsindex 3db31910a..825b817e1 100644--- a/engine/crates/xerj-engine/src/index.rs+++ b/engine/crates/xerj-engine/src/index.rs@@ -11076,6 +11076,16 @@ impl Index {tracing::info!(index=%self.name, field, k, "semantic_phase=start_brute");}+ // #423: evaluate the kNN `filter` through the SAME schema-aware matcher+ // the rest of search uses (`search_inner` runs its filter through+ // `doc_matches_query_typed(.., &dmq_schema)`). The schemaless+ // `doc_matches_query` has no field types, so on a `keyword`/`text` field+ // it folds case and OVER-MATCHES — a `prefix`/`match`/`fuzzy` filter kept+ // documents a plain (non-knn) search would reject. A filtered knn ALWAYS+ // takes this brute-force path (HNSW serves only `filter.is_none()`), so+ // this is the whole filtered-kNN path. Owned clone; the lock is released.+ let dmq_schema = self.schema().await;+// ── Collect all candidate (doc_id, source) pairs ──────────────let mut candidates: Vec<(String, Value)> = Vec::new();// Memtable first (newest writes).@@ -11213,7 +11223,7 @@ impl Index {if let Some(obj) = src_with_id.as_object_mut() {obj.insert("_id".to_string(), Value::String(id.clone()));}- if !doc_matches_query(f, &src_with_id) {+ if !doc_matches_query_typed(f, &src_with_id, &dmq_schema) {continue;}}@@ -11331,7 +11341,7 @@ impl Index {if let Some(obj) = src_with_id.as_object_mut() {obj.insert("_id".to_string(), Value::String(id.clone()));}- if !doc_matches_query(f, &src_with_id) {+ if !doc_matches_query_typed(f, &src_with_id, &dmq_schema) {continue;}}diff --git a/engine/crates/xerj-engine/tests/integration.rs b/engine/crates/xerj-engine/tests/integration.rsindex 3defb6cb3..baedd10a4 100644--- a/engine/crates/xerj-engine/tests/integration.rs+++ b/engine/crates/xerj-engine/tests/integration.rs@@ -8653,3 +8653,146 @@ async fn a_refusal_is_not_acknowledged_while_the_sidecar_is_unreadable() {"and nothing may be left behind that only this process can see");}++/// #423 (schemaless-matcher root cause, kNN slice): a filtered `knn` applies+/// its `filter` through the SCHEMALESS matcher inside+/// `run_knn_brute_force_with_deadline` (`doc_matches_query`), whereas the rest+/// of search evaluates the same filter through the schema-aware path+/// (`search_inner`'s `doc_matches_query_typed(.., &dmq_schema)`). A `knn` that+/// carries a `filter` ALWAYS routes to brute force — the dispatch serves HNSW+/// only when `filter.is_none()`, so "filters remain exact scans" — hence this+/// is the WHOLE filtered-kNN path, reachable from the public search API, not an+/// edge case.+///+/// Divergence: `tag` is a declared `keyword`, so a keyword `prefix` is+/// case-SENSITIVE in ES ("Hello" does not start with lowercase "hel"). The+/// schemaless matcher has no schema, folds case, and wrongly keeps the doc.+/// Fail-before (schemaless): 1 hit. After (schema-aware): 0 hits — consistent+/// with the identical filter run outside a knn.+#[tokio::test]+async fn knn_filter_prefix_on_keyword_is_case_sensitive_like_plain_search() {+ let dir = TempDir::new().unwrap();+ let mut schema = Schema::empty();+ schema+ .fields+ .push(FieldConfig::new("tag", FieldType::Keyword));+ let mut emb = FieldConfig::new("emb", FieldType::Vector);+ emb.options.dimensions = Some(3);+ emb.options.similarity = Some("cosine".to_string());+ schema.fields.push(emb);++ let engine = make_engine(&dir);+ engine.create_index("knn_filter_kw", schema).unwrap();+ let idx = engine.get_index("knn_filter_kw").unwrap();++ idx.index_document(+ Some("d0".to_string()),+ json!({"tag": "Hello", "emb": [1.0, 0.0, 0.0]}),+ )+ .await+ .unwrap();+ idx.index_document(+ Some("d1".to_string()),+ json!({"tag": "World", "emb": [0.0, 1.0, 0.0]}),+ )+ .await+ .unwrap();+ idx.refresh().await.unwrap();++ let knn_body = json!({+ "size": 10,+ "query": {"knn": {+ "field": "emb",+ "query_vector": [1.0, 0.0, 0.0],+ "k": 10,+ "filter": {"prefix": {"tag": "hel"}}+ }}+ });++ // Control: the SAME keyword `prefix` filter run as a plain query already+ // matches nothing (the schema-aware path is case-sensitive). The filtered+ // knn must agree.+ let plain = idx+ .search(&parse_request(&json!({"size": 10, "query": {"prefix": {"tag": "hel"}}})).unwrap())+ .await+ .unwrap();+ assert_eq!(+ plain.hits.len(),+ 0,+ "control: plain keyword `prefix:hel` must match no case-preserved term; got {:?}",+ plain.hits.iter().map(|h| &h.id).collect::<Vec<_>>()+ );++ let knn = idx+ .search(&parse_request(&knn_body).unwrap())+ .await+ .unwrap();+ assert_eq!(+ knn.hits.len(),+ 0,+ "filtered kNN must apply the keyword `prefix` filter case-sensitively \+ (consistent with plain search); the schemaless matcher folded case and \+ wrongly kept: {:?}",+ knn.hits.iter().map(|h| &h.id).collect::<Vec<_>>()+ );+}++/// #423 companion for the scalar8-quantised brute-force loop (the `use_sq8`+/// branch of `run_knn_brute_force_with_deadline`, a distinct filter call site+/// from the default f32 loop above). Same keyword-`prefix` divergence; the only+/// change is `quantization = scalar8`, which routes candidate filtering through+/// the SQ8 pre-filter loop instead of the default one.+#[tokio::test]+async fn knn_filter_prefix_on_keyword_is_case_sensitive_for_scalar8() {+ let dir = TempDir::new().unwrap();+ let mut schema = Schema::empty();+ schema+ .fields+ .push(FieldConfig::new("tag", FieldType::Keyword));+ let mut emb = FieldConfig::new("emb", FieldType::Vector);+ emb.options.dimensions = Some(3);+ emb.options.similarity = Some("cosine".to_string());+ emb.options.quantization = Some("scalar8".to_string());+ schema.fields.push(emb);++ let engine = make_engine(&dir);+ engine.create_index("knn_filter_kw_sq8", schema).unwrap();+ let idx = engine.get_index("knn_filter_kw_sq8").unwrap();++ idx.index_document(+ Some("d0".to_string()),+ json!({"tag": "Hello", "emb": [1.0, 0.0, 0.0]}),+ )+ .await+ .unwrap();+ idx.index_document(+ Some("d1".to_string()),+ json!({"tag": "World", "emb": [0.0, 1.0, 0.0]}),+ )+ .await+ .unwrap();+ idx.refresh().await.unwrap();++ let knn = idx+ .search(+ &parse_request(&json!({+ "size": 10,+ "query": {"knn": {+ "field": "emb",+ "query_vector": [1.0, 0.0, 0.0],+ "k": 10,+ "filter": {"prefix": {"tag": "hel"}}+ }}+ }))+ .unwrap(),+ )+ .await+ .unwrap();+ assert_eq!(+ knn.hits.len(),+ 0,+ "scalar8 filtered kNN must apply the keyword `prefix` filter \+ case-sensitively; schemaless matcher folded case and wrongly kept: {:?}",+ knn.hits.iter().map(|h| &h.id).collect::<Vec<_>>()+ );+}