Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

fix(engine): filtered kNN evaluates its filter schema-aware (#423)

xerj-org8a5791b29ccd
diff --git a/engine/crates/xerj-engine/src/index.rs b/engine/crates/xerj-engine/src/index.rs
index 3db31910a..825b817e1 100644
--- a/engine/crates/xerj-engine/src/index.rs
+++ b/engine/crates/xerj-engine/src/index.rs
@@ -11076,6 +11076,16 @@ impl Index {
tracing::info!(index=%self.name, field, k, "semantic_phase=start_brute");
}
+ // #423: evaluate the kNN `filter` through the SAME schema-aware matcher
+ // the rest of search uses (`search_inner` runs its filter through
+ // `doc_matches_query_typed(.., &dmq_schema)`). The schemaless
+ // `doc_matches_query` has no field types, so on a `keyword`/`text` field
+ // it folds case and OVER-MATCHES — a `prefix`/`match`/`fuzzy` filter kept
+ // documents a plain (non-knn) search would reject. A filtered knn ALWAYS
+ // takes this brute-force path (HNSW serves only `filter.is_none()`), so
+ // this is the whole filtered-kNN path. Owned clone; the lock is released.
+ let dmq_schema = self.schema().await;
+
// ── Collect all candidate (doc_id, source) pairs ──────────────
let mut candidates: Vec<(String, Value)> = Vec::new();
// Memtable first (newest writes).
@@ -11213,7 +11223,7 @@ impl Index {
if let Some(obj) = src_with_id.as_object_mut() {
obj.insert("_id".to_string(), Value::String(id.clone()));
}
- if !doc_matches_query(f, &src_with_id) {
+ if !doc_matches_query_typed(f, &src_with_id, &dmq_schema) {
continue;
}
}
@@ -11331,7 +11341,7 @@ impl Index {
if let Some(obj) = src_with_id.as_object_mut() {
obj.insert("_id".to_string(), Value::String(id.clone()));
}
- if !doc_matches_query(f, &src_with_id) {
+ if !doc_matches_query_typed(f, &src_with_id, &dmq_schema) {
continue;
}
}
diff --git a/engine/crates/xerj-engine/tests/integration.rs b/engine/crates/xerj-engine/tests/integration.rs
index 3defb6cb3..baedd10a4 100644
--- a/engine/crates/xerj-engine/tests/integration.rs
+++ b/engine/crates/xerj-engine/tests/integration.rs
@@ -8653,3 +8653,146 @@ async fn a_refusal_is_not_acknowledged_while_the_sidecar_is_unreadable() {
"and nothing may be left behind that only this process can see"
);
}
+
+/// #423 (schemaless-matcher root cause, kNN slice): a filtered `knn` applies
+/// its `filter` through the SCHEMALESS matcher inside
+/// `run_knn_brute_force_with_deadline` (`doc_matches_query`), whereas the rest
+/// of search evaluates the same filter through the schema-aware path
+/// (`search_inner`'s `doc_matches_query_typed(.., &dmq_schema)`). A `knn` that
+/// carries a `filter` ALWAYS routes to brute force — the dispatch serves HNSW
+/// only when `filter.is_none()`, so "filters remain exact scans" — hence this
+/// is the WHOLE filtered-kNN path, reachable from the public search API, not an
+/// edge case.
+///
+/// Divergence: `tag` is a declared `keyword`, so a keyword `prefix` is
+/// case-SENSITIVE in ES ("Hello" does not start with lowercase "hel"). The
+/// schemaless matcher has no schema, folds case, and wrongly keeps the doc.
+/// Fail-before (schemaless): 1 hit. After (schema-aware): 0 hits — consistent
+/// with the identical filter run outside a knn.
+#[tokio::test]
+async fn knn_filter_prefix_on_keyword_is_case_sensitive_like_plain_search() {
+ let dir = TempDir::new().unwrap();
+ let mut schema = Schema::empty();
+ schema
+ .fields
+ .push(FieldConfig::new("tag", FieldType::Keyword));
+ let mut emb = FieldConfig::new("emb", FieldType::Vector);
+ emb.options.dimensions = Some(3);
+ emb.options.similarity = Some("cosine".to_string());
+ schema.fields.push(emb);
+
+ let engine = make_engine(&dir);
+ engine.create_index("knn_filter_kw", schema).unwrap();
+ let idx = engine.get_index("knn_filter_kw").unwrap();
+
+ idx.index_document(
+ Some("d0".to_string()),
+ json!({"tag": "Hello", "emb": [1.0, 0.0, 0.0]}),
+ )
+ .await
+ .unwrap();
+ idx.index_document(
+ Some("d1".to_string()),
+ json!({"tag": "World", "emb": [0.0, 1.0, 0.0]}),
+ )
+ .await
+ .unwrap();
+ idx.refresh().await.unwrap();
+
+ let knn_body = json!({
+ "size": 10,
+ "query": {"knn": {
+ "field": "emb",
+ "query_vector": [1.0, 0.0, 0.0],
+ "k": 10,
+ "filter": {"prefix": {"tag": "hel"}}
+ }}
+ });
+
+ // Control: the SAME keyword `prefix` filter run as a plain query already
+ // matches nothing (the schema-aware path is case-sensitive). The filtered
+ // knn must agree.
+ let plain = idx
+ .search(&parse_request(&json!({"size": 10, "query": {"prefix": {"tag": "hel"}}})).unwrap())
+ .await
+ .unwrap();
+ assert_eq!(
+ plain.hits.len(),
+ 0,
+ "control: plain keyword `prefix:hel` must match no case-preserved term; got {:?}",
+ plain.hits.iter().map(|h| &h.id).collect::<Vec<_>>()
+ );
+
+ let knn = idx
+ .search(&parse_request(&knn_body).unwrap())
+ .await
+ .unwrap();
+ assert_eq!(
+ knn.hits.len(),
+ 0,
+ "filtered kNN must apply the keyword `prefix` filter case-sensitively \
+ (consistent with plain search); the schemaless matcher folded case and \
+ wrongly kept: {:?}",
+ knn.hits.iter().map(|h| &h.id).collect::<Vec<_>>()
+ );
+}
+
+/// #423 companion for the scalar8-quantised brute-force loop (the `use_sq8`
+/// branch of `run_knn_brute_force_with_deadline`, a distinct filter call site
+/// from the default f32 loop above). Same keyword-`prefix` divergence; the only
+/// change is `quantization = scalar8`, which routes candidate filtering through
+/// the SQ8 pre-filter loop instead of the default one.
+#[tokio::test]
+async fn knn_filter_prefix_on_keyword_is_case_sensitive_for_scalar8() {
+ let dir = TempDir::new().unwrap();
+ let mut schema = Schema::empty();
+ schema
+ .fields
+ .push(FieldConfig::new("tag", FieldType::Keyword));
+ let mut emb = FieldConfig::new("emb", FieldType::Vector);
+ emb.options.dimensions = Some(3);
+ emb.options.similarity = Some("cosine".to_string());
+ emb.options.quantization = Some("scalar8".to_string());
+ schema.fields.push(emb);
+
+ let engine = make_engine(&dir);
+ engine.create_index("knn_filter_kw_sq8", schema).unwrap();
+ let idx = engine.get_index("knn_filter_kw_sq8").unwrap();
+
+ idx.index_document(
+ Some("d0".to_string()),
+ json!({"tag": "Hello", "emb": [1.0, 0.0, 0.0]}),
+ )
+ .await
+ .unwrap();
+ idx.index_document(
+ Some("d1".to_string()),
+ json!({"tag": "World", "emb": [0.0, 1.0, 0.0]}),
+ )
+ .await
+ .unwrap();
+ idx.refresh().await.unwrap();
+
+ let knn = idx
+ .search(
+ &parse_request(&json!({
+ "size": 10,
+ "query": {"knn": {
+ "field": "emb",
+ "query_vector": [1.0, 0.0, 0.0],
+ "k": 10,
+ "filter": {"prefix": {"tag": "hel"}}
+ }}
+ }))
+ .unwrap(),
+ )
+ .await
+ .unwrap();
+ assert_eq!(
+ knn.hits.len(),
+ 0,
+ "scalar8 filtered kNN must apply the keyword `prefix` filter \
+ case-sensitively; schemaless matcher folded case and wrongly kept: {:?}",
+ knn.hits.iter().map(|h| &h.id).collect::<Vec<_>>()
+ );
+}