Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

fix(mapping): semantic_text is Text on the lexical side — match/BM25 survives flush

xerj-org198fbf88a16e
diff --git a/engine/crates/xerj-api/src/es_compat.rs b/engine/crates/xerj-api/src/es_compat.rs
index 27496643b..076805c08 100644
--- a/engine/crates/xerj-api/src/es_compat.rs
+++ b/engine/crates/xerj-api/src/es_compat.rs
@@ -10894,7 +10894,16 @@ fn es_properties_to_fields(properties: &Value) -> Vec<FieldConfig> {
fn es_type_to_native(es_type: &str) -> FieldType {
match es_type {
- "text" => FieldType::Text,
+ // `semantic_text` carries plain text in `_source` (the derived
+ // embedding lives in the companion `<field>_vector` field), so on
+ // the LEXICAL side it must behave exactly like `text`: schema type
+ // Text puts it in the search path's `text_fields` set and gives it
+ // the standard analyzer + positions in the segment FTS built at
+ // flush. It used to fall through to `Object`, which FTS-indexes
+ // the whole value as ONE keyword token — match/BM25 then found the
+ // doc pre-flush (memtable scan) but ZERO hits post-flush (segment
+ // FTS had no standard-analyzed tokens for the field).
+ "text" | "semantic_text" => FieldType::Text,
"keyword" | "constant_keyword" | "wildcard" => FieldType::Keyword,
"long" | "integer" | "short" | "byte" | "unsigned_long" => FieldType::Long,
"double" | "float" | "half_float" | "scaled_float" => FieldType::Double,
diff --git a/engine/crates/xerj-engine/tests/integration.rs b/engine/crates/xerj-engine/tests/integration.rs
index 3c71ddc99..7f01e1de4 100644
--- a/engine/crates/xerj-engine/tests/integration.rs
+++ b/engine/crates/xerj-engine/tests/integration.rs
@@ -4713,3 +4713,58 @@ async fn test_bulk_index_malformed_doc_rejected_per_item() {
assert_eq!(all2.total.value, 1, "non-object body must not be stored");
}
+/// Blocker 4: match/BM25 over a `semantic_text` field returned hits from the
+/// memtable but ZERO once flushed — the field's schema type (Object) gave it
+/// the whole-value keyword analyzer in the segment FTS. The es_compat mapper
+/// now types it Text (+ embedding config); this exercises exactly that shape.
+#[tokio::test]
+async fn test_semantic_text_match_survives_flush() {
+ let dir = TempDir::new().unwrap();
+ let engine = make_engine(&dir);
+
+ // The schema shape es_compat produces for `"type": "semantic_text"`:
+ // lexical side = Text (standard analyzer, positions), plus an embedding
+ // config producing the companion `content_vector`.
+ let mut schema = Schema::empty();
+ let mut fc = FieldConfig::new("content", FieldType::Text);
+ fc.options.dimensions = Some(16);
+ fc.options.similarity = Some("cosine".to_string());
+ fc.embedding = Some(xerj_common::types::EmbeddingConfig {
+ endpoint: None,
+ model: None,
+ target_field: Some("content_vector".to_string()),
+ });
+ schema.fields.push(fc);
+ engine.create_index("sem", schema).unwrap();
+ let idx = engine.get_index("sem").unwrap();
+
+ idx.index_document(
+ Some("1".into()),
+ json!({"content": "the quick brown fox jumps over the lazy dog"}),
+ )
+ .await
+ .unwrap();
+
+ let pre = idx
+ .search(&make_search(json!({"match": {"content": "quick fox"}})))
+ .await
+ .unwrap();
+ assert_eq!(pre.total.value, 1, "pre-flush match must hit");
+
+ idx.flush().await.unwrap();
+
+ let post = idx
+ .search(&make_search(json!({"match": {"content": "quick fox"}})))
+ .await
+ .unwrap();
+ assert_eq!(
+ post.total.value, 1,
+ "post-flush match must still hit (segment FTS must standard-analyze semantic_text)"
+ );
+ // The auto-embed side must be unaffected by the lexical type change.
+ assert!(
+ post.hits[0].source.get("content_vector").is_some(),
+ "companion embedding vector missing from _source"
+ );
+}
+