AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
fix(mapping): semantic_text is Text on the lexical side — match/BM25 survives flush
xerj-org198fbf88a16e
diff --git a/engine/crates/xerj-api/src/es_compat.rs b/engine/crates/xerj-api/src/es_compat.rsindex 27496643b..076805c08 100644--- a/engine/crates/xerj-api/src/es_compat.rs+++ b/engine/crates/xerj-api/src/es_compat.rs@@ -10894,7 +10894,16 @@ fn es_properties_to_fields(properties: &Value) -> Vec<FieldConfig> {fn es_type_to_native(es_type: &str) -> FieldType {match es_type {- "text" => FieldType::Text,+ // `semantic_text` carries plain text in `_source` (the derived+ // embedding lives in the companion `<field>_vector` field), so on+ // the LEXICAL side it must behave exactly like `text`: schema type+ // Text puts it in the search path's `text_fields` set and gives it+ // the standard analyzer + positions in the segment FTS built at+ // flush. It used to fall through to `Object`, which FTS-indexes+ // the whole value as ONE keyword token — match/BM25 then found the+ // doc pre-flush (memtable scan) but ZERO hits post-flush (segment+ // FTS had no standard-analyzed tokens for the field).+ "text" | "semantic_text" => FieldType::Text,"keyword" | "constant_keyword" | "wildcard" => FieldType::Keyword,"long" | "integer" | "short" | "byte" | "unsigned_long" => FieldType::Long,"double" | "float" | "half_float" | "scaled_float" => FieldType::Double,diff --git a/engine/crates/xerj-engine/tests/integration.rs b/engine/crates/xerj-engine/tests/integration.rsindex 3c71ddc99..7f01e1de4 100644--- a/engine/crates/xerj-engine/tests/integration.rs+++ b/engine/crates/xerj-engine/tests/integration.rs@@ -4713,3 +4713,58 @@ async fn test_bulk_index_malformed_doc_rejected_per_item() {assert_eq!(all2.total.value, 1, "non-object body must not be stored");}+/// Blocker 4: match/BM25 over a `semantic_text` field returned hits from the+/// memtable but ZERO once flushed — the field's schema type (Object) gave it+/// the whole-value keyword analyzer in the segment FTS. The es_compat mapper+/// now types it Text (+ embedding config); this exercises exactly that shape.+#[tokio::test]+async fn test_semantic_text_match_survives_flush() {+ let dir = TempDir::new().unwrap();+ let engine = make_engine(&dir);++ // The schema shape es_compat produces for `"type": "semantic_text"`:+ // lexical side = Text (standard analyzer, positions), plus an embedding+ // config producing the companion `content_vector`.+ let mut schema = Schema::empty();+ let mut fc = FieldConfig::new("content", FieldType::Text);+ fc.options.dimensions = Some(16);+ fc.options.similarity = Some("cosine".to_string());+ fc.embedding = Some(xerj_common::types::EmbeddingConfig {+ endpoint: None,+ model: None,+ target_field: Some("content_vector".to_string()),+ });+ schema.fields.push(fc);+ engine.create_index("sem", schema).unwrap();+ let idx = engine.get_index("sem").unwrap();++ idx.index_document(+ Some("1".into()),+ json!({"content": "the quick brown fox jumps over the lazy dog"}),+ )+ .await+ .unwrap();++ let pre = idx+ .search(&make_search(json!({"match": {"content": "quick fox"}})))+ .await+ .unwrap();+ assert_eq!(pre.total.value, 1, "pre-flush match must hit");++ idx.flush().await.unwrap();++ let post = idx+ .search(&make_search(json!({"match": {"content": "quick fox"}})))+ .await+ .unwrap();+ assert_eq!(+ post.total.value, 1,+ "post-flush match must still hit (segment FTS must standard-analyze semantic_text)"+ );+ // The auto-embed side must be unaffected by the lexical type change.+ assert!(+ post.hits[0].source.get("content_vector").is_some(),+ "companion embedding vector missing from _source"+ );+}+