AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
feat(ai): auto-embed-on-ingest for semantic_text (zero-config built-in embedder)
xerj-org0687c0c8a70b
diff --git a/engine/crates/xerj-ai/src/lib.rs b/engine/crates/xerj-ai/src/lib.rsindex 68f545697..fed59df45 100644--- a/engine/crates/xerj-ai/src/lib.rs+++ b/engine/crates/xerj-ai/src/lib.rs@@ -4,15 +4,18 @@//!//! Provides://! - [`embed`] — Embedding proxy: async HTTP client for OpenAI-compatible embedding APIs+//! - [`local`] — Built-in zero-config deterministic text embedder (feature hashing)//! - [`chunker`] — Text chunking with sentence-aware splitting and overlap//! - [`memory`] — Agent memory: semantic dedup + recency-blended recallpub mod chunker;pub mod embed;+pub mod local;pub mod memory;pub use chunker::{Chunk, TextChunker};pub use embed::{EmbeddingProxy, EmbeddingProxyConfig};+pub use local::{local_embed, DEFAULT_DIMS};pub use memory::{AgentMemory, MemoryEntry};pub use xerj_common::Result;diff --git a/engine/crates/xerj-ai/src/local.rs b/engine/crates/xerj-ai/src/local.rsnew file mode 100644index 000000000..2c1ee2745--- /dev/null+++ b/engine/crates/xerj-ai/src/local.rs@@ -0,0 +1,158 @@+//! Built-in, zero-config deterministic text embedder.+//!+//! [`local_embed`] turns arbitrary text into a fixed-dimensional, L2-normalised+//! `f32` vector using the *feature-hashing trick* (a.k.a. the "hashing+//! vectoriser"): word unigrams and intra-word character trigrams are hashed+//! into a fixed number of buckets with a signed contribution, then the whole+//! accumulator is L2-normalised so that cosine similarity is comparable across+//! documents of different lengths.+//!+//! This is **not** a neural model — it captures lexical / sub-word overlap, not+//! deep semantics. Its job is to make the `semantic_text` field type work+//! end-to-end with **zero external dependencies** (offline, no API key), giving+//! a sensible out-of-the-box experience: paraphrases that share vocabulary rank+//! above unrelated text. When a real+//! [`crate::embed::EmbeddingProxy`] is configured the engine uses that instead+//! for production-quality embeddings — the *same* embedder is always used at+//! ingest and query time so the vectors are comparable.+//!+//! Key properties:+//! * **Deterministic** — identical input always yields the identical vector,+//! across processes and restarts (no learned weights, no RNG).+//! * **Configurable dimensionality** — `dims` (default [`DEFAULT_DIMS`]).+//! * **Comparable** — L2-normalised, so cosine similarity is well-behaved.++/// Default embedding dimensionality for the built-in embedder.+///+/// 384 mirrors the popular `all-MiniLM-L6-v2` output size, so switching from+/// the built-in embedder to a real proxy of that width needs no mapping change.+pub const DEFAULT_DIMS: usize = 384;++/// 64-bit FNV-1a hash — small, fast, stable across platforms.+#[inline]+fn fnv1a64(bytes: &[u8]) -> u64 {+ let mut h: u64 = 0xcbf29ce484222325;+ for &b in bytes {+ h ^= b as u64;+ h = h.wrapping_mul(0x00000100000001b3);+ }+ h+}++/// Accumulate one hashed feature into `acc` with a signed weight. The low bits+/// pick the bucket; one high bit picks the sign (signed hashing reduces the+/// collision bias of unsigned feature hashing).+#[inline]+fn add_feature(acc: &mut [f32], token: &[u8], weight: f32) {+ let dims = acc.len();+ if dims == 0 {+ return;+ }+ let h = fnv1a64(token);+ let idx = (h % dims as u64) as usize;+ let sign = if (h >> 63) & 1 == 0 { 1.0 } else { -1.0 };+ acc[idx] += weight * sign;+}++/// Embed `text` into a deterministic, L2-normalised vector of length `dims`.+///+/// Empty / whitespace-only text (or `dims == 0`) yields an all-zero vector;+/// cosine similarity against a zero vector is defined as 0 by+/// `compute_vector_similarity`, so this degrades gracefully rather than+/// panicking.+pub fn local_embed(text: &str, dims: usize) -> Vec<f32> {+ let mut acc = vec![0.0f32; dims];+ if dims == 0 {+ return acc;+ }++ let lower = text.to_lowercase();+ for token in lower.split(|c: char| !c.is_alphanumeric()).filter(|t| !t.is_empty()) {+ // Whole-word unigram — the dominant lexical signal.+ add_feature(&mut acc, token.as_bytes(), 1.0);++ // Intra-word character trigrams (padded so word boundaries matter),+ // at a lower weight. These give partial credit to morphological+ // variants (e.g. "run" / "running" share the "run" shingle), which+ // nudges the embedding a little past exact-keyword matching.+ let padded: Vec<char> = std::iter::once('#')+ .chain(token.chars())+ .chain(std::iter::once('#'))+ .collect();+ if padded.len() >= 3 {+ for w in padded.windows(3) {+ let mut buf = [0u8; 12];+ let mut n = 0;+ for &ch in w {+ let s = ch.len_utf8();+ ch.encode_utf8(&mut buf[n..]);+ n += s;+ }+ add_feature(&mut acc, &buf[..n], 0.35);+ }+ }+ }++ // L2-normalise so cosine similarity is length-invariant.+ let norm: f32 = acc.iter().map(|x| x * x).sum::<f32>().sqrt();+ if norm > 0.0 {+ for x in acc.iter_mut() {+ *x /= norm;+ }+ }+ acc+}++#[cfg(test)]+mod tests {+ use super::*;++ fn dot(a: &[f32], b: &[f32]) -> f32 {+ a.iter().zip(b).map(|(x, y)| x * y).sum()+ }++ #[test]+ fn dims_and_determinism() {+ let a = local_embed("the quick brown fox", 128);+ let b = local_embed("the quick brown fox", 128);+ assert_eq!(a.len(), 128);+ assert_eq!(a, b, "embedding must be deterministic");+ }++ #[test]+ fn l2_normalised() {+ let v = local_embed("hello world of vectors", DEFAULT_DIMS);+ let norm: f32 = v.iter().map(|x| x * x).sum::<f32>().sqrt();+ assert!((norm - 1.0).abs() < 1e-4, "expected unit norm, got {norm}");+ }++ #[test]+ fn empty_text_is_zero_vector() {+ let v = local_embed(" ", 64);+ assert_eq!(v.len(), 64);+ assert!(v.iter().all(|x| *x == 0.0));+ }++ #[test]+ fn zero_dims_is_empty() {+ assert!(local_embed("anything", 0).is_empty());+ }++ #[test]+ fn paraphrase_ranks_above_unrelated() {+ // A paraphrase sharing vocabulary must be closer (higher cosine) than+ // a topically-unrelated sentence — the property `semantic_text` relies+ // on for kNN retrieval.+ let dims = DEFAULT_DIMS;+ let doc_relevant = local_embed("a hungry dog chased the ball across the green park", dims);+ let doc_unrelated = local_embed("investors sold technology shares amid rising interest rates", dims);+ let query = local_embed("a dog ran after a ball in the park", dims);++ let sim_relevant = dot(&query, &doc_relevant);+ let sim_unrelated = dot(&query, &doc_unrelated);+ assert!(+ sim_relevant > sim_unrelated,+ "relevant {sim_relevant} should beat unrelated {sim_unrelated}"+ );+ }+}diff --git a/engine/crates/xerj-api/src/es_compat.rs b/engine/crates/xerj-api/src/es_compat.rsindex 46b556be2..dff0b07ed 100644--- a/engine/crates/xerj-api/src/es_compat.rs+++ b/engine/crates/xerj-api/src/es_compat.rs@@ -9558,6 +9558,48 @@ fn es_properties_to_fields(properties: &Value) -> Vec<FieldConfig> {fc.options.null_value = Some(Value::String(format!("__copy_to__:{}", copy_target)));}+ // `semantic_text` — auto-embed on ingest. Attach an EmbeddingConfig so+ // the engine vectorises this field's text into a companion vector field+ // (`<field>_vector`) at index time, making it kNN-searchable via the+ // `semantic` query with zero external configuration (built-in embedder)+ // or via the configured EmbeddingProxy when present. The native type+ // stays `Object` (the raw text still lives in `_source`); the derived+ // vector field carries the searchable embedding.+ if es_type == "semantic_text" {+ // Dimensionality for the built-in embedder — honour an explicit+ // `dimensions`/`dims` override, else the built-in default (384).+ // Default 384 mirrors xerj_ai::local::DEFAULT_DIMS (the built-in+ // embedder width); kept as a literal here to avoid a build-graph+ // dependency from xerj-api onto xerj-ai.+ let dims = field_def+ .get("dimensions")+ .or_else(|| field_def.get("dims"))+ .and_then(Value::as_u64)+ .map(|d| d as usize)+ .unwrap_or(384);+ fc.options.dimensions = Some(dims);+ // Built-in embeddings are L2-normalised → cosine similarity.+ fc.options.similarity = Some("cosine".to_string());+ let target_field = format!("{field_name}_vector");+ // Allow a mapping to point at an external inference endpoint/model+ // (`inference_id` mirrors ES 8.x semantic_text wiring); when absent+ // the engine falls back to the zero-config built-in embedder.+ let endpoint = field_def+ .get("inference_endpoint")+ .and_then(Value::as_str)+ .map(str::to_string);+ let model = field_def+ .get("inference_id")+ .or_else(|| field_def.get("model_id"))+ .and_then(Value::as_str)+ .map(str::to_string);+ fc.embedding = Some(xerj_common::types::EmbeddingConfig {+ endpoint,+ model,+ target_field: Some(target_field),+ });+ }+// Propagate dense_vector config into FieldOptions so the search// path can look up similarity + dims without re-parsing the raw// mapping.diff --git a/engine/crates/xerj-engine/src/index.rs b/engine/crates/xerj-engine/src/index.rsindex cf95c2294..6b7417071 100644--- a/engine/crates/xerj-engine/src/index.rs+++ b/engine/crates/xerj-engine/src/index.rs@@ -987,6 +987,12 @@ impl Index {}}+ // Auto-embed `semantic_text` fields: vectorise the field's text into+ // its companion vector field (`<field>_vector`) so it becomes+ // kNN-searchable. Runs before copy_to / storage so the derived vector+ // is part of `_source` and gets picked up by HNSW indexing below.+ let source = self.apply_semantic_embeddings(source).await?;+// Apply copy_to: expand the source by copying field values to their target fields.let source = {let schema_guard = self.schema.read().await;@@ -3355,43 +3361,144 @@ impl Index {k: usize,filter: Option<Box<QueryNode>>,) -> Result<SearchResult> {- let proxy = match &*self.embedding_proxy {- Some(p) => p,- None => {- return Err(EngineError::Common(- xerj_common::XerjError::invalid_query(- "semantic query requires `embedding.default_endpoint` to be \- configured (currently empty). Set Config.embedding.default_endpoint \- and Config.embedding.default_model to an OpenAI-compatible \- /v1/embeddings endpoint.",- ),- ));+ // Resolve whether `field` is a `semantic_text` field (has an embedding+ // config). If so, the vector lives in the companion `target_field` and+ // was produced by the same embedder used at ingest — so we must embed+ // the query with that same embedder and kNN against `target_field`.+ let (knn_field, dims, is_semantic_text, similarity) = {+ let schema = self.schema.read().await;+ match schema.schema.field(field) {+ Some(fc) if fc.embedding.is_some() => {+ let emb = fc.embedding.as_ref().unwrap();+ let target = emb+ .target_field+ .clone()+ .unwrap_or_else(|| format!("{field}_vector"));+ let dims = fc.options.dimensions.unwrap_or(xerj_ai::local::DEFAULT_DIMS);+ let sim = fc+ .options+ .similarity+ .clone()+ .unwrap_or_else(|| "cosine".to_string());+ (target, dims, true, sim)+ }+ _ => (+ field.to_string(),+ xerj_ai::local::DEFAULT_DIMS,+ false,+ lookup_vector_similarity(&schema.schema, field),+ ),}};- // Embed the query text. `embed_batch` takes Vec<String>; we- // only have one text but the proxy's batching keeps the- // wire format stable for callers.- let mut vectors = proxy- .embed_batch(vec![text.to_string()])- .await- .map_err(|e| EngineError::Common(xerj_common::XerjError::invalid_query(- format!("semantic embed failed: {e}"),- )))?;- let query_vec = match vectors.pop() {- Some(v) if !v.is_empty() => v,- _ => {- return Err(EngineError::Common(- xerj_common::XerjError::invalid_query(- "embedding proxy returned no vector for semantic query",- ),- ));++ // Embed the query text with the effective embedder:+ // * EmbeddingProxy when configured (high-quality, same as ingest),+ // * else the built-in deterministic embedder — but ONLY for+ // `semantic_text` fields, which were embedded that same way at+ // ingest. A `semantic` query against a plain dense_vector field+ // with no proxy configured still returns the original 400 (there+ // is no comparable stored vector to match against).+ let query_vec = if let Some(proxy) = &*self.embedding_proxy {+ // `embed_batch` takes Vec<String>; we only have one text but the+ // proxy's batching keeps the wire format stable for callers.+ let mut vectors = proxy+ .embed_batch(vec![text.to_string()])+ .await+ .map_err(|e| EngineError::Common(xerj_common::XerjError::invalid_query(+ format!("semantic embed failed: {e}"),+ )))?;+ match vectors.pop() {+ Some(v) if !v.is_empty() => v,+ _ => {+ return Err(EngineError::Common(+ xerj_common::XerjError::invalid_query(+ "embedding proxy returned no vector for semantic query",+ ),+ ));+ }}+ } else if is_semantic_text {+ xerj_ai::local::local_embed(text, dims)+ } else {+ return Err(EngineError::Common(+ xerj_common::XerjError::invalid_query(+ "semantic query requires either a `semantic_text` field (auto-embedded \+ with the built-in embedder) or `embedding.default_endpoint` configured \+ to an OpenAI-compatible /v1/embeddings endpoint.",+ ),+ ));};- let similarity = {++ self.run_knn_brute_force(request, &knn_field, &query_vec, k, filter, &similarity).await+ }++ /// Auto-embed `semantic_text` fields on ingest.+ ///+ /// For each schema field carrying an [`EmbeddingConfig`], reads the field's+ /// text from `source` and writes an embedding into the config's+ /// `target_field` (default `<field>_vector`). Uses the configured+ /// [`EmbeddingProxy`] when present, else the zero-config built-in+ /// deterministic embedder — the *same* choice made at query time so the+ /// vectors are comparable. Fields where the target already holds a value+ /// (caller pre-embedded) or whose value is not a string are left untouched.+ async fn apply_semantic_embeddings(&self, mut source: Value) -> Result<Value> {+ // Collect (field, target_field, dims) specs without holding the schema+ // lock across the (possibly async) embedding calls.+ let specs: Vec<(String, String, usize)> = {let schema = self.schema.read().await;- lookup_vector_similarity(&schema.schema, field)+ schema+ .schema+ .fields+ .iter()+ .filter_map(|fc| {+ fc.embedding.as_ref().map(|emb| {+ let target = emb+ .target_field+ .clone()+ .unwrap_or_else(|| format!("{}_vector", fc.name));+ let dims =+ fc.options.dimensions.unwrap_or(xerj_ai::local::DEFAULT_DIMS);+ (fc.name.clone(), target, dims)+ })+ })+ .collect()};- self.run_knn_brute_force(request, field, &query_vec, k, filter, &similarity).await+ if specs.is_empty() {+ return Ok(source);+ }+ let Some(obj) = source.as_object_mut() else {+ return Ok(source);+ };+ for (field, target, dims) in specs {+ // Respect a caller-supplied vector (e.g. pre-computed offline).+ if obj.get(&target).map(|v| !v.is_null()).unwrap_or(false) {+ continue;+ }+ let Some(text) = obj.get(&field).and_then(Value::as_str) else {+ continue;+ };+ let text = text.to_string();+ let vec = if let Some(proxy) = &*self.embedding_proxy {+ let mut vs = proxy+ .embed_batch(vec![text])+ .await+ .map_err(|e| EngineError::Common(xerj_common::XerjError::invalid_query(+ format!("semantic_text embed failed for field [{field}]: {e}"),+ )))?;+ match vs.pop() {+ Some(v) if !v.is_empty() => v,+ _ => continue,+ }+ } else {+ xerj_ai::local::local_embed(&text, dims)+ };+ let arr: Vec<Value> = vec+ .into_iter()+ .map(|f| Value::from(f as f64))+ .collect();+ obj.insert(target, Value::Array(arr));+ }+ Ok(source)}/// Hybrid (multi-query + fusion) executor. Recursively runs each