Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

feat(ai): auto-embed-on-ingest for semantic_text (zero-config built-in embedder)

xerj-org0687c0c8a70b
diff --git a/engine/crates/xerj-ai/src/lib.rs b/engine/crates/xerj-ai/src/lib.rs
index 68f545697..fed59df45 100644
--- a/engine/crates/xerj-ai/src/lib.rs
+++ b/engine/crates/xerj-ai/src/lib.rs
@@ -4,15 +4,18 @@
//!
//! Provides:
//! - [`embed`] — Embedding proxy: async HTTP client for OpenAI-compatible embedding APIs
+//! - [`local`] — Built-in zero-config deterministic text embedder (feature hashing)
//! - [`chunker`] — Text chunking with sentence-aware splitting and overlap
//! - [`memory`] — Agent memory: semantic dedup + recency-blended recall
pub mod chunker;
pub mod embed;
+pub mod local;
pub mod memory;
pub use chunker::{Chunk, TextChunker};
pub use embed::{EmbeddingProxy, EmbeddingProxyConfig};
+pub use local::{local_embed, DEFAULT_DIMS};
pub use memory::{AgentMemory, MemoryEntry};
pub use xerj_common::Result;
diff --git a/engine/crates/xerj-ai/src/local.rs b/engine/crates/xerj-ai/src/local.rs
new file mode 100644
index 000000000..2c1ee2745
--- /dev/null
+++ b/engine/crates/xerj-ai/src/local.rs
@@ -0,0 +1,158 @@
+//! Built-in, zero-config deterministic text embedder.
+//!
+//! [`local_embed`] turns arbitrary text into a fixed-dimensional, L2-normalised
+//! `f32` vector using the *feature-hashing trick* (a.k.a. the "hashing
+//! vectoriser"): word unigrams and intra-word character trigrams are hashed
+//! into a fixed number of buckets with a signed contribution, then the whole
+//! accumulator is L2-normalised so that cosine similarity is comparable across
+//! documents of different lengths.
+//!
+//! This is **not** a neural model — it captures lexical / sub-word overlap, not
+//! deep semantics. Its job is to make the `semantic_text` field type work
+//! end-to-end with **zero external dependencies** (offline, no API key), giving
+//! a sensible out-of-the-box experience: paraphrases that share vocabulary rank
+//! above unrelated text. When a real
+//! [`crate::embed::EmbeddingProxy`] is configured the engine uses that instead
+//! for production-quality embeddings — the *same* embedder is always used at
+//! ingest and query time so the vectors are comparable.
+//!
+//! Key properties:
+//! * **Deterministic** — identical input always yields the identical vector,
+//! across processes and restarts (no learned weights, no RNG).
+//! * **Configurable dimensionality** — `dims` (default [`DEFAULT_DIMS`]).
+//! * **Comparable** — L2-normalised, so cosine similarity is well-behaved.
+
+/// Default embedding dimensionality for the built-in embedder.
+///
+/// 384 mirrors the popular `all-MiniLM-L6-v2` output size, so switching from
+/// the built-in embedder to a real proxy of that width needs no mapping change.
+pub const DEFAULT_DIMS: usize = 384;
+
+/// 64-bit FNV-1a hash — small, fast, stable across platforms.
+#[inline]
+fn fnv1a64(bytes: &[u8]) -> u64 {
+ let mut h: u64 = 0xcbf29ce484222325;
+ for &b in bytes {
+ h ^= b as u64;
+ h = h.wrapping_mul(0x00000100000001b3);
+ }
+ h
+}
+
+/// Accumulate one hashed feature into `acc` with a signed weight. The low bits
+/// pick the bucket; one high bit picks the sign (signed hashing reduces the
+/// collision bias of unsigned feature hashing).
+#[inline]
+fn add_feature(acc: &mut [f32], token: &[u8], weight: f32) {
+ let dims = acc.len();
+ if dims == 0 {
+ return;
+ }
+ let h = fnv1a64(token);
+ let idx = (h % dims as u64) as usize;
+ let sign = if (h >> 63) & 1 == 0 { 1.0 } else { -1.0 };
+ acc[idx] += weight * sign;
+}
+
+/// Embed `text` into a deterministic, L2-normalised vector of length `dims`.
+///
+/// Empty / whitespace-only text (or `dims == 0`) yields an all-zero vector;
+/// cosine similarity against a zero vector is defined as 0 by
+/// `compute_vector_similarity`, so this degrades gracefully rather than
+/// panicking.
+pub fn local_embed(text: &str, dims: usize) -> Vec<f32> {
+ let mut acc = vec![0.0f32; dims];
+ if dims == 0 {
+ return acc;
+ }
+
+ let lower = text.to_lowercase();
+ for token in lower.split(|c: char| !c.is_alphanumeric()).filter(|t| !t.is_empty()) {
+ // Whole-word unigram — the dominant lexical signal.
+ add_feature(&mut acc, token.as_bytes(), 1.0);
+
+ // Intra-word character trigrams (padded so word boundaries matter),
+ // at a lower weight. These give partial credit to morphological
+ // variants (e.g. "run" / "running" share the "run" shingle), which
+ // nudges the embedding a little past exact-keyword matching.
+ let padded: Vec<char> = std::iter::once('#')
+ .chain(token.chars())
+ .chain(std::iter::once('#'))
+ .collect();
+ if padded.len() >= 3 {
+ for w in padded.windows(3) {
+ let mut buf = [0u8; 12];
+ let mut n = 0;
+ for &ch in w {
+ let s = ch.len_utf8();
+ ch.encode_utf8(&mut buf[n..]);
+ n += s;
+ }
+ add_feature(&mut acc, &buf[..n], 0.35);
+ }
+ }
+ }
+
+ // L2-normalise so cosine similarity is length-invariant.
+ let norm: f32 = acc.iter().map(|x| x * x).sum::<f32>().sqrt();
+ if norm > 0.0 {
+ for x in acc.iter_mut() {
+ *x /= norm;
+ }
+ }
+ acc
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ fn dot(a: &[f32], b: &[f32]) -> f32 {
+ a.iter().zip(b).map(|(x, y)| x * y).sum()
+ }
+
+ #[test]
+ fn dims_and_determinism() {
+ let a = local_embed("the quick brown fox", 128);
+ let b = local_embed("the quick brown fox", 128);
+ assert_eq!(a.len(), 128);
+ assert_eq!(a, b, "embedding must be deterministic");
+ }
+
+ #[test]
+ fn l2_normalised() {
+ let v = local_embed("hello world of vectors", DEFAULT_DIMS);
+ let norm: f32 = v.iter().map(|x| x * x).sum::<f32>().sqrt();
+ assert!((norm - 1.0).abs() < 1e-4, "expected unit norm, got {norm}");
+ }
+
+ #[test]
+ fn empty_text_is_zero_vector() {
+ let v = local_embed(" ", 64);
+ assert_eq!(v.len(), 64);
+ assert!(v.iter().all(|x| *x == 0.0));
+ }
+
+ #[test]
+ fn zero_dims_is_empty() {
+ assert!(local_embed("anything", 0).is_empty());
+ }
+
+ #[test]
+ fn paraphrase_ranks_above_unrelated() {
+ // A paraphrase sharing vocabulary must be closer (higher cosine) than
+ // a topically-unrelated sentence — the property `semantic_text` relies
+ // on for kNN retrieval.
+ let dims = DEFAULT_DIMS;
+ let doc_relevant = local_embed("a hungry dog chased the ball across the green park", dims);
+ let doc_unrelated = local_embed("investors sold technology shares amid rising interest rates", dims);
+ let query = local_embed("a dog ran after a ball in the park", dims);
+
+ let sim_relevant = dot(&query, &doc_relevant);
+ let sim_unrelated = dot(&query, &doc_unrelated);
+ assert!(
+ sim_relevant > sim_unrelated,
+ "relevant {sim_relevant} should beat unrelated {sim_unrelated}"
+ );
+ }
+}
diff --git a/engine/crates/xerj-api/src/es_compat.rs b/engine/crates/xerj-api/src/es_compat.rs
index 46b556be2..dff0b07ed 100644
--- a/engine/crates/xerj-api/src/es_compat.rs
+++ b/engine/crates/xerj-api/src/es_compat.rs
@@ -9558,6 +9558,48 @@ fn es_properties_to_fields(properties: &Value) -> Vec<FieldConfig> {
fc.options.null_value = Some(Value::String(format!("__copy_to__:{}", copy_target)));
}
+ // `semantic_text` — auto-embed on ingest. Attach an EmbeddingConfig so
+ // the engine vectorises this field's text into a companion vector field
+ // (`<field>_vector`) at index time, making it kNN-searchable via the
+ // `semantic` query with zero external configuration (built-in embedder)
+ // or via the configured EmbeddingProxy when present. The native type
+ // stays `Object` (the raw text still lives in `_source`); the derived
+ // vector field carries the searchable embedding.
+ if es_type == "semantic_text" {
+ // Dimensionality for the built-in embedder — honour an explicit
+ // `dimensions`/`dims` override, else the built-in default (384).
+ // Default 384 mirrors xerj_ai::local::DEFAULT_DIMS (the built-in
+ // embedder width); kept as a literal here to avoid a build-graph
+ // dependency from xerj-api onto xerj-ai.
+ let dims = field_def
+ .get("dimensions")
+ .or_else(|| field_def.get("dims"))
+ .and_then(Value::as_u64)
+ .map(|d| d as usize)
+ .unwrap_or(384);
+ fc.options.dimensions = Some(dims);
+ // Built-in embeddings are L2-normalised → cosine similarity.
+ fc.options.similarity = Some("cosine".to_string());
+ let target_field = format!("{field_name}_vector");
+ // Allow a mapping to point at an external inference endpoint/model
+ // (`inference_id` mirrors ES 8.x semantic_text wiring); when absent
+ // the engine falls back to the zero-config built-in embedder.
+ let endpoint = field_def
+ .get("inference_endpoint")
+ .and_then(Value::as_str)
+ .map(str::to_string);
+ let model = field_def
+ .get("inference_id")
+ .or_else(|| field_def.get("model_id"))
+ .and_then(Value::as_str)
+ .map(str::to_string);
+ fc.embedding = Some(xerj_common::types::EmbeddingConfig {
+ endpoint,
+ model,
+ target_field: Some(target_field),
+ });
+ }
+
// Propagate dense_vector config into FieldOptions so the search
// path can look up similarity + dims without re-parsing the raw
// mapping.
diff --git a/engine/crates/xerj-engine/src/index.rs b/engine/crates/xerj-engine/src/index.rs
index cf95c2294..6b7417071 100644
--- a/engine/crates/xerj-engine/src/index.rs
+++ b/engine/crates/xerj-engine/src/index.rs
@@ -987,6 +987,12 @@ impl Index {
}
}
+ // Auto-embed `semantic_text` fields: vectorise the field's text into
+ // its companion vector field (`<field>_vector`) so it becomes
+ // kNN-searchable. Runs before copy_to / storage so the derived vector
+ // is part of `_source` and gets picked up by HNSW indexing below.
+ let source = self.apply_semantic_embeddings(source).await?;
+
// Apply copy_to: expand the source by copying field values to their target fields.
let source = {
let schema_guard = self.schema.read().await;
@@ -3355,43 +3361,144 @@ impl Index {
k: usize,
filter: Option<Box<QueryNode>>,
) -> Result<SearchResult> {
- let proxy = match &*self.embedding_proxy {
- Some(p) => p,
- None => {
- return Err(EngineError::Common(
- xerj_common::XerjError::invalid_query(
- "semantic query requires `embedding.default_endpoint` to be \
- configured (currently empty). Set Config.embedding.default_endpoint \
- and Config.embedding.default_model to an OpenAI-compatible \
- /v1/embeddings endpoint.",
- ),
- ));
+ // Resolve whether `field` is a `semantic_text` field (has an embedding
+ // config). If so, the vector lives in the companion `target_field` and
+ // was produced by the same embedder used at ingest — so we must embed
+ // the query with that same embedder and kNN against `target_field`.
+ let (knn_field, dims, is_semantic_text, similarity) = {
+ let schema = self.schema.read().await;
+ match schema.schema.field(field) {
+ Some(fc) if fc.embedding.is_some() => {
+ let emb = fc.embedding.as_ref().unwrap();
+ let target = emb
+ .target_field
+ .clone()
+ .unwrap_or_else(|| format!("{field}_vector"));
+ let dims = fc.options.dimensions.unwrap_or(xerj_ai::local::DEFAULT_DIMS);
+ let sim = fc
+ .options
+ .similarity
+ .clone()
+ .unwrap_or_else(|| "cosine".to_string());
+ (target, dims, true, sim)
+ }
+ _ => (
+ field.to_string(),
+ xerj_ai::local::DEFAULT_DIMS,
+ false,
+ lookup_vector_similarity(&schema.schema, field),
+ ),
}
};
- // Embed the query text. `embed_batch` takes Vec<String>; we
- // only have one text but the proxy's batching keeps the
- // wire format stable for callers.
- let mut vectors = proxy
- .embed_batch(vec![text.to_string()])
- .await
- .map_err(|e| EngineError::Common(xerj_common::XerjError::invalid_query(
- format!("semantic embed failed: {e}"),
- )))?;
- let query_vec = match vectors.pop() {
- Some(v) if !v.is_empty() => v,
- _ => {
- return Err(EngineError::Common(
- xerj_common::XerjError::invalid_query(
- "embedding proxy returned no vector for semantic query",
- ),
- ));
+
+ // Embed the query text with the effective embedder:
+ // * EmbeddingProxy when configured (high-quality, same as ingest),
+ // * else the built-in deterministic embedder — but ONLY for
+ // `semantic_text` fields, which were embedded that same way at
+ // ingest. A `semantic` query against a plain dense_vector field
+ // with no proxy configured still returns the original 400 (there
+ // is no comparable stored vector to match against).
+ let query_vec = if let Some(proxy) = &*self.embedding_proxy {
+ // `embed_batch` takes Vec<String>; we only have one text but the
+ // proxy's batching keeps the wire format stable for callers.
+ let mut vectors = proxy
+ .embed_batch(vec![text.to_string()])
+ .await
+ .map_err(|e| EngineError::Common(xerj_common::XerjError::invalid_query(
+ format!("semantic embed failed: {e}"),
+ )))?;
+ match vectors.pop() {
+ Some(v) if !v.is_empty() => v,
+ _ => {
+ return Err(EngineError::Common(
+ xerj_common::XerjError::invalid_query(
+ "embedding proxy returned no vector for semantic query",
+ ),
+ ));
+ }
}
+ } else if is_semantic_text {
+ xerj_ai::local::local_embed(text, dims)
+ } else {
+ return Err(EngineError::Common(
+ xerj_common::XerjError::invalid_query(
+ "semantic query requires either a `semantic_text` field (auto-embedded \
+ with the built-in embedder) or `embedding.default_endpoint` configured \
+ to an OpenAI-compatible /v1/embeddings endpoint.",
+ ),
+ ));
};
- let similarity = {
+
+ self.run_knn_brute_force(request, &knn_field, &query_vec, k, filter, &similarity).await
+ }
+
+ /// Auto-embed `semantic_text` fields on ingest.
+ ///
+ /// For each schema field carrying an [`EmbeddingConfig`], reads the field's
+ /// text from `source` and writes an embedding into the config's
+ /// `target_field` (default `<field>_vector`). Uses the configured
+ /// [`EmbeddingProxy`] when present, else the zero-config built-in
+ /// deterministic embedder — the *same* choice made at query time so the
+ /// vectors are comparable. Fields where the target already holds a value
+ /// (caller pre-embedded) or whose value is not a string are left untouched.
+ async fn apply_semantic_embeddings(&self, mut source: Value) -> Result<Value> {
+ // Collect (field, target_field, dims) specs without holding the schema
+ // lock across the (possibly async) embedding calls.
+ let specs: Vec<(String, String, usize)> = {
let schema = self.schema.read().await;
- lookup_vector_similarity(&schema.schema, field)
+ schema
+ .schema
+ .fields
+ .iter()
+ .filter_map(|fc| {
+ fc.embedding.as_ref().map(|emb| {
+ let target = emb
+ .target_field
+ .clone()
+ .unwrap_or_else(|| format!("{}_vector", fc.name));
+ let dims =
+ fc.options.dimensions.unwrap_or(xerj_ai::local::DEFAULT_DIMS);
+ (fc.name.clone(), target, dims)
+ })
+ })
+ .collect()
};
- self.run_knn_brute_force(request, field, &query_vec, k, filter, &similarity).await
+ if specs.is_empty() {
+ return Ok(source);
+ }
+ let Some(obj) = source.as_object_mut() else {
+ return Ok(source);
+ };
+ for (field, target, dims) in specs {
+ // Respect a caller-supplied vector (e.g. pre-computed offline).
+ if obj.get(&target).map(|v| !v.is_null()).unwrap_or(false) {
+ continue;
+ }
+ let Some(text) = obj.get(&field).and_then(Value::as_str) else {
+ continue;
+ };
+ let text = text.to_string();
+ let vec = if let Some(proxy) = &*self.embedding_proxy {
+ let mut vs = proxy
+ .embed_batch(vec![text])
+ .await
+ .map_err(|e| EngineError::Common(xerj_common::XerjError::invalid_query(
+ format!("semantic_text embed failed for field [{field}]: {e}"),
+ )))?;
+ match vs.pop() {
+ Some(v) if !v.is_empty() => v,
+ _ => continue,
+ }
+ } else {
+ xerj_ai::local::local_embed(&text, dims)
+ };
+ let arr: Vec<Value> = vec
+ .into_iter()
+ .map(|f| Value::from(f as f64))
+ .collect();
+ obj.insert(target, Value::Array(arr));
+ }
+ Ok(source)
}
/// Hybrid (multi-query + fusion) executor. Recursively runs each