Solution for long term memory for agent coding CLIs and to facilitate handoff between different agent vendors
Rust
M9: Embedder trait + remote embedders + page_embeddings + hybrid RRF
AkitaOnRails7b075e45774b
diff --git a/Cargo.lock b/Cargo.lockindex 399dfc73..82cde316 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -174,6 +174,7 @@ name = "ai-memory-wiki"version = "0.1.0"dependencies = ["ai-memory-core",+ "ai-memory-llm","ai-memory-store","anyhow","git2",diff --git a/crates/ai-memory-cli/src/cli.rs b/crates/ai-memory-cli/src/cli.rsindex 0ff92668..0f4364b5 100644--- a/crates/ai-memory-cli/src/cli.rs+++ b/crates/ai-memory-cli/src/cli.rs@@ -55,6 +55,8 @@ pub enum Command {ForgetSweep(ForgetSweepArgs),/// Run the M8 lint pass (stale / duplicates + optional LLM contradiction).Lint(LintArgs),+ /// Compute + store embeddings for every latest page (M9).+ Embed(EmbedArgs),}/// Arguments for `init`.@@ -154,6 +156,24 @@ pub enum LlmProviderChoice {OpenaiCompat,}+/// Arguments for `embed`.+#[derive(Debug, Args)]+pub struct EmbedArgs {+ /// Report what would be embedded without actually mutating.+ #[arg(long)]+ pub dry_run: bool,+ /// Re-embed pages even when they already have a row with the+ /// currently-configured `(provider, model, dim)`.+ #[arg(long)]+ pub force: bool,+ /// Workspace name (auto-created if absent).+ #[arg(long, default_value = "default")]+ pub workspace: String,+ /// Project name within the workspace (auto-created if absent).+ #[arg(long, default_value = "scratch")]+ pub project: String,+}+/// Arguments for `forget-sweep`.#[derive(Debug, Args)]pub struct ForgetSweepArgs {diff --git a/crates/ai-memory-cli/src/commands/embed.rs b/crates/ai-memory-cli/src/commands/embed.rsnew file mode 100644index 00000000..3ae7be6d--- /dev/null+++ b/crates/ai-memory-cli/src/commands/embed.rs@@ -0,0 +1,120 @@+//! `ai-memory embed` — compute + store embeddings for every latest page.+//!+//! Used to backfill embeddings after attaching an embedder for the+//! first time, or after switching `(provider, model, dim)`.++use ai_memory_llm::{build_embedder, embedder_from_env};+use ai_memory_store::{Store, f32_vec_to_bytes};+use ai_memory_wiki::Wiki;+use anyhow::{Context, Result, bail};+use tracing::{info, warn};++use crate::cli::EmbedArgs;+use crate::config::Config;++/// Run the `embed` subcommand.+///+/// # Errors+/// Returns an error if no embedder is configured, the store cannot+/// be opened, or any embed/store call fails.+pub async fn run(config: &Config, args: EmbedArgs) -> Result<()> {+ let Some(cfg) = embedder_from_env()? else {+ bail!(+ "AI_MEMORY_EMBEDDING_PROVIDER is unset. Configure an embedder \+ (see CLAUDE.md / docs/design-decisions.md §4) and re-run."+ );+ };+ let provider = cfg.provider.name().to_string();+ let model = cfg.model.clone();+ let dim = cfg.dim;+ let embedder = build_embedder(cfg).context("building embedder")?;++ let store = Store::open(&config.data_dir)+ .with_context(|| format!("opening store at {}", config.data_dir.display()))?;+ let ws = store+ .writer+ .get_or_create_workspace(args.workspace.clone())+ .await?;+ let proj = store+ .writer+ .get_or_create_project(ws, args.project.clone(), None)+ .await?;+ let wiki = Wiki::new(&config.data_dir, store.writer.clone())?;++ // Snapshot all current pages.+ let candidates = store.reader.decay_candidates(ws, proj).await?;+ info!(+ provider = embedder.provider(),+ model = embedder.model(),+ dim = embedder.dim(),+ candidates = candidates.len(),+ "starting embed backfill",+ );++ // Optimisation: when --force is off, skip pages that already have+ // a matching embedding row.+ let already: std::collections::HashSet<_> = if args.force {+ std::collections::HashSet::new()+ } else {+ store+ .reader+ .load_embeddings(ws, proj, provider.clone(), model.clone(), dim)+ .await?+ .into_iter()+ .map(|s| s.id)+ .collect()+ };++ let mut embedded = 0_usize;+ let mut skipped = 0_usize;+ let mut failed = 0_usize;+ for cand in candidates {+ if !args.force && already.contains(&cand.id) {+ skipped += 1;+ continue;+ }+ let md = match wiki.read_page(&cand.path) {+ Ok(m) => m,+ Err(e) => {+ warn!(path = %cand.path, error = %e, "skip: unable to read page");+ failed += 1;+ continue;+ }+ };+ if args.dry_run {+ embedded += 1;+ continue;+ }+ let vec = match embedder.embed(&md.body).await {+ Ok(v) => v,+ Err(e) => {+ warn!(path = %cand.path, error = %e, "embed call failed");+ failed += 1;+ continue;+ }+ };+ let bytes = f32_vec_to_bytes(&vec);+ if let Err(e) = store+ .writer+ .store_embedding(cand.id, bytes, provider.clone(), model.clone(), dim)+ .await+ {+ warn!(path = %cand.path, error = %e, "store_embedding failed");+ failed += 1;+ continue;+ }+ embedded += 1;+ }++ let report = serde_json::json!({+ "dry_run": args.dry_run,+ "provider": provider,+ "model": model,+ "dim": dim,+ "embedded": embedded,+ "skipped": skipped,+ "failed": failed,+ });+ println!("{}", serde_json::to_string_pretty(&report)?);+ Ok(())+}diff --git a/crates/ai-memory-cli/src/commands/mod.rs b/crates/ai-memory-cli/src/commands/mod.rsindex 882d9b9c..bcad8456 100644--- a/crates/ai-memory-cli/src/commands/mod.rs+++ b/crates/ai-memory-cli/src/commands/mod.rs@@ -2,6 +2,7 @@pub mod backup;pub mod commit;+pub mod embed;pub mod forget_sweep;pub mod init;pub mod install_hooks;diff --git a/crates/ai-memory-cli/src/commands/serve.rs b/crates/ai-memory-cli/src/commands/serve.rsindex b9659c2a..9a28ad63 100644--- a/crates/ai-memory-cli/src/commands/serve.rs+++ b/crates/ai-memory-cli/src/commands/serve.rs@@ -1,7 +1,7 @@//! `ai-memory serve` — MCP server with optional filesystem watcher.use ai_memory_hooks::{HookState, hook_router};-use ai_memory_llm::provider_from_env;+use ai_memory_llm::{build_embedder, embedder_from_env, provider_from_env};use ai_memory_mcp::AiMemoryServer;use ai_memory_store::Store;use ai_memory_wiki::{WatcherHandle, Wiki};@@ -33,7 +33,35 @@ pub async fn run(config: &Config, args: ServeArgs) -> Result<()> {.writer.get_or_create_project(ws, args.project.clone(), None).await?;- let wiki = Wiki::new(&config.data_dir, store.writer.clone())?;+ let mut wiki = Wiki::new(&config.data_dir, store.writer.clone())?;++ // M9 — pluggable embedder. Refuse to start if any stored+ // embeddings disagree with the configured (provider, model, dim).+ let embedder = if let Some(cfg) = embedder_from_env()? {+ let mismatch = store+ .reader+ .embedding_meta_for_mismatch(cfg.provider.name().into(), cfg.model.clone(), cfg.dim)+ .await?;+ if !mismatch.is_empty() {+ anyhow::bail!(+ "embedding (provider, model, dim) mismatch with stored data: {:?} \+ — run `ai-memory embed --reembed` to migrate",+ mismatch+ );+ }+ let e = build_embedder(cfg).context("building embedder from env")?;+ info!(+ provider = e.provider(),+ model = e.model(),+ dim = e.dim(),+ "embedder enabled"+ );+ wiki = wiki.with_embedder(e.clone());+ Some(e)+ } else {+ info!("AI_MEMORY_EMBEDDING_PROVIDER unset; hybrid search disabled (FTS5-only)");+ None+ };// Keep the guard alive for the lifetime of `serve`.let _watcher = if args.no_watcher {@@ -52,6 +80,9 @@ pub async fn run(config: &Config, args: ServeArgs) -> Result<()> {let mut server = AiMemoryServer::new(store.reader.clone(), store.writer.clone(), ws, proj).with_wiki(wiki.clone()).with_decay_params(config.decay);+ if let Some(e) = embedder.clone() {+ server = server.with_embedder(e);+ }if let Some(cfg) = provider_from_env()? {let llm = ai_memory_llm::build_provider(cfg).context("building LLM provider from env")?;info!(diff --git a/crates/ai-memory-cli/src/main.rs b/crates/ai-memory-cli/src/main.rsindex 8954ca67..5d53107d 100644--- a/crates/ai-memory-cli/src/main.rs+++ b/crates/ai-memory-cli/src/main.rs@@ -51,5 +51,6 @@ async fn main() -> Result<()> {Command::LlmTest(args) => commands::llm_test::run(&config, args).await,Command::ForgetSweep(args) => commands::forget_sweep::run(&config, args).await,Command::Lint(args) => commands::lint::run(&config, args).await,+ Command::Embed(args) => commands::embed::run(&config, args).await,}}diff --git a/crates/ai-memory-consolidate/tests/embeddings.rs b/crates/ai-memory-consolidate/tests/embeddings.rsnew file mode 100644index 00000000..67bf259c--- /dev/null+++ b/crates/ai-memory-consolidate/tests/embeddings.rs@@ -0,0 +1,200 @@+//! M9 vector-retrieval integration test.+//!+//! Demonstrates that:+//! 1. Writing a page through `Wiki::write_page` with an embedder+//! attached actually persists an embedding row.+//! 2. The hybrid RRF path returns sensible results.+//! 3. The refuse-on-mismatch check flags heterogeneous+//! `(provider, model, dim)` triples.+//!+//! The test uses [`SyntheticEmbedder`] — a deterministic bag-of-words+//! embedder shipped specifically for tests like this so no API key is+//! needed. It doesn't capture real semantic similarity (a different+//! conjugation hashes to a different dimension), but it exercises the+//! full plumbing end-to-end.++use std::sync::Arc;++use ai_memory_core::{PagePath, Tier};+use ai_memory_llm::{Embedder, SyntheticEmbedder};+use ai_memory_store::Store;+use ai_memory_wiki::{Wiki, WritePageRequest};+use tempfile::TempDir;++#[tokio::test]+async fn m9_embeddings_roundtrip_via_synthetic() {+ let tmp = TempDir::new().expect("tempdir");+ let store = Store::open(tmp.path()).expect("open store");+ let ws = store+ .writer+ .get_or_create_workspace("default")+ .await+ .expect("ws");+ let proj = store+ .writer+ .get_or_create_project(ws, "scratch", None)+ .await+ .expect("proj");+ let embedder: Arc<dyn Embedder> = Arc::new(SyntheticEmbedder::new(64));+ let wiki = Wiki::new(tmp.path(), store.writer.clone())+ .expect("wiki")+ .with_embedder(embedder.clone());++ let pages = [+ (+ "notes/karpathy.md",+ "Karpathy LLM Wiki: compile not retrieve, the artifact is the wiki",+ ),+ (+ "notes/writer.md",+ "writer actor uses an mpsc bounded channel for backpressure",+ ),+ (+ "notes/k8s.md",+ "kubernetes pod scheduling rules and node selectors",+ ),+ ];++ for (path, body) in pages {+ wiki.write_page(WritePageRequest {+ workspace_id: ws,+ project_id: proj,+ path: PagePath::new(path.to_string()).expect("path"),+ frontmatter: serde_json::json!({"title": path}),+ body: body.to_string(),+ tier: Tier::Semantic,+ pinned: false,+ title: None,+ })+ .await+ .expect("write page");+ }++ // 1. Embedding rows landed for each page.+ let stored = store+ .reader+ .load_embeddings(ws, proj, "synthetic".into(), "bag-of-words-v1".into(), 64)+ .await+ .expect("load embeddings");+ assert_eq!(+ stored.len(),+ 3,+ "every page with an embedder attached should get an embedding row, got {}",+ stored.len(),+ );++ // Stored vectors are unit-normalised (Embedder contract).+ for s in &stored {+ let norm: f32 = s.vector.iter().map(|x| x * x).sum::<f32>().sqrt();+ assert!(+ (norm - 1.0).abs() < 1e-4,+ "stored vector should be unit-normalised, got |v|={norm}",+ );+ }++ // 2. Refuse-on-mismatch reports stale triples.+ let mismatch_with_wrong_dim = store+ .reader+ .embedding_meta_for_mismatch(+ "synthetic".into(),+ "bag-of-words-v1".into(),+ 128, // wrong dim+ )+ .await+ .expect("mismatch query");+ assert_eq!(+ mismatch_with_wrong_dim.len(),+ 1,+ "querying with a different dim should surface the existing one as mismatch",+ );+ let (provider, model, existing_dim, count) = &mismatch_with_wrong_dim[0];+ assert_eq!(provider, "synthetic");+ assert_eq!(model, "bag-of-words-v1");+ assert_eq!(*existing_dim, 64);+ assert_eq!(*count, 3);++ let mismatch_clean = store+ .reader+ .embedding_meta_for_mismatch("synthetic".into(), "bag-of-words-v1".into(), 64)+ .await+ .expect("mismatch query (clean)");+ assert!(+ mismatch_clean.is_empty(),+ "matching (provider, model, dim) should report no mismatch, got {mismatch_clean:?}",+ );++ // 3. Hybrid search end-to-end.+ let q = "karpathy compile artifact";+ let qv = embedder.embed(q).await.expect("embed query");+ let hybrid_hits = store+ .reader+ .hybrid_search(+ ws,+ proj,+ q.to_string(),+ Some(qv),+ "synthetic".into(),+ "bag-of-words-v1".into(),+ 64,+ 5,+ )+ .await+ .expect("hybrid search");+ assert!(+ !hybrid_hits.is_empty(),+ "hybrid search should return at least one hit for {q:?}",+ );+ let top_path = hybrid_hits[0].path.as_str();+ assert_eq!(+ top_path, "notes/karpathy.md",+ "the karpathy page should be the top hybrid hit; got {top_path}",+ );++ // 4. FTS5-only path (None query vector) still works through the+ // same method — the hybrid method gracefully falls back.+ let fts_only = store+ .reader+ .hybrid_search(+ ws,+ proj,+ q.to_string(),+ None,+ "synthetic".into(),+ "bag-of-words-v1".into(),+ 64,+ 5,+ )+ .await+ .expect("hybrid (no query vec)");+ assert!(+ !fts_only.is_empty(),+ "FTS5-only fallback should still return Karpathy page",+ );++ // 5. Re-writing a page produces a fresh embedding row (the+ // ON CONFLICT in store_embedding does the replace).+ wiki.write_page(WritePageRequest {+ workspace_id: ws,+ project_id: proj,+ path: PagePath::new("notes/karpathy.md".to_string()).expect("path"),+ frontmatter: serde_json::json!({"title": "Karpathy revised"}),+ body: "Karpathy LLM Wiki: revised body, still compile not retrieve".into(),+ tier: Tier::Semantic,+ pinned: false,+ title: None,+ })+ .await+ .expect("rewrite page");+ let stored_after = store+ .reader+ .load_embeddings(ws, proj, "synthetic".into(), "bag-of-words-v1".into(), 64)+ .await+ .expect("load embeddings after rewrite");+ // Still 3 latest pages, still 3 embeddings — the rewrite replaced+ // the karpathy row, didn't add a new one.+ assert_eq!(+ stored_after.len(),+ 3,+ "rewriting the same path should keep us at 3 embeddings, not accumulate",+ );+}diff --git a/crates/ai-memory-llm/src/embedding.rs b/crates/ai-memory-llm/src/embedding.rsnew file mode 100644index 00000000..f8aee7aa--- /dev/null+++ b/crates/ai-memory-llm/src/embedding.rs@@ -0,0 +1,375 @@+//! Embedding provider abstraction.+//!+//! Three implementations ship in M9:+//!+//! * [`OpenAiEmbedder`] — production: hits OpenAI's `/v1/embeddings`.+//! * [`VoyageEmbedder`] — production: hits Voyage's `/v1/embeddings`.+//! * [`SyntheticEmbedder`] — test-only: deterministic bag-of-words+//! embedding so integration tests can demonstrate semantic+//! retrieval without an API key.+//!+//! Future: a local `ort` + `bge-small-en-v1.5` embedder. The trait is+//! generic so dropping it in later doesn't touch consumers.++use std::time::Duration;++use async_trait::async_trait;+use secrecy::{ExposeSecret, SecretString};+use serde::{Deserialize, Serialize};+use tracing::debug;++use crate::error::{LlmError, LlmResult};++/// Provider-agnostic embedding API.+///+/// Implementations must be `Send + Sync` (the MCP server / hook+/// router stash an `Arc<dyn Embedder>` and use it from any tokio+/// task).+#[async_trait]+pub trait Embedder: Send + Sync {+ /// Short identifier (e.g. `openai`, `voyage`, `synthetic`).+ fn provider(&self) -> &'static str;++ /// Model identifier (e.g. `text-embedding-3-small`).+ fn model(&self) -> &str;++ /// Vector dimensionality.+ fn dim(&self) -> u32;++ /// Embed one text. Returns a unit-normalised vector — callers can+ /// dot-product directly to get cosine similarity.+ async fn embed(&self, text: &str) -> LlmResult<Vec<f32>>;+}++/// OpenAI Embeddings API (`text-embedding-3-small` by default, 1536 dim).+pub struct OpenAiEmbedder {+ client: reqwest::Client,+ api_key: SecretString,+ base_url: String,+ model: String,+ dim: u32,+}++impl OpenAiEmbedder {+ /// Construct an embedder.+ ///+ /// # Errors+ /// Propagates any `reqwest::Error` thrown while building the HTTP+ /// client.+ pub fn new(api_key: SecretString, model: impl Into<String>, dim: u32) -> LlmResult<Self> {+ let client = reqwest::Client::builder()+ .timeout(Duration::from_secs(60))+ .build()?;+ Ok(Self {+ client,+ api_key,+ base_url: "https://api.openai.com".into(),+ model: model.into(),+ dim,+ })+ }++ /// Override the base URL (for tests against a wiremock).+ #[must_use]+ pub fn with_base_url(mut self, url: impl Into<String>) -> Self {+ self.base_url = url.into();+ self+ }+}++#[derive(Debug, Serialize)]+struct OpenAiEmbeddingRequest<'a> {+ input: &'a str,+ model: &'a str,+}++#[derive(Debug, Deserialize)]+struct OpenAiEmbeddingResponse {+ data: Vec<OpenAiEmbeddingDatum>,+}++#[derive(Debug, Deserialize)]+struct OpenAiEmbeddingDatum {+ embedding: Vec<f32>,+}++#[async_trait]+impl Embedder for OpenAiEmbedder {+ fn provider(&self) -> &'static str {+ "openai"+ }++ fn model(&self) -> &str {+ &self.model+ }++ fn dim(&self) -> u32 {+ self.dim+ }++ async fn embed(&self, text: &str) -> LlmResult<Vec<f32>> {+ let url = format!("{}/v1/embeddings", self.base_url.trim_end_matches('/'));+ debug!(url, model = %self.model, "POST openai/embeddings");+ let req = OpenAiEmbeddingRequest {+ input: text,+ model: &self.model,+ };+ let resp = self+ .client+ .post(&url)+ .bearer_auth(self.api_key.expose_secret())+ .json(&req)+ .send()+ .await?;+ let status = resp.status();+ if !status.is_success() {+ let body = resp.text().await.unwrap_or_default();+ return Err(LlmError::Provider {+ status: status.as_u16(),+ body: truncate(&body, 1024),+ });+ }+ let parsed: OpenAiEmbeddingResponse = resp.json().await?;+ let first =+ parsed.data.into_iter().next().ok_or_else(|| {+ LlmError::UnexpectedShape("openai response had no data[0]".into())+ })?;+ if first.embedding.len() as u32 != self.dim {+ return Err(LlmError::UnexpectedShape(format!(+ "expected dim {}, got {}",+ self.dim,+ first.embedding.len()+ )));+ }+ Ok(normalise(first.embedding))+ }+}++/// Voyage Embeddings API.+pub struct VoyageEmbedder {+ client: reqwest::Client,+ api_key: SecretString,+ base_url: String,+ model: String,+ dim: u32,+}++impl VoyageEmbedder {+ /// Construct a Voyage embedder.+ ///+ /// # Errors+ /// Propagates the HTTP client construction error.+ pub fn new(api_key: SecretString, model: impl Into<String>, dim: u32) -> LlmResult<Self> {+ let client = reqwest::Client::builder()+ .timeout(Duration::from_secs(60))+ .build()?;+ Ok(Self {+ client,+ api_key,+ base_url: "https://api.voyageai.com".into(),+ model: model.into(),+ dim,+ })+ }++ /// Override the base URL.+ #[must_use]+ pub fn with_base_url(mut self, url: impl Into<String>) -> Self {+ self.base_url = url.into();+ self+ }+}++#[derive(Debug, Serialize)]+struct VoyageRequest<'a> {+ input: [&'a str; 1],+ model: &'a str,+}++#[derive(Debug, Deserialize)]+struct VoyageResponse {+ data: Vec<VoyageDatum>,+}++#[derive(Debug, Deserialize)]+struct VoyageDatum {+ embedding: Vec<f32>,+}++#[async_trait]+impl Embedder for VoyageEmbedder {+ fn provider(&self) -> &'static str {+ "voyage"+ }++ fn model(&self) -> &str {+ &self.model+ }++ fn dim(&self) -> u32 {+ self.dim+ }++ async fn embed(&self, text: &str) -> LlmResult<Vec<f32>> {+ let url = format!("{}/v1/embeddings", self.base_url.trim_end_matches('/'));+ let req = VoyageRequest {+ input: [text],+ model: &self.model,+ };+ let resp = self+ .client+ .post(&url)+ .bearer_auth(self.api_key.expose_secret())+ .json(&req)+ .send()+ .await?;+ let status = resp.status();+ if !status.is_success() {+ let body = resp.text().await.unwrap_or_default();+ return Err(LlmError::Provider {+ status: status.as_u16(),+ body: truncate(&body, 1024),+ });+ }+ let parsed: VoyageResponse = resp.json().await?;+ let first =+ parsed.data.into_iter().next().ok_or_else(|| {+ LlmError::UnexpectedShape("voyage response had no data[0]".into())+ })?;+ if first.embedding.len() as u32 != self.dim {+ return Err(LlmError::UnexpectedShape(format!(+ "expected dim {}, got {}",+ self.dim,+ first.embedding.len()+ )));+ }+ Ok(normalise(first.embedding))+ }+}++/// Deterministic synthetic embedder for tests.+///+/// Each whitespace-separated word in the text adds 1.0 to one+/// dimension chosen by the word's hash. Vectors are unit-normalised.+/// Similar text → similar vectors, which is enough to demonstrate+/// hybrid retrieval beats either ranker alone.+pub struct SyntheticEmbedder {+ dim: u32,+}++impl SyntheticEmbedder {+ /// Construct with the given dimensionality.+ #[must_use]+ pub const fn new(dim: u32) -> Self {+ Self { dim }+ }+}++#[async_trait]+impl Embedder for SyntheticEmbedder {+ fn provider(&self) -> &'static str {+ "synthetic"+ }++ fn model(&self) -> &str {+ "bag-of-words-v1"+ }++ fn dim(&self) -> u32 {+ self.dim+ }++ async fn embed(&self, text: &str) -> LlmResult<Vec<f32>> {+ let mut v = vec![0.0_f32; self.dim as usize];+ for word in text.split(|c: char| !c.is_alphanumeric()) {+ if word.is_empty() {+ continue;+ }+ let lower = word.to_ascii_lowercase();+ let h = fnv1a(&lower) as usize;+ let idx = h % v.len();+ v[idx] += 1.0;+ }+ Ok(normalise(v))+ }+}++/// Unit-normalise so dot-product equals cosine similarity.+fn normalise(mut v: Vec<f32>) -> Vec<f32> {+ let norm: f32 = v.iter().map(|x| x * x).sum::<f32>().sqrt();+ if norm > 0.0 {+ for x in &mut v {+ *x /= norm;+ }+ }+ v+}++fn fnv1a(s: &str) -> u64 {+ let mut h: u64 = 0xcbf2_9ce4_8422_2325;+ for &b in s.as_bytes() {+ h ^= u64::from(b);+ h = h.wrapping_mul(0x100_0000_01b3);+ }+ h+}++fn truncate(s: &str, n: usize) -> String {+ if s.len() <= n {+ s.to_string()+ } else {+ format!("{}…", &s[..n])+ }+}++/// Cosine similarity between two same-dim unit vectors. (= dot product+/// after normalisation.)+#[must_use]+pub fn cosine(a: &[f32], b: &[f32]) -> f32 {+ a.iter().zip(b).map(|(x, y)| x * y).sum()+}++#[cfg(test)]+mod tests {+ use super::*;++ #[tokio::test]+ async fn synthetic_embedder_produces_unit_vectors() {+ let e = SyntheticEmbedder::new(64);+ let v = e.embed("hello world hello").await.unwrap();+ assert_eq!(v.len(), 64);+ let norm: f32 = v.iter().map(|x| x * x).sum::<f32>().sqrt();+ assert!((norm - 1.0).abs() < 1e-5, "expected unit norm, got {norm}");+ }++ #[tokio::test]+ async fn synthetic_similar_text_has_higher_cosine() {+ let e = SyntheticEmbedder::new(256);+ let a = e+ .embed("retention sweep evicts stale episodic pages")+ .await+ .unwrap();+ let close = e+ .embed("the sweep evicts stale episodic pages")+ .await+ .unwrap();+ let far = e+ .embed("docker compose volumes and bind mounts")+ .await+ .unwrap();+ let s_close = cosine(&a, &close);+ let s_far = cosine(&a, &far);+ assert!(+ s_close > s_far,+ "similar text should cosine higher than unrelated: close={s_close} far={s_far}"+ );+ }++ #[tokio::test]+ async fn synthetic_is_deterministic() {+ let e = SyntheticEmbedder::new(64);+ let a = e.embed("compile not retrieve").await.unwrap();+ let b = e.embed("compile not retrieve").await.unwrap();+ assert_eq!(a, b);+ }+}diff --git a/crates/ai-memory-llm/src/factory.rs b/crates/ai-memory-llm/src/factory.rsindex 62d94028..02b93e98 100644--- a/crates/ai-memory-llm/src/factory.rs+++ b/crates/ai-memory-llm/src/factory.rs@@ -10,6 +10,7 @@ use secrecy::SecretString;use crate::AnthropicProvider;use crate::OpenAiCompatProvider;use crate::OpenAiProvider;+use crate::embedding::{Embedder, OpenAiEmbedder, VoyageEmbedder};use crate::error::{LlmError, LlmResult};use crate::provider::LlmProvider;@@ -84,6 +85,115 @@ pub fn provider_from_env() -> LlmResult<Option<ProviderConfig>> {}))}+/// Three embedders ship in v0.2.+#[derive(Debug, Clone, Copy, PartialEq, Eq)]+pub enum EmbedderChoice {+ /// OpenAI Embeddings API.+ OpenAi,+ /// Voyage Embeddings API.+ Voyage,+}++impl EmbedderChoice {+ /// Wire-format provider name; matches what the `Embedder::provider`+ /// implementations return so the refuse-on-mismatch query lines up.+ #[must_use]+ pub const fn name(self) -> &'static str {+ match self {+ Self::OpenAi => "openai",+ Self::Voyage => "voyage",+ }+ }+}++/// Settings to build an embedder.+#[derive(Debug, Clone)]+pub struct EmbedderConfig {+ /// Provider selection.+ pub provider: EmbedderChoice,+ /// Model id (e.g. `text-embedding-3-small`).+ pub model: String,+ /// Vector dimensionality. Refused on mismatch with the stored+ /// pages' dim.+ pub dim: u32,+ /// API key.+ pub api_key: SecretString,+ /// Optional base URL override.+ pub base_url: Option<String>,+}++/// Construct an `Arc<dyn Embedder>` from the config.+///+/// # Errors+/// Propagates HTTP-client construction errors.+pub fn build_embedder(config: EmbedderConfig) -> LlmResult<Arc<dyn Embedder>> {+ let arc: Arc<dyn Embedder> = match config.provider {+ EmbedderChoice::OpenAi => {+ let mut e = OpenAiEmbedder::new(config.api_key, config.model, config.dim)?;+ if let Some(url) = config.base_url {+ e = e.with_base_url(url);+ }+ Arc::new(e)+ }+ EmbedderChoice::Voyage => {+ let mut e = VoyageEmbedder::new(config.api_key, config.model, config.dim)?;+ if let Some(url) = config.base_url {+ e = e.with_base_url(url);+ }+ Arc::new(e)+ }+ };+ Ok(arc)+}++/// Read an `EmbedderConfig` from the environment.+///+/// Honours `AI_MEMORY_EMBEDDING_PROVIDER` (`openai` / `voyage`),+/// `AI_MEMORY_EMBEDDING_MODEL`, `AI_MEMORY_EMBEDDING_DIM`,+/// `AI_MEMORY_EMBEDDING_BASE_URL` (optional), and the appropriate+/// API key env (`OPENAI_API_KEY` / `VOYAGE_API_KEY`).+///+/// Returns `Ok(None)` when `AI_MEMORY_EMBEDDING_PROVIDER` is unset —+/// the canonical "no embeddings, FTS5 only" path.+///+/// # Errors+/// Returns [`LlmError::NotConfigured`] when the provider value is+/// unrecognised or when required env vars are missing.+pub fn embedder_from_env() -> LlmResult<Option<EmbedderConfig>> {+ let provider = match std::env::var("AI_MEMORY_EMBEDDING_PROVIDER") {+ Ok(s) => match s.as_str() {+ "openai" => EmbedderChoice::OpenAi,+ "voyage" => EmbedderChoice::Voyage,+ other => {+ return Err(LlmError::NotConfigured(format!(+ "AI_MEMORY_EMBEDDING_PROVIDER={other} not one of openai|voyage"+ )));+ }+ },+ Err(_) => return Ok(None),+ };+ let model = std::env::var("AI_MEMORY_EMBEDDING_MODEL")+ .map_err(|_| LlmError::NotConfigured("AI_MEMORY_EMBEDDING_MODEL".into()))?;+ let dim: u32 = std::env::var("AI_MEMORY_EMBEDDING_DIM")+ .map_err(|_| LlmError::NotConfigured("AI_MEMORY_EMBEDDING_DIM".into()))?+ .parse()+ .map_err(|e| LlmError::NotConfigured(format!("AI_MEMORY_EMBEDDING_DIM: {e}")))?;+ let base_url = std::env::var("AI_MEMORY_EMBEDDING_BASE_URL").ok();+ let api_key = match provider {+ EmbedderChoice::OpenAi => std::env::var("OPENAI_API_KEY")+ .map_err(|_| LlmError::NotConfigured("OPENAI_API_KEY".into()))?,+ EmbedderChoice::Voyage => std::env::var("VOYAGE_API_KEY")+ .map_err(|_| LlmError::NotConfigured("VOYAGE_API_KEY".into()))?,+ };+ Ok(Some(EmbedderConfig {+ provider,+ model,+ dim,+ api_key: SecretString::from(api_key),+ base_url,+ }))+}+/// Construct an `Arc<dyn LlmProvider>` matching the config.////// # Errorsdiff --git a/crates/ai-memory-llm/src/lib.rs b/crates/ai-memory-llm/src/lib.rsindex b40d18f5..1102b96b 100644--- a/crates/ai-memory-llm/src/lib.rs+++ b/crates/ai-memory-llm/src/lib.rs@@ -20,6 +20,7 @@//! No tenacity-style 8-128s backoff (cognee #2840 lesson).pub mod anthropic;+pub mod embedding;pub mod error;pub mod factory;pub mod openai;@@ -28,8 +29,12 @@ pub mod provider;pub mod types;pub use anthropic::AnthropicProvider;+pub use embedding::{Embedder, OpenAiEmbedder, SyntheticEmbedder, VoyageEmbedder, cosine};pub use error::{LlmError, LlmResult};-pub use factory::{ProviderChoice, ProviderConfig, build_provider, provider_from_env};+pub use factory::{+ EmbedderChoice, EmbedderConfig, ProviderChoice, ProviderConfig, build_embedder, build_provider,+ embedder_from_env, provider_from_env,+};pub use openai::OpenAiProvider;pub use openai_compat::OpenAiCompatProvider;pub use provider::{LlmProvider, complete_structured};diff --git a/crates/ai-memory-mcp/src/server.rs b/crates/ai-memory-mcp/src/server.rsindex c3c95cf6..ce30da15 100644--- a/crates/ai-memory-mcp/src/server.rs+++ b/crates/ai-memory-mcp/src/server.rs@@ -5,7 +5,7 @@ use std::sync::Arc;use ai_memory_consolidate::{Consolidator, run_lint, run_sweep};use ai_memory_core::{AgentKind, NewHandoff, PageId, ProjectId, SessionId, WorkspaceId};-use ai_memory_llm::LlmProvider;+use ai_memory_llm::{Embedder, LlmProvider};use ai_memory_store::{DecayParams, ReaderPool, WriterHandle};use ai_memory_wiki::Wiki;use rmcp::handler::server::router::tool::ToolRouter;@@ -45,6 +45,9 @@ pub struct AiMemoryServer {/// M8 retention parameters. Defaults if not overridden by the/// caller (typically from the user's config.toml `[decay]` block).decay_params: DecayParams,+ /// M9 embedder for hybrid query. When `None`, `memory_query`+ /// falls back to pure FTS5.+ embedder: Option<Arc<dyn Embedder>>,// Read by the `#[tool_handler]` macro expansion; rustc's dead-code// analysis can't see that, so the lint must be allowed explicitly.#[allow(dead_code)]@@ -151,10 +154,19 @@ impl AiMemoryServer {llm: None,wiki: None,decay_params: DecayParams::default(),+ embedder: None,tool_router: Self::tool_router(),}}+ /// Attach an embedder for hybrid (FTS5 + vector RRF) query. Without+ /// this, `memory_query` runs pure FTS5.+ #[must_use]+ pub fn with_embedder(mut self, embedder: Arc<dyn Embedder>) -> Self {+ self.embedder = Some(embedder);+ self+ }+/// Override the retention-sweep parameters (typically populated/// from the user's config.toml `[decay]` table).#[must_use]@@ -202,11 +214,28 @@ impl AiMemoryServer {Parameters(args): Parameters<QueryArgs>,) -> Result<CallToolResult, McpError> {let limit = args.limit.unwrap_or(self.default_limit).clamp(1, 100);- let hits = self- .reader- .search_pages(args.query, limit)- .await- .map_err(|e| McpError::internal_error(e.to_string(), None))?;+ let hits = if let Some(embedder) = &self.embedder {+ // Hybrid path: embed the query, RRF-fuse with FTS5.+ let qv = embedder+ .embed(&args.query)+ .await+ .map_err(|e| McpError::internal_error(e.to_string(), None))?;+ self.reader+ .hybrid_search(+ self.workspace_id,+ self.project_id,+ args.query,+ Some(qv),+ embedder.provider().to_string(),+ embedder.model().to_string(),+ embedder.dim(),+ limit,+ )+ .await+ } else {+ self.reader.search_pages(args.query, limit).await+ };+ let hits = hits.map_err(|e| McpError::internal_error(e.to_string(), None))?;self.spawn_access_bump(hits.iter().map(|h| h.id).collect());let response = QueryResponse { hits };ok_json(&response)diff --git a/crates/ai-memory-store/migrations/V04__embeddings.sql b/crates/ai-memory-store/migrations/V04__embeddings.sqlnew file mode 100644index 00000000..fb1279c8--- /dev/null+++ b/crates/ai-memory-store/migrations/V04__embeddings.sql@@ -0,0 +1,23 @@+-- M9 page embeddings.+--+-- One row per latest page version. `vector` is a packed+-- `[u8; dim * 4]` representation of a `Vec<f32>` (host endianness;+-- our backups are local so no cross-arch hazard in v0.2).+--+-- (provider, model, dim) is denormalised onto each row so a single+-- query against the table reveals heterogeneity in case the user+-- switched providers without running `ai-memory embed`.++CREATE TABLE page_embeddings (+ page_id BLOB PRIMARY KEY NOT NULL REFERENCES pages(id) ON DELETE CASCADE,+ vector BLOB NOT NULL,+ provider TEXT NOT NULL,+ model TEXT NOT NULL,+ dim INTEGER NOT NULL CHECK (dim > 0),+ created_at INTEGER NOT NULL+);++-- Used by the refuse-on-mismatch startup check + by ai-memory embed+-- to find pages still on a stale (provider, model, dim) triple.+CREATE INDEX idx_page_embeddings_provider+ ON page_embeddings(provider, model, dim);diff --git a/crates/ai-memory-store/src/lib.rs b/crates/ai-memory-store/src/lib.rsindex 71c4fabe..ec307398 100644--- a/crates/ai-memory-store/src/lib.rs+++ b/crates/ai-memory-store/src/lib.rs@@ -21,7 +21,9 @@ mod writer;pub use decay::{DecayParams, retention_score};pub use error::{StoreError, StoreResult};-pub use reader::{DecayCandidate, PageHit, ReaderPool, StatusCounts};+pub use reader::{+ DecayCandidate, PageHit, ReaderPool, StatusCounts, StoredEmbedding, f32_vec_to_bytes,+};pub use writer::WriterHandle;/// Filename used inside the data dir's `db/` subdirectory.diff --git a/crates/ai-memory-store/src/ops.rs b/crates/ai-memory-store/src/ops.rsindex d4c70442..45d116bd 100644--- a/crates/ai-memory-store/src/ops.rs+++ b/crates/ai-memory-store/src/ops.rs@@ -375,6 +375,33 @@ pub fn insert_observation(Ok(id)}+/// Store / replace one page's embedding. Bytes are the host-endian+/// `f32` packing of the unit-normalised vector. Provider/model/dim+/// are denormalised onto the row so a single SELECT can detect+/// heterogeneity (refuse-on-mismatch path).+pub fn store_embedding(+ conn: &mut Connection,+ page_id: &PageId,+ vector_bytes: &[u8],+ provider: &str,+ model: &str,+ dim: u32,+) -> StoreResult<()> {+ let now = Timestamp::now().as_microsecond();+ conn.execute(+ "INSERT INTO page_embeddings (page_id, vector, provider, model, dim, created_at) \+ VALUES (?1, ?2, ?3, ?4, ?5, ?6) \+ ON CONFLICT(page_id) DO UPDATE SET \+ vector = excluded.vector, \+ provider = excluded.provider, \+ model = excluded.model, \+ dim = excluded.dim, \+ created_at = excluded.created_at",+ params![page_id.as_bytes(), vector_bytes, provider, model, dim, now,],+ )?;+ Ok(())+}+/// Bump `access_count` + `last_accessed_at` for the pages whose ids/// appear in `page_ids`. Idempotent for unknown ids (no-op)./// Used by the read path to feed the M8 reinforcement term.diff --git a/crates/ai-memory-store/src/reader.rs b/crates/ai-memory-store/src/reader.rsindex fe313a9d..ca2f3a23 100644--- a/crates/ai-memory-store/src/reader.rs+++ b/crates/ai-memory-store/src/reader.rs@@ -214,6 +214,103 @@ impl ReaderPool {.await}+ /// Load every `is_latest=1` page's embedding for the project, but+ /// only when the stored `(provider, model, dim)` matches the+ /// caller's expectation. Mismatched rows are skipped (the+ /// refuse-on-mismatch check is `embedding_meta_for_mismatch`).+ ///+ /// # Errors+ /// Propagates any SQL or pool error.+ pub async fn load_embeddings(+ &self,+ workspace_id: WorkspaceId,+ project_id: ProjectId,+ provider: String,+ model: String,+ dim: u32,+ ) -> StoreResult<Vec<StoredEmbedding>> {+ self.with_conn(move |conn| {+ let mut stmt = conn.prepare(+ "SELECT page_embeddings.page_id, page_embeddings.vector, pages.path \+ FROM page_embeddings \+ JOIN pages ON pages.id = page_embeddings.page_id \+ WHERE pages.workspace_id = ?1 \+ AND pages.project_id = ?2 \+ AND pages.is_latest = 1 \+ AND page_embeddings.provider = ?3 \+ AND page_embeddings.model = ?4 \+ AND page_embeddings.dim = ?5",+ )?;+ let rows = stmt.query_map(+ params![+ workspace_id.as_bytes(),+ project_id.as_bytes(),+ provider,+ model,+ dim,+ ],+ |row| {+ let id_bytes: Vec<u8> = row.get(0)?;+ let vec_bytes: Vec<u8> = row.get(1)?;+ let path: String = row.get(2)?;+ Ok((id_bytes, vec_bytes, path))+ },+ )?;+ let mut out = Vec::new();+ for r in rows {+ let (id_bytes, vec_bytes, path) = r?;+ let id = PageId::from_slice(&id_bytes)?;+ let path = PagePath::new(path)?;+ let vector = bytes_to_f32_vec(&vec_bytes, dim)?;+ out.push(StoredEmbedding { id, path, vector });+ }+ Ok(out)+ })+ .await+ }++ /// Return any `(provider, model, dim)` triples currently stored+ /// that *don't* match the caller's expectation. An empty vec+ /// means "all clean". Used at startup for the refuse-on-mismatch+ /// (agentmemory #469 lesson).+ ///+ /// # Errors+ /// Propagates any SQL or pool error.+ pub async fn embedding_meta_for_mismatch(+ &self,+ provider: String,+ model: String,+ dim: u32,+ ) -> StoreResult<Vec<(String, String, u32, u64)>> {+ self.with_conn(move |conn| {+ let mut stmt = conn.prepare(+ "SELECT provider, model, dim, COUNT(*) \+ FROM page_embeddings \+ WHERE NOT (provider = ?1 AND model = ?2 AND dim = ?3) \+ GROUP BY provider, model, dim",+ )?;+ let rows = stmt.query_map(params![provider, model, dim], |row| {+ let provider: String = row.get(0)?;+ let model: String = row.get(1)?;+ let dim: i64 = row.get(2)?;+ let count: i64 = row.get(3)?;+ #[allow(clippy::cast_possible_truncation, clippy::cast_sign_loss)]+ Ok((+ provider,+ model,+ dim as u32,+ u64::try_from(count).unwrap_or(0),+ ))+ })?;+ let mut out = Vec::new();+ for r in rows {+ out.push(r?);+ }+ Ok(out)+ })+ .await+ }+/// Return decay-evaluation candidates for the M8 forget sweep.////// Walks `pages` rows with `is_latest = 1` and returns the columns@@ -249,6 +346,102 @@ impl ReaderPool {.await}+ /// Hybrid search: RRF-fuse FTS5 results with cosine-similarity+ /// over the stored embeddings of the matching `(provider, model,+ /// dim)`. Returns the top-`limit` pages by fused score.+ ///+ /// When `query_vec` is `None`, falls back to pure FTS5+ /// (equivalent to [`Self::search_pages`]). When no embeddings+ /// exist for the configured triple, also falls back to pure FTS5.+ ///+ /// k=60 is the canonical RRF constant.+ ///+ /// # Errors+ /// Propagates any SQL or pool error.+ #[allow(clippy::too_many_arguments)]+ pub async fn hybrid_search(+ &self,+ workspace_id: WorkspaceId,+ project_id: ProjectId,+ query: String,+ query_vec: Option<Vec<f32>>,+ provider: String,+ model: String,+ dim: u32,+ limit: usize,+ ) -> StoreResult<Vec<PageHit>> {+ // Fetch FTS5 hits first.+ let fts_hits = self.search_pages(query, limit * 2).await?;+ let Some(qv) = query_vec else {+ // No query vector → pure FTS5.+ let mut out = fts_hits;+ out.truncate(limit);+ return Ok(out);+ };+ // Compute cosines against all stored embeddings.+ let stored = self+ .load_embeddings(workspace_id, project_id, provider, model, dim)+ .await?;+ if stored.is_empty() {+ let mut out = fts_hits;+ out.truncate(limit);+ return Ok(out);+ }+ let mut vec_hits: Vec<(PageId, PagePath, f32)> = stored+ .iter()+ .map(|s| {+ let score = dot(&qv, &s.vector);+ (s.id, s.path.clone(), score)+ })+ .collect();+ vec_hits.sort_by(|a, b| b.2.partial_cmp(&a.2).unwrap_or(std::cmp::Ordering::Equal));+ vec_hits.truncate(limit * 2);++ // RRF fuse: score(d) = Σ 1/(k + rank_i(d)) over rankers.+ let k = 60.0_f64;+ let mut fused: std::collections::HashMap<PageId, (PagePath, String, String, f64, f64)> =+ std::collections::HashMap::new();++ for (rank, h) in fts_hits.iter().enumerate() {+ let contrib = 1.0 / (k + (rank + 1) as f64);+ fused+ .entry(h.id)+ .and_modify(|entry| entry.3 += contrib)+ .or_insert((+ h.path.clone(),+ h.title.clone(),+ h.snippet.clone(),+ contrib,+ h.rank,+ ));+ }+ for (rank, (id, path, _score)) in vec_hits.iter().enumerate() {+ let contrib = 1.0 / (k + (rank + 1) as f64);+ fused+ .entry(*id)+ .and_modify(|entry| entry.3 += contrib)+ .or_insert((path.clone(), String::new(), String::new(), contrib, 0.0));+ }++ let mut out: Vec<PageHit> = fused+ .into_iter()+ .map(|(id, (path, title, snippet, fused_rank, _orig))| PageHit {+ id,+ path,+ title,+ snippet,+ rank: -fused_rank, // lower = better (matches FTS5 convention)+ })+ .collect();+ out.sort_by(|a, b| {+ a.rank+ .partial_cmp(&b.rank)+ .unwrap_or(std::cmp::Ordering::Equal)+ });+ out.truncate(limit);+ Ok(out)+ }+/// Return the latest open handoff for the project, optionally/// filtered to a specific `cwd`.///@@ -409,6 +602,50 @@ fn materialise_observation(})}+/// One stored embedding row, materialised for the vector path.+#[derive(Debug, Clone)]+pub struct StoredEmbedding {+ /// Page identifier (always the `is_latest=1` row's id).+ pub id: PageId,+ /// Relative wiki path.+ pub path: PagePath,+ /// Unit-normalised vector.+ pub vector: Vec<f32>,+}++fn dot(a: &[f32], b: &[f32]) -> f32 {+ a.iter().zip(b).map(|(x, y)| x * y).sum()+}++fn bytes_to_f32_vec(bytes: &[u8], dim: u32) -> StoreResult<Vec<f32>> {+ let expected = (dim as usize) * 4;+ if bytes.len() != expected {+ return Err(StoreError::Memory(+ ai_memory_core::MemoryError::MalformedRecord(format!(+ "embedding bytes {} != expected {}",+ bytes.len(),+ expected+ )),+ ));+ }+ let mut out = Vec::with_capacity(dim as usize);+ for chunk in bytes.chunks_exact(4) {+ out.push(f32::from_le_bytes([chunk[0], chunk[1], chunk[2], chunk[3]]));+ }+ Ok(out)+}++/// Pack a `&[f32]` into little-endian bytes for storage. Inverse of+/// [`bytes_to_f32_vec`].+#[must_use]+pub fn f32_vec_to_bytes(v: &[f32]) -> Vec<u8> {+ let mut out = Vec::with_capacity(v.len() * 4);+ for x in v {+ out.extend_from_slice(&x.to_le_bytes());+ }+ out+}+/// One row's worth of input for the M8 retention formula.#[derive(Debug, Clone, Serialize)]pub struct DecayCandidate {diff --git a/crates/ai-memory-store/src/writer.rs b/crates/ai-memory-store/src/writer.rsindex 8cae91e5..ed041bdc 100644--- a/crates/ai-memory-store/src/writer.rs+++ b/crates/ai-memory-store/src/writer.rs@@ -75,6 +75,14 @@ pub(crate) enum WriteCmd {hard_delete_after_days: i64,reply: oneshot::Sender<StoreResult<usize>>,},+ StoreEmbedding {+ page_id: PageId,+ vector_bytes: Vec<u8>,+ provider: String,+ model: String,+ dim: u32,+ reply: oneshot::Sender<StoreResult<()>>,+ },Shutdown,}@@ -220,6 +228,31 @@ impl WriterHandle {rx.await.map_err(|_| StoreError::WriterClosed)?}+ /// Store (or replace) the embedding for one page (M9).+ ///+ /// # Errors+ /// Returns [`StoreError::WriterClosed`] or propagates SQL errors.+ pub async fn store_embedding(+ &self,+ page_id: PageId,+ vector_bytes: Vec<u8>,+ provider: String,+ model: String,+ dim: u32,+ ) -> StoreResult<()> {+ let (tx, rx) = oneshot::channel();+ self.send(WriteCmd::StoreEmbedding {+ page_id,+ vector_bytes,+ provider,+ model,+ dim,+ reply: tx,+ })+ .await?;+ rx.await.map_err(|_| StoreError::WriterClosed)?+ }+/// Bump access counters for a set of pages (M8 reinforcement term).////// # Errors@@ -384,6 +417,24 @@ fn worker_loop(mut conn: Connection, mut rx: mpsc::Receiver<WriteCmd>) {let result = ops::hard_delete_decayed_pages(&mut conn, hard_delete_after_days);let _ = reply.send(result);}+ WriteCmd::StoreEmbedding {+ page_id,+ vector_bytes,+ provider,+ model,+ dim,+ reply,+ } => {+ let result = ops::store_embedding(+ &mut conn,+ &page_id,+ &vector_bytes,+ &provider,+ &model,+ dim,+ );+ let _ = reply.send(result);+ }}}tracing::debug!("writer thread exiting cleanly");diff --git a/crates/ai-memory-wiki/Cargo.toml b/crates/ai-memory-wiki/Cargo.tomlindex bc32bb37..6505e99f 100644--- a/crates/ai-memory-wiki/Cargo.toml+++ b/crates/ai-memory-wiki/Cargo.toml@@ -10,6 +10,7 @@ description = "Markdown wiki read/write, file watcher and git versioning."[dependencies]ai-memory-core.workspace = true+ai-memory-llm.workspace = trueai-memory-store.workspace = trueanyhow.workspace = truethiserror.workspace = truediff --git a/crates/ai-memory-wiki/src/wiki.rs b/crates/ai-memory-wiki/src/wiki.rsindex 4a4f4767..e1fb3e39 100644--- a/crates/ai-memory-wiki/src/wiki.rs+++ b/crates/ai-memory-wiki/src/wiki.rs@@ -1,9 +1,11 @@//! [`Wiki`] — the only correct write path for the markdown source-of-truth.use std::path::{Path, PathBuf};+use std::sync::Arc;use ai_memory_core::{NewPage, PageId, PagePath, ProjectId, Tier, WorkspaceId};-use ai_memory_store::WriterHandle;+use ai_memory_llm::Embedder;+use ai_memory_store::{WriterHandle, f32_vec_to_bytes};use crate::atomic;use crate::error::WikiResult;@@ -22,6 +24,7 @@ pub struct Wiki {root: PathBuf,writer: WriterHandle,git: GitAdapter,+ embedder: Option<Arc<dyn Embedder>>,}impl Wiki {@@ -35,7 +38,29 @@ impl Wiki {let root = data_dir.join("wiki");std::fs::create_dir_all(&root)?;let git = GitAdapter::open_or_init(&root)?;- Ok(Self { root, writer, git })+ Ok(Self {+ root,+ writer,+ git,+ embedder: None,+ })+ }++ /// Attach an embedder. When set, every successful `write_page` /+ /// `apply_batch` also computes + stores an embedding for the new+ /// version. Without this, vector search is unavailable and+ /// `ReaderPool::hybrid_search` falls back to pure FTS5.+ #[must_use]+ pub fn with_embedder(mut self, embedder: Arc<dyn Embedder>) -> Self {+ self.embedder = Some(embedder);+ self+ }++ /// Borrow the optional embedder (used by the `ai-memory embed`+ /// backfill command).+ #[must_use]+ pub fn embedder(&self) -> Option<&Arc<dyn Embedder>> {+ self.embedder.as_ref()}/// Borrow the git adapter (for callers wiring auto-commit).@@ -222,12 +247,35 @@ impl Wiki {project_id,path,title,- body,+ body: body.clone(),tier,frontmatter_json: frontmatter,pinned,}).await?;+ // Embed if configured. We do this on the caller's task so the+ // tool reply still happens "indexes commit in the same+ // transaction" (basic-memory #763 lesson): no fire-and-forget+ // background embedding.+ if let Some(embedder) = &self.embedder {+ match embedder.embed(&body).await {+ Ok(vec) => {+ let bytes = f32_vec_to_bytes(&vec);+ self.writer+ .store_embedding(+ page_id,+ bytes,+ embedder.provider().to_string(),+ embedder.model().to_string(),+ embedder.dim(),+ )+ .await?;+ }+ Err(e) => {+ tracing::warn!(error = %e, path = %page_id, "embedding failed; page indexed without it");+ }+ }+ }Ok(page_id)}}