Solution for long term memory for agent coding CLIs and to facilitate handoff between different agent vendors
Rust
fix(embed,store): re-embed all projects and purge stale embeddings
pedrofjre6207019af7b
diff --git a/crates/ai-memory-cli/src/commands/embed.rs b/crates/ai-memory-cli/src/commands/embed.rsindex 57fd62db..4e621fe2 100644--- a/crates/ai-memory-cli/src/commands/embed.rs+++ b/crates/ai-memory-cli/src/commands/embed.rs@@ -14,6 +14,7 @@ struct EmbedRequest {project: String,reembed: bool,dry_run: bool,+ all_projects: bool,}/// Run the `embed` subcommand.@@ -27,7 +28,15 @@ struct EmbedRequest {/// response.pub async fn run(config: &Config, args: EmbedArgs) -> Result<()> {let endpoint = ServerEndpoint::from_config(config);- let project = super::resolve_project_name(config, args.project.as_deref())?;+ // Model migrations must reach stale rows in every project namespace.+ // Without an explicit `--project`, `--force` / `--reembed` fans out+ // across the whole workspace instead of the CWD-derived project only.+ let all_projects = args.force && args.project.is_none();+ let project = if all_projects {+ String::new()+ } else {+ super::resolve_project_name(config, args.project.as_deref())?+ };let report: serde_json::Value = post_json(&endpoint,"/admin/embed",@@ -38,6 +47,7 @@ pub async fn run(config: &Config, args: EmbedArgs) -> Result<()> {// field is `reembed` — map them here.reembed: args.force,dry_run: args.dry_run,+ all_projects,},).await?;@@ -51,3 +61,38 @@ pub async fn run(config: &Config, args: EmbedArgs) -> Result<()> {}Ok(())}++#[cfg(test)]+mod tests {+ use crate::cli::{Cli, Command};+ use clap::Parser;++ #[test]+ fn force_without_project_enables_all_projects() {+ let cli = Cli::try_parse_from(["ai-memory", "embed", "--force"]).unwrap();+ let Command::Embed(args) = cli.command else {+ panic!("expected embed command");+ };+ assert!(args.force);+ assert!(args.project.is_none());+ assert!(+ args.force && args.project.is_none(),+ "--force without --project must fan out to all projects"+ );+ }++ #[test]+ fn force_with_explicit_project_stays_scoped() {+ let cli =+ Cli::try_parse_from(["ai-memory", "embed", "--force", "--project", "consisanet"])+ .unwrap();+ let Command::Embed(args) = cli.command else {+ panic!("expected embed command");+ };+ let all_projects = args.force && args.project.is_none();+ assert!(+ !all_projects,+ "--force with --project must stay scoped to that project"+ );+ }+}diff --git a/crates/ai-memory-mcp/src/admin.rs b/crates/ai-memory-mcp/src/admin.rsindex 9cfd475e..3506d9a1 100644--- a/crates/ai-memory-mcp/src/admin.rs+++ b/crates/ai-memory-mcp/src/admin.rs@@ -899,7 +899,8 @@ struct EmbedRequest {/// Workspace name (auto-created if absent).#[serde(default = "default_workspace")]workspace: String,- /// Project name (auto-created if absent).+ /// Project name (auto-created if absent). Ignored when+ /// [`Self::all_projects`] is true.#[serde(default = "default_project")]project: String,/// When true, regenerates embeddings even for pages that already@@ -910,6 +911,12 @@ struct EmbedRequest {/// calling the embedder or writing anything.#[serde(default)]dry_run: bool,+ /// When true, embed every project in `workspace` instead of a+ /// single `project`. The CLI sets this for `--force` /+ /// `--reembed` without an explicit `--project` so model migrations+ /// reach stale rows in every namespace.+ #[serde(default)]+ all_projects: bool,}/// Summary response from `POST /admin/embed`.@@ -932,27 +939,51 @@ pub struct EmbedReport {pub dim: u32,}-async fn handle_embed(- State(state): State<Arc<AdminState>>,- Json(req): Json<EmbedRequest>,-) -> Result<impl IntoResponse, (StatusCode, Json<serde_json::Value>)> {- let embedder = match state.embedder.clone() {- Some(e) => e,- None => {- return Err((- StatusCode::SERVICE_UNAVAILABLE,- Json(serde_json::json!({- "error": "embedder not configured on server"- })),- ));- }- };+#[derive(Default)]+struct EmbedCounts {+ embedded: usize,+ skipped: usize,+ failed: usize,+ would_embed: usize,+}++impl EmbedCounts {+ fn absorb(&mut self, other: Self) {+ self.embedded += other.embedded;+ self.skipped += other.skipped;+ self.failed += other.failed;+ self.would_embed += other.would_embed;+ }+}+async fn embed_project_pages(+ state: &AdminState,+ embedder: &Arc<dyn Embedder>,+ ws: WorkspaceId,+ proj: ProjectId,+ reembed: bool,+ dry_run: bool,+) -> Result<EmbedCounts, (StatusCode, Json<serde_json::Value>)> {let provider = embedder.provider().to_string();let model = embedder.model().to_string();let dim = embedder.dim();- let (ws, proj) = resolve_ws_proj(&state, &req.workspace, &req.project).await?;+ if reembed && !dry_run {+ let purged = state+ .writer+ .delete_stale_page_embeddings(provider.clone(), model.clone(), dim)+ .await+ .map_err(|e| internal_err(e.to_string()))?;+ if purged > 0 {+ info!(+ purged,+ provider = %provider,+ model = %model,+ dim,+ "embed: purged stale page_embeddings before re-embed"+ );+ }+ }let candidates = state.reader@@ -960,8 +991,7 @@ async fn handle_embed(.await.map_err(|e| internal_err(e.to_string()))?;- // Build the set of page ids that already have a matching embedding.- let already: std::collections::HashSet<_> = if req.reembed {+ let already: std::collections::HashSet<_> = if reembed {std::collections::HashSet::new()} else {state@@ -973,37 +1003,42 @@ async fn handle_embed(.collect()};- let mut embedded = 0_usize;- let mut skipped = 0_usize;- let mut failed = 0_usize;- let mut would_embed = 0_usize;+ let mut counts = EmbedCounts::default();let mut pending = Vec::with_capacity(EMBEDDING_WRITE_BATCH);for cand in candidates {- if !req.reembed && already.contains(&cand.id) {- skipped += 1;+ if !reembed && already.contains(&cand.id) {+ counts.skipped += 1;continue;}- if req.dry_run {- would_embed += 1;+ if dry_run {+ counts.would_embed += 1;continue;}let md = match state.wiki.read_page(ws, proj, &cand.path) {Ok(m) => m,Err(e) => {warn!(path = %cand.path, error = %e, "embed: skip unreadable page");- failed += 1;+ counts.failed += 1;continue;}};+ if md.body.trim().is_empty() {+ counts.skipped += 1;+ continue;+ }let vec = match embedder.embed_document(&md.body).await {Ok(v) => v,Err(e) => {warn!(path = %cand.path, error = %e, "embed: provider call failed");- failed += 1;+ counts.failed += 1;continue;}};+ // Gentle pacing for provider rate limits (OpenRouter free tier, etc.).+ if !dry_run {+ tokio::time::sleep(std::time::Duration::from_millis(800)).await;+ }pending.push(EmbeddingWrite {page_id: cand.id,vector_bytes: f32_vec_to_bytes(&vec),@@ -1012,16 +1047,98 @@ async fn handle_embed(dim,});if pending.len() >= EMBEDDING_WRITE_BATCH {- flush_embedding_batch(&state.writer, &mut pending, &mut embedded, &mut failed).await;+ flush_embedding_batch(+ &state.writer,+ &mut pending,+ &mut counts.embedded,+ &mut counts.failed,+ )+ .await;}}- flush_embedding_batch(&state.writer, &mut pending, &mut embedded, &mut failed).await;+ flush_embedding_batch(+ &state.writer,+ &mut pending,+ &mut counts.embedded,+ &mut counts.failed,+ )+ .await;++ Ok(counts)+}++async fn handle_embed(+ State(state): State<Arc<AdminState>>,+ Json(req): Json<EmbedRequest>,+) -> Result<impl IntoResponse, (StatusCode, Json<serde_json::Value>)> {+ let embedder = match state.embedder.clone() {+ Some(e) => e,+ None => {+ return Err((+ StatusCode::SERVICE_UNAVAILABLE,+ Json(serde_json::json!({+ "error": "embedder not configured on server"+ })),+ ));+ }+ };++ let provider = embedder.provider().to_string();+ let model = embedder.model().to_string();+ let dim = embedder.dim();++ let mut totals = EmbedCounts::default();++ if req.reembed && !req.dry_run {+ let purged = state+ .writer+ .delete_stale_page_embeddings(provider.clone(), model.clone(), dim)+ .await+ .map_err(|e| internal_err(e.to_string()))?;+ info!(purged, provider = %provider, model = %model, "purged stale page_embeddings before re-embed");+ }++ if req.all_projects {+ let summaries = state+ .reader+ .list_projects_with_stats()+ .await+ .map_err(|e| internal_err(e.to_string()))?;+ for summary in summaries+ .into_iter()+ .filter(|p| p.workspace_name == req.workspace)+ {+ let (ws, proj) =+ resolve_ws_proj(&state, &summary.workspace_name, &summary.project_name).await?;+ let partial = embed_project_pages(+ &state,+ &embedder,+ ws,+ proj,+ req.reembed,+ req.dry_run,+ )+ .await?;+ totals.absorb(partial);+ }+ } else {+ let (ws, proj) = resolve_ws_proj(&state, &req.workspace, &req.project).await?;+ totals = embed_project_pages(+ &state,+ &embedder,+ ws,+ proj,+ req.reembed,+ req.dry_run,+ )+ .await?;+ }let report = EmbedReport {- embedded,- skipped,- failed,- would_embed,+ embedded: totals.embedded,+ skipped: totals.skipped,+ failed: totals.failed,+ would_embed: totals.would_embed,provider,model,dim,diff --git a/crates/ai-memory-store/src/lib.rs b/crates/ai-memory-store/src/lib.rsindex e2d0cf17..589165ed 100644--- a/crates/ai-memory-store/src/lib.rs+++ b/crates/ai-memory-store/src/lib.rs@@ -694,4 +694,88 @@ mod tests {.unwrap();assert!(none.is_none());}++ #[tokio::test]+ async fn delete_stale_page_embeddings_removes_mismatched_rows() {+ let tmp = TempDir::new().unwrap();+ let store = Store::open(tmp.path()).unwrap();+ let ws = store+ .writer+ .get_or_create_workspace("default")+ .await+ .unwrap();+ let proj = store+ .writer+ .get_or_create_project(ws, "test", None)+ .await+ .unwrap();+ let p1 = store+ .writer+ .upsert_page(sample_page(ws, proj, "a.md", "body a"))+ .await+ .unwrap();+ let p2 = store+ .writer+ .upsert_page(sample_page(ws, proj, "b.md", "body b"))+ .await+ .unwrap();+ store+ .writer+ .store_embedding(+ p1,+ vec![0u8; 4],+ "google".into(),+ "models/gemini-embedding-001".into(),+ 768,+ )+ .await+ .unwrap();+ store+ .writer+ .store_embedding(+ p2,+ vec![1u8; 4],+ "openai".into(),+ "openai/text-embedding-3-small".into(),+ 1536,+ )+ .await+ .unwrap();+ let n = store+ .writer+ .delete_stale_page_embeddings(+ "openai".into(),+ "openai/text-embedding-3-small".into(),+ 1536,+ )+ .await+ .unwrap();+ assert_eq!(n, 1);+ let mismatch = store+ .reader+ .embedding_meta_for_mismatch(+ "openai".into(),+ "openai/text-embedding-3-small".into(),+ 1536,+ )+ .await+ .unwrap();+ assert!(mismatch.is_empty());+ }++ #[test]+ fn v07_migration_metadata_for_fork_repair() {+ let tmp = TempDir::new().unwrap();+ let db_path = tmp.path().join("repair-meta.db");+ let mut conn = Connection::open(&db_path).unwrap();+ crate::migrations::run_to(&mut conn, 7).unwrap();+ let (name, checksum): (String, String) = conn+ .query_row(+ "SELECT name, checksum FROM refinery_schema_history WHERE version = 7",+ [],+ |r| Ok((r.get(0)?, r.get(1)?)),+ )+ .unwrap();+ eprintln!("V7_NAME={name} V7_CHECKSUM={checksum}");+ }}diff --git a/crates/ai-memory-store/src/ops.rs b/crates/ai-memory-store/src/ops.rsindex 02949c52..15385129 100644--- a/crates/ai-memory-store/src/ops.rs+++ b/crates/ai-memory-store/src/ops.rs@@ -822,6 +822,28 @@ pub fn purge_project(})}+/// Remove embedding rows for latest pages whose `(provider, model, dim)`+/// does not match the configured triple (used before a full re-embed).+pub fn delete_stale_page_embeddings(+ conn: &mut Connection,+ provider: &str,+ model: &str,+ dim: u32,+) -> StoreResult<u64> {+ let n = conn.execute(+ "DELETE FROM page_embeddings \+ WHERE page_id IN (SELECT id FROM pages WHERE is_latest = 1) \+ AND NOT (provider = ?1 AND model = ?2 AND dim = CAST(?3 AS INTEGER))",+ params![provider, model, dim],+ )?;+ let orphans = conn.execute(+ "DELETE FROM page_embeddings \+ WHERE page_id IN (SELECT id FROM pages WHERE is_latest = 0)",+ [],+ )?;+ Ok(u64::try_from(n.saturating_add(orphans)).unwrap_or(0))+}+#[cfg(test)]mod tests {//! Focused unit tests for the load-bearing mutating SQL paths.@@ -1228,4 +1250,49 @@ mod tests {.unwrap();assert_eq!(count, 2);}++ #[test]+ fn delete_stale_page_embeddings_removes_mismatched_rows() {+ let (_tmp, mut conn, ws, proj) = fresh_db();+ let p1 = upsert_page(&mut conn, &page(ws, proj, "a.md", "body a")).unwrap();+ let p2 = upsert_page(&mut conn, &page(ws, proj, "b.md", "body b")).unwrap();+ store_embedding(+ &mut conn,+ &p1,+ &[0u8; 4],+ "google",+ "models/gemini-embedding-001",+ 768,+ )+ .unwrap();+ store_embedding(+ &mut conn,+ &p2,+ &[1u8; 4],+ "openai",+ "openai/text-embedding-3-small",+ 1536,+ )+ .unwrap();+ let n = super::delete_stale_page_embeddings(+ &mut conn,+ "openai",+ "openai/text-embedding-3-small",+ 1536,+ )+ .unwrap();+ assert_eq!(n, 1);+ let remaining: i64 = conn+ .query_row("SELECT COUNT(*) FROM page_embeddings", [], |r| r.get(0))+ .unwrap();+ assert_eq!(remaining, 1);+ let model: String = conn+ .query_row(+ "SELECT model FROM page_embeddings WHERE page_id = ?1",+ params![&p2.as_bytes()[..]],+ |r| r.get(0),+ )+ .unwrap();+ assert_eq!(model, "openai/text-embedding-3-small");+ }}diff --git a/crates/ai-memory-store/src/reader.rs b/crates/ai-memory-store/src/reader.rsindex 2ad66444..3242ab59 100644--- a/crates/ai-memory-store/src/reader.rs+++ b/crates/ai-memory-store/src/reader.rs@@ -875,10 +875,11 @@ impl ReaderPool {) -> StoreResult<Vec<(String, String, u32, u64)>> {self.with_conn(move |conn| {let mut stmt = conn.prepare(- "SELECT provider, model, dim, COUNT(*) \- FROM page_embeddings \- WHERE NOT (provider = ?1 AND model = ?2 AND dim = ?3) \- GROUP BY provider, model, dim",+ "SELECT pe.provider, pe.model, pe.dim, COUNT(*) \+ FROM page_embeddings pe \+ JOIN pages pg ON pg.id = pe.page_id AND pg.is_latest = 1 \+ WHERE NOT (pe.provider = ?1 AND pe.model = ?2 AND pe.dim = ?3) \+ GROUP BY pe.provider, pe.model, pe.dim",)?;let rows = stmt.query_map(params![provider, model, dim], |row| {let provider: String = row.get(0)?;diff --git a/crates/ai-memory-store/src/writer.rs b/crates/ai-memory-store/src/writer.rsindex ce9b1e08..9764ee16 100644--- a/crates/ai-memory-store/src/writer.rs+++ b/crates/ai-memory-store/src/writer.rs@@ -94,6 +94,12 @@ pub(crate) enum WriteCmd {embeddings: Vec<EmbeddingWrite>,reply: oneshot::Sender<StoreResult<()>>,},+ DeleteStalePageEmbeddings {+ provider: String,+ model: String,+ dim: u32,+ reply: oneshot::Sender<StoreResult<u64>>,+ },/// Delete a project and all its data (pages, sessions, observations,/// handoffs, embeddings) in one transaction. Returns the paths of/// every page file that must be removed from disk by the caller.@@ -304,6 +310,29 @@ impl WriterHandle {rx.await.map_err(|_| StoreError::WriterClosed)?}+ /// Remove embedding rows whose triple does not match the configured provider/model/dim.+ ///+ /// Used when re-embedding after a model migration (e.g. Gemini → OpenRouter).+ ///+ /// # Errors+ /// Returns [`StoreError::WriterClosed`] or propagates SQL errors.+ pub async fn delete_stale_page_embeddings(+ &self,+ provider: String,+ model: String,+ dim: u32,+ ) -> StoreResult<u64> {+ let (tx, rx) = oneshot::channel();+ self.send(WriteCmd::DeleteStalePageEmbeddings {+ provider,+ model,+ dim,+ reply: tx,+ })+ .await?;+ rx.await.map_err(|_| StoreError::WriterClosed)?+ }+/// Bump access counters for a set of pages (M8 reinforcement term).////// # Errors@@ -597,6 +626,15 @@ fn worker_loop(mut conn: Connection, mut rx: mpsc::Receiver<WriteCmd>) {let result = ops::store_embeddings(&mut conn, &embeddings);send_or_warn(reply, result, "store_embeddings");}+ WriteCmd::DeleteStalePageEmbeddings {+ provider,+ model,+ dim,+ reply,+ } => {+ let result = ops::delete_stale_page_embeddings(&mut conn, &provider, &model, dim);+ send_or_warn(reply, result, "delete_stale_page_embeddings");+ }WriteCmd::PurgeProject {workspace_id,project_id,