Solution for long term memory for agent coding CLIs and to facilitate handoff between different agent vendors
Rust
fix(embed): scope stale embedding cleanup
AkitaOnRails73e2d1a34eac
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 14d7cebc..4d23e9d8 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -15,6 +15,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0- OpenAI-compatible embedding calls now truncate oversized page bodies, surfaceprovider errors returned in HTTP 200 bodies, retry bounded HTTP 429 responses,and may reuse `LLM_API_KEY` when a custom embedding base URL is configured.+- `ai-memory embed --force` without `--project` now re-embeds every project in+ the workspace and purges stale/superseded embedding rows in the same scope.## [0.2.0] - 2026-05-26### Addeddiff --git a/crates/ai-memory-cli/src/commands/embed.rs b/crates/ai-memory-cli/src/commands/embed.rsindex 4e621fe2..8c01fee5 100644--- a/crates/ai-memory-cli/src/commands/embed.rs+++ b/crates/ai-memory-cli/src/commands/embed.rs@@ -83,9 +83,8 @@ mod tests {#[test]fn force_with_explicit_project_stays_scoped() {- let cli =- Cli::try_parse_from(["ai-memory", "embed", "--force", "--project", "consisanet"])- .unwrap();+ let cli = Cli::try_parse_from(["ai-memory", "embed", "--force", "--project", "consisanet"])+ .unwrap();let Command::Embed(args) = cli.command else {panic!("expected embed command");};diff --git a/crates/ai-memory-mcp/src/admin.rs b/crates/ai-memory-mcp/src/admin.rsindex 3506d9a1..863b5858 100644--- a/crates/ai-memory-mcp/src/admin.rs+++ b/crates/ai-memory-mcp/src/admin.rs@@ -968,23 +968,6 @@ async fn embed_project_pages(let model = embedder.model().to_string();let dim = embedder.dim();- if reembed && !dry_run {- let purged = state- .writer- .delete_stale_page_embeddings(provider.clone(), model.clone(), dim)- .await- .map_err(|e| internal_err(e.to_string()))?;- if purged > 0 {- info!(- purged,- provider = %provider,- model = %model,- dim,- "embed: purged stale page_embeddings before re-embed"- );- }- }-let candidates = state.reader.decay_candidates(ws, proj)@@ -1035,10 +1018,6 @@ async fn embed_project_pages(continue;}};- // Gentle pacing for provider rate limits (OpenRouter free tier, etc.).- if !dry_run {- tokio::time::sleep(std::time::Duration::from_millis(800)).await;- }pending.push(EmbeddingWrite {page_id: cand.id,vector_bytes: f32_vec_to_bytes(&vec),@@ -1089,49 +1068,56 @@ async fn handle_embed(let mut totals = EmbedCounts::default();- if req.reembed && !req.dry_run {- let purged = state- .writer- .delete_stale_page_embeddings(provider.clone(), model.clone(), dim)- .await- .map_err(|e| internal_err(e.to_string()))?;- info!(purged, provider = %provider, model = %model, "purged stale page_embeddings before re-embed");- }-if req.all_projects {- let summaries = state+ if let Some(ws) = state.reader- .list_projects_with_stats()+ .find_workspace(req.workspace.clone()).await- .map_err(|e| internal_err(e.to_string()))?;- for summary in summaries- .into_iter()- .filter(|p| p.workspace_name == req.workspace)+ .map_err(|e| internal_err(e.to_string()))?{- let (ws, proj) =- resolve_ws_proj(&state, &summary.workspace_name, &summary.project_name).await?;- let partial = embed_project_pages(- &state,- &embedder,- ws,- proj,- req.reembed,- req.dry_run,- )- .await?;- totals.absorb(partial);+ if req.reembed && !req.dry_run {+ let purged = state+ .writer+ .delete_stale_page_embeddings(ws, None, provider.clone(), model.clone(), dim)+ .await+ .map_err(|e| internal_err(e.to_string()))?;+ info!(purged, provider = %provider, model = %model, "purged stale page_embeddings before workspace re-embed");+ }++ let summaries = state+ .reader+ .list_projects_with_stats()+ .await+ .map_err(|e| internal_err(e.to_string()))?;+ for summary in summaries+ .into_iter()+ .filter(|p| p.workspace_name == req.workspace)+ {+ let Some(proj) = state+ .reader+ .find_project(ws, summary.project_name.clone())+ .await+ .map_err(|e| internal_err(e.to_string()))?+ else {+ continue;+ };+ let partial =+ embed_project_pages(&state, &embedder, ws, proj, req.reembed, req.dry_run)+ .await?;+ totals.absorb(partial);+ }}} else {let (ws, proj) = resolve_ws_proj(&state, &req.workspace, &req.project).await?;- totals = embed_project_pages(- &state,- &embedder,- ws,- proj,- req.reembed,- req.dry_run,- )- .await?;+ if req.reembed && !req.dry_run {+ let purged = state+ .writer+ .delete_stale_page_embeddings(ws, Some(proj), provider.clone(), model.clone(), dim)+ .await+ .map_err(|e| internal_err(e.to_string()))?;+ info!(purged, provider = %provider, model = %model, "purged stale page_embeddings before project re-embed");+ }+ totals = embed_project_pages(&state, &embedder, ws, proj, req.reembed, req.dry_run).await?;}let report = EmbedReport {diff --git a/crates/ai-memory-mcp/tests/admin_phase3.rs b/crates/ai-memory-mcp/tests/admin_phase3.rsindex 1dcebfcd..0553c68c 100644--- a/crates/ai-memory-mcp/tests/admin_phase3.rs+++ b/crates/ai-memory-mcp/tests/admin_phase3.rs@@ -10,6 +10,7 @@use ai_memory_core::{AgentKind, NewObservation, NewPage, NewSession, ObservationKind, PagePath, SessionId, Tier,};+use ai_memory_llm::SyntheticEmbedder;use ai_memory_mcp::{AdminState, admin_router};use ai_memory_store::{DecayParams, Store};use ai_memory_wiki::Wiki;@@ -17,6 +18,7 @@ use ai_memory_wiki::WritePageRequest;use axum::body::Body;use axum::http::{Request, StatusCode};use serde_json::json;+use std::sync::Arc;use tempfile::TempDir;use tower::ServiceExt;@@ -326,6 +328,90 @@ async fn embed_without_embedder_returns_503() {);}+#[tokio::test]+async fn embed_all_projects_rebuilds_workspace_projects() {+ let tmp = TempDir::new().unwrap();+ let (mut state, store) = make_state(&tmp).await;+ let ws = store+ .writer+ .get_or_create_workspace("default")+ .await+ .unwrap();+ let alpha = store+ .writer+ .get_or_create_project(ws, "alpha", None)+ .await+ .unwrap();+ let beta = store+ .writer+ .get_or_create_project(ws, "beta", None)+ .await+ .unwrap();++ state+ .wiki+ .write_page(WritePageRequest {+ workspace_id: ws,+ project_id: alpha,+ path: PagePath::new("notes/a.md").unwrap(),+ frontmatter: serde_json::json!({"title": "A", "tier": "semantic"}),+ body: "alpha content".into(),+ tier: Tier::Semantic,+ pinned: false,+ title: Some("A".into()),+ })+ .await+ .unwrap();+ state+ .wiki+ .write_page(WritePageRequest {+ workspace_id: ws,+ project_id: beta,+ path: PagePath::new("notes/b.md").unwrap(),+ frontmatter: serde_json::json!({"title": "B", "tier": "semantic"}),+ body: "beta content".into(),+ tier: Tier::Semantic,+ pinned: false,+ title: Some("B".into()),+ })+ .await+ .unwrap();++ state.embedder = Some(Arc::new(SyntheticEmbedder::new(64)));+ let resp = post(+ state,+ "/admin/embed",+ json!({+ "workspace": "default",+ "reembed": true,+ "all_projects": true,+ }),+ )+ .await;+ assert_eq!(resp.status(), StatusCode::OK);++ let body = body_json(resp).await;+ assert_eq!(body["embedded"].as_u64().unwrap(), 2, "{body}");+ assert_eq!(+ store+ .reader+ .embedded_page_ids(ws, alpha, "synthetic".into(), "bag-of-words-v1".into(), 64)+ .await+ .unwrap()+ .len(),+ 1+ );+ assert_eq!(+ store+ .reader+ .embedded_page_ids(ws, beta, "synthetic".into(), "bag-of-words-v1".into(), 64)+ .await+ .unwrap()+ .len(),+ 1+ );+}+// ---------------------------------------------------------------------------// commit// ---------------------------------------------------------------------------diff --git a/crates/ai-memory-store/src/lib.rs b/crates/ai-memory-store/src/lib.rsindex 589165ed..61834b03 100644--- a/crates/ai-memory-store/src/lib.rs+++ b/crates/ai-memory-store/src/lib.rs@@ -744,6 +744,8 @@ mod tests {let n = store.writer.delete_stale_page_embeddings(+ ws,+ Some(proj),"openai".into(),"openai/text-embedding-3-small".into(),1536,@@ -762,20 +764,4 @@ mod tests {.unwrap();assert!(mismatch.is_empty());}-- #[test]- fn v07_migration_metadata_for_fork_repair() {- let tmp = TempDir::new().unwrap();- let db_path = tmp.path().join("repair-meta.db");- let mut conn = Connection::open(&db_path).unwrap();- crate::migrations::run_to(&mut conn, 7).unwrap();- let (name, checksum): (String, String) = conn- .query_row(- "SELECT name, checksum FROM refinery_schema_history WHERE version = 7",- [],- |r| Ok((r.get(0)?, r.get(1)?)),- )- .unwrap();- eprintln!("V7_NAME={name} V7_CHECKSUM={checksum}");- }}diff --git a/crates/ai-memory-store/src/ops.rs b/crates/ai-memory-store/src/ops.rsindex 15385129..59acf782 100644--- a/crates/ai-memory-store/src/ops.rs+++ b/crates/ai-memory-store/src/ops.rs@@ -822,25 +822,63 @@ pub fn purge_project(})}-/// Remove embedding rows for latest pages whose `(provider, model, dim)`-/// does not match the configured triple (used before a full re-embed).+/// Remove embedding rows in a workspace/project scope whose `(provider, model, dim)`+/// does not match the configured triple, plus rows tied to superseded pages.pub fn delete_stale_page_embeddings(conn: &mut Connection,+ workspace_id: &WorkspaceId,+ project_id: Option<&ProjectId>,provider: &str,model: &str,dim: u32,) -> StoreResult<u64> {- let n = conn.execute(- "DELETE FROM page_embeddings \- WHERE page_id IN (SELECT id FROM pages WHERE is_latest = 1) \- AND NOT (provider = ?1 AND model = ?2 AND dim = CAST(?3 AS INTEGER))",- params![provider, model, dim],- )?;- let orphans = conn.execute(- "DELETE FROM page_embeddings \- WHERE page_id IN (SELECT id FROM pages WHERE is_latest = 0)",- [],- )?;+ let tx = conn.transaction()?;+ let (n, orphans) = if let Some(project_id) = project_id {+ let n = tx.execute(+ "DELETE FROM page_embeddings \+ WHERE page_id IN (\+ SELECT id FROM pages \+ WHERE workspace_id = ?1 AND project_id = ?2 AND is_latest = 1\+ ) \+ AND NOT (provider = ?3 AND model = ?4 AND dim = CAST(?5 AS INTEGER))",+ params![+ workspace_id.as_bytes(),+ project_id.as_bytes(),+ provider,+ model,+ dim+ ],+ )?;+ let orphans = tx.execute(+ "DELETE FROM page_embeddings \+ WHERE page_id IN (\+ SELECT id FROM pages \+ WHERE workspace_id = ?1 AND project_id = ?2 AND is_latest = 0\+ )",+ params![workspace_id.as_bytes(), project_id.as_bytes()],+ )?;+ (n, orphans)+ } else {+ let n = tx.execute(+ "DELETE FROM page_embeddings \+ WHERE page_id IN (\+ SELECT id FROM pages \+ WHERE workspace_id = ?1 AND is_latest = 1\+ ) \+ AND NOT (provider = ?2 AND model = ?3 AND dim = CAST(?4 AS INTEGER))",+ params![workspace_id.as_bytes(), provider, model, dim],+ )?;+ let orphans = tx.execute(+ "DELETE FROM page_embeddings \+ WHERE page_id IN (\+ SELECT id FROM pages \+ WHERE workspace_id = ?1 AND is_latest = 0\+ )",+ params![workspace_id.as_bytes()],+ )?;+ (n, orphans)+ };+ tx.commit()?;Ok(u64::try_from(n.saturating_add(orphans)).unwrap_or(0))}@@ -1254,8 +1292,12 @@ mod tests {#[test]fn delete_stale_page_embeddings_removes_mismatched_rows() {let (_tmp, mut conn, ws, proj) = fresh_db();+ let other = get_or_create_project(&mut conn, &ws, "other", None).unwrap();let p1 = upsert_page(&mut conn, &page(ws, proj, "a.md", "body a")).unwrap();let p2 = upsert_page(&mut conn, &page(ws, proj, "b.md", "body b")).unwrap();+ let p3 = upsert_page(&mut conn, &page(ws, other, "c.md", "body c")).unwrap();+ let old = upsert_page(&mut conn, &page(ws, proj, "old.md", "old body")).unwrap();+ let _new = upsert_page(&mut conn, &page(ws, proj, "old.md", "new body")).unwrap();store_embedding(&mut conn,&p1,@@ -1265,6 +1307,15 @@ mod tests {768,).unwrap();+ store_embedding(+ &mut conn,+ &p3,+ &[2u8; 4],+ "google",+ "models/gemini-embedding-001",+ 768,+ )+ .unwrap();store_embedding(&mut conn,&p2,@@ -1274,18 +1325,29 @@ mod tests {1536,).unwrap();+ store_embedding(+ &mut conn,+ &old,+ &[3u8; 4],+ "openai",+ "openai/text-embedding-3-small",+ 1536,+ )+ .unwrap();let n = super::delete_stale_page_embeddings(&mut conn,+ &ws,+ Some(&proj),"openai","openai/text-embedding-3-small",1536,).unwrap();- assert_eq!(n, 1);+ assert_eq!(n, 2);let remaining: i64 = conn.query_row("SELECT COUNT(*) FROM page_embeddings", [], |r| r.get(0)).unwrap();- assert_eq!(remaining, 1);+ assert_eq!(remaining, 2);let model: String = conn.query_row("SELECT model FROM page_embeddings WHERE page_id = ?1",@@ -1294,5 +1356,16 @@ mod tests {).unwrap();assert_eq!(model, "openai/text-embedding-3-small");+ let other_rows: i64 = conn+ .query_row(+ "SELECT COUNT(*) FROM page_embeddings WHERE page_id = ?1",+ params![&p3.as_bytes()[..]],+ |r| r.get(0),+ )+ .unwrap();+ assert_eq!(+ other_rows, 1,+ "explicit project purge must not touch siblings"+ );}}diff --git a/crates/ai-memory-store/src/writer.rs b/crates/ai-memory-store/src/writer.rsindex 9764ee16..127badae 100644--- a/crates/ai-memory-store/src/writer.rs+++ b/crates/ai-memory-store/src/writer.rs@@ -95,6 +95,8 @@ pub(crate) enum WriteCmd {reply: oneshot::Sender<StoreResult<()>>,},DeleteStalePageEmbeddings {+ workspace_id: WorkspaceId,+ project_id: Option<ProjectId>,provider: String,model: String,dim: u32,@@ -310,7 +312,7 @@ impl WriterHandle {rx.await.map_err(|_| StoreError::WriterClosed)?}- /// Remove embedding rows whose triple does not match the configured provider/model/dim.+ /// Remove embedding rows in a workspace/project scope whose triple does not match the configured provider/model/dim.////// Used when re-embedding after a model migration (e.g. Gemini → OpenRouter).///@@ -318,12 +320,16 @@ impl WriterHandle {/// Returns [`StoreError::WriterClosed`] or propagates SQL errors.pub async fn delete_stale_page_embeddings(&self,+ workspace_id: WorkspaceId,+ project_id: Option<ProjectId>,provider: String,model: String,dim: u32,) -> StoreResult<u64> {let (tx, rx) = oneshot::channel();self.send(WriteCmd::DeleteStalePageEmbeddings {+ workspace_id,+ project_id,provider,model,dim,@@ -627,12 +633,21 @@ fn worker_loop(mut conn: Connection, mut rx: mpsc::Receiver<WriteCmd>) {send_or_warn(reply, result, "store_embeddings");}WriteCmd::DeleteStalePageEmbeddings {+ workspace_id,+ project_id,provider,model,dim,reply,} => {- let result = ops::delete_stale_page_embeddings(&mut conn, &provider, &model, dim);+ let result = ops::delete_stale_page_embeddings(+ &mut conn,+ &workspace_id,+ project_id.as_ref(),+ &provider,+ &model,+ dim,+ );send_or_warn(reply, result, "delete_stale_page_embeddings");}WriteCmd::PurgeProject {diff --git a/docs/deploy.md b/docs/deploy.mdindex 12b5b708..87c1390b 100644--- a/docs/deploy.md+++ b/docs/deploy.md@@ -221,8 +221,10 @@ ssh "$SERVER" "tail -100 $DEPLOY_DIR/data/logs/ai-memory.log.$(date +%F)"- **Embedding mismatch after a model change**: startup logs a warningwhen stored `(provider, model, dim)` triples differ from config.Hybrid search ignores stale rows until they are re-embedded. Start- the server normally, then run `ai-memory embed --force` (or rely- on scheduled embedding backfill if enabled).+ the server normally, then run `ai-memory embed --force` to rebuild+ every project in the workspace, or add `--project <name>` to scope+ the rebuild. Scheduled embedding backfill can also fill missing+ rows when enabled.- **Container restart loop**: check`docker logs ai-memory` - the `ai-memory starting` line at the topreports the resolved config; a missing required env var (e.g.