Skip to content

Solution for long term memory for agent coding CLIs and to facilitate handoff between different agent vendors

Rust

fix(embed): scope stale embedding cleanup

AkitaOnRails73e2d1a34eac
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 14d7cebc..4d23e9d8 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -15,6 +15,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
- OpenAI-compatible embedding calls now truncate oversized page bodies, surface
provider errors returned in HTTP 200 bodies, retry bounded HTTP 429 responses,
and may reuse `LLM_API_KEY` when a custom embedding base URL is configured.
+- `ai-memory embed --force` without `--project` now re-embeds every project in
+ the workspace and purges stale/superseded embedding rows in the same scope.
## [0.2.0] - 2026-05-26
### Added
diff --git a/crates/ai-memory-cli/src/commands/embed.rs b/crates/ai-memory-cli/src/commands/embed.rs
index 4e621fe2..8c01fee5 100644
--- a/crates/ai-memory-cli/src/commands/embed.rs
+++ b/crates/ai-memory-cli/src/commands/embed.rs
@@ -83,9 +83,8 @@ mod tests {
#[test]
fn force_with_explicit_project_stays_scoped() {
- let cli =
- Cli::try_parse_from(["ai-memory", "embed", "--force", "--project", "consisanet"])
- .unwrap();
+ let cli = Cli::try_parse_from(["ai-memory", "embed", "--force", "--project", "consisanet"])
+ .unwrap();
let Command::Embed(args) = cli.command else {
panic!("expected embed command");
};
diff --git a/crates/ai-memory-mcp/src/admin.rs b/crates/ai-memory-mcp/src/admin.rs
index 3506d9a1..863b5858 100644
--- a/crates/ai-memory-mcp/src/admin.rs
+++ b/crates/ai-memory-mcp/src/admin.rs
@@ -968,23 +968,6 @@ async fn embed_project_pages(
let model = embedder.model().to_string();
let dim = embedder.dim();
- if reembed && !dry_run {
- let purged = state
- .writer
- .delete_stale_page_embeddings(provider.clone(), model.clone(), dim)
- .await
- .map_err(|e| internal_err(e.to_string()))?;
- if purged > 0 {
- info!(
- purged,
- provider = %provider,
- model = %model,
- dim,
- "embed: purged stale page_embeddings before re-embed"
- );
- }
- }
-
let candidates = state
.reader
.decay_candidates(ws, proj)
@@ -1035,10 +1018,6 @@ async fn embed_project_pages(
continue;
}
};
- // Gentle pacing for provider rate limits (OpenRouter free tier, etc.).
- if !dry_run {
- tokio::time::sleep(std::time::Duration::from_millis(800)).await;
- }
pending.push(EmbeddingWrite {
page_id: cand.id,
vector_bytes: f32_vec_to_bytes(&vec),
@@ -1089,49 +1068,56 @@ async fn handle_embed(
let mut totals = EmbedCounts::default();
- if req.reembed && !req.dry_run {
- let purged = state
- .writer
- .delete_stale_page_embeddings(provider.clone(), model.clone(), dim)
- .await
- .map_err(|e| internal_err(e.to_string()))?;
- info!(purged, provider = %provider, model = %model, "purged stale page_embeddings before re-embed");
- }
-
if req.all_projects {
- let summaries = state
+ if let Some(ws) = state
.reader
- .list_projects_with_stats()
+ .find_workspace(req.workspace.clone())
.await
- .map_err(|e| internal_err(e.to_string()))?;
- for summary in summaries
- .into_iter()
- .filter(|p| p.workspace_name == req.workspace)
+ .map_err(|e| internal_err(e.to_string()))?
{
- let (ws, proj) =
- resolve_ws_proj(&state, &summary.workspace_name, &summary.project_name).await?;
- let partial = embed_project_pages(
- &state,
- &embedder,
- ws,
- proj,
- req.reembed,
- req.dry_run,
- )
- .await?;
- totals.absorb(partial);
+ if req.reembed && !req.dry_run {
+ let purged = state
+ .writer
+ .delete_stale_page_embeddings(ws, None, provider.clone(), model.clone(), dim)
+ .await
+ .map_err(|e| internal_err(e.to_string()))?;
+ info!(purged, provider = %provider, model = %model, "purged stale page_embeddings before workspace re-embed");
+ }
+
+ let summaries = state
+ .reader
+ .list_projects_with_stats()
+ .await
+ .map_err(|e| internal_err(e.to_string()))?;
+ for summary in summaries
+ .into_iter()
+ .filter(|p| p.workspace_name == req.workspace)
+ {
+ let Some(proj) = state
+ .reader
+ .find_project(ws, summary.project_name.clone())
+ .await
+ .map_err(|e| internal_err(e.to_string()))?
+ else {
+ continue;
+ };
+ let partial =
+ embed_project_pages(&state, &embedder, ws, proj, req.reembed, req.dry_run)
+ .await?;
+ totals.absorb(partial);
+ }
}
} else {
let (ws, proj) = resolve_ws_proj(&state, &req.workspace, &req.project).await?;
- totals = embed_project_pages(
- &state,
- &embedder,
- ws,
- proj,
- req.reembed,
- req.dry_run,
- )
- .await?;
+ if req.reembed && !req.dry_run {
+ let purged = state
+ .writer
+ .delete_stale_page_embeddings(ws, Some(proj), provider.clone(), model.clone(), dim)
+ .await
+ .map_err(|e| internal_err(e.to_string()))?;
+ info!(purged, provider = %provider, model = %model, "purged stale page_embeddings before project re-embed");
+ }
+ totals = embed_project_pages(&state, &embedder, ws, proj, req.reembed, req.dry_run).await?;
}
let report = EmbedReport {
diff --git a/crates/ai-memory-mcp/tests/admin_phase3.rs b/crates/ai-memory-mcp/tests/admin_phase3.rs
index 1dcebfcd..0553c68c 100644
--- a/crates/ai-memory-mcp/tests/admin_phase3.rs
+++ b/crates/ai-memory-mcp/tests/admin_phase3.rs
@@ -10,6 +10,7 @@
use ai_memory_core::{
AgentKind, NewObservation, NewPage, NewSession, ObservationKind, PagePath, SessionId, Tier,
};
+use ai_memory_llm::SyntheticEmbedder;
use ai_memory_mcp::{AdminState, admin_router};
use ai_memory_store::{DecayParams, Store};
use ai_memory_wiki::Wiki;
@@ -17,6 +18,7 @@ use ai_memory_wiki::WritePageRequest;
use axum::body::Body;
use axum::http::{Request, StatusCode};
use serde_json::json;
+use std::sync::Arc;
use tempfile::TempDir;
use tower::ServiceExt;
@@ -326,6 +328,90 @@ async fn embed_without_embedder_returns_503() {
);
}
+#[tokio::test]
+async fn embed_all_projects_rebuilds_workspace_projects() {
+ let tmp = TempDir::new().unwrap();
+ let (mut state, store) = make_state(&tmp).await;
+ let ws = store
+ .writer
+ .get_or_create_workspace("default")
+ .await
+ .unwrap();
+ let alpha = store
+ .writer
+ .get_or_create_project(ws, "alpha", None)
+ .await
+ .unwrap();
+ let beta = store
+ .writer
+ .get_or_create_project(ws, "beta", None)
+ .await
+ .unwrap();
+
+ state
+ .wiki
+ .write_page(WritePageRequest {
+ workspace_id: ws,
+ project_id: alpha,
+ path: PagePath::new("notes/a.md").unwrap(),
+ frontmatter: serde_json::json!({"title": "A", "tier": "semantic"}),
+ body: "alpha content".into(),
+ tier: Tier::Semantic,
+ pinned: false,
+ title: Some("A".into()),
+ })
+ .await
+ .unwrap();
+ state
+ .wiki
+ .write_page(WritePageRequest {
+ workspace_id: ws,
+ project_id: beta,
+ path: PagePath::new("notes/b.md").unwrap(),
+ frontmatter: serde_json::json!({"title": "B", "tier": "semantic"}),
+ body: "beta content".into(),
+ tier: Tier::Semantic,
+ pinned: false,
+ title: Some("B".into()),
+ })
+ .await
+ .unwrap();
+
+ state.embedder = Some(Arc::new(SyntheticEmbedder::new(64)));
+ let resp = post(
+ state,
+ "/admin/embed",
+ json!({
+ "workspace": "default",
+ "reembed": true,
+ "all_projects": true,
+ }),
+ )
+ .await;
+ assert_eq!(resp.status(), StatusCode::OK);
+
+ let body = body_json(resp).await;
+ assert_eq!(body["embedded"].as_u64().unwrap(), 2, "{body}");
+ assert_eq!(
+ store
+ .reader
+ .embedded_page_ids(ws, alpha, "synthetic".into(), "bag-of-words-v1".into(), 64)
+ .await
+ .unwrap()
+ .len(),
+ 1
+ );
+ assert_eq!(
+ store
+ .reader
+ .embedded_page_ids(ws, beta, "synthetic".into(), "bag-of-words-v1".into(), 64)
+ .await
+ .unwrap()
+ .len(),
+ 1
+ );
+}
+
// ---------------------------------------------------------------------------
// commit
// ---------------------------------------------------------------------------
diff --git a/crates/ai-memory-store/src/lib.rs b/crates/ai-memory-store/src/lib.rs
index 589165ed..61834b03 100644
--- a/crates/ai-memory-store/src/lib.rs
+++ b/crates/ai-memory-store/src/lib.rs
@@ -744,6 +744,8 @@ mod tests {
let n = store
.writer
.delete_stale_page_embeddings(
+ ws,
+ Some(proj),
"openai".into(),
"openai/text-embedding-3-small".into(),
1536,
@@ -762,20 +764,4 @@ mod tests {
.unwrap();
assert!(mismatch.is_empty());
}
-
- #[test]
- fn v07_migration_metadata_for_fork_repair() {
- let tmp = TempDir::new().unwrap();
- let db_path = tmp.path().join("repair-meta.db");
- let mut conn = Connection::open(&db_path).unwrap();
- crate::migrations::run_to(&mut conn, 7).unwrap();
- let (name, checksum): (String, String) = conn
- .query_row(
- "SELECT name, checksum FROM refinery_schema_history WHERE version = 7",
- [],
- |r| Ok((r.get(0)?, r.get(1)?)),
- )
- .unwrap();
- eprintln!("V7_NAME={name} V7_CHECKSUM={checksum}");
- }
}
diff --git a/crates/ai-memory-store/src/ops.rs b/crates/ai-memory-store/src/ops.rs
index 15385129..59acf782 100644
--- a/crates/ai-memory-store/src/ops.rs
+++ b/crates/ai-memory-store/src/ops.rs
@@ -822,25 +822,63 @@ pub fn purge_project(
})
}
-/// Remove embedding rows for latest pages whose `(provider, model, dim)`
-/// does not match the configured triple (used before a full re-embed).
+/// Remove embedding rows in a workspace/project scope whose `(provider, model, dim)`
+/// does not match the configured triple, plus rows tied to superseded pages.
pub fn delete_stale_page_embeddings(
conn: &mut Connection,
+ workspace_id: &WorkspaceId,
+ project_id: Option<&ProjectId>,
provider: &str,
model: &str,
dim: u32,
) -> StoreResult<u64> {
- let n = conn.execute(
- "DELETE FROM page_embeddings \
- WHERE page_id IN (SELECT id FROM pages WHERE is_latest = 1) \
- AND NOT (provider = ?1 AND model = ?2 AND dim = CAST(?3 AS INTEGER))",
- params![provider, model, dim],
- )?;
- let orphans = conn.execute(
- "DELETE FROM page_embeddings \
- WHERE page_id IN (SELECT id FROM pages WHERE is_latest = 0)",
- [],
- )?;
+ let tx = conn.transaction()?;
+ let (n, orphans) = if let Some(project_id) = project_id {
+ let n = tx.execute(
+ "DELETE FROM page_embeddings \
+ WHERE page_id IN (\
+ SELECT id FROM pages \
+ WHERE workspace_id = ?1 AND project_id = ?2 AND is_latest = 1\
+ ) \
+ AND NOT (provider = ?3 AND model = ?4 AND dim = CAST(?5 AS INTEGER))",
+ params![
+ workspace_id.as_bytes(),
+ project_id.as_bytes(),
+ provider,
+ model,
+ dim
+ ],
+ )?;
+ let orphans = tx.execute(
+ "DELETE FROM page_embeddings \
+ WHERE page_id IN (\
+ SELECT id FROM pages \
+ WHERE workspace_id = ?1 AND project_id = ?2 AND is_latest = 0\
+ )",
+ params![workspace_id.as_bytes(), project_id.as_bytes()],
+ )?;
+ (n, orphans)
+ } else {
+ let n = tx.execute(
+ "DELETE FROM page_embeddings \
+ WHERE page_id IN (\
+ SELECT id FROM pages \
+ WHERE workspace_id = ?1 AND is_latest = 1\
+ ) \
+ AND NOT (provider = ?2 AND model = ?3 AND dim = CAST(?4 AS INTEGER))",
+ params![workspace_id.as_bytes(), provider, model, dim],
+ )?;
+ let orphans = tx.execute(
+ "DELETE FROM page_embeddings \
+ WHERE page_id IN (\
+ SELECT id FROM pages \
+ WHERE workspace_id = ?1 AND is_latest = 0\
+ )",
+ params![workspace_id.as_bytes()],
+ )?;
+ (n, orphans)
+ };
+ tx.commit()?;
Ok(u64::try_from(n.saturating_add(orphans)).unwrap_or(0))
}
@@ -1254,8 +1292,12 @@ mod tests {
#[test]
fn delete_stale_page_embeddings_removes_mismatched_rows() {
let (_tmp, mut conn, ws, proj) = fresh_db();
+ let other = get_or_create_project(&mut conn, &ws, "other", None).unwrap();
let p1 = upsert_page(&mut conn, &page(ws, proj, "a.md", "body a")).unwrap();
let p2 = upsert_page(&mut conn, &page(ws, proj, "b.md", "body b")).unwrap();
+ let p3 = upsert_page(&mut conn, &page(ws, other, "c.md", "body c")).unwrap();
+ let old = upsert_page(&mut conn, &page(ws, proj, "old.md", "old body")).unwrap();
+ let _new = upsert_page(&mut conn, &page(ws, proj, "old.md", "new body")).unwrap();
store_embedding(
&mut conn,
&p1,
@@ -1265,6 +1307,15 @@ mod tests {
768,
)
.unwrap();
+ store_embedding(
+ &mut conn,
+ &p3,
+ &[2u8; 4],
+ "google",
+ "models/gemini-embedding-001",
+ 768,
+ )
+ .unwrap();
store_embedding(
&mut conn,
&p2,
@@ -1274,18 +1325,29 @@ mod tests {
1536,
)
.unwrap();
+ store_embedding(
+ &mut conn,
+ &old,
+ &[3u8; 4],
+ "openai",
+ "openai/text-embedding-3-small",
+ 1536,
+ )
+ .unwrap();
let n = super::delete_stale_page_embeddings(
&mut conn,
+ &ws,
+ Some(&proj),
"openai",
"openai/text-embedding-3-small",
1536,
)
.unwrap();
- assert_eq!(n, 1);
+ assert_eq!(n, 2);
let remaining: i64 = conn
.query_row("SELECT COUNT(*) FROM page_embeddings", [], |r| r.get(0))
.unwrap();
- assert_eq!(remaining, 1);
+ assert_eq!(remaining, 2);
let model: String = conn
.query_row(
"SELECT model FROM page_embeddings WHERE page_id = ?1",
@@ -1294,5 +1356,16 @@ mod tests {
)
.unwrap();
assert_eq!(model, "openai/text-embedding-3-small");
+ let other_rows: i64 = conn
+ .query_row(
+ "SELECT COUNT(*) FROM page_embeddings WHERE page_id = ?1",
+ params![&p3.as_bytes()[..]],
+ |r| r.get(0),
+ )
+ .unwrap();
+ assert_eq!(
+ other_rows, 1,
+ "explicit project purge must not touch siblings"
+ );
}
}
diff --git a/crates/ai-memory-store/src/writer.rs b/crates/ai-memory-store/src/writer.rs
index 9764ee16..127badae 100644
--- a/crates/ai-memory-store/src/writer.rs
+++ b/crates/ai-memory-store/src/writer.rs
@@ -95,6 +95,8 @@ pub(crate) enum WriteCmd {
reply: oneshot::Sender<StoreResult<()>>,
},
DeleteStalePageEmbeddings {
+ workspace_id: WorkspaceId,
+ project_id: Option<ProjectId>,
provider: String,
model: String,
dim: u32,
@@ -310,7 +312,7 @@ impl WriterHandle {
rx.await.map_err(|_| StoreError::WriterClosed)?
}
- /// Remove embedding rows whose triple does not match the configured provider/model/dim.
+ /// Remove embedding rows in a workspace/project scope whose triple does not match the configured provider/model/dim.
///
/// Used when re-embedding after a model migration (e.g. Gemini → OpenRouter).
///
@@ -318,12 +320,16 @@ impl WriterHandle {
/// Returns [`StoreError::WriterClosed`] or propagates SQL errors.
pub async fn delete_stale_page_embeddings(
&self,
+ workspace_id: WorkspaceId,
+ project_id: Option<ProjectId>,
provider: String,
model: String,
dim: u32,
) -> StoreResult<u64> {
let (tx, rx) = oneshot::channel();
self.send(WriteCmd::DeleteStalePageEmbeddings {
+ workspace_id,
+ project_id,
provider,
model,
dim,
@@ -627,12 +633,21 @@ fn worker_loop(mut conn: Connection, mut rx: mpsc::Receiver<WriteCmd>) {
send_or_warn(reply, result, "store_embeddings");
}
WriteCmd::DeleteStalePageEmbeddings {
+ workspace_id,
+ project_id,
provider,
model,
dim,
reply,
} => {
- let result = ops::delete_stale_page_embeddings(&mut conn, &provider, &model, dim);
+ let result = ops::delete_stale_page_embeddings(
+ &mut conn,
+ &workspace_id,
+ project_id.as_ref(),
+ &provider,
+ &model,
+ dim,
+ );
send_or_warn(reply, result, "delete_stale_page_embeddings");
}
WriteCmd::PurgeProject {
diff --git a/docs/deploy.md b/docs/deploy.md
index 12b5b708..87c1390b 100644
--- a/docs/deploy.md
+++ b/docs/deploy.md
@@ -221,8 +221,10 @@ ssh "$SERVER" "tail -100 $DEPLOY_DIR/data/logs/ai-memory.log.$(date +%F)"
- **Embedding mismatch after a model change**: startup logs a warning
when stored `(provider, model, dim)` triples differ from config.
Hybrid search ignores stale rows until they are re-embedded. Start
- the server normally, then run `ai-memory embed --force` (or rely
- on scheduled embedding backfill if enabled).
+ the server normally, then run `ai-memory embed --force` to rebuild
+ every project in the workspace, or add `--project <name>` to scope
+ the rebuild. Scheduled embedding backfill can also fill missing
+ rows when enabled.
- **Container restart loop**: check
`docker logs ai-memory` - the `ai-memory starting` line at the top
reports the resolved config; a missing required env var (e.g.