Skip to content

Solution for long term memory for agent coding CLIs and to facilitate handoff between different agent vendors

Rust

fix(embed,store): re-embed all projects and purge stale embeddings

pedrofjre6207019af7b
diff --git a/crates/ai-memory-cli/src/commands/embed.rs b/crates/ai-memory-cli/src/commands/embed.rs
index 57fd62db..4e621fe2 100644
--- a/crates/ai-memory-cli/src/commands/embed.rs
+++ b/crates/ai-memory-cli/src/commands/embed.rs
@@ -14,6 +14,7 @@ struct EmbedRequest {
project: String,
reembed: bool,
dry_run: bool,
+ all_projects: bool,
}
/// Run the `embed` subcommand.
@@ -27,7 +28,15 @@ struct EmbedRequest {
/// response.
pub async fn run(config: &Config, args: EmbedArgs) -> Result<()> {
let endpoint = ServerEndpoint::from_config(config);
- let project = super::resolve_project_name(config, args.project.as_deref())?;
+ // Model migrations must reach stale rows in every project namespace.
+ // Without an explicit `--project`, `--force` / `--reembed` fans out
+ // across the whole workspace instead of the CWD-derived project only.
+ let all_projects = args.force && args.project.is_none();
+ let project = if all_projects {
+ String::new()
+ } else {
+ super::resolve_project_name(config, args.project.as_deref())?
+ };
let report: serde_json::Value = post_json(
&endpoint,
"/admin/embed",
@@ -38,6 +47,7 @@ pub async fn run(config: &Config, args: EmbedArgs) -> Result<()> {
// field is `reembed` — map them here.
reembed: args.force,
dry_run: args.dry_run,
+ all_projects,
},
)
.await?;
@@ -51,3 +61,38 @@ pub async fn run(config: &Config, args: EmbedArgs) -> Result<()> {
}
Ok(())
}
+
+#[cfg(test)]
+mod tests {
+ use crate::cli::{Cli, Command};
+ use clap::Parser;
+
+ #[test]
+ fn force_without_project_enables_all_projects() {
+ let cli = Cli::try_parse_from(["ai-memory", "embed", "--force"]).unwrap();
+ let Command::Embed(args) = cli.command else {
+ panic!("expected embed command");
+ };
+ assert!(args.force);
+ assert!(args.project.is_none());
+ assert!(
+ args.force && args.project.is_none(),
+ "--force without --project must fan out to all projects"
+ );
+ }
+
+ #[test]
+ fn force_with_explicit_project_stays_scoped() {
+ let cli =
+ Cli::try_parse_from(["ai-memory", "embed", "--force", "--project", "consisanet"])
+ .unwrap();
+ let Command::Embed(args) = cli.command else {
+ panic!("expected embed command");
+ };
+ let all_projects = args.force && args.project.is_none();
+ assert!(
+ !all_projects,
+ "--force with --project must stay scoped to that project"
+ );
+ }
+}
diff --git a/crates/ai-memory-mcp/src/admin.rs b/crates/ai-memory-mcp/src/admin.rs
index 9cfd475e..3506d9a1 100644
--- a/crates/ai-memory-mcp/src/admin.rs
+++ b/crates/ai-memory-mcp/src/admin.rs
@@ -899,7 +899,8 @@ struct EmbedRequest {
/// Workspace name (auto-created if absent).
#[serde(default = "default_workspace")]
workspace: String,
- /// Project name (auto-created if absent).
+ /// Project name (auto-created if absent). Ignored when
+ /// [`Self::all_projects`] is true.
#[serde(default = "default_project")]
project: String,
/// When true, regenerates embeddings even for pages that already
@@ -910,6 +911,12 @@ struct EmbedRequest {
/// calling the embedder or writing anything.
#[serde(default)]
dry_run: bool,
+ /// When true, embed every project in `workspace` instead of a
+ /// single `project`. The CLI sets this for `--force` /
+ /// `--reembed` without an explicit `--project` so model migrations
+ /// reach stale rows in every namespace.
+ #[serde(default)]
+ all_projects: bool,
}
/// Summary response from `POST /admin/embed`.
@@ -932,27 +939,51 @@ pub struct EmbedReport {
pub dim: u32,
}
-async fn handle_embed(
- State(state): State<Arc<AdminState>>,
- Json(req): Json<EmbedRequest>,
-) -> Result<impl IntoResponse, (StatusCode, Json<serde_json::Value>)> {
- let embedder = match state.embedder.clone() {
- Some(e) => e,
- None => {
- return Err((
- StatusCode::SERVICE_UNAVAILABLE,
- Json(serde_json::json!({
- "error": "embedder not configured on server"
- })),
- ));
- }
- };
+#[derive(Default)]
+struct EmbedCounts {
+ embedded: usize,
+ skipped: usize,
+ failed: usize,
+ would_embed: usize,
+}
+
+impl EmbedCounts {
+ fn absorb(&mut self, other: Self) {
+ self.embedded += other.embedded;
+ self.skipped += other.skipped;
+ self.failed += other.failed;
+ self.would_embed += other.would_embed;
+ }
+}
+async fn embed_project_pages(
+ state: &AdminState,
+ embedder: &Arc<dyn Embedder>,
+ ws: WorkspaceId,
+ proj: ProjectId,
+ reembed: bool,
+ dry_run: bool,
+) -> Result<EmbedCounts, (StatusCode, Json<serde_json::Value>)> {
let provider = embedder.provider().to_string();
let model = embedder.model().to_string();
let dim = embedder.dim();
- let (ws, proj) = resolve_ws_proj(&state, &req.workspace, &req.project).await?;
+ if reembed && !dry_run {
+ let purged = state
+ .writer
+ .delete_stale_page_embeddings(provider.clone(), model.clone(), dim)
+ .await
+ .map_err(|e| internal_err(e.to_string()))?;
+ if purged > 0 {
+ info!(
+ purged,
+ provider = %provider,
+ model = %model,
+ dim,
+ "embed: purged stale page_embeddings before re-embed"
+ );
+ }
+ }
let candidates = state
.reader
@@ -960,8 +991,7 @@ async fn handle_embed(
.await
.map_err(|e| internal_err(e.to_string()))?;
- // Build the set of page ids that already have a matching embedding.
- let already: std::collections::HashSet<_> = if req.reembed {
+ let already: std::collections::HashSet<_> = if reembed {
std::collections::HashSet::new()
} else {
state
@@ -973,37 +1003,42 @@ async fn handle_embed(
.collect()
};
- let mut embedded = 0_usize;
- let mut skipped = 0_usize;
- let mut failed = 0_usize;
- let mut would_embed = 0_usize;
+ let mut counts = EmbedCounts::default();
let mut pending = Vec::with_capacity(EMBEDDING_WRITE_BATCH);
for cand in candidates {
- if !req.reembed && already.contains(&cand.id) {
- skipped += 1;
+ if !reembed && already.contains(&cand.id) {
+ counts.skipped += 1;
continue;
}
- if req.dry_run {
- would_embed += 1;
+ if dry_run {
+ counts.would_embed += 1;
continue;
}
let md = match state.wiki.read_page(ws, proj, &cand.path) {
Ok(m) => m,
Err(e) => {
warn!(path = %cand.path, error = %e, "embed: skip unreadable page");
- failed += 1;
+ counts.failed += 1;
continue;
}
};
+ if md.body.trim().is_empty() {
+ counts.skipped += 1;
+ continue;
+ }
let vec = match embedder.embed_document(&md.body).await {
Ok(v) => v,
Err(e) => {
warn!(path = %cand.path, error = %e, "embed: provider call failed");
- failed += 1;
+ counts.failed += 1;
continue;
}
};
+ // Gentle pacing for provider rate limits (OpenRouter free tier, etc.).
+ if !dry_run {
+ tokio::time::sleep(std::time::Duration::from_millis(800)).await;
+ }
pending.push(EmbeddingWrite {
page_id: cand.id,
vector_bytes: f32_vec_to_bytes(&vec),
@@ -1012,16 +1047,98 @@ async fn handle_embed(
dim,
});
if pending.len() >= EMBEDDING_WRITE_BATCH {
- flush_embedding_batch(&state.writer, &mut pending, &mut embedded, &mut failed).await;
+ flush_embedding_batch(
+ &state.writer,
+ &mut pending,
+ &mut counts.embedded,
+ &mut counts.failed,
+ )
+ .await;
}
}
- flush_embedding_batch(&state.writer, &mut pending, &mut embedded, &mut failed).await;
+ flush_embedding_batch(
+ &state.writer,
+ &mut pending,
+ &mut counts.embedded,
+ &mut counts.failed,
+ )
+ .await;
+
+ Ok(counts)
+}
+
+async fn handle_embed(
+ State(state): State<Arc<AdminState>>,
+ Json(req): Json<EmbedRequest>,
+) -> Result<impl IntoResponse, (StatusCode, Json<serde_json::Value>)> {
+ let embedder = match state.embedder.clone() {
+ Some(e) => e,
+ None => {
+ return Err((
+ StatusCode::SERVICE_UNAVAILABLE,
+ Json(serde_json::json!({
+ "error": "embedder not configured on server"
+ })),
+ ));
+ }
+ };
+
+ let provider = embedder.provider().to_string();
+ let model = embedder.model().to_string();
+ let dim = embedder.dim();
+
+ let mut totals = EmbedCounts::default();
+
+ if req.reembed && !req.dry_run {
+ let purged = state
+ .writer
+ .delete_stale_page_embeddings(provider.clone(), model.clone(), dim)
+ .await
+ .map_err(|e| internal_err(e.to_string()))?;
+ info!(purged, provider = %provider, model = %model, "purged stale page_embeddings before re-embed");
+ }
+
+ if req.all_projects {
+ let summaries = state
+ .reader
+ .list_projects_with_stats()
+ .await
+ .map_err(|e| internal_err(e.to_string()))?;
+ for summary in summaries
+ .into_iter()
+ .filter(|p| p.workspace_name == req.workspace)
+ {
+ let (ws, proj) =
+ resolve_ws_proj(&state, &summary.workspace_name, &summary.project_name).await?;
+ let partial = embed_project_pages(
+ &state,
+ &embedder,
+ ws,
+ proj,
+ req.reembed,
+ req.dry_run,
+ )
+ .await?;
+ totals.absorb(partial);
+ }
+ } else {
+ let (ws, proj) = resolve_ws_proj(&state, &req.workspace, &req.project).await?;
+ totals = embed_project_pages(
+ &state,
+ &embedder,
+ ws,
+ proj,
+ req.reembed,
+ req.dry_run,
+ )
+ .await?;
+ }
let report = EmbedReport {
- embedded,
- skipped,
- failed,
- would_embed,
+ embedded: totals.embedded,
+ skipped: totals.skipped,
+ failed: totals.failed,
+ would_embed: totals.would_embed,
provider,
model,
dim,
diff --git a/crates/ai-memory-store/src/lib.rs b/crates/ai-memory-store/src/lib.rs
index e2d0cf17..589165ed 100644
--- a/crates/ai-memory-store/src/lib.rs
+++ b/crates/ai-memory-store/src/lib.rs
@@ -694,4 +694,88 @@ mod tests {
.unwrap();
assert!(none.is_none());
}
+
+ #[tokio::test]
+ async fn delete_stale_page_embeddings_removes_mismatched_rows() {
+ let tmp = TempDir::new().unwrap();
+ let store = Store::open(tmp.path()).unwrap();
+ let ws = store
+ .writer
+ .get_or_create_workspace("default")
+ .await
+ .unwrap();
+ let proj = store
+ .writer
+ .get_or_create_project(ws, "test", None)
+ .await
+ .unwrap();
+ let p1 = store
+ .writer
+ .upsert_page(sample_page(ws, proj, "a.md", "body a"))
+ .await
+ .unwrap();
+ let p2 = store
+ .writer
+ .upsert_page(sample_page(ws, proj, "b.md", "body b"))
+ .await
+ .unwrap();
+ store
+ .writer
+ .store_embedding(
+ p1,
+ vec![0u8; 4],
+ "google".into(),
+ "models/gemini-embedding-001".into(),
+ 768,
+ )
+ .await
+ .unwrap();
+ store
+ .writer
+ .store_embedding(
+ p2,
+ vec![1u8; 4],
+ "openai".into(),
+ "openai/text-embedding-3-small".into(),
+ 1536,
+ )
+ .await
+ .unwrap();
+ let n = store
+ .writer
+ .delete_stale_page_embeddings(
+ "openai".into(),
+ "openai/text-embedding-3-small".into(),
+ 1536,
+ )
+ .await
+ .unwrap();
+ assert_eq!(n, 1);
+ let mismatch = store
+ .reader
+ .embedding_meta_for_mismatch(
+ "openai".into(),
+ "openai/text-embedding-3-small".into(),
+ 1536,
+ )
+ .await
+ .unwrap();
+ assert!(mismatch.is_empty());
+ }
+
+ #[test]
+ fn v07_migration_metadata_for_fork_repair() {
+ let tmp = TempDir::new().unwrap();
+ let db_path = tmp.path().join("repair-meta.db");
+ let mut conn = Connection::open(&db_path).unwrap();
+ crate::migrations::run_to(&mut conn, 7).unwrap();
+ let (name, checksum): (String, String) = conn
+ .query_row(
+ "SELECT name, checksum FROM refinery_schema_history WHERE version = 7",
+ [],
+ |r| Ok((r.get(0)?, r.get(1)?)),
+ )
+ .unwrap();
+ eprintln!("V7_NAME={name} V7_CHECKSUM={checksum}");
+ }
}
diff --git a/crates/ai-memory-store/src/ops.rs b/crates/ai-memory-store/src/ops.rs
index 02949c52..15385129 100644
--- a/crates/ai-memory-store/src/ops.rs
+++ b/crates/ai-memory-store/src/ops.rs
@@ -822,6 +822,28 @@ pub fn purge_project(
})
}
+/// Remove embedding rows for latest pages whose `(provider, model, dim)`
+/// does not match the configured triple (used before a full re-embed).
+pub fn delete_stale_page_embeddings(
+ conn: &mut Connection,
+ provider: &str,
+ model: &str,
+ dim: u32,
+) -> StoreResult<u64> {
+ let n = conn.execute(
+ "DELETE FROM page_embeddings \
+ WHERE page_id IN (SELECT id FROM pages WHERE is_latest = 1) \
+ AND NOT (provider = ?1 AND model = ?2 AND dim = CAST(?3 AS INTEGER))",
+ params![provider, model, dim],
+ )?;
+ let orphans = conn.execute(
+ "DELETE FROM page_embeddings \
+ WHERE page_id IN (SELECT id FROM pages WHERE is_latest = 0)",
+ [],
+ )?;
+ Ok(u64::try_from(n.saturating_add(orphans)).unwrap_or(0))
+}
+
#[cfg(test)]
mod tests {
//! Focused unit tests for the load-bearing mutating SQL paths.
@@ -1228,4 +1250,49 @@ mod tests {
.unwrap();
assert_eq!(count, 2);
}
+
+ #[test]
+ fn delete_stale_page_embeddings_removes_mismatched_rows() {
+ let (_tmp, mut conn, ws, proj) = fresh_db();
+ let p1 = upsert_page(&mut conn, &page(ws, proj, "a.md", "body a")).unwrap();
+ let p2 = upsert_page(&mut conn, &page(ws, proj, "b.md", "body b")).unwrap();
+ store_embedding(
+ &mut conn,
+ &p1,
+ &[0u8; 4],
+ "google",
+ "models/gemini-embedding-001",
+ 768,
+ )
+ .unwrap();
+ store_embedding(
+ &mut conn,
+ &p2,
+ &[1u8; 4],
+ "openai",
+ "openai/text-embedding-3-small",
+ 1536,
+ )
+ .unwrap();
+ let n = super::delete_stale_page_embeddings(
+ &mut conn,
+ "openai",
+ "openai/text-embedding-3-small",
+ 1536,
+ )
+ .unwrap();
+ assert_eq!(n, 1);
+ let remaining: i64 = conn
+ .query_row("SELECT COUNT(*) FROM page_embeddings", [], |r| r.get(0))
+ .unwrap();
+ assert_eq!(remaining, 1);
+ let model: String = conn
+ .query_row(
+ "SELECT model FROM page_embeddings WHERE page_id = ?1",
+ params![&p2.as_bytes()[..]],
+ |r| r.get(0),
+ )
+ .unwrap();
+ assert_eq!(model, "openai/text-embedding-3-small");
+ }
}
diff --git a/crates/ai-memory-store/src/reader.rs b/crates/ai-memory-store/src/reader.rs
index 2ad66444..3242ab59 100644
--- a/crates/ai-memory-store/src/reader.rs
+++ b/crates/ai-memory-store/src/reader.rs
@@ -875,10 +875,11 @@ impl ReaderPool {
) -> StoreResult<Vec<(String, String, u32, u64)>> {
self.with_conn(move |conn| {
let mut stmt = conn.prepare(
- "SELECT provider, model, dim, COUNT(*) \
- FROM page_embeddings \
- WHERE NOT (provider = ?1 AND model = ?2 AND dim = ?3) \
- GROUP BY provider, model, dim",
+ "SELECT pe.provider, pe.model, pe.dim, COUNT(*) \
+ FROM page_embeddings pe \
+ JOIN pages pg ON pg.id = pe.page_id AND pg.is_latest = 1 \
+ WHERE NOT (pe.provider = ?1 AND pe.model = ?2 AND pe.dim = ?3) \
+ GROUP BY pe.provider, pe.model, pe.dim",
)?;
let rows = stmt.query_map(params![provider, model, dim], |row| {
let provider: String = row.get(0)?;
diff --git a/crates/ai-memory-store/src/writer.rs b/crates/ai-memory-store/src/writer.rs
index ce9b1e08..9764ee16 100644
--- a/crates/ai-memory-store/src/writer.rs
+++ b/crates/ai-memory-store/src/writer.rs
@@ -94,6 +94,12 @@ pub(crate) enum WriteCmd {
embeddings: Vec<EmbeddingWrite>,
reply: oneshot::Sender<StoreResult<()>>,
},
+ DeleteStalePageEmbeddings {
+ provider: String,
+ model: String,
+ dim: u32,
+ reply: oneshot::Sender<StoreResult<u64>>,
+ },
/// Delete a project and all its data (pages, sessions, observations,
/// handoffs, embeddings) in one transaction. Returns the paths of
/// every page file that must be removed from disk by the caller.
@@ -304,6 +310,29 @@ impl WriterHandle {
rx.await.map_err(|_| StoreError::WriterClosed)?
}
+ /// Remove embedding rows whose triple does not match the configured provider/model/dim.
+ ///
+ /// Used when re-embedding after a model migration (e.g. Gemini → OpenRouter).
+ ///
+ /// # Errors
+ /// Returns [`StoreError::WriterClosed`] or propagates SQL errors.
+ pub async fn delete_stale_page_embeddings(
+ &self,
+ provider: String,
+ model: String,
+ dim: u32,
+ ) -> StoreResult<u64> {
+ let (tx, rx) = oneshot::channel();
+ self.send(WriteCmd::DeleteStalePageEmbeddings {
+ provider,
+ model,
+ dim,
+ reply: tx,
+ })
+ .await?;
+ rx.await.map_err(|_| StoreError::WriterClosed)?
+ }
+
/// Bump access counters for a set of pages (M8 reinforcement term).
///
/// # Errors
@@ -597,6 +626,15 @@ fn worker_loop(mut conn: Connection, mut rx: mpsc::Receiver<WriteCmd>) {
let result = ops::store_embeddings(&mut conn, &embeddings);
send_or_warn(reply, result, "store_embeddings");
}
+ WriteCmd::DeleteStalePageEmbeddings {
+ provider,
+ model,
+ dim,
+ reply,
+ } => {
+ let result = ops::delete_stale_page_embeddings(&mut conn, &provider, &model, dim);
+ send_or_warn(reply, result, "delete_stale_page_embeddings");
+ }
WriteCmd::PurgeProject {
workspace_id,
project_id,