Skip to content

Solution for long term memory for agent coding CLIs and to facilitate handoff between different agent vendors

Rust

perf: batch graph and embedding writes

AkitaOnRailsed2d29a895b0
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 8b363437..b1b44904 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -31,6 +31,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
### Changed
- Same-body page upserts are now true no-ops, avoiding periodic watcher
reconcile writes, FTS churn, and misleading recent-page timestamps.
+- Graph-neighbor expansion for hybrid search now batches all seed pages into
+ one SQL query instead of issuing incoming/outgoing lookups per seed.
+- Embedding backfill stores embeddings in chunks instead of one writer
+ command and SQLite transaction per page.
- Documented the vector backend policy and the measured criteria required
before adding `sqlite-vec`.
- Clarified Gemini CLI support docs: MCP registration, lifecycle hooks,
diff --git a/crates/ai-memory-cli/src/commands/serve.rs b/crates/ai-memory-cli/src/commands/serve.rs
index 09c3e289..3bf06035 100644
--- a/crates/ai-memory-cli/src/commands/serve.rs
+++ b/crates/ai-memory-cli/src/commands/serve.rs
@@ -7,7 +7,7 @@ use ai_memory_core::{ActiveProject, ProjectId, Sanitizer, WorkspaceId};
use ai_memory_hooks::{HookState, hook_router};
use ai_memory_llm::{Embedder, LlmProvider, build_embedder, build_provider};
use ai_memory_mcp::{AdminState, AiMemoryServer, admin_router};
-use ai_memory_store::{ReaderPool, Store, WriterHandle, f32_vec_to_bytes};
+use ai_memory_store::{EmbeddingWrite, ReaderPool, Store, WriterHandle, f32_vec_to_bytes};
use ai_memory_web;
use ai_memory_wiki::{WatcherHandle, Wiki, migrations, run_wiki_migrations};
use anyhow::{Context, Result};
@@ -35,6 +35,7 @@ use crate::config::{Config, MaintenanceSettings};
/// 10 MB is generous headroom; without a cap, axum streams unbounded
/// bodies into memory (audit critical #2).
const MAX_BODY_BYTES: usize = 10 * 1024 * 1024;
+const EMBEDDING_WRITE_BATCH: usize = 100;
struct ConsolidatorSetup {
server: AiMemoryServer,
@@ -359,6 +360,7 @@ async fn run_embedding_backfill(
let mut embedded = 0usize;
let mut failed = 0usize;
+ let mut pending = Vec::with_capacity(EMBEDDING_WRITE_BATCH);
for cand in candidates {
if already.contains(&cand.id) {
continue;
@@ -379,25 +381,40 @@ async fn run_embedding_backfill(
continue;
}
};
- if let Err(e) = writer
- .store_embedding(
- cand.id,
- f32_vec_to_bytes(&vec),
- provider.clone(),
- model.clone(),
- dim,
- )
- .await
- {
- failed += 1;
- tracing::warn!(path = %cand.path, error = %e, "scheduled embed: store failed");
- continue;
+ pending.push(EmbeddingWrite {
+ page_id: cand.id,
+ vector_bytes: f32_vec_to_bytes(&vec),
+ provider: provider.clone(),
+ model: model.clone(),
+ dim,
+ });
+ if pending.len() >= EMBEDDING_WRITE_BATCH {
+ flush_embedding_batch(writer, &mut pending, &mut embedded, &mut failed).await;
}
- embedded += 1;
}
+ flush_embedding_batch(writer, &mut pending, &mut embedded, &mut failed).await;
Ok((embedded, failed))
}
+async fn flush_embedding_batch(
+ writer: &WriterHandle,
+ pending: &mut Vec<EmbeddingWrite>,
+ embedded: &mut usize,
+ failed: &mut usize,
+) {
+ if pending.is_empty() {
+ return;
+ }
+ let batch = std::mem::replace(pending, Vec::with_capacity(EMBEDDING_WRITE_BATCH));
+ let count = batch.len();
+ if let Err(e) = writer.store_embeddings(batch).await {
+ *failed += count;
+ tracing::warn!(count, error = %e, "scheduled embed: batch store failed");
+ } else {
+ *embedded += count;
+ }
+}
+
async fn configure_embedder(
config: &Config,
store: &Store,
diff --git a/crates/ai-memory-mcp/src/admin.rs b/crates/ai-memory-mcp/src/admin.rs
index 4b590930..dfd00487 100644
--- a/crates/ai-memory-mcp/src/admin.rs
+++ b/crates/ai-memory-mcp/src/admin.rs
@@ -32,7 +32,9 @@ use ai_memory_core::{
DEFAULT_PROJECT_NAME, DEFAULT_WORKSPACE_NAME, PagePath, ProjectId, SessionId, Tier, WorkspaceId,
};
use ai_memory_llm::{Embedder, LlmProvider};
-use ai_memory_store::{DecayParams, ReaderPool, StoreError, WriterHandle, f32_vec_to_bytes};
+use ai_memory_store::{
+ DecayParams, EmbeddingWrite, ReaderPool, StoreError, WriterHandle, f32_vec_to_bytes,
+};
use ai_memory_wiki::{Wiki, WritePageRequest};
use axum::Json;
use axum::Router;
@@ -47,6 +49,8 @@ use serde::{Deserialize, Serialize};
use tokio_util::io::ReaderStream;
use tracing::{info, warn};
+const EMBEDDING_WRITE_BATCH: usize = 100;
+
/// Shared state for the admin router.
#[derive(Clone)]
pub struct AdminState {
@@ -939,6 +943,7 @@ async fn handle_embed(
let mut skipped = 0_usize;
let mut failed = 0_usize;
let mut would_embed = 0_usize;
+ let mut pending = Vec::with_capacity(EMBEDDING_WRITE_BATCH);
for cand in candidates {
if !req.reembed && already.contains(&cand.id) {
@@ -965,18 +970,18 @@ async fn handle_embed(
continue;
}
};
- let bytes = f32_vec_to_bytes(&vec);
- if let Err(e) = state
- .writer
- .store_embedding(cand.id, bytes, provider.clone(), model.clone(), dim)
- .await
- {
- warn!(path = %cand.path, error = %e, "embed: store_embedding failed");
- failed += 1;
- continue;
+ pending.push(EmbeddingWrite {
+ page_id: cand.id,
+ vector_bytes: f32_vec_to_bytes(&vec),
+ provider: provider.clone(),
+ model: model.clone(),
+ dim,
+ });
+ if pending.len() >= EMBEDDING_WRITE_BATCH {
+ flush_embedding_batch(&state.writer, &mut pending, &mut embedded, &mut failed).await;
}
- embedded += 1;
}
+ flush_embedding_batch(&state.writer, &mut pending, &mut embedded, &mut failed).await;
let report = EmbedReport {
embedded,
@@ -993,6 +998,25 @@ async fn handle_embed(
))
}
+async fn flush_embedding_batch(
+ writer: &WriterHandle,
+ pending: &mut Vec<EmbeddingWrite>,
+ embedded: &mut usize,
+ failed: &mut usize,
+) {
+ if pending.is_empty() {
+ return;
+ }
+ let batch = std::mem::replace(pending, Vec::with_capacity(EMBEDDING_WRITE_BATCH));
+ let count = batch.len();
+ if let Err(e) = writer.store_embeddings(batch).await {
+ *failed += count;
+ warn!(count, error = %e, "embed: store_embeddings failed");
+ } else {
+ *embedded += count;
+ }
+}
+
// ---------------------------------------------------------------------
// commit
// ---------------------------------------------------------------------
diff --git a/crates/ai-memory-store/src/lib.rs b/crates/ai-memory-store/src/lib.rs
index e2d3c2f3..4c28a23a 100644
--- a/crates/ai-memory-store/src/lib.rs
+++ b/crates/ai-memory-store/src/lib.rs
@@ -21,7 +21,7 @@ mod writer;
pub use decay::{DecayParams, retention_score};
pub use error::{StoreError, StoreResult};
-pub use ops::{PurgeSummary, ReorgSummary};
+pub use ops::{EmbeddingWrite, PurgeSummary, ReorgSummary};
pub use reader::{
ActivityWindow, BriefingPage, BriefingSnapshot, DecayCandidate, DerivedIndexStatus,
EmbeddingTripleCount, ObservationHit, PageHit, PageHitWithMeta, PageMeta, PageSummary,
diff --git a/crates/ai-memory-store/src/ops.rs b/crates/ai-memory-store/src/ops.rs
index 8844f502..d37855f8 100644
--- a/crates/ai-memory-store/src/ops.rs
+++ b/crates/ai-memory-store/src/ops.rs
@@ -51,6 +51,21 @@ use sha2::{Digest, Sha256};
use crate::error::{StoreError, StoreResult};
+/// One embedding upsert requested by a backfill or embed command.
+#[derive(Debug)]
+pub struct EmbeddingWrite {
+ /// Page receiving the embedding.
+ pub page_id: PageId,
+ /// Packed little-endian `f32` vector bytes.
+ pub vector_bytes: Vec<u8>,
+ /// Embedding provider name.
+ pub provider: String,
+ /// Embedding model name.
+ pub model: String,
+ /// Vector dimension.
+ pub dim: u32,
+}
+
/// Upsert a page by path, superseding any existing latest version when the
/// content (sha256 of body) has changed.
///
@@ -415,6 +430,39 @@ pub fn store_embedding(
Ok(())
}
+/// Store / replace a batch of page embeddings in one transaction.
+pub fn store_embeddings(conn: &mut Connection, embeddings: &[EmbeddingWrite]) -> StoreResult<()> {
+ if embeddings.is_empty() {
+ return Ok(());
+ }
+ let now = Timestamp::now().as_microsecond();
+ let tx = conn.transaction()?;
+ {
+ let mut stmt = tx.prepare(
+ "INSERT INTO page_embeddings (page_id, vector, provider, model, dim, created_at) \
+ VALUES (?1, ?2, ?3, ?4, ?5, ?6) \
+ ON CONFLICT(page_id) DO UPDATE SET \
+ vector = excluded.vector, \
+ provider = excluded.provider, \
+ model = excluded.model, \
+ dim = excluded.dim, \
+ created_at = excluded.created_at",
+ )?;
+ for embedding in embeddings {
+ stmt.execute(params![
+ embedding.page_id.as_bytes(),
+ embedding.vector_bytes.as_slice(),
+ embedding.provider.as_str(),
+ embedding.model.as_str(),
+ embedding.dim,
+ now,
+ ])?;
+ }
+ }
+ tx.commit()?;
+ Ok(())
+}
+
/// Bump `access_count` + `last_accessed_at` for the pages whose ids
/// appear in `page_ids`. Idempotent for unknown ids (no-op).
/// Used by the read path to feed the M8 reinforcement term.
@@ -1097,4 +1145,37 @@ mod tests {
.unwrap();
assert_eq!(model, "model-b", "latest model metadata wins");
}
+
+ #[test]
+ fn store_embeddings_batches_rows_in_one_call() {
+ let (_tmp, mut conn, ws, proj) = fresh_db();
+ let p1 = upsert_page(&mut conn, &page(ws, proj, "notes/a.md", "body a")).unwrap();
+ let p2 = upsert_page(&mut conn, &page(ws, proj, "notes/b.md", "body b")).unwrap();
+
+ store_embeddings(
+ &mut conn,
+ &[
+ EmbeddingWrite {
+ page_id: p1,
+ vector_bytes: vec![0u8; 4],
+ provider: "test".into(),
+ model: "model".into(),
+ dim: 1,
+ },
+ EmbeddingWrite {
+ page_id: p2,
+ vector_bytes: vec![1u8; 4],
+ provider: "test".into(),
+ model: "model".into(),
+ dim: 1,
+ },
+ ],
+ )
+ .unwrap();
+
+ let count: i64 = conn
+ .query_row("SELECT COUNT(*) FROM page_embeddings", [], |r| r.get(0))
+ .unwrap();
+ assert_eq!(count, 2);
+ }
}
diff --git a/crates/ai-memory-store/src/reader.rs b/crates/ai-memory-store/src/reader.rs
index f06484db..251f585f 100644
--- a/crates/ai-memory-store/src/reader.rs
+++ b/crates/ai-memory-store/src/reader.rs
@@ -6,6 +6,7 @@
//! soft cap: a connection that comes back when the pool is already full
//! is simply dropped.
+use std::fmt::Write as _;
use std::path::{Path, PathBuf};
use std::sync::Arc;
@@ -14,7 +15,8 @@ use ai_memory_core::{
PageId, PagePath, ProjectId, SessionId, WorkspaceId,
};
use parking_lot::Mutex;
-use rusqlite::{Connection, OpenFlags, OptionalExtension, params};
+use rusqlite::types::Value;
+use rusqlite::{Connection, OpenFlags, OptionalExtension, params, params_from_iter};
use serde::Serialize;
// `ai_memory_core::Tier` is referenced via fully-qualified path inside the
// DecayCandidate struct definition above to avoid a top-level import
@@ -938,94 +940,76 @@ impl ReaderPool {
return Ok(Vec::new());
}
self.with_conn(move |conn| {
- let seed_set: std::collections::HashSet<PageId> = seed_ids.iter().copied().collect();
let mut seen = std::collections::HashSet::new();
let mut out = Vec::new();
- let mut outgoing = conn.prepare(
- "SELECT tp.id, tp.path, tp.title, substr(tp.body, 1, 240) \
- FROM links l \
- JOIN pages tp ON tp.id = l.to_page_id \
- WHERE l.from_page_id = ?1 \
- AND tp.workspace_id = ?2 \
- AND tp.project_id = ?3 \
- AND tp.is_latest = 1 \
- ORDER BY tp.updated_at DESC",
- )?;
- let mut incoming = conn.prepare(
- "SELECT fp.id, fp.path, fp.title, substr(fp.body, 1, 240) \
- FROM links l \
- JOIN pages fp ON fp.id = l.from_page_id \
- WHERE l.to_page_id = ?1 \
- AND fp.workspace_id = ?2 \
- AND fp.project_id = ?3 \
- AND fp.is_latest = 1 \
- ORDER BY fp.updated_at DESC",
- )?;
-
- for seed_id in &seed_ids {
- let rows = outgoing.query_map(
- params![
- seed_id.as_bytes(),
- workspace_id.as_bytes(),
- project_id.as_bytes()
- ],
- |row| {
- let id_bytes: Vec<u8> = row.get(0)?;
- let path: String = row.get(1)?;
- let title: String = row.get(2)?;
- let snippet: String = row.get(3)?;
- Ok((id_bytes, path, title, snippet))
- },
- )?;
- for row in rows {
- let (id_bytes, path, title, snippet) = row?;
- let id = PageId::from_slice(&id_bytes)?;
- if seed_set.contains(&id) || !seen.insert(id) {
- continue;
- }
- out.push(PageHit {
- id,
- path: PagePath::new(path)?,
- title,
- snippet,
- rank: 0.0,
- });
- if out.len() >= limit {
- return Ok(out);
- }
+ let mut values_clause = String::with_capacity(seed_ids.len() * 8);
+ let mut sql_params = Vec::with_capacity(seed_ids.len() * 2 + 4);
+ for (idx, seed_id) in seed_ids.iter().enumerate() {
+ if idx > 0 {
+ values_clause.push_str(", ");
}
+ values_clause.push_str("(?, ?)");
+ sql_params.push(Value::Blob(seed_id.as_bytes().to_vec()));
+ sql_params.push(Value::Integer(idx as i64));
+ }
+ sql_params.push(Value::Blob(workspace_id.as_bytes().to_vec()));
+ sql_params.push(Value::Blob(project_id.as_bytes().to_vec()));
+ sql_params.push(Value::Blob(workspace_id.as_bytes().to_vec()));
+ sql_params.push(Value::Blob(project_id.as_bytes().to_vec()));
+
+ let mut sql = String::with_capacity(values_clause.len() + 1_400);
+ write!(
+ &mut sql,
+ "WITH seeds(seed_id, seed_ord) AS (VALUES {values_clause}), \
+ neighbors AS ( \
+ SELECT tp.id AS id, tp.path AS path, tp.title AS title, \
+ substr(tp.body, 1, 240) AS snippet, \
+ seeds.seed_ord * 2 AS stream_ord, tp.updated_at AS updated_at \
+ FROM seeds \
+ JOIN links l ON l.from_page_id = seeds.seed_id \
+ JOIN pages tp ON tp.id = l.to_page_id \
+ WHERE tp.workspace_id = ? AND tp.project_id = ? AND tp.is_latest = 1 \
+ UNION ALL \
+ SELECT fp.id AS id, fp.path AS path, fp.title AS title, \
+ substr(fp.body, 1, 240) AS snippet, \
+ seeds.seed_ord * 2 + 1 AS stream_ord, fp.updated_at AS updated_at \
+ FROM seeds \
+ JOIN links l ON l.to_page_id = seeds.seed_id \
+ JOIN pages fp ON fp.id = l.from_page_id \
+ WHERE fp.workspace_id = ? AND fp.project_id = ? AND fp.is_latest = 1 \
+ ) \
+ SELECT id, path, title, snippet \
+ FROM neighbors \
+ WHERE NOT EXISTS (SELECT 1 FROM seeds s WHERE s.seed_id = neighbors.id) \
+ ORDER BY stream_ord ASC, updated_at DESC"
+ )
+ .expect("writing SQL into String cannot fail");
- let rows = incoming.query_map(
- params![
- seed_id.as_bytes(),
- workspace_id.as_bytes(),
- project_id.as_bytes()
- ],
- |row| {
- let id_bytes: Vec<u8> = row.get(0)?;
- let path: String = row.get(1)?;
- let title: String = row.get(2)?;
- let snippet: String = row.get(3)?;
- Ok((id_bytes, path, title, snippet))
- },
- )?;
- for row in rows {
- let (id_bytes, path, title, snippet) = row?;
- let id = PageId::from_slice(&id_bytes)?;
- if seed_set.contains(&id) || !seen.insert(id) {
- continue;
- }
- out.push(PageHit {
- id,
- path: PagePath::new(path)?,
- title,
- snippet,
- rank: 0.0,
- });
- if out.len() >= limit {
- return Ok(out);
- }
+ let mut stmt = conn.prepare(&sql)?;
+ let rows = stmt.query_map(params_from_iter(sql_params.iter()), |row| {
+ let id_bytes: Vec<u8> = row.get(0)?;
+ let path: String = row.get(1)?;
+ let title: String = row.get(2)?;
+ let snippet: String = row.get(3)?;
+ Ok((id_bytes, path, title, snippet))
+ })?;
+
+ for row in rows {
+ let (id_bytes, path, title, snippet) = row?;
+ let id = PageId::from_slice(&id_bytes)?;
+ if !seen.insert(id) {
+ continue;
+ }
+ out.push(PageHit {
+ id,
+ path: PagePath::new(path)?,
+ title,
+ snippet,
+ rank: 0.0,
+ });
+ if out.len() >= limit {
+ break;
}
}
Ok(out)
diff --git a/crates/ai-memory-store/src/writer.rs b/crates/ai-memory-store/src/writer.rs
index 14bbc8c6..ce9b1e08 100644
--- a/crates/ai-memory-store/src/writer.rs
+++ b/crates/ai-memory-store/src/writer.rs
@@ -18,7 +18,7 @@ use rusqlite::Connection;
use tokio::sync::{mpsc, oneshot};
use crate::error::{StoreError, StoreResult};
-use crate::ops::{self, PurgeSummary, ReorgSummary};
+use crate::ops::{self, EmbeddingWrite, PurgeSummary, ReorgSummary};
/// Commands accepted by the writer thread.
pub(crate) enum WriteCmd {
@@ -90,6 +90,10 @@ pub(crate) enum WriteCmd {
dim: u32,
reply: oneshot::Sender<StoreResult<()>>,
},
+ StoreEmbeddingBatch {
+ embeddings: Vec<EmbeddingWrite>,
+ reply: oneshot::Sender<StoreResult<()>>,
+ },
/// Delete a project and all its data (pages, sessions, observations,
/// handoffs, embeddings) in one transaction. Returns the paths of
/// every page file that must be removed from disk by the caller.
@@ -286,6 +290,20 @@ impl WriterHandle {
rx.await.map_err(|_| StoreError::WriterClosed)?
}
+ /// Store or replace a batch of embeddings in one SQLite transaction.
+ ///
+ /// # Errors
+ /// Returns [`StoreError::WriterClosed`] or propagates SQL errors.
+ pub async fn store_embeddings(&self, embeddings: Vec<EmbeddingWrite>) -> StoreResult<()> {
+ let (tx, rx) = oneshot::channel();
+ self.send(WriteCmd::StoreEmbeddingBatch {
+ embeddings,
+ reply: tx,
+ })
+ .await?;
+ rx.await.map_err(|_| StoreError::WriterClosed)?
+ }
+
/// Bump access counters for a set of pages (M8 reinforcement term).
///
/// # Errors
@@ -575,6 +593,10 @@ fn worker_loop(mut conn: Connection, mut rx: mpsc::Receiver<WriteCmd>) {
);
send_or_warn(reply, result, "store_embedding");
}
+ WriteCmd::StoreEmbeddingBatch { embeddings, reply } => {
+ let result = ops::store_embeddings(&mut conn, &embeddings);
+ send_or_warn(reply, result, "store_embeddings");
+ }
WriteCmd::PurgeProject {
workspace_id,
project_id,