Solution for long term memory for agent coding CLIs and to facilitate handoff between different agent vendors
Rust
M2.5-B: ai-memory backup / restore with SQLite online snapshot
AkitaOnRailsabf1c8a7c0e4
diff --git a/Cargo.lock b/Cargo.lockindex 1f84fe66..d305c2d8 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -2,6 +2,12 @@# It is not intended for manual editing.version = 4+[[package]]+name = "adler2"+version = "2.0.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa"+[[package]]name = "ahash"version = "0.8.12"@@ -36,10 +42,12 @@ dependencies = ["clap","dirs","figment",+ "flate2","rmcp","serde","serde_json","sysinfo",+ "tar","tempfile","tokio","tokio-util",@@ -434,6 +442,15 @@ dependencies = ["libc",]+[[package]]+name = "crc32fast"+version = "1.5.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511"+dependencies = [+ "cfg-if",+]+[[package]]name = "crossbeam-channel"version = "0.5.15"@@ -632,12 +649,32 @@ dependencies = ["windows-sys 0.60.2",]+[[package]]+name = "filetime"+version = "0.2.29"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "5c287a33c7f0a620c38e641e7f60827713987b3c0f26e8ddc9462cc69cf75759"+dependencies = [+ "cfg-if",+ "libc",+]+[[package]]name = "find-msvc-tools"version = "0.1.9"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582"+[[package]]+name = "flate2"+version = "1.1.9"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c"+dependencies = [+ "crc32fast",+ "miniz_oxide",+]+[[package]]name = "foldhash"version = "0.1.5"@@ -1258,6 +1295,16 @@ version = "0.3.17"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a"+[[package]]+name = "miniz_oxide"+version = "0.8.9"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316"+dependencies = [+ "adler2",+ "simd-adler32",+]+[[package]]name = "mio"version = "1.2.0"@@ -1923,6 +1970,12 @@ dependencies = ["libc",]+[[package]]+name = "simd-adler32"+version = "0.3.9"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "703d5c7ef118737c72f1af64ad2f6f8c5e1921f818cdcb97b8fe6fc69bf66214"+[[package]]name = "siphasher"version = "1.0.3"@@ -2024,6 +2077,17 @@ dependencies = ["windows",]+[[package]]+name = "tar"+version = "0.4.46"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "3f6221d9a6003c78398e3b239969f352578258df48c8eb051caadae0015bc840"+dependencies = [+ "filetime",+ "libc",+ "xattr",+]+[[package]]name = "tempfile"version = "3.27.0"@@ -2949,6 +3013,16 @@ version = "0.6.3"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4"+[[package]]+name = "xattr"+version = "1.6.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "32e45ad4206f6d2479085147f02bc2ef834ac85886624a23575ae137c8aa8156"+dependencies = [+ "libc",+ "rustix",+]+[[package]]name = "yansi"version = "1.0.1"diff --git a/Cargo.toml b/Cargo.tomlindex c16d25d5..3a4c5695 100644--- a/Cargo.toml+++ b/Cargo.toml@@ -60,7 +60,7 @@ dirs = "5"secrecy = { version = "0.10", features = ["serde"] }# Storage.-rusqlite = { version = "0.32", features = ["bundled"] }+rusqlite = { version = "0.32", features = ["bundled", "backup"] }refinery = { version = "0.8", features = ["rusqlite"] }sha2 = "0.10"parking_lot = "0.12"@@ -75,6 +75,10 @@ rmcp = { version = "1.7", features = ["server", "macros", "transport-io", "transschemars = "1"axum = "0.8"+# Backup / restore archive format.+tar = "0.4"+flate2 = "1"+# Testing.tempfile = "3"diff --git a/crates/ai-memory-cli/Cargo.toml b/crates/ai-memory-cli/Cargo.tomlindex 20c61f21..6b00a5ca 100644--- a/crates/ai-memory-cli/Cargo.toml+++ b/crates/ai-memory-cli/Cargo.toml@@ -22,16 +22,20 @@ axum.workspace = trueclap.workspace = truedirs.workspace = truefigment.workspace = true+flate2.workspace = truermcp.workspace = trueserde.workspace = trueserde_json.workspace = truesysinfo.workspace = true+tar.workspace = truetokio.workspace = truetokio-util.workspace = truetracing.workspace = truetracing-appender.workspace = truetracing-subscriber.workspace = true+tempfile.workspace = true+[dev-dependencies]tempfile.workspace = truediff --git a/crates/ai-memory-cli/src/cli.rs b/crates/ai-memory-cli/src/cli.rsindex 65205e53..71921865 100644--- a/crates/ai-memory-cli/src/cli.rs+++ b/crates/ai-memory-cli/src/cli.rs@@ -41,6 +41,10 @@ pub enum Command {Serve(ServeArgs),/// Wipe the data directory's wiki/, db/, raw/ contents.Reset(ResetArgs),+ /// Snapshot wiki/, db/, and config.toml into a gzipped tarball.+ Backup(BackupArgs),+ /// Restore a backup tarball into the data directory.+ Restore(RestoreArgs),}/// Arguments for `init`.@@ -91,6 +95,25 @@ pub struct ResetArgs {pub confirm: bool,}+/// Arguments for `backup`.+#[derive(Debug, Args)]+pub struct BackupArgs {+ /// Destination tarball (`.tar.gz`).+ #[arg(long, short = 'o')]+ pub to: PathBuf,+}++/// Arguments for `restore`.+#[derive(Debug, Args)]+pub struct RestoreArgs {+ /// Source tarball.+ #[arg(long, short = 'i')]+ pub from: PathBuf,+ /// Overwrite an existing non-empty data dir.+ #[arg(long)]+ pub force: bool,+}+/// Transport for the MCP server.#[derive(Debug, Clone, Copy, clap::ValueEnum)]pub enum TransportKind {diff --git a/crates/ai-memory-cli/src/commands/backup.rs b/crates/ai-memory-cli/src/commands/backup.rsnew file mode 100644index 00000000..b85261dd--- /dev/null+++ b/crates/ai-memory-cli/src/commands/backup.rs@@ -0,0 +1,102 @@+//! `ai-memory backup --to <tarball>` — make a consistent point-in-time+//! snapshot of the data dir.+//!+//! Uses SQLite's online backup API so the source DB stays writable for+//! the duration of the copy. The tarball is gzip-compressed and+//! contains `wiki/`, a fresh `db/memory.sqlite` snapshot, and the+//! `config.toml` (if present).++use std::path::Path;++use ai_memory_store::Store;+use anyhow::{Context, Result, bail};+use flate2::Compression;+use flate2::write::GzEncoder;+use tracing::info;++use crate::cli::BackupArgs;+use crate::config::Config;+use crate::process_guard::{busy_message, sibling_processes};++/// Run the `backup` subcommand.+///+/// # Errors+/// Returns an error if another `ai-memory` process is running, the+/// store cannot be opened, the SQLite snapshot fails, or the tarball+/// cannot be written.+pub async fn run(config: &Config, args: BackupArgs) -> Result<()> {+ let siblings = sibling_processes();+ if !siblings.is_empty() {+ bail!(busy_message("backup", &siblings));+ }++ let store = Store::open(&config.data_dir)+ .with_context(|| format!("opening store at {}", config.data_dir.display()))?;++ let staging = tempfile::tempdir()?;+ let snapshot_path = staging.path().join("memory.sqlite");+ info!(snapshot = %snapshot_path.display(), "snapshotting SQLite");+ store+ .reader+ .snapshot_to(snapshot_path.clone())+ .await+ .context("running SQLite online backup")?;++ let dest = &args.to;+ if let Some(parent) = dest.parent()+ && !parent.as_os_str().is_empty()+ {+ std::fs::create_dir_all(parent)?;+ }+ let file =+ std::fs::File::create(dest).with_context(|| format!("creating {}", dest.display()))?;+ let encoder = GzEncoder::new(file, Compression::default());+ let mut tar = tar::Builder::new(encoder);+ tar.mode(tar::HeaderMode::Deterministic);++ // wiki/ goes in as-is.+ let wiki_dir = config.data_dir.join("wiki");+ if wiki_dir.is_dir() {+ tar.append_dir_all("wiki", &wiki_dir)+ .with_context(|| format!("archiving {}", wiki_dir.display()))?;+ }++ // db/memory.sqlite from the snapshot, not the live file.+ tar.append_path_with_name(&snapshot_path, "db/memory.sqlite")+ .context("archiving db snapshot")?;++ // config.toml, if present.+ let cfg = config.data_dir.join("config.toml");+ if cfg.is_file() {+ tar.append_path_with_name(&cfg, "config.toml")+ .context("archiving config.toml")?;+ }++ let encoder = tar.into_inner()?;+ encoder.finish()?;++ let size = std::fs::metadata(dest).map(|m| m.len()).unwrap_or(0);+ info!(path = %dest.display(), bytes = size, "backup written");+ println!("backup -> {} ({})", dest.display(), human_bytes(size));+ Ok(())+}++fn human_bytes(n: u64) -> String {+ const UNITS: &[&str] = &["B", "KiB", "MiB", "GiB"];+ let mut value = n as f64;+ let mut unit = 0;+ while value >= 1024.0 && unit + 1 < UNITS.len() {+ value /= 1024.0;+ unit += 1;+ }+ if unit == 0 {+ format!("{n} {}", UNITS[0])+ } else {+ format!("{value:.2} {}", UNITS[unit])+ }+}++#[allow(dead_code)] // Reserved for cross-platform diagnostics later.+fn rel_path<'a>(base: &Path, full: &'a Path) -> Option<&'a Path> {+ full.strip_prefix(base).ok()+}diff --git a/crates/ai-memory-cli/src/commands/mod.rs b/crates/ai-memory-cli/src/commands/mod.rsindex 66e26656..3e94ffec 100644--- a/crates/ai-memory-cli/src/commands/mod.rs+++ b/crates/ai-memory-cli/src/commands/mod.rs@@ -1,7 +1,9 @@//! Subcommand implementations.+pub mod backup;pub mod init;pub mod reset;+pub mod restore;pub mod search;pub mod serve;pub mod status;diff --git a/crates/ai-memory-cli/src/commands/reset.rs b/crates/ai-memory-cli/src/commands/reset.rsindex b17286ae..fbaf679a 100644--- a/crates/ai-memory-cli/src/commands/reset.rs+++ b/crates/ai-memory-cli/src/commands/reset.rs@@ -5,14 +5,12 @@//! inode caused phantom search results after a reset).use anyhow::{Result, bail};-use std::ffi::OsStr;-use sysinfo::System;use crate::cli::ResetArgs;use crate::config::Config;+use crate::process_guard::{busy_message, sibling_processes};const SUBDIRS: &[&str] = &["wiki", "db", "raw"];-const BIN_NAME: &str = "ai-memory";/// Run the `reset` subcommand.///@@ -22,13 +20,7 @@ const BIN_NAME: &str = "ai-memory";pub fn run(config: &Config, args: ResetArgs) -> Result<()> {let siblings = sibling_processes();if !siblings.is_empty() {- let pids: Vec<u32> = siblings.iter().map(|p| p.as_u32()).collect();- bail!(- "refusing to reset: {} other ai-memory process(es) running (pids: {:?}). \- Stop them first, then re-run.",- pids.len(),- pids,- );+ bail!(busy_message("reset", &siblings));}if !args.confirm {@@ -54,18 +46,3 @@ pub fn run(config: &Config, args: ResetArgs) -> Result<()> {tracing::info!("reset complete");Ok(())}--fn sibling_processes() -> Vec<sysinfo::Pid> {- let mut sys = System::new();- sys.refresh_processes(sysinfo::ProcessesToUpdate::All, true);- let me = sysinfo::Pid::from_u32(std::process::id());- let bin_os: &OsStr = OsStr::new(BIN_NAME);- sys.processes_by_exact_name(bin_os)- // On Linux, sysinfo lists tokio worker threads alongside the main- // process under the same comm name. thread_kind() == None means- // we're looking at the process leader, not one of its threads.- .filter(|p| p.thread_kind().is_none())- .map(sysinfo::Process::pid)- .filter(|pid| *pid != me)- .collect()-}diff --git a/crates/ai-memory-cli/src/commands/restore.rs b/crates/ai-memory-cli/src/commands/restore.rsnew file mode 100644index 00000000..434802b5--- /dev/null+++ b/crates/ai-memory-cli/src/commands/restore.rs@@ -0,0 +1,73 @@+//! `ai-memory restore --from <tarball>` — restore a backup tarball.+//!+//! Refuses to overwrite a non-empty data dir unless `--force` is given.+//! Refuses while another `ai-memory` process is alive. After extraction,+//! re-opens the store so any pending migrations run (and a corrupt+//! snapshot fails loudly).++use ai_memory_store::Store;+use anyhow::{Context, Result, bail};+use flate2::read::GzDecoder;+use tracing::info;++use crate::cli::RestoreArgs;+use crate::config::Config;+use crate::process_guard::{busy_message, sibling_processes};++/// Run the `restore` subcommand.+///+/// # Errors+/// Returns an error if another `ai-memory` process is running, the+/// data dir is non-empty without `--force`, the tarball cannot be+/// extracted, or the restored store fails to open.+pub fn run(config: &Config, args: RestoreArgs) -> Result<()> {+ let siblings = sibling_processes();+ if !siblings.is_empty() {+ bail!(busy_message("restore", &siblings));+ }++ if !args.from.is_file() {+ bail!("source tarball {} not found", args.from.display());+ }++ let wiki = config.data_dir.join("wiki");+ let db = config.data_dir.join("db").join("memory.sqlite");+ if (wiki.is_dir() && std::fs::read_dir(&wiki)?.next().is_some()) || db.is_file() {+ if !args.force {+ bail!(+ "refusing to restore: data dir at {} is non-empty (pass --force to overwrite)",+ config.data_dir.display(),+ );+ }+ // Force path: drop the existing wiki + db so the tarball can+ // populate them cleanly. Keep config.toml, logs/, models/.+ for sub in ["wiki", "db"] {+ let path = config.data_dir.join(sub);+ if path.exists() {+ std::fs::remove_dir_all(&path)?;+ }+ }+ }+ std::fs::create_dir_all(&config.data_dir)?;++ let file = std::fs::File::open(&args.from)+ .with_context(|| format!("opening {}", args.from.display()))?;+ let decoder = GzDecoder::new(file);+ let mut archive = tar::Archive::new(decoder);+ archive.set_preserve_permissions(false);+ archive+ .unpack(&config.data_dir)+ .with_context(|| format!("extracting into {}", config.data_dir.display()))?;+ info!(from = %args.from.display(), into = %config.data_dir.display(), "tarball extracted");++ // Open + drop the store so refinery applies any pending migrations+ // and the SQLite file is validated.+ let _store = Store::open(&config.data_dir).context("opening restored store")?;+ info!("restore complete");+ println!(+ "restored {} -> {}",+ args.from.display(),+ config.data_dir.display()+ );+ Ok(())+}diff --git a/crates/ai-memory-cli/src/main.rs b/crates/ai-memory-cli/src/main.rsindex f2e952e8..258da7e7 100644--- a/crates/ai-memory-cli/src/main.rs+++ b/crates/ai-memory-cli/src/main.rs@@ -17,6 +17,7 @@ mod cli;mod commands;mod config;mod logging;+mod process_guard;use cli::{Cli, Command};use config::Config;@@ -43,5 +44,7 @@ async fn main() -> Result<()> {Command::Watch(args) => commands::watch::run(&config, args).await,Command::Serve(args) => commands::serve::run(&config, args).await,Command::Reset(args) => commands::reset::run(&config, args),+ Command::Backup(args) => commands::backup::run(&config, args).await,+ Command::Restore(args) => commands::restore::run(&config, args),}}diff --git a/crates/ai-memory-cli/src/process_guard.rs b/crates/ai-memory-cli/src/process_guard.rsnew file mode 100644index 00000000..2a3a31c0--- /dev/null+++ b/crates/ai-memory-cli/src/process_guard.rs@@ -0,0 +1,43 @@+//! Shared "is another ai-memory process alive?" check.+//!+//! Used by every destructive command (`reset`, `backup`, `restore`) so+//! we never race a live writer (lesson from basic-memory #765).++use std::ffi::OsStr;++use sysinfo::System;++/// Binary name to match against `/proc/*/comm` (or platform equivalent).+pub const BIN_NAME: &str = "ai-memory";++/// Return PIDs of *other* `ai-memory` processes (excluding the current+/// process and any threads of it).+#[must_use]+pub fn sibling_processes() -> Vec<sysinfo::Pid> {+ let mut sys = System::new();+ sys.refresh_processes(sysinfo::ProcessesToUpdate::All, true);+ let me = sysinfo::Pid::from_u32(std::process::id());+ let bin_os: &OsStr = OsStr::new(BIN_NAME);+ sys.processes_by_exact_name(bin_os)+ // On Linux, sysinfo lists tokio worker threads alongside the main+ // process under the same comm name. thread_kind() == None means+ // we're looking at the process leader, not one of its threads.+ .filter(|p| p.thread_kind().is_none())+ .map(sysinfo::Process::pid)+ .filter(|pid| *pid != me)+ .collect()+}++/// Format a "refusing to ..." error message for the given operation,+/// quoting sibling PIDs.+#[must_use]+pub fn busy_message(verb: &str, siblings: &[sysinfo::Pid]) -> String {+ let pids: Vec<u32> = siblings.iter().copied().map(sysinfo::Pid::as_u32).collect();+ format!(+ "refusing to {}: {} other ai-memory process(es) running (pids: {:?}). \+ Stop them first, then re-run.",+ verb,+ pids.len(),+ pids,+ )+}diff --git a/crates/ai-memory-store/src/reader.rs b/crates/ai-memory-store/src/reader.rsindex abfa8651..209809b5 100644--- a/crates/ai-memory-store/src/reader.rs+++ b/crates/ai-memory-store/src/reader.rs@@ -181,6 +181,19 @@ impl ReaderPool {.await}+ /// Snapshot the database to `dest_path` using SQLite's online backup+ /// API. The source DB stays writable for the duration of the copy.+ ///+ /// # Errors+ /// Propagates any SQL or pool error.+ pub async fn snapshot_to(&self, dest_path: PathBuf) -> StoreResult<()> {+ self.with_conn(move |conn| {+ conn.backup(rusqlite::DatabaseName::Main, &dest_path, None)+ .map_err(StoreError::from)+ })+ .await+ }+/// Return aggregate counts for the `status` view.////// # Errors