Skip to content

Solution for long term memory for agent coding CLIs and to facilitate handoff between different agent vendors

Rust

M2.5-B: ai-memory backup / restore with SQLite online snapshot

AkitaOnRailsabf1c8a7c0e4
diff --git a/Cargo.lock b/Cargo.lock
index 1f84fe66..d305c2d8 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -2,6 +2,12 @@
# It is not intended for manual editing.
version = 4
+[[package]]
+name = "adler2"
+version = "2.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa"
+
[[package]]
name = "ahash"
version = "0.8.12"
@@ -36,10 +42,12 @@ dependencies = [
"clap",
"dirs",
"figment",
+ "flate2",
"rmcp",
"serde",
"serde_json",
"sysinfo",
+ "tar",
"tempfile",
"tokio",
"tokio-util",
@@ -434,6 +442,15 @@ dependencies = [
"libc",
]
+[[package]]
+name = "crc32fast"
+version = "1.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511"
+dependencies = [
+ "cfg-if",
+]
+
[[package]]
name = "crossbeam-channel"
version = "0.5.15"
@@ -632,12 +649,32 @@ dependencies = [
"windows-sys 0.60.2",
]
+[[package]]
+name = "filetime"
+version = "0.2.29"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5c287a33c7f0a620c38e641e7f60827713987b3c0f26e8ddc9462cc69cf75759"
+dependencies = [
+ "cfg-if",
+ "libc",
+]
+
[[package]]
name = "find-msvc-tools"
version = "0.1.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582"
+[[package]]
+name = "flate2"
+version = "1.1.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c"
+dependencies = [
+ "crc32fast",
+ "miniz_oxide",
+]
+
[[package]]
name = "foldhash"
version = "0.1.5"
@@ -1258,6 +1295,16 @@ version = "0.3.17"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a"
+[[package]]
+name = "miniz_oxide"
+version = "0.8.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316"
+dependencies = [
+ "adler2",
+ "simd-adler32",
+]
+
[[package]]
name = "mio"
version = "1.2.0"
@@ -1923,6 +1970,12 @@ dependencies = [
"libc",
]
+[[package]]
+name = "simd-adler32"
+version = "0.3.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "703d5c7ef118737c72f1af64ad2f6f8c5e1921f818cdcb97b8fe6fc69bf66214"
+
[[package]]
name = "siphasher"
version = "1.0.3"
@@ -2024,6 +2077,17 @@ dependencies = [
"windows",
]
+[[package]]
+name = "tar"
+version = "0.4.46"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3f6221d9a6003c78398e3b239969f352578258df48c8eb051caadae0015bc840"
+dependencies = [
+ "filetime",
+ "libc",
+ "xattr",
+]
+
[[package]]
name = "tempfile"
version = "3.27.0"
@@ -2949,6 +3013,16 @@ version = "0.6.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4"
+[[package]]
+name = "xattr"
+version = "1.6.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "32e45ad4206f6d2479085147f02bc2ef834ac85886624a23575ae137c8aa8156"
+dependencies = [
+ "libc",
+ "rustix",
+]
+
[[package]]
name = "yansi"
version = "1.0.1"
diff --git a/Cargo.toml b/Cargo.toml
index c16d25d5..3a4c5695 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -60,7 +60,7 @@ dirs = "5"
secrecy = { version = "0.10", features = ["serde"] }
# Storage.
-rusqlite = { version = "0.32", features = ["bundled"] }
+rusqlite = { version = "0.32", features = ["bundled", "backup"] }
refinery = { version = "0.8", features = ["rusqlite"] }
sha2 = "0.10"
parking_lot = "0.12"
@@ -75,6 +75,10 @@ rmcp = { version = "1.7", features = ["server", "macros", "transport-io", "trans
schemars = "1"
axum = "0.8"
+# Backup / restore archive format.
+tar = "0.4"
+flate2 = "1"
+
# Testing.
tempfile = "3"
diff --git a/crates/ai-memory-cli/Cargo.toml b/crates/ai-memory-cli/Cargo.toml
index 20c61f21..6b00a5ca 100644
--- a/crates/ai-memory-cli/Cargo.toml
+++ b/crates/ai-memory-cli/Cargo.toml
@@ -22,16 +22,20 @@ axum.workspace = true
clap.workspace = true
dirs.workspace = true
figment.workspace = true
+flate2.workspace = true
rmcp.workspace = true
serde.workspace = true
serde_json.workspace = true
sysinfo.workspace = true
+tar.workspace = true
tokio.workspace = true
tokio-util.workspace = true
tracing.workspace = true
tracing-appender.workspace = true
tracing-subscriber.workspace = true
+tempfile.workspace = true
+
[dev-dependencies]
tempfile.workspace = true
diff --git a/crates/ai-memory-cli/src/cli.rs b/crates/ai-memory-cli/src/cli.rs
index 65205e53..71921865 100644
--- a/crates/ai-memory-cli/src/cli.rs
+++ b/crates/ai-memory-cli/src/cli.rs
@@ -41,6 +41,10 @@ pub enum Command {
Serve(ServeArgs),
/// Wipe the data directory's wiki/, db/, raw/ contents.
Reset(ResetArgs),
+ /// Snapshot wiki/, db/, and config.toml into a gzipped tarball.
+ Backup(BackupArgs),
+ /// Restore a backup tarball into the data directory.
+ Restore(RestoreArgs),
}
/// Arguments for `init`.
@@ -91,6 +95,25 @@ pub struct ResetArgs {
pub confirm: bool,
}
+/// Arguments for `backup`.
+#[derive(Debug, Args)]
+pub struct BackupArgs {
+ /// Destination tarball (`.tar.gz`).
+ #[arg(long, short = 'o')]
+ pub to: PathBuf,
+}
+
+/// Arguments for `restore`.
+#[derive(Debug, Args)]
+pub struct RestoreArgs {
+ /// Source tarball.
+ #[arg(long, short = 'i')]
+ pub from: PathBuf,
+ /// Overwrite an existing non-empty data dir.
+ #[arg(long)]
+ pub force: bool,
+}
+
/// Transport for the MCP server.
#[derive(Debug, Clone, Copy, clap::ValueEnum)]
pub enum TransportKind {
diff --git a/crates/ai-memory-cli/src/commands/backup.rs b/crates/ai-memory-cli/src/commands/backup.rs
new file mode 100644
index 00000000..b85261dd
--- /dev/null
+++ b/crates/ai-memory-cli/src/commands/backup.rs
@@ -0,0 +1,102 @@
+//! `ai-memory backup --to <tarball>` — make a consistent point-in-time
+//! snapshot of the data dir.
+//!
+//! Uses SQLite's online backup API so the source DB stays writable for
+//! the duration of the copy. The tarball is gzip-compressed and
+//! contains `wiki/`, a fresh `db/memory.sqlite` snapshot, and the
+//! `config.toml` (if present).
+
+use std::path::Path;
+
+use ai_memory_store::Store;
+use anyhow::{Context, Result, bail};
+use flate2::Compression;
+use flate2::write::GzEncoder;
+use tracing::info;
+
+use crate::cli::BackupArgs;
+use crate::config::Config;
+use crate::process_guard::{busy_message, sibling_processes};
+
+/// Run the `backup` subcommand.
+///
+/// # Errors
+/// Returns an error if another `ai-memory` process is running, the
+/// store cannot be opened, the SQLite snapshot fails, or the tarball
+/// cannot be written.
+pub async fn run(config: &Config, args: BackupArgs) -> Result<()> {
+ let siblings = sibling_processes();
+ if !siblings.is_empty() {
+ bail!(busy_message("backup", &siblings));
+ }
+
+ let store = Store::open(&config.data_dir)
+ .with_context(|| format!("opening store at {}", config.data_dir.display()))?;
+
+ let staging = tempfile::tempdir()?;
+ let snapshot_path = staging.path().join("memory.sqlite");
+ info!(snapshot = %snapshot_path.display(), "snapshotting SQLite");
+ store
+ .reader
+ .snapshot_to(snapshot_path.clone())
+ .await
+ .context("running SQLite online backup")?;
+
+ let dest = &args.to;
+ if let Some(parent) = dest.parent()
+ && !parent.as_os_str().is_empty()
+ {
+ std::fs::create_dir_all(parent)?;
+ }
+ let file =
+ std::fs::File::create(dest).with_context(|| format!("creating {}", dest.display()))?;
+ let encoder = GzEncoder::new(file, Compression::default());
+ let mut tar = tar::Builder::new(encoder);
+ tar.mode(tar::HeaderMode::Deterministic);
+
+ // wiki/ goes in as-is.
+ let wiki_dir = config.data_dir.join("wiki");
+ if wiki_dir.is_dir() {
+ tar.append_dir_all("wiki", &wiki_dir)
+ .with_context(|| format!("archiving {}", wiki_dir.display()))?;
+ }
+
+ // db/memory.sqlite from the snapshot, not the live file.
+ tar.append_path_with_name(&snapshot_path, "db/memory.sqlite")
+ .context("archiving db snapshot")?;
+
+ // config.toml, if present.
+ let cfg = config.data_dir.join("config.toml");
+ if cfg.is_file() {
+ tar.append_path_with_name(&cfg, "config.toml")
+ .context("archiving config.toml")?;
+ }
+
+ let encoder = tar.into_inner()?;
+ encoder.finish()?;
+
+ let size = std::fs::metadata(dest).map(|m| m.len()).unwrap_or(0);
+ info!(path = %dest.display(), bytes = size, "backup written");
+ println!("backup -> {} ({})", dest.display(), human_bytes(size));
+ Ok(())
+}
+
+fn human_bytes(n: u64) -> String {
+ const UNITS: &[&str] = &["B", "KiB", "MiB", "GiB"];
+ let mut value = n as f64;
+ let mut unit = 0;
+ while value >= 1024.0 && unit + 1 < UNITS.len() {
+ value /= 1024.0;
+ unit += 1;
+ }
+ if unit == 0 {
+ format!("{n} {}", UNITS[0])
+ } else {
+ format!("{value:.2} {}", UNITS[unit])
+ }
+}
+
+#[allow(dead_code)] // Reserved for cross-platform diagnostics later.
+fn rel_path<'a>(base: &Path, full: &'a Path) -> Option<&'a Path> {
+ full.strip_prefix(base).ok()
+}
diff --git a/crates/ai-memory-cli/src/commands/mod.rs b/crates/ai-memory-cli/src/commands/mod.rs
index 66e26656..3e94ffec 100644
--- a/crates/ai-memory-cli/src/commands/mod.rs
+++ b/crates/ai-memory-cli/src/commands/mod.rs
@@ -1,7 +1,9 @@
//! Subcommand implementations.
+pub mod backup;
pub mod init;
pub mod reset;
+pub mod restore;
pub mod search;
pub mod serve;
pub mod status;
diff --git a/crates/ai-memory-cli/src/commands/reset.rs b/crates/ai-memory-cli/src/commands/reset.rs
index b17286ae..fbaf679a 100644
--- a/crates/ai-memory-cli/src/commands/reset.rs
+++ b/crates/ai-memory-cli/src/commands/reset.rs
@@ -5,14 +5,12 @@
//! inode caused phantom search results after a reset).
use anyhow::{Result, bail};
-use std::ffi::OsStr;
-use sysinfo::System;
use crate::cli::ResetArgs;
use crate::config::Config;
+use crate::process_guard::{busy_message, sibling_processes};
const SUBDIRS: &[&str] = &["wiki", "db", "raw"];
-const BIN_NAME: &str = "ai-memory";
/// Run the `reset` subcommand.
///
@@ -22,13 +20,7 @@ const BIN_NAME: &str = "ai-memory";
pub fn run(config: &Config, args: ResetArgs) -> Result<()> {
let siblings = sibling_processes();
if !siblings.is_empty() {
- let pids: Vec<u32> = siblings.iter().map(|p| p.as_u32()).collect();
- bail!(
- "refusing to reset: {} other ai-memory process(es) running (pids: {:?}). \
- Stop them first, then re-run.",
- pids.len(),
- pids,
- );
+ bail!(busy_message("reset", &siblings));
}
if !args.confirm {
@@ -54,18 +46,3 @@ pub fn run(config: &Config, args: ResetArgs) -> Result<()> {
tracing::info!("reset complete");
Ok(())
}
-
-fn sibling_processes() -> Vec<sysinfo::Pid> {
- let mut sys = System::new();
- sys.refresh_processes(sysinfo::ProcessesToUpdate::All, true);
- let me = sysinfo::Pid::from_u32(std::process::id());
- let bin_os: &OsStr = OsStr::new(BIN_NAME);
- sys.processes_by_exact_name(bin_os)
- // On Linux, sysinfo lists tokio worker threads alongside the main
- // process under the same comm name. thread_kind() == None means
- // we're looking at the process leader, not one of its threads.
- .filter(|p| p.thread_kind().is_none())
- .map(sysinfo::Process::pid)
- .filter(|pid| *pid != me)
- .collect()
-}
diff --git a/crates/ai-memory-cli/src/commands/restore.rs b/crates/ai-memory-cli/src/commands/restore.rs
new file mode 100644
index 00000000..434802b5
--- /dev/null
+++ b/crates/ai-memory-cli/src/commands/restore.rs
@@ -0,0 +1,73 @@
+//! `ai-memory restore --from <tarball>` — restore a backup tarball.
+//!
+//! Refuses to overwrite a non-empty data dir unless `--force` is given.
+//! Refuses while another `ai-memory` process is alive. After extraction,
+//! re-opens the store so any pending migrations run (and a corrupt
+//! snapshot fails loudly).
+
+use ai_memory_store::Store;
+use anyhow::{Context, Result, bail};
+use flate2::read::GzDecoder;
+use tracing::info;
+
+use crate::cli::RestoreArgs;
+use crate::config::Config;
+use crate::process_guard::{busy_message, sibling_processes};
+
+/// Run the `restore` subcommand.
+///
+/// # Errors
+/// Returns an error if another `ai-memory` process is running, the
+/// data dir is non-empty without `--force`, the tarball cannot be
+/// extracted, or the restored store fails to open.
+pub fn run(config: &Config, args: RestoreArgs) -> Result<()> {
+ let siblings = sibling_processes();
+ if !siblings.is_empty() {
+ bail!(busy_message("restore", &siblings));
+ }
+
+ if !args.from.is_file() {
+ bail!("source tarball {} not found", args.from.display());
+ }
+
+ let wiki = config.data_dir.join("wiki");
+ let db = config.data_dir.join("db").join("memory.sqlite");
+ if (wiki.is_dir() && std::fs::read_dir(&wiki)?.next().is_some()) || db.is_file() {
+ if !args.force {
+ bail!(
+ "refusing to restore: data dir at {} is non-empty (pass --force to overwrite)",
+ config.data_dir.display(),
+ );
+ }
+ // Force path: drop the existing wiki + db so the tarball can
+ // populate them cleanly. Keep config.toml, logs/, models/.
+ for sub in ["wiki", "db"] {
+ let path = config.data_dir.join(sub);
+ if path.exists() {
+ std::fs::remove_dir_all(&path)?;
+ }
+ }
+ }
+ std::fs::create_dir_all(&config.data_dir)?;
+
+ let file = std::fs::File::open(&args.from)
+ .with_context(|| format!("opening {}", args.from.display()))?;
+ let decoder = GzDecoder::new(file);
+ let mut archive = tar::Archive::new(decoder);
+ archive.set_preserve_permissions(false);
+ archive
+ .unpack(&config.data_dir)
+ .with_context(|| format!("extracting into {}", config.data_dir.display()))?;
+ info!(from = %args.from.display(), into = %config.data_dir.display(), "tarball extracted");
+
+ // Open + drop the store so refinery applies any pending migrations
+ // and the SQLite file is validated.
+ let _store = Store::open(&config.data_dir).context("opening restored store")?;
+ info!("restore complete");
+ println!(
+ "restored {} -> {}",
+ args.from.display(),
+ config.data_dir.display()
+ );
+ Ok(())
+}
diff --git a/crates/ai-memory-cli/src/main.rs b/crates/ai-memory-cli/src/main.rs
index f2e952e8..258da7e7 100644
--- a/crates/ai-memory-cli/src/main.rs
+++ b/crates/ai-memory-cli/src/main.rs
@@ -17,6 +17,7 @@ mod cli;
mod commands;
mod config;
mod logging;
+mod process_guard;
use cli::{Cli, Command};
use config::Config;
@@ -43,5 +44,7 @@ async fn main() -> Result<()> {
Command::Watch(args) => commands::watch::run(&config, args).await,
Command::Serve(args) => commands::serve::run(&config, args).await,
Command::Reset(args) => commands::reset::run(&config, args),
+ Command::Backup(args) => commands::backup::run(&config, args).await,
+ Command::Restore(args) => commands::restore::run(&config, args),
}
}
diff --git a/crates/ai-memory-cli/src/process_guard.rs b/crates/ai-memory-cli/src/process_guard.rs
new file mode 100644
index 00000000..2a3a31c0
--- /dev/null
+++ b/crates/ai-memory-cli/src/process_guard.rs
@@ -0,0 +1,43 @@
+//! Shared "is another ai-memory process alive?" check.
+//!
+//! Used by every destructive command (`reset`, `backup`, `restore`) so
+//! we never race a live writer (lesson from basic-memory #765).
+
+use std::ffi::OsStr;
+
+use sysinfo::System;
+
+/// Binary name to match against `/proc/*/comm` (or platform equivalent).
+pub const BIN_NAME: &str = "ai-memory";
+
+/// Return PIDs of *other* `ai-memory` processes (excluding the current
+/// process and any threads of it).
+#[must_use]
+pub fn sibling_processes() -> Vec<sysinfo::Pid> {
+ let mut sys = System::new();
+ sys.refresh_processes(sysinfo::ProcessesToUpdate::All, true);
+ let me = sysinfo::Pid::from_u32(std::process::id());
+ let bin_os: &OsStr = OsStr::new(BIN_NAME);
+ sys.processes_by_exact_name(bin_os)
+ // On Linux, sysinfo lists tokio worker threads alongside the main
+ // process under the same comm name. thread_kind() == None means
+ // we're looking at the process leader, not one of its threads.
+ .filter(|p| p.thread_kind().is_none())
+ .map(sysinfo::Process::pid)
+ .filter(|pid| *pid != me)
+ .collect()
+}
+
+/// Format a "refusing to ..." error message for the given operation,
+/// quoting sibling PIDs.
+#[must_use]
+pub fn busy_message(verb: &str, siblings: &[sysinfo::Pid]) -> String {
+ let pids: Vec<u32> = siblings.iter().copied().map(sysinfo::Pid::as_u32).collect();
+ format!(
+ "refusing to {}: {} other ai-memory process(es) running (pids: {:?}). \
+ Stop them first, then re-run.",
+ verb,
+ pids.len(),
+ pids,
+ )
+}
diff --git a/crates/ai-memory-store/src/reader.rs b/crates/ai-memory-store/src/reader.rs
index abfa8651..209809b5 100644
--- a/crates/ai-memory-store/src/reader.rs
+++ b/crates/ai-memory-store/src/reader.rs
@@ -181,6 +181,19 @@ impl ReaderPool {
.await
}
+ /// Snapshot the database to `dest_path` using SQLite's online backup
+ /// API. The source DB stays writable for the duration of the copy.
+ ///
+ /// # Errors
+ /// Propagates any SQL or pool error.
+ pub async fn snapshot_to(&self, dest_path: PathBuf) -> StoreResult<()> {
+ self.with_conn(move |conn| {
+ conn.backup(rusqlite::DatabaseName::Main, &dest_path, None)
+ .map_err(StoreError::from)
+ })
+ .await
+ }
+
/// Return aggregate counts for the `status` view.
///
/// # Errors