Solution for long term memory for agent coding CLIs and to facilitate handoff between different agent vendors
Rust
//! `ai-memory init` — create the data directory layout.
use std::fs;
use std::io::Write;
#[cfg(unix)]
use std::os::unix::fs::{DirBuilderExt as _, OpenOptionsExt as _};
use std::path::{Path, PathBuf};
use anyhow::{Context, Result};
use crate::cli::InitArgs;
use crate::commands::path_util::display_path;
use crate::config::Config;
use ai_memory_mcp::auth::generate_token_hex;
const DEFAULT_CONFIG_TOML: &str = include_str!("../../templates/config.default.toml");
const SUBDIRS: &[&str] = &["wiki", "raw", "db", "models"];
/// Bytes of OS CSPRNG entropy used for the auto-generated
/// `[auth].token_pepper`. 32 bytes → 64 hex chars.
const TOKEN_PEPPER_BYTES: usize = 32;
/// Build the `[auth]` block appended to the freshly-rendered default
/// config on `ai-memory init`. The pepper is auto-generated so it's
/// stable from install onwards (rotating it invalidates every existing
/// native API key — see `ai-memory-store::api_credentials`).
fn render_default_auth_block(pepper: &str) -> String {
format!(
"
# Per-server token pepper. Keeps stolen `api_credentials.token_hash` rows useless
# to an offline attacker by mixing each token with this secret before
# hashing. Auto-generated on `ai-memory init`. **Do NOT change after
# the first native API key is added** — rotating invalidates every key.
# Human password login does not use this pepper.
token_pepper = \"{pepper}\"
# Multi-user attribution (all optional).
#
# Without these, the bearer token authenticates the request but
# leaves it anonymous. Set them to label root-token writes in the
# audit log + page frontmatter.
#
# Add human users with `ai-memory user add-human --username <name>`, then issue
# programmatic access separately with `ai-memory api-key add`.
#
# root_username = \"boss\"
# root_email = \"[email protected]\"
# root_name = \"Boss\"
#
# Trusted authenticating proxy (optional). Give the proxy a bearer token that
# differs from bearer_token. It must replace client-supplied X-Memory-Actor-*
# headers and assert either User or the Issuer+Sub pair on every request.
# actor_proxy_bearer_token = \"<distinct-random-token>\"
# root_issuer = \"https://idp.example\"
# root_subject = \"<stable-root-subject>\"
"
)
}
/// The `bearer_token_file` line appended to a rendered config.
fn render_bearer_token_file_line(raw: &str) -> String {
format!(
"
# Root bearer token, one line in an owner-only file. A relative path resolves
# against the data dir. Written by `ai-memory init` on a fresh data dir; the
# server requires it on every route but /healthz, and hooks, the CLI and
# `ai-memory mcp-bridge` read it from there. Harness configs never hold it.
bearer_token_file = {}
",
toml_edit::Value::from(raw)
)
}
/// Which `bearer_token_file` a config being written should record, minting the
/// token when this is a fresh install.
///
/// - A config being rewritten with `--force` that already named a token file
/// keeps naming it, so the rewrite does not silently turn auth off.
/// - A fresh install (no config and no database yet) with no bearer supplied
/// another way gets a new token at `auth-tokens/default`.
/// - Anything else, an existing install above all, records nothing and keeps
/// the loopback no-auth behaviour it had (`ai-memory auth enable` opts in).
fn root_bearer_for_new_config(config: &Config, fresh: bool) -> Result<Option<RecordedBearer>> {
if let Some(raw) = config
.auth
.bearer_token_file
.as_ref()
.filter(|raw| !raw.as_os_str().is_empty())
{
let token = config
.auth
.token_file_path()
.map(crate::local_auth::provision_root_token)
.transpose()?;
return Ok(Some(RecordedBearer {
raw: raw.to_string_lossy().into_owned(),
token,
}));
}
if !fresh || config.auth.bearer_token.is_some() {
return Ok(None);
}
let root = &config.data_dir;
let minted =
crate::local_auth::provision_root_token(&crate::server_profiles::root_token_path(root))?;
crate::local_auth::seed_hook_credential(root, &minted.token)?;
println!(
"Generated a root bearer token in {} (owner-only). The server requires it on every \
route but /healthz; hooks, the CLI and `ai-memory mcp-bridge` read it from there.",
display_path(&minted.path)
);
Ok(Some(RecordedBearer {
raw: crate::server_profiles::ROOT_TOKEN_RELATIVE.to_owned(),
token: Some(minted),
}))
}
/// The `bearer_token_file` a written config records, and the token read from
/// (or minted into) that file.
struct RecordedBearer {
raw: String,
token: Option<crate::local_auth::RootToken>,
}
/// What one `init` did, for a caller (`setup`) that keeps going in the same
/// process after it.
pub(crate) struct InitOutcome {
/// The config file `init` wrote or found.
pub(crate) config_path: PathBuf,
/// False when a config file already existed and was left untouched.
pub(crate) config_written: bool,
/// The root bearer the written config names in `bearer_token_file`, so the
/// caller can adopt it the way `Config::load` would have.
pub(crate) root_token: Option<crate::local_auth::RootToken>,
}
/// Run the `init` subcommand.
///
/// Creates `<data_dir>/{wiki,raw,db,models}` (idempotent) and writes a default
/// config file unless one already exists (use `--force` to overwrite). With no
/// explicit `--config`, the config remains at `<data_dir>/config.toml`; packaged
/// systemd units can pass `--config /etc/ai-memory/config.toml` or
/// `--config ~/.config/ai-memory/config.toml` without changing the data root.
///
/// On a fresh data dir it also mints the root bearer token (see
/// [`root_bearer_for_new_config`]).
///
/// # Errors
/// Returns an error if directories cannot be created or the config file
/// cannot be written.
pub fn run(config: &Config, args: InitArgs, config_path: Option<&Path>) -> Result<()> {
initialize(config, &args, config_path).map(|_| ())
}
/// [`run`], reporting what it did.
///
/// # Errors
/// As [`run`].
pub(crate) fn initialize(
config: &Config,
args: &InitArgs,
config_path: Option<&Path>,
) -> Result<InitOutcome> {
let root = &config.data_dir;
let cfg_path = init_config_path(root, config_path);
// Decided before anything is created: a data dir that already has a
// database is an existing install even when it never had a config file
// (env-configured and container installs), and it must keep running the
// way it did.
let fresh = !cfg_path.exists() && !root.join("db").join(ai_memory_store::DB_FILENAME).exists();
create_private_dir_all(root)
.with_context(|| format!("creating data root {}", display_path(root)))?;
for sub in SUBDIRS {
let path = root.join(sub);
create_private_dir_all(&path)
.with_context(|| format!("creating {}", display_path(&path)))?;
tracing::info!(path = %display_path(&path), "ensured directory");
}
let mut outcome = InitOutcome {
config_path: cfg_path.clone(),
config_written: false,
root_token: None,
};
if cfg_path.exists() && !args.force {
tracing::info!(
path = %display_path(&cfg_path),
"config already exists; leaving untouched (pass --force to overwrite)",
);
} else {
if let Some(parent) = cfg_path.parent() {
create_private_dir_all(parent)
.with_context(|| format!("creating config dir {}", display_path(parent)))?;
}
// Pepper is generated NOW (not at first server start) so it's
// stable from install onwards: see comment in render_default_
// auth_block. getrandom::Error doesn't impl `std::error::Error`
// (anyhow's `with_context` would conflict), so map manually.
let pepper = generate_token_hex(TOKEN_PEPPER_BYTES)
.map_err(|e| anyhow::anyhow!("generating auth token_pepper: {e}"))?;
let recorded = root_bearer_for_new_config(config, fresh)?;
let bearer_line = recorded
.as_ref()
.map(|bearer| render_bearer_token_file_line(&bearer.raw))
.unwrap_or_default();
outcome.root_token = recorded.and_then(|bearer| bearer.token);
let body = format!(
"{}{}{}",
DEFAULT_CONFIG_TOML,
render_default_auth_block(&pepper),
bearer_line
);
let mut f = private_config_file(&cfg_path, args.force)
.with_context(|| format!("creating {}", display_path(&cfg_path)))?;
f.write_all(body.as_bytes())
.with_context(|| format!("writing {}", display_path(&cfg_path)))?;
tracing::info!(path = %display_path(&cfg_path), "wrote default config");
outcome.config_written = true;
}
tracing::info!("init complete");
Ok(outcome)
}
/// Create missing directory components with owner-only access without changing
/// the permissions of an existing installation.
fn create_private_dir_all(path: &Path) -> std::io::Result<()> {
let mut builder = fs::DirBuilder::new();
builder.recursive(true);
#[cfg(unix)]
builder.mode(0o700);
builder.create(path)
}
/// Open a config file so a newly created file is private before its secret
/// token pepper is written. Forced rewrites preserve existing permissions.
fn private_config_file(path: &Path, force: bool) -> std::io::Result<fs::File> {
let mut options = fs::OpenOptions::new();
options.write(true).truncate(true).create(true);
if !force {
options.create_new(true);
}
#[cfg(unix)]
options.mode(0o600);
options.open(path)
}
fn init_config_path(data_dir: &Path, explicit: Option<&Path>) -> PathBuf {
explicit
.map(PathBuf::from)
.unwrap_or_else(|| data_dir.join("config.toml"))
}
#[cfg(test)]
mod tests {
use super::*;
use tempfile::TempDir;
fn cfg_in(dir: &std::path::Path) -> Config {
Config {
data_dir: dir.to_path_buf(),
..Config::default()
}
}
#[test]
fn init_creates_subdirs_and_config() {
let tmp = TempDir::new().unwrap();
let config = cfg_in(tmp.path());
run(&config, InitArgs { force: false }, None).unwrap();
for sub in SUBDIRS {
assert!(tmp.path().join(sub).is_dir(), "missing {sub}");
}
assert!(tmp.path().join("config.toml").exists());
}
#[test]
fn init_is_idempotent() {
let tmp = TempDir::new().unwrap();
let config = cfg_in(tmp.path());
run(&config, InitArgs { force: false }, None).unwrap();
// Touch the config to detect a clobber.
let stamp = std::fs::metadata(tmp.path().join("config.toml"))
.unwrap()
.modified()
.unwrap();
std::thread::sleep(std::time::Duration::from_millis(20));
run(&config, InitArgs { force: false }, None).unwrap();
let stamp2 = std::fs::metadata(tmp.path().join("config.toml"))
.unwrap()
.modified()
.unwrap();
assert_eq!(stamp, stamp2, "second init clobbered the config");
}
#[test]
fn init_writes_token_pepper_into_auth_block() {
let tmp = TempDir::new().unwrap();
let config = cfg_in(tmp.path());
run(&config, InitArgs { force: false }, None).unwrap();
let body = std::fs::read_to_string(tmp.path().join("config.toml")).unwrap();
assert!(
body.contains("[auth]"),
"rendered config must include the [auth] section"
);
assert!(
body.contains("token_pepper = "),
"rendered config must include the auto-generated token_pepper"
);
// Sanity: pepper is 64 hex chars (32 bytes).
let pepper_line = body
.lines()
.find(|l| l.trim_start().starts_with("token_pepper"))
.expect("token_pepper line present");
let hex_value = pepper_line
.split('"')
.nth(1)
.expect("quoted token_pepper value");
assert_eq!(hex_value.len(), 64, "token_pepper must be 64 hex chars");
assert!(hex_value.chars().all(|c| c.is_ascii_hexdigit()));
}
#[test]
fn init_renders_config_that_parses_back_into_auth_settings() {
// Regression guard: the appended [auth] block must be valid TOML
// *and* round-trip into the same AuthSettings the server uses.
// Catches a future drift between the template and the schema.
// Uses the same figment pipeline `Config::load` runs in production
// (Serialized defaults + Toml file merge) so the test exercises
// the actual loader, not a parallel parser.
use figment::Figment;
use figment::providers::{Format as _, Serialized, Toml};
let tmp = TempDir::new().unwrap();
let cfg = cfg_in(tmp.path());
run(&cfg, InitArgs { force: false }, None).unwrap();
let cfg_path = tmp.path().join("config.toml");
let loaded: Config = Figment::from(Serialized::defaults(Config::default()))
.merge(Toml::file(&cfg_path))
.extract()
.expect("rendered config must round-trip via figment (full Config extract)");
assert!(
!loaded.auth.secure_cookie,
"generated config must preserve direct loopback HTTP browser compatibility"
);
assert!(
loaded
.auth
.token_pepper
.as_deref()
.is_some_and(|p| p.len() == 64),
"parsed token_pepper must round-trip from the rendered template"
);
// `[auth]` must stay the LAST section of config.default.toml: the
// block appended here emits bare keys, so any section added below it
// silently swallows them into the wrong table. Assert the structural
// rule directly rather than leaving it to whichever `[auth]` key
// happens to be checked above.
let rendered = std::fs::read_to_string(&cfg_path).unwrap();
let last_section = rendered
.lines()
.rfind(|line| line.trim_start().starts_with('['))
.expect("rendered config declares sections");
assert_eq!(
last_section.trim(),
"[auth]",
"[auth] must remain the last section; init appends bare keys after it"
);
assert!(loaded.auth.bearer_token.is_none());
assert!(!loaded.slots.per_user);
assert_eq!(
loaded.routing.mid_session,
ai_memory_core::MidSessionRouting::FollowCwd,
"generated config must preserve historical per-event attribution"
);
assert_eq!(
loaded.consolidation.max_input_tokens,
ai_memory_consolidate::DEFAULT_CONSOLIDATION_MAX_INPUT_TOKENS
);
assert_eq!(
loaded.consolidation.max_output_tokens,
ai_memory_consolidate::DEFAULT_CONSOLIDATION_MAX_OUTPUT_TOKENS
);
assert!(loaded.auth.root_username.is_none());
}
#[test]
fn init_generates_a_unique_pepper_per_run() {
// Two fresh installs must not collide on the pepper. Anyone
// hand-patching this to a constant for testing would break the
// security model on every new server.
let tmp_a = TempDir::new().unwrap();
let tmp_b = TempDir::new().unwrap();
run(&cfg_in(tmp_a.path()), InitArgs { force: false }, None).unwrap();
run(&cfg_in(tmp_b.path()), InitArgs { force: false }, None).unwrap();
let a = std::fs::read_to_string(tmp_a.path().join("config.toml")).unwrap();
let b = std::fs::read_to_string(tmp_b.path().join("config.toml")).unwrap();
let pa = a
.lines()
.find(|l| l.trim_start().starts_with("token_pepper"))
.unwrap();
let pb = b
.lines()
.find(|l| l.trim_start().starts_with("token_pepper"))
.unwrap();
assert_ne!(
pa, pb,
"two fresh inits must generate distinct peppers (got {pa})"
);
}
#[test]
fn init_honors_explicit_config_path_outside_data_dir() {
let tmp = TempDir::new().unwrap();
let data_dir = tmp.path().join("data");
let config_dir = tmp.path().join("config").join("ai-memory");
let config_path = config_dir.join("config.toml");
let config = cfg_in(&data_dir);
run(
&config,
InitArgs { force: false },
Some(config_path.as_path()),
)
.unwrap();
assert!(data_dir.join("wiki").is_dir());
assert!(data_dir.join("raw").is_dir());
assert!(config_path.is_file());
assert!(
!data_dir.join("config.toml").exists(),
"explicit --config must not also write data-dir config"
);
}
/// Load the rendered config through the production loader, with no
/// process environment, the way the server and every client will.
fn load(data_dir: &Path, config_path: Option<&Path>) -> Config {
Config::load_with_runtime_env(
config_path,
Some(data_dir.to_path_buf()),
crate::config::RuntimeEnv::default(),
)
.unwrap()
}
/// New-install default: a fresh data dir gets a root bearer, recorded as a relative
/// `bearer_token_file` the loader resolves, and the hook credential files
/// carry the same token. The token is in no rendered config.
#[test]
fn a_fresh_init_mints_a_root_bearer_the_loader_reads() {
let tmp = TempDir::new().unwrap();
let root = tmp.path().join("data");
run(&cfg_in(&root), InitArgs { force: false }, None).unwrap();
let token_path = crate::server_profiles::root_token_path(&root);
let token = crate::config::read_trimmed_secret(&token_path).expect("token minted");
let body = std::fs::read_to_string(root.join("config.toml")).unwrap();
assert!(
body.contains("bearer_token_file = \"auth-tokens/default\""),
"{body}"
);
assert!(
!body.contains(&token),
"the token never goes into config.toml"
);
let loaded = load(&root, None);
assert_eq!(loaded.auth.bearer_token.as_deref(), Some(token.as_str()));
assert_eq!(
loaded.auth.renderable_bearer_token(),
None,
"a file-sourced token is never handed to a harness renderer"
);
assert_eq!(
crate::config::read_hook_auth_token(&root).as_deref(),
Some(token.as_str()),
"hooks read the same token from the files they already know"
);
}
/// Re-running init leaves the token alone; `--force` rewrites the config
/// but keeps naming (and reusing) the same token file, so a rewrite never
/// silently turns auth off or rotates a credential clients hold.
#[test]
fn re_running_init_keeps_the_same_root_bearer() {
let tmp = TempDir::new().unwrap();
let root = tmp.path().join("data");
run(&cfg_in(&root), InitArgs { force: false }, None).unwrap();
let token_path = crate::server_profiles::root_token_path(&root);
let first = crate::config::read_trimmed_secret(&token_path).unwrap();
run(&load(&root, None), InitArgs { force: false }, None).unwrap();
run(&load(&root, None), InitArgs { force: true }, None).unwrap();
assert_eq!(
crate::config::read_trimmed_secret(&token_path),
Some(first.clone())
);
assert_eq!(load(&root, None).auth.bearer_token, Some(first));
}
/// Control: a data dir that already holds a database is an existing
/// install, even with no config file. It gets no token and keeps the
/// loopback no-auth behaviour.
#[test]
fn init_on_an_existing_install_does_not_turn_auth_on() {
let tmp = TempDir::new().unwrap();
let root = tmp.path().join("data");
std::fs::create_dir_all(root.join("db")).unwrap();
std::fs::write(root.join("db").join(ai_memory_store::DB_FILENAME), b"").unwrap();
run(&cfg_in(&root), InitArgs { force: false }, None).unwrap();
let body = std::fs::read_to_string(root.join("config.toml")).unwrap();
assert!(!body.contains("bearer_token_file"), "{body}");
assert!(!crate::server_profiles::root_token_path(&root).exists());
assert!(crate::config::read_hook_auth_token(&root).is_none());
assert!(load(&root, None).auth.bearer_token.is_none());
}
/// An operator who already supplies a bearer (`AI_MEMORY_AUTH_TOKEN` or
/// an inline key) chose their credential; a second one would leave the
/// hooks holding a token the server does not enforce.
#[test]
fn init_with_a_supplied_bearer_mints_nothing() {
let tmp = TempDir::new().unwrap();
let root = tmp.path().join("data");
let mut config = cfg_in(&root);
config.auth.bearer_token = Some("operator-token".into());
run(&config, InitArgs { force: false }, None).unwrap();
assert!(!crate::server_profiles::root_token_path(&root).exists());
let body = std::fs::read_to_string(root.join("config.toml")).unwrap();
assert!(!body.contains("bearer_token_file"), "{body}");
}
/// With `--config` outside the data dir the token still lives in the data
/// dir, and the relative path resolves there.
#[test]
fn a_fresh_init_with_an_external_config_resolves_the_token_in_the_data_dir() {
let tmp = TempDir::new().unwrap();
let root = tmp.path().join("data");
let config_path = tmp.path().join("etc").join("config.toml");
run(
&cfg_in(&root),
InitArgs { force: false },
Some(&config_path),
)
.unwrap();
let token =
crate::config::read_trimmed_secret(&crate::server_profiles::root_token_path(&root))
.unwrap();
assert_eq!(
load(&root, Some(&config_path)).auth.bearer_token,
Some(token)
);
}
#[cfg(unix)]
#[test]
fn init_creates_private_directories_and_config() {
use std::os::unix::fs::PermissionsExt as _;
let tmp = TempDir::new().unwrap();
let root = tmp.path().join("data");
run(&cfg_in(&root), InitArgs { force: false }, None).unwrap();
for path in std::iter::once(root.clone()).chain(SUBDIRS.iter().map(|sub| root.join(sub))) {
assert_eq!(
fs::metadata(&path).unwrap().permissions().mode() & 0o777,
0o700,
"{}",
path.display()
);
}
assert_eq!(
fs::metadata(root.join("config.toml"))
.unwrap()
.permissions()
.mode()
& 0o777,
0o600
);
}
}