Open source Grok Bot alternative, Bring your own agent and subscription: message Claude Code, Codex, Gemini CLI, Cursor, Copilot and 40+ AI agents as bots on your own computer, from iPhone, Mac, Windows, Linux or terminal. Agent-agnostic and free. Also a Muse and Dots alternative. Built in Rust.
Rust acpagent-client-protocolai-agentsai-assistantclaudeclaude-codecodexcoding-agentscursorgeminigemini-cligrok-bot-alternativeiosiphonelinuxmacosmcpopen-sourcerustself-hosted
//! Memory on disk ([design](../../../../docs/design/bot-memory.md#memory-is-files)): the folders, the scan that
//! turns what changed in them into versions, and the host's own compare-and-swap writes. Which bot's turn a change
//! is credited to is `turns`. The rules (names, checks, who wrote each fact, what a bot may change, the index) are
//! `codync_mem::files`.
use crate::LockExt;
use crate::hub::Hub;
use crate::store::{BotConfig, Store};
use anyhow::{Context, Result, anyhow, bail};
use codync_mem::files::{self as mf, Author, Cause, Change, Version};
use std::collections::{HashMap, HashSet};
use std::path::{Path, PathBuf};
use std::sync::Mutex;
use std::time::UNIX_EPOCH;
/// A file changed more recently than this may still be mid-write: the next scan takes it.
pub(super) const SETTLE_MS: i64 = 2_000;
/// How long a file's earlier versions are kept, and with them what a bot forgot.
const KEEP_MS: i64 = 30 * 24 * 60 * 60 * 1000;
/// A layer's folder: the shared one (`None`), or a bot's own.
pub fn folder(bot: Option<&str>) -> PathBuf {
let data = crate::service::data_dir();
match bot {
Some(id) => data.join("bots").join(id).join("bot-memory"),
None => data.join("shared-memory"),
}
}
/// The layers a bot sees: its own, and the shared one when it uses it.
pub fn layers(cfg: &BotConfig) -> Vec<Option<String>> {
let mut layers = vec![Some(cfg.id.clone())];
if super::shared(cfg) {
layers.push(None);
}
layers
}
/// The folders a bot sees, created so an agent can be given them.
pub fn folders(cfg: &BotConfig) -> Vec<PathBuf> {
layers(cfg)
.iter()
.map(|layer| {
let dir = folder(layer.as_deref());
if let Err(error) = std::fs::create_dir_all(&dir) {
tracing::warn!(dir = %dir.display(), error = %error, "couldn't create a memory folder");
}
dir
})
.collect()
}
/// The index of the notes a bot sees, for its opening.
pub fn index(store: &Store, cfg: &BotConfig) -> String {
match store.memory_read(|c| mf::index(c, &cfg.id, super::shared(cfg))) {
Ok(index) => index.render(|bot| folder(bot).display().to_string()),
Err(error) => {
tracing::warn!(bot = %cfg.id, error = format!("{error:#}"), "couldn't read the memory files");
String::new()
}
}
}
/// One scan or host write at a time, so two can't record the same change, nor two writes pass the same
/// compare-and-swap.
// ponytail: one lock for every folder; per-folder locks if scans ever queue behind each other.
static SCANNING: Mutex<()> = Mutex::new(());
/// Records what changed in the folders a bot sees since the last scan. Returns whether a file was too fresh to
/// take yet.
pub fn scan(hub: &Hub, cfg: &BotConfig) -> bool {
let _one = SCANNING.locked();
prune(&hub.store);
let mut deferred = false;
for layer in layers(cfg) {
match scan_layer(hub, layer.as_deref()) {
Ok(fresh) => deferred |= fresh,
Err(error) => tracing::warn!(bot = %cfg.id, error = format!("{error:#}"), "couldn't scan the memory files"),
}
}
deferred
}
/// Once a day: versions older than [`KEEP_MS`] go.
fn prune(store: &Store) {
let today = chrono::Local::now().format("%Y-%m-%d").to_string();
if store.kv_get("memory.pruned").as_deref() == Some(today.as_str()) {
return;
}
match store.memory_write(|c, now, _| mf::prune(c, now - KEEP_MS)) {
Ok(()) => {
let _ = store.kv_set("memory.pruned", &today);
}
Err(error) => tracing::warn!(error = format!("{error:#}"), "couldn't prune old memory versions"),
}
}
/// What is on disk for one file.
struct OnDisk {
content: String,
mtime: i64,
size: u64,
}
/// Scans one folder; true when a file was too fresh to take yet.
fn scan_layer(hub: &Hub, layer: Option<&str>) -> Result<bool> {
let (mut items, unread) = listing(&folder(layer));
let mut deferred = false;
let recorded: HashMap<String, mf::File> =
hub.store.memory_read(|c| mf::layer(c, layer))?.into_iter().map(|f| (f.name.clone(), f)).collect();
let now = crate::store::now_ms();
let mut present = HashSet::new();
// MEMORY.md last: a fact moved from it into a new note is then found in the note, not taken as forgotten.
items.sort_by_key(|(name, _)| name == mf::MAIN);
for (name, path) in items {
let Some((mtime, size)) = regular(&path, &name) else { continue };
present.insert(name.clone());
let known = recorded.get(&name);
if known.is_some_and(|f| !f.deleted && f.mtime == mtime && f.size == size) {
continue;
}
// A time in the future isn't waited for: it would outlast the turn that set it.
if (0..SETTLE_MS).contains(&(now - mtime)) {
deferred = true;
continue;
}
let content = match std::fs::read_to_string(&path) {
Ok(content) => content,
Err(error) => {
tracing::warn!(file = %name, error = %error, "couldn't read a memory file");
continue;
}
};
if known.is_some_and(|f| f.hash == mf::hash(&content)) {
hub.store.memory_write(|c, _, _| mf::observed(c, layer, &name, mtime, size))?;
continue;
}
found(hub, layer, &name, Some(&OnDisk { content, mtime, size }))?;
}
let gone = |f: &&mf::File| {
!f.deleted && !present.contains(&f.name) && !unread.iter().any(|prefix| f.name.starts_with(prefix.as_str()))
};
for name in recorded.values().filter(gone).map(|f| &f.name) {
found(hub, layer, name, None)?;
}
Ok(deferred)
}
/// Every entry of a layer's folder and its topic folders, named from the layer's folder (`health/allergies.md`),
/// and the folders that couldn't be read whole (`health/`, or `""` for the layer's own): a folder that went missing
/// or can't be read now is not every note in it deleted. Only folders a note may sit in are entered: never a link
/// (a `DirEntry`'s type doesn't follow one), a hidden one or one too deep.
fn listing(dir: &Path) -> (Vec<(String, PathBuf)>, Vec<String>) {
let (mut found, mut unread) = (Vec::new(), Vec::new());
let mut folders = vec![(String::new(), dir.to_path_buf())];
while let Some((prefix, dir)) = folders.pop() {
let read = std::fs::read_dir(&dir).and_then(|items| {
for item in items {
let item = item?;
let Some(name) = item.file_name().to_str().map(|n| format!("{prefix}{n}")) else { continue };
if item.file_type()?.is_dir() {
if mf::valid_folder(&name) {
folders.push((format!("{name}/"), item.path()));
}
continue;
}
found.push((name, item.path()));
}
Ok(())
});
if let Err(error) = read {
if error.kind() != std::io::ErrorKind::NotFound {
tracing::warn!(dir = %dir.display(), error = %error, "couldn't read a memory folder");
}
unread.push(prefix);
}
}
(found, unread)
}
/// A memory file's path, when each topic folder on the way is a real folder or not there yet: never a link, so
/// nothing can make the host read or write a file outside the memory folders.
fn path_of(layer: Option<&str>, name: &str) -> Result<PathBuf> {
let mut path = folder(layer);
let mut parts = name.split('/').peekable();
while let Some(part) = parts.next() {
path.push(part);
if parts.peek().is_some() && std::fs::symlink_metadata(&path).is_ok_and(|meta| !meta.is_dir()) {
bail!("a topic folder of {name} is a link");
}
}
Ok(path)
}
/// A regular `.md` file with a valid name: its modification time and size. Links, hidden files and folders are
/// left alone, so nothing can make the host read, journal or restore another file.
fn regular(path: &Path, name: &str) -> Option<(i64, u64)> {
if !mf::valid_name(name) {
return None;
}
let meta = std::fs::symlink_metadata(path).ok().filter(std::fs::Metadata::is_file)?;
let mtime = meta.modified().ok()?.duration_since(UNIX_EPOCH).ok()?.as_millis();
Some((i64::try_from(mtime).ok()?, meta.len()))
}
/// A change the scan found: made in a bot's turn, checked, and put back when it may not stand; otherwise just
/// recorded as the computer's.
fn found(hub: &Hub, layer: Option<&str>, name: &str, after: Option<&OnDisk>) -> Result<()> {
let before = hub.store.memory_read(|c| mf::current(c, layer, name))?;
let (mtime, size) = after.map_or((0, 0), |a| (a.mtime, a.size));
let content = after.map(|a| a.content.as_str());
let change = Change {
bot: layer,
name,
content,
author: Author::Computer,
author_bot: None,
cause: if content.is_some() { Cause::Edit } else { Cause::Delete },
message: None,
mtime,
size,
};
let Some((cfg, turn)) = super::turns::turn_of(hub, layer, name, after.map(|a| a.mtime))? else {
record(hub, &change)?;
return Ok(());
};
let asked = turn.message.is_some();
if let Err(error) = admit(hub, layer, name, before.as_deref(), content, asked) {
// Put back what was there, as it was recorded, and tell the bot why.
let path = path_of(layer, name)?;
put(&path, before.as_deref())?;
let (mtime, size) = regular(&path, name).unwrap_or_default();
if before.is_some() {
hub.store.memory_write(|c, _, _| mf::observed(c, layer, name, mtime, size))?;
}
tracing::info!(bot = %cfg.id, file = name, reason = %error, "put back a bot's memory change");
super::put_back(&hub.store, &cfg.id, &turn.session, name, &error.to_string());
return Ok(());
}
let change = Change { author: Author::Bot, author_bot: Some(&cfg.id), message: turn.message.as_deref(), ..change };
record_bot_change(hub, &cfg, Some(&turn.session), before.as_deref(), &change, asked)?;
Ok(())
}
/// Whether a bot's change to a file may stand: what it brings in may be memory, and a bot may change it outside a
/// turn the user started in its own chat (`asked`).
fn admit(
hub: &Hub,
layer: Option<&str>,
name: &str,
before: Option<&str>,
after: Option<&str>,
asked: bool,
) -> Result<()> {
if let Some(content) = after {
mf::check_change(layer, name, before, content, asked)?;
}
hub.store.memory_read(|c| mf::admit(c, layer, before, after, asked))?;
Ok(())
}
/// Records a bot's change, with what it had forgotten when the user asked for it, and tells the other sessions;
/// the one it was made in (`session`) already knows.
fn record_bot_change(
hub: &Hub,
cfg: &BotConfig,
session: Option<&str>,
before: Option<&str>,
change: &Change<'_>,
asked: bool,
) -> Result<Version> {
let heard = super::current_revision(&hub.store, &cfg.id);
let version = hub.store.memory_write(|c, now, rev| {
mf::settle(c, change.bot, change.name, before, change.content, asked)?;
mf::record(c, change, now, rev)
})?;
super::changed(hub, change.bot);
if let Some(session) = session
&& let Err(error) = super::heard_own_write(&hub.store, &cfg.id, session, &heard)
{
tracing::warn!(bot = %cfg.id, error = format!("{error:#}"), "couldn't record the bot's own edit");
}
Ok(version)
}
/// The file's content on disk now, `None` when there is none. Refuses names the folders don't hold.
pub fn read(layer: Option<&str>, name: &str) -> Result<Option<String>> {
if !mf::valid_name(name) {
bail!(
"Memory files are MEMORY.md, or notes named in lowercase letters, digits and dashes, ending in .md, in up to {} topic folders (like people.md or health/allergies.md).",
mf::DEPTH_MAX
);
}
let Some(path) = path_of(layer, name).ok().filter(|path| regular(path, name).is_some()) else { return Ok(None) };
std::fs::read_to_string(&path).map(Some).with_context(|| format!("reading {name}"))
}
/// Who a host write is for.
#[derive(Clone, Copy)]
pub struct Writer<'a> {
pub author: Author,
pub bot: Option<&'a BotConfig>,
pub cause: Cause,
/// A bot's write: the user's message that started the turn, when it was one the user started in the bot's own
/// chat. Only then may it change or take out what memory says.
pub message: Option<&'a str>,
}
/// Writes a file the way every host writer does: only over the content whose hash is `expected` (`None`: no file
/// yet), through a temporary file in the same folder renamed over it, or by deleting it (`content` `None`). A file
/// that changed meanwhile isn't overwritten: its change is recorded and the write fails. A bot's write passes the
/// checks and the rules on what it may change first.
pub fn write(
hub: &Hub,
layer: Option<&str>,
name: &str,
content: Option<&str>,
expected: Option<&str>,
writer: Writer<'_>,
) -> Result<Version> {
// Held until the change is recorded, so another writer can't pass the same compare-and-swap meanwhile.
let _one = SCANNING.locked();
let now = read(layer, name)?;
if now.as_deref().map(mf::hash).as_deref() != expected {
// The other change is recorded, so it isn't lost.
scan_layer(hub, layer)?;
bail!("{name} changed meanwhile. Read it again and redo the change on top of it.");
}
let asked = writer.message.is_some();
if writer.author == Author::Bot {
admit(hub, layer, name, now.as_deref(), content, asked)?;
}
if now.as_deref() == content {
return hub
.store
.memory_read(|c| {
let file = mf::get(c, layer, name)?.ok_or(codync_mem::store::Error::NotFound)?;
mf::version(c, file.version)?.ok_or(codync_mem::store::Error::NotFound)
})
.context("the file has no recorded version");
}
let path = path_of(layer, name)?;
put(&path, content)?;
let (mtime, size) = regular(&path, name).unwrap_or_default();
let change = Change {
bot: layer,
name,
content,
author: writer.author,
author_bot: writer.bot.map(|cfg| cfg.id.as_str()),
cause: writer.cause,
message: writer.message,
mtime,
size,
};
match writer.bot {
Some(cfg) if writer.author == Author::Bot => {
let session = hub.runtime(&cfg.id).session;
record_bot_change(hub, cfg, session.as_deref(), now.as_deref(), &change, asked)
}
_ => record(hub, &change),
}
}
/// Records a change no rule on a bot's changes applies to, and tells the apps and sessions.
fn record(hub: &Hub, change: &Change<'_>) -> Result<Version> {
let version = hub.store.memory_write(|c, now, rev| mf::record(c, change, now, rev))?;
super::changed(hub, change.bot);
Ok(version)
}
/// Replaces or deletes a file on disk at a [`path_of`] path, its missing folders made first: a temporary sibling
/// renamed over it, so a reader never sees half of it.
fn put(path: &Path, content: Option<&str>) -> Result<()> {
let Some(content) = content else {
return match std::fs::remove_file(path) {
Err(error) if error.kind() != std::io::ErrorKind::NotFound => Err(error.into()),
_ => Ok(()),
};
};
let dir = path.parent().ok_or_else(|| anyhow!("a memory file has a folder"))?;
std::fs::create_dir_all(dir).context("creating the memory folder")?;
let temp = dir.join(format!(".{}.tmp", uuid::Uuid::new_v4()));
std::fs::write(&temp, content).context("writing the memory file")?;
if let Err(error) = std::fs::rename(&temp, path) {
let _ = std::fs::remove_file(&temp);
return Err(error).context("replacing the memory file");
}
Ok(())
}
/// A deleted bot's folder.
pub fn remove_bot(id: &str) {
if let Err(error) = std::fs::remove_dir_all(folder(Some(id)))
&& error.kind() != std::io::ErrorKind::NotFound
{
tracing::warn!(error = %error, "couldn't remove a deleted bot's memory files");
}
}