Open source Grok Bot alternative, Bring your own agent and subscription: message Claude Code, Codex, Gemini CLI, Cursor, Copilot and 40+ AI agents as bots on your own computer, from iPhone, Mac, Windows, Linux or terminal. Agent-agnostic and free. Also a Muse and Dots alternative. Built in Rust.
Rust acpagent-client-protocolai-agentsai-assistantclaudeclaude-codecodexcoding-agentscursorgeminigemini-cligrok-bot-alternativeiosiphonelinuxmacosmcpopen-sourcerustself-hosted
feat(host): keep bot memory in the host instead of Engram
leepokai41ca73ac46e4
diff --git a/.githooks/pre-commit b/.githooks/pre-commitindex 5950e386..385731b6 100755--- a/.githooks/pre-commit+++ b/.githooks/pre-commit@@ -36,15 +36,12 @@ staged_copy() {git archive "$(git write-tree)" | tar -x -C "$snapshot"}-if touches '^(host/|packaging/(updates|engram)/|docs/reference/fixtures/remote-relay-vectors\.json)'; then+if touches '^(host/|packaging/updates/|docs/reference/fixtures/remote-relay-vectors\.json)'; thenstaged_copyecho "Checking the host (cargo fmt, clippy, test)..."- # CI's pinned memory runtime (fetched once): without it every test turn waits on installing one.- engram=$(python3 "$repo/packaging/engram/test-runtime.py")- (cd "$snapshot/host" && export CARGO_TARGET_DIR="$repo/host/target/precommit" CODYNC_TEST_ENGRAM="$engram" &&+ (cd "$snapshot/host" && export CARGO_TARGET_DIR="$repo/host/target/precommit" &&cargo fmt --all --check && cargo clippy --workspace --all-targets -- -D warnings && cargo test --workspace --no-run &&- within 300 sh -c 'cargo test --workspace && cargo test --test memory_e2e -- --ignored &&- cargo test --bin codync-host native_memory_survives_restart -- --ignored')+ within 300 cargo test --workspace)fiif touches '^apps/desktop/'; thendiff --git a/.github/workflows/host.yml b/.github/workflows/host.ymlindex 1f914cee..e545e918 100644--- a/.github/workflows/host.yml+++ b/.github/workflows/host.yml@@ -4,7 +4,7 @@ on:push:tags: ["v*"] # one version tag ships the host and the desktop app (release-desktop.yml)pull_request:- paths: ["host/**", "packaging/updates/**", "packaging/engram/**", "docs/reference/fixtures/remote-relay-vectors.json", ".github/workflows/host.yml"]+ paths: ["host/**", "packaging/updates/**", "docs/reference/fixtures/remote-relay-vectors.json", ".github/workflows/host.yml"]# A newer push to the same PR supersedes the run in progress; tag runs always finish.concurrency:@@ -32,11 +32,7 @@ jobs:workspaces: host- run: cargo fmt --all --check- run: cargo clippy --workspace --all-targets -- -D warnings- - name: Fetch pinned memory runtime for integration tests- run: python ../packaging/engram/test-runtime.py- run: cargo test --workspace- - run: cargo test --test memory_e2e -- --ignored- - run: cargo test --bin codync-host native_memory_survives_restart -- --ignoredrelease:if: startsWith(github.ref, 'refs/tags/v')diff --git a/host/crates/memory/src/check.rs b/host/crates/memory/src/check.rsindex b601998c..f7ea3227 100644--- a/host/crates/memory/src/check.rs+++ b/host/crates/memory/src/check.rs@@ -17,6 +17,8 @@ pub enum Refusal {Secret,/// The user deleted this memory before.Forgotten,+ /// The layer already holds this memory.+ Duplicate,/// The layer is at its cap.Full,}@@ -31,6 +33,7 @@ impl Refusal {Self::Instruction => "Memory holds facts about the user, not instructions to an assistant.",Self::Secret => "Passwords, keys, ID numbers and card or account numbers are never stored.",Self::Forgotten => "The user deleted this memory before, so it isn't saved again.",+ Self::Duplicate => "Memory already has this.",Self::Full => "This memory layer is full. Revise or archive an entry first.",}}diff --git a/host/crates/memory/src/store.rs b/host/crates/memory/src/store.rsindex 43bf92da..dca1ec37 100644--- a/host/crates/memory/src/store.rs+++ b/host/crates/memory/src/store.rs@@ -141,25 +141,41 @@ pub fn cap(bot: Option<&str>) -> usize {}pub fn add(conn: &Connection, writer: Writer<'_>, new: New<'_>, now: i64, rev: i64) -> Result<Entry> {+ admissible(conn, writer, new.bot, new.text)?;let text = new.text.trim();+ conn.execute(+ "INSERT INTO memory_entries (bot, state, text, author, author_bot, room, evidence, created_at, updated_at, rev)+ VALUES (?1, 'active', ?2, ?3, ?4, ?5, ?6, ?7, ?7, ?8)",+ params![new.bot, text, author_name(writer.author()), writer.bot(), new.room, new.evidence.join(","), now, rev],+ )?;+ get(conn, conn.last_insert_rowid())?.ok_or(Error::NotFound)+}++/// Whether [`add`] would take `text` into a layer (`None`: "yours") from `writer`; nothing is written. Lets a+/// caller refuse at once what it would only write later, such as a bot's proposal waiting for the user.+pub fn admissible(conn: &Connection, writer: Writer<'_>, bot: Option<&str>, text: &str) -> Result<()> {+ let text = text.trim();if text.is_empty() {return Err(Refusal::Empty.into());}if writer != Writer::User {- check::text(text)?;- if forgotten(conn, new.bot, text)? {+ match check::text(text) {+ // Moved memory was written under older, looser limits; it stays whole.+ Err(Refusal::TooLong) if writer == Writer::Import => {}+ checked => checked?,+ }+ if forgotten(conn, bot, text)? {return Err(Refusal::Forgotten.into());}- if used(conn, new.bot)? + tokens::estimate(text) > cap(new.bot) {- return Err(Refusal::Full.into());+ if duplicate(conn, bot, text)? {+ return Err(Refusal::Duplicate.into());}}- conn.execute(- "INSERT INTO memory_entries (bot, state, text, author, author_bot, room, evidence, created_at, updated_at, rev)- VALUES (?1, 'active', ?2, ?3, ?4, ?5, ?6, ?7, ?7, ?8)",- params![new.bot, text, author_name(writer.author()), writer.bot(), new.room, new.evidence.join(","), now, rev],- )?;- get(conn, conn.last_insert_rowid())?.ok_or(Error::NotFound)+ // An import may run past the cap: the opening takes what fits, and the importer archives the rest.+ if matches!(writer, Writer::Bot(_) | Writer::Dream(_)) && used(conn, bot)? + tokens::estimate(text) > cap(bot) {+ return Err(Refusal::Full.into());+ }+ Ok(())}pub fn revise(conn: &Connection, writer: Writer<'_>, id: i64, text: &str, now: i64, rev: i64) -> Result<Entry> {@@ -228,6 +244,16 @@ pub fn forget(conn: &Connection, id: i64) -> Result<Entry> {Ok(entry)}+/// Keeps `text` out of a layer (`None`: "yours") for bots and dreaming, as if it had been forgotten: the user+/// declined it.+pub fn forbid(conn: &Connection, bot: Option<&str>, text: &str) -> Result<()> {+ conn.execute(+ "INSERT OR IGNORE INTO memory_tombstones (scope, hash) VALUES (?1, ?2)",+ params![scope(bot), fingerprint(text)],+ )?;+ Ok(())+}+/// Forgets every entry in one layer (`None`: "yours"). Returns what was deleted.pub fn forget_layer(conn: &Connection, bot: Option<&str>) -> Result<Vec<Entry>> {let entries = layer(conn, bot)?;@@ -238,6 +264,19 @@ pub fn forget_layer(conn: &Connection, bot: Option<&str>) -> Result<Vec<Entry>>Ok(entries)}+/// Removes what was kept for a deleted bot: its own layer, its tombstones and its dream log. What it wrote to+/// "yours" stays, since the user may still want it.+pub fn drop_bot(conn: &Connection, bot: &str) -> Result<()> {+ conn.execute("DELETE FROM memory_entries WHERE bot = ?1", [bot])?;+ conn.execute("DELETE FROM memory_tombstones WHERE scope = ?1", [scope(Some(bot))])?;+ conn.execute(+ "DELETE FROM memory_dream_changes WHERE dream IN (SELECT id FROM memory_dreams WHERE bot = ?1)",+ [bot],+ )?;+ conn.execute("DELETE FROM memory_dreams WHERE bot = ?1", [bot])?;+ Ok(())+}+pub fn get(conn: &Connection, id: i64) -> Result<Option<Entry>> {Ok(conn.query_row(&format!("{SELECT} WHERE id = ?1"), [id], entry).optional()?)}@@ -297,6 +336,18 @@ fn set_state(get(conn, id)?.ok_or(Error::NotFound)}+fn duplicate(conn: &Connection, bot: Option<&str>, text: &str) -> Result<bool> {+ let mut stmt = conn.prepare("SELECT text FROM memory_entries WHERE bot IS ?1")?;+ let texts = stmt.query_map([bot], |row| row.get::<_, String>(0))?;+ let wanted = fingerprint(text);+ for existing in texts {+ if fingerprint(&existing?) == wanted {+ return Ok(true);+ }+ }+ Ok(false)+}+fn forgotten(conn: &Connection, bot: Option<&str>, text: &str) -> Result<bool> {Ok(conn.query_row(@@ -447,6 +498,16 @@ mod tests {assert!(search(&conn, "a", "100%", 10).unwrap().is_empty());}+ #[test]+ fn a_bot_cant_write_what_the_layer_already_holds() {+ let conn = db();+ add(&conn, Writer::User, new(Some("a"), "Likes tea"), 1, 1).unwrap();+ let again = add(&conn, Writer::Bot("a"), new(Some("a"), "likes TEA."), 2, 2);+ assert!(matches!(again, Err(Error::Refused(Refusal::Duplicate))));+ assert!(admissible(&conn, Writer::Bot("a"), Some("a"), "Likes coffee").is_ok());+ assert!(admissible(&conn, Writer::Bot("a"), Some("a"), "my password is 1234").is_err());+ }+#[test]fn forgetting_a_layer_leaves_the_others() {let conn = db();diff --git a/host/src/agent/bot/mod.rs b/host/src/agent/bot/mod.rsindex 02628c3c..62fd230e 100644--- a/host/src/agent/bot/mod.rs+++ b/host/src/agent/bot/mod.rs@@ -7,8 +7,8 @@//! "full conversation" sheet, and surface live as the roster activity line.//!//! Context (from Grok Bot too): the bot's instructions and memory are a frozen-//! snapshot per session and compaction epoch (`context`), finished exchanges feed-//! its memory keeper (`memory`), messages sent while it works are folded into+//! snapshot per session and compaction epoch (`context`), finished turns feed+//! its dreaming (`memory`), messages sent while it works are folded into//! its next turn, and a turn cut off by a host restart is resumed.//!//! Lanes: besides its own chat, a bot talks in threads (each a session of its own,@@ -113,7 +113,7 @@ pub fn spawn(hub: Arc<Hub>, cfg: BotConfig) -> BotHandle {(now_ms() - started < STALE_RESUME_MS).then(|| Lane { chat: cfg.id.clone(), thread: (!thread.is_empty()).then(|| thread.to_owned()) })});- let keeper = memory::spawn_keeper(hub.clone(), cfg.id.clone());+ let dreamer = memory::spawn_dreamer(hub.clone(), cfg.id.clone());let lane = Lane::main(&cfg.id);let fast_support = FastModeSupport::from_stored(hub.store.kv_get(&fast_mode::stored_key(&cfg.id)).as_deref());let support = fast_support.clone();@@ -134,11 +134,9 @@ pub fn spawn(hub: Arc<Hub>, cfg: BotConfig) -> BotHandle {routine_completion: None,session_fresh: false,applied_system: HashMap::new(),- keeper,+ dreamer,turn_text: None,turn_entries: Vec::new(),- memory_session: None,- memory_warned: false,turn_memory_revision: String::new(),announce: None,turn: None,@@ -227,14 +225,12 @@ struct Actor {session_fresh: bool,/// Session -> the instructions this process last handed Claude as its system prompt.applied_system: HashMap<String, String>,- keeper: mpsc::UnboundedSender<memory::KeeperEvent>,+ dreamer: mpsc::UnboundedSender<memory::DreamEvent>,/// The user's words for this turn (None for a hidden turn or one not from the user); feeds memory.turn_text: Option<String>,/// The user messages this turn answers (none when it isn't the user's).turn_entries: Vec<String>,- memory_session: Option<String>,- /// The user was told memory is unavailable.- memory_warned: bool,+ /// The memory change this turn's notice told the agent about (see `memory::change_notice`).turn_memory_revision: String,/// A profile update this turn carried, recorded once the agent got it.announce: Option<(Snapshot, Identity)>,diff --git a/host/src/agent/bot/queue.rs b/host/src/agent/bot/queue.rsindex 2778162c..675e6307 100644--- a/host/src/agent/bot/queue.rs+++ b/host/src/agent/bot/queue.rs@@ -71,7 +71,7 @@ impl Actor {}self.forget_session();// Retain and summarize completed exchanges even when the user starts afresh.- let _ = self.keeper.send(memory::KeeperEvent::Flush);+ let _ = self.dreamer.send(memory::DreamEvent::Flush);self.notice("New session — the agent starts with a fresh context.", NoticeStyle::Divider);}Cmd::Reconfigure(cfg) => {@@ -254,6 +254,7 @@ impl Actor {r.activity = "Couldn't start".into();r.started_at = None;r.lane = None;+ r.message = None;});self.hub.team.cancel_from(&self.cfg.id);}diff --git a/host/src/agent/bot/session.rs b/host/src/agent/bot/session.rsindex 007a8a7c..bd8a2ce4 100644--- a/host/src/agent/bot/session.rs+++ b/host/src/agent/bot/session.rs@@ -134,10 +134,6 @@ impl Actor {/// Makes sure the agent runs and `slot`'s session (main, or a thread's) is live;/// sets `turn_session`. A thread without one forks the main session when it can.pub(super) async fn ensure_session(&mut self, slot: Option<&str>) -> Result<()> {- // Memory is best-effort: a bot without it still answers.- if let Err(error) = crate::chat::memory::prepare(&self.cfg.id).await {- self.memory_unavailable(&error);- }if !self.cfg.connectors.is_empty()|| self.hub.store.kv_read(&format!("agent-env:{}", self.cfg.backend))?.is_some(){@@ -159,7 +155,11 @@ impl Actor {.await?;self.hub.set_runtime(&self.id(), |r| r.activity = "Starting agent…".into());// Keys saved from an "environment variable" sign-in.- let env = crate::agent::auth::env(&self.hub.store, &self.cfg.backend)?;+ let mut env = crate::agent::auth::env(&self.hub.store, &self.cfg.backend)?;+ if crate::agent::workspace::is_managed(&self.cfg) {+ // Codync's memory is the only one a bot in its own workspace has (see `claude_meta`).+ env.push(("CLAUDE_CODE_DISABLE_AUTO_MEMORY".into(), "1".into()));+ }let mut last_err = None;for (i, command) in candidates.iter().enumerate() {let fallback_left = i + 1 < candidates.len();@@ -190,7 +190,13 @@ impl Actor {{self.hub.set_runtime(&self.cfg.id, |r| r.activity = "Opening the thread…".into());let res = self- .session_request("session/fork", json!({"sessionId": main, "cwd": self.cfg.cwd, "mcpServers": servers}))+ .session_request("session/fork", {+ let mut params = json!({"sessionId": main, "cwd": self.cfg.cwd, "mcpServers": servers});+ if claude {+ params["_meta"] = json!({"claudeCode": claude_options(&self.cfg)});+ }+ params+ }).await.ok();if let Some(res) = res@@ -235,7 +241,7 @@ impl Actor {self.hub.set_runtime(&self.cfg.id, |r| r.activity = "Resuming session…".into());let mut params = json!({"sessionId": sid, "cwd": self.cfg.cwd, "mcpServers": servers});if let Some(s) = &snapshot {- params["_meta"] = system_meta(&s.system);+ params["_meta"] = claude_meta(&self.cfg, &s.system);}let res = self.session_request("session/load", params).await;let Some(conn) = self.conn.as_mut() else { return res.map(drop) };@@ -277,7 +283,7 @@ impl Actor {};let mut params = json!({"cwd": self.cfg.cwd, "mcpServers": servers});if claude {- params["_meta"] = system_meta(&system);+ params["_meta"] = claude_meta(&self.cfg, &system);}let res = self.session_request("session/new", params).await?;let Some(conn) = self.conn.as_mut() else { bail!("agent not running") };@@ -312,17 +318,6 @@ impl Actor {Err(acp::Failure::error(acp::FailureKind::Timeout, message))}- /// Memory failed to start: said once per actor, then the bot works without it.- pub(super) fn memory_unavailable(&mut self, error: &anyhow::Error) {- tracing::warn!(bot = %self.cfg.id, error = format!("{error:#}"), "memory unavailable");- if !std::mem::replace(&mut self.memory_warned, true) {- self.notice(- &format!("Memory is unavailable; the bot works without it for now. {error:#}"),- NoticeStyle::Info,- );- }- }-/// Drops the main ACP session so the next turn starts a fresh one.pub(super) fn forget_session(&mut self) {self.set_session(None, None);@@ -332,9 +327,20 @@ impl Actor {/// How long `session/new`, `session/load` and `session/fork` may take.const SESSION_TIMEOUT: Duration = Duration::from_secs(180);-/// Claude's `_meta`: the bot's instructions appended to Claude Code's own system prompt.-fn system_meta(system: &str) -> Value {- json!({"systemPrompt": {"append": system}})+/// Claude's `_meta`: the bot's instructions appended to Claude Code's own system prompt, and its options.+fn claude_meta(cfg: &BotConfig, system: &str) -> Value {+ json!({"systemPrompt": {"append": system}, "claudeCode": claude_options(cfg)})+}++/// Claude Code's own memory stays off where Codync's is the bot's only one (a bot in its own workspace); in a+/// project folder it is the project's working memory and stays on. Its auto-dream rewrites memory with no log, so+/// it is off everywhere ([design](../../../../docs/design/bot-memory.md#claude-codes-own-memory)).+fn claude_options(cfg: &BotConfig) -> Value {+ let mut settings = json!({"autoDreamEnabled": false});+ if crate::agent::workspace::is_managed(cfg) {+ settings["autoMemoryEnabled"] = false.into();+ }+ json!({"options": {"settings": settings}})}/// Shell commands that may start the bot's agent over ACP, best first.diff --git a/host/src/agent/bot/turn.rs b/host/src/agent/bot/turn.rsindex ff15eaa3..8a17cd14 100644--- a/host/src/agent/bot/turn.rs+++ b/host/src/agent/bot/turn.rs@@ -50,7 +50,7 @@ impl Actor {!hidden && self.active_request.is_none() && self.active_group.is_none() && self.active_routine.is_none();self.turn_text = from_user.then(|| text.to_owned());self.turn_entries = if from_user { entry_ids.to_vec() } else { Vec::new() };- self.turn_memory_revision = memory::maintenance::revision(&self.hub.store, &self.cfg.id);+ self.turn_memory_revision.clear();self.announce = None;self.stop_requested = false;self.cancel_deadline = None;@@ -73,6 +73,7 @@ impl Actor {r.activity = if hidden { "Picking up where it left off…" } else { "Starting…" }.into();r.started_at = Some(now_ms());r.lane = Some(lane.clone());+ r.message = if from_user { entry_ids.last().cloned() } else { None };});let slot = self.slot(&lane);@@ -103,6 +104,7 @@ impl Actor {r.activity = String::new();r.started_at = None;r.lane = None;+ r.message = None;});return Ok(());}@@ -120,24 +122,17 @@ impl Actor {}let claude = self.conn.as_ref().is_some_and(|c| c.claude);let sid = self.turn_session.clone().ok_or_else(|| anyhow!("no session"))?;- self.memory_session = match memory::lifecycle::begin(&self.hub, &self.cfg.id, &sid).await {- Ok(session) => Some(session),- Err(error) => {- self.memory_unavailable(&error);- None- }- };- if from_user- && let Some(session) = &self.memory_session- && let Err(error) = memory::lifecycle::capture_prompt(&self.cfg.id, session, text).await- {- tracing::warn!(bot = %self.cfg.id, %error, "memory prompt capture failed");- }let snapshot = self.snapshot(&sid).await?;let remind_files = !snapshot.system.contains("send_file");let mut prompt = text.to_owned();- if let Some(notice) = memory::lifecycle::change_notice(&self.hub, &self.cfg.id, &sid) {+ if self.session_fresh {+ // A new session's instructions already hold memory as it is now.+ let revision = memory::current_revision(&self.hub.store, &self.cfg.id);+ memory::mark_announced(&self.hub.store, &self.cfg.id, &sid, &revision)?;+ }+ if let Some((notice, revision)) = memory::change_notice(&self.hub.store, &self.cfg, &sid) {prompt = format!("{notice}\n\n{prompt}");+ self.turn_memory_revision = revision;}if let Some(intro) = self.thread_intro.take() {prompt = format!("{intro}\n\n{prompt}");@@ -169,7 +164,8 @@ impl Actor {}// Written before this returns, so a Stop right after is sent after the prompt.let answer = acp.send("session/prompt", params).await?;- if let Err(error) = memory::lifecycle::mark_announced(&self.hub, &self.cfg.id, &sid, &self.turn_memory_revision)+ if !self.turn_memory_revision.is_empty()+ && let Err(error) = memory::mark_announced(&self.hub.store, &self.cfg.id, &sid, &self.turn_memory_revision){tracing::warn!(%error, "could not acknowledge memory update notice");}@@ -306,22 +302,16 @@ impl Actor {{tracing::warn!(bot = %self.cfg.id, error = format!("{error:#}"), "couldn't record the profile update");}- if let (Some(user), Some(agent), "end_turn") = (self.turn_text.take(), &final_text, stop_reason.as_str())- && memory::is_memorable(&user)- {- let _ = self.keeper.send(memory::KeeperEvent::Exchange(memory::Exchange {- user,- agent: agent.clone(),- at: now_ms(),- session: self.memory_session.clone().unwrap_or_default(),- revision: self.turn_memory_revision.clone(),- }));+ if let (Some(user), Some(agent), "end_turn") = (self.turn_text.take(), &final_text, stop_reason.as_str()) {+ let turn = crate::chat::naming::Turn { user, agent: agent.clone() };+ let _ = self.dreamer.send(memory::DreamEvent::Turn(turn));}self.hub.set_runtime(&self.id(), |r| {r.status = if failed { BotStatus::Error } else { BotStatus::Idle };r.activity = String::new();r.started_at = None;r.lane = None;+ r.message = None;});self.hub.team.cancel_from(&self.cfg.id);// A room turn's news comes from the group once the room is done (see `group`).diff --git a/host/src/agent/bot/updates.rs b/host/src/agent/bot/updates.rsindex 94fb7bf5..36b7c562 100644--- a/host/src/agent/bot/updates.rs+++ b/host/src/agent/bot/updates.rs@@ -107,7 +107,7 @@ impl Actor {let id = u["compactionId"].as_str().unwrap_or_default();match context::bump_epoch(&self.hub.store, &self.cfg.id, sid, id) {Ok(true) => {- let _ = self.keeper.send(crate::chat::memory::KeeperEvent::Flush);+ let _ = self.dreamer.send(crate::chat::memory::DreamEvent::Flush);self.notice("Earlier context was summarized to make room.", NoticeStyle::Info);}Ok(false) => {}diff --git a/host/src/api/bots.rs b/host/src/api/bots.rsindex d2658f1e..669ad478 100644--- a/host/src/api/bots.rs+++ b/host/src/api/bots.rs@@ -39,17 +39,8 @@ pub(super) async fn call(hub: &Arc<Hub>, method: &str, b: Value) -> Result<Contr)?,"routineCall" => crate::routines::call(hub, str_arg(&b, "botId")?, str_arg(&b, "name")?, &b["arguments"])?,"memoryCall" => {- let bot = str_arg(&b, "botId")?;- let name = str_arg(&b, "name")?;- let args = if name.starts_with("mem_") {- crate::chat::memory::lifecycle::bound_args(hub, bot, b["memoryLane"].as_str(), name, &b["arguments"])- .await?- } else {- b["arguments"].clone()- };- crate::chat::memory::call(hub, bot, name, &args).await?+ crate::chat::memory::tool_call(hub, str_arg(&b, "botId")?, str_arg(&b, "name")?, &b["arguments"]).await?}- "memoryTools" => crate::chat::memory::available_tools(hub, str_arg(&b, "botId")?).await?,"chatCall" => {crate::chat::outbox::call(hub, str_arg(&b, "botId")?, str_arg(&b, "name")?, &b["arguments"]).await?}@@ -184,13 +175,21 @@ pub(super) async fn call(hub: &Arc<Hub>, method: &str, b: Value) -> Result<Contrlet turn = hub.store.max_turn(bot) + 1;// A group has no agent of its own to queue behind: its message is simply sent.let status = if row.config.is_group() { "sent" } else { "queued" };+ let mut data = json!({"text": text, "clientNonce": nonce, "status": status, "attachments": attachments});+ // What the user pasted rather than typed: memory never takes it as the user's own words.+ let pasted: Vec<&str> = b["pasted"]+ .as_array()+ .into_iter()+ .flatten()+ .filter_map(Value::as_str)+ .map(str::trim)+ .filter(|p| !p.is_empty() && text.contains(p))+ .collect();+ if !pasted.is_empty() {+ data["pasted"] = json!(pasted);+ }let e = hub- .add_entry(- &lane,- EntryKind::User,- turn,- &json!({"text": text, "clientNonce": nonce, "status": status, "attachments": attachments}),- )+ .add_entry(&lane, EntryKind::User, turn, &data).ok_or_else(|| anyhow!("couldn't save the message"))?;// Before the turn starts, so its reply lands unread.let scope = lane.thread.as_deref().map_or(ReadScope::Chat, ReadScope::Thread);@@ -225,8 +224,8 @@ pub(super) async fn call(hub: &Arc<Hub>, method: &str, b: Value) -> Result<Contrhub.send_cmd(str_arg(&b, "botId")?, Cmd::NewSession)?;json!({})}- "memory" | "saveMemory" | "memoryDetail" | "pinMemory" | "reviewMemory" | "forgetMemory" | "clearMemory"- | "exportMemory" | "importMemory" => crate::chat::memory::manage::call(hub, method, &b).await?,+ "memory" | "saveMemory" | "pinMemory" | "archiveMemory" | "forgetMemory" | "clearMemory" | "undoDream"+ | "answerMemory" | "undoMemory" => crate::chat::memory::call(hub, method, &b).await?,"react" => {let e = hub.react(str_arg(&b, "entryId")?, str_arg(&b, "emoji")?)?;json!({"entry": e})diff --git a/host/src/api/devices.rs b/host/src/api/devices.rsindex 76ebfdc9..9edad2f6 100644--- a/host/src/api/devices.rs+++ b/host/src/api/devices.rs@@ -48,7 +48,6 @@ const LOCAL_ONLY: &[&str] = &["chatCall","teamCall","memoryCall",- "memoryTools","routineCall","composioCall","connectorTarget",diff --git a/host/src/chat/context.rs b/host/src/chat/context.rsindex 829303e9..7c03c8b8 100644--- a/host/src/chat/context.rs+++ b/host/src/chat/context.rs@@ -9,9 +9,9 @@//! agent compacts its context (a `compaction_update`) or a new session starts.//! Profile edits in between are announced once as an `<agent_profile_update>`//! block on the next message and folded into the snapshot after the next-//! compaction. Memory learned in between reaches the prompt the same way.+//! compaction. Memory learned in between reaches the prompt the same way; a change+//! the user makes is also told on the next message (see `memory::change_notice`).-use crate::chat::memory::{self, Memory};use crate::store::{BotConfig, Store};use anyhow::Result;use serde::{Deserialize, Serialize};@@ -92,7 +92,7 @@ fn save(store: &Store, bot_id: &str, s: &Snapshot) -> Result<()> {}/// The snapshot for `session` at its current epoch, rendering a fresh one when-/// the stored one belongs to another session or an earlier epoch. Reads memory files.+/// the stored one belongs to another session or an earlier epoch.pub fn resolve(store: &Store, cfg: &BotConfig, session: &str) -> Result<Snapshot> {let epoch = epoch(store, &cfg.id, session);let stored = load(store, &cfg.id, session);@@ -141,7 +141,7 @@ pub fn adopt(store: &Store, cfg: &BotConfig, session: &str, system: String) -> ROk(s)}-/// The bot's instructions, rendered live. Reads memory files.+/// The bot's instructions, rendered live.pub fn render(store: &Store, cfg: &BotConfig) -> String {let name = cfg.name.trim();let mut lines = vec!["Agent profile:".to_owned()];@@ -173,13 +173,10 @@ pub fn render(store: &Store, cfg: &BotConfig) -> String {lines.push(skills);}let mut out = lines.join("\n");- match Memory::for_bot(&cfg.id).and_then(|mem| Ok((mem.recall(memory::RECENT_PROMPT_LIMIT)?, mem))) {- Ok((recall, mem)) => {- let (section, _) = memory::render(&recall, mem.location());- out.push_str("\n\n");- out.push_str(§ion);- }- Err(error) => tracing::warn!(bot = %cfg.id, error = format!("{error:#}"), "no memory folder"),+ let memory = crate::chat::memory::opening(store, cfg);+ if !memory.is_empty() {+ out.push_str("\n\n");+ out.push_str(&memory);}out}diff --git a/host/src/chat/memory/api.rs b/host/src/chat/memory/api.rsnew file mode 100644index 00000000..ec7e7cf6--- /dev/null+++ b/host/src/chat/memory/api.rs@@ -0,0 +1,154 @@+//! The apps' memory methods: the two layers, the user's own edits, the dream log, and the chat cards.++use super::{Change, notices};+use crate::hub::Hub;+use crate::store::BotConfig;+use anyhow::{Result, anyhow, bail};+use codync_mem::store::{self as mem, Entry, Error, New, State, Writer};+use serde_json::{Value, json};+use std::sync::Arc;++/// Dream runs the memory screen lists.+const DREAMS_SHOWN: usize = 20;+/// The most of a project's `MEMORY.md` the screen shows.+const PROJECT_MEMORY_BYTES: u64 = 64 * 1024;++pub async fn call(hub: &Arc<Hub>, method: &str, b: &Value) -> Result<Value> {+ let hub = hub.clone();+ let method = method.to_owned();+ let b = b.clone();+ tokio::task::spawn_blocking(move || dispatch(&hub, &method, &b)).await?+}++fn dispatch(hub: &Hub, method: &str, b: &Value) -> Result<Value> {+ match method {+ "answerMemory" => return notices::answer(hub, str_arg(b, "entryId")?, b["accept"] == true),+ "undoMemory" => return notices::undo(hub, str_arg(b, "entryId")?),+ _ => {}+ }+ let cfg = bot(hub, str_arg(b, "botId")?)?;+ let id = || b["id"].as_i64().ok_or_else(|| anyhow!("id required"));+ match method {+ "memory" => view(hub, &cfg),+ "saveMemory" => {+ let text = str_arg(b, "text")?;+ let entry = if let Some(id) = b["id"].as_i64() {+ edit(hub, &cfg, id, |c, now, rev| mem::revise(c, Writer::User, id, text, now, rev))?+ } else {+ let layer = (b["shared"] != true).then_some(cfg.id.as_str());+ let new = New { bot: layer, text, room: None, evidence: &[] };+ let entry = hub.store.memory_write(|c, now, rev| mem::add(c, Writer::User, new, now, rev))?;+ super::changed(hub, entry.bot.as_deref(), Change::User);+ entry+ };+ Ok(json!({"entry": entry}))+ }+ "pinMemory" => {+ let (id, pinned) = (id()?, b["pinned"] == true);+ edit(hub, &cfg, id, |c, now, rev| mem::pin(c, id, pinned, now, rev))?;+ Ok(json!({}))+ }+ "archiveMemory" => {+ let id = id()?;+ if b["archived"] == false {+ edit(hub, &cfg, id, |c, now, rev| mem::activate(c, id, now, rev))?;+ } else {+ edit(hub, &cfg, id, |c, now, rev| mem::archive(c, Writer::User, id, None, now, rev))?;+ }+ Ok(json!({}))+ }+ "forgetMemory" => {+ let id = id()?;+ edit(hub, &cfg, id, |c, _, _| mem::forget(c, id))?;+ Ok(json!({}))+ }+ "clearMemory" => {+ let layer = (b["shared"] != true).then_some(cfg.id.as_str());+ hub.store.memory_write(|c, _, _| mem::forget_layer(c, layer))?;+ super::changed(hub, layer, Change::User);+ Ok(json!({}))+ }+ "undoDream" => {+ let id = id()?;+ let dream = hub.store.memory_read(|c| codync_mem::dream::get(c, id))?;+ if dream.is_none_or(|d| d.bot != cfg.id) {+ bail!("unknown dream");+ }+ hub.store.memory_write(|c, now, rev| codync_mem::dream::undo(c, id, now, rev))?;+ // A run may have written to either layer.+ super::changed(hub, None, Change::User);+ Ok(json!({}))+ }+ _ => bail!("unknown method: {method}"),+ }+}++/// Changes one entry the bot can see, then tells the bots that see it.+fn edit(+ hub: &Hub,+ cfg: &BotConfig,+ id: i64,+ change: impl FnOnce(&rusqlite::Connection, i64, i64) -> mem::Result<Entry>,+) -> Result<Entry> {+ let bot = cfg.id.clone();+ let entry = hub.store.memory_write(|c, now, rev| {+ let entry = mem::get(c, id)?.ok_or(Error::NotFound)?;+ if entry.bot.as_deref().is_some_and(|owner| owner != bot) {+ return Err(Error::NotFound);+ }+ change(c, now, rev)+ })?;+ super::changed(hub, entry.bot.as_deref(), Change::User);+ Ok(entry)+}++fn view(hub: &Hub, cfg: &BotConfig) -> Result<Value> {+ let bot = cfg.id.as_str();+ let (yours, own, dreams) = hub.store.memory_read(|c| {+ Ok((mem::layer(c, None)?, mem::layer(c, Some(bot))?, codync_mem::dream::recent(c, bot, DREAMS_SHOWN)?))+ })?;+ let layer = |entries: Vec<Entry>, layer: Option<&str>| {+ let used: usize =+ entries.iter().filter(|e| e.state == State::Active).map(|e| codync_mem::tokens::estimate(&e.text)).sum();+ json!({"entries": entries, "used": used, "cap": mem::cap(layer)})+ };+ Ok(json!({+ "shared": super::shared(cfg),+ "yours": layer(yours, None),+ "bot": layer(own, Some(bot)),+ "dreams": dreams,+ "projectMemory": project_memory(cfg),+ "memoryRev": hub.store.memory_rev(),+ }))+}++/// Claude Code's own memory of a project bot's folder (`~/.claude/projects/<folder>/memory/MEMORY.md`), shown+/// read-only so nothing a bot remembers is out of the user's sight.+fn project_memory(cfg: &BotConfig) -> Value {+ if crate::agent::workspace::is_managed(cfg) || cfg.backend != "claude" {+ return Value::Null;+ }+ let folder: String = cfg.cwd.chars().map(|c| if c.is_ascii_alphanumeric() { c } else { '-' }).collect();+ let Some(path) = dirs::home_dir().map(|home| home.join(".claude/projects").join(folder).join("memory/MEMORY.md"))+ else {+ return Value::Null;+ };+ let read = std::fs::File::open(&path).and_then(|file| {+ let mut bytes = Vec::new();+ std::io::Read::read_to_end(&mut std::io::Read::take(file, PROJECT_MEMORY_BYTES), &mut bytes)?;+ Ok(String::from_utf8_lossy(&bytes).into_owned())+ });+ read.map_or(Value::Null, |text| json!({"path": path.to_string_lossy(), "text": text}))+}++fn bot(hub: &Hub, id: &str) -> Result<BotConfig> {+ let cfg = hub.store.bot(id)?.filter(|b| !b.deleted).ok_or_else(|| anyhow!("unknown bot"))?.config;+ if cfg.is_group() {+ bail!("a group has no memory of its own");+ }+ Ok(cfg)+}++fn str_arg<'a>(b: &'a Value, k: &str) -> Result<&'a str> {+ b[k].as_str().ok_or_else(|| anyhow!("{k} required"))+}diff --git a/host/src/chat/memory/dates.rs b/host/src/chat/memory/dates.rsindex 1c0af333..5fa69187 100644--- a/host/src/chat/memory/dates.rs+++ b/host/src/chat/memory/dates.rs@@ -7,8 +7,3 @@ pub fn ymd(ms: i64) -> String {chrono::DateTime::from_timestamp_millis(ms).map_or_else(|| "unknown date".into(), |date| date.format("%Y-%m-%d").to_string())}--#[cfg(test)]-pub(super) fn parse_ymd(raw: &str) -> Option<i64> {- Some(chrono::NaiveDate::parse_from_str(raw, "%Y-%m-%d").ok()?.and_hms_opt(0, 0, 0)?.and_utc().timestamp_millis())-}diff --git a/host/src/chat/memory/dream.rs b/host/src/chat/memory/dream.rsnew file mode 100644index 00000000..cf9b86bf--- /dev/null+++ b/host/src/chat/memory/dream.rs@@ -0,0 +1,230 @@+//! Dreaming in the host: when a bot's run starts, what it reads, and running its model on the bot's own harness.+//! What a run may change is the crate's decision (`codync_mem::dream`).++use super::Change;+use crate::LockExt;+use crate::chat::{naming, oneshot};+use crate::hub::Hub;+use crate::store::{BotConfig, Entry, EntryKind, Store};+use anyhow::{Result, anyhow};+use codync_mem::dream::{self, Message, Outcome};+use std::sync::Arc;+use std::time::Duration;+use tokio::sync::mpsc;+use tokio::time::Instant;++/// Quiet time after the last turn before a run.+const IDLE: Duration = Duration::from_secs(5 * 60);+/// User messages that start a run without waiting, and the most one run reads.+const BATCH: usize = 8;+/// Entries (the user's messages and final replies) one run looks at.+const READ_LIMIT: i64 = 40;+const MESSAGE_CHARS: usize = 4_000;+/// Usage of the bot's plan above which runs wait, so dreaming never spends the user's last headroom.+const HEADROOM_PERCENT: f64 = 75.0;+/// How often an idle bot checks whether its daily time review is due.+const REVIEW_CHECK: Duration = Duration::from_secs(60 * 60);++pub enum DreamEvent {+ /// A turn the user started has finished.+ Turn(naming::Turn),+ /// Run soon: a new session started or the context was compacted.+ Flush,+}++enum Ran {+ /// Read up to here; more may wait.+ Done { more: bool },+ /// Try again later without counting a failure: usage is high, or memory changed during the run.+ Later,+}++fn bookmark_key(bot: &str) -> String {+ format!("memory.dream.{bot}")+}++fn reviewed_key(bot: &str) -> String {+ format!("memory.dream.reviewed.{bot}")+}++/// Dreaming starts reading after `seq`.+pub(super) fn set_bookmark(store: &Store, bot: &str, seq: i64) -> Result<()> {+ store.kv_set(&bookmark_key(bot), &seq.to_string())+}++fn bookmark(store: &Store, bot: &str) -> i64 {+ store.kv_get(&bookmark_key(bot)).and_then(|v| v.parse().ok()).unwrap_or(0)+}++/// Starts the bot's dreamer. It runs once the bot has been quiet for [`IDLE`] after the user's turns, or right+/// away when [`BATCH`] of them are waiting, and once a day for the time review. It also names an unnamed bot. It+/// stops when the bot actor drops the sender; what it hasn't read waits for the next one.+pub fn spawn_dreamer(hub: Arc<Hub>, bot_id: String) -> mpsc::UnboundedSender<DreamEvent> {+ let (tx, mut rx) = mpsc::unbounded_channel::<DreamEvent>();+ tokio::spawn(async move {+ let mut waiting = waiting(&hub.store, &bot_id);+ let idle = tokio::time::sleep(IDLE);+ tokio::pin!(idle);+ let mut failures = 0u32;+ let mut retry_at: Option<Instant> = None;+ let quiet_until = |retry_at: Option<Instant>| {+ let after = Instant::now() + IDLE;+ retry_at.map_or(after, |at| at.max(after))+ };+ loop {+ let due = waiting > 0 || review_due(&hub.store, &bot_id);+ let run = tokio::select! {+ event = rx.recv() => match event {+ None => break,+ Some(DreamEvent::Turn(turn)) => {+ if let Err(e) = naming::observe(&hub, &bot_id, turn).await {+ tracing::warn!(bot = %bot_id, error = format!("{e:#}"), "naming the bot failed");+ }+ waiting += 1;+ idle.as_mut().reset(quiet_until(retry_at));+ waiting >= BATCH && retry_at.is_none()+ }+ Some(DreamEvent::Flush) => {+ idle.as_mut().reset(retry_at.unwrap_or_else(Instant::now));+ false+ }+ },+ () = &mut idle, if due => true,+ () = tokio::time::sleep(REVIEW_CHECK), if !due => false,+ };+ if !run {+ continue;+ }+ let next = match self::run(&hub, &bot_id).await {+ Ok(Ran::Done { more }) => {+ failures = 0;+ retry_at = None;+ waiting = if more { waiting.max(1) } else { self::waiting(&hub.store, &bot_id) };+ if more { Instant::now() } else { quiet_until(None) }+ }+ Ok(Ran::Later) => quiet_until(retry_at),+ Err(e) => {+ failures += 1;+ retry_at = Some(Instant::now() + IDLE * 2u32.pow(failures.min(5)));+ tracing::warn!(bot = %bot_id, failures, error = format!("{e:#}"), "dreaming failed; it retries later");+ quiet_until(retry_at)+ }+ };+ idle.as_mut().reset(next);+ }+ });+ tx+}++/// The user's messages since the bookmark.+fn waiting(store: &Store, bot: &str) -> usize {+ store+ .conversation_since(bot, bookmark(store, bot), READ_LIMIT)+ .map_or(0, |entries| entries.iter().filter(|e| e.kind == EntryKind::User.as_str()).count())+}++/// Once a day, when the bot's memory holds a dated entry it may rewrite (the time review).+fn review_due(store: &Store, bot: &str) -> bool {+ if store.kv_get(&reviewed_key(bot)).as_deref() == Some(today().as_str()) {+ return false;+ }+ store.memory_read(|c| codync_mem::store::layer(c, Some(bot))).is_ok_and(|entries| {+ entries.iter().any(|e| {+ e.state == codync_mem::store::State::Active+ && e.author != codync_mem::store::Author::User+ && e.text.chars().any(|c| c.is_ascii_digit())+ })+ })+}++fn today() -> String {+ chrono::Local::now().format("%Y-%m-%d").to_string()+}++async fn run(hub: &Arc<Hub>, bot_id: &str) -> Result<Ran> {+ let Some(cfg) = hub.store.bot(bot_id)?.filter(|b| !b.deleted).map(|b| b.config) else {+ return Ok(Ran::Done { more: false });+ };+ if near_limit(hub, &cfg) {+ return Ok(Ran::Later);+ }+ let after = bookmark(&hub.store, bot_id);+ let entries = hub.store.conversation_since(bot_id, after, READ_LIMIT)?;+ let read = within_batch(&entries);+ let more = read.len() < entries.len() || i64::try_from(entries.len()).is_ok_and(|n| n == READ_LIMIT);+ let last_seq = read.last().map_or(after, |e| e.seq);+ let messages: Vec<Message> = read.iter().filter_map(message).collect();+ let today = today();+ let review = review_due(&hub.store, bot_id);+ let worth = messages.iter().any(|m| m.from_user && m.text.iter().any(|run| codync_mem::evidence::substantive(run)));+ if !worth && !review {+ set_bookmark(&hub.store, bot_id, last_seq)?;+ return Ok(Ran::Done { more });+ }++ let shared = super::shared(&cfg);+ let snapshot = hub.store.memory_read(|c| dream::snapshot(c, bot_id, shared, &today))?;+ let (system, material) = dream::prompt(&snapshot, &messages);+ let raw = oneshot::one_shot(hub, &cfg, &system, &material).await?;+ let plan = dream::parse(&raw).ok_or_else(|| anyhow!("dreaming answered without a plan"))?;+ let (kept, rejected) = dream::audit(&snapshot, &messages, plan);+ if !rejected.is_empty() {+ tracing::info!(bot = %bot_id, dropped = rejected.len(), reasons = ?rejected, "dreaming dropped changes");+ }+ let outcome = hub.store.memory_write(|c, now, rev| dream::apply(c, &snapshot, &kept, &messages, now, rev))?;+ match outcome {+ Outcome::Stale => return Ok(Ran::Later),+ Outcome::Changed(dream) => {+ tracing::info!(bot = %bot_id, added = dream.added.len(), archived = dream.archived.len(), "memory dreamed");+ let layer = if kept.add.iter().any(|a| a.shared) { None } else { Some(bot_id) };+ super::changed(hub, layer, Change::Background);+ }+ Outcome::Unchanged => {}+ }+ set_bookmark(&hub.store, bot_id, last_seq)?;+ if review {+ hub.store.kv_set(&reviewed_key(bot_id), &today)?;+ }+ Ok(Ran::Done { more })+}++/// Usage of the bot's harness above [`HEADROOM_PERCENT`] in any window.+fn near_limit(hub: &Hub, cfg: &BotConfig) -> bool {+ hub.usage+ .locked()+ .providers+ .iter()+ .filter(|p| p.id == cfg.backend)+ .flat_map(|p| &p.windows)+ .any(|w| w.percent > HEADROOM_PERCENT)+}++/// The entries up to the [`BATCH`]th user message and the replies after it.+fn within_batch(entries: &[Entry]) -> &[Entry] {+ let mut users = 0;+ for (i, e) in entries.iter().enumerate() {+ if e.kind == EntryKind::User.as_str() {+ if users == BATCH {+ return &entries[..i];+ }+ users += 1;+ }+ }+ entries+}++/// What a run sees of an entry. A user message counts only for what the user typed themselves; one the host wrote+/// in the user's place (`from: host`) isn't the user's words.+fn message(e: &Entry) -> Option<Message> {+ let clip = |text: &str| crate::agent::acp::truncate(text, MESSAGE_CHARS);+ let (from_user, text) = if e.kind == EntryKind::User.as_str() {+ if e.data["from"] == "host" {+ return None;+ }+ (true, super::notices::typed(&e.data).iter().map(|run| clip(run)).collect::<Vec<_>>())+ } else {+ (false, vec![clip(e.data["text"].as_str().unwrap_or_default())])+ };+ let text: Vec<String> = text.into_iter().filter(|run| !run.trim().is_empty()).collect();+ (!text.is_empty()).then(|| Message { id: e.id.clone(), from_user, text, date: super::dates::ymd(e.created_at) })+}diff --git a/host/src/chat/memory/engram/backup.rs b/host/src/chat/memory/engram/backup.rsdeleted file mode 100644index 01e8a7bc..00000000--- a/host/src/chat/memory/engram/backup.rs+++ /dev/null@@ -1,104 +0,0 @@-//! Native JSON backup/restore; no application-written SQLite rows.--use super::{directory, install};-use anyhow::{Context, Result, ensure};-use serde_json::Value;-use std::{path::Path, time::Duration};--pub(in crate::chat::memory) async fn export(bot: &str) -> Result<Value> {- super::prepare(bot).await?;- let dir = directory(bot)?;- let file = dir.join(format!("export-{}.json", uuid::Uuid::new_v4()));- let result = async {- command(bot, "export", &file).await?;- Ok(serde_json::from_slice(&tokio::fs::read(&file).await?)?)- }- .await;- cleanup(&file).await;- result-}--pub(in crate::chat::memory) async fn import(bot: &str, data: &Value) -> Result<Value> {- ensure!(data["sessions"].is_array() && data["observations"].is_array(), "invalid Engram backup");- let data = scoped_backup(bot, data)?;- let bytes = serde_json::to_vec(&data)?;- ensure!(bytes.len() <= 100 * 1024 * 1024, "memory backup exceeds 100 MB");- let before = export(bot).await?;- let dir = directory(bot)?;- let backup = dir.join(format!("before-import-{}.json", uuid::Uuid::new_v4()));- tokio::fs::write(&backup, serde_json::to_vec(&before)?).await?;- let file = dir.join(format!("import-{}.json", uuid::Uuid::new_v4()));- tokio::fs::write(&file, bytes).await?;- let result = command(bot, "import", &file).await;- cleanup(&file).await;- result?;- Ok(serde_json::json!({"backup":backup}))-}--fn scoped_backup(bot: &str, data: &Value) -> Result<Value> {- let project = super::project(bot);- let mut data = data.clone();- let mut sessions = std::collections::HashMap::new();- for session in data["sessions"].as_array_mut().context("backup has no sessions")? {- let old = session["id"].as_str().context("backup session has no id")?.to_owned();- let new = if session["project"] == project {- old.clone()- } else {- format!("{project}:import:{}", crate::chat::memory::read::content_hash(&old))- };- if new != old {- session["directory"] = "".into();- }- session["id"] = new.clone().into();- session["project"] = project.clone().into();- sessions.insert(old, new);- }- for collection in ["observations", "prompts", "prompt_tombstones"] {- for record in data[collection].as_array_mut().into_iter().flatten() {- if let Some(original) = record["project"].as_str().filter(|p| *p != project)- && collection == "observations"- {- record["tool_name"] =- format!("Imported from {original}; {}", record["tool_name"].as_str().unwrap_or_default()).into();- }- record["project"] = project.clone().into();- let source = record["session_id"].as_str().context("backup record has no session")?;- record["session_id"] = sessions.get(source).context("backup references an unknown session")?.clone().into();- }- }- for relation in data["relations"].as_array_mut().into_iter().flatten() {- if let Some(source) = relation["session_id"].as_str() {- relation["session_id"] =- sessions.get(source).context("relation references an unknown session")?.clone().into();- }- }- Ok(data)-}--async fn command(bot: &str, action: &str, file: &Path) -> Result<()> {- let mut command = tokio::process::Command::new(install::binary().await?);- command.arg(action).arg(file);- if action == "export" {- command.arg("--all");- }- let output = tokio::time::timeout(- Duration::from_secs(120),- command- .env("ENGRAM_DATA_DIR", directory(bot)?)- .env("ENGRAM_CLOUD_AUTOSYNC", "0")- .current_dir(directory(bot)?)- .kill_on_drop(true)- .output(),- )- .await??;- ensure!(output.status.success(), "Engram {action} failed: {}", String::from_utf8_lossy(&output.stderr));- Ok(())-}--async fn cleanup(file: &Path) {- if let Err(error) = tokio::fs::remove_file(file).await- && error.kind() != std::io::ErrorKind::NotFound- {- tracing::warn!(%error, path = %file.display(), "removing temporary memory backup failed");- }-}diff --git a/host/src/chat/memory/engram/clear.rs b/host/src/chat/memory/engram/clear.rsdeleted file mode 100644index b1f98ea4..00000000--- a/host/src/chat/memory/engram/clear.rs+++ /dev/null@@ -1,39 +0,0 @@-//! Clear observations, captured prompts and session summaries through the native CLI.--use anyhow::{Result, ensure};-use rusqlite::{Connection, OpenFlags};-use std::time::Duration;--pub(super) async fn run(bot: &str) -> Result<()> {- let dir = super::directory(bot)?;- let path = dir.join("engram.db");- let projects = tokio::task::spawn_blocking(move || -> Result<Vec<String>> {- let db = Connection::open_with_flags(path, OpenFlags::SQLITE_OPEN_READ_ONLY)?;- let mut query = db.prepare("SELECT project FROM sessions UNION SELECT project FROM observations")?;- Ok(query- .query_map([], |row| row.get::<_, Option<String>>(0))?- .collect::<rusqlite::Result<Vec<_>>>()?- .into_iter()- .flatten()- .filter(|s| !s.is_empty())- .collect())- })- .await??;- let binary = super::install::binary().await?;- for project in projects {- let result = tokio::time::timeout(- Duration::from_secs(120),- tokio::process::Command::new(&binary)- .args(["delete", "project", &project, "--hard"])- .env("ENGRAM_DATA_DIR", &dir)- .env("ENGRAM_CLOUD_AUTOSYNC", "0")- .current_dir(&dir)- .kill_on_drop(true)- .output(),- )- .await??;- ensure!(result.status.success(), "clearing Engram memory failed: {}", String::from_utf8_lossy(&result.stderr));- }- // The migration journal and completion marker remain: old Markdown must not be imported again.- Ok(())-}diff --git a/host/src/chat/memory/engram/install.rs b/host/src/chat/memory/engram/install.rsdeleted file mode 100644index 22941a31..00000000--- a/host/src/chat/memory/engram/install.rs+++ /dev/null@@ -1,134 +0,0 @@-//! Install one checksum-pinned binary, shared by every bot on this host.--use super::VERSION;-use crate::LockExt;-use anyhow::{Context, Result, bail, ensure};-use sha2::{Digest, Sha256};-use std::{- fmt::Write as _,- path::PathBuf,- sync::Mutex,- time::{Duration, Instant},-};--const RELEASES: &str = "https://github.com/Gentleman-Programming/engram/releases/download";-const MAX_ARCHIVE_BYTES: usize = 128 * 1024 * 1024;-static INSTALL: tokio::sync::Mutex<()> = tokio::sync::Mutex::const_new(());-/// When the last download failed, and how long until another is tried.-static FAILED_AT: Mutex<Option<Instant>> = Mutex::new(None);-const RETRY_AFTER: Duration = Duration::from_secs(5 * 60);--pub(super) async fn binary() -> Result<PathBuf> {- #[cfg(test)]- if let Some(binary) = std::env::var_os("CODYNC_TEST_ENGRAM") {- return Ok(binary.into());- }- let _guard = INSTALL.lock().await;- let root = crate::service::data_dir().join("engram").join(VERSION);- let name = if cfg!(windows) { "engram.exe" } else { "engram" };- let binary = root.join(name);- if tokio::fs::try_exists(root.join(".installed")).await? && tokio::fs::try_exists(&binary).await? {- return Ok(binary);- }- // Every turn asks for memory: after a failed download (offline), don't make each one wait on another.- if FAILED_AT.locked().is_some_and(|at| at.elapsed() < RETRY_AFTER) {- bail!("Engram couldn't be downloaded; trying again in a few minutes");- }- let installed = install(root, name).await;- if installed.is_err() {- *FAILED_AT.locked() = Some(Instant::now());- }- installed.map(|()| binary)-}--async fn install(root: PathBuf, name: &str) -> Result<()> {- let (platform, checksum) = distribution(std::env::consts::OS, std::env::consts::ARCH)?;- let extension = if cfg!(windows) { "zip" } else { "tar.gz" };- let archive = format!("engram_{VERSION}_{platform}.{extension}");- let mut response = crate::http()- .get(format!("{RELEASES}/v{VERSION}/{archive}"))- .timeout(Duration::from_secs(180))- .send()- .await?- .error_for_status()?;- let mut bytes = Vec::new();- while let Some(chunk) = response.chunk().await? {- ensure!(bytes.len() + chunk.len() <= MAX_ARCHIVE_BYTES, "Engram download exceeds size limit");- bytes.extend_from_slice(&chunk);- }- verify(&bytes, &checksum)?;- let staging = root.with_extension(format!("staging-{}", uuid::Uuid::new_v4()));- let (target, stage, executable) = (root, staging.clone(), name.to_owned());- let result = tokio::task::spawn_blocking(move || -> Result<()> {- std::fs::create_dir_all(&stage)?;- let file = stage.join(&archive);- std::fs::write(&file, bytes)?;- crate::agent::registry::extract(&file, &stage, std::path::Path::new(&executable))?;- let output = std::process::Command::new(stage.join(&executable)).arg("version").output()?;- ensure!(output.status.success(), "Engram version check failed");- let version = String::from_utf8(output.stdout)?;- ensure!(- version.split_whitespace().any(|v| v.trim_start_matches('v') == VERSION),- "unexpected Engram version: {version}"- );- std::fs::write(stage.join(".installed"), VERSION)?;- // A previous incomplete installation has never been used by a process.- if target.exists() {- std::fs::remove_dir_all(&target)?;- }- std::fs::rename(&stage, &target)?;- Ok(())- })- .await?;- if result.is_err()- && let Err(error) = tokio::fs::remove_dir_all(&staging).await- {- tracing::warn!(%error, "could not remove incomplete Engram installation");- }- result.context("installing Engram")-}--fn verify(bytes: &[u8], expected: &str) -> Result<()> {- let mut hash = String::with_capacity(64);- for byte in Sha256::digest(bytes) {- let _ = write!(hash, "{byte:02x}");- }- ensure!(hash == expected, "Engram download checksum mismatch");- Ok(())-}--fn distribution(os: &str, arch: &str) -> Result<(String, String)> {- let os = match os {- "macos" => "darwin",- "linux" => "linux",- "windows" => "windows",- _ => bail!("Engram does not support {os}"),- };- let arch = match arch {- "x86_64" => "amd64",- "aarch64" => "arm64",- _ => bail!("Engram does not support {arch}"),- };- let target = format!("{os}_{arch}");- let release: serde_json::Value =- serde_json::from_str(include_str!("../../../../../packaging/engram/release.json"))?;- ensure!(release["version"] == VERSION, "Engram release manifest version mismatch");- let checksum = release["assets"][&target].as_str().context("missing Engram release checksum")?.to_owned();- Ok((target, checksum))-}--#[cfg(test)]-mod tests {- use super::*;-- #[test]- fn rejects_corrupt_release_and_unsupported_platform() {- assert!(verify(b"corrupt", &distribution("macos", "aarch64").unwrap().1).is_err());- assert!(distribution("linux", "riscv64").is_err());- for os in ["macos", "linux", "windows"] {- for arch in ["x86_64", "aarch64"] {- assert_eq!(distribution(os, arch).unwrap().1.len(), 64);- }- }- }-}diff --git a/host/src/chat/memory/engram/migrate.rs b/host/src/chat/memory/engram/migrate.rsdeleted file mode 100644index 001abffc..00000000--- a/host/src/chat/memory/engram/migrate.rs+++ /dev/null@@ -1,171 +0,0 @@-//! One-way, retry-safe import of Codync's Markdown memory through Engram's native importer.--use anyhow::{Context, Result, ensure};-use serde_json::{Value, json};-use sha2::{Digest, Sha256};-use std::{collections::BTreeMap, fmt::Write as _, path::Path, time::Duration};--const MARKER: &str = ".codync-markdown-imported";-const BACKUP: &str = "markdown-backup.json";--pub(super) async fn run(binary: &Path, dir: &Path, project: &str) -> Result<()> {- if tokio::fs::try_exists(dir.join(MARKER)).await? {- return Ok(());- }- tokio::fs::create_dir_all(dir).await?;- let backup = dir.join(BACKUP);- // The immutable backup is also the import journal. A retry always reads the same input,- // even when the old Markdown was edited after an interrupted migration.- if !tokio::fs::try_exists(&backup).await? {- let legacy = dir.parent().context("memory directory has no parent")?.join("memory");- let files = tokio::task::spawn_blocking(move || read_legacy(&legacy)).await??;- let staging = backup.with_extension("tmp");- tokio::fs::write(&staging, serde_json::to_vec_pretty(&files)?).await?;- tokio::fs::rename(&staging, &backup).await?;- }- let files: BTreeMap<String, String> = serde_json::from_slice(&tokio::fs::read(&backup).await?)?;- let payload = bundle(project, &files);- let count = payload["observations"].as_array().map_or(0, Vec::len);- if count > 0 {- let input = dir.join("markdown-import.json");- tokio::fs::write(&input, serde_json::to_vec(&payload)?).await?;- let output = tokio::time::timeout(- Duration::from_secs(180),- tokio::process::Command::new(binary)- .arg("import")- .arg(&input)- .env("ENGRAM_DATA_DIR", dir)- .env("ENGRAM_CLOUD_AUTOSYNC", "0")- .current_dir(dir)- .kill_on_drop(true)- .output(),- )- .await??;- ensure!(output.status.success(), "Engram memory import failed: {}", String::from_utf8_lossy(&output.stderr));- let database = dir.join("engram.db");- let expected: Vec<String> = payload["observations"]- .as_array()- .into_iter()- .flatten()- .filter_map(|v| v["sync_id"].as_str().map(str::to_owned))- .collect();- tokio::task::spawn_blocking(move || -> Result<()> {- let db = rusqlite::Connection::open_with_flags(database, rusqlite::OpenFlags::SQLITE_OPEN_READ_ONLY)?;- for id in expected {- let found: bool =- db.query_row("SELECT EXISTS(SELECT 1 FROM observations WHERE sync_id = ?)", [&id], |r| r.get(0))?;- ensure!(found, "Engram import omitted a memory: {id}");- }- Ok(())- })- .await??;- tokio::fs::remove_file(input).await?;- }- // This marker stays even after Forget everything: old files must never resurrect facts.- tokio::fs::write(dir.join(MARKER), format!("Engram {}: {count} memories\n", super::VERSION)).await?;- Ok(())-}--fn read_legacy(dir: &Path) -> Result<BTreeMap<String, String>> {- let mut files = BTreeMap::new();- match std::fs::read_to_string(dir.join("profile.md")) {- Ok(text) => {- files.insert("profile.md".into(), text);- }- Err(e) if e.kind() == std::io::ErrorKind::NotFound => {}- Err(e) => return Err(e).context("reading profile memory"),- }- match std::fs::read_dir(dir.join("log")) {- Ok(entries) => {- for entry in entries {- let path = entry?.path();- if path.extension().is_some_and(|x| x == "md") {- let name = path- .file_name()- .context("memory log has no filename")?- .to_str()- .context("memory log filename is not UTF-8")?;- files.insert(format!("log/{name}"), std::fs::read_to_string(&path)?);- }- }- }- Err(e) if e.kind() == std::io::ErrorKind::NotFound => {}- Err(e) => return Err(e).context("reading memory logs"),- }- Ok(files)-}--fn bundle(project: &str, files: &BTreeMap<String, String>) -> Value {- let session = format!("{project}:markdown-import");- let mut observations = Vec::new();- for (path, text) in files {- for (line, raw) in text.lines().enumerate() {- let Some(rest) = raw.trim().strip_prefix("- (") else {- continue;- };- // The backup is frozen, so a line that can't be read must not block the import forever:- // a malformed one is kept whole, dated by the import.- let dated = rest.split_once(')').and_then(|(date, content)| {- Some((chrono::NaiveDate::parse_from_str(date, "%Y-%m-%d").ok()?, content.trim()))- });- let (date, content) = dated.unwrap_or_else(|| (chrono::Utc::now().date_naive(), &raw.trim()[2..]));- if content.is_empty() {- continue;- }- let timestamp = format!("{date} 00:00:00");- let profile = path == "profile.md";- let source = format!("{project}\n{path}\n{}\n{raw}", line + 1);- let mut id = String::from("obs-codync-");- for b in Sha256::digest(source.as_bytes()) {- let _ = write!(id, "{b:02x}");- }- observations.push(json!({- "sync_id": id, "session_id": session, "project": project,- "type": if profile { "learning" } else { "discovery" },- "title": content.chars().take(100).collect::<String>(), "content": content,- "scope": if profile { "personal" } else { "project" },- "topic_key": format!("codync/import/{}/{}", path, line + 1),- "tool_name": format!("codync-import:{path}:{}", line + 1),- "created_at": timestamp, "updated_at": timestamp, "revision_count": 1,- }));- }- }- let start = observations.first().and_then(|o| o["created_at"].as_str()).unwrap_or("1970-01-01 00:00:00");- json!({- "version": "0.2.0", "exported_at": chrono::Utc::now().format("%Y-%m-%d %H:%M:%S").to_string(),- "sessions": [{"id": session, "project": project, "directory": "", "started_at": start,- "ended_at": start, "summary": "Imported Codync Markdown memory; original files preserved in markdown-backup.json"}],- "observations": observations, "prompts": [],- })-}--#[cfg(test)]-mod tests {- use super::*;-- #[test]- fn preserves_dates_sources_and_repeatable_import_identity() {- let files = BTreeMap::from([- ("profile.md".into(), "# Profile\n- (2026-01-01) 喜歡繁體中文\n".into()),- ("log/2026-02.md".into(), "- (2026-02-02) 發布 Codync\n".into()),- ]);- let a = bundle("bot-a", &files);- let b = bundle("bot-a", &files);- assert_eq!(a["observations"], b["observations"]);- let profile = &a["observations"][1];- assert_eq!(profile["created_at"], "2026-01-01 00:00:00");- assert_eq!(profile["content"], "喜歡繁體中文");- assert_eq!(profile["tool_name"], "codync-import:profile.md:2");- assert_ne!(profile["sync_id"], bundle("bot-b", &files)["observations"][1]["sync_id"]);- }-- #[test]- fn malformed_dated_fact_is_imported_whole_instead_of_blocking() {- let files =- BTreeMap::from([("profile.md".into(), "- (2026-02-31) fact\n- (no date\n- (2026-01-01) \n".into())]);- let payload = bundle("bot", &files);- let contents: Vec<&str> =- payload["observations"].as_array().unwrap().iter().filter_map(|o| o["content"].as_str()).collect();- assert_eq!(contents, ["(2026-02-31) fact", "(no date"]);- }-}diff --git a/host/src/chat/memory/engram/mod.rs b/host/src/chat/memory/engram/mod.rsdeleted file mode 100644index 1d7ec94d..00000000--- a/host/src/chat/memory/engram/mod.rs+++ /dev/null@@ -1,38 +0,0 @@-//! Pinned Engram runtime. Each bot owns a separate database and stdio process.-//! Engram owns schema migrations and every write; Codync never writes its tables.--mod backup;-mod clear;-mod install;-mod migrate;-mod process;--use anyhow::{Result, bail};-use std::path::PathBuf;--pub(super) use backup::{export, import};-pub(super) use process::{call, clear, prepare, tools};--pub const VERSION: &str = "3.2.1";--pub fn directory(bot_id: &str) -> Result<PathBuf> {- if bot_id.is_empty() || !bot_id.bytes().all(|b| b.is_ascii_alphanumeric() || b == b'-') {- bail!("invalid bot id");- }- #[cfg(not(test))]- let root = crate::service::data_dir();- // Unit fixtures run several hosts with the same synthetic bot IDs on separate runtimes.- // Keep their real native databases isolated from each other and from the user's bots.- #[cfg(test)]- let root = {- static ROOT: std::sync::LazyLock<PathBuf> = std::sync::LazyLock::new(|| {- std::env::temp_dir().join(format!("codync-memory-unit-{}", uuid::Uuid::new_v4()))- });- ROOT.join(tokio::runtime::Handle::try_current()?.id().to_string())- };- Ok(root.join("bots").join(bot_id).join("engram"))-}--pub fn project(bot_id: &str) -> String {- format!("codync-{bot_id}")-}diff --git a/host/src/chat/memory/engram/process.rs b/host/src/chat/memory/engram/process.rsdeleted file mode 100644index 61319d7a..00000000--- a/host/src/chat/memory/engram/process.rs+++ /dev/null@@ -1,230 +0,0 @@-//! Serialize native MCP requests per bot. A failed write is never replayed automatically.--use super::{directory, install, project};-use anyhow::{Context, Result, anyhow, bail, ensure};-use serde_json::{Value, json};-use std::{- collections::HashMap,- path::Path,- process::Stdio,- sync::{Arc, LazyLock, Once},- time::{Duration, Instant},-};-use tokio::{- io::{AsyncBufReadExt, AsyncReadExt, AsyncWriteExt, BufReader},- process::{Child, ChildStdin, ChildStdout},- sync::Mutex,-};--type Slot = Arc<Mutex<Option<Process>>>;-static PROCESSES: LazyLock<Mutex<HashMap<String, Slot>>> = LazyLock::new(|| Mutex::new(HashMap::new()));-const REQUEST_TIMEOUT: Duration = Duration::from_secs(90);-const MAX_MESSAGE_BYTES: u64 = 8 * 1024 * 1024;--struct Process {- child: Child,- input: ChildStdin,- output: BufReader<ChildStdout>,- last_used: Instant,- next_id: u64,-}--impl Process {- async fn start(binary: &Path, dir: &Path, project: &str) -> Result<Self> {- tokio::fs::create_dir_all(dir).await?;- let config_dir = dir.join(".engram");- tokio::fs::create_dir_all(&config_dir).await?;- let config = config_dir.join("config.json");- if tokio::fs::try_exists(&config).await? {- let saved: Value = serde_json::from_slice(&tokio::fs::read(&config).await?)?;- ensure!(saved["project_name"] == project, "Engram project binding does not match this bot");- } else {- tokio::fs::write(&config, serde_json::to_vec(&json!({"project_name":project}))?).await?;- }- let mut child = tokio::process::Command::new(binary)- .args(["mcp", "--project", project])- .env("ENGRAM_DATA_DIR", dir)- // Memory stays local even if the user's shell opts another Engram into cloud sync.- .env("ENGRAM_CLOUD_AUTOSYNC", "0")- .current_dir(dir)- .stdin(Stdio::piped())- .stdout(Stdio::piped())- .stderr(Stdio::null())- .kill_on_drop(true)- .spawn()- .context("starting Engram")?;- let input = child.stdin.take().context("Engram stdin unavailable")?;- let output = BufReader::new(child.stdout.take().context("Engram stdout unavailable")?);- let mut process = Self { child, input, output, next_id: 0, last_used: Instant::now() };- process- .request(- "initialize",- json!({- "protocolVersion": crate::mcp::PROTOCOL_VERSION,- "capabilities": {},- "clientInfo": {"name": "codync", "version": env!("CARGO_PKG_VERSION")},- }),- )- .await?;- process.send(&json!({"jsonrpc":"2.0", "method":"notifications/initialized"})).await?;- let registered = process- .request(- "tools/call",- json!({"name":"mem_session_start", "arguments":{- "id": format!("{project}:codync-background"), "directory": dir,- }}),- )- .await?;- ensure!(registered["isError"] != true, "Engram session registration failed: {registered}");- Ok(process)- }-- async fn send(&mut self, value: &Value) -> Result<()> {- let mut bytes = serde_json::to_vec(value)?;- bytes.push(b'\n');- self.input.write_all(&bytes).await?;- self.input.flush().await?;- Ok(())- }-- async fn request(&mut self, method: &str, params: Value) -> Result<Value> {- self.next_id += 1;- let id = self.next_id;- self.send(&json!({"jsonrpc":"2.0", "id":id, "method":method, "params":params})).await?;- loop {- let mut line = Vec::new();- let read = (&mut self.output).take(MAX_MESSAGE_BYTES + 1).read_until(b'\n', &mut line).await?;- ensure!(read > 0, "Engram closed its output");- ensure!(read as u64 <= MAX_MESSAGE_BYTES, "Engram response exceeds size limit");- let value: Value = serde_json::from_slice(&line).context("invalid Engram response")?;- if value.get("id") != Some(&json!(id)) {- continue;- }- if let Some(error) = value.get("error") {- bail!("Engram: {error}");- }- return value.get("result").cloned().context("Engram response has no result");- }- }-}--fn start_idle_cleanup() {- static START: Once = Once::new();- START.call_once(|| {- tokio::spawn(async {- loop {- tokio::time::sleep(Duration::from_secs(60)).await;- let slots: Vec<Slot> = PROCESSES.lock().await.values().cloned().collect();- for slot in slots {- if let Ok(mut process) = slot.try_lock()- && process.as_ref().is_some_and(|p| p.last_used.elapsed() >= Duration::from_secs(300))- {- *process = None;- }- }- }- });- });-}--async fn request(bot: &str, method: &str, params: Value) -> Result<Value> {- start_idle_cleanup();- let dir = directory(bot)?;- let slot = PROCESSES.lock().await.entry(directory(bot)?.to_string_lossy().into_owned()).or_default().clone();- let mut process = slot.lock().await;- if let Some(p) = process.as_mut()- && p.child.try_wait()?.is_some()- {- *process = None;- }- if process.is_none() {- let binary = install::binary().await?;- super::migrate::run(&binary, &dir, &project(bot)).await?;- *process = Some(tokio::time::timeout(REQUEST_TIMEOUT, Process::start(&binary, &dir, &project(bot))).await??);- }- let result =- tokio::time::timeout(REQUEST_TIMEOUT, process.as_mut().context("Engram not running")?.request(method, params))- .await;- match result {- Ok(Ok(value)) => {- if let Some(p) = process.as_mut() {- p.last_used = Instant::now();- }- Ok(value)- }- failure => {- // Dropping stdin and the child closes the session. The next request starts fresh;- // replaying this request could duplicate a write whose acknowledgement was lost.- *process = None;- match failure {- Ok(Err(error)) => Err(error),- Err(error) => Err(anyhow!("Engram request timed out; check whether the change was saved: {error}")),- Ok(Ok(_)) => unreachable!(),- }- }- }-}--/// Stop the MCP writer while the native CLI removes all memory entities.-pub(in crate::chat::memory) async fn clear(bot: &str) -> Result<()> {- prepare(bot).await?;- let slot = PROCESSES.lock().await.entry(directory(bot)?.to_string_lossy().into_owned()).or_default().clone();- let mut process = slot.lock().await;- if let Some(mut p) = process.take() {- p.child.kill().await?;- }- super::clear::run(bot).await-}--pub(in crate::chat::memory) async fn tools(bot: &str) -> Result<Value> {- request(bot, "tools/list", json!({})).await-}--pub(in crate::chat::memory) async fn prepare(bot: &str) -> Result<()> {- request(bot, "ping", json!({})).await?;- Ok(())-}--pub(in crate::chat::memory) async fn call(bot: &str, name: &str, args: &Value) -> Result<Value> {- ensure!(name.starts_with("mem_"), "unknown Engram tool");- request(bot, "tools/call", json!({"name":name, "arguments":args})).await-}--#[cfg(test)]-mod tests {- use super::*;-- #[tokio::test]- #[ignore = "requires the pinned release binary at CODYNC_TEST_ENGRAM"]- async fn native_memory_survives_restart_and_cannot_read_another_bot() {- let binary = std::env::var("CODYNC_TEST_ENGRAM").expect("set CODYNC_TEST_ENGRAM");- let root = std::env::temp_dir().join(format!("codync-engram-test-{}", uuid::Uuid::new_v4()));- let result = async {- let mut a = Process::start(Path::new(&binary), &root.join("a"), "bot-a").await?;- let saved = a- .request(- "tools/call",- json!({"name":"mem_save", "arguments":{- "title":"使用者偏好", "type":"learning", "content":"使用者偏好繁體中文", "project":"bot-a"- }}),- )- .await?;- ensure!(saved["isError"] != true, "save failed: {saved}");- a.child.kill().await?;- drop(a);- let mut a = Process::start(Path::new(&binary), &root.join("a"), "bot-a").await?;- let query = json!({"name":"mem_search", "arguments":{"query":"繁體中文", "all_projects":true}});- let found = a.request("tools/call", query.clone()).await?;- ensure!(found.to_string().contains("使用者偏好"), "restart lost memory: {found}");- let mut b = Process::start(Path::new(&binary), &root.join("b"), "bot-b").await?;- let isolated = b.request("tools/call", query).await?;- ensure!(!isolated.to_string().contains("使用者偏好"), "cross-bot read: {isolated}");- a.child.kill().await?;- b.child.kill().await?;- Ok::<_, anyhow::Error>(())- }- .await;- std::fs::remove_dir_all(root).expect("cleanup test databases");- result.expect("native Engram integration");- }-}diff --git a/host/src/chat/memory/extract.rs b/host/src/chat/memory/extract.rsdeleted file mode 100644index b05ec6be..00000000--- a/host/src/chat/memory/extract.rs+++ /dev/null@@ -1,290 +0,0 @@-//! What the keeper is asked and what it answers: extraction, consolidation and episode-//! prompts (as in Grok Bot), parsing its reply and applying it to the memory folder.--use super::facts::{Fact, Kind, Memory, dedupe_key, normalize, ymd};-use super::{NONE, NOTE_PREFIX, PROFILE_PROMPT_LIMIT, PROFILE_TARGET, RECENT_PROMPT_LIMIT, RELEVANT_LIMIT};-use crate::store::Store;-use anyhow::Result;-use serde::{Deserialize, Serialize};-use std::collections::HashSet;--const TRIVIAL: &[&str] = &[- "hi",- "hey",- "hello",- "yo",- "sup",- "thanks",- "thank you",- "ty",- "thx",- "ok",- "okay",- "k",- "kk",- "cool",- "nice",- "great",- "awesome",- "perfect",- "yes",- "yep",- "yeah",- "no",- "nope",- "sure",- "got it",- "gotcha",- "lol",- "haha",- "np",- "done",- "good",- "bye",-];--/// Small talk ("thanks", "ok") isn't worth a keeper run.-pub fn is_memorable(user: &str) -> bool {- let user = user.trim();- if user.is_empty() {- return false;- }- if user.chars().count() > 40 || user.contains('?') {- return true;- }- let bare = user.to_lowercase();- let bare = bare.trim_end_matches(|c: char| c.is_whitespace() || "!.…,~)]".contains(c));- !TRIVIAL.contains(&bare.split_whitespace().collect::<Vec<_>>().join(" ").as_str())-}--pub fn extraction_system_prompt() -> String {- [- "You maintain the long-term memory of a personal assistant. Read the latest exchanges and decide what — if anything — is worth remembering for future, unrelated conversations.",- "",- "Tag each fact you keep with a category:",- "- \"profile\": enduring facts about who the user is and how to work with them — their name and how to address them, role, location, languages, lasting preferences and constraints, and important people or relationships. These are remembered indefinitely.",- "- \"log\": substantive history worth keeping — ongoing projects and tasks, decisions, commitments, and time-bound details.",- "- \"note\": minor, low-stakes details that might help someday but are not worth keeping in mind every turn (small one-off preferences, incidental context). Notes fade from the always-visible list fastest but stay on disk.",- "",- "Do NOT record one-off request mechanics, what the assistant did this turn, general knowledge, or anything already present in the existing memory list.",- "",- "If a new exchange updates or contradicts a fact in the existing memory list (e.g. the user moved, changed jobs, or renamed something), drop anything clearly superseded: output a line \"remove: <the exact existing fact text>\" and then add the corrected fact. Only remove facts that appear verbatim in the existing list — never invent removals.",- "",- "Write each fact as a self-contained statement, one per line: \"profile: <fact>\", \"log: <fact>\", or \"note: <fact>\" to add (e.g. \"profile: The user's name is Ian\", \"log: Planning a trip to Tokyo in October 2025\"), or \"remove: <existing fact>\" to drop a superseded one.",- "Output exactly NONE (and nothing else) when there is nothing to add or remove.",- ]- .join("\n")-}--pub fn extraction_user_prompt(turns: &[EpisodeTurn], existing: &[String]) -> String {- let existing = if existing.is_empty() {- "(empty)".to_owned()- } else {- existing.iter().map(|m| format!("- {m}")).collect::<Vec<_>>().join("\n")- };- let side = |s: &str| if s.trim().is_empty() { "(no message)".to_owned() } else { s.trim().to_owned() };- let body = turns- .iter()- .map(|t| format!("({})\nUser: {}\nAssistant: {}", ymd(t.ts), side(&t.user), side(&t.agent)))- .collect::<Vec<_>>()- .join("\n\n");- format!("Existing memory:\n{existing}\n\nLatest exchanges, oldest first:\n\n{body}")-}--pub fn consolidation_system_prompt() -> String {- [- format!("You maintain the long-term memory of a personal assistant. Its profile of the user (enduring facts about who they are and how to work with them) has grown past {PROFILE_PROMPT_LIMIT} facts and must be merged down to at most {PROFILE_TARGET}."),- "Merge duplicates and near-duplicates into one self-contained statement, drop facts clearly superseded by a newer one (each fact shows the date it was learned), and move anything that is history rather than who the user is (projects, events, one-off details) to the log.".to_owned(),- "Keep every distinct enduring fact and keep the user's wording where you can. Never invent facts.".to_owned(),- "Output one fact per line: \"profile: <fact>\" for the new profile, \"log: <fact>\" for what moves to the log. Output nothing else.".to_owned(),- ]- .join("\n")-}--pub fn consolidation_user_prompt(facts: &[Fact]) -> String {- let mut facts: Vec<&Fact> = facts.iter().collect();- facts.sort_by_key(|f| f.created_at);- let body = facts.iter().map(|f| format!("- ({}) {}", ymd(f.created_at), f.content)).collect::<Vec<_>>().join("\n");- format!("Profile, oldest first:\n{body}")-}--#[derive(Debug, Default, PartialEq, Eq)]-pub struct Extraction {- pub additions: Vec<(String, Kind)>,- pub removals: Vec<String>,-}--pub fn parse_extraction(raw: &str, existing: &[String]) -> Extraction {- let mut out = Extraction::default();- let raw = raw.trim();- if raw.is_empty() || raw.eq_ignore_ascii_case(NONE) {- return out;- }- let mut seen: HashSet<String> = existing.iter().map(|m| dedupe_key(m)).collect();- for line in raw.lines() {- let line = strip_bullet(line);- let (tag, rest) = match line.split_once(':') {- Some((t, r)) if ["profile", "log", "note", "remove"].contains(&t.trim().to_lowercase().as_str()) => {- (Some(t.trim().to_lowercase()), r)- }- _ => (None, line),- };- let bare = normalize(rest);- if bare.is_empty() || bare.eq_ignore_ascii_case(NONE) {- continue;- }- if tag.as_deref() == Some("remove") {- out.removals.push(bare);- continue;- }- let content = if tag.as_deref() == Some("note") { normalize(&format!("{NOTE_PREFIX}{bare}")) } else { bare };- if seen.insert(dedupe_key(&content)) {- out.additions.push((content, if tag.as_deref() == Some("profile") { Kind::Profile } else { Kind::Log }));- }- }- out-}--fn strip_bullet(line: &str) -> &str {- let t = line.trim_start();- for b in ["- ", "* ", "• "] {- if let Some(r) = t.strip_prefix(b) {- return r;- }- }- let digits = t.chars().take_while(char::is_ascii_digit).count();- if digits > 0- && let Some(r) = t[digits..].strip_prefix(". ").or_else(|| t[digits..].strip_prefix(") "))- {- return r;- }- t-}--/// What the keeper is shown as "existing memory": everything in the prompt plus-/// archived facts that share words with the exchange.-pub fn existing_for_extraction(mem: &Memory, exchange: &str) -> Result<Vec<String>> {- let recall = mem.recall(RECENT_PROMPT_LIMIT)?;- let in_prompt: Vec<&Fact> = recall.profile.iter().chain(&recall.recent).collect();- let shown: HashSet<String> = in_prompt.iter().map(|f| dedupe_key(&f.content)).collect();- let relevant = mem.relevant(exchange, RELEVANT_LIMIT)?;- Ok(in_prompt- .into_iter()- .map(|f| f.content.clone())- .chain(relevant.into_iter().filter(|f| !shown.contains(&dedupe_key(&f.content))).map(|f| f.content))- .collect())-}--pub fn apply(mem: &Memory, ex: &Extraction, known: &[String], now: i64) -> Result<(usize, usize)> {- let known: HashSet<String> = known.iter().map(|k| dedupe_key(k)).collect();- let mut removed = 0;- for r in &ex.removals {- if known.contains(&dedupe_key(r)) && mem.remove_by_content(r)? {- removed += 1;- }- }- let mut added = 0;- for (content, kind) in &ex.additions {- if mem.add(content, *kind, now)? {- added += 1;- }- }- Ok((added, removed))-}--#[derive(Serialize, Deserialize, Clone, Debug, PartialEq, Eq)]-pub struct EpisodeTurn {- pub ts: i64,- pub user: String,- pub agent: String,- #[serde(default)]- pub session: String,-}--pub fn episode_system_prompt(bot_name: &str) -> String {- [- format!("You maintain the long-term memory of a personal assistant named {bot_name}."),- format!("You are given the most recent turns of a conversation between the user and {bot_name}, in order, each tagged with its date."),- format!("Write a concise session summary of goals, decisions, outcomes and unfinished next steps capturing what the user and {bot_name} were actually working on across these turns — the throughline, key decisions, and outcomes — so it stays useful months from now."),- "Anchor any time references with the absolute dates shown, never relative words like \"yesterday\". Drop greetings, acknowledgements, and anything ephemeral. Never invent details.".to_owned(),- "Output only the summary, no preamble. Output exactly NONE if nothing in this stretch is worth remembering.".to_owned(),- ]- .join("\n")-}--pub fn episode_user_prompt(bot_name: &str, turns: &[EpisodeTurn]) -> String {- let body = turns- .iter()- .map(|t| {- let mut parts = vec![format!("({})", ymd(t.ts))];- if !t.user.trim().is_empty() {- parts.push(format!("User: {}", t.user.trim()));- }- if !t.agent.trim().is_empty() {- parts.push(format!("{bot_name}: {}", t.agent.trim()));- }- parts.join("\n")- })- .collect::<Vec<_>>()- .join("\n\n");- format!("Recent turns, oldest first:\n\n{body}")-}--fn episode_key(bot_id: &str) -> String {- format!("memory.episode.{bot_id}")-}--pub fn pending_episode(store: &Store, bot_id: &str) -> Vec<EpisodeTurn> {- store.kv_get(&episode_key(bot_id)).and_then(|v| serde_json::from_str(&v).ok()).unwrap_or_default()-}--pub fn set_pending_episode(store: &Store, bot_id: &str, turns: &[EpisodeTurn]) -> Result<()> {- store.kv_set(&episode_key(bot_id), &serde_json::to_string(turns)?)-}--#[cfg(test)]-mod tests {- use super::super::dates::parse_ymd;- use super::*;-- #[test]- fn extraction_output_is_parsed() {- let existing = vec!["Lives in Taipei".to_owned()];- let ex = parse_extraction(- "- profile: The user's name is Ian\n2. log: Planning a trip\nnote: likes tea\nremove: Lives in Taipei\nprofile: lives in taipei\nNONE",- &existing,- );- assert_eq!(- ex.additions,- vec![- ("The user's name is Ian".to_owned(), Kind::Profile),- ("Planning a trip".to_owned(), Kind::Log),- ("[note] likes tea".to_owned(), Kind::Log),- ]- );- assert_eq!(ex.removals, vec!["Lives in Taipei".to_owned()]);- assert_eq!(parse_extraction(" none ", &[]), Extraction::default());- }-- #[test]- fn batched_exchanges_are_dated_in_order() {- let t = parse_ymd("2026-09-30").expect("valid date");- let turns = [- EpisodeTurn { ts: t, user: "I use Swift 6".into(), agent: "Noted".into(), session: String::new() },- EpisodeTurn { ts: t, user: String::new(), agent: "Done".into(), session: String::new() },- ];- let prompt = extraction_user_prompt(&turns, &[]);- assert!(prompt.contains("Existing memory:\n(empty)"));- assert!(- prompt.contains("(2026-09-30)\nUser: I use Swift 6\nAssistant: Noted\n\n(2026-09-30)\nUser: (no message)")- );- }-- #[test]- fn small_talk_is_not_memorable() {- assert!(!is_memorable("thanks!"));- assert!(!is_memorable(" OK. "));- assert!(is_memorable("my name is Kai"));- assert!(is_memorable("why?"));- }-}diff --git a/host/src/chat/memory/facts.rs b/host/src/chat/memory/facts.rsdeleted file mode 100644index 7ee03548..00000000--- a/host/src/chat/memory/facts.rs+++ /dev/null@@ -1,212 +0,0 @@-//! Read-only projection of Engram's pinned schema for Codync's UI and prompt.-//! These blocking methods run on `spawn_blocking`; every write uses native MCP.--use super::{MAX_FACT_CHARS, PROFILE_PROMPT_LIMIT, engram};-use anyhow::{Context, Result, bail};-use rusqlite::{Connection, OpenFlags};-use serde::{Deserialize, Serialize};-use serde_json::{Value, json};-use std::path::{Path, PathBuf};--pub use super::dates::ymd;--#[derive(Serialize, Deserialize, Clone, Copy, Debug, PartialEq, Eq)]-#[serde(rename_all = "lowercase")]-pub enum Kind {- Profile,- Log,-}--#[derive(Serialize, Clone, Debug)]-#[serde(rename_all = "camelCase")]-pub struct Fact {- pub id: String,- pub content: String,- pub created_at: i64,- pub kind: Kind,- pub title: String,- pub memory_type: String,- pub scope: String,- pub project: String,- pub topic_key: Option<String>,- pub session_id: String,- pub pinned: bool,- pub review_after: Option<String>,- pub revision_count: i64,- pub source: Option<String>,-}--pub struct Recall {- pub profile: Vec<Fact>,- pub recent: Vec<Fact>,-}--pub struct Memory {- bot: String,- dir: PathBuf,- session: String,-}--impl Memory {- pub fn for_bot(bot: &str) -> Result<Self> {- let dir = engram::directory(bot)?;- tokio::runtime::Handle::try_current()- .context("memory access requires a host runtime")?- .block_on(engram::prepare(bot))?;- Ok(Self { bot: bot.to_owned(), dir, session: format!("{}:codync-background", engram::project(bot)) })- }-- pub fn for_session(bot: &str, session: &str) -> Result<Self> {- let mut memory = Self::for_bot(bot)?;- if !session.is_empty() {- session.clone_into(&mut memory.session);- }- Ok(memory)- }-- pub fn location(&self) -> &Path {- &self.dir- }-- pub fn project(&self) -> String {- engram::project(&self.bot)- }-- fn database(&self) -> Result<Connection> {- let db = Connection::open_with_flags(self.dir.join("engram.db"), OpenFlags::SQLITE_OPEN_READ_ONLY)?;- db.busy_timeout(std::time::Duration::from_secs(5))?;- Ok(db)- }-- pub fn relevant(&self, query: &str, limit: usize) -> Result<Vec<Fact>> {- super::read::relevant(&self.database()?, query, limit)- }-- pub fn browse(&self, query: &str, filter: &str, limit: usize, offset: usize) -> Result<super::read::Page> {- super::read::browse(&mut self.database()?, query, filter, limit, offset)- }-- pub fn find(&self, id: &str) -> Result<Fact> {- super::read::find(&self.database()?, id.parse()?)?.context("memory not found")- }-- pub fn recall(&self, recent_limit: usize) -> Result<Recall> {- let db = self.database()?;- Ok(Recall {- profile: super::read::list(&db, "profile", PROFILE_PROMPT_LIMIT)?,- recent: super::read::list(&db, "log", recent_limit)?,- })- }-- pub fn profile_facts(&self) -> Result<Vec<Fact>> {- super::read::list(&self.database()?, "profile", usize::MAX)- }-- pub fn add(&self, content: &str, kind: Kind, at_ms: i64) -> Result<bool> {- let content = normalize(content);- if content.is_empty() {- return Ok(false);- }- if super::read::same_content(&self.database()?, &content)?.is_some() {- return Ok(false);- }- self.tool(- "mem_save",- &json!({- "title": content.chars().take(100).collect::<String>(), "content": content,- "type": if kind == Kind::Profile { "learning" } else { "discovery" },- "scope": if kind == Kind::Profile { "personal" } else { "project" },- "project": engram::project(&self.bot), "capture_prompt": false,- "session_id": self.session,- "topic_key": format!("codync/keeper/{at_ms}/{}", super::read::content_hash(&content)),- }),- )?;- Ok(true)- }-- pub fn summary(&self, content: &str) -> Result<()> {- self.tool(- "mem_session_summary",- &json!({"session_id":self.session, "project":self.project(), "content":content}),- )?;- Ok(())- }-- pub fn remove(&self, id: &str) -> Result<bool> {- let Some(fact) = super::read::find(&self.database()?, id.parse()?)? else {- return Ok(false);- };- self.tool("mem_delete", &json!({"id":id.parse::<i64>()?, "expected_project":fact.project}))?;- Ok(true)- }-- pub fn remove_by_content(&self, content: &str) -> Result<bool> {- let Some(fact) = super::read::same_content(&self.database()?, content)? else {- return Ok(false);- };- self.remove(&fact.id)- }-- pub fn clear(&self) -> Result<()> {- tokio::runtime::Handle::try_current()?.block_on(engram::clear(&self.bot))- }-- /// Consolidation changes the scope of old facts; their IDs and history remain intact.- pub fn rewrite_profile(&self, keep: &[(String, i64)], demote: &[(String, i64)]) -> Result<()> {- for (content, _) in demote {- if let Some(fact) = super::read::same_content(&self.database()?, content)? {- self.tool(- "mem_update",- &json!({"id":fact.id.parse::<i64>()?, "expected_project":fact.project, "scope":"project"}),- )?;- }- }- for (content, at) in keep {- if let Some(fact) = super::read::same_content(&self.database()?, content)? {- if fact.scope != "personal" {- self.tool(- "mem_update",- &json!({"id":fact.id.parse::<i64>()?, "expected_project":fact.project, "scope":"personal"}),- )?;- }- } else {- self.add(content, Kind::Profile, *at)?;- }- }- Ok(())- }-- pub fn tool(&self, name: &str, args: &Value) -> Result<Value> {- let result = tokio::runtime::Handle::try_current()?.block_on(engram::call(&self.bot, name, args))?;- if result["isError"] == true {- bail!("Engram: {}", result["content"]);- }- Ok(result)- }-}--pub(super) fn parse_timestamp(raw: &str) -> Result<i64> {- if let Ok(date) = chrono::DateTime::parse_from_rfc3339(raw) {- return Ok(date.timestamp_millis());- }- Ok(chrono::NaiveDateTime::parse_from_str(raw, "%Y-%m-%d %H:%M:%S%.f")?.and_utc().timestamp_millis())-}--pub fn normalize(raw: &str) -> String {- raw.split_whitespace().collect::<Vec<_>>().join(" ").chars().take(MAX_FACT_CHARS).collect()-}--pub(super) fn dedupe_key(content: &str) -> String {- normalize(content).to_lowercase()-}--#[cfg(test)]-mod tests {- use super::*;-- #[test]- fn reads_both_native_timestamp_formats_without_losing_date() {- assert_eq!(parse_timestamp("2026-01-01 00:00:00").unwrap(), parse_timestamp("2026-01-01T00:00:00Z").unwrap());- assert!(parse_timestamp("bad date").is_err());- }-}diff --git a/host/src/chat/memory/import.rs b/host/src/chat/memory/import.rsnew file mode 100644index 00000000..e02330a0--- /dev/null+++ b/host/src/chat/memory/import.rs@@ -0,0 +1,196 @@+//! A one-time move of the memory Codync kept before (an Engram database per bot, and the Markdown files before+//! that) into the host's memory tables. Each bot moves once; then the old files go.+//!+//! Facts the old keeper filed about the user come in active, as far as the bot's cap allows; the rest are+//! archived, still searchable. Dreaming changes moved entries only on the user's own contrary words.++use crate::hub::Hub;+use crate::store::Store;+use anyhow::{Context, Result};+use codync_mem::store::{self as mem, Error, New, Writer};+use std::path::Path;++/// Prefix the old keeper put on notes.+const NOTE_PREFIX: &str = "[note] ";++struct Fact {+ text: String,+ /// About the user (the old profile) rather than a log entry.+ profile: bool,+ pinned: bool,+ /// Unix milliseconds.+ at: i64,+}++/// Moves every bot's old memory once, then removes what the old memory left on disk. Errors are logged per bot,+/// and the bot keeps its old files until a later start moves them.+pub fn import_old_memory(hub: &Hub) {+ let data = crate::service::data_dir();+ let rows = match hub.store.bots() {+ Ok(rows) => rows,+ Err(error) => {+ tracing::warn!(error = format!("{error:#}"), "couldn't list bots to move their memory");+ return;+ }+ };+ for row in rows.iter().filter(|r| !r.deleted && !r.config.is_group()) {+ let id = &row.config.id;+ let dir = data.join("bots").join(id);+ if let Err(error) = import_bot(&hub.store, id, &dir) {+ tracing::warn!(bot = %id, error = format!("{error:#}"), "couldn't move the bot's old memory");+ continue;+ }+ for old in ["engram", "memory"] {+ remove(&dir.join(old));+ }+ }+ if hub.store.kv_get("memory.moved").is_none() {+ // The old keeper's queues and session bindings, and its notices' revisions (uuids, not revs).+ for prefix in+ ["memory.unprocessed.", "memory.episode.", "memory.session.", "memory.revision.", "memory.announced."]+ {+ for key in hub.store.kv_prefix(prefix) {+ let _ = hub.store.kv_delete(&key);+ }+ }+ for row in rows.iter().filter(|r| !r.deleted && !r.config.is_group()) {+ if let Err(error) = crate::chat::context::invalidate(&hub.store, &row.config.id) {+ tracing::warn!(bot = %row.config.id, error = format!("{error:#}"), "couldn't re-render instructions");+ }+ }+ let _ = hub.store.kv_set("memory.moved", "1");+ }+ for old in ["engram", "memory-keeper"] {+ remove(&data.join(old));+ }+}++fn import_bot(store: &Store, bot: &str, dir: &Path) -> Result<()> {+ let marker = format!("memory.imported.{bot}");+ if store.kv_get(&marker).is_some() {+ return Ok(());+ }+ let engram = dir.join("engram/engram.db");+ let facts = if engram.exists() { read_engram(&engram)? } else { read_markdown(&dir.join("memory"))? };+ let count = facts.len();+ store.memory_write(|c, _, rev| {+ let mut used = 0;+ for fact in &facts {+ let new = New { bot: Some(bot), text: &fact.text, room: None, evidence: &[] };+ let entry = match mem::add(c, Writer::Import, new, fact.at, rev) {+ Ok(entry) => entry,+ Err(Error::Refused(_)) => continue,+ Err(e) => return Err(e),+ };+ if fact.pinned {+ mem::pin(c, entry.id, true, fact.at, rev)?;+ }+ let size = codync_mem::tokens::estimate(&entry.text);+ if fact.profile && used + size <= codync_mem::tokens::BOT_CAP {+ used += size;+ } else {+ mem::archive(c, Writer::Import, entry.id, None, fact.at, rev)?;+ }+ }+ Ok(())+ })?;+ // Dreaming starts from now: what came before is in the moved memory already.+ super::dream::set_bookmark(store, bot, store.last_seq(bot))?;+ store.kv_set(&marker, &count.to_string())?;+ tracing::info!(bot, facts = count, "moved the bot's old memory");+ Ok(())+}++/// The old Engram observations, profile and pinned first, newest first: the order the opening takes them in.+fn read_engram(path: &Path) -> Result<Vec<Fact>> {+ let db = rusqlite::Connection::open_with_flags(path, rusqlite::OpenFlags::SQLITE_OPEN_READ_ONLY)+ .context("opening the old memory database")?;+ let mut stmt = db.prepare(+ "SELECT content, scope, pinned, created_at FROM observations+ WHERE deleted_at IS NULL AND type NOT LIKE '%summary%'+ ORDER BY pinned DESC, (scope = 'personal') DESC, created_at DESC, id DESC",+ )?;+ let rows = stmt.query_map([], |row| {+ Ok((+ row.get::<_, String>(0)?,+ row.get::<_, Option<String>>(1)?,+ row.get::<_, bool>(2)?,+ row.get::<_, String>(3)?,+ ))+ })?;+ let mut facts = Vec::new();+ for row in rows {+ let (content, scope, pinned, created) = row?;+ let text = content.strip_prefix(NOTE_PREFIX).unwrap_or(&content).trim().to_owned();+ if text.is_empty() {+ continue;+ }+ let at = chrono::NaiveDateTime::parse_from_str(&created, "%Y-%m-%d %H:%M:%S%.f")+ .map_or_else(|_| crate::store::now_ms(), |t| t.and_utc().timestamp_millis());+ facts.push(Fact { text, profile: scope.as_deref() == Some("personal") || pinned, pinned, at });+ }+ Ok(facts)+}++/// The Markdown memory from before Engram: `profile.md` about the user, `log/*.md` dated notes, as bullets+/// `- (YYYY-MM-DD) fact`.+fn read_markdown(dir: &Path) -> Result<Vec<Fact>> {+ let mut files = Vec::new();+ match std::fs::read_to_string(dir.join("profile.md")) {+ Ok(text) => files.push((true, text)),+ Err(e) if e.kind() == std::io::ErrorKind::NotFound => {}+ Err(e) => return Err(e).context("reading profile memory"),+ }+ match std::fs::read_dir(dir.join("log")) {+ Ok(entries) => {+ for entry in entries {+ let path = entry?.path();+ if path.extension().is_some_and(|x| x == "md") {+ files.push((false, std::fs::read_to_string(&path)?));+ }+ }+ }+ Err(e) if e.kind() == std::io::ErrorKind::NotFound => {}+ Err(e) => return Err(e).context("reading memory logs"),+ }+ let mut facts: Vec<Fact> =+ files.iter().flat_map(|(profile, text)| text.lines().filter_map(move |line| bullet(line, *profile))).collect();+ facts.sort_by_key(|f| (!f.profile, std::cmp::Reverse(f.at)));+ Ok(facts)+}++fn bullet(line: &str, profile: bool) -> Option<Fact> {+ let rest = line.trim().strip_prefix("- ")?;+ let (at, text) = rest+ .strip_prefix('(')+ .and_then(|r| r.split_once(')'))+ .and_then(|(date, text)| {+ let day = chrono::NaiveDate::parse_from_str(date, "%Y-%m-%d").ok()?;+ Some((day.and_hms_opt(0, 0, 0)?.and_utc().timestamp_millis(), text))+ })+ .unwrap_or((crate::store::now_ms(), rest));+ let text = text.trim();+ (!text.is_empty()).then(|| Fact { text: text.to_owned(), profile, pinned: false, at })+}++fn remove(path: &Path) {+ match std::fs::remove_dir_all(path) {+ Ok(()) => tracing::info!(path = %path.display(), "removed the old memory files"),+ Err(e) if e.kind() == std::io::ErrorKind::NotFound => {}+ Err(e) => tracing::warn!(path = %path.display(), error = %e, "couldn't remove the old memory files"),+ }+}++#[cfg(test)]+mod tests {+ use super::*;++ #[test]+ fn markdown_bullets_keep_their_dates() {+ let fact = bullet("- (2026-03-01) Prefers tea", true).unwrap();+ assert_eq!(fact.text, "Prefers tea");+ assert_eq!(super::super::dates::ymd(fact.at), "2026-03-01");+ assert_eq!(bullet("- undated fact", false).unwrap().text, "undated fact");+ assert!(bullet("plain line", true).is_none());+ }+}diff --git a/host/src/chat/memory/keeper.rs b/host/src/chat/memory/keeper.rsdeleted file mode 100644index 3e503097..00000000--- a/host/src/chat/memory/keeper.rs+++ /dev/null@@ -1,350 +0,0 @@-//! The bot's memory keeper: queued exchanges, the one-shot agent runs that remember them,-//! episodes and profile consolidation.--use super::extract::{- EpisodeTurn, apply, consolidation_system_prompt, consolidation_user_prompt, episode_system_prompt,- episode_user_prompt, existing_for_extraction, extraction_system_prompt, extraction_user_prompt, parse_extraction,- pending_episode, set_pending_episode,-};-use super::facts::{Kind, Memory, dedupe_key};-use super::{- EXCHANGE_CHARS, KEEPER_BATCH, KEEPER_IDLE, KEEPER_QUEUE_LIMIT, KEEPER_TIMEOUT, NONE, PROFILE_PROMPT_LIMIT,-};-use crate::agent::acp::{self, Acp, Incoming};-use crate::hub::Hub;-use crate::store::{BotConfig, Store};-use anyhow::{Result, anyhow, bail};-use serde_json::json;-use std::collections::HashSet;-use std::sync::Arc;-use std::time::Duration;-use tokio::sync::mpsc;--/// A finished user ↔ agent exchange worth remembering.-pub struct Exchange {- pub user: String,- pub agent: String,- pub at: i64,- pub session: String,- pub revision: String,-}--pub enum KeeperEvent {- Exchange(Exchange),- Flush,-}--fn unprocessed_key(bot_id: &str) -> String {- format!("memory.unprocessed.{bot_id}")-}--fn unprocessed(store: &Store, bot_id: &str) -> Vec<EpisodeTurn> {- store.kv_get(&unprocessed_key(bot_id)).and_then(|v| serde_json::from_str(&v).ok()).unwrap_or_default()-}--/// Starts the bot's keeper. Exchanges queue up (persisted, so a restart keeps them)-/// and are remembered together once the bot has been quiet for [`KEEPER_IDLE`] or-/// [`KEEPER_BATCH`] have piled up: the frozen prompt only picks facts up at the next-/// compaction or session anyway, and one run over several exchanges sees their-/// context. An unnamed bot is named right away (see `naming`). It stops when the-/// bot actor drops the sender; what's queued is picked up by the next keeper.-pub fn spawn_keeper(hub: Arc<Hub>, bot_id: String) -> mpsc::UnboundedSender<KeeperEvent> {- let (tx, mut rx) = mpsc::unbounded_channel::<KeeperEvent>();- tokio::spawn(async move {- let mut queued = unprocessed(&hub.store, &bot_id);- let idle = tokio::time::sleep(KEEPER_IDLE);- tokio::pin!(idle);- // After a failed run the next one waits, longer each time, instead of retrying on every exchange.- let mut failures = 0u32;- let mut retry_at: Option<tokio::time::Instant> = None;- let quiet_until = |retry_at: Option<tokio::time::Instant>| {- let after = tokio::time::Instant::now() + KEEPER_IDLE;- retry_at.map_or(after, |at| at.max(after))- };- loop {- let flush = tokio::select! {- x = rx.recv() => {- let Some(event) = x else { break };- let KeeperEvent::Exchange(x) = event else {- idle.as_mut().reset(retry_at.unwrap_or_else(tokio::time::Instant::now));- continue;- };- if x.revision != super::maintenance::revision(&hub.store, &bot_id) { continue; }- let turn = EpisodeTurn {- ts: x.at,- user: acp::truncate(&x.user, EXCHANGE_CHARS),- agent: acp::truncate(&x.agent, EXCHANGE_CHARS),- session: x.session,- };- let mutation = super::maintenance::lock(&bot_id);- let guard = mutation.lock().await;- if x.revision != super::maintenance::revision(&hub.store, &bot_id) { continue; }- // Re-read: another keeper of this bot (before a restart) may have flushed.- queued = unprocessed(&hub.store, &bot_id);- queued.push(turn.clone());- // ponytail: while the keeper can't run, only the newest exchanges wait for it (the- // transcript keeps them all); keep a longer backlog if older ones must still be learned.- let over = queued.len().saturating_sub(KEEPER_QUEUE_LIMIT);- queued.drain(..over);- if let Err(e) = hub.store.kv_set(&unprocessed_key(&bot_id), &serde_json::to_string(&queued).unwrap_or_default()) {- tracing::warn!(bot = %bot_id, error = format!("{e:#}"), "couldn't queue an exchange for memory");- }- drop(guard);- if let Err(e) = crate::chat::naming::observe(&hub, &bot_id, turn.clone()).await {- tracing::warn!(bot = %bot_id, error = format!("{e:#}"), "naming the bot failed");- }- idle.as_mut().reset(quiet_until(retry_at));- queued.len() >= KEEPER_BATCH && retry_at.is_none()- }- () = &mut idle, if !queued.is_empty() => true,- };- if !flush {- continue;- }- let processing = super::maintenance::keeper_lock(&bot_id);- let _processing = processing.lock().await;- // One batch per run keeps the keeper's prompt bounded; the rest follows right after.- let batch: Vec<EpisodeTurn> = unprocessed(&hub.store, &bot_id).into_iter().take(KEEPER_BATCH).collect();- match remember(&hub, &bot_id, batch.clone()).await {- Ok(()) => {- failures = 0;- retry_at = None;- let mutation = super::maintenance::lock(&bot_id);- let _guard = mutation.lock().await;- let mut current = unprocessed(&hub.store, &bot_id);- acknowledge(&mut current, &batch);- if let Err(e) = hub- .store- .kv_set(&unprocessed_key(&bot_id), &serde_json::to_string(¤t).unwrap_or_default())- {- tracing::warn!(bot = %bot_id, error = format!("{e:#}"), "couldn't acknowledge the memory queue");- } else {- queued = current;- }- }- Err(e) => {- failures += 1;- retry_at = Some(tokio::time::Instant::now() + KEEPER_IDLE * 2u32.pow(failures.min(5)));- tracing::warn!(bot = %bot_id, failures, error = format!("{e:#}"), "memory keeper failed; batch retained for retry");- }- }- let next = if retry_at.is_none() && queued.len() >= KEEPER_BATCH {- tokio::time::Instant::now()- } else {- quiet_until(retry_at)- };- idle.as_mut().reset(next);- }- });- tx-}--/// Another actor may have appended turns while an old keeper was working.-fn acknowledge(current: &mut Vec<EpisodeTurn>, completed: &[EpisodeTurn]) {- if current.starts_with(completed) {- current.drain(..completed.len());- }-}--async fn remember(hub: &Arc<Hub>, bot_id: &str, turns: Vec<EpisodeTurn>) -> Result<()> {- let mut groups = std::collections::BTreeMap::<String, Vec<EpisodeTurn>>::new();- for turn in pending_episode(&hub.store, bot_id).into_iter().chain(turns) {- groups.entry(turn.session.clone()).or_default().push(turn);- }- for (session, turns) in groups {- remember_session(hub, bot_id, &session, turns).await?;- }- set_pending_episode(&hub.store, bot_id, &[])?;- Ok(())-}--async fn remember_session(hub: &Arc<Hub>, bot_id: &str, session: &str, turns: Vec<EpisodeTurn>) -> Result<()> {- let Some(cfg) = hub.store.bot(bot_id)?.filter(|b| !b.deleted).map(|b| b.config) else { return Ok(()) };- let Some(at) = turns.last().map(|t| t.ts) else { return Ok(()) };- let revision = super::maintenance::revision(&hub.store, bot_id);-- let id = bot_id.to_owned();- let exchange_text = turns.iter().map(|t| format!("{}\n{}", t.user, t.agent)).collect::<Vec<_>>().join("\n");- let existing = tokio::task::spawn_blocking(move || {- Memory::for_bot(&id).and_then(|mem| existing_for_extraction(&mem, &exchange_text))- })- .await??;- let raw = one_shot(hub, &cfg, &extraction_system_prompt(), &extraction_user_prompt(&turns, &existing)).await?;- let extraction = parse_extraction(&raw, &existing);- let mutation = super::maintenance::lock(bot_id);- let guard = mutation.lock().await;- if super::maintenance::revision(&hub.store, bot_id) != revision {- return Ok(()); // A user's edit/forget supersedes this extraction from old context.- }- let id = bot_id.to_owned();- let source_session =- if session.is_empty() { String::new() } else { super::lifecycle::begin(hub, bot_id, session).await? };- let summary_session = source_session.clone();- let (added, removed, crowded) = tokio::task::spawn_blocking(move || {- Memory::for_session(&id, &source_session).and_then(|mem| {- let (added, removed) = apply(&mem, &extraction, &existing, at)?;- Ok((added, removed, mem.profile_facts()?.len() > PROFILE_PROMPT_LIMIT))- })- })- .await??;- tracing::info!(bot = %bot_id, exchanges = turns.len(), added, removed, "memory updated");- drop(guard);- if crowded && let Err(e) = consolidate(hub, &cfg, &revision).await {- tracing::warn!(bot = %bot_id, error = format!("{e:#}"), "consolidating the profile failed");- }-- let raw = one_shot(hub, &cfg, &episode_system_prompt(&cfg.name), &episode_user_prompt(&cfg.name, &turns)).await?;- let narrative: String = raw.trim().chars().take(8_000).collect();- let _guard = mutation.lock().await;- if super::maintenance::revision(&hub.store, bot_id) != revision {- return Ok(());- }- if !narrative.is_empty() && !narrative.eq_ignore_ascii_case(NONE) {- let id = bot_id.to_owned();- tokio::task::spawn_blocking(move || {- let mem = Memory::for_session(&id, &summary_session)?;- mem.summary(&narrative)- })- .await??;- }- Ok(())-}--/// The profile outgrew what the prompt shows: have the keeper merge it down to-/// [`PROFILE_TARGET`] facts. Nothing is lost: facts it drops move to the log.-async fn consolidate(hub: &Arc<Hub>, cfg: &BotConfig, revision: &str) -> Result<()> {- let id = cfg.id.clone();- let facts = tokio::task::spawn_blocking(move || Memory::for_bot(&id).and_then(|m| m.profile_facts())).await??;- let raw = one_shot(hub, cfg, &consolidation_system_prompt(), &consolidation_user_prompt(&facts)).await?;- let plan = parse_extraction(&raw, &[]);- let keep: Vec<&String> = plan.additions.iter().filter(|(_, k)| *k == Kind::Profile).map(|(c, _)| c).collect();- if keep.is_empty() || keep.len() > PROFILE_PROMPT_LIMIT {- bail!("the consolidated profile had {} facts", keep.len());- }- let now = crate::store::now_ms();- let dated = |c: &str| facts.iter().find(|f| dedupe_key(&f.content) == dedupe_key(c)).map_or(now, |f| f.created_at);- let kept: Vec<(String, i64)> = keep.iter().map(|c| ((*c).clone(), dated(c))).collect();- let kept_keys: HashSet<String> = kept.iter().map(|(c, _)| dedupe_key(c)).collect();- let mut demoted: Vec<(String, i64)> = facts- .iter()- .filter(|f| !kept_keys.contains(&dedupe_key(&f.content)))- .map(|f| (f.content.clone(), f.created_at))- .collect();- demoted.extend(plan.additions.iter().filter(|(_, k)| *k == Kind::Log).map(|(c, _)| (c.clone(), now)));- let mutation = super::maintenance::lock(&cfg.id);- let _guard = mutation.lock().await;- if super::maintenance::revision(&hub.store, &cfg.id) != revision {- return Ok(());- }- let id = cfg.id.clone();- let (before, after) = (facts.len(), kept.len());- tokio::task::spawn_blocking(move || Memory::for_bot(&id).and_then(|m| m.rewrite_profile(&kept, &demoted)))- .await??;- tracing::info!(bot = %cfg.id, before, after, "profile consolidated");- Ok(())-}--/// Runs one prompt on a throwaway agent of the bot's harness and returns its reply.-/// Claude gets a real system prompt, no tools, no settings and no saved session;-/// other harnesses get the instructions inline.-pub(crate) async fn one_shot(hub: &Arc<Hub>, cfg: &BotConfig, system: &str, user: &str) -> Result<String> {- let cwd = crate::service::data_dir().join("memory-keeper");- tokio::fs::create_dir_all(&cwd).await?;- let cwd = cwd.to_string_lossy().into_owned();- if hub.store.kv_read(&format!("agent-env:{}", cfg.backend))?.is_some() {- crate::market::vault::unlock(hub.clone()).await?;- }- let env = crate::agent::auth::env(&hub.store, &cfg.backend)?;- let mut conn = None;- let mut last_err = None;- for command in crate::agent::bot::launch_commands(cfg, |_| {}).await? {- match crate::agent::bot::start_agent(&command, &cwd, &env, Duration::from_secs(60)).await {- Ok(c) => {- conn = Some(c);- break;- }- Err(e) => last_err = Some(e),- }- }- let mut conn = conn.ok_or_else(|| last_err.unwrap_or_else(|| anyhow!("no agent to run the memory keeper")))?;- let acp = conn.acp.clone();- let result = async {- let mut params = json!({"cwd": cwd, "mcpServers": []});- let text = if conn.claude {- params["_meta"] = json!({- "systemPrompt": system,- "claudeCode": {"options": {"tools": [], "persistSession": false, "settingSources": [], "model": "haiku"}},- });- user.to_owned()- } else {- format!("{system}\n\n---\n\n{user}")- };- let res = acp.request("session/new", params).await?;- let sid = res["sessionId"].as_str().ok_or_else(|| anyhow!("agent returned no sessionId"))?.to_owned();- // Same model as the bot: the harness default may be another (local) model that- // would load next to the bot's, or a cloud one the user kept these chats away from.- if !conn.claude {- crate::agent::bot::select_model(&acp, &res, &sid, cfg).await?;- }- let prompt = acp.request("session/prompt", json!({"sessionId": sid, "prompt": [{"type": "text", "text": text}]}));- tokio::pin!(prompt);- let deadline = tokio::time::sleep(KEEPER_TIMEOUT);- tokio::pin!(deadline);- let mut out = String::new();- loop {- tokio::select! {- r = &mut prompt => { r?; break }- inc = conn.rx.recv() => {- if !collect(&acp, inc, &mut out).await {- bail!("the memory keeper's agent exited");- }- }- () = &mut deadline => bail!("the memory keeper timed out"),- }- }- while let Ok(inc) = conn.rx.try_recv() {- collect(&acp, Some(inc), &mut out).await;- }- Ok(out)- }- .await;- acp.kill().await;- result-}--/// Keeps reply text; refuses anything the agent asks of us. False once the agent is gone.-async fn collect(acp: &Acp, inc: Option<Incoming>, out: &mut String) -> bool {- match inc {- None | Some(Incoming::Closed { .. }) => false,- Some(Incoming::Notification { method, params }) => {- let u = ¶ms["update"];- if method == "session/update" && u["sessionUpdate"] == "agent_message_chunk" {- out.push_str(&acp::content_text(&u["content"]));- }- true- }- Some(Incoming::Request { id, method, .. }) => {- let _ = if method == "session/request_permission" {- acp.respond(id, json!({"outcome": {"outcome": "cancelled"}})).await- } else {- acp.respond_error(id, -32601, "not supported").await- };- true- }- }-}--#[cfg(test)]-mod tests {- use super::*;-- #[test]- fn acknowledging_a_batch_preserves_newer_or_replaced_work() {- let turn = |ts| EpisodeTurn { ts, user: "preference".into(), agent: "noted".into(), session: "session".into() };- let mut current = vec![turn(1), turn(2), turn(3)];- acknowledge(&mut current, &[turn(1), turn(2)]);- assert_eq!(current, vec![turn(3)]);- acknowledge(&mut current, &[turn(1)]);- assert_eq!(current, vec![turn(3)]);- }-}diff --git a/host/src/chat/memory/lifecycle.rs b/host/src/chat/memory/lifecycle.rsdeleted file mode 100644index 5864989c..00000000--- a/host/src/chat/memory/lifecycle.rs+++ /dev/null@@ -1,88 +0,0 @@-//! Bind memory writes to the actual ACP session, including independent reply threads.--use super::{engram, manage::payload};-use crate::{- hub::Hub,- store::{Lane, lane_key},-};-use anyhow::{Context, Result, ensure};-use serde_json::{Value, json};-use std::sync::Arc;--fn key(bot: &str, session: &str) -> String {- format!("memory.session.{bot}.{session}")-}--pub async fn begin(hub: &Arc<Hub>, bot: &str, session: &str) -> Result<String> {- let binding = key(bot, session);- let mut effective = hub.store.kv_get(&binding).filter(|v| !v.is_empty()).unwrap_or_else(|| session.to_owned());- for attempt in 0..2 {- let result =- engram::call(bot, "mem_session_start", &json!({"id":effective, "directory":engram::directory(bot)?}))- .await?;- if result["isError"] != true {- hub.store.kv_set(&binding, &effective)?;- return Ok(effective);- }- if attempt == 0 && payload(&result)["error_code"] == "session_already_ended" {- effective = format!("{session}:resume:{}", uuid::Uuid::new_v4());- } else {- anyhow::bail!("Engram session registration failed: {}", result["content"]);- }- }- anyhow::bail!("could not register memory session")-}--pub async fn bound_args(hub: &Arc<Hub>, bot: &str, lane: Option<&str>, name: &str, args: &Value) -> Result<Value> {- let row = hub.store.bot(bot)?.filter(|b| !b.deleted).context("unknown bot")?;- let sid = match lane.filter(|s| !s.is_empty()) {- Some(root) => hub.store.kv_get(&lane_key("session", bot, &Lane::in_thread(bot, root))),- None => row.session_id,- }- .filter(|s| !s.is_empty());- let mut args = args.clone();- ensure!(args.is_object(), "memory arguments must be an object");- // Search/management still works without a running agent (e.g. the settings screen).- if let Some(sid) = sid {- let effective = begin(hub, bot, &sid).await?;- if matches!(name, "mem_save" | "mem_save_prompt" | "mem_session_summary" | "mem_capture_passive") {- args["session_id"] = effective.clone().into();- }- if matches!(name, "mem_session_start" | "mem_session_end") {- args["id"] = effective.into();- }- }- if name == "mem_session_start" {- args["directory"] = json!(engram::directory(bot)?);- }- if args.get("project").is_some() {- args["project"] = engram::project(bot).into();- }- Ok(args)-}--pub async fn capture_prompt(bot: &str, session: &str, text: &str) -> Result<()> {- let result = engram::call(- bot,- "mem_save_prompt",- &json!({"session_id":session, "content":text, "project":engram::project(bot)}),- )- .await?;- ensure!(result["isError"] != true, "Engram prompt capture failed: {}", result["content"]);- Ok(())-}--/// An explicit edit/forget must reach non-Claude harnesses too, even mid-session.-pub fn change_notice(hub: &Hub, bot: &str, session: &str) -> Option<String> {- let revision = super::maintenance::revision(&hub.store, bot);- let key = format!("memory.announced.{bot}.{session}");- if hub.store.kv_get(&key).unwrap_or_default() == revision {- return None;- }- Some("<memory_update>Saved memories have changed. Earlier memory excerpts may be outdated or deleted. Call mem_context and mem_search before relying on them; respect removals and corrections.</memory_update>".into())-}--/// A notice is acknowledged only after the prompt was delivered to the harness.-pub fn mark_announced(hub: &Hub, bot: &str, session: &str, revision: &str) -> Result<()> {- hub.store.kv_set(&format!("memory.announced.{bot}.{session}"), revision)-}diff --git a/host/src/chat/memory/maintenance.rs b/host/src/chat/memory/maintenance.rsdeleted file mode 100644index 706ffdab..00000000--- a/host/src/chat/memory/maintenance.rs+++ /dev/null@@ -1,37 +0,0 @@-//! Serialize forgetting against keeper writes, and invalidate every session's memory snapshot.--use crate::{LockExt, store::Store};-use anyhow::Result;-use std::{- collections::HashMap,- sync::{Arc, LazyLock, Mutex},-};--static LOCKS: LazyLock<Mutex<HashMap<String, Arc<tokio::sync::Mutex<()>>>>> =- LazyLock::new(|| Mutex::new(HashMap::new()));--pub fn lock(bot: &str) -> Arc<tokio::sync::Mutex<()>> {- LOCKS.locked().entry(bot.to_owned()).or_default().clone()-}--pub fn keeper_lock(bot: &str) -> Arc<tokio::sync::Mutex<()>> {- static KEEPERS: LazyLock<Mutex<HashMap<String, Arc<tokio::sync::Mutex<()>>>>> =- LazyLock::new(|| Mutex::new(HashMap::new()));- KEEPERS.locked().entry(bot.to_owned()).or_default().clone()-}--pub fn revision(store: &Store, bot: &str) -> String {- store.kv_get(&format!("memory.revision.{bot}")).unwrap_or_default()-}--/// A correction (edit, forget, clear): it supersedes what the keeper is extracting from older-/// context, sessions are told memory changed, and their instructions re-render.-pub fn changed(store: &Store, bot: &str) -> Result<()> {- store.kv_set(&format!("memory.revision.{bot}"), &uuid::Uuid::new_v4().to_string())?;- crate::chat::context::invalidate(store, bot)-}--pub fn discard_pending(store: &Store, bot: &str) -> Result<()> {- store.kv_set(&format!("memory.unprocessed.{bot}"), "[]")?;- super::set_pending_episode(store, bot, &[])-}diff --git a/host/src/chat/memory/manage.rs b/host/src/chat/memory/manage.rsdeleted file mode 100644index 657d0bc4..00000000--- a/host/src/chat/memory/manage.rs+++ /dev/null@@ -1,164 +0,0 @@-//! Memory management shared by the phone, desktop and terminal clients.--use super::{Memory, maintenance};-use crate::hub::Hub;-use anyhow::{Context, Result, bail, ensure};-use serde_json::{Value, json};-use std::sync::Arc;--pub async fn call(hub: &Arc<Hub>, method: &str, args: &Value) -> Result<Value> {- let bot = args["botId"].as_str().context("botId is required")?.to_owned();- let row = hub.store.bot(&bot)?.filter(|b| !b.deleted && !b.config.is_group()).context("unknown bot")?;- let clears = method == "clearMemory";- let corrects = matches!(method, "saveMemory" | "forgetMemory");- // Not corrections: the keeper's work stands, only what the instructions show changes.- let reorders = matches!(method, "pinMemory" | "importMemory");- let mutation = maintenance::lock(&bot);- let _guard = mutation.lock().await;- if clears {- // Invalidate before deletion too: partial native failures must not resurrect old queued facts.- maintenance::changed(&hub.store, &bot)?;- maintenance::discard_pending(&hub.store, &bot)?;- }- if method == "exportMemory" {- let text = serde_json::to_string_pretty(&super::engram::export(&bot).await?)?;- if text.len() <= 384 * 1024 {- return Ok(json!({"json":text}));- }- let id = uuid::Uuid::new_v4().to_string();- let root = crate::chat::uploads::root(&row.config);- let upload = id.clone();- tokio::task::spawn_blocking(move || {- crate::chat::uploads::append(&root, &upload, "engram-backup.json", 0, text.as_bytes(), true)- })- .await??;- return Ok(json!({"uploadId":id}));- }- if method == "importMemory" {- let data = if let Some(upload) = args["uploadId"].as_str() {- let root = crate::chat::uploads::root(&row.config);- let upload = upload.to_owned();- tokio::task::spawn_blocking(move || -> Result<Value> {- let (path, _) = crate::chat::uploads::resolve(&root, &upload)?;- ensure!(std::fs::metadata(&path)?.len() <= 100 * 1024 * 1024, "memory backup exceeds 100 MB");- Ok(serde_json::from_slice(&std::fs::read(path)?)?)- })- .await??- } else {- match args["json"].as_str() {- Some(text) => serde_json::from_str(text).context("invalid Engram JSON backup")?,- None => args["data"].clone(),- }- };- let result = super::engram::import(&bot, &data).await?;- crate::chat::context::invalidate(&hub.store, &bot)?;- return Ok(result);- }- let (method, args, id) = (method.to_owned(), args.clone(), bot.clone());- let result = tokio::task::spawn_blocking(move || {- let memory = Memory::for_bot(&id)?;- dispatch(&memory, &method, &args)- })- .await??;- if clears || corrects {- maintenance::changed(&hub.store, &bot)?;- maintenance::discard_pending(&hub.store, &bot)?;- } else if reorders {- crate::chat::context::invalidate(&hub.store, &bot)?;- }- Ok(result)-}--fn dispatch(memory: &Memory, method: &str, args: &Value) -> Result<Value> {- match method {- "memory" => listing(memory, args),- "saveMemory" => save(memory, args),- "memoryDetail" => {- let fact = memory.find(required(args, "id")?)?;- let id = fact.id.parse::<i64>()?;- let mut history_args = json!({"id":id, "include_history":true, "project":fact.project});- if let Some(cursor) = args.get("historyCursor") {- history_args["history_cursor"] = cursor.clone();- }- let history = memory.tool("mem_get_observation", &history_args)?;- let timeline = memory.tool("mem_timeline", &json!({"observation_id":id, "project":fact.project}))?;- Ok(json!({"fact":fact, "history":payload(&history), "timeline":payload(&timeline)}))- }- "pinMemory" => {- let fact = memory.find(required(args, "id")?)?;- memory.tool(- if args["pinned"] == true { "mem_pin" } else { "mem_unpin" },- &json!({"id":fact.id.parse::<i64>()?}),- )?;- Ok(json!({}))- }- "reviewMemory" => {- let fact = memory.find(required(args, "id")?)?;- memory.tool(- "mem_review",- &json!({"action":"mark_reviewed", "observation_id":fact.id.parse::<i64>()?, "project":fact.project}),- )?;- Ok(json!({}))- }- "forgetMemory" => Ok(json!({"removed":memory.remove(required(args, "id")?)?})),- "clearMemory" => {- memory.clear()?;- Ok(json!({}))- }- _ => bail!("unknown memory operation"),- }-}--fn listing(memory: &Memory, args: &Value) -> Result<Value> {- let query = args["query"].as_str().unwrap_or_default().trim();- let offset = usize::try_from(args["offset"].as_u64().unwrap_or(0)).context("offset is too large")?;- // Old clients only send botId and have no Load more control.- let legacy = ["query", "filter", "offset", "limit"].iter().all(|key| args.get(key).is_none());- let limit = if legacy { usize::MAX } else { usize::try_from(args["limit"].as_u64().unwrap_or(50).clamp(1, 100))? };- let filter = args["filter"].as_str().unwrap_or("all");- let page = memory.browse(query, filter, limit, offset)?;- let next = (offset + page.facts.len() < page.total).then_some(offset + page.facts.len());- Ok(json!({"location":memory.location(), "engine":"engram", "version":super::engram::VERSION,- "facts":page.facts, "total":page.total, "nextOffset":next,- "query":query, "filter":filter, "offset":offset}))-}--fn save(memory: &Memory, args: &Value) -> Result<Value> {- let title = required(args, "title")?.trim();- let content = required(args, "content")?.trim();- ensure!(!title.is_empty() && !content.is_empty(), "title and content cannot be empty");- let scope = args["scope"].as_str().unwrap_or("project");- ensure!(["project", "personal", "global"].contains(&scope), "invalid memory scope");- let mut fields = json!({"title":title, "content":content, "scope":scope,- "type":args["memoryType"].as_str().unwrap_or("learning")});- if let Some(topic) = args["topicKey"].as_str() {- fields["topic_key"] = topic.into();- }- let name = if let Some(id) = args["id"].as_str() {- let existing = memory.find(id)?;- fields["id"] = id.parse::<i64>()?.into();- fields["expected_project"] = existing.project.into();- "mem_update"- } else {- fields["project"] = memory.project().into();- fields["capture_prompt"] = false.into();- fields["session_id"] = format!("{}:codync-background", memory.project()).into();- "mem_save"- };- Ok(payload(&memory.tool(name, &fields)?))-}--fn required<'a>(args: &'a Value, key: &str) -> Result<&'a str> {- args[key].as_str().with_context(|| format!("{key} is required"))-}--pub(super) fn payload(result: &Value) -> Value {- let text = result["content"]- .as_array()- .into_iter()- .flatten()- .filter_map(|v| v["text"].as_str())- .collect::<Vec<_>>()- .join("\n");- serde_json::from_str(&text).unwrap_or_else(|_| json!({"result":text}))-}diff --git a/host/src/chat/memory/mod.rs b/host/src/chat/memory/mod.rsindex 2ed45d77..5b213e5a 100644--- a/host/src/chat/memory/mod.rs+++ b/host/src/chat/memory/mod.rs@@ -1,135 +1,102 @@-//! Per-bot Engram memory with automatic extraction, recall and native MCP tools.-//! Transcripts stay in Codync; all durable memory lives in Engram's SQLite database.+//! Bot memory ([design](../../../../docs/design/bot-memory.md)): it records the person, not the work. The rules+//! (store, checks, opening, dreaming) live in the `codync_mem` crate; this module runs them in the host: the+//! session opening, the bot's memory tools, the client API, the chat lines that show and undo a bot's writes,+//! dreaming, and the one-time move from the old Engram memory.+mod api;mod dates;-mod engram;-mod extract;-mod facts;-mod keeper;-pub(crate) mod lifecycle;-pub(crate) mod maintenance;-pub(crate) mod manage;-mod read;-mod search;+mod dream;+mod import;+mod notices;+mod tools;-pub use extract::{EpisodeTurn, is_memorable, set_pending_episode};-pub use facts::Memory;-pub(crate) use keeper::one_shot;-pub use keeper::{Exchange, KeeperEvent, spawn_keeper};-pub use search::{INSTRUCTIONS, available_tools, call, tools};+pub use api::call;+pub use dream::{DreamEvent, spawn_dreamer};+pub use import::import_old_memory;+pub use tools::{INSTRUCTIONS, call as tool_call, tools};-use anyhow::Result;-use facts::{Fact, Recall, ymd};-use std::path::Path;-use std::time::Duration;+use crate::hub::Hub;+use crate::store::{BotConfig, Store};-/// Profile facts shown in the prompt; past this the keeper consolidates the profile.-const PROFILE_PROMPT_LIMIT: usize = 100;-/// What a consolidation merges the profile down to, leaving room to grow.-const PROFILE_TARGET: usize = 60;-/// Log facts shown in the prompt (newest first), within [`RECENT_CHAR_BUDGET`].-pub const RECENT_PROMPT_LIMIT: usize = 30;-const RECENT_CHAR_BUDGET: usize = 4_000;-const MAX_FACT_CHARS: usize = 500;-const RELEVANT_LIMIT: usize = 10;-/// An exchange side is cut to this before it goes to the keeper.-const EXCHANGE_CHARS: usize = 8_000;-const KEEPER_TIMEOUT: Duration = Duration::from_secs(180);-/// Quiet time after the last exchange before the keeper runs over the queued ones.-const KEEPER_IDLE: Duration = Duration::from_secs(5 * 60);-/// Queued exchanges that make the keeper run without waiting, and the most one run takes.-const KEEPER_BATCH: usize = 8;-/// Exchanges kept waiting while the keeper fails; older ones are dropped.-const KEEPER_QUEUE_LIMIT: usize = 4 * KEEPER_BATCH;+/// Whether a bot uses the memory all bots share. Unless the user chose, a bot in its own workspace does and one+/// working in a project folder doesn't.+pub fn shared(cfg: &BotConfig) -> bool {+ cfg.shared_memory.unwrap_or_else(|| crate::agent::workspace::is_managed(cfg))+}-const NOTE_PREFIX: &str = "[note] ";-const NONE: &str = "NONE";+/// The memory section of a bot's instructions.+pub fn opening(store: &Store, cfg: &BotConfig) -> String {+ match store.memory_read(|c| codync_mem::opening::load(c, &cfg.id, shared(cfg))) {+ Ok(opening) => opening.render(),+ Err(error) => {+ tracing::warn!(bot = %cfg.id, error = format!("{error:#}"), "couldn't read memory");+ String::new()+ }+ }+}-fn fact_line(f: &Fact) -> String {- format!("- (learned {}) {}", ymd(f.created_at), f.content)+#[derive(Clone, Copy, Debug, PartialEq, Eq)]+pub enum Change {+ /// The user changed memory: every session of the bot hears it on its next turn.+ User,+ /// A bot or dreaming wrote it. The session that wrote it knows; others pick it up at their next compaction or+ /// new session.+ Background,}-/// The memory section of the bot's instructions, and whether it holds any facts.-pub fn render(recall: &Recall, location: &Path) -> (String, bool) {- let mut lines = vec![- "Memory: durable facts you have learned about the user and their world.".to_owned(),- "These persist across every session with this bot, even after a new session starts. Rely on them so you stay consistent and avoid re-asking what you already know.".to_owned(),- format!(- "Your memory is managed by Engram in {}. Use its memory tools to read and write it; do not edit the database or the legacy Markdown backup directly.",- location.display()- ),- "At the start of related work, call mem_context and mem_search to recall prior knowledge; retrieve full observations before relying on previews. Save durable preferences, decisions, discoveries and fixes proactively with mem_save. Reuse a stable topic_key for evolving knowledge. When asked to remember or forget, perform the memory operation before confirming success. A background keeper also extracts durable facts from completed user conversations.".to_owned(),- "Before ending a session, save a mem_session_summary. After compaction, recover context with mem_context. Use mem_review, mem_judge and mem_compare to review stale or conflicting knowledge; never treat a proposed conflict as resolved until it has been judged. Use search_history for original chat messages. Only this bot's memory is available through these tools.".to_owned(),- ];- if !recall.profile.is_empty() {- lines.push("About the user:".into());- lines.extend(recall.profile.iter().map(fact_line));- }- if !recall.recent.is_empty() {- lines.push("Recently:".into());- let mut budget = RECENT_CHAR_BUDGET;- let mut shown = 0;- for f in &recall.recent {- // Characters, not bytes, so CJK facts get the same room; a long one (a session summary) is clipped.- let mut line = fact_line(f);- if line.chars().count() > MAX_FACT_CHARS {- line = line.chars().take(MAX_FACT_CHARS).chain(['…']).collect();- }- let size = line.chars().count();- if size > budget {- break;- }- budget -= size;- lines.push(line);- shown += 1;- }- let omitted = recall.recent.len() - shown;- if omitted > 0 {- lines.push(format!("({omitted} more recent memories — use mem_search to retrieve them.)"));+/// After memory changed in `bot`'s own layer, or in the shared one (`None`): clients fetch it again, and a change+/// the user made reaches the bots' sessions.+pub fn changed(hub: &Hub, bot: Option<&str>, change: Change) {+ let bots: Vec<String> = match bot {+ Some(bot) => vec![bot.to_owned()],+ None => hub+ .store+ .bots()+ .map(|rows| rows.into_iter().filter(|r| !r.deleted && !r.config.is_group()).map(|r| r.config.id).collect())+ .unwrap_or_default(),+ };+ let revision = hub.store.memory_rev().to_string();+ for id in &bots {+ if change == Change::User+ && let Err(error) = hub+ .store+ .kv_set(&revision_key(id), &revision)+ .and_then(|()| crate::chat::context::invalidate(&hub.store, id))+ {+ tracing::warn!(bot = %id, error = format!("{error:#}"), "couldn't flag the memory change");}+ hub.emit_bot(id);}- let has_facts = !recall.profile.is_empty() || !recall.recent.is_empty();- if !has_facts {- lines.push("No facts recorded yet.".into());- }- (lines.join("\n"), has_facts)}-/// Ensure installation, migration and the native memory process before starting a session.-pub async fn prepare(bot_id: &str) -> Result<()> {- engram::prepare(bot_id).await+fn revision_key(bot: &str) -> String {+ format!("memory.revision.{bot}")}-#[cfg(test)]-mod tests {- use super::*;+fn announced_key(bot: &str, session: &str) -> String {+ format!("memory.announced.{bot}.{session}")+}- fn fact(content: &str) -> Fact {- Fact {- id: "1".into(),- content: content.into(),- created_at: 0,- kind: facts::Kind::Log,- title: String::new(),- memory_type: "discovery".into(),- scope: "project".into(),- project: String::new(),- topic_key: None,- session_id: String::new(),- pinned: false,- review_after: None,- revision_count: 1,- source: None,- }+/// When the user changed memory since this session last heard: the whole memory as it is now, to put before the+/// next message, and the revision to acknowledge once it was delivered.+pub fn change_notice(store: &Store, cfg: &BotConfig, session: &str) -> Option<(String, String)> {+ let revision = store.kv_get(&revision_key(&cfg.id)).unwrap_or_default();+ if revision.is_empty() || store.kv_get(&announced_key(&cfg.id, session)).unwrap_or_default() == revision {+ return None;}+ let notice = format!(+ "<memory_update>\nThe user changed your memory. This is all of it now and replaces any earlier memory in this conversation.\n{}\n</memory_update>",+ opening(store, cfg)+ );+ Some((notice, revision))+}- #[test]- fn recent_memories_share_a_character_budget() {- let mut recent = vec![fact(&"摘".repeat(9_000))];- recent.extend((0..20).map(|i| fact(&format!("使用者偏好繁體中文 {i}"))));- let (text, _) = render(&Recall { profile: vec![], recent }, Path::new("/tmp"));- let facts: Vec<&str> = text.lines().filter(|l| l.starts_with("- (learned")).collect();- assert!(facts.iter().all(|l| l.chars().count() <= MAX_FACT_CHARS + 1), "a long memory is clipped");- assert_eq!(facts.len(), 21, "short CJK facts still fit after a long summary");- }+/// The session heard memory up to `revision`: from a notice, or from instructions rendered just now.+pub fn mark_announced(store: &Store, bot: &str, session: &str, revision: &str) -> anyhow::Result<()> {+ store.kv_set(&announced_key(bot, session), revision)+}++/// The revision a fresh session's instructions already include.+pub fn current_revision(store: &Store, bot: &str) -> String {+ store.kv_get(&revision_key(bot)).unwrap_or_default()}diff --git a/host/src/chat/memory/notices.rs b/host/src/chat/memory/notices.rsnew file mode 100644index 00000000..fcbaa768--- /dev/null+++ b/host/src/chat/memory/notices.rs@@ -0,0 +1,253 @@+//! A bot's memory writes show in its chat: a "Remembered" line the user can undo, or, for a bot that asks before+//! acting, a card the user approves or declines. The line is an ordinary notice whose `memory` field says what+//! happened, so an app that doesn't know it still shows the text.++use super::Change;+use crate::hub::Hub;+use crate::store::{BotConfig, EntryKind, Lane};+use anyhow::{Result, anyhow, bail};+use codync_mem::store::{self as mem, Author, Entry, Error, New, Writer};+use serde::{Deserialize, Serialize};+use serde_json::{Value, json};++#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]+#[serde(tag = "action", rename_all = "camelCase")]+pub enum Action {+ Remember {+ text: String,+ #[serde(default)]+ shared: bool,+ },+ Revise {+ entry: i64,+ text: String,+ },+ Archive {+ entry: i64,+ },+}++#[derive(Clone, Copy, Debug, Serialize, Deserialize, PartialEq, Eq)]+#[serde(rename_all = "camelCase")]+pub enum Status {+ /// Written; the user can undo it.+ Done,+ /// Waiting for the user's approval.+ Pending,+ Accepted,+ Declined,+ Undone,+ /// Approved, but the write was refused (the layer is full, say).+ Failed,+}++/// The runs of a user message the user typed themselves: not pasted, quoted or forwarded text.+pub fn typed(data: &Value) -> Vec<String> {+ let pasted: Vec<String> =+ data["pasted"].as_array().into_iter().flatten().filter_map(|p| p.as_str().map(str::to_owned)).collect();+ codync_mem::evidence::typed(data["text"].as_str().unwrap_or_default(), &pasted)+}++/// Writes now, and says so in the chat with an undo.+pub fn apply(+ hub: &Hub,+ cfg: &BotConfig,+ lane: Option<Lane>,+ message: Option<String>,+ action: &Action,+) -> Result<Entry> {+ let lane = lane.unwrap_or_else(|| Lane::main(&cfg.id));+ let (entry, previous) = write(hub, cfg, &lane, message.clone(), action)?;+ let card = Card { status: Status::Done, entry: Some(entry.id), previous, message, error: None };+ post(hub, cfg, &lane, action, &card);+ super::changed(hub, entry.bot.as_deref(), Change::Background);+ Ok(entry)+}++/// Asks the user first: a card in the chat. The text is checked now, so the bot hears a refusal at once, and+/// again when the user approves.+pub fn propose(hub: &Hub, cfg: &BotConfig, lane: Option<Lane>, message: Option<String>, action: &Action) -> Result<()> {+ let lane = lane.unwrap_or_else(|| Lane::main(&cfg.id));+ let shared = super::shared(cfg);+ let previous = match action {+ Action::Remember { text, shared: wants } => {+ let layer = (!(*wants && shared)).then_some(cfg.id.as_str());+ hub.store.memory_read(|c| mem::admissible(c, Writer::Bot(&cfg.id), layer, text))?;+ None+ }+ Action::Revise { entry, .. } | Action::Archive { entry } => {+ if let Action::Revise { text, .. } = action {+ codync_mem::check::text(text)?;+ }+ let current = hub.store.memory_read(|c| visible(c, *entry, &cfg.id, shared))?;+ if current.author == Author::User {+ return Err(Error::UsersOwn.into());+ }+ Some(current.text)+ }+ };+ let card = Card { status: Status::Pending, entry: None, previous, message, error: None };+ post(hub, cfg, &lane, action, &card);+ Ok(())+}++/// The user approved or declined a card.+pub fn answer(hub: &Hub, notice_id: &str, accept: bool) -> Result<Value> {+ let (notice, cfg, action, mut card) = open(hub, notice_id)?;+ if card.status != Status::Pending {+ bail!("This card was already answered.");+ }+ let lane = Lane { chat: notice.bot_id.clone(), thread: notice.thread_id.clone() };+ if accept {+ match write(hub, &cfg, &lane, card.message.clone(), &action) {+ Ok((entry, previous)) => {+ card.status = Status::Accepted;+ card.entry = Some(entry.id);+ card.previous = previous.or(card.previous);+ super::changed(hub, entry.bot.as_deref(), Change::User);+ }+ Err(error) => {+ card.status = Status::Failed;+ card.error = Some(format!("{error:#}"));+ }+ }+ } else {+ card.status = Status::Declined;+ if let Action::Remember { text, shared } = &action {+ let layer = (!(*shared && super::shared(&cfg))).then_some(cfg.id.as_str());+ hub.store.memory_write(|c, _, _| mem::forbid(c, layer, text))?;+ }+ }+ Ok(update(hub, ¬ice, &cfg, &action, &card))+}++/// The user took back what a bot wrote.+pub fn undo(hub: &Hub, notice_id: &str) -> Result<Value> {+ let (notice, cfg, action, mut card) = open(hub, notice_id)?;+ let id = card.entry.filter(|_| matches!(card.status, Status::Done | Status::Accepted));+ let id = id.ok_or_else(|| anyhow!("There's nothing to undo."))?;+ let previous = card.previous.clone();+ let layer = hub.store.memory_write(|c, now, rev| {+ let entry = mem::get(c, id)?.ok_or(Error::NotFound)?;+ match &action {+ Action::Remember { .. } => {+ mem::forget(c, id)?;+ }+ Action::Revise { .. } => {+ mem::revise(c, Writer::User, id, previous.as_deref().unwrap_or(&entry.text), now, rev)?;+ }+ Action::Archive { .. } => {+ mem::activate(c, id, now, rev)?;+ }+ }+ Ok(entry.bot)+ })?;+ card.status = Status::Undone;+ super::changed(hub, layer.as_deref(), Change::User);+ Ok(update(hub, ¬ice, &cfg, &action, &card))+}++/// What a card records beside its action.+#[derive(Clone, Debug, Serialize, Deserialize)]+struct Card {+ status: Status,+ /// The memory entry written.+ #[serde(default, skip_serializing_if = "Option::is_none")]+ entry: Option<i64>,+ /// The text before a revision, or of the entry being archived.+ #[serde(default, skip_serializing_if = "Option::is_none")]+ previous: Option<String>,+ /// The user's message the turn answered: where the fact was learned.+ #[serde(default, skip_serializing_if = "Option::is_none")]+ message: Option<String>,+ #[serde(default, skip_serializing_if = "Option::is_none")]+ error: Option<String>,+}++fn write(+ hub: &Hub,+ cfg: &BotConfig,+ lane: &Lane,+ message: Option<String>,+ action: &Action,+) -> Result<(Entry, Option<String>)> {+ let bot = cfg.id.as_str();+ let shared = super::shared(cfg);+ let room = (lane.chat != cfg.id).then(|| lane.chat.clone());+ let evidence: Vec<String> = message.into_iter().collect();+ let writer = Writer::Bot(bot);+ hub.store.memory_write(|c, now, rev| match action {+ Action::Remember { text, shared: wants } => {+ let layer = (!(*wants && shared)).then_some(bot);+ let new = New { bot: layer, text, room: room.as_deref(), evidence: &evidence };+ mem::add(c, writer, new, now, rev).map(|e| (e, None))+ }+ Action::Revise { entry, text } => {+ let old = visible(c, *entry, bot, shared)?;+ mem::revise(c, writer, *entry, text, now, rev).map(|e| (e, Some(old.text)))+ }+ Action::Archive { entry } => {+ let old = visible(c, *entry, bot, shared)?;+ mem::archive(c, writer, *entry, None, now, rev).map(|e| (e, Some(old.text)))+ }+ })+}++/// An entry in this bot's own layer, or in the shared one when the bot uses it.+fn visible(c: &rusqlite::Connection, id: i64, bot: &str, shared: bool) -> mem::Result<Entry> {+ let entry = mem::get(c, id)?.ok_or(Error::NotFound)?;+ if entry.bot.as_deref() == Some(bot) || (entry.bot.is_none() && shared) { Ok(entry) } else { Err(Error::NotFound) }+}++fn post(hub: &Hub, cfg: &BotConfig, lane: &Lane, action: &Action, card: &Card) {+ let turn = hub.store.max_turn(&lane.chat);+ hub.add_entry(lane, EntryKind::Notice, turn, &data(cfg, action, card));+}++fn update(hub: &Hub, notice: &crate::store::Entry, cfg: &BotConfig, action: &Action, card: &Card) -> Value {+ let mut data = notice.data.clone();+ let fresh = self::data(cfg, action, card);+ data["text"] = fresh["text"].clone();+ data["memory"] = fresh["memory"].clone();+ hub.set_entry(¬ice.id, &data).map_or(Value::Null, |e| json!({"entry": e}))+}++fn data(cfg: &BotConfig, action: &Action, card: &Card) -> Value {+ let mut memory = serde_json::to_value(action).expect("an action is plain data");+ if let (Some(memory), Value::Object(card)) =+ (memory.as_object_mut(), serde_json::to_value(card).expect("a card is plain data"))+ {+ memory.extend(card);+ }+ json!({"text": summary(action, card), "style": "info", "author": cfg.id, "memory": memory})+}++/// What an app that doesn't know memory cards shows.+fn summary(action: &Action, card: &Card) -> String {+ let previous = card.previous.as_deref().unwrap_or_default();+ let what = match action {+ Action::Remember { text, .. } | Action::Revise { text, .. } => text.as_str(),+ Action::Archive { .. } => previous,+ };+ match (card.status, action) {+ (Status::Pending, Action::Remember { .. }) => format!("Wants to remember: {what}"),+ (Status::Pending, Action::Revise { .. }) => format!("Wants to update memory: {previous} → {what}"),+ (Status::Pending, Action::Archive { .. }) => format!("Wants to archive from memory: {what}"),+ (Status::Done | Status::Accepted, Action::Remember { .. }) => format!("Remembered: {what}"),+ (Status::Done | Status::Accepted, Action::Revise { .. }) => format!("Updated memory: {what}"),+ (Status::Done | Status::Accepted, Action::Archive { .. }) => format!("Archived from memory: {what}"),+ (Status::Declined, _) => format!("Declined: {what}"),+ (Status::Undone, _) => format!("Undone: {what}"),+ (Status::Failed, _) => format!("Couldn't save to memory: {what}"),+ }+}++fn open(hub: &Hub, notice_id: &str) -> Result<(crate::store::Entry, BotConfig, Action, Card)> {+ let notice = hub.store.entry(notice_id).ok_or_else(|| anyhow!("unknown card"))?;+ let memory = ¬ice.data["memory"];+ let action: Action = serde_json::from_value(memory.clone()).map_err(|_| anyhow!("not a memory card"))?;+ let card: Card = serde_json::from_value(memory.clone())?;+ let bot = notice.data["author"].as_str().unwrap_or(¬ice.bot_id);+ let cfg = hub.store.bot(bot)?.filter(|b| !b.deleted).ok_or_else(|| anyhow!("unknown bot"))?.config;+ Ok((notice, cfg, action, card))+}diff --git a/host/src/chat/memory/read.rs b/host/src/chat/memory/read.rsdeleted file mode 100644index 226c5d79..00000000--- a/host/src/chat/memory/read.rs+++ /dev/null@@ -1,148 +0,0 @@-//! Bounded read-only queries over Engram's own indexes. Never creates or changes schema.--use super::facts::{Fact, Kind};-use anyhow::{Result, bail};-use rusqlite::{Connection, Row, types::Value};-use sha2::{Digest, Sha256};-use std::fmt::Write as _;--const COLUMNS: &str = "id, content, created_at, title, type, scope, project, topic_key,- session_id, pinned, review_after, revision_count, tool_name";-const ORDER: &str = "pinned DESC, (scope = 'personal') DESC, created_at DESC, id DESC";--pub struct Page {- pub facts: Vec<Fact>,- pub total: usize,-}--pub fn browse(db: &mut Connection, query: &str, filter: &str, limit: usize, offset: usize) -> Result<Page> {- let (condition, mut args) = conditions(query, filter)?;- let tx = db.transaction()?;- let total: i64 = tx.query_row(- &format!("SELECT COUNT(*) FROM observations WHERE {condition}"),- rusqlite::params_from_iter(&args),- |r| r.get(0),- )?;- args.push(Value::Integer(i64::try_from(limit).unwrap_or(i64::MAX)));- args.push(Value::Integer(i64::try_from(offset)?));- let facts = select(&tx, &format!("{condition} ORDER BY {ORDER} LIMIT ? OFFSET ?"), args)?;- tx.commit()?;- Ok(Page { facts, total: usize::try_from(total)? })-}--pub fn list(db: &Connection, filter: &str, limit: usize) -> Result<Vec<Fact>> {- let (condition, mut args) = conditions("", filter)?;- args.push(Value::Integer(i64::try_from(limit).unwrap_or(i64::MAX)));- select(db, &format!("{condition} ORDER BY {ORDER} LIMIT ?"), args)-}--pub fn find(db: &Connection, id: i64) -> Result<Option<Fact>> {- Ok(select(db, "deleted_at IS NULL AND id = ?", vec![Value::Integer(id)])?.pop())-}--/// Recall older relevant facts using Engram's FTS index instead of scanning a recent window.-pub fn relevant(db: &Connection, text: &str, limit: usize) -> Result<Vec<Fact>> {- let mut terms = Vec::new();- let mut seen = std::collections::HashSet::new();- for word in text.split(|c: char| !c.is_alphanumeric()) {- let chars: Vec<char> = word.to_lowercase().chars().collect();- let candidates = if chars.iter().any(|c| ('\u{3400}'..='\u{9fff}').contains(c)) {- chars.windows(3).map(|w| w.iter().collect::<String>()).collect::<Vec<_>>()- } else if chars.len() >= 3 {- vec![chars.iter().collect()]- } else {- Vec::new()- };- for term in candidates {- if seen.insert(term.clone()) {- terms.push(format!("\"{}\"", term.replace('"', "\"\"")));- }- if terms.len() == 64 {- break;- }- }- if terms.len() == 64 {- break;- }- }- if terms.is_empty() {- return Ok(Vec::new());- }- select(- db,- "deleted_at IS NULL AND id IN (SELECT rowid FROM observations_fts WHERE observations_fts MATCH ? ORDER BY rank LIMIT ?)",- vec![Value::Text(terms.join(" OR ")), Value::Integer(i64::try_from(limit)?)],- )-}--pub fn same_content(db: &Connection, content: &str) -> Result<Option<Fact>> {- Ok(select(db, "deleted_at IS NULL AND normalized_hash = ? LIMIT 1", vec![Value::Text(content_hash(content))])?- .pop())-}--pub fn content_hash(content: &str) -> String {- let normalized = content.split_whitespace().collect::<Vec<_>>().join(" ").to_lowercase();- let mut hash = String::with_capacity(64);- for b in Sha256::digest(normalized.as_bytes()) {- let _ = write!(hash, "{b:02x}");- }- hash-}--fn select(db: &Connection, condition: &str, args: Vec<Value>) -> Result<Vec<Fact>> {- let mut query = db.prepare(&format!("SELECT {COLUMNS} FROM observations WHERE {condition}"))?;- Ok(query.query_map(rusqlite::params_from_iter(args), fact)?.collect::<rusqlite::Result<Vec<_>>>()?)-}--fn conditions(query: &str, filter: &str) -> Result<(String, Vec<Value>)> {- let mut clauses = vec!["deleted_at IS NULL".to_owned()];- match filter {- "profile" => clauses.push("scope = 'personal'".into()),- "log" => clauses.push("scope != 'personal'".into()),- "pinned" => clauses.push("pinned = 1".into()),- "review" => clauses.push("datetime(review_after) <= datetime('now')".into()),- "all" => {}- _ => bail!("unknown memory filter"),- }- let mut args = Vec::new();- let mut indexed = Vec::new();- // Engram uses a trigram FTS5 index. One/two-character terms need literal LIKE;- // escaping wildcards keeps punctuation and short CJK searches predictable.- for term in query.split_whitespace().take(16) {- if term.chars().count() >= 3 {- indexed.push(format!("\"{}\"", term.replace('"', "\"\"")));- } else {- let escaped = term.replace('\\', "\\\\").replace('%', "\\%").replace('_', "\\_");- clauses.push("(title || ' ' || content || ' ' || COALESCE(topic_key, '')) LIKE ? ESCAPE '\\'".into());- args.push(Value::Text(format!("%{escaped}%")));- }- }- if !indexed.is_empty() {- clauses.push("id IN (SELECT rowid FROM observations_fts WHERE observations_fts MATCH ?)".into());- args.push(Value::Text(indexed.join(" AND ")));- }- Ok((clauses.join(" AND "), args))-}--fn fact(row: &Row<'_>) -> rusqlite::Result<Fact> {- let scope: String = row.get(5)?;- let date: String = row.get(2)?;- let created_at = super::facts::parse_timestamp(&date)- .map_err(|e| rusqlite::Error::FromSqlConversionFailure(2, rusqlite::types::Type::Text, e.into()))?;- Ok(Fact {- id: row.get::<_, i64>(0)?.to_string(),- content: row.get(1)?,- created_at,- kind: if scope == "personal" { Kind::Profile } else { Kind::Log },- title: row.get(3)?,- memory_type: row.get(4)?,- scope,- project: row.get::<_, Option<String>>(6)?.unwrap_or_default(),- topic_key: row.get(7)?,- session_id: row.get(8)?,- pinned: row.get(9)?,- review_after: row.get(10)?,- revision_count: row.get(11)?,- source: row.get(12)?,- })-}diff --git a/host/src/chat/memory/search.rs b/host/src/chat/memory/search.rsdeleted file mode 100644index 9cf53789..00000000--- a/host/src/chat/memory/search.rs+++ /dev/null@@ -1,92 +0,0 @@-//! History search, the built-in `memory` MCP server.--use super::facts::ymd;-use crate::agent::acp;-use crate::hub::Hub;-use anyhow::{Result, anyhow, bail};-use serde_json::{Value, json};-use std::sync::Arc;--const SEARCH_DEFAULT: i64 = 10;-const SEARCH_MAX: i64 = 30;-const SEARCH_TERMS: usize = 8;-const SEARCH_TEXT_CHARS: usize = 1_500;--pub const INSTRUCTIONS: &str = "Use search_history to find what was actually said in your past chats with the user \-(their messages and yours, including threads), beyond what your memory notes kept. \-Search for distinctive words; every word must appear. Results are newest first.";--pub fn tools() -> Value {- json!([{- "name": "search_history",- "description": "Search your full chat history with the user for messages containing every given word (case-insensitive). Returns dated messages, newest first.",- "inputSchema": {- "type": "object",- "properties": {- "query": {"type": "string", "description": "Words that must all appear, separated by spaces."},- "limit": {"type": "integer", "description": "Maximum results (default 10, at most 30)."},- },- "required": ["query"],- },- "annotations": {"readOnlyHint": true},- }])-}--pub async fn call(hub: &Arc<Hub>, bot_id: &str, name: &str, args: &Value) -> Result<Value> {- hub.store.bot(bot_id)?.filter(|b| !b.deleted).ok_or_else(|| anyhow!("unknown bot"))?;- if name.starts_with("mem_") {- let read_only = matches!(- name,- "mem_search"- | "mem_context"- | "mem_timeline"- | "mem_get_observation"- | "mem_stats"- | "mem_current_project"- | "mem_list_projects"- | "mem_doctor"- | "mem_suggest_topic_key"- ) || (name == "mem_review" && args["action"] == "list");- let mutation = super::maintenance::lock(bot_id);- let _guard = if read_only { None } else { Some(mutation.lock().await) };- let result = super::engram::call(bot_id, name, args).await?;- if matches!(name, "mem_update" | "mem_delete" | "mem_merge_projects") && result["isError"] != true {- super::maintenance::changed(&hub.store, bot_id)?;- super::maintenance::discard_pending(&hub.store, bot_id)?;- }- return Ok(result);- }- if name != "search_history" {- bail!("unknown memory tool: {name}");- }- let bot = hub.store.bot(bot_id)?.filter(|b| !b.deleted).ok_or_else(|| anyhow!("unknown bot"))?;- let terms: Vec<String> =- args["query"].as_str().unwrap_or_default().split_whitespace().take(SEARCH_TERMS).map(str::to_owned).collect();- if terms.is_empty() {- bail!("query must contain at least one word");- }- let limit = args["limit"].as_i64().unwrap_or(SEARCH_DEFAULT).clamp(1, SEARCH_MAX);- let hub = hub.clone();- let id = bot.config.id.clone();- let rows = tokio::task::spawn_blocking(move || hub.store.search_messages(&id, &terms, limit)).await??;- let results: Vec<Value> = rows- .iter()- .map(|e| {- json!({- "date": ymd(e.created_at),- "from": if e.kind == crate::store::EntryKind::User.as_str() { "user" } else { "you" },- "inThread": e.thread_id.is_some(),- "text": acp::truncate(e.data["text"].as_str().unwrap_or_default(), SEARCH_TEXT_CHARS),- })- })- .collect();- Ok(json!({"results": results}))-}--pub async fn available_tools(hub: &Arc<Hub>, bot_id: &str) -> Result<Value> {- hub.store.bot(bot_id)?.filter(|b| !b.deleted).ok_or_else(|| anyhow!("unknown bot"))?;- let mut native = super::engram::tools(bot_id).await?;- let list = native["tools"].as_array_mut().ok_or_else(|| anyhow!("Engram returned no tools"))?;- list.extend(tools().as_array().into_iter().flatten().cloned());- Ok(native)-}diff --git a/host/src/chat/memory/tools.rs b/host/src/chat/memory/tools.rsnew file mode 100644index 00000000..8febfc79--- /dev/null+++ b/host/src/chat/memory/tools.rs@@ -0,0 +1,160 @@+//! The built-in `memory` MCP server: the bot's own memory tools, and search over its chat history.++use super::dates::ymd;+use super::notices::{self, Action};+use crate::agent::acp;+use crate::hub::Hub;+use crate::store::{BotConfig, Permission};+use anyhow::{Result, anyhow, bail};+use serde_json::{Value, json};+use std::sync::Arc;++const SEARCH_DEFAULT: i64 = 10;+const SEARCH_MAX: i64 = 30;+const SEARCH_TERMS: usize = 8;+const SEARCH_TEXT_CHARS: usize = 1_500;++pub const INSTRUCTIONS: &str = "Your memory holds what you know about the user, and every session opens with it. \+Save lasting facts about the user with remember, and keep entries true with revise_memory and archive_memory \+(the [id] is in your instructions). Project and code knowledge belongs in the project's own files, not here. \+search_memory finds archived entries; search_history finds what was actually said in your past chats.";++pub fn tools() -> Value {+ json!([+ {+ "name": "remember",+ "description": "Save one short fact about the user to your memory: who they are, their people, preferences, standing requests, plans or commitments. Use it when the user asks you to remember something or tells you something lasting about themselves. Write one self-contained fact in the user's language, with absolute dates. The user sees it in the chat and can undo it; a bot that asks before acting shows it to the user for approval first.",+ "inputSchema": {+ "type": "object",+ "properties": {+ "text": {"type": "string", "description": "The fact, in one short sentence."},+ "shared": {"type": "boolean", "description": "True for facts all the user's bots should know (name, time zone, language, important people, lasting preferences); false (default) keeps it with you."},+ },+ "required": ["text"],+ },+ },+ {+ "name": "revise_memory",+ "description": "Correct an entry of your memory, by the [id] your instructions show. Entries the user wrote can't be changed.",+ "inputSchema": {+ "type": "object",+ "properties": {+ "id": {"type": "integer"},+ "text": {"type": "string", "description": "The corrected fact."},+ },+ "required": ["id", "text"],+ },+ },+ {+ "name": "archive_memory",+ "description": "Take an entry out of your memory when it's no longer true, by its [id]. It stays searchable. Entries the user wrote can't be archived.",+ "inputSchema": {+ "type": "object",+ "properties": {"id": {"type": "integer"}},+ "required": ["id"],+ },+ },+ {+ "name": "search_memory",+ "description": "Search your memory, including archived entries, for text containing the query (case-insensitive).",+ "inputSchema": {+ "type": "object",+ "properties": {"query": {"type": "string"}},+ "required": ["query"],+ },+ "annotations": {"readOnlyHint": true},+ },+ {+ "name": "search_history",+ "description": "Search your full chat history with the user for messages containing every given word (case-insensitive). Returns dated messages, newest first.",+ "inputSchema": {+ "type": "object",+ "properties": {+ "query": {"type": "string", "description": "Words that must all appear, separated by spaces."},+ "limit": {"type": "integer", "description": "Maximum results (default 10, at most 30)."},+ },+ "required": ["query"],+ },+ "annotations": {"readOnlyHint": true},+ },+ ])+}++pub async fn call(hub: &Arc<Hub>, bot_id: &str, name: &str, args: &Value) -> Result<Value> {+ let cfg = hub.store.bot(bot_id)?.filter(|b| !b.deleted).ok_or_else(|| anyhow!("unknown bot"))?.config;+ let text = || args["text"].as_str().map(str::to_owned).ok_or_else(|| anyhow!("text required"));+ let id = || args["id"].as_i64().ok_or_else(|| anyhow!("id required"));+ match name {+ "remember" => write(hub, &cfg, Action::Remember { text: text()?, shared: args["shared"] == true }).await,+ "revise_memory" => write(hub, &cfg, Action::Revise { entry: id()?, text: text()? }).await,+ "archive_memory" => write(hub, &cfg, Action::Archive { entry: id()? }).await,+ "search_memory" => search_memory(hub, &cfg, args["query"].as_str().unwrap_or_default()).await,+ "search_history" => search_history(hub, &cfg, args).await,+ _ => bail!("unknown memory tool: {name}"),+ }+}++/// A bot that acts without asking writes at once; one that asks puts a confirm card in the chat, unless the user's+/// own message in this turn asked it to remember.+async fn write(hub: &Arc<Hub>, cfg: &BotConfig, action: Action) -> Result<Value> {+ let hub = hub.clone();+ let cfg = cfg.clone();+ tokio::task::spawn_blocking(move || {+ let turn = hub.runtime(&cfg.id);+ let asked = turn+ .message+ .as_deref()+ .and_then(|id| hub.store.entry(id))+ .is_some_and(|e| codync_mem::evidence::asks_to_remember(¬ices::typed(&e.data).join("\n")));+ if cfg.permission == Permission::Auto || asked {+ let entry = notices::apply(&hub, &cfg, turn.lane, turn.message, &action)?;+ Ok(json!({"status": "saved", "id": entry.id}))+ } else {+ notices::propose(&hub, &cfg, turn.lane, turn.message, &action)?;+ Ok(json!({"status": "waitingForUser", "note": "The user approves or declines this in the chat. Don't say it's saved."}))+ }+ })+ .await?+}++async fn search_memory(hub: &Arc<Hub>, cfg: &BotConfig, query: &str) -> Result<Value> {+ if query.trim().is_empty() {+ bail!("query required");+ }+ let (hub, bot, shared, query) = (hub.clone(), cfg.id.clone(), super::shared(cfg), query.to_owned());+ let found =+ tokio::task::spawn_blocking(move || hub.store.memory_read(|c| codync_mem::store::search(c, &bot, &query, 20)))+ .await??;+ let results: Vec<Value> = found+ .into_iter()+ .filter(|e| shared || e.bot.is_some())+ .map(|e| {+ json!({"id": e.id, "text": e.text, "state": e.state, "shared": e.bot.is_none(), "date": ymd(e.updated_at)})+ })+ .collect();+ Ok(json!({"results": results}))+}++async fn search_history(hub: &Arc<Hub>, cfg: &BotConfig, args: &Value) -> Result<Value> {+ let terms: Vec<String> =+ args["query"].as_str().unwrap_or_default().split_whitespace().take(SEARCH_TERMS).map(str::to_owned).collect();+ if terms.is_empty() {+ bail!("query must contain at least one word");+ }+ let limit = args["limit"].as_i64().unwrap_or(SEARCH_DEFAULT).clamp(1, SEARCH_MAX);+ let hub = hub.clone();+ let id = cfg.id.clone();+ let rows = tokio::task::spawn_blocking(move || hub.store.search_messages(&id, &terms, limit)).await??;+ let results: Vec<Value> = rows+ .iter()+ .map(|e| {+ json!({+ "date": ymd(e.created_at),+ "from": if e.kind == crate::store::EntryKind::User.as_str() { "user" } else { "you" },+ "inThread": e.thread_id.is_some(),+ "text": acp::truncate(e.data["text"].as_str().unwrap_or_default(), SEARCH_TEXT_CHARS),+ })+ })+ .collect();+ Ok(json!({"results": results}))+}diff --git a/host/src/chat/mod.rs b/host/src/chat/mod.rsindex c94de594..41836a7b 100644--- a/host/src/chat/mod.rs+++ b/host/src/chat/mod.rs@@ -6,6 +6,7 @@ pub mod files;pub mod group;pub mod memory;pub mod naming;+pub mod oneshot;pub mod outbox;pub mod prompt;pub mod team;diff --git a/host/src/chat/naming.rs b/host/src/chat/naming.rsindex 1762ebfe..0c515765 100644--- a/host/src/chat/naming.rs+++ b/host/src/chat/naming.rs@@ -1,16 +1,23 @@//! Automatic bot names, as in Grok Bot: a bot created without a name is called//! [`PLACEHOLDER`] until its first few conversations show what it's for. Then-//! the keeper (see `memory`) asks a one-shot agent of the bot's harness for a-//! short name. The user renaming it ends this. The description stays the+//! a one-shot agent of the bot's harness (see `oneshot`) is asked for a short+//! name, at the end of a turn. The user renaming it ends this. The description stays the//! user's: it holds standing instructions.-use crate::chat::memory::{self, EpisodeTurn};use crate::hub::Hub;use crate::store::Store;use anyhow::Result;+use serde::{Deserialize, Serialize};use serde_json::json;use std::sync::Arc;+/// A finished user ↔ bot exchange.+#[derive(Clone, Debug, Serialize, Deserialize)]+pub struct Turn {+ pub user: String,+ pub agent: String,+}+pub const PLACEHOLDER: &str = "New Bot";/// Conversations before the first naming attempt.const NAME_AFTER: usize = 3;@@ -22,12 +29,12 @@ fn key(bot_id: &str) -> String {format!("naming.turns.{bot_id}")}-fn pending(store: &Store, bot_id: &str) -> Vec<EpisodeTurn> {+fn pending(store: &Store, bot_id: &str) -> Vec<Turn> {store.kv_get(&key(bot_id)).and_then(|v| serde_json::from_str(&v).ok()).unwrap_or_default()}/// Records a finished conversation and, once there are enough, names the bot.-pub async fn observe(hub: &Arc<Hub>, bot_id: &str, turn: EpisodeTurn) -> Result<()> {+pub async fn observe(hub: &Arc<Hub>, bot_id: &str, turn: Turn) -> Result<()> {let Some(cfg) = hub.store.bot(bot_id)?.filter(|b| !b.deleted && b.config.auto_name).map(|b| b.config) else {return Ok(());};@@ -39,7 +46,7 @@ pub async fn observe(hub: &Arc<Hub>, bot_id: &str, turn: EpisodeTurn) -> Result<if turns.len() < NAME_AFTER {return Ok(());}- let raw = memory::one_shot(hub, &cfg, SYSTEM_PROMPT, &user_prompt(&turns)).await?;+ let raw = crate::chat::oneshot::one_shot(hub, &cfg, SYSTEM_PROMPT, &user_prompt(&turns)).await?;let Some(name) = parse(&raw) else {tracing::info!(bot = %bot_id, "purpose not clear yet; naming later");return Ok(());@@ -64,7 +71,7 @@ name: <a short name for the bot, 1-4 words, saying what it does, like a job titlWrite it in the language and script the user writes in (Traditional and Simplified Chinese are different). No quotes, no emoji, no trailing punctuation.Output exactly NONE if the conversations don't show a clear purpose yet (only greetings, tests or small talk).";-fn user_prompt(turns: &[EpisodeTurn]) -> String {+fn user_prompt(turns: &[Turn]) -> String {let body = turns.iter().map(|t| format!("User: {}\nBot: {}", t.user.trim(), t.agent.trim()))diff --git a/host/src/chat/oneshot.rs b/host/src/chat/oneshot.rsnew file mode 100644index 00000000..f91d2215--- /dev/null+++ b/host/src/chat/oneshot.rs@@ -0,0 +1,101 @@+//! Background jobs on the bot's own harness (dreaming, automatic naming): one prompt to a throwaway agent.++use crate::agent::acp::{self, Acp, Incoming};+use crate::hub::Hub;+use crate::store::BotConfig;+use anyhow::{Result, anyhow, bail};+use serde_json::json;+use std::sync::Arc;+use std::time::Duration;++const TIMEOUT: Duration = Duration::from_secs(180);++/// Runs one prompt on a throwaway agent of the bot's harness and returns its reply.+/// Claude gets a real system prompt, no tools, no settings and no saved session;+/// other harnesses get the instructions inline.+pub async fn one_shot(hub: &Arc<Hub>, cfg: &BotConfig, system: &str, user: &str) -> Result<String> {+ let cwd = crate::service::data_dir().join("background");+ tokio::fs::create_dir_all(&cwd).await?;+ let cwd = cwd.to_string_lossy().into_owned();+ if hub.store.kv_read(&format!("agent-env:{}", cfg.backend))?.is_some() {+ crate::market::vault::unlock(hub.clone()).await?;+ }+ let env = crate::agent::auth::env(&hub.store, &cfg.backend)?;+ let mut conn = None;+ let mut last_err = None;+ for command in crate::agent::bot::launch_commands(cfg, |_| {}).await? {+ match crate::agent::bot::start_agent(&command, &cwd, &env, Duration::from_secs(60)).await {+ Ok(c) => {+ conn = Some(c);+ break;+ }+ Err(e) => last_err = Some(e),+ }+ }+ let mut conn = conn.ok_or_else(|| last_err.unwrap_or_else(|| anyhow!("no agent to run a background job")))?;+ let acp = conn.acp.clone();+ let result = async {+ let mut params = json!({"cwd": cwd, "mcpServers": []});+ let text = if conn.claude {+ params["_meta"] = json!({+ "systemPrompt": system,+ "claudeCode": {"options": {"tools": [], "persistSession": false, "settingSources": [], "model": "haiku"}},+ });+ user.to_owned()+ } else {+ format!("{system}\n\n---\n\n{user}")+ };+ let res = acp.request("session/new", params).await?;+ let sid = res["sessionId"].as_str().ok_or_else(|| anyhow!("agent returned no sessionId"))?.to_owned();+ // Same model as the bot: the harness default may be another (local) model that+ // would load next to the bot's, or a cloud one the user kept these chats away from.+ if !conn.claude {+ crate::agent::bot::select_model(&acp, &res, &sid, cfg).await?;+ }+ let prompt = acp.request("session/prompt", json!({"sessionId": sid, "prompt": [{"type": "text", "text": text}]}));+ tokio::pin!(prompt);+ let deadline = tokio::time::sleep(TIMEOUT);+ tokio::pin!(deadline);+ let mut out = String::new();+ loop {+ tokio::select! {+ r = &mut prompt => { r?; break }+ inc = conn.rx.recv() => {+ if !collect(&acp, inc, &mut out).await {+ bail!("the background job's agent exited");+ }+ }+ () = &mut deadline => bail!("the background job timed out"),+ }+ }+ while let Ok(inc) = conn.rx.try_recv() {+ collect(&acp, Some(inc), &mut out).await;+ }+ Ok(out)+ }+ .await;+ acp.kill().await;+ result+}++/// Keeps reply text; refuses anything the agent asks of us. False once the agent is gone.+async fn collect(acp: &Acp, inc: Option<Incoming>, out: &mut String) -> bool {+ match inc {+ None | Some(Incoming::Closed { .. }) => false,+ Some(Incoming::Notification { method, params }) => {+ let u = ¶ms["update"];+ if method == "session/update" && u["sessionUpdate"] == "agent_message_chunk" {+ out.push_str(&acp::content_text(&u["content"]));+ }+ true+ }+ Some(Incoming::Request { id, method, .. }) => {+ let _ = if method == "session/request_permission" {+ acp.respond(id, json!({"outcome": {"outcome": "cancelled"}})).await+ } else {+ acp.respond_error(id, -32601, "not supported").await+ };+ true+ }+ }+}diff --git a/host/src/hub.rs b/host/src/hub.rsindex 5f1af5fc..4dbf445d 100644--- a/host/src/hub.rs+++ b/host/src/hub.rs@@ -39,6 +39,9 @@ pub struct Runtime {pub started_at: Option<i64>,/// Where the running turn talks: its own chat, a thread, or a group.pub lane: Option<Lane>,+ /// The user's message that started the running turn; `None` for a turn a routine, a room or another bot+ /// started. Memory cites it as where a fact was learned.+ pub message: Option<String>,pub fast_mode: crate::agent::config::FastModeSupport,}@@ -118,6 +121,7 @@ impl Hub {pub fn start(self: &Arc<Self>) -> Result<()> {// Approval cards and queued sends from a previous run can't be honored anymore.self.store.expire_pending()?;+ crate::chat::memory::import_old_memory(self);self.routines.restore(&self.store)?;crate::market::turn_on_installed_once(&self.store)?;for row in self.store.bots()? {@@ -247,6 +251,9 @@ impl Hub {self.store.save_bot(&cfg)?;}if !cfg.is_group() {+ if crate::chat::memory::shared(&cfg) != crate::chat::memory::shared(&row.config) {+ crate::chat::memory::changed(self, Some(&id), crate::chat::memory::Change::User);+ }self.send_cmd(&id, Cmd::Reconfigure(Box::new(cfg)))?;}self.emit_bot(&id);diff --git a/host/src/hub/sync.rs b/host/src/hub/sync.rsindex da8ff7e3..21fa8858 100644--- a/host/src/hub/sync.rs+++ b/host/src/hub/sync.rs@@ -11,6 +11,10 @@ impl Hub {let cfg = &row.config;let mut v = serde_json::to_value(cfg).expect("BotConfig is plain data and always serializes");v["managedWorkspace"] = crate::agent::workspace::is_managed(cfg).into();+ if !cfg.is_group() {+ v["sharedMemory"] = crate::chat::memory::shared(cfg).into();+ v["memoryRev"] = self.store.memory_rev().into();+ }let rt = if cfg.is_group() {// A group has no agent: Fast mode belongs to its members.crate::hub::Runtime {diff --git a/host/src/market/requests.rs b/host/src/market/requests.rsindex cecf469a..eff9b604 100644--- a/host/src/market/requests.rs+++ b/host/src/market/requests.rs@@ -317,7 +317,8 @@ fn complete(hub: &Arc<Hub>, e: &Entry, id: &str, note: String) -> Result<Value>&format!("connection-ack-{id}"),&lane,EntryKind::User,- &json!({"text":note,"status":"queued"}),+ // Written for the user, not by them: memory doesn't take it as their words.+ &json!({"text":note,"status":"queued","from":"host"}),)?;if added {hub.send_cmd(&bot, Cmd::Send { lane, entry_id: ack.id, text: note })?;diff --git a/host/src/mcp.rs b/host/src/mcp.rsindex 1c6866a3..74a47b55 100644--- a/host/src/mcp.rs+++ b/host/src/mcp.rs@@ -78,11 +78,6 @@ pub async fn serve(bot: String, port: u16, server: Server) -> Result<()> {"ping" => Ok(json!({})),"tools/list" => Ok(match server {Server::Computer => computer_tools(port, &token).await,- // Without Engram, history search still works.- Server::Memory => memory_tools(port, &token, &bot).await.unwrap_or_else(|error| {- tracing::warn!(error = format!("{error:#}"), "memory tools unavailable");- json!({"tools": available_tools})- }),_ => json!({"tools": available_tools}),}),"tools/call" => Ok(call(port, &token, &bot, params, server).await),@@ -98,19 +93,6 @@ pub async fn serve(bot: String, port: u16, server: Server) -> Result<()> {Ok(())}-async fn memory_tools(port: u16, token: &str, bot: &str) -> Result<Value> {- Ok(crate::http()- .post(format!("http://127.0.0.1:{port}/api/memoryTools"))- .bearer_auth(token)- .json(&json!({"botId": bot}))- .timeout(Duration::from_secs(240))- .send()- .await?- .error_for_status()?- .json()- .await?)-}-/// The `computer` tools the host offers; none when it can't be reached.async fn computer_tools(port: u16, token: &str) -> Value {let res = crate::http()@@ -158,7 +140,6 @@ async fn call(port: u16, token: &str, bot: &str, params: &Value, server: Server)return match server {Server::Computer if v["isError"] == true => json!({"content": v["content"], "isError": true}),Server::Computer => json!({"content": v["content"]}),- Server::Memory if v["content"].is_array() => v,Server::Chat | Server::Team | Server::Memory | Server::Routines | Server::Connectors => {json!({"content": [{"type": "text", "text": v.to_string()}]})}diff --git a/host/src/store/bots.rs b/host/src/store/bots.rsindex a8e970d8..e4ec4a43 100644--- a/host/src/store/bots.rs+++ b/host/src/store/bots.rs@@ -65,6 +65,9 @@ impl Store {c.execute("UPDATE bots SET rev = ?2, deleted = 1, session_id = NULL WHERE id = ?1", params![id, rev])?;c.execute("DELETE FROM kv WHERE k IN (SELECT 'read.' || id FROM entries WHERE bot_id = ?1)", [id])?;c.execute("DELETE FROM entries WHERE bot_id = ?1", [id])?;+ codync_mem::store::drop_bot(&c, id)?;+ // Its dreaming bookmark, notices' revisions and the record of its moved memory.+ c.execute("DELETE FROM kv WHERE k GLOB ?1", [format!("memory.*{id}*")])?;// Lane state (see `lane_key`) it owned, or that bots kept in it.c.execute("DELETE FROM kv WHERE k GLOB ?1 OR k GLOB ?2", [format!("lane.*.{id}@*"), format!("lane.*@{id}*")])?;// ponytail: snapshots of replaced sessions stay until the bot goes; prune by session if kv grows.@@ -223,6 +226,7 @@ mod tests {connectors: vec![],skills: vec![],computer: false,+ shared_memory: None,auto_name: false,created_at: 0,};diff --git a/host/src/store/entries.rs b/host/src/store/entries.rsindex d01cf9d8..5ffe14ad 100644--- a/host/src/store/entries.rs+++ b/host/src/store/entries.rs@@ -198,6 +198,28 @@ impl Store {Ok(rows)}+ /// Messages in all of `bot_id`'s own lanes (its chat and threads) after `after_seq`, oldest first: the+ /// user's, and final replies. Dreaming reads these.+ pub fn conversation_since(&self, bot_id: &str, after_seq: i64, limit: i64) -> Result<Vec<Entry>> {+ let c = self.db.locked();+ let mut st = c.prepare(&format!(+ "SELECT {ENTRY_COLS} FROM entries WHERE bot_id = ?1 AND seq > ?2 AND+ (kind = 'user' OR (kind = 'agent' AND json_extract(data, '$.final') = 1))+ ORDER BY seq LIMIT ?3"+ ))?;+ Ok(st.query_map(params![bot_id, after_seq, limit], row_entry)?.collect::<rusqlite::Result<_>>()?)+ }++ /// The newest entry of a bot's chat and threads.+ pub fn last_seq(&self, bot_id: &str) -> i64 {+ let c = self.db.locked();+ logged(+ "last seq",+ c.query_row("SELECT COALESCE(MAX(seq), 0) FROM entries WHERE bot_id = ?1", [bot_id], |r| r.get(0)),+ )+ .unwrap_or(0)+ }+/// Chat-visible messages (the user's, final replies) in `lane` after `after_seq`, oldest first.pub fn messages_after(&self, lane: &Lane, after_seq: i64, limit: i64) -> Result<Vec<Entry>> {let c = self.db.locked();diff --git a/host/src/store/memory.rs b/host/src/store/memory.rsnew file mode 100644index 00000000..01e91b38--- /dev/null+++ b/host/src/store/memory.rs@@ -0,0 +1,32 @@+//! The memory crate's tables live in this database. Each change is one transaction stamped with the next `rev`,+//! like every other mutation; see `codync_mem` and `chat::memory`.++use super::{Store, next_rev, now_ms};+use crate::LockExt;+use anyhow::Result;+use rusqlite::Connection;++impl Store {+ /// A memory change, in one transaction with its `rev`. The closure gets the time and the `rev` to stamp.+ pub fn memory_write<T>(+ &self,+ change: impl FnOnce(&Connection, i64, i64) -> codync_mem::store::Result<T>,+ ) -> Result<T> {+ let mut c = self.db.locked();+ let tx = c.transaction()?;+ let rev = next_rev(&tx)?;+ let out = change(&tx, now_ms(), rev)?;+ tx.execute("INSERT INTO kv(k, v) VALUES('memory.rev', ?1) ON CONFLICT(k) DO UPDATE SET v = ?1", [rev])?;+ tx.commit()?;+ Ok(out)+ }++ pub fn memory_read<T>(&self, read: impl FnOnce(&Connection) -> codync_mem::store::Result<T>) -> Result<T> {+ Ok(read(&self.db.locked())?)+ }++ /// The `rev` of the last memory change anywhere: clients fetch a bot's memory again when it moves.+ pub fn memory_rev(&self) -> i64 {+ self.kv_get("memory.rev").and_then(|v| v.parse().ok()).unwrap_or(0)+ }+}diff --git a/host/src/store/mod.rs b/host/src/store/mod.rsindex 963c938e..09f9fd8a 100644--- a/host/src/store/mod.rs+++ b/host/src/store/mod.rs@@ -8,6 +8,7 @@mod bots;mod devices;mod entries;+mod memory;mod model;mod schema;@@ -62,6 +63,11 @@ impl Store {Ok(())}+ pub fn kv_delete(&self, k: &str) -> Result<()> {+ self.db.locked().execute("DELETE FROM kv WHERE k = ?1", [k])?;+ Ok(())+ }+/// Keys starting with `prefix`.pub fn kv_prefix(&self, prefix: &str) -> Vec<String> {let c = self.db.locked();diff --git a/host/src/store/model.rs b/host/src/store/model.rsindex 499c077d..6c2a82c8 100644--- a/host/src/store/model.rs+++ b/host/src/store/model.rs@@ -106,6 +106,10 @@ pub struct BotConfig {/// The bot gets the built-in `computer` MCP server (see `screen`).#[serde(default)]pub computer: bool,+ /// Uses the memory shared by all bots ("yours"). Unset: on for a bot in its own workspace, off for one in a+ /// project folder (see `chat::memory::shared`).+ #[serde(default, skip_serializing_if = "Option::is_none")]+ pub shared_memory: Option<bool>,/// Created without a name: the host names it from its first conversations (see `naming`)/// until the user renames it.#[serde(default)]diff --git a/host/src/store/schema.rs b/host/src/store/schema.rsindex c788d60d..198923ba 100644--- a/host/src/store/schema.rs+++ b/host/src/store/schema.rs@@ -71,6 +71,7 @@ impl Store {CREATE INDEX IF NOT EXISTS entries_bot ON entries(bot_id, seq);",)?;migrate(&c)?;+ codync_mem::schema::migrate(&c)?;let secret_key = if path == Path::new(":memory:") {let mut key = vec![0; 32];getrandom::fill(&mut key).map_err(|_| anyhow::anyhow!("random key unavailable"))?;diff --git a/host/tests/common/mod.rs b/host/tests/common/mod.rsindex e6ed28f9..07bac032 100644--- a/host/tests/common/mod.rs+++ b/host/tests/common/mod.rs@@ -1,38 +1,9 @@//! Shared by the end-to-end tests.-use std::path::{Path, PathBuf};+use std::path::Path;use std::process::{Child, Command, Stdio};-use std::sync::OnceLock;use std::time::{Duration, Instant};-/// The checksum-verified Engram release the test hosts share: `CODYNC_TEST_ENGRAM`, else the copy-/// `packaging/engram/test-runtime.py` fetches once into `target/engram-test-runtime`. Without it-/// each host downloads its own, slow enough that tests time out. `None` when offline.-pub fn engram_fixture() -> Option<PathBuf> {- static FIXTURE: OnceLock<Option<PathBuf>> = OnceLock::new();- FIXTURE- .get_or_init(|| {- if let Some(binary) = std::env::var_os("CODYNC_TEST_ENGRAM") {- return Some(binary.into());- }- let script = Path::new(env!("CARGO_MANIFEST_DIR")).join("../packaging/engram/test-runtime.py");- let output = Command::new(python()).arg(script).output().ok().filter(|o| o.status.success())?;- Some(String::from_utf8_lossy(&output.stdout).trim().into())- })- .clone()-}--/// Use the checksum-verified release fixture so isolated hosts don't each download it.-pub fn seed_memory_runtime(home: &Path) {- let Some(binary) = engram_fixture() else {- return;- };- let dir = home.join("engram/3.2.1");- std::fs::create_dir_all(&dir).expect("memory runtime folder");- std::fs::copy(binary, dir.join(if cfg!(windows) { "engram.exe" } else { "engram" })).expect("copy Engram fixture");- std::fs::write(dir.join(".installed"), "3.2.1").expect("mark fixture installed");-}-/// Runs `codync-host serve` for `home` until it answers: the process and its base URL. A port/// picked here is only free for a moment; when something else takes it before the host binds, the/// host exits and this starts it again on another one.diff --git a/host/tests/e2e.rs b/host/tests/e2e.rsindex 7e7d04e1..25c2adab 100644--- a/host/tests/e2e.rs+++ b/host/tests/e2e.rs@@ -34,7 +34,6 @@ impl Drop for Host {async fn start_host() -> Host {let home = std::env::temp_dir().join(format!("codync-e2e-{}", uuid::Uuid::new_v4()));std::fs::create_dir_all(&home).unwrap();- common::seed_memory_runtime(&home);let (child, base) = common::serve(&home).await;let token = std::fs::read_to_string(home.join("token")).unwrap().trim().to_owned();Host { child, base, token, home }diff --git a/host/tests/files_e2e.rs b/host/tests/files_e2e.rsindex 6921c113..06e4c459 100644--- a/host/tests/files_e2e.rs+++ b/host/tests/files_e2e.rs@@ -27,7 +27,6 @@ impl Host {async fn start() -> Self {let home = std::env::temp_dir().join(format!("codync-files-e2e-{}", uuid::Uuid::new_v4()));std::fs::create_dir_all(&home).unwrap();- common::seed_memory_runtime(&home);let (child, base) = common::serve(&home).await;let mut host = Self { child, home, base, token: String::new() };std::fs::read_to_string(host.home.join("token")).unwrap().trim().clone_into(&mut host.token);diff --git a/host/tests/fixtures/memory_agent.py b/host/tests/fixtures/memory_agent.pyindex bdbd3816..d29542c9 100644--- a/host/tests/fixtures/memory_agent.py+++ b/host/tests/fixtures/memory_agent.py@@ -1,10 +1,27 @@-"""Deterministic ACP replies for automatic memory extraction, without an LLM provider."""+"""Deterministic ACP replies for dreaming, without an LLM provider.++A dream run gets a plan with two additions: one quoting what the user typed, one quoting what they pasted. Only+the first may survive the host's audit.+"""import jsonimport sysimport uuidsession = ""+PLAN = {+ "add": [+ {+ "text": "Prefers concise Traditional Chinese explanations",+ "quote": "explain things to me concisely in Traditional Chinese",+ "message": "m1",+ },+ {"text": "Loves durian", "quote": "I love durian", "message": "m1"},+ ],+ "archive": [],+ "review": [],+}+def send(value):print(json.dumps({"jsonrpc": "2.0", **value}), flush=True)@@ -21,12 +38,12 @@ for line in sys.stdin:result = {"sessionId": session}elif method == "session/prompt":text = params["prompt"][0]["text"]- if "Tag each fact you keep with a category" in text:- reply = "profile: The user prefers concise Traditional Chinese explanations."- elif "Write a concise session summary" in text:- reply = "The user established a preference for concise Traditional Chinese explanations."+ if "You keep the memory a personal assistant bot has" in text:+ reply = "Here is the plan:\n" + json.dumps(PLAN)+ elif "You name a personal assistant bot" in text:+ reply = "NONE"else:- reply = "I will use concise Traditional Chinese explanations."+ reply = "I will keep my explanations short."send({"method": "session/update", "params": {"sessionId": session, "update": {"sessionUpdate": "agent_message_chunk", "content": {"type": "text", "text": reply}}}})diff --git a/host/tests/fixtures/wire-shape.json b/host/tests/fixtures/wire-shape.jsonindex 39524002..c6316f0a 100644--- a/host/tests/fixtures/wire-shape.json+++ b/host/tests/fixtures/wire-shape.json@@ -25,6 +25,7 @@"event:bot.bot.lastMessage": "string","event:bot.bot.managedWorkspace": "bool","event:bot.bot.members": "array",+ "event:bot.bot.memoryRev": "number","event:bot.bot.model": "null","event:bot.bot.modelOptions": "object","event:bot.bot.name": "string",@@ -33,6 +34,7 @@"event:bot.bot.pinned": "bool","event:bot.bot.position": "number","event:bot.bot.rev": "number",+ "event:bot.bot.sharedMemory": "bool","event:bot.bot.skills": "array","event:bot.bot.startedAt": "null","event:bot.bot.status": "string",diff --git a/host/tests/memory_e2e.rs b/host/tests/memory_e2e.rsindex e1015208..737f4c58 100644--- a/host/tests/memory_e2e.rs+++ b/host/tests/memory_e2e.rs@@ -1,14 +1,13 @@-//! A real host and a checksum-verified Engram release, with isolated disposable data.+//! Bot memory on a real host with disposable data: moving the old memory, the user's edits, a bot's writes by+//! its permission, and dreaming.#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]#[allow(dead_code)]mod common;-use base64::Engine as _;use serde_json::{Value, json};use std::{- fmt::Write as _,path::PathBuf,process::Child,time::{Duration, Instant},@@ -30,187 +29,239 @@ impl Drop for Host {impl Host {async fn start() -> Self {- assert!(common::engram_fixture().is_some(), "needs the pinned Engram binary");let home = std::env::temp_dir().join(format!("codync-memory-e2e-{}", uuid::Uuid::new_v4()));- common::seed_memory_runtime(&home);let (child, base) = common::serve(&home).await;let token = std::fs::read_to_string(home.join("token")).unwrap().trim().to_owned();Self { child, home, base, token }}- async fn call(&self, method: &str, body: Value) -> Value {+ /// Stops the host, lets `between` change its files, and starts it again on the same data.+ async fn restart(&mut self, between: impl FnOnce(&std::path::Path)) {+ common::stop(&mut self.child);+ between(&self.home);+ let (child, base) = common::serve(&self.home).await;+ self.child = child;+ self.base = base;+ }++ async fn post(&self, method: &str, body: &Value) -> (bool, Value) {let response = reqwest::Client::new().post(format!("{}/api/{method}", self.base)).bearer_auth(&self.token)- .json(&body)+ .json(body).timeout(Duration::from_secs(120)).send().await.unwrap();- let status = response.status();- let body: Value = response.json().await.unwrap();- assert!(status.is_success(), "{method}: {body}");+ let ok = response.status().is_success();+ (ok, response.json().await.unwrap())+ }++ async fn call(&self, method: &str, body: Value) -> Value {+ let (ok, body) = self.post(method, &body).await;+ assert!(ok, "{method}: {body}");body}- async fn bot(&self, name: &str) -> String {- self.call("createBot", json!({"name":name, "backend":"custom", "command":"unused", "connectors":[]}))- .await["bot"]["id"].as_str().unwrap().to_owned()+ /// A call that must be refused: its error text.+ async fn refused(&self, method: &str, body: Value) -> String {+ let (ok, body) = self.post(method, &body).await;+ assert!(!ok, "{method} should have been refused: {body}");+ body["error"].as_str().unwrap_or_default().to_owned()+ }++ async fn bot(&self, name: &str, permission: &str) -> String {+ let bot =+ json!({"name": name, "backend": "custom", "command": "unused", "connectors": [], "permission": permission});+ self.call("createBot", bot).await["bot"]["id"].as_str().unwrap().to_owned()+ }++ async fn memory(&self, bot: &str) -> Value {+ self.call("memory", json!({"botId": bot})).await+ }++ async fn tool(&self, bot: &str, name: &str, arguments: Value) -> Value {+ self.call("memoryCall", json!({"botId": bot, "name": name, "arguments": arguments})).await+ }++ async fn tool_refused(&self, bot: &str, name: &str, arguments: Value) -> String {+ self.refused("memoryCall", json!({"botId": bot, "name": name, "arguments": arguments})).await+ }++ /// The newest memory line in a bot's chat.+ async fn card(&self, bot: &str) -> Value {+ let history = self.call("history", json!({"botId": bot})).await;+ history["entries"].as_array().unwrap().iter().rev().find(|e| e["data"]["memory"].is_object()).unwrap().clone()}}+fn texts(layer: &Value, state: &str) -> Vec<String> {+ layer["entries"]+ .as_array()+ .unwrap()+ .iter()+ .filter(|e| e["state"] == state)+ .map(|e| e["text"].as_str().unwrap().to_owned())+ .collect()+}+#[tokio::test]-#[ignore = "requires the pinned Engram release at CODYNC_TEST_ENGRAM"]-async fn migrate_manage_backup_and_forget_without_cross_bot_reads() {- let host = Host::start().await;- let a = host.bot("Memory A").await;- let b = host.bot("Memory B").await;- let legacy = host.home.join("bots").join(&a).join("memory");- std::fs::create_dir_all(legacy.join("log")).unwrap();- let mut facts = String::new();- for i in 0..65 {- writeln!(facts, "- (2026-01-01) 台灣繁體中文偏好 {i}").unwrap();- }- std::fs::write(legacy.join("profile.md"), &facts).unwrap();- std::fs::write(legacy.join("log/2026-02.md"), "- (2026-02-02) 發布 Codync\n").unwrap();-- let first = host.call("memory", json!({"botId":a,"offset":0})).await;- assert_eq!(first["total"], 66);- assert_eq!(first["facts"].as_array().unwrap().len(), 50);- assert_eq!(first["nextOffset"], 50);- let legacy_client = host.call("memory", json!({"botId":a})).await;- assert_eq!(legacy_client["facts"].as_array().unwrap().len(), 66, "older clients cannot paginate");- let search = host.call("memory", json!({"botId":a, "query":"台灣繁體中文"})).await;- assert_eq!(search["total"], 65, "UI search must not inherit the MCP top-20 cap");- let short = host.call("memory", json!({"botId":a, "query":"台灣", "offset":50})).await;- assert_eq!(short["facts"].as_array().unwrap().len(), 15);- assert!(short["nextOffset"].is_null());- assert_eq!(host.call("memory", json!({"botId":b, "query":"台灣"})).await["total"], 0);- assert_eq!(std::fs::read_to_string(legacy.join("profile.md")).unwrap(), facts);- let one = &first["facts"][0];- assert_eq!(one["createdAt"], 1_767_225_600_000_i64);- let id = one["id"].as_str().unwrap();- host.call("saveMemory", json!({"botId":a,"id":id,"title":"Updated preference","content":"Prefers English now","scope":"personal","memoryType":"learning"})).await;- host.call("pinMemory", json!({"botId":a,"id":id,"pinned":true})).await;- let pinned = host.call("memory", json!({"botId":a,"filter":"pinned"})).await;- assert_eq!(pinned["total"], 1);- let detail = host.call("memoryDetail", json!({"botId":a,"id":id})).await;- assert!(detail["history"]["result"].as_str().unwrap().contains("台灣繁體中文"));- let exported = host.call("exportMemory", json!({"botId":a})).await;- let backup: Value = serde_json::from_str(exported["json"].as_str().unwrap()).unwrap();- assert_eq!(backup["observations"].as_array().unwrap().len(), 66);- host.call("forgetMemory", json!({"botId":a,"id":id})).await;- assert_eq!(host.call("memory", json!({"botId":a,"filter":"pinned"})).await["total"], 0);- let native_session = format!("codync-{a}:codync-background");- let prompt = host.call("memoryCall", json!({"botId":a,"name":"mem_save_prompt","arguments":{"session_id":native_session,"content":"UNIQUE_SECRET_CAPTURE"}})).await;- assert_ne!(prompt["isError"], true);- host.call("clearMemory", json!({"botId":a})).await;- let cleared = host.call("exportMemory", json!({"botId":a})).await;- let cleared: Value = serde_json::from_str(cleared["json"].as_str().unwrap()).unwrap();- assert!(!cleared.to_string().contains("UNIQUE_SECRET_CAPTURE"));- assert_eq!(cleared["observations"], json!([]));- assert_eq!(host.call("memory", json!({"botId":a})).await["total"], 0);- // Import into another bot's isolated store, then search the source content.- host.call("importMemory", json!({"botId":b,"json":exported["json"]})).await;- assert_eq!(host.call("memory", json!({"botId":b})).await["total"], 66);- assert_eq!(- host.call("memory", json!({"botId":a})).await["total"],- 0,- "legacy files must not resurrect cleared facts"- );+async fn moves_old_memory_once_and_keeps_the_layers_apart() {+ let mut host = Host::start().await;+ let a = host.bot("Memory A", "auto").await;+ let b = host.bot("Memory B", "auto").await;+ let (legacy, engram) =+ (host.home.join("bots").join(&a).join("memory"), host.home.join("bots").join(&b).join("engram"));+ host.restart(|_| {+ // Markdown from before Engram for A; an Engram database for B.+ std::fs::create_dir_all(legacy.join("log")).unwrap();+ std::fs::write(legacy.join("profile.md"), "- (2026-01-01) 偏好繁體中文\n- (2026-01-02) Lives in Taipei\n")+ .unwrap();+ std::fs::write(legacy.join("log/2026-02.md"), "- (2026-02-02) Shipped Codync\n").unwrap();+ std::fs::create_dir_all(&engram).unwrap();+ let db = rusqlite::Connection::open(engram.join("engram.db")).unwrap();+ db.execute_batch(+ "CREATE TABLE observations (id INTEGER PRIMARY KEY, content TEXT, scope TEXT, pinned INTEGER,+ created_at TEXT, type TEXT, deleted_at TEXT);+ INSERT INTO observations (content, scope, pinned, created_at, type, deleted_at) VALUES+ ('Has a daughter named Mei', 'personal', 1, '2026-03-01 10:00:00', 'learning', NULL),+ ('[note] Uses vim keybindings', 'project', 0, '2026-03-02 10:00:00', 'discovery', NULL),+ ('A forgotten fact', 'personal', 0, '2026-03-03 10:00:00', 'learning', '2026-03-04 10:00:00'),+ ('What one session did', 'project', 0, '2026-03-05 10:00:00', 'session_summary', NULL);",+ )+ .unwrap();+ })+ .await;++ let view = host.memory(&a).await;+ assert_eq!(texts(&view["bot"], "active"), ["Lives in Taipei", "偏好繁體中文"]);+ assert_eq!(texts(&view["bot"], "archived"), ["Shipped Codync"], "log entries stay searchable");+ assert_eq!(view["bot"]["entries"][0]["author"], "imported");+ assert!(!legacy.exists(), "the old files go once moved");+ let view = host.memory(&b).await;+ assert_eq!(texts(&view["bot"], "active"), ["Has a daughter named Mei"]);+ assert_eq!(texts(&view["bot"], "archived"), ["Uses vim keybindings"]);+ assert_eq!(view["bot"]["entries"][0]["pinned"], true);+ assert!(!engram.exists());+ host.restart(|_| {}).await;+ assert_eq!(host.memory(&a).await["bot"]["entries"].as_array().unwrap().len(), 3, "nothing moves twice");++ // A shared fact reaches every bot that uses the shared layer; a bot's own stays with it.+ host.call("saveMemory", json!({"botId": a, "shared": true, "text": "Time zone is Asia/Taipei"})).await;+ let own = host.call("saveMemory", json!({"botId": b, "text": "Reviews PRs on Fridays"})).await;+ let own = own["entry"]["id"].as_i64().unwrap();+ let view = host.memory(&a).await;+ assert_eq!(view["shared"], true, "a bot in its own workspace uses the shared layer");+ assert_eq!(texts(&view["yours"], "active"), ["Time zone is Asia/Taipei"]);+ assert!(!view.to_string().contains("Reviews PRs"));+ assert!(host.refused("forgetMemory", json!({"botId": a, "id": own})).await.contains("No such memory"));++ host.call("pinMemory", json!({"botId": b, "id": own, "pinned": true})).await;+ host.call("archiveMemory", json!({"botId": b, "id": own, "archived": true})).await;+ assert_eq!(texts(&host.memory(&b).await["bot"], "archived").len(), 2);+ host.call("archiveMemory", json!({"botId": b, "id": own, "archived": false})).await;+ assert_eq!(texts(&host.memory(&b).await["bot"], "active")[0], "Reviews PRs on Fridays", "pinned first");+ host.call("forgetMemory", json!({"botId": b, "id": own})).await;+ host.call("clearMemory", json!({"botId": a, "shared": true})).await;+ let view = host.memory(&b).await;+ assert!(view["yours"]["entries"].as_array().unwrap().is_empty());+ assert!(!view.to_string().contains("Reviews PRs"));}#[tokio::test]-#[ignore = "requires the pinned Engram release at CODYNC_TEST_ENGRAM"]-async fn large_backups_round_trip_through_chunked_transport() {+async fn bots_write_memory_by_their_permission() {let host = Host::start().await;- let bot = host.bot("Large backup").await;- let session = "external-session";- let observations: Vec<Value> = (0..80)- .map(|i| {- json!({- "id":i+1, "sync_id":format!("large-{i}"), "session_id":session,- "project":"external-project", "title":format!("Durable fact {i}"), "type":"learning", "scope":"project",- "content":format!("Record {i}: {}", "long-lived knowledge ".repeat(500)),- "created_at":"2026-01-01T00:00:00Z", "updated_at":"2026-01-01T00:00:00Z",- })- })- .collect();- let backup = serde_json::to_vec(&json!({"version":"0.2.0", "sessions":[{- "id":session,"project":"external-project","directory":"", "started_at":"2026-01-01T00:00:00Z"- }],"observations":observations,"prompts":[]}))- .unwrap();- assert!(backup.len() > 384 * 1024);- let upload = uuid::Uuid::new_v4().to_string();- for (index, chunk) in backup.chunks(384 * 1024).enumerate() {- let offset = index * 384 * 1024;- host.call(- "upload",- json!({"botId":bot,"uploadId":upload,"name":"backup.json","offset":offset,- "data":base64::engine::general_purpose::STANDARD.encode(chunk),"done":offset+chunk.len()==backup.len()}),- )- .await;- }- host.call("importMemory", json!({"botId":bot,"uploadId":upload})).await;- assert_eq!(host.call("memory", json!({"botId":bot})).await["total"], 80);- let result = host.call("exportMemory", json!({"botId":bot})).await;- let upload = result["uploadId"].as_str().expect("large export must use chunks");- let mut bytes = Vec::new();- loop {- let part = host.call("readUpload", json!({"botId":bot,"uploadId":upload,"offset":bytes.len()})).await;- bytes.extend(base64::engine::general_purpose::STANDARD.decode(part["data"].as_str().unwrap()).unwrap());- if bytes.len() as u64 == part["size"].as_u64().unwrap() {- break;- }- }- let exported: Value = serde_json::from_slice(&bytes).unwrap();- assert_eq!(exported["observations"].as_array().unwrap().len(), 80);- for fact in exported["observations"].as_array().unwrap() {- assert_eq!(fact["project"], format!("codync-{bot}"));- assert!(fact["content"].as_str().unwrap().len() > 9_000);- }+ let auto = host.bot("Auto", "auto").await;+ let ask = host.bot("Ask", "ask").await;++ // A bot that acts on its own writes at once, and the chat says so with an undo.+ let saved = host.tool(&auto, "remember", json!({"text": "Prefers short replies"})).await;+ assert_eq!(saved["status"], "saved");+ let card = host.card(&auto).await;+ assert_eq!(card["data"]["memory"]["status"], "done");+ assert_eq!(card["data"]["text"], "Remembered: Prefers short replies");+ assert_eq!(texts(&host.memory(&auto).await["bot"], "active"), ["Prefers short replies"]);+ host.call("undoMemory", json!({"entryId": card["id"]})).await;+ assert_eq!(host.card(&auto).await["data"]["memory"]["status"], "undone");+ assert!(host.memory(&auto).await["bot"]["entries"].as_array().unwrap().is_empty());+ let again = host.tool_refused(&auto, "remember", json!({"text": "prefers short replies."})).await;+ assert!(again.contains("deleted this memory before"), "{again}");++ // What the user wrote stays theirs; secrets never go in.+ let mine = host.call("saveMemory", json!({"botId": auto, "text": "Has two cats"})).await;+ let mine = mine["entry"]["id"].as_i64().unwrap();+ let revise = host.tool_refused(&auto, "revise_memory", json!({"id": mine, "text": "Has three cats"})).await;+ assert!(revise.contains("only the user"), "{revise}");+ let secret = host.tool_refused(&auto, "remember", json!({"text": "The Wi-Fi password is hunter22"})).await;+ assert!(secret.contains("never stored"), "{secret}");+ let found = host.tool(&auto, "search_memory", json!({"query": "cats"})).await;+ assert_eq!(found["results"][0]["text"], "Has two cats");++ // A bot that asks first puts a card in the chat; nothing is written until the user approves.+ let pending = host.tool(&ask, "remember", json!({"text": "Works from home on Mondays"})).await;+ assert_eq!(pending["status"], "waitingForUser");+ assert!(host.memory(&ask).await["bot"]["entries"].as_array().unwrap().is_empty());+ let card = host.card(&ask).await;+ assert_eq!(card["data"]["memory"]["status"], "pending");+ host.call("answerMemory", json!({"entryId": card["id"], "accept": true})).await;+ assert_eq!(texts(&host.memory(&ask).await["bot"], "active"), ["Works from home on Mondays"]);+ assert!(host.refused("answerMemory", json!({"entryId": card["id"], "accept": true})).await.contains("already"));++ host.tool(&ask, "remember", json!({"text": "Hates long meetings"})).await;+ let card = host.card(&ask).await;+ host.call("answerMemory", json!({"entryId": card["id"], "accept": false})).await;+ assert_eq!(host.card(&ask).await["data"]["memory"]["status"], "declined");+ let declined = host.tool_refused(&ask, "remember", json!({"text": "Hates long meetings"})).await;+ assert!(declined.contains("deleted this memory before"), "a declined memory isn't proposed again: {declined}");}#[tokio::test]-#[ignore = "requires the pinned Engram release at CODYNC_TEST_ENGRAM"]-async fn new_session_flushes_automatic_facts_and_native_summary() {+async fn dreaming_keeps_only_what_the_user_typed_and_can_be_undone() {let host = Host::start().await;let agent = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures/memory_agent.py");- let command = common::python_agent(&agent);- let created = host+ let bot = json!({"name": "Dreamer", "backend": "custom", "command": common::python_agent(&agent), "connectors": [], "notify": false});+ let bot = host.call("createBot", bot).await["bot"]["id"].as_str().unwrap().to_owned();+ let paste = "I love durian, remember it";+ let sent = host.call(- "createBot",- json!({"name":"Keeper test","backend":"custom","command":command,"connectors":[],"notify":false}),+ "send",+ json!({+ "botId": bot,+ "text": format!("Please always explain things to me concisely in Traditional Chinese.\n{paste}"),+ "clientNonce": "dream-1",+ "pasted": [paste],+ }),).await;- let bot = created["bot"]["id"].as_str().unwrap();- host.call(- "send",- json!({"botId":bot,"text":"Please always explain things to me concisely in Traditional Chinese."}),- )- .await;+ let message = sent["entry"]["id"].as_str().unwrap().to_owned();let deadline = Instant::now() + Duration::from_secs(30);loop {- let history = host.call("history", json!({"botId":bot})).await;+ let history = host.call("history", json!({"botId": bot})).await;if history["entries"].as_array().unwrap().iter().any(|e| e["kind"] == "agent" && e["data"]["final"] == true) {break;}assert!(Instant::now() < deadline, "agent did not finish: {history}");tokio::time::sleep(Duration::from_millis(50)).await;}- host.call("newSession", json!({"botId":bot})).await;- loop {- let memory = host.call("memory", json!({"botId":bot})).await;- let facts = memory["facts"].as_array().unwrap();- if facts.iter().any(|f| f["memoryType"] == "session_summary") {- assert!(- facts- .iter()- .any(|f| f["kind"] == "profile" && f["content"].as_str().unwrap().contains("Traditional Chinese"))- );- break;+ // A new session asks dreaming to run now rather than after five quiet minutes.+ host.call("newSession", json!({"botId": bot})).await;+ let view = loop {+ let view = host.memory(&bot).await;+ if !view["dreams"].as_array().unwrap().is_empty() {+ break view;}- assert!(Instant::now() < deadline, "keeper did not save summary: {memory}");+ assert!(Instant::now() < deadline, "dreaming didn't run: {view}");tokio::time::sleep(Duration::from_millis(100)).await;- }+ };+ assert_eq!(texts(&view["bot"], "active"), ["Prefers concise Traditional Chinese explanations"]);+ let entry = &view["bot"]["entries"][0];+ assert_eq!(entry["author"], "dream");+ assert_eq!(entry["evidence"], json!([message]), "it cites the message it learned from");+ assert!(!view.to_string().contains("durian"), "pasted text isn't the user's own words");++ let dream = view["dreams"][0]["id"].as_i64().unwrap();+ host.call("undoDream", json!({"botId": bot, "id": dream})).await;+ let view = host.memory(&bot).await;+ assert!(view["bot"]["entries"].as_array().unwrap().is_empty());+ assert_eq!(view["dreams"][0]["undone"], true);}diff --git a/host/tests/routines_e2e.rs b/host/tests/routines_e2e.rsindex 2395bdd2..f5474f6e 100644--- a/host/tests/routines_e2e.rs+++ b/host/tests/routines_e2e.rs@@ -29,7 +29,6 @@ impl Host {async fn start() -> Self {let home = std::env::temp_dir().join(format!("codync-routines-e2e-{}", uuid::Uuid::new_v4()));std::fs::create_dir_all(&home).unwrap();- common::seed_memory_runtime(&home);let (child, base) = common::serve(&home).await;let mut host = Self { child, home, base, token: String::new() };std::fs::read_to_string(host.home.join("token")).unwrap().trim().clone_into(&mut host.token);diff --git a/host/tests/team_e2e.rs b/host/tests/team_e2e.rsindex 8dc285cf..802250b1 100644--- a/host/tests/team_e2e.rs+++ b/host/tests/team_e2e.rs@@ -28,7 +28,6 @@ impl Host {async fn start() -> Self {let home = std::env::temp_dir().join(format!("codync-team-e2e-{}", uuid::Uuid::new_v4()));std::fs::create_dir_all(&home).unwrap();- common::seed_memory_runtime(&home);let (child, base) = common::serve(&home).await;let mut host = Self { child, home, base, token: String::new() };std::fs::read_to_string(host.home.join("token")).unwrap().trim().clone_into(&mut host.token);diff --git a/host/tests/turns_e2e.rs b/host/tests/turns_e2e.rsindex 38605aad..76a8b25a 100644--- a/host/tests/turns_e2e.rs+++ b/host/tests/turns_e2e.rs@@ -28,7 +28,6 @@ impl Host {async fn start() -> Self {let home = std::env::temp_dir().join(format!("codync-turns-e2e-{}", uuid::Uuid::new_v4()));std::fs::create_dir_all(&home).unwrap();- common::seed_memory_runtime(&home);let (child, base) = common::serve(&home).await;let token = std::fs::read_to_string(home.join("token")).unwrap().trim().to_owned();Self { child, base, token, home }diff --git a/packaging/engram/release.json b/packaging/engram/release.jsondeleted file mode 100644index c2ead93a..00000000--- a/packaging/engram/release.json+++ /dev/null@@ -1,11 +0,0 @@-{- "version": "3.2.1",- "assets": {- "darwin_amd64": "bc529ce47844a59c8091ca444e5f3f1ba126caf8df5b4dff01312df345c5c3f9",- "darwin_arm64": "32901d25a1a311565c382111f138fc7788e80685e575e9ec11e7c2ae5fbc5128",- "linux_amd64": "c5e0bbf77b3c4b196713d64d68593966d30a94582fad8dafe8e458a182c155a8",- "linux_arm64": "e1ed861f8e745bd93e679a91e283ae6668e64ad0afcdd552f61625d0e45480dd",- "windows_amd64": "d1951fec7810e5ff2213a24865c3ee63b7ecfd10a4b8631494faf02d1c2eae65",- "windows_arm64": "8d09fe37b22c02ac3717751a083b20af99e60f43d919e9e6bcde50e2827cabec"- }-}diff --git a/packaging/engram/test-runtime.py b/packaging/engram/test-runtime.pydeleted file mode 100644index aa31828e..00000000--- a/packaging/engram/test-runtime.py+++ /dev/null@@ -1,54 +0,0 @@-#!/usr/bin/env python3-"""Fetch the same pinned Engram release as the host for isolated integration tests."""--import hashlib-import io-import json-import os-from pathlib import Path-import platform-import tarfile-import urllib.request-import zipfile--root = Path(__file__).resolve().parents[2]-release = json.loads((Path(__file__).parent / "release.json").read_text())-system = {"Darwin": "darwin", "Linux": "linux", "Windows": "windows"}[platform.system()]-arch = {"arm64": "arm64", "aarch64": "arm64", "x86_64": "amd64", "amd64": "amd64"}[platform.machine().lower()]-target = f"{system}_{arch}"-version = release["version"]-extension = "zip" if system == "windows" else "tar.gz"-name = "engram.exe" if system == "windows" else "engram"-archive = f"engram_{version}_{target}.{extension}"-url = f"https://github.com/Gentleman-Programming/engram/releases/download/v{version}/{archive}"-destination = root / "host" / "target" / "engram-test-runtime" / version / name---def fetch():- with urllib.request.urlopen(url, timeout=120) as response:- data = response.read(128 * 1024 * 1024 + 1)- if hashlib.sha256(data).hexdigest() != release["assets"][target]:- raise SystemExit("Engram release checksum mismatch")- if extension == "zip":- with zipfile.ZipFile(io.BytesIO(data)) as files:- binary = files.read(name)- else:- with tarfile.open(fileobj=io.BytesIO(data), mode="r:gz") as files:- member = files.getmember(name)- if not member.isfile():- raise SystemExit("Engram release does not contain a regular binary")- binary = files.extractfile(member).read()- destination.parent.mkdir(parents=True, exist_ok=True)- partial = destination.with_name(name + ".part")- partial.write_bytes(binary)- partial.chmod(0o755)- partial.replace(destination)---# A copy fetched earlier is reused: the pre-commit hook asks for it on every host commit.-if not destination.exists():- fetch()-if "GITHUB_ENV" in os.environ:- with open(os.environ["GITHUB_ENV"], "a", encoding="utf-8") as env:- env.write(f"CODYNC_TEST_ENGRAM={destination}\n")-print(destination)