Solution for long term memory for agent coding CLIs and to facilitate handoff between different agent vendors
Rust
fix(wiki): refuse page paths that collide on a case-folding filesystem
Gabe Tobias3859cfe76789
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 08a74e62..5c500b63 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -7,6 +7,18 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0## [Unreleased]+### Fixed+- A page write is refused when another live page in the same project differs+ from it only by case or Unicode normalization. Such a pair is one file on+ macOS (APFS) and Windows (NTFS), so creating the second silently overwrote+ the first page's file while the index kept both rows, so reads for either path+ then returned the survivor's body, and the wiki watcher superseded the+ overwritten row, losing the original content from disk and index alike. The+ refusal names both paths, applies on every platform (the wiki is synced+ between them), and leaves supersedes of an existing path untouched.+ `reindex` skips such a pair instead of failing the whole rebuild, reports the+ count, and logs each one.+## [2.3.2] - 2026-09-20### Changeddiff --git a/Cargo.lock b/Cargo.lockindex 25bd2249..c45fd6d2 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -108,6 +108,7 @@ dependencies = [name = "ai-memory-core"version = "2.3.2"dependencies = [+ "icu_normalizer","jiff","regex","schemars",@@ -1956,6 +1957,9 @@ dependencies = ["icu_properties","icu_provider","smallvec",+ "utf16_iter",+ "utf8_iter",+ "write16","zerovec",]@@ -4405,6 +4409,12 @@ dependencies = ["serde",]+[[package]]+name = "utf16_iter"+version = "1.0.5"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "c8232dd3cdaed5356e0f716d285e4b40b932ac434100fe9b7e0e8e935b9e6246"+[[package]]name = "utf8_iter"version = "1.0.4"@@ -5082,6 +5092,12 @@ dependencies = ["wasmparser",]+[[package]]+name = "write16"+version = "1.0.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "d1890f4022759daae28ed4fe62859b1236caebfc61ede2f63ed4e695f3f6d936"+[[package]]name = "writeable"version = "0.6.3"diff --git a/Cargo.toml b/Cargo.tomlindex 849d66a6..82665ba2 100644--- a/Cargo.toml+++ b/Cargo.toml@@ -152,6 +152,9 @@ flate2 = "1"# Text munging / hook payload sanitisation.regex = "1"+# Unicode NFC for the portable page-path key. Already in the tree via+# idna -> idna_adapter, so this adds no new code to the binary.+icu_normalizer = "2"# Wiki git versioning.git2 = { version = "0.21", default-features = false, features = ["vendored-libgit2"] }diff --git a/crates/ai-memory-cli/src/commands/reindex.rs b/crates/ai-memory-cli/src/commands/reindex.rsindex e4aaa87d..4148ebaa 100644--- a/crates/ai-memory-cli/src/commands/reindex.rs+++ b/crates/ai-memory-cli/src/commands/reindex.rs@@ -79,6 +79,13 @@ pub async fn run(config: &Config, _args: ReindexArgs) -> Result<()> {summary.workspaces,config.data_dir.join("wiki").display(),);+ if summary.skipped_collisions > 0 {+ println!(+ "skipped {} page(s) whose path differs from an indexed page only by case or Unicode \+ normalization (they cannot coexist on macOS/Windows); the log names each pair",+ summary.skipped_collisions,+ );+ }Ok(())}diff --git a/crates/ai-memory-core/Cargo.toml b/crates/ai-memory-core/Cargo.tomlindex 8fe95df1..3d6a5f7d 100644--- a/crates/ai-memory-core/Cargo.toml+++ b/crates/ai-memory-core/Cargo.toml@@ -15,6 +15,7 @@ thiserror.workspace = trueuuid.workspace = truejiff.workspace = trueregex.workspace = true+icu_normalizer.workspace = truetracing.workspace = true# JsonSchema on Tier/PageKind so the consolidation prompt's# structured-output schema enforces enum membership instead ofdiff --git a/crates/ai-memory-core/src/ids.rs b/crates/ai-memory-core/src/ids.rsindex 4296b786..d779627f 100644--- a/crates/ai-memory-core/src/ids.rs+++ b/crates/ai-memory-core/src/ids.rs@@ -203,6 +203,23 @@ impl fmt::Display for PagePath {}}+/// The key a case-folding or Unicode-normalizing filesystem effectively+/// stores a page path under: two paths with the same key are one file on+/// macOS (APFS) and Windows (NTFS), whatever they look like here.+///+/// Lowercase first, then compose: `İ` lowercases to `i` + U+0307, which only+/// composes back to a single scalar after the fold.+#[must_use]+pub fn portable_page_key(path: &str) -> String {+ let lowered = path.to_lowercase();+ let nfc = icu_normalizer::ComposingNormalizer::new_nfc();+ if nfc.is_normalized(&lowered) {+ lowered+ } else {+ nfc.normalize(&lowered).into_owned()+ }+}+/// Discriminator for the agent CLI that captured an observation or handoff.#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)]#[serde(rename_all = "kebab-case")]@@ -761,7 +778,7 @@ fn ensure_portable_component(component: &str, full: &str) -> Result<(), MemoryEr#[cfg(test)]mod portable_page_path_tests {- use super::PagePath;+ use super::{PagePath, portable_page_key};/// Every shape #462 reproduced on native Windows. Each either fails late/// with a 500, or writes but cannot be checkpointed by libgit2 and cannot@@ -891,4 +908,32 @@ mod portable_page_path_tests {);}}++ #[test]+ fn portable_key_collapses_case_and_normalization() {+ for (a, b) in [+ ("concepts/alpha.md", "concepts/Alpha.md"),+ ("Concepts/alpha.md", "concepts/ALPHA.md"),+ ("concepts/caf\u{00e9}.md", "concepts/cafe\u{0301}.md"),+ ("concepts/CAF\u{00c9}.md", "concepts/cafe\u{0301}.md"),+ ] {+ assert_eq!(+ portable_page_key(a),+ portable_page_key(b),+ "{a:?} and {b:?} are one file on macOS/Windows"+ );+ }+ }++ #[test]+ fn portable_key_keeps_distinct_paths_distinct() {+ for (a, b) in [+ ("concepts/alpha.md", "concepts/alphabet.md"),+ ("concepts/alpha.md", "decisions/alpha.md"),+ ("concepts/nested/a.md", "concepts/a.md"),+ ("concepts/cafe.md", "concepts/caf\u{00e9}.md"),+ ] {+ assert_ne!(portable_page_key(a), portable_page_key(b));+ }+ }}diff --git a/crates/ai-memory-core/src/lib.rs b/crates/ai-memory-core/src/lib.rsindex 058dc038..97436922 100644--- a/crates/ai-memory-core/src/lib.rs+++ b/crates/ai-memory-core/src/lib.rs@@ -58,7 +58,7 @@ pub use handoff::{pub use ids::{AgentKind, ApiCredentialId, AutoImproveProposalId, AutoImproveRunId, EntityId, HandoffId,ManagedRunId, MessageId, ObservationId, PageFeedbackId, PageId, PagePath, ProjectId, SessionId,- UserId, WorkspaceId, WorkstreamId, is_git_reserved_component,+ UserId, WorkspaceId, WorkstreamId, is_git_reserved_component, portable_page_key,};pub use message::{AgentMessage, MessageBox, MessageClaim, MessageOrigin, MessageState, NewAgentMessage,diff --git a/crates/ai-memory-store/src/error.rs b/crates/ai-memory-store/src/error.rsindex 1b0302a8..698d51f4 100644--- a/crates/ai-memory-store/src/error.rs+++ b/crates/ai-memory-store/src/error.rs@@ -118,6 +118,24 @@ pub enum StoreError {#[error("workspace name '{0}' is already taken")]WorkspaceNameTaken(String),+ /// A page create was refused because another live page in the same scope+ /// differs from it only by case or Unicode normalization. Such a pair is+ /// one file on macOS and Windows, so creating the second destroys the+ /// first (#1/#2: the index kept both rows and served the survivor's body+ /// for either path).+ #[error(+ "page path '{requested}' collides with existing page '{existing}': they differ only in \+ case or Unicode normalization, so a case-insensitive or normalizing filesystem (macOS, \+ Windows) stores both at one file and this write would destroy the other page. Choose a \+ distinct path, or delete/rename the existing page first."+ )]+ PagePathCollides {+ /// The path this write asked for.+ requested: String,+ /// The live page it would land on.+ existing: String,+ },+/// A session move was rejected because the destination scope already has/// a latest page at the session page path (`idx_pages_latest_path`), so/// re-stamping the source versions would collide. The caller retries withdiff --git a/crates/ai-memory-store/src/ops.rs b/crates/ai-memory-store/src/ops.rsindex f30a8867..67c23aed 100644--- a/crates/ai-memory-store/src/ops.rs+++ b/crates/ai-memory-store/src/ops.rs@@ -1070,6 +1070,12 @@ pub(crate) fn upsert_page_in_tx()?;return Ok(new_id);}+ if let Some(existing) = colliding_live_path(tx, page)? {+ return Err(StoreError::PagePathCollides {+ requested: page.path.as_str().to_owned(),+ existing,+ });+ }let frontmatter_str = stamped_frontmatter(conformed, now)?;let new_id = PageId::new();tx.execute(@@ -1112,6 +1118,36 @@ pub(crate) fn upsert_page_in_tx(Ok(new_id)}+/// The live page this create would share a file with on a case-folding or+/// normalizing filesystem, if any.+///+/// Runs on creates only: a supersede targets a path that already has its own+/// row, so it cannot introduce a pair that did not exist before. That keeps+/// the scan off the rewrite-heavy path, where it would repeat for every+/// consolidation pass over the same page.+fn colliding_live_path(+ tx: &rusqlite::Transaction<'_>,+ page: &NewPage,+) -> StoreResult<Option<String>> {+ let key = ai_memory_core::portable_page_key(page.path.as_str());+ let mut stmt = tx.prepare_cached(+ "SELECT path FROM pages \+ WHERE workspace_id = ?1 AND project_id = ?2 AND is_latest = 1 AND path <> ?3",+ )?;+ let mut rows = stmt.query(params![+ page.workspace_id.as_bytes(),+ page.project_id.as_bytes(),+ page.path.as_str(),+ ])?;+ while let Some(row) = rows.next()? {+ let candidate: String = row.get(0)?;+ if ai_memory_core::portable_page_key(&candidate) == key {+ return Ok(Some(candidate));+ }+ }+ Ok(None)+}+/// Attach the normalized entity set to a new page version (V38). Entity/// rows are shared within one project; links stay attached to immutable/// page versions and latest-page filtering controls retrieval.diff --git a/crates/ai-memory-wiki/src/wiki.rs b/crates/ai-memory-wiki/src/wiki.rsindex 871f0987..5055c12b 100644--- a/crates/ai-memory-wiki/src/wiki.rs+++ b/crates/ai-memory-wiki/src/wiki.rs@@ -50,6 +50,12 @@ pub struct ReindexSummary {/// (#701) — the same case one level down, for a purge whose page-file/// removal did not complete.pub skipped_purged_sessions: usize,+ /// Pages skipped because another page in the same scope differs from them+ /// only by case or Unicode normalization. A wiki authored on a+ /// case-sensitive filesystem can hold such a pair; indexing both is what+ /// the write guard now refuses, and one bad pair must not fail the whole+ /// rebuild.+ pub skipped_collisions: usize,}/// The session a page belongs to, for paths that name one.@@ -1710,8 +1716,22 @@ impl Wiki {summary.skipped_purged_sessions += 1;continue;}- self.reindex_page(ws, proj, path).await?;- summary.pages += 1;+ match self.reindex_page(ws, proj, path).await {+ Ok(_) => summary.pages += 1,+ Err(WikiError::Store(ai_memory_store::StoreError::PagePathCollides {+ requested,+ existing,+ })) => {+ tracing::warn!(+ path = %requested,+ existing = %existing,+ "skipping page: it differs from an already-indexed page only by case \+ or Unicode normalization, so both cannot coexist on macOS/Windows",+ );+ summary.skipped_collisions += 1;+ }+ Err(e) => return Err(e),+ }}}Ok(summary)@@ -2864,6 +2884,138 @@ mod tests {assert!(ai_memory_core::okf::is_conformant(&parsed.frontmatter));}+ /// Refused on every platform, not only the case-insensitive ones: the+ /// wiki is synced between them, so what it accepts cannot depend on the+ /// filesystem underneath.+ #[tokio::test]+ async fn a_case_only_path_variant_cannot_destroy_an_existing_page() {+ let tmp = TempDir::new().unwrap();+ let store = Store::open(tmp.path()).unwrap();+ let ws = store.writer.get_or_create_workspace("w").await.unwrap();+ let proj = store+ .writer+ .get_or_create_project(ws, "p", None)+ .await+ .unwrap();+ let wiki = Wiki::new(tmp.path(), store.writer.clone()).unwrap();++ wiki.write_page(req(+ ws,+ proj,+ "concepts/alpha.md",+ "the original alpha body",+ serde_json::json!({"title": "Alpha"}),+ ))+ .await+ .unwrap();++ let collision = wiki+ .write_page(req(+ ws,+ proj,+ "concepts/Alpha.md",+ "a different page entirely",+ serde_json::json!({"title": "Alpha Upper"}),+ ))+ .await;++ let err = collision.expect_err("case-only variant must be refused");+ let msg = err.to_string();+ assert!(+ msg.contains("concepts/alpha.md") && msg.contains("concepts/Alpha.md"),+ "the error must name both pages so the caller can act on it: {msg}"+ );+ let raw =+ std::fs::read_to_string(wiki.project_root(ws, proj).join("concepts/alpha.md")).unwrap();+ assert!(+ raw.contains("the original alpha body"),+ "the existing page's file must survive the refused write: {raw}"+ );+ }++ /// The same collision through Unicode normalization rather than case.+ #[tokio::test]+ async fn an_nfd_path_variant_cannot_destroy_its_nfc_page() {+ let tmp = TempDir::new().unwrap();+ let store = Store::open(tmp.path()).unwrap();+ let ws = store.writer.get_or_create_workspace("w").await.unwrap();+ let proj = store+ .writer+ .get_or_create_project(ws, "p", None)+ .await+ .unwrap();+ let wiki = Wiki::new(tmp.path(), store.writer.clone()).unwrap();++ let nfc = "concepts/caf\u{00e9}.md";+ let nfd = "concepts/cafe\u{0301}.md";++ wiki.write_page(req(+ ws,+ proj,+ nfc,+ "the composed page body",+ serde_json::json!({"title": "Cafe NFC"}),+ ))+ .await+ .unwrap();++ let collision = wiki+ .write_page(req(+ ws,+ proj,+ nfd,+ "a different page entirely",+ serde_json::json!({"title": "Cafe NFD"}),+ ))+ .await;++ assert!(+ collision.is_err(),+ "a decomposed variant of an existing composed path must be refused"+ );+ let raw = std::fs::read_to_string(wiki.project_root(ws, proj).join(nfc)).unwrap();+ assert!(+ raw.contains("the composed page body"),+ "the existing page's file must survive the refused write: {raw}"+ );+ }++ /// The guard must not turn page paths into case-insensitive keys.+ #[tokio::test]+ async fn distinct_paths_are_unaffected_by_the_collision_guard() {+ let tmp = TempDir::new().unwrap();+ let store = Store::open(tmp.path()).unwrap();+ let ws = store.writer.get_or_create_workspace("w").await.unwrap();+ let proj = store+ .writer+ .get_or_create_project(ws, "p", None)+ .await+ .unwrap();+ let wiki = Wiki::new(tmp.path(), store.writer.clone()).unwrap();++ for path in [+ "concepts/alpha.md",+ "concepts/alphabet.md",+ "concepts/nested/alpha.md",+ "decisions/alpha.md",+ ] {+ wiki.write_page(req(ws, proj, path, "body", serde_json::json!({})))+ .await+ .unwrap_or_else(|e| panic!("{path} must be accepted: {e}"));+ }++ // A rewrite is a supersede, not a collision.+ wiki.write_page(req(+ ws,+ proj,+ "concepts/alpha.md",+ "revised body",+ serde_json::json!({}),+ ))+ .await+ .unwrap();+ }+/// An unchanged rewrite must emit byte-identical markdown: the/// `generated.at` inheritance keeps the timestamp, so neither git/// nor the store sees a phantom new version.