AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
feat(sessions): import a MemPalace palace with --format mempalace (#2874) (#2894)
Jake Elbirtff3b1d28d511
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 329dde937..ab2757a2c 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -62,6 +62,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0Code session UUID, and marks the model-written observations and summaries`is_summary: true`.+- Offline `xerj sessions import --format mempalace` exports a MemPalace+ palace's drawers verbatim, with their wing, room and hall, through the+ existing text redactor (#2874). It reads the `sqlite_exact` backend+ read-only in one snapshot without creating side files on a clean store,+ skips registry and sentinel rows and closets, decodes MemPalace's escaped+ text, and refuses a palace on the default Chroma backend.+- Offline `xerj sessions import --format codex` exports a documented subsetof one archived Codex rollout through the existing text redactor. It skipstools and reasoning, retains physical source locations and archive eventdiff --git a/engine/crates/xerj-autoindex/src/sessions.rs b/engine/crates/xerj-autoindex/src/sessions.rsindex 04402f598..8a8acf3a2 100644--- a/engine/crates/xerj-autoindex/src/sessions.rs+++ b/engine/crates/xerj-autoindex/src/sessions.rs@@ -16,7 +16,7 @@ const MAX_RECORDS: usize = 100_000;pub const HELP: &str = "xerj sessions import — offline session preparation for autoindex-USAGE: xerj sessions import --format <claude-code|codex|claude-mem> --input FILE --output NEW.jsonl+USAGE: xerj sessions import --format <claude-code|codex|claude-mem|mempalace> --input PATH --output NEW.jsonlRead one explicit Claude Code transcript or Codex rollout JSONL; write user/assistant textrecords through the existing native PII redactor before any output is stored.@@ -37,8 +37,11 @@ or files. exec --json and external history_base are unsupported.claude-mem opens ~/.claude-mem/claude-mem.db read-only in one snapshot and exportsuser_prompts, observations and session_summaries rows (is_summary=true for themodel-written two), each with the session ID claude-mem recorded (a UUID for-Claude Code). Input <=512 MiB (.db plus -wal); line <=1 MiB; turn text-<=256 KiB; <=100000 messages or rows.+Claude Code). mempalace reads a palace directory's sqlite_exact.sqlite3+read-only and exports mempalace_drawers drawers verbatim with wing/room/hall,+skipping registry/sentinel rows and closets; the Chroma backend is unsupported.+Input <=512 MiB (database plus -wal); line <=1 MiB; turn text <=256 KiB;+<=100000 messages or rows.Exit 0: exported, JSON summary on stdout; 2: usage; 1: import/publication error.Optional automation (Claude Code, macOS/Linux): xerj init --memory.@@ -270,6 +273,7 @@ fn claude_turn(mod claude_mem;mod codex;+mod mempalace;#[derive(Clone, Copy)]enum SessionFormat {@@ -289,6 +293,10 @@ pub fn import_claude_mem(input: &Path, output: &Path) -> Result<ImportSummary> {claude_mem::import(input, output)}+pub fn import_mempalace(input: &Path, output: &Path) -> Result<ImportSummary> {+ mempalace::import(input, output)+}+fn ensure_new_output(output: &Path) -> Result<()> {match fs::symlink_metadata(output) {Ok(_) => bail!("session output already exists; choose a new output path"),@@ -494,7 +502,8 @@ pub fn run_cli(args: &[String]) -> i32 {Some("claude-code") => import_claude_code,Some("codex") => import_codex,Some("claude-mem") => import_claude_mem,- _ => bail!("--format claude-code, codex or claude-mem is required"),+ Some("mempalace") => import_mempalace,+ _ => bail!("--format claude-code, codex, claude-mem or mempalace is required"),};Ok((PathBuf::from(input.context("--input is required")?),diff --git a/engine/crates/xerj-autoindex/src/sessions/mempalace.rs b/engine/crates/xerj-autoindex/src/sessions/mempalace.rsnew file mode 100644index 000000000..53d0ffba2--- /dev/null+++ b/engine/crates/xerj-autoindex/src/sessions/mempalace.rs@@ -0,0 +1,624 @@+//! Read-only MemPalace sqlite_exact subset. Reference and licence are pinned in the import guide.+use super::*;+use rusqlite::{Connection, OpenFlags, OptionalExtension};++/// The sqlite_exact backend's file inside a palace directory.+const STORE: &str = "sqlite_exact.sqlite3";+/// MemPalace's default backend; its store is not read by this importer.+const CHROMA_STORE: &str = "chroma.sqlite3";+/// The collection holding verbatim drawers; closets and other collections are skipped.+const DRAWERS: &str = "mempalace_drawers";+/// Metadata flag on a record whose strings MemPalace stored escaped.+const ESCAPED: &str = "_mempalace_escaped";+const ESCAPE_PREFIX: &str = "\u{1}mpesc:";+/// Drawer metadata kept as redacted text when it is a string.+const TEXT_METADATA: [&str; 5] = ["wing", "room", "hall", "added_by", "content_date"];+/// Drawer metadata kept when it is an integer.+const NUMERIC_METADATA: [&str; 3] = ["chunk_index", "line_start", "line_end"];++/// Undo MemPalace's stored-text escape (`\\`, `\0`, `\uXXXX` after the prefix).+/// NUL becomes a space and an unpaired surrogate U+FFFD, as MemPalace's own search+/// index and MCP output render them; other text is returned as written.+fn unescape(value: &str) -> String {+ let Some(rest) = value.strip_prefix(ESCAPE_PREFIX) else {+ return value.to_string();+ };+ let mut out = String::with_capacity(rest.len());+ let mut chars = rest.chars();+ while let Some(c) = chars.next() {+ if c != '\\' {+ out.push(c);+ continue;+ }+ let mut ahead = chars.clone();+ match ahead.next() {+ Some('\\') => out.push('\\'),+ Some('0') => out.push(' '),+ Some('u') => {+ let hex: String = ahead.by_ref().take(4).collect();+ let valid = hex.len() == 4+ && hex+ .chars()+ .all(|h| h.is_ascii_digit() || ('A'..='F').contains(&h));+ if !valid {+ out.push('\\');+ continue;+ }+ let code = u32::from_str_radix(&hex, 16).unwrap();+ out.push(char::from_u32(code).unwrap_or('\u{FFFD}'));+ }+ _ => {+ out.push('\\');+ continue;+ }+ }+ chars = ahead;+ }+ out+}++/// The values MemPalace's own reader treats as a set flag: 1, true, "1", "true", "True".+fn flagged(value: Option<&Value>) -> bool {+ match value {+ Some(Value::Bool(b)) => *b,+ Some(Value::Number(n)) => n.as_i64() == Some(1),+ Some(Value::String(s)) => matches!(s.as_str(), "1" | "true" | "True"),+ _ => false,+ }+}++/// RFC 3339 instant normalised like the other formats; anything else is unknown.+fn timestamp(candidates: [Option<&str>; 3]) -> Value {+ candidates+ .into_iter()+ .flatten()+ .find_map(|s| chrono::DateTime::parse_from_rfc3339(s).ok())+ .map(|t| {+ json!(t+ .with_timezone(&chrono::Utc)+ .to_rfc3339_opts(chrono::SecondsFormat::Millis, true))+ })+ .unwrap_or(Value::Null)+}++fn sidecar(db: &Path, suffix: &str) -> PathBuf {+ let mut path = db.as_os_str().to_owned();+ path.push(suffix);+ PathBuf::from(path)+}++/// What a writer starting mid-import changes: each file's presence, size and mtime.+type Signature = Vec<Option<(u64, Option<std::time::SystemTime>)>>;++fn signature(db: &Path) -> Signature {+ ["", "-wal", "-shm"]+ .iter()+ .map(|suffix| {+ fs::metadata(sidecar(db, suffix))+ .ok()+ .map(|m| (m.len(), m.modified().ok()))+ })+ .collect()+}++/// The store file for a palace directory or an explicit store path, and the+/// canonical path SQLite will open (it names sidecars after the symlink target).+fn store_path(input: &Path) -> Result<(PathBuf, PathBuf)> {+ let meta = fs::metadata(input).context("cannot open MemPalace palace or store")?;+ let chroma = |dir: &Path| dir.join(CHROMA_STORE).is_file();+ let store = if meta.is_dir() {+ let store = input.join(STORE);+ ensure!(+ store.is_file() || !chroma(input),+ "this palace uses MemPalace's default Chroma backend; only the sqlite_exact \+ backend is supported"+ );+ ensure!(+ !(store.is_file() && chroma(input)),+ "this palace holds both a Chroma and a sqlite_exact store, which MemPalace \+ itself refuses to open; pass the store file to read it anyway"+ );+ ensure!(+ store.is_file(),+ "no {STORE} in the palace directory; pass a MemPalace palace (~/.mempalace/palace)"+ );+ store+ } else {+ ensure!(+ meta.is_file(),+ "MemPalace input must be a directory or a regular file"+ );+ ensure!(+ input.file_name().and_then(|n| n.to_str()) != Some(CHROMA_STORE),+ "this is MemPalace's Chroma store; only the sqlite_exact backend is supported"+ );+ input.to_path_buf()+ };+ let canonical = fs::canonicalize(&store).context("cannot open MemPalace store")?;+ Ok((store, canonical))+}++/// Read-only without creating sidecars on a clean store, as MemPalace's own reader does.+/// Returns whether the store was opened immutable.+fn open(db: &Path) -> Result<(Connection, bool)> {+ let (wal, shm) = (sidecar(db, "-wal").is_file(), sidecar(db, "-shm").is_file());+ ensure!(+ wal == shm,+ "MemPalace store has an incomplete WAL sidecar set; let its writer exit cleanly, then rerun"+ );+ let len = fs::metadata(db)?.len()+ + fs::metadata(sidecar(db, "-wal"))+ .map(|m| m.len())+ .unwrap_or(0);+ ensure!(+ len <= MAX_INPUT_BYTES,+ "MemPalace store exceeds 512 MiB; export a smaller copy"+ );+ // A live writer's rows are in the WAL, which immutable mode would skip.+ let mode = if wal {+ "mode=ro"+ } else {+ "mode=ro&immutable=1"+ };+ let path = db.to_str().context("MemPalace store path must be UTF-8")?;+ let uri = format!(+ "file:{}?{mode}",+ path.replace('%', "%25")+ .replace('?', "%3f")+ .replace('#', "%23")+ );+ let conn = Connection::open_with_flags(+ &uri,+ OpenFlags::SQLITE_OPEN_READ_ONLY+ | OpenFlags::SQLITE_OPEN_URI+ | OpenFlags::SQLITE_OPEN_NO_MUTEX,+ )+ .context("cannot open MemPalace store read-only")?;+ conn.execute_batch("PRAGMA query_only = 1; BEGIN")+ .context("cannot read MemPalace store")?;+ Ok((conn, !wal))+}++pub(super) fn import(input: &Path, output: &Path) -> Result<ImportSummary> {+ ensure_new_output(output)?;+ let (store, db) = store_path(input)?;+ let before = signature(&db);+ let (conn, immutable) = open(&db)?;+ let tables: i64 = conn+ .query_row(+ "SELECT count(*) FROM sqlite_master WHERE type = 'table' \+ AND name IN ('collections', 'documents')",+ [],+ |row| row.get(0),+ )+ .context("cannot read the input as a SQLite database; pass ~/.mempalace/palace")?;+ ensure!(+ tables == 2,+ "not a MemPalace sqlite_exact store: no collections and documents tables"+ );+ let collection: Option<i64> = conn+ .query_row(+ "SELECT id FROM collections WHERE name = ?1",+ [DRAWERS],+ |row| row.get(0),+ )+ .optional()?;+ let collection =+ collection.with_context(|| format!("MemPalace store has no {DRAWERS} collection"))?;+ let source_file = store+ .file_name()+ .and_then(|s| s.to_str())+ .context("MemPalace store filename must be UTF-8")?;+ let mut export = Export::create(output)?;+ let mut summary = ImportSummary {+ schema_version: 1,+ records: 0,+ source_lines: 0,+ skipped_records: 0,+ skipped_blocks: 0,+ source_sha256: String::new(),+ output_sha256: String::new(),+ };+ let mut source_hash = Sha256::new();+ let mut stmt = conn.prepare(+ "SELECT rowid, id, document, metadata_json, created_at FROM documents \+ WHERE collection_id = ?1 ORDER BY created_at, id",+ )?;+ let mut rows = stmt.query([collection])?;+ while let Some(row) = rows.next()? {+ summary.source_lines += 1;+ ensure!(+ summary.source_lines <= MAX_RECORDS as u64,+ "MemPalace store exceeds 100000 drawers; export a smaller copy"+ );+ let rowid: i64 = row.get(0)?;+ let at = format!("documents row {rowid}");+ let read = |i| {+ row.get::<_, String>(i)+ .with_context(|| format!("{at}: expected text"))+ };+ let (drawer, document, metadata, created) = (read(1)?, read(2)?, read(3)?, read(4)?);+ source_hash.update(serde_json::to_vec(&json!([+ rowid, drawer, document, metadata, created+ ]))?);+ let metadata: serde_json::Map<String, Value> = serde_json::from_str(&metadata)+ .map_err(|_| anyhow::anyhow!("{at}: metadata_json is not a JSON object"))?;+ // Registry and sentinel rows are MemPalace bookkeeping, not memories.+ if metadata.get("ingest_mode").and_then(Value::as_str) == Some("registry")+ || flagged(metadata.get("is_sentinel"))+ {+ summary.skipped_records += 1;+ continue;+ }+ let escaped = flagged(metadata.get(ESCAPED));+ let decode = |s: &str| {+ if escaped {+ unescape(s)+ } else {+ s.to_string()+ }+ };+ let meta_text = |key: &str| metadata.get(key).and_then(Value::as_str).map(decode);+ let body = decode(&document).trim().to_string();+ if body.is_empty() {+ summary.skipped_records += 1;+ continue;+ }+ ensure!(body.len() <= MAX_TEXT_BYTES, "{at}: text exceeds 256 KiB");+ let origin = meta_text("source_file");+ // A project drawer is a chunk of that file; a conversation drawer's source is a transcript.+ let convo = metadata.get("ingest_mode").and_then(Value::as_str) == Some("convos");+ let files: Vec<&String> = origin.iter().filter(|_| !convo).collect();+ let mut text_fields = json!({+ "body": body,+ "files": files,+ "source_file": source_file,+ "drawer_id": drawer,+ });+ if let Some(origin) = &origin {+ text_fields["origin_file"] = json!(origin);+ }+ for key in TEXT_METADATA {+ if let Some(value) = meta_text(key) {+ text_fields[key] = json!(value);+ }+ }+ // The drawer ID is stable across `mempalace compact`, which can renumber+ // rowids, but is built from wing/room names, so only its digest is identity.+ let key = format!("{:x}", Sha256::digest(drawer.as_bytes()));+ let key = &key[..32];+ let mut record = json!({});+ record["id"] = json!(format!("mempalace/drawer/{key}"));+ record["session_schema_version"] = json!(1);+ record["session_format"] = json!("mempalace");+ // Drawers are keyed by palace location, not by agent session.+ record["session_id"] = Value::Null;+ record["message_id"] = json!(format!("drawer/{key}"));+ record["parent_id"] = Value::Null;+ record["role"] = json!("drawer");+ // When the content was written, else when it was mined, else when stored.+ let (authored, filed) = (meta_text("authored_at"), meta_text("filed_at"));+ record["timestamp"] = timestamp([+ authored.as_deref(),+ filed.as_deref(),+ Some(created.as_str()),+ ]);+ record["source_table"] = json!("documents");+ record["source_line"] = json!(rowid);+ for key in NUMERIC_METADATA {+ record[key] = metadata+ .get(key)+ .filter(|v| v.is_i64())+ .cloned()+ .unwrap_or(Value::Null);+ }+ // Drawers hold the mined text verbatim, not a model's condensation.+ record["is_summary"] = json!(false);+ export.push(+ ParsedTurn {+ text_fields,+ identity_fields: record,+ },+ &mut summary,+ )?;+ }+ drop(rows);+ drop(stmt);+ conn.execute_batch("COMMIT")?;+ // Immutable mode trusts the file not to change; a writer that started meanwhile+ // (its sidecars appear, or a checkpoint rewrites the file) voids that snapshot.+ ensure!(+ !immutable || signature(&db) == before,+ "MemPalace store changed during import; rerun when its writer is idle"+ );+ ensure!(+ summary.records > 0,+ "no text-bearing MemPalace drawers; no export published"+ );+ summary.source_sha256 = format!("{:x}", source_hash.finalize());+ export.publish(output, &mut summary)?;+ Ok(summary)+}++#[cfg(test)]+mod tests {+ use super::*;++ fn palace(dir: &Path) -> PathBuf {+ let palace = dir.join("palace");+ fs::create_dir(&palace).unwrap();+ Connection::open(palace.join(STORE))+ .unwrap()+ .execute_batch(include_str!("../../tests/fixtures/mempalace.sql"))+ .unwrap();+ palace+ }++ fn fresh(dir: &Path, name: &str) -> PathBuf {+ let path = dir.join(name);+ fs::create_dir(&path).unwrap();+ path+ }++ fn rows(path: &Path) -> Vec<Value> {+ fs::read_to_string(path)+ .unwrap()+ .lines()+ .map(|s| serde_json::from_str(s).unwrap())+ .collect()+ }++ #[test]+ fn mempalace_drawers_export_redacted_without_bookkeeping_or_closets() {+ let dir = tempfile::tempdir().unwrap();+ let palace = palace(dir.path());+ let store = palace.join(STORE);+ let original = fs::read(&store).unwrap();+ let output = dir.path().join("safe.jsonl");+ let summary = import_mempalace(&store, &output).unwrap();+ // Registry and sentinel rows are read and skipped; the closet is never read.+ assert_eq!(+ (+ summary.records,+ summary.source_lines,+ summary.skipped_records+ ),+ (3, 5, 2)+ );+ let safe = fs::read_to_string(&output).unwrap();+ assert!(!safe.contains("DO_NOT_EXPORT"));+ assert!(!safe.contains("[email protected]"));+ assert!(safe.contains("[REDACTED_EMAIL]"));+ assert!(!safe.contains(ESCAPE_PREFIX));+ let rows = rows(&output);+ let project = &rows[0];+ let key = format!("{:x}", Sha256::digest(b"drawer_watcher_design_0001"));+ assert_eq!(project["id"], format!("mempalace/drawer/{}", &key[..32]));+ assert_eq!(project["message_id"], format!("drawer/{}", &key[..32]));+ assert_eq!(project["source_line"], 1);+ assert_eq!(project["session_format"], "mempalace");+ assert_eq!(project["role"], "drawer");+ assert_eq!(project["is_summary"], false);+ assert_eq!(project["session_id"], Value::Null);+ assert_eq!(+ project["title"],+ "drawer: We dropped polling because file events avoid repeated scans while the \+ durable journal preserves recovery."+ );+ assert_eq!(project["files"], json!(["/work/watcher/docs/decisions.md"]));+ assert_eq!(project["origin_file"], "/work/watcher/docs/decisions.md");+ assert_eq!(project["source_file"], STORE);+ assert_eq!(project["drawer_id"], "drawer_watcher_design_0001");+ assert_eq!(+ (&project["wing"], &project["room"]),+ (&json!("watcher"), &json!("design"))+ );+ assert_eq!(+ (&project["line_start"], &project["line_end"]),+ (&json!(3), &json!(4))+ );+ assert_eq!(project["timestamp"], "2026-10-25T00:00:01.000Z");+ let convo = &rows[1];+ assert_eq!(convo["hall"], "hall_facts");+ // A transcript's own time wins over when MemPalace mined it.+ assert_eq!(convo["timestamp"], "2026-10-20T09:30:00.000Z");+ assert_eq!(convo["files"], json!([]));+ assert_eq!(+ convo["origin_file"],+ "/home/me/.claude/projects/watcher/session.jsonl"+ );+ let escaped = &rows[2];+ assert_eq!(escaped["body"], "Build log one two \\ path \u{FFFD} end");+ assert_eq!(escaped["files"], json!(["/work/watcher/build .log"]));+ assert!(escaped.get(ESCAPED).is_none());+ // Same snapshot, same bytes; the store is unchanged and gains no sidecars.+ let again = dir.path().join("again.jsonl");+ let rerun = import_mempalace(&palace, &again).unwrap();+ assert_eq!(rerun.output_sha256, summary.output_sha256);+ assert_eq!(rerun.source_sha256, summary.source_sha256);+ assert_eq!(fs::read(&store).unwrap(), original);+ assert!(!sidecar(&store, "-wal").exists() && !sidecar(&store, "-shm").exists());+ assert!(import_mempalace(&store, &output)+ .unwrap_err()+ .to_string()+ .contains("already exists"));+ }++ #[test]+ fn mempalace_drawers_still_in_a_live_writers_wal_are_exported() {+ let dir = tempfile::tempdir().unwrap();+ let palace = palace(dir.path());+ let writer = Connection::open(palace.join(STORE)).unwrap();+ writer+ .execute_batch(+ "PRAGMA wal_autocheckpoint = 0;+ INSERT INTO documents VALUES (1, 'drawer_late', 'Written after the snapshot',+ '{\"wing\":\"watcher\",\"room\":\"design\"}', x'00', 1,+ '2026-10-25T00:00:09.000000+00:00', '2026-10-25T00:00:09.000000+00:00');",+ )+ .unwrap();+ let output = dir.path().join("safe.jsonl");+ assert_eq!(import_mempalace(&palace, &output).unwrap().records, 4);+ assert_eq!(rows(&output)[3]["body"], "Written after the snapshot");+ drop(writer);+ }++ #[test]+ fn mempalace_chroma_and_malformed_stores_publish_nothing() {+ let dir = tempfile::tempdir().unwrap();+ let chroma = dir.path().join("chroma-palace");+ fs::create_dir(&chroma).unwrap();+ fs::write(chroma.join(CHROMA_STORE), b"").unwrap();+ let not_sqlite = dir.path().join("not-sqlite");+ fs::create_dir(¬_sqlite).unwrap();+ fs::write(not_sqlite.join(STORE), b"not a database").unwrap();+ let empty = dir.path().join("empty.sqlite3");+ Connection::open(&empty)+ .unwrap()+ .execute_batch("CREATE TABLE t(x)")+ .unwrap();+ let no_drawers = palace(&fresh(dir.path(), "no-drawers"));+ Connection::open(no_drawers.join(STORE))+ .unwrap()+ .execute_batch("UPDATE collections SET name = 'renamed' WHERE id = 1")+ .unwrap();+ let bad_metadata = palace(&fresh(dir.path(), "bad-metadata"));+ Connection::open(bad_metadata.join(STORE))+ .unwrap()+ .execute_batch("UPDATE documents SET metadata_json = '[1]' WHERE rowid = 2")+ .unwrap();+ let both = palace(&fresh(dir.path(), "both"));+ fs::write(both.join(CHROMA_STORE), b"").unwrap();+ let torn = palace(&fresh(dir.path(), "torn"));+ fs::write(sidecar(&torn.join(STORE), "-wal"), b"").unwrap();+ for (name, input, reason) in [+ ("chroma", chroma.clone(), "default Chroma backend"),+ ("chroma_file", chroma.join(CHROMA_STORE), "Chroma store"),+ (+ "missing",+ dir.path().join("missing"),+ "cannot open MemPalace",+ ),+ (+ "not_sqlite",+ not_sqlite,+ "pass ~/.mempalace/palace: file is not a database",+ ),+ ("empty", empty, "no collections and documents tables"),+ ("no_drawers", no_drawers, "no mempalace_drawers collection"),+ (+ "bad_metadata",+ bad_metadata,+ "row 2: metadata_json is not a JSON object",+ ),+ ("both", both, "both a Chroma and a sqlite_exact store"),+ ("torn", torn, "incomplete WAL sidecar set"),+ ] {+ let output = dir.path().join(format!("{name}.jsonl"));+ let error = format!("{:#}", import_mempalace(&input, &output).unwrap_err());+ assert!(error.contains(reason), "{name}: {error}");+ assert!(!output.exists(), "{name}");+ }+ }++ #[test]+ fn mempalace_ids_survive_a_compact_that_renumbers_rows() {+ let dir = tempfile::tempdir().unwrap();+ let palace = palace(dir.path());+ let before = dir.path().join("before.jsonl");+ import_mempalace(&palace, &before).unwrap();+ // Rewrite the drawers in reverse order so every rowid changes.+ Connection::open(palace.join(STORE))+ .unwrap()+ .execute_batch(+ "CREATE TABLE copy AS SELECT * FROM documents ORDER BY rowid DESC;+ DELETE FROM documents;+ INSERT INTO documents SELECT * FROM copy;+ DROP TABLE copy;",+ )+ .unwrap();+ let after = dir.path().join("after.jsonl");+ import_mempalace(&palace, &after).unwrap();+ let ids = |path: &Path| -> Vec<(Value, Value)> {+ rows(path)+ .into_iter()+ .map(|r| (r["id"].clone(), r["source_line"].clone()))+ .collect()+ };+ let (before, after) = (ids(&before), ids(&after));+ for ((id, line), (id_after, line_after)) in before.iter().zip(&after) {+ assert_eq!(id, id_after);+ assert_ne!(line, line_after);+ }+ }++ #[test]+ fn mempalace_flags_match_its_own_reader() {+ for (value, set) in [+ (json!(true), true),+ (json!(1), true),+ (json!("1"), true),+ (json!("true"), true),+ (json!("True"), true),+ (json!(false), false),+ (json!(0), false),+ (json!(2), false),+ (json!("yes"), false),+ (json!("TRUE"), false),+ (Value::Null, false),+ ] {+ assert_eq!(flagged(Some(&value)), set, "{value}");+ }+ assert!(!flagged(None));+ }++ #[test]+ fn mempalace_writer_starting_mid_import_is_detected() {+ let dir = tempfile::tempdir().unwrap();+ let store = palace(dir.path()).join(STORE);+ let before = signature(&store);+ let (reader, immutable) = open(&store).unwrap();+ assert!(immutable);+ // A writer that opens the clean store now creates its sidecars, so the+ // check after COMMIT sees a different signature and refuses.+ let writer = Connection::open(&store).unwrap();+ writer+ .execute_batch(+ "INSERT INTO documents VALUES (1, 'drawer_late', 'Late', '{}', x'00', 1, \+ '2026-10-25T00:00:09+00:00', '2026-10-25T00:00:09+00:00');",+ )+ .unwrap();+ reader.execute_batch("COMMIT").unwrap();+ assert_ne!(signature(&store), before);+ }++ #[test]+ fn mempalace_unescape_follows_stored_text_rules() {+ for (stored, read) in [+ ("plain \\0 text", "plain \\0 text"),+ ("\u{1}mpesc:a\\\\b", "a\\b"),+ ("\u{1}mpesc:nul\\0end", "nul end"),+ ("\u{1}mpesc:\\uDC80", "\u{FFFD}"),+ ("\u{1}mpesc:\\u00E9", "\u{e9}"),+ ("\u{1}mpesc:keep \\x and \\u12", "keep \\x and \\u12"),+ ("\u{1}mpesc:\\\\0", "\\0"),+ ] {+ assert_eq!(unescape(stored), read, "{stored:?}");+ }+ }++ #[test]+ fn mempalace_is_a_cli_format() {+ let dir = tempfile::tempdir().unwrap();+ let input = palace(dir.path());+ let output = dir.path().join("safe.jsonl");+ let args: Vec<String> = ["import", "--format", "mempalace", "--input"]+ .iter()+ .map(|s| s.to_string())+ .chain([input.display().to_string(), "--output".into()])+ .chain([output.display().to_string()])+ .collect();+ assert_eq!(run_cli(&args), 0);+ assert_eq!(rows(&output).len(), 3);+ }+}diff --git a/engine/crates/xerj-autoindex/tests/fixtures/mempalace.sql b/engine/crates/xerj-autoindex/tests/fixtures/mempalace.sqlnew file mode 100644index 000000000..4d2923848--- /dev/null+++ b/engine/crates/xerj-autoindex/tests/fixtures/mempalace.sql@@ -0,0 +1,69 @@+-- A small MemPalace sqlite_exact store (mempalace/backends/sqlite_exact.py schema at+-- MemPalace/mempalace 4f7745afbd9d0197c6475b4c7ac846d4932eaeda). DO_NOT_EXPORT marks+-- rows and columns the importer must never write: bookkeeping rows, closets, embeddings.+PRAGMA journal_mode = WAL;++CREATE TABLE meta (+ key TEXT PRIMARY KEY,+ value TEXT NOT NULL+);+CREATE TABLE collections (+ id INTEGER PRIMARY KEY AUTOINCREMENT,+ name TEXT NOT NULL UNIQUE,+ dimension INTEGER,+ created_at TEXT NOT NULL+);+CREATE TABLE documents (+ collection_id INTEGER NOT NULL,+ id TEXT NOT NULL,+ document TEXT NOT NULL,+ metadata_json TEXT NOT NULL,+ embedding BLOB NOT NULL,+ dim INTEGER NOT NULL,+ created_at TEXT NOT NULL,+ updated_at TEXT NOT NULL,+ PRIMARY KEY (collection_id, id),+ FOREIGN KEY(collection_id) REFERENCES collections(id) ON DELETE CASCADE+);++INSERT INTO meta VALUES ('schema_version', '1');+INSERT INTO collections VALUES (1, 'mempalace_drawers', 1, '2026-10-25T00:00:00.000000+00:00');+INSERT INTO collections VALUES (2, 'mempalace_closets', 1, '2026-10-25T00:00:00.000000+00:00');++-- A drawer mined from a project file.+INSERT INTO documents VALUES (+ 1, 'drawer_watcher_design_0001',+ 'We dropped polling because file events avoid repeated scans while the durable journal preserves recovery.',+ '{"wing":"watcher","room":"design","source_file":"/work/watcher/docs/decisions.md","chunk_index":0,"line_start":3,"line_end":4,"added_by":"miner","filed_at":"2026-10-25T00:00:01.000000+00:00"}',+ x'00', 1, '2026-10-25T00:00:01.000000+00:00', '2026-10-25T00:00:01.000000+00:00'+);+-- A drawer mined from a conversation, holding personal data the redactor must remove.+INSERT INTO documents VALUES (+ 1, 'drawer_watcher_chat_0002',+ '> Why did we drop polling? Contact [email protected] or 202-555-0147.+File events, with the journal for recovery.',+ '{"wing":"watcher","room":"chat","hall":"hall_facts","source_file":"/home/me/.claude/projects/watcher/session.jsonl","chunk_index":0,"added_by":"convo_miner","authored_at":"2026-10-20T09:30:00+00:00","filed_at":"2026-10-25T00:00:02.000000+00:00","entities":"DO_NOT_EXPORT_ENTITIES","ingest_mode":"convos"}',+ x'00', 1, '2026-10-25T00:00:02.000000+00:00', '2026-10-25T00:00:02.000000+00:00'+);+-- A drawer MemPalace stored escaped because its text held a NUL and a stray byte.+INSERT INTO documents VALUES (+ 1, 'drawer_watcher_logs_0003',+ char(1) || 'mpesc:Build log one\0two \\ path \uDC80 end',+ '{"wing":"watcher","room":"logs","source_file":' || json_quote(char(1) || 'mpesc:/work/watcher/build\0.log') || ',"filed_at":"2026-10-25T00:00:03.000000+00:00","_mempalace_escaped":1}',+ x'00', 1, '2026-10-25T00:00:03.000000+00:00', '2026-10-25T00:00:03.000000+00:00'+);+-- Bookkeeping rows that share the drawers collection.+INSERT INTO documents VALUES (+ 1, 'registry_watcher_0004', 'DO_NOT_EXPORT_REGISTRY',+ '{"wing":"watcher","room":"_registry","source_file":"/home/me/.claude/projects/watcher/session.jsonl","ingest_mode":"registry"}',+ x'00', 1, '2026-10-25T00:00:04.000000+00:00', '2026-10-25T00:00:04.000000+00:00'+);+INSERT INTO documents VALUES (+ 1, 'sentinel_0005', 'DO_NOT_EXPORT_SENTINEL', '{"is_sentinel":true}',+ x'00', 1, '2026-10-25T00:00:05.000000+00:00', '2026-10-25T00:00:05.000000+00:00'+);+-- A closet: MemPalace's pointer index over drawers, not a memory of its own.+INSERT INTO documents VALUES (+ 2, 'closet_watcher_0006', 'DO_NOT_EXPORT_CLOSET', '{"wing":"watcher","room":"design"}',+ x'00', 1, '2026-10-25T00:00:06.000000+00:00', '2026-10-25T00:00:06.000000+00:00'+);diff --git a/tools/session-import/README.md b/tools/session-import/README.mdindex 54dc51dba..efa534a0f 100644--- a/tools/session-import/README.md+++ b/tools/session-import/README.md@@ -1,9 +1,9 @@# Import an archived agent transcript for decision retrieval`xerj sessions import` is the explicit preparation step for a single Claude Code-transcript, Codex rollout JSONL file or claude-mem store. It runs locally, calls no model or node, and exports text+transcript, Codex rollout JSONL file, claude-mem store or MemPalace palace. It runs locally, calls no model or node, and exports textturns through XERJ's existing native PII plugin. Review the export, then give its-folder to ordinary `xerj autoindex`. This covers three explicit formats for+folder to ordinary `xerj autoindex`. This covers four explicit formats for[#1793](https://github.com/xerj-org/xerj/issues/1793) and[#2874](https://github.com/xerj-org/xerj/issues/2874), not automatic transcriptdiscovery, a live conversation view or incremental sync.@@ -314,3 +314,82 @@ private-node lifecycle check:```shpython3 tools/session-import/smoke.py --binary engine/target/release/xerj --format claude-mem```++## MemPalace palace adapter++[MemPalace](https://github.com/MemPalace/mempalace) files conversation and+project text verbatim as drawers, organised into wings, rooms and halls. Pass+the palace directory (MemPalace's default is `~/.mempalace/palace`) or its+`sqlite_exact.sqlite3` file explicitly; the command does not look for it or+change it:++```sh+set -eu+SESSION_EXPORT=$(mktemp -d)+xerj sessions import --format mempalace \+ --input ~/.mempalace/palace \+ --output "$SESSION_EXPORT/records.jsonl"+```++Only MemPalace's `sqlite_exact` backend is read. A palace on the default Chroma+backend (a `chroma.sqlite3` and no `sqlite_exact.sqlite3`) refuses with that+reason rather than exporting nothing, and a palace holding both stores refuses,+as MemPalace itself does (pass the store file to read it anyway). The store is opened the way MemPalace's+own read-only reader opens it: with both `-wal` and `-shm` present it is opened+read-only so a live writer's uncheckpointed drawers are included; with neither+it is opened immutable so no side files are created; with only one it refuses.+All drawers come from one read transaction. An immutable open trusts the file+not to change, so if the store or its side files change while it is read (a+writer started), the import refuses and publishes nothing; rerun when the writer+is idle. The 512 MiB input cap counts the store and its `-wal` file together.++Only the `mempalace_drawers` collection exports, ordered by `created_at` then+drawer ID. Closets (MemPalace's pointer index over drawers) and other+collections are not read, and registry and sentinel rows, which share the+drawers collection as bookkeeping, count as skipped records. A drawer MemPalace+stored escaped (flagged `_mempalace_escaped`) is decoded first; a NUL in it+becomes a space and an unpaired surrogate U+FFFD, as MemPalace's own search+index and MCP output render them. Embeddings, entity lists and other metadata+are not exported.++MemPalace output keeps schema 1 and the shared text/source fields, with these+format-specific meanings:++| Field | MemPalace meaning |+|---|---|+| `session_format`, `role`, `is_summary` | `mempalace`, `drawer`, `false`: a drawer is mined text, not a model's condensation |+| `session_id`, `parent_id` | null: drawers are keyed by palace location, not by agent session |+| `id`, `message_id` | `mempalace/drawer/<key>` and `drawer/<key>`, where `<key>` is the first 32 hex digits of the SHA-256 of MemPalace's drawer ID, so a `mempalace compact` that renumbers rows keeps the IDs |+| `drawer_id` | MemPalace's drawer ID, as redacted text |+| `wing`, `room`, `hall`, `added_by`, `content_date` | Drawer metadata, as redacted text when present |+| `origin_file` | The file MemPalace mined the drawer from, as redacted text |+| `files` | `origin_file` for a project drawer; empty for a conversation drawer (`ingest_mode: convos`), whose origin is a transcript |+| `chunk_index`, `line_start`, `line_end` | MemPalace's chunk position, integers or null |+| `timestamp` | `authored_at` (when a conversation was written), else `filed_at` (when it was mined), else the row's `created_at`, as RFC 3339 UTC with milliseconds; null when none is an RFC 3339 instant |+| `source_file`, `source_table`, `source_line` | Store basename, `documents`, and SQLite row id |++`source_lines` counts drawer-collection rows read, bookkeeping included, and+`source_sha256` hashes them in export order, so two exports of an unchanged+palace match. A missing `mempalace_drawers` collection, a `metadata_json` that+is not a JSON object or a non-text column refuses the whole export; more than+100,000 rows refuses rather than truncating. The redaction, size limits,+staging, no-clobber publication, exit codes and new-folder/prefix rerun rules+above apply unchanged.++Reference retrieved before implementation: MemPalace, MIT, pinned at+`4f7745afbd9d0197c6475b4c7ac846d4932eaeda`:++- [`backends/sqlite_exact.py:57`](https://github.com/MemPalace/mempalace/blob/4f7745afbd9d0197c6475b4c7ac846d4932eaeda/mempalace/backends/sqlite_exact.py#L57) and [`:1923-1951`](https://github.com/MemPalace/mempalace/blob/4f7745afbd9d0197c6475b4c7ac846d4932eaeda/mempalace/backends/sqlite_exact.py#L1923-L1951): the store filename and its `collections` and `documents` tables.+- [`backends/sqlite_exact.py:1790-1817`](https://github.com/MemPalace/mempalace/blob/4f7745afbd9d0197c6475b4c7ac846d4932eaeda/mempalace/backends/sqlite_exact.py#L1790-L1817): the read-only open rule for WAL side files.+- [`backends/sqlite_exact.py:290-294`](https://github.com/MemPalace/mempalace/blob/4f7745afbd9d0197c6475b4c7ac846d4932eaeda/mempalace/backends/sqlite_exact.py#L290-L294): registry and sentinel rows excluded as bookkeeping.+- [`config.py:288`](https://github.com/MemPalace/mempalace/blob/4f7745afbd9d0197c6475b4c7ac846d4932eaeda/mempalace/config.py#L288) and [`palace/collection.py:218`](https://github.com/MemPalace/mempalace/blob/4f7745afbd9d0197c6475b4c7ac846d4932eaeda/mempalace/palace/collection.py#L218): the drawers and closets collection names.+- [`stored_text.py:40-41`](https://github.com/MemPalace/mempalace/blob/4f7745afbd9d0197c6475b4c7ac846d4932eaeda/mempalace/stored_text.py#L40-L41) and [`:85-89`](https://github.com/MemPalace/mempalace/blob/4f7745afbd9d0197c6475b4c7ac846d4932eaeda/mempalace/stored_text.py#L85-L89): the escape flag, prefix and decoder.++These are format evidence, not copied implementation. The fixture+(`engine/crates/xerj-autoindex/tests/fixtures/mempalace.sql`) is a synthetic+store in that column shape, not a live MemPalace install. Run it through the+private-node lifecycle check:++```sh+python3 tools/session-import/smoke.py --binary engine/target/release/xerj --format mempalace+```diff --git a/tools/session-import/smoke.py b/tools/session-import/smoke.pyindex 20df4a2db..f1290d1ba 100644--- a/tools/session-import/smoke.py+++ b/tools/session-import/smoke.py@@ -1,5 +1,5 @@#!/usr/bin/env python3-"""Synthetic session or claude-mem store -> redacted JSONL -> autoindex -> cited decision, on a private node."""+"""Synthetic session, claude-mem store or MemPalace palace -> redacted JSONL -> autoindex -> cited decision, on a private node."""import argparseimport jsonimport os@@ -18,6 +18,7 @@ FORMATS = {'claude-code': ('claude-session.jsonl', 2, 3, ['src/watch.rs']),'codex': ('codex-session.jsonl', 2, 9, []),'claude-mem': ('claude-mem.sql', 3, 1, ['src/watch.rs', 'src/poll.rs']),+ 'mempalace': ('mempalace.sql', 3, 1, ['/work/watcher/docs/decisions.md']),}@@ -41,8 +42,11 @@ def main():with tempfile.TemporaryDirectory(prefix='xerj-sessions-smoke-') as temp:work = Path(temp)fixture, records, expected_line, expected_files = FORMATS[args.format]- if args.format == 'claude-mem':- source = work / 'claude-mem.db'+ if args.format in ('claude-mem', 'mempalace'):+ # MemPalace is passed as its palace directory, the store inside it.+ palace = work / 'palace'+ source = work / 'claude-mem.db' if args.format == 'claude-mem' else palace / 'sqlite_exact.sqlite3'+ source.parent.mkdir(exist_ok=True)with sqlite3.connect(source) as store:store.executescript((FIXTURES / fixture).read_text())store.close()@@ -52,7 +56,8 @@ def main():exports = work / 'export'exports.mkdir()output = exports / 'records.jsonl'- command = ['sessions', 'import', '--format', args.format, '--input', str(source), '--output', str(output)]+ given = palace if args.format == 'mempalace' else source+ command = ['sessions', 'import', '--format', args.format, '--input', str(given), '--output', str(output)]imported = json.loads(run(binary, command, env, work).stdout)assert imported['records'] == records, importedsafe = output.read_bytes()