AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
feat(sessions): import a claude-mem store with --format claude-mem (#2874)
Jacob Elbirt6b77110f48fe
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 78b7e3466..1f3a94856 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -46,6 +46,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0one with `/` is anchored at the root. The run reports how many paths it leftout (#2483).+- Offline `xerj sessions import --format claude-mem` exports a claude-mem+ store's prompts, observations and session summaries through the existing+ text redactor (#2874). It reads `claude-mem.db` read-only in one snapshot,+ including rows still in a running worker's WAL, joins each row to its Claude+ Code session UUID, and marks the model-written observations and summaries+ `is_summary: true`.+- Offline `xerj sessions import --format codex` exports a documented subsetof one archived Codex rollout through the existing text redactor. It skipstools and reasoning, retains physical source locations and archive eventdiff --git a/engine/crates/xerj-autoindex/src/sessions.rs b/engine/crates/xerj-autoindex/src/sessions.rsindex 8a3263257..bcbae9602 100644--- a/engine/crates/xerj-autoindex/src/sessions.rs+++ b/engine/crates/xerj-autoindex/src/sessions.rs@@ -16,7 +16,7 @@ const MAX_RECORDS: usize = 100_000;pub const HELP: &str = "xerj sessions import — offline session preparation for autoindex-USAGE: xerj sessions import --format <claude-code|codex> --input FILE --output NEW.jsonl+USAGE: xerj sessions import --format <claude-code|codex|claude-mem> --input FILE --output NEW.jsonlRead one explicit Claude Code transcript or Codex rollout JSONL; write user/assistant textrecords through the existing native PII redactor before any output is stored.@@ -34,7 +34,10 @@ The redactor handles email/IP/card/SSN/phone patterns, not every secret or name.Codex reads response_item messages after session_meta, skips duplicate event_msgtext and tools/reasoning; uses line-and-byte-digest event IDs, no inferred parentsor files. exec --json and external history_base are unsupported.-Input <=512 MiB; line <=1 MiB; turn text <=256 KiB; <=100000 messages.+claude-mem opens ~/.claude-mem/claude-mem.db read-only in one snapshot and exports+user_prompts, observations and session_summaries rows (is_summary=true for the+model-written two), joining each to its Claude Code session UUID via sdk_sessions.+Input <=512 MiB; line <=1 MiB; turn text <=256 KiB; <=100000 messages or rows.Exit 0: exported, JSON summary on stdout; 2: usage; 1: import/publication error.Optional automation (Claude Code, macOS/Linux): xerj init --memory.@@ -264,6 +267,7 @@ fn claude_turn(}))}+mod claude_mem;mod codex;#[derive(Clone, Copy)]@@ -280,12 +284,95 @@ pub fn import_codex(input: &Path, output: &Path) -> Result<ImportSummary> {import_session(input, output, SessionFormat::Codex)}-fn import_session(input: &Path, output: &Path, format: SessionFormat) -> Result<ImportSummary> {+pub fn import_claude_mem(input: &Path, output: &Path) -> Result<ImportSummary> {+ claude_mem::import(input, output)+}++fn ensure_new_output(output: &Path) -> Result<()> {match fs::symlink_metadata(output) {Ok(_) => bail!("session output already exists; choose a new output path"),- Err(e) if e.kind() == std::io::ErrorKind::NotFound => {}- Err(e) => return Err(e).context("cannot inspect session output"),+ Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(()),+ Err(e) => Err(e).context("cannot inspect session output"),+ }+}++/// Redacted, allowlisted records staged privately until a no-clobber publish.+struct Export {+ staging: tempfile::NamedTempFile,+ redactor: PiiRedactionPlugin,+ hash: Sha256,+ written: u64,+}++impl Export {+ fn create(output: &Path) -> Result<Self> {+ let parent = output+ .parent()+ .filter(|p| !p.as_os_str().is_empty())+ .unwrap_or(Path::new("."));+ let staging = tempfile::Builder::new()+ .prefix(".xerj-session-")+ .suffix(".tmp")+ .tempfile_in(parent)+ .context("cannot create private export staging file; output parent must exist")?;+ Ok(Self {+ staging,+ redactor: PiiRedactionPlugin::new(Vec::new()),+ hash: Sha256::new(),+ written: 0,+ })+ }++ fn push(&mut self, turn: ParsedTurn, summary: &mut ImportSummary) -> Result<()> {+ let ParsedTurn {+ text_fields: mut record,+ identity_fields,+ } = turn;+ ensure!(+ self.redactor.process(&mut record) == ProcessAction::Pass,+ "PII redaction refused session record"+ );+ // Only validated enums/IDs and numeric provenance bypass text redaction.+ record+ .as_object_mut()+ .unwrap()+ .extend(identity_fields.as_object().unwrap().clone());+ let kind = record["role"].as_str().unwrap();+ let title: String = record["body"]+ .as_str()+ .unwrap()+ .lines()+ .next()+ .unwrap_or("")+ .chars()+ .take(120)+ .collect();+ record["title"] = json!(format!("{kind}: {title}"));+ let mut encoded = serde_json::to_vec(&record)?;+ encoded.push(b'\n');+ self.written += encoded.len() as u64;+ ensure!(+ self.written <= MAX_INPUT_BYTES,+ "session export exceeds 512 MiB; use a smaller transcript"+ );+ self.staging.write_all(&encoded)?;+ self.hash.update(&encoded);+ summary.records += 1;+ Ok(())+ }++ fn publish(self, output: &Path, summary: &mut ImportSummary) -> Result<()> {+ let Self {+ mut staging, hash, ..+ } = self;+ summary.output_sha256 = format!("{:x}", hash.finalize());+ staging.as_file_mut().sync_all()?;+ publish(staging, output)}+}++fn import_session(input: &Path, output: &Path, format: SessionFormat) -> Result<ImportSummary> {+ ensure_new_output(output)?;let mut options = fs::OpenOptions::new();options.read(true);#[cfg(unix)]@@ -305,16 +392,7 @@ fn import_session(input: &Path, output: &Path, format: SessionFormat) -> Result<.file_name().and_then(|s| s.to_str()).context("session input filename must be UTF-8")?;- let parent = output- .parent()- .filter(|p| !p.as_os_str().is_empty())- .unwrap_or(Path::new("."));- let mut staging = tempfile::Builder::new()- .prefix(".xerj-session-")- .suffix(".tmp")- .tempfile_in(parent)- .context("cannot create private export staging file; output parent must exist")?;- let redactor = PiiRedactionPlugin::new(Vec::new());+ let mut export = Export::create(output)?;let mut summary = ImportSummary {schema_version: 1,records: 0,@@ -325,11 +403,9 @@ fn import_session(input: &Path, output: &Path, format: SessionFormat) -> Result<output_sha256: String::new(),};let mut source_hash = Sha256::new();- let mut output_hash = Sha256::new();let mut reader = BufReader::new((&file).take(before.len() + 1));let mut bytes = Vec::new();let mut read = 0;- let mut written = 0u64;let mut session: Option<String> = None;let mut seen = HashSet::new();let mut codex = codex::CodexState::default();@@ -370,43 +446,9 @@ fn import_session(input: &Path, output: &Path, format: SessionFormat) -> Result<codex.turn(&entry, line, offset, source_file, &bytes, &mut summary)?}};- let Some(ParsedTurn {- text_fields: mut record,- identity_fields,- }) = parsed- else {- continue;- };- ensure!(- redactor.process(&mut record) == ProcessAction::Pass,- "PII redaction refused session record"- );- // Only validated enums/IDs and numeric provenance bypass text redaction.- record- .as_object_mut()- .unwrap()- .extend(identity_fields.as_object().unwrap().clone());- let kind = record["role"].as_str().unwrap();- let title: String = record["body"]- .as_str()- .unwrap()- .lines()- .next()- .unwrap_or("")- .chars()- .take(120)- .collect();- record["title"] = json!(format!("{kind}: {title}"));- let mut encoded = serde_json::to_vec(&record)?;- encoded.push(b'\n');- written += encoded.len() as u64;- ensure!(- written <= MAX_INPUT_BYTES,- "session export exceeds 512 MiB; use a smaller transcript"- );- staging.write_all(&encoded)?;- output_hash.update(&encoded);- summary.records += 1;+ if let Some(turn) = parsed {+ export.push(turn, &mut summary)?;+ }}ensure!(summary.records > 0,@@ -414,9 +456,7 @@ fn import_session(input: &Path, output: &Path, format: SessionFormat) -> Result<);unchanged(&file, &before, input, read)?;summary.source_sha256 = format!("{:x}", source_hash.finalize());- summary.output_sha256 = format!("{:x}", output_hash.finalize());- staging.as_file_mut().sync_all()?;- publish(staging, output)?;+ export.publish(output, &mut summary)?;Ok(summary)}@@ -428,7 +468,8 @@ pub fn run_cli(args: &[String]) -> i32 {print!("{HELP}");return 0;}- let parsed = (|| -> Result<(PathBuf, PathBuf, SessionFormat)> {+ type Import = fn(&Path, &Path) -> Result<ImportSummary>;+ let parsed = (|| -> Result<(PathBuf, PathBuf, Import)> {ensure!(args[0] == "import", "expected sessions import");let mut input = None;let mut output = None;@@ -448,25 +489,26 @@ pub fn run_cli(args: &[String]) -> i32 {.context("session import option requires a value")?;*slot = Some(value.clone());}- let format = match format.as_deref() {- Some("claude-code") => SessionFormat::ClaudeCode,- Some("codex") => SessionFormat::Codex,- _ => bail!("--format claude-code or --format codex is required"),+ let import: Import = match format.as_deref() {+ Some("claude-code") => import_claude_code,+ Some("codex") => import_codex,+ Some("claude-mem") => import_claude_mem,+ _ => bail!("--format claude-code, codex or claude-mem is required"),};Ok((PathBuf::from(input.context("--input is required")?),PathBuf::from(output.context("--output is required")?),- format,+ import,))})();- let (input, output, format) = match parsed {+ let (input, output, import) = match parsed {Ok(paths) => paths,Err(error) => {eprintln!("error: {error}");return 2;}};- match import_session(&input, &output, format) {+ match import(&input, &output) {Ok(summary) => {let mut stdout = std::io::stdout().lock();if serde_json::to_writer(&mut stdout, &summary).is_err() || writeln!(stdout).is_err() {diff --git a/engine/crates/xerj-autoindex/src/sessions/claude_mem.rs b/engine/crates/xerj-autoindex/src/sessions/claude_mem.rsnew file mode 100644index 000000000..a177e5903--- /dev/null+++ b/engine/crates/xerj-autoindex/src/sessions/claude_mem.rs@@ -0,0 +1,540 @@+//! Read-only claude-mem SQLite subset. Reference and licence are pinned in the import guide.+use super::*;+use rusqlite::{types::ValueRef, Connection, OpenFlags, Row};++/// One exported claude-mem table: which columns become text, which stay optional.+struct Table {+ name: &'static str,+ role: &'static str,+ /// Columns every supported claude-mem schema has.+ required: &'static [&'static str],+ /// Columns added by later claude-mem migrations; read as NULL when absent.+ optional: &'static [&'static str],+ /// Session join: the column that matches `sdk_sessions`, and the column it matches.+ session: (&'static str, &'static str),+}++const PROMPTS: Table = Table {+ name: "user_prompts",+ role: "user",+ required: &["id", "prompt_text", "created_at_epoch"],+ optional: &["prompt_number", "content_session_id"],+ session: ("content_session_id", "content_session_id"),+};++const OBSERVATIONS: Table = Table {+ name: "observations",+ role: "observation",+ required: &[+ "id",+ "memory_session_id",+ "project",+ "type",+ "created_at_epoch",+ ],+ optional: &[+ "prompt_number",+ "title",+ "subtitle",+ "narrative",+ "text",+ "facts",+ "concepts",+ "files_read",+ "files_modified",+ ],+ session: ("memory_session_id", "memory_session_id"),+};++const SUMMARIES: Table = Table {+ name: "session_summaries",+ role: "summary",+ required: &["id", "memory_session_id", "project", "created_at_epoch"],+ optional: &[+ "prompt_number",+ "request",+ "investigated",+ "learned",+ "completed",+ "next_steps",+ "notes",+ "files_read",+ "files_edited",+ ],+ session: ("memory_session_id", "memory_session_id"),+};++const SUMMARY_SECTIONS: [(&str, &str); 6] = [+ ("request", "Request"),+ ("investigated", "Investigated"),+ ("learned", "Learned"),+ ("completed", "Completed"),+ ("next_steps", "Next steps"),+ ("notes", "Notes"),+];++fn columns(conn: &Connection, table: &str) -> Result<HashSet<String>> {+ let mut stmt = conn.prepare("SELECT name FROM pragma_table_info(?1)")?;+ let names = stmt+ .query_map([table], |row| row.get::<_, String>(0))?+ .collect::<rusqlite::Result<HashSet<_>>>()?;+ Ok(names)+}++/// A row as JSON: NULL, integer and text only; anything else refuses the export.+fn row_json(row: &Row, names: &[&str], at: &str) -> Result<serde_json::Map<String, Value>> {+ let mut out = serde_json::Map::new();+ for (i, name) in names.iter().enumerate() {+ let value = match row.get_ref(i)? {+ ValueRef::Null => Value::Null,+ ValueRef::Integer(n) => json!(n),+ ValueRef::Text(t) => json!(std::str::from_utf8(t)+ .map_err(|_| anyhow::anyhow!("{at}: {name} is not UTF-8 text"))?),+ _ => bail!("{at}: unsupported value type in {name}"),+ };+ out.insert((*name).to_string(), value);+ }+ Ok(out)+}++fn text<'a>(row: &'a serde_json::Map<String, Value>, field: &str, at: &str) -> Result<&'a str> {+ match &row[field] {+ Value::Null => Ok(""),+ Value::String(s) => Ok(s.trim()),+ _ => bail!("{at}: expected text in {field}"),+ }+}++/// claude-mem stores `facts`, `concepts` and `files_*` as JSON.stringify'd string arrays.+fn string_array(+ row: &serde_json::Map<String, Value>,+ field: &str,+ at: &str,+) -> Result<Vec<String>> {+ let raw = text(row, field, at)?;+ if raw.is_empty() {+ return Ok(Vec::new());+ }+ serde_json::from_str(raw)+ .map_err(|_| anyhow::anyhow!("{at}: {field} is not a JSON string array"))+}++fn push_line(body: &mut String, line: &str) {+ if line.is_empty() {+ return;+ }+ if !body.is_empty() {+ body.push('\n');+ }+ body.push_str(line);+}++/// Text body and file paths for one row, in the order claude-mem renders them.+fn body_and_files(+ table: &Table,+ row: &serde_json::Map<String, Value>,+ at: &str,+) -> Result<(String, Vec<String>)> {+ let mut body = String::new();+ let mut files = Vec::new();+ match table.role {+ "user" => push_line(&mut body, text(row, "prompt_text", at)?),+ "observation" => {+ for field in ["title", "subtitle", "narrative", "text"] {+ push_line(&mut body, text(row, field, at)?);+ }+ for fact in string_array(row, "facts", at)? {+ push_line(&mut body, &format!("- {}", fact.trim()));+ }+ files.extend(string_array(row, "files_modified", at)?);+ files.extend(string_array(row, "files_read", at)?);+ }+ _ => {+ for (field, label) in SUMMARY_SECTIONS {+ let value = text(row, field, at)?;+ if !value.is_empty() {+ push_line(&mut body, &format!("{label}: {value}"));+ }+ }+ files.extend(string_array(row, "files_edited", at)?);+ files.extend(string_array(row, "files_read", at)?);+ }+ }+ ensure!(body.len() <= MAX_TEXT_BYTES, "{at}: text exceeds 256 KiB");+ let mut unique = HashSet::new();+ files.retain(|f| unique.insert(f.clone()));+ Ok((body, files))+}++fn timestamp(row: &serde_json::Map<String, Value>, at: &str) -> Result<Value> {+ let millis = row["created_at_epoch"]+ .as_i64()+ .with_context(|| format!("{at}: expected integer created_at_epoch"))?;+ let time = chrono::DateTime::from_timestamp_millis(millis)+ .with_context(|| format!("{at}: created_at_epoch out of range"))?;+ Ok(json!(+ time.to_rfc3339_opts(chrono::SecondsFormat::Millis, true)+ ))+}++#[allow(clippy::too_many_arguments)]+fn export_table(+ conn: &Connection,+ table: &Table,+ sessions: bool,+ source_file: &str,+ summary: &mut ImportSummary,+ source_hash: &mut Sha256,+ export: &mut Export,+) -> Result<()> {+ let present = columns(conn, table.name)?;+ if present.is_empty() {+ return Ok(());+ }+ for name in table.required {+ ensure!(+ present.contains(*name),+ "claude-mem table {} has no {name} column; unsupported claude-mem version",+ table.name+ );+ }+ let names: Vec<&str> = table+ .required+ .iter()+ .chain(table.optional)+ .copied()+ .collect();+ // Column and table names come only from the allowlists above.+ let mut select: Vec<String> = names+ .iter()+ .map(|c| match present.contains(*c) {+ true => format!("t.{c}"),+ false => format!("NULL AS {c}"),+ })+ .collect();+ let (local, remote) = table.session;+ let join = if sessions && present.contains(local) {+ select.push("s.content_session_id".to_string());+ format!("LEFT JOIN sdk_sessions s ON s.{remote} = t.{local}")+ } else {+ select.push("NULL".to_string());+ String::new()+ };+ let sql = format!(+ "SELECT {} FROM {} t {join} ORDER BY t.created_at_epoch, t.id",+ select.join(", "),+ table.name+ );+ let mut stmt = conn.prepare(&sql)?;+ let mut rows = stmt.query([])?;+ let mut all = names.clone();+ all.push("session");+ while let Some(row) = rows.next()? {+ summary.source_lines += 1;+ ensure!(+ summary.source_lines <= MAX_RECORDS as u64,+ "claude-mem store exceeds 100000 rows; export a smaller copy"+ );+ let at = format!("{} row {}", table.name, row.get::<_, i64>(0)?);+ let row = row_json(row, &all, &at)?;+ source_hash.update(table.name.as_bytes());+ source_hash.update(serde_json::to_vec(&row)?);+ let (body, files) = body_and_files(table, &row, &at)?;+ if body.is_empty() {+ summary.skipped_records += 1;+ continue;+ }+ let rowid = row["id"].as_i64().unwrap();+ let mut text_fields = json!({"body": body, "files": files, "source_file": source_file});+ for field in ["project", "memory_session_id"] {+ if let Some(value) = row.get(field).filter(|v| !v.is_null()) {+ text_fields[field] = value.clone();+ }+ }+ if table.role == "observation" {+ text_fields["observation_type"] = row["type"].clone();+ text_fields["concepts"] = json!(string_array(&row, "concepts", &at)?);+ }+ let mut record = json!({});+ record["id"] = json!(format!("claude-mem/{}/{rowid}", table.role));+ record["session_schema_version"] = json!(1);+ record["session_format"] = json!("claude-mem");+ // claude-mem keys sessions by its own ID; the Claude Code session UUID is joined in.+ let session = row["session"]+ .as_str()+ .or_else(|| row.get("content_session_id").and_then(Value::as_str));+ record["session_id"] = Value::Null;+ if let Some(session) = session {+ match uuid::Uuid::parse_str(session) {+ Ok(id) if session.len() == 36 => record["session_id"] = json!(id.to_string()),+ // Not a UUID (another claude-mem platform): keep it, but as redacted text.+ _ => text_fields["session_id"] = json!(session),+ }+ }+ record["message_id"] = json!(format!("{}/{rowid}", table.role));+ record["parent_id"] = Value::Null;+ record["role"] = json!(table.role);+ record["timestamp"] = timestamp(&row, &at)?;+ record["source_table"] = json!(table.name);+ record["source_line"] = json!(rowid);+ record["prompt_number"] = row+ .get("prompt_number")+ .filter(|v| v.is_i64())+ .cloned()+ .unwrap_or(Value::Null);+ // Observations and summaries are claude-mem's model-written condensations.+ record["is_summary"] = json!(table.role != "user");+ export.push(+ ParsedTurn {+ text_fields,+ identity_fields: record,+ },+ summary,+ )?;+ }+ Ok(())+}++pub(super) fn import(input: &Path, output: &Path) -> Result<ImportSummary> {+ ensure_new_output(output)?;+ let meta = fs::metadata(input).context("cannot open claude-mem database")?;+ ensure!(meta.is_file(), "claude-mem input must be a regular file");+ ensure!(+ meta.len() <= MAX_INPUT_BYTES,+ "claude-mem database exceeds 512 MiB; export a smaller copy"+ );+ let source_file = input+ .file_name()+ .and_then(|s| s.to_str())+ .context("claude-mem input filename must be UTF-8")?;+ // Read-only but not immutable: a running claude-mem worker keeps recent rows in the WAL.+ let conn = Connection::open_with_flags(+ input,+ OpenFlags::SQLITE_OPEN_READ_ONLY | OpenFlags::SQLITE_OPEN_NO_MUTEX,+ )+ .context("cannot open claude-mem database read-only")?;+ conn.execute_batch("PRAGMA query_only = 1; BEGIN")+ .context("cannot read claude-mem database")?;+ let tables = columns(&conn, "observations").map_err(|_| {+ anyhow::anyhow!("input is not a readable SQLite database; pass ~/.claude-mem/claude-mem.db")+ })?;+ ensure!(+ !tables.is_empty(),+ "not a claude-mem database: no observations table"+ );+ let sessions = !columns(&conn, "sdk_sessions")?.is_empty();+ let mut export = Export::create(output)?;+ let mut summary = ImportSummary {+ schema_version: 1,+ records: 0,+ source_lines: 0,+ skipped_records: 0,+ skipped_blocks: 0,+ source_sha256: String::new(),+ output_sha256: String::new(),+ };+ let mut source_hash = Sha256::new();+ // One read transaction: all three tables come from the same snapshot.+ for table in [&PROMPTS, &OBSERVATIONS, &SUMMARIES] {+ export_table(+ &conn,+ table,+ sessions,+ source_file,+ &mut summary,+ &mut source_hash,+ &mut export,+ )?;+ }+ conn.execute_batch("COMMIT")?;+ ensure!(+ summary.records > 0,+ "no text-bearing claude-mem prompts, observations or summaries; no export published"+ );+ summary.source_sha256 = format!("{:x}", source_hash.finalize());+ export.publish(output, &mut summary)?;+ Ok(summary)+}++#[cfg(test)]+mod tests {+ use super::*;++ fn store(dir: &Path) -> PathBuf {+ let path = dir.join("claude-mem.db");+ Connection::open(&path)+ .unwrap()+ .execute_batch(include_str!("../../tests/fixtures/claude-mem.sql"))+ .unwrap();+ path+ }++ fn rows(path: &Path) -> Vec<Value> {+ fs::read_to_string(path)+ .unwrap()+ .lines()+ .map(|s| serde_json::from_str(s).unwrap())+ .collect()+ }++ #[test]+ fn claude_mem_prompts_observations_and_summaries_export_redacted_with_sessions() {+ let dir = tempfile::tempdir().unwrap();+ let input = store(dir.path());+ let original = fs::read(&input).unwrap();+ let output = dir.path().join("safe.jsonl");+ let summary = import_claude_mem(&input, &output).unwrap();+ assert_eq!((summary.records, summary.source_lines), (3, 3));+ let safe = fs::read_to_string(&output).unwrap();+ assert!(!safe.contains("DO_NOT_EXPORT"));+ assert!(!safe.contains("[email protected]"));+ assert!(safe.contains("[REDACTED_EMAIL]"));+ let rows = rows(&output);+ let roles: Vec<_> = rows.iter().map(|r| r["role"].as_str().unwrap()).collect();+ assert_eq!(roles, ["user", "observation", "summary"]);+ for row in &rows {+ assert_eq!(row["session_format"], "claude-mem");+ assert_eq!(row["session_id"], "550e8400-e29b-41d4-a716-446655440000");+ }+ let observation = &rows[1];+ assert_eq!(observation["id"], "claude-mem/observation/1");+ assert_eq!(+ observation["title"],+ "observation: Dropped polling for file events"+ );+ assert!(observation["body"]+ .as_str()+ .unwrap()+ .contains("- The durable journal preserves recovery"));+ assert_eq!(observation["files"], json!(["src/watch.rs", "src/poll.rs"]));+ assert_eq!(observation["observation_type"], "decision");+ assert_eq!(observation["concepts"], json!(["watcher", "recovery"]));+ assert_eq!(observation["source_table"], "observations");+ assert_eq!(observation["source_line"], 1);+ assert_eq!(observation["timestamp"], "2026-10-25T00:00:01.000Z");+ assert_eq!(+ [+ &rows[0]["is_summary"],+ &rows[1]["is_summary"],+ &rows[2]["is_summary"]+ ],+ [false, true, true]+ );+ assert!(rows[2]["body"]+ .as_str()+ .unwrap()+ .starts_with("Request: Explain why polling was dropped\nInvestigated:"));+ assert_eq!(rows[2]["files"], json!(["src/watch.rs", "src/poll.rs"]));+ // Same snapshot, same bytes; the store itself is never written.+ let again = dir.path().join("again.jsonl");+ let second = import_claude_mem(&input, &again).unwrap();+ assert_eq!(fs::read_to_string(&again).unwrap(), safe);+ assert_eq!(second.source_sha256, summary.source_sha256);+ assert_eq!(fs::read(&input).unwrap(), original);+ assert!(import_claude_mem(&input, &output).is_err());+ assert_eq!(fs::read_to_string(&output).unwrap(), safe);+ }++ #[test]+ fn claude_mem_rows_still_in_a_live_workers_wal_are_exported() {+ let dir = tempfile::tempdir().unwrap();+ let input = store(dir.path());+ // A running worker: WAL mode, no checkpoint, connection held open.+ let worker = Connection::open(&input).unwrap();+ worker+ .execute_batch(+ "PRAGMA journal_mode=WAL; PRAGMA wal_autocheckpoint=0;+ INSERT INTO observations (memory_session_id, project, type, created_at,+ created_at_epoch, title) VALUES ('mem-0001', 'watcher', 'discovery',+ '2026-10-25T00:00:03.000Z', 1792886403000, 'Written after the snapshot');",+ )+ .unwrap();+ assert!(dir.path().join("claude-mem.db-wal").exists());+ let output = dir.path().join("live.jsonl");+ let summary = import_claude_mem(&input, &output).unwrap();+ assert_eq!(summary.records, 4);+ assert!(fs::read_to_string(&output)+ .unwrap()+ .contains("Written after the snapshot"));+ drop(worker);+ }++ #[test]+ fn claude_mem_older_schema_without_migrated_columns_still_exports() {+ let dir = tempfile::tempdir().unwrap();+ let input = dir.path().join("claude-mem.db");+ Connection::open(&input)+ .unwrap()+ .execute_batch(+ "CREATE TABLE observations (id INTEGER PRIMARY KEY, memory_session_id TEXT,+ project TEXT, text TEXT, type TEXT, created_at TEXT, created_at_epoch INTEGER);+ INSERT INTO observations VALUES (7, 'mem-1', 'p', 'Plain v1 text', 'change',+ '2026-10-25T00:00:00.000Z', 1792886400000);",+ )+ .unwrap();+ let output = dir.path().join("safe.jsonl");+ let summary = import_claude_mem(&input, &output).unwrap();+ assert_eq!(summary.records, 1);+ let rows = rows(&output);+ assert_eq!(rows[0]["body"], "Plain v1 text");+ assert_eq!(rows[0]["session_id"], Value::Null);+ assert_eq!(rows[0]["files"], json!([]));+ }++ #[test]+ fn claude_mem_wrong_or_malformed_stores_publish_nothing() {+ let dir = tempfile::tempdir().unwrap();+ let output = dir.path().join("safe.jsonl");+ let not_sqlite = dir.path().join("notes.db");+ fs::write(¬_sqlite, "not a database").unwrap();+ let other = dir.path().join("other.db");+ Connection::open(&other)+ .unwrap()+ .execute_batch("CREATE TABLE t (x)")+ .unwrap();+ let malformed = store(dir.path());+ Connection::open(&malformed)+ .unwrap()+ .execute_batch("UPDATE observations SET facts = 'not json'")+ .unwrap();+ for (input, reason) in [+ (¬_sqlite, "not a readable SQLite database"),+ (&other, "no observations table"),+ (+ &malformed,+ "observations row 1: facts is not a JSON string array",+ ),+ ] {+ let error = format!("{:#}", import_claude_mem(input, &output).unwrap_err());+ assert!(error.contains(reason), "{error}");+ assert!(!output.exists());+ }+ assert_eq!(+ fs::read_dir(dir.path())+ .unwrap()+ .filter(|e| e+ .as_ref()+ .unwrap()+ .file_name()+ .to_string_lossy()+ .ends_with(".tmp"))+ .count(),+ 0+ );+ }++ #[test]+ fn claude_mem_is_a_cli_format() {+ let dir = tempfile::tempdir().unwrap();+ let input = store(dir.path());+ let output = dir.path().join("safe.jsonl");+ let args: Vec<String> = ["import", "--format", "claude-mem", "--input"]+ .iter()+ .map(|s| s.to_string())+ .chain([input.display().to_string(), "--output".into()])+ .chain([output.display().to_string()])+ .collect();+ assert_eq!(run_cli(&args), 0);+ assert_eq!(rows(&output).len(), 3);+ }+}diff --git a/engine/crates/xerj-autoindex/tests/fixtures/claude-mem.sql b/engine/crates/xerj-autoindex/tests/fixtures/claude-mem.sqlnew file mode 100644index 000000000..fa0d80c03--- /dev/null+++ b/engine/crates/xerj-autoindex/tests/fixtures/claude-mem.sql@@ -0,0 +1,97 @@+-- Synthetic claude-mem store: the four tables `xerj sessions import --format claude-mem`+-- reads, in the column shape of thedotmack/claude-mem at 999a0c40 (SessionStore.ts).+-- Columns the importer never exports carry DO_NOT_EXPORT markers.+PRAGMA journal_mode = WAL;++CREATE TABLE sdk_sessions (+ id INTEGER PRIMARY KEY AUTOINCREMENT,+ content_session_id TEXT NOT NULL,+ memory_session_id TEXT UNIQUE,+ project TEXT NOT NULL,+ platform_source TEXT NOT NULL DEFAULT 'claude',+ user_prompt TEXT,+ started_at TEXT NOT NULL,+ started_at_epoch INTEGER NOT NULL,+ status TEXT NOT NULL DEFAULT 'active',+ custom_title TEXT,+ cwd TEXT+);++CREATE TABLE observations (+ id INTEGER PRIMARY KEY AUTOINCREMENT,+ memory_session_id TEXT NOT NULL,+ project TEXT NOT NULL,+ text TEXT,+ type TEXT NOT NULL,+ created_at TEXT NOT NULL,+ created_at_epoch INTEGER NOT NULL,+ prompt_number INTEGER,+ title TEXT,+ subtitle TEXT,+ facts TEXT,+ narrative TEXT,+ concepts TEXT,+ files_read TEXT,+ files_modified TEXT,+ discovery_tokens INTEGER DEFAULT 0,+ content_hash TEXT,+ merged_into_project TEXT,+ metadata TEXT+);++CREATE TABLE session_summaries (+ id INTEGER PRIMARY KEY AUTOINCREMENT,+ memory_session_id TEXT UNIQUE NOT NULL,+ project TEXT NOT NULL,+ request TEXT,+ investigated TEXT,+ learned TEXT,+ completed TEXT,+ next_steps TEXT,+ files_read TEXT,+ files_edited TEXT,+ notes TEXT,+ created_at TEXT NOT NULL,+ created_at_epoch INTEGER NOT NULL,+ prompt_number INTEGER,+ discovery_tokens INTEGER DEFAULT 0+);++CREATE TABLE user_prompts (+ id INTEGER PRIMARY KEY AUTOINCREMENT,+ session_db_id INTEGER,+ content_session_id TEXT NOT NULL,+ prompt_number INTEGER NOT NULL,+ prompt_text TEXT NOT NULL,+ created_at TEXT NOT NULL,+ created_at_epoch INTEGER NOT NULL+);++INSERT INTO sdk_sessions VALUES (+ 1, '550e8400-e29b-41d4-a716-446655440000', 'mem-0001', 'watcher', 'claude',+ 'DO_NOT_EXPORT_SESSION_PROMPT', '2026-10-25T00:00:00.000Z', 1792886400000,+ 'completed', 'DO_NOT_EXPORT_TITLE', '/work/watcher'+);++INSERT INTO user_prompts VALUES (+ 1, 1, '550e8400-e29b-41d4-a716-446655440000', 1,+ 'Why did we drop polling? Contact [email protected] or 202-555-0147.',+ '2026-10-25T00:00:00.000Z', 1792886400000+);++INSERT INTO observations VALUES (+ 1, 'mem-0001', 'watcher', NULL, 'decision', '2026-10-25T00:00:01.000Z', 1792886401000, 1,+ 'Dropped polling for file events', 'Watcher design',+ '["File events avoid repeated scans","The durable journal preserves recovery"]',+ 'We dropped polling because file events avoid repeated scans while the durable journal preserves recovery.',+ '["watcher","recovery"]', '["src/poll.rs"]', '["src/watch.rs"]', 1200,+ 'DO_NOT_EXPORT_HASH', NULL, '{"note":"DO_NOT_EXPORT_METADATA"}'+);++INSERT INTO session_summaries VALUES (+ 1, 'mem-0001', 'watcher', 'Explain why polling was dropped',+ 'Compared polling with file events', 'Event loss is recoverable from the journal',+ 'Switched the watcher to file events', 'Measure event latency',+ '["src/poll.rs"]', '["src/watch.rs"]', NULL,+ '2026-10-25T00:00:02.000Z', 1792886402000, 1, 900+);diff --git a/tools/session-import/README.md b/tools/session-import/README.mdindex e5b14aa0e..4d778a202 100644--- a/tools/session-import/README.md+++ b/tools/session-import/README.md@@ -1,10 +1,11 @@# Import an archived agent transcript for decision retrieval`xerj sessions import` is the explicit preparation step for a single Claude Code-transcript or Codex rollout JSONL file. It runs locally, calls no model or node, and exports text+transcript, Codex rollout JSONL file or claude-mem store. It runs locally, calls no model or node, and exports textturns through XERJ's existing native PII plugin. Review the export, then give its-folder to ordinary `xerj autoindex`. This covers two explicit formats for-[#1793](https://github.com/xerj-org/xerj/issues/1793), not automatic transcript+folder to ordinary `xerj autoindex`. This covers three explicit formats for+[#1793](https://github.com/xerj-org/xerj/issues/1793) and+[#2874](https://github.com/xerj-org/xerj/issues/2874), not automatic transcriptdiscovery, a live conversation view or incremental sync.## Try the synthetic example@@ -231,3 +232,80 @@ Run the Codex fixture through the same private-node lifecycle check:```shpython3 tools/session-import/smoke.py --binary engine/target/release/xerj --format codex```++## claude-mem store adapter++[claude-mem](https://github.com/thedotmack/claude-mem) is a Claude Code plugin+that has a model condense each session into observations and summaries, kept in+one SQLite file. Pass that file explicitly; the command does not look for it,+start or stop the claude-mem worker, or write to the store:++```sh+set -eu+SESSION_EXPORT=$(mktemp -d)+xerj sessions import --format claude-mem \+ --input ~/.claude-mem/claude-mem.db \+ --output "$SESSION_EXPORT/records.jsonl"+```++The store is opened read-only, not immutable, inside one read transaction, so+rows a running worker still holds in `claude-mem.db-wal` are included and all+three tables come from the same snapshot. SQLite needs the store's `-shm` file+or a writable folder to read a WAL database; on a read-only copy without either,+copy the `.db`, `-wal` and `-shm` files together. A file with no `observations`+table refuses as not a claude-mem store.++Three tables export, each ordered by `created_at_epoch` then row id:++| Table | `role` | `body` |+|---|---|---|+| `user_prompts` | `user` | `prompt_text`, what was typed |+| `observations` | `observation` | `title`, `subtitle`, `narrative`, `text`, then each of `facts` as a `- ` line |+| `session_summaries` | `summary` | `Request:`, `Investigated:`, `Learned:`, `Completed:`, `Next steps:`, `Notes:` lines |++Observations and summaries are what claude-mem's model wrote about a session,+not what was said in it: they carry `is_summary: true`, and a search hit on one+is a model's account to check against the code, not a quote. A row whose text+columns are all empty counts as a skipped record. Session metadata (the stored+first prompt and custom title), content hashes, token counts, the `metadata`+column, `pending_messages` and the Chroma vector store are not exported.++claude-mem output keeps schema 1 and the shared text/source fields, with these+format-specific meanings:++| Field | claude-mem meaning |+|---|---|+| `session_format`, `session_id` | `claude-mem`; the Claude Code session UUID (`content_session_id`), joined through `sdk_sessions` for observations and summaries. A non-UUID value is kept as redacted text; no session row gives null |+| `id`, `message_id` | `claude-mem/<role>/<row id>` and `<role>/<row id>`: stable for a row across exports of the same store |+| `parent_id` | null |+| `files` | `files_modified` (summaries: `files_edited`) then `files_read`, deduplicated, as claude-mem recorded them |+| `project`, `memory_session_id` | claude-mem's project name and its own session key, as redacted text |+| `observation_type`, `concepts` | Observations only: claude-mem's type (`decision`, `bugfix` and so on) and concept tags, as redacted text |+| `prompt_number` | The prompt within the session the row belongs to, or null |+| `timestamp` | `created_at_epoch` as RFC 3339 UTC with milliseconds |+| `source_file`, `source_table`, `source_line` | Store basename, table, and SQLite row id (there is no line or byte offset in a database) |++`source_lines` in the summary counts rows read and `source_sha256` hashes those+rows' selected columns in export order, so two exports of an unchanged store+match. Columns later claude-mem migrations added are read as null on an older+store; a missing required column, a value that is not text or an integer, or a+`facts`, `concepts` or `files_*` value that is not a JSON string array refuses+the whole export. More than 100,000 rows refuses rather than truncating. The+redaction, size limits, staging, no-clobber publication, exit codes and+new-folder/prefix rerun rules above apply unchanged.++Reference retrieved before implementation: claude-mem, Apache-2.0, pinned at+`999a0c4010a12befecb1686e695126c04a4faa6e`:++- [`SessionStore.ts:1273-1325`](https://github.com/thedotmack/claude-mem/blob/999a0c4010a12befecb1686e695126c04a4faa6e/src/services/sqlite/SessionStore.ts#L1273-L1325): base `sdk_sessions`, `observations` and `session_summaries` tables and their session keys.+- [`SessionStore.ts:1697-1712`](https://github.com/thedotmack/claude-mem/blob/999a0c4010a12befecb1686e695126c04a4faa6e/src/services/sqlite/SessionStore.ts#L1697-L1712): `user_prompts`, keyed by `content_session_id` and `session_db_id`, with no project column.+- [`SessionStore.ts:4166-4180`](https://github.com/thedotmack/claude-mem/blob/999a0c4010a12befecb1686e695126c04a4faa6e/src/services/sqlite/SessionStore.ts#L4166-L4180): observation fields, with `facts`, `concepts` and `files_*` stored as `JSON.stringify` arrays.++These are format evidence, not copied implementation. The fixture+(`engine/crates/xerj-autoindex/tests/fixtures/claude-mem.sql`) is a synthetic+store in that column shape, not a live claude-mem install. Run it through the+private-node lifecycle check:++```sh+python3 tools/session-import/smoke.py --binary engine/target/release/xerj --format claude-mem+```diff --git a/tools/session-import/smoke.py b/tools/session-import/smoke.pyindex b5bdce844..20df4a2db 100644--- a/tools/session-import/smoke.py+++ b/tools/session-import/smoke.py@@ -1,16 +1,24 @@#!/usr/bin/env python3-"""Synthetic session -> redacted JSONL -> autoindex -> cited decision, on a private node."""+"""Synthetic session or claude-mem store -> redacted JSONL -> autoindex -> cited decision, on a private node."""import argparseimport jsonimport osfrom pathlib import Pathimport socket+import sqlite3import subprocessimport tempfileimport timeimport urllib.requestROOT = Path(__file__).resolve().parents[2]+FIXTURES = ROOT / 'engine/crates/xerj-autoindex/tests/fixtures'+# fixture, records, decision source_line, decision files+FORMATS = {+ 'claude-code': ('claude-session.jsonl', 2, 3, ['src/watch.rs']),+ 'codex': ('codex-session.jsonl', 2, 9, []),+ 'claude-mem': ('claude-mem.sql', 3, 1, ['src/watch.rs', 'src/poll.rs']),+}def run(binary, arguments, env, work, expected=0):@@ -24,7 +32,7 @@ def run(binary, arguments, env, work, expected=0):def main():parser = argparse.ArgumentParser(description=__doc__)parser.add_argument('--binary', type=Path, required=True)- parser.add_argument('--format', choices=('claude-code', 'codex'), default='claude-code')+ parser.add_argument('--format', choices=tuple(FORMATS), default='claude-code')args = parser.parse_args()binary = args.binary.resolve(strict=True)env = {k: v for k, v in os.environ.items() if not k.startswith('XERJ_')@@ -32,20 +40,26 @@ def main():env.update(XERJ_DISABLE_FEEDBACK='true', XERJ_AUTH='', XERJ_NUM_CPUS='4')with tempfile.TemporaryDirectory(prefix='xerj-sessions-smoke-') as temp:work = Path(temp)- source = work / 'session.jsonl'- fixture = 'codex-session.jsonl' if args.format == 'codex' else 'claude-session.jsonl'- source.write_bytes((ROOT / 'engine/crates/xerj-autoindex/tests/fixtures' / fixture).read_bytes())+ fixture, records, expected_line, expected_files = FORMATS[args.format]+ if args.format == 'claude-mem':+ source = work / 'claude-mem.db'+ with sqlite3.connect(source) as store:+ store.executescript((FIXTURES / fixture).read_text())+ store.close()+ else:+ source = work / 'session.jsonl'+ source.write_bytes((FIXTURES / fixture).read_bytes())exports = work / 'export'exports.mkdir()output = exports / 'records.jsonl'command = ['sessions', 'import', '--format', args.format, '--input', str(source), '--output', str(output)]imported = json.loads(run(binary, command, env, work).stdout)- assert imported['records'] == 2, imported+ assert imported['records'] == records, importedsafe = output.read_bytes()assert b'[email protected]' not in safe and b'DO_NOT_EXPORT' not in saferun(binary, command, env, work, expected=1)assert output.read_bytes() == safe- print('offline import: 2 redacted text turns; overwrite refused', flush=True)+ print(f'offline import: {records} redacted text records; overwrite refused', flush=True)ports = []sockets = []@@ -128,29 +142,27 @@ def main():assert len(done) == 1 and done[0]['ok'] is True and done[0]['exit'] == 0, doneassert not done[0].get('datasets_refused') and not done[0].get('files_refused'), donecount = request('/session-smoke-*/_count')['count']- assert count == 2, count+ assert count == records, countprint(f'autoindex pass {generation + 1}: {time.monotonic() - before:.3f}s; {count} live documents', flush=True)result = request('/session-smoke-*/_search', {'size': 10, 'query': {'match_all': {}}})stored = result['hits']['hits']- assert len(stored) == 2, result+ assert len(stored) == records, resultassert '[email protected]' not in json.dumps(stored)assert 'DO_NOT_EXPORT' not in json.dumps(stored)decision = next(h['_source'] for h in stored if 'durable journal' in h['_source']['body'])- expected_line = 9 if args.format == 'codex' else 3- expected_files = [] if args.format == 'codex' else ['src/watch.rs']- assert decision['source_file'] == 'session.jsonl' and decision['source_line'] == expected_line, decision+ assert decision['source_file'] == source.name and decision['source_line'] == expected_line, decisionassert decision['files'] == expected_files, decisionassert decision['session_format'] == args.format, decisionsearch = run(binary, ['search', '--prefix', 'session-smoke', '--url', url,'--json', 'why did we drop polling'], env, work)assert 'durable journal' in search.stdout, search.stdout- print(f'search: decision text found; original session.jsonl:{expected_line}; files={expected_files}', flush=True)+ print(f'search: decision text found; original {source.name}:{expected_line}; files={expected_files}', flush=True)finally:stop(child)child = start(log)try:- assert request('/session-smoke-*/_count')['count'] == 2- print('graceful restart: 2 documents preserved', flush=True)+ assert request('/session-smoke-*/_count')['count'] == records+ print(f'graceful restart: {records} documents preserved', flush=True)finally:stop(child)return 0